1#!/usr/bin/env bash
2
3# Copyright 2017 The Kubernetes Authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17# A library of helper functions and constant for the local config.
18
19# Use the config file specified in $KUBE_CONFIG_FILE, or default to
20# config-default.sh.
21readonly GCE_MAX_LOCAL_SSD=8
22
23KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
24source "${KUBE_ROOT}/cluster/gce/${KUBE_CONFIG_FILE-"config-default.sh"}"
25source "${KUBE_ROOT}/cluster/common.sh"
26source "${KUBE_ROOT}/hack/lib/util.sh"
27
28if [[ "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "ubuntu" || "${NODE_OS_DISTRIBUTION}" == "custom" ]]; then
29 source "${KUBE_ROOT}/cluster/gce/${NODE_OS_DISTRIBUTION}/node-helper.sh"
30else
31 echo "Cannot operate on cluster using node os distro: ${NODE_OS_DISTRIBUTION}" >&2
32 exit 1
33fi
34
35source "${KUBE_ROOT}/cluster/gce/windows/node-helper.sh"
36
37if [[ "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then
38 source "${KUBE_ROOT}/cluster/gce/${MASTER_OS_DISTRIBUTION}/master-helper.sh"
39else
40 echo "Cannot operate on cluster using master os distro: ${MASTER_OS_DISTRIBUTION}" >&2
41 exit 1
42fi
43
44if [[ ${NODE_LOCAL_SSDS:-} -ge 1 ]] && [[ -n ${NODE_LOCAL_SSDS_EXT:-} ]] ; then
45 echo -e "${color_red:-}Local SSD: Only one of NODE_LOCAL_SSDS and NODE_LOCAL_SSDS_EXT can be specified at once${color_norm:-}" >&2
46 exit 2
47fi
48
49if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then
50 DEFAULT_GCI_PROJECT=google-containers
51 if [[ "${GCI_VERSION}" == "cos"* ]] || [[ "${MASTER_IMAGE_FAMILY}" == "cos"* ]]; then
52 DEFAULT_GCI_PROJECT=cos-cloud
53 fi
54 export MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-${DEFAULT_GCI_PROJECT}}
55
56 # If the master image is not set, we use the latest image based on image
57 # family.
58 kube_master_image="${KUBE_GCE_MASTER_IMAGE:-${GCI_VERSION}}"
59 if [[ -z "${kube_master_image}" ]]; then
60 kube_master_image=$(gcloud compute images list --project="${MASTER_IMAGE_PROJECT}" --no-standard-images --filter="family:${MASTER_IMAGE_FAMILY}" --format 'value(name)')
61 fi
62
63 echo "Using image: ${kube_master_image} from project: ${MASTER_IMAGE_PROJECT} as master image" >&2
64 export MASTER_IMAGE="${kube_master_image}"
65fi
66
67# Sets node image based on the specified os distro. Currently this function only
68# supports gci and debian.
69#
70# Requires:
71# NODE_OS_DISTRIBUTION
72# Sets:
73# DEFAULT_GCI_PROJECT
74# NODE_IMAGE
75# NODE_IMAGE_PROJECT
76function set-linux-node-image() {
77 if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then
78 DEFAULT_GCI_PROJECT=google-containers
79 if [[ "${GCI_VERSION}" == "cos"* ]] || [[ "${NODE_IMAGE_FAMILY}" == "cos"* ]]; then
80 DEFAULT_GCI_PROJECT=cos-cloud
81 fi
82
83 # If the node image is not set, we use the latest image based on image
84 # family.
85 # Otherwise, we respect whatever is set by the user.
86 NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-${DEFAULT_GCI_PROJECT}}
87 local kube_node_image
88
89 kube_node_image="${KUBE_GCE_NODE_IMAGE:-${GCI_VERSION}}"
90 if [[ -z "${kube_node_image}" ]]; then
91 kube_node_image=$(gcloud compute images list --project="${NODE_IMAGE_PROJECT}" --no-standard-images --filter="family:${NODE_IMAGE_FAMILY}" --format 'value(name)')
92 fi
93
94 echo "Using image: ${kube_node_image} from project: ${NODE_IMAGE_PROJECT} as node image" >&2
95 export NODE_IMAGE="${kube_node_image}"
96 fi
97}
98
99# Requires:
100# WINDOWS_NODE_OS_DISTRIBUTION
101# Sets:
102# WINDOWS_NODE_IMAGE_PROJECT
103# WINDOWS_NODE_IMAGE
104function set-windows-node-image() {
105 WINDOWS_NODE_IMAGE_PROJECT="windows-cloud"
106 if [[ "${WINDOWS_NODE_OS_DISTRIBUTION}" == "win2019" ]]; then
107 WINDOWS_NODE_IMAGE="windows-server-2019-dc-core-v20210914"
108 elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION}" == "win1909" ]]; then
109 WINDOWS_NODE_IMAGE="windows-server-1909-dc-core-v20210413"
110 elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION}" == "win2004" ]]; then
111 WINDOWS_NODE_IMAGE="windows-server-2004-dc-core-v20210914"
112 elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION,,}" == "win20h2" ]]; then
113 WINDOWS_NODE_IMAGE="windows-server-20h2-dc-core-v20210914"
114 elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION,,}" == "win2022" ]]; then
115 WINDOWS_NODE_IMAGE="windows-server-2022-dc-core-v20220513"
116 else
117 echo "Unknown WINDOWS_NODE_OS_DISTRIBUTION ${WINDOWS_NODE_OS_DISTRIBUTION}" >&2
118 exit 1
119 fi
120}
121
122set-linux-node-image
123set-windows-node-image
124
125# Verify cluster autoscaler configuration.
126if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then
127 if [[ -z $AUTOSCALER_MIN_NODES ]]; then
128 echo "AUTOSCALER_MIN_NODES not set."
129 exit 1
130 fi
131 if [[ -z $AUTOSCALER_MAX_NODES ]]; then
132 echo "AUTOSCALER_MAX_NODES not set."
133 exit 1
134 fi
135fi
136
137# These prefixes must not be prefixes of each other, so that they can be used to
138# detect mutually exclusive sets of nodes.
139NODE_INSTANCE_PREFIX=${NODE_INSTANCE_PREFIX:-"${INSTANCE_PREFIX}-minion"}
140WINDOWS_NODE_INSTANCE_PREFIX=${WINDOWS_NODE_INSTANCE_PREFIX:-"${INSTANCE_PREFIX}-windows-node"}
141
142# NODE_TAG (expected to be) defined by caller
143# shellcheck disable=SC2153
144NODE_TAGS="${NODE_TAG}"
145
146ALLOCATE_NODE_CIDRS=true
147PREEXISTING_NETWORK=false
148PREEXISTING_NETWORK_MODE=""
149
150KUBE_PROMPT_FOR_UPDATE=${KUBE_PROMPT_FOR_UPDATE:-"n"}
151# How long (in seconds) to wait for cluster initialization.
152KUBE_CLUSTER_INITIALIZATION_TIMEOUT=${KUBE_CLUSTER_INITIALIZATION_TIMEOUT:-300}
153
154function join_csv() {
155 local IFS=','; echo "$*";
156}
157
158# This function returns the first string before the comma
159function split_csv() {
160 echo "$*" | cut -d',' -f1
161}
162
163# Verify prereqs
164function verify-prereqs() {
165 local cmd
166
167 # we use openssl to generate certs
168 kube::util::test_openssl_installed
169
170 # ensure a version supported by easyrsa is installed
171 if [ "$(openssl version | cut -d\ -f1)" == "LibreSSL" ]; then
172 echo "LibreSSL is not supported. Please ensure openssl points to an OpenSSL binary"
173 if [ "$(uname -s)" == "Darwin" ]; then
174 # We want this print just the way it is
175 # shellcheck disable=SC2016
176 echo 'On macOS we recommend using homebrew and adding "$(brew --prefix openssl)/bin" to your PATH'
177 fi
178 exit 1
179 fi
180
181 # we use gcloud to create the cluster, gsutil to stage binaries and data
182 for cmd in gcloud gsutil; do
183 if ! which "${cmd}" >/dev/null; then
184 echo "Can't find ${cmd} in PATH, please fix and retry. The Google Cloud " >&2
185 echo "SDK can be downloaded from https://cloud.google.com/sdk/." >&2
186 exit 1
187 fi
188 done
189 update-or-verify-gcloud
190}
191
192# Use the gcloud defaults to find the project. If it is already set in the
193# environment then go with that.
194#
195# Vars set:
196# PROJECT
197# NETWORK_PROJECT
198# PROJECT_REPORTED
199function detect-project() {
200 if [[ -z "${PROJECT-}" ]]; then
201 PROJECT=$(gcloud config list project --format 'value(core.project)')
202 fi
203
204 NETWORK_PROJECT=${NETWORK_PROJECT:-${PROJECT}}
205
206 if [[ -z "${PROJECT-}" ]]; then
207 echo "Could not detect Google Cloud Platform project. Set the default project using " >&2
208 echo "'gcloud config set project <PROJECT>'" >&2
209 exit 1
210 fi
211 if [[ -z "${PROJECT_REPORTED-}" ]]; then
212 echo "Project: ${PROJECT}" >&2
213 echo "Network Project: ${NETWORK_PROJECT}" >&2
214 echo "Zone: ${ZONE}" >&2
215 PROJECT_REPORTED=true
216 fi
217}
218
219# Use gsutil to get the md5 hash for a particular tar
220function gsutil_get_tar_md5() {
221 # location_tar could be local or in the cloud
222 # local tar_location example ./_output/release-tars/kubernetes-server-linux-amd64.tar.gz
223 # cloud tar_location example gs://kubernetes-staging-PROJECT/kubernetes-devel/kubernetes-server-linux-amd64.tar.gz
224 local -r tar_location=$1
225 #parse the output and return the md5 hash
226 #the sed command at the end removes whitespace
227 local -r tar_md5=$(gsutil hash -h -m "${tar_location}" 2>/dev/null | grep "Hash (md5):" | awk -F ':' '{print $2}' | sed 's/^[[:space:]]*//g')
228 echo "${tar_md5}"
229}
230
231# Copy a release tar and its accompanying hash.
232function copy-to-staging() {
233 local -r staging_path=$1
234 local -r gs_url=$2
235 local -r tar=$3
236 local -r hash=$4
237 local -r basename_tar=$(basename "${tar}")
238
239 #check whether this tar alread exists and has the same hash
240 #if it matches, then don't bother uploading it again
241
242 #remote_tar_md5 checks the remote location for the existing tarball and its md5
243 #staging_path example gs://kubernetes-staging-PROJECT/kubernetes-devel
244 #basename_tar example kubernetes-server-linux-amd64.tar.gz
245 local -r remote_tar_md5=$(gsutil_get_tar_md5 "${staging_path}/${basename_tar}")
246 if [[ -n ${remote_tar_md5} ]]; then
247 #local_tar_md5 checks the remote location for the existing tarball and its md5 hash
248 #tar example ./_output/release-tars/kubernetes-server-linux-amd64.tar.gz
249 local -r local_tar_md5=$(gsutil_get_tar_md5 "${tar}")
250 if [[ "${remote_tar_md5}" == "${local_tar_md5}" ]]; then
251 echo "+++ ${basename_tar} uploaded earlier, cloud and local file md5 match (md5 = ${local_tar_md5})"
252 return 0
253 fi
254 fi
255
256 echo "${hash}" > "${tar}.sha512"
257 gsutil -m -q -h "Cache-Control:private, max-age=0" cp "${tar}" "${tar}.sha512" "${staging_path}"
258 gsutil -m acl ch -g all:R "${gs_url}" "${gs_url}.sha512" >/dev/null 2>&1 || true
259 echo "+++ ${basename_tar} uploaded (sha512 = ${hash})"
260}
261
262
263# Given the cluster zone, return the list of regional GCS release
264# bucket suffixes for the release in preference order. GCS doesn't
265# give us an API for this, so we hardcode it.
266#
267# Assumed vars:
268# RELEASE_REGION_FALLBACK
269# REGIONAL_KUBE_ADDONS
270# ZONE
271# Vars set:
272# PREFERRED_REGION
273function set-preferred-region() {
274 case ${ZONE} in
275 asia-*)
276 PREFERRED_REGION=("asia-northeast1" "us-central1" "europe-west6")
277 ;;
278 europe-*)
279 PREFERRED_REGION=("europe-west6" "us-central1" "asia-northeast1")
280 ;;
281 *)
282 PREFERRED_REGION=("us-central1" "europe-west6" "asia-northeast1")
283 ;;
284 esac
285
286 if [[ "${RELEASE_REGION_FALLBACK}" != "true" ]]; then
287 PREFERRED_REGION=( "${PREFERRED_REGION[0]}" )
288 fi
289}
290
291# Take the local tar files and upload them to Google Storage. They will then be
292# downloaded by the master as part of the start up script for the master.
293#
294# Assumed vars:
295# PROJECT
296# SERVER_BINARY_TAR
297# KUBE_MANIFESTS_TAR
298# ZONE
299# Vars set:
300# SERVER_BINARY_TAR_URL
301# SERVER_BINARY_TAR_HASH
302# NODE_BINARY_TAR_URL
303# NODE_BINARY_TAR_HASH
304# KUBE_MANIFESTS_TAR_URL
305# KUBE_MANIFESTS_TAR_HASH
306function upload-tars() {
307 SERVER_BINARY_TAR_URL=
308 SERVER_BINARY_TAR_HASH=
309 NODE_BINARY_TAR_URL=
310 NODE_BINARY_TAR_HASH=
311 KUBE_MANIFESTS_TAR_URL=
312 KUBE_MANIFESTS_TAR_HASH=
313
314 local project_hash
315 if which md5 > /dev/null 2>&1; then
316 project_hash=$(md5 -q -s "$PROJECT")
317 else
318 project_hash=$(echo -n "$PROJECT" | md5sum)
319 project_hash=${project_hash%%[[:blank:]]*}
320 fi
321
322 # This requires 1 million projects before the probability of collision is 50%
323 # that's probably good enough for now :P
324 project_hash=${project_hash:0:10}
325
326 set-preferred-region
327
328 if [[ "${ENABLE_DOCKER_REGISTRY_CACHE:-}" == "true" ]]; then
329 DOCKER_REGISTRY_MIRROR_URL="https://mirror.gcr.io"
330 fi
331
332 SERVER_BINARY_TAR_HASH=$(sha512sum-file "${SERVER_BINARY_TAR}")
333
334 if [[ -n "${NODE_BINARY_TAR:-}" ]]; then
335 NODE_BINARY_TAR_HASH=$(sha512sum-file "${NODE_BINARY_TAR}")
336 fi
337 if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then
338 KUBE_MANIFESTS_TAR_HASH=$(sha512sum-file "${KUBE_MANIFESTS_TAR}")
339 fi
340
341 local server_binary_tar_urls=()
342 local node_binary_tar_urls=()
343
344 for region in "${PREFERRED_REGION[@]}"; do
345 suffix="-${region}"
346 local staging_bucket="gs://kubernetes-staging-${project_hash}${suffix}"
347
348 # Ensure the buckets are created
349 if ! gsutil ls "${staging_bucket}" >/dev/null; then
350 echo "Creating ${staging_bucket}"
351 gsutil mb -l "${region}" -p "${PROJECT}" "${staging_bucket}"
352 fi
353
354 local staging_path="${staging_bucket}/${INSTANCE_PREFIX}-devel"
355
356 echo "+++ Staging tars to Google Storage: ${staging_path}"
357 local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}"
358 copy-to-staging "${staging_path}" "${server_binary_gs_url}" "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_HASH}"
359
360 if [[ -n "${NODE_BINARY_TAR:-}" ]]; then
361 local node_binary_gs_url="${staging_path}/${NODE_BINARY_TAR##*/}"
362 copy-to-staging "${staging_path}" "${node_binary_gs_url}" "${NODE_BINARY_TAR}" "${NODE_BINARY_TAR_HASH}"
363 fi
364
365 # Convert from gs:// URL to an https:// URL
366 server_binary_tar_urls+=("${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}")
367 if [[ -n "${NODE_BINARY_TAR:-}" ]]; then
368 node_binary_tar_urls+=("${node_binary_gs_url/gs:\/\//https://storage.googleapis.com/}")
369 fi
370 if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then
371 local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}"
372 copy-to-staging "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}"
373 # Convert from gs:// URL to an https:// URL
374 kube_manifests_tar_urls+=("${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}")
375 fi
376 done
377
378 SERVER_BINARY_TAR_URL=$(join_csv "${server_binary_tar_urls[@]}")
379 if [[ -n "${NODE_BINARY_TAR:-}" ]]; then
380 NODE_BINARY_TAR_URL=$(join_csv "${node_binary_tar_urls[@]}")
381 fi
382 if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then
383 KUBE_MANIFESTS_TAR_URL=$(join_csv "${kube_manifests_tar_urls[@]}")
384 fi
385}
386
387# Detect Linux and Windows nodes created in the instance group.
388#
389# Assumed vars:
390# NODE_INSTANCE_PREFIX
391# WINDOWS_NODE_INSTANCE_PREFIX
392# Vars set:
393# NODE_NAMES
394# INSTANCE_GROUPS
395# WINDOWS_NODE_NAMES
396# WINDOWS_INSTANCE_GROUPS
397function detect-node-names() {
398 detect-project
399 INSTANCE_GROUPS=()
400 kube::util::read-array INSTANCE_GROUPS < <(gcloud compute instance-groups managed list \
401 --project "${PROJECT}" \
402 --filter "name ~ '${NODE_INSTANCE_PREFIX}-.+' AND zone:(${ZONE})" \
403 --format='value(name)' || true)
404 WINDOWS_INSTANCE_GROUPS=()
405 kube::util::read-array WINDOWS_INSTANCE_GROUPS < <(gcloud compute instance-groups managed list \
406 --project "${PROJECT}" \
407 --filter "name ~ '${WINDOWS_NODE_INSTANCE_PREFIX}-.+' AND zone:(${ZONE})" \
408 --format='value(name)' || true)
409
410 NODE_NAMES=()
411 if [[ -n "${INSTANCE_GROUPS[*]:-}" ]]; then
412 for group in "${INSTANCE_GROUPS[@]}"; do
413 kube::util::read-array NODE_NAMES < <(gcloud compute instance-groups managed list-instances \
414 "${group}" --zone "${ZONE}" --project "${PROJECT}" \
415 --format='value(name)')
416 done
417 fi
418 # Add heapster node name to the list too (if it exists).
419 if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then
420 NODE_NAMES+=("${NODE_INSTANCE_PREFIX}-heapster")
421 fi
422 export NODE_NAMES
423 WINDOWS_NODE_NAMES=()
424 if [[ -n "${WINDOWS_INSTANCE_GROUPS[*]:-}" ]]; then
425 for group in "${WINDOWS_INSTANCE_GROUPS[@]}"; do
426 kube::util::read-array WINDOWS_NODE_NAMES < <(gcloud compute instance-groups managed \
427 list-instances "${group}" --zone "${ZONE}" --project "${PROJECT}" \
428 --format='value(name)')
429 done
430 fi
431 export WINDOWS_NODE_NAMES
432
433 echo "INSTANCE_GROUPS=${INSTANCE_GROUPS[*]:-}" >&2
434 echo "NODE_NAMES=${NODE_NAMES[*]:-}" >&2
435}
436
437# Detect the information about the minions
438#
439# Assumed vars:
440# ZONE
441# Vars set:
442# NODE_NAMES
443# KUBE_NODE_IP_ADDRESSES (array)
444function detect-nodes() {
445 detect-project
446 detect-node-names
447 KUBE_NODE_IP_ADDRESSES=()
448 for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
449 local node_ip
450 node_ip=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \
451 "${NODE_NAMES[$i]}" --format='value(networkInterfaces[0].accessConfigs[0].natIP)')
452 if [[ -z "${node_ip-}" ]] ; then
453 echo "Did not find ${NODE_NAMES[$i]}" >&2
454 else
455 echo "Found ${NODE_NAMES[$i]} at ${node_ip}"
456 KUBE_NODE_IP_ADDRESSES+=("${node_ip}")
457 fi
458 done
459 if [[ -z "${KUBE_NODE_IP_ADDRESSES-}" ]]; then
460 echo "Could not detect Kubernetes minion nodes. Make sure you've launched a cluster with 'kube-up.sh'" >&2
461 exit 1
462 fi
463}
464
465# Detect the IP for the master
466#
467# Assumed vars:
468# MASTER_NAME
469# ZONE
470# REGION
471# Vars set:
472# KUBE_MASTER
473# KUBE_MASTER_IP
474function detect-master() {
475 detect-project
476 KUBE_MASTER=${MASTER_NAME}
477 echo "Trying to find master named '${MASTER_NAME}'" >&2
478 if [[ -z "${KUBE_MASTER_IP-}" ]]; then
479 local master_address_name="${MASTER_NAME}-ip"
480 echo "Looking for address '${master_address_name}'" >&2
481 if ! KUBE_MASTER_IP=$(gcloud compute addresses describe "${master_address_name}" \
482 --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') || \
483 [[ -z "${KUBE_MASTER_IP-}" ]]; then
484 echo "Could not detect Kubernetes master node. Make sure you've launched a cluster with 'kube-up.sh'" >&2
485 exit 1
486 fi
487 fi
488 if [[ -z "${KUBE_MASTER_INTERNAL_IP-}" ]] && [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
489 local master_address_name="${MASTER_NAME}-internal-ip"
490 echo "Looking for address '${master_address_name}'" >&2
491 if ! KUBE_MASTER_INTERNAL_IP=$(gcloud compute addresses describe "${master_address_name}" \
492 --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') || \
493 [[ -z "${KUBE_MASTER_INTERNAL_IP-}" ]]; then
494 echo "Could not detect Kubernetes master node. Make sure you've launched a cluster with 'kube-up.sh'" >&2
495 exit 1
496 fi
497 fi
498 echo "Using master: $KUBE_MASTER (external IP: $KUBE_MASTER_IP; internal IP: ${KUBE_MASTER_INTERNAL_IP:-(not set)})" >&2
499}
500
501function load-or-gen-kube-bearertoken() {
502 if [[ -n "${KUBE_CONTEXT:-}" ]]; then
503 get-kubeconfig-bearertoken
504 fi
505 if [[ -z "${KUBE_BEARER_TOKEN:-}" ]]; then
506 gen-kube-bearertoken
507 fi
508}
509
510# Figure out which binary use on the server and assure it is available.
511# If KUBE_VERSION is specified use binaries specified by it, otherwise
512# use local dev binaries.
513#
514# Assumed vars:
515# KUBE_VERSION
516# KUBE_RELEASE_VERSION_REGEX
517# KUBE_CI_VERSION_REGEX
518# Vars set:
519# KUBE_TAR_HASH
520# SERVER_BINARY_TAR_URL
521# SERVER_BINARY_TAR_HASH
522function tars_from_version() {
523 local sha512sum=""
524 if which sha512sum >/dev/null 2>&1; then
525 sha512sum="sha512sum"
526 else
527 sha512sum="shasum -a512"
528 fi
529
530 if [[ -z "${KUBE_VERSION-}" ]]; then
531 find-release-tars
532 upload-tars
533 elif [[ ${KUBE_VERSION} =~ ${KUBE_RELEASE_VERSION_REGEX} ]]; then
534 SERVER_BINARY_TAR_URL="https://dl.k8s.io/release/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz"
535 # TODO: Clean this up.
536 KUBE_MANIFESTS_TAR_URL="${SERVER_BINARY_TAR_URL/server-linux-amd64/manifests}"
537 KUBE_MANIFESTS_TAR_HASH=$(curl -L "${KUBE_MANIFESTS_TAR_URL}" --silent --show-error | ${sha512sum})
538 KUBE_MANIFESTS_TAR_HASH=${KUBE_MANIFESTS_TAR_HASH%%[[:blank:]]*}
539 elif [[ ${KUBE_VERSION} =~ ${KUBE_CI_VERSION_REGEX} ]]; then
540 SERVER_BINARY_TAR_URL="https://storage.googleapis.com/k8s-release-dev/ci/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz"
541 # TODO: Clean this up.
542 KUBE_MANIFESTS_TAR_URL="${SERVER_BINARY_TAR_URL/server-linux-amd64/manifests}"
543 KUBE_MANIFESTS_TAR_HASH=$(curl "${KUBE_MANIFESTS_TAR_URL}" --silent --show-error | ${sha512sum})
544 KUBE_MANIFESTS_TAR_HASH=${KUBE_MANIFESTS_TAR_HASH%%[[:blank:]]*}
545 else
546 echo "Version doesn't match regexp" >&2
547 exit 1
548 fi
549 if ! SERVER_BINARY_TAR_HASH=$(curl -Ss --fail "${SERVER_BINARY_TAR_URL}.sha512"); then
550 echo "Failure trying to curl release .sha512"
551 fi
552
553 if ! curl -Ss --head "${SERVER_BINARY_TAR_URL}" >&/dev/null; then
554 echo "Can't find release at ${SERVER_BINARY_TAR_URL}" >&2
555 exit 1
556 fi
557}
558
559# Reads kube-env metadata from master
560#
561# Assumed vars:
562# KUBE_MASTER
563# PROJECT
564# ZONE
565function get-master-env() {
566 # TODO(zmerlynn): Make this more reliable with retries.
567 gcloud compute --project "${PROJECT}" ssh --zone "${ZONE}" "${KUBE_MASTER}" --command \
568 "curl --fail --silent -H 'Metadata-Flavor: Google' \
569 'http://metadata/computeMetadata/v1/instance/attributes/kube-env'" 2>/dev/null
570 gcloud compute --project "${PROJECT}" ssh --zone "${ZONE}" "${KUBE_MASTER}" --command \
571 "curl --fail --silent -H 'Metadata-Flavor: Google' \
572 'http://metadata/computeMetadata/v1/instance/attributes/kube-master-certs'" 2>/dev/null
573}
574
575# Quote something appropriate for a yaml string.
576#
577# TODO(zmerlynn): Note that this function doesn't so much "quote" as
578# "strip out quotes", and we really should be using a YAML library for
579# this, but PyYAML isn't shipped by default, and *rant rant rant ... SIGH*
580function yaml-quote {
581 echo "${@:-}" | sed -e "s/'/''/g;s/^/'/i;s/$/'/i"
582}
583
584# Writes the cluster location into a temporary file.
585# Assumed vars
586# ZONE
587function write-cluster-location {
588 cat >"${KUBE_TEMP}/cluster-location.txt" << EOF
589${ZONE}
590EOF
591}
592
593# Writes the cluster name into a temporary file.
594# Assumed vars
595# CLUSTER_NAME
596function write-cluster-name {
597 cat >"${KUBE_TEMP}/cluster-name.txt" << EOF
598${CLUSTER_NAME}
599EOF
600}
601
602function write-master-env {
603 # If the user requested that the master be part of the cluster, set the
604 # environment variable to program the master kubelet to register itself.
605 if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" && -z "${KUBELET_APISERVER:-}" ]]; then
606 KUBELET_APISERVER="${MASTER_NAME}"
607 fi
608 if [[ -z "${KUBERNETES_MASTER_NAME:-}" ]]; then
609 KUBERNETES_MASTER_NAME="${MASTER_NAME}"
610 fi
611
612 construct-linux-kubelet-flags "master"
613 build-linux-kube-env true "${KUBE_TEMP}/master-kube-env.yaml"
614 build-kubelet-config true "linux" "${KUBE_TEMP}/master-kubelet-config.yaml"
615 build-kube-master-certs "${KUBE_TEMP}/kube-master-certs.yaml"
616}
617
618function write-linux-node-env {
619 if [[ -z "${KUBERNETES_MASTER_NAME:-}" ]]; then
620 KUBERNETES_MASTER_NAME="${MASTER_NAME}"
621 fi
622
623 construct-linux-kubelet-flags "heapster"
624 build-linux-kube-env false "${KUBE_TEMP}/heapster-kube-env.yaml"
625 construct-linux-kubelet-flags "node"
626 build-linux-kube-env false "${KUBE_TEMP}/node-kube-env.yaml"
627 build-kubelet-config false "linux" "${KUBE_TEMP}/node-kubelet-config.yaml"
628}
629
630function write-windows-node-env {
631 construct-windows-kubelet-flags
632 construct-windows-kubeproxy-flags
633 build-windows-kube-env "${KUBE_TEMP}/windows-node-kube-env.yaml"
634 build-kubelet-config false "windows" "${KUBE_TEMP}/windows-node-kubelet-config.yaml"
635}
636
637function build-linux-node-labels {
638 local node_type=$1
639 local node_labels=""
640 if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${node_type}" != "master" ]]; then
641 # Add kube-proxy daemonset label to node to avoid situation during cluster
642 # upgrade/downgrade when there are two instances of kube-proxy running on a node.
643 node_labels="node.kubernetes.io/kube-proxy-ds-ready=true"
644 fi
645 if [[ -n "${NODE_LABELS:-}" ]]; then
646 node_labels="${node_labels:+${node_labels},}${NODE_LABELS}"
647 fi
648 if [[ -n "${NON_MASTER_NODE_LABELS:-}" && "${node_type}" != "master" ]]; then
649 node_labels="${node_labels:+${node_labels},}${NON_MASTER_NODE_LABELS}"
650 fi
651 if [[ -n "${MASTER_NODE_LABELS:-}" && "${node_type}" == "master" ]]; then
652 node_labels="${node_labels:+${node_labels},}${MASTER_NODE_LABELS}"
653 fi
654 echo "$node_labels"
655}
656
657function build-windows-node-labels {
658 local node_labels=""
659 if [[ -n "${WINDOWS_NODE_LABELS:-}" ]]; then
660 node_labels="${node_labels:+${node_labels},}${WINDOWS_NODE_LABELS}"
661 fi
662 if [[ -n "${WINDOWS_NON_MASTER_NODE_LABELS:-}" ]]; then
663 node_labels="${node_labels:+${node_labels},}${WINDOWS_NON_MASTER_NODE_LABELS}"
664 fi
665 echo "$node_labels"
666}
667
668# yaml-map-string-stringarray converts the encoded structure to yaml format, and echoes the result
669# under the provided name. If the encoded structure is empty, echoes nothing.
670# 1: name to be output in yaml
671# 2: encoded map-string-string (which may contain duplicate keys - resulting in map-string-stringarray)
672# 3: key-value separator (defaults to ':')
673# 4: item separator (defaults to ',')
674function yaml-map-string-stringarray {
675 declare -r name="${1}"
676 declare -r encoded="${2}"
677 declare -r kv_sep="${3:-:}"
678 declare -r item_sep="${4:-,}"
679
680 declare -a pairs # indexed array
681 declare -A map # associative array
682 IFS="${item_sep}" read -ra pairs <<<"${encoded}" # split on item_sep
683 for pair in "${pairs[@]}"; do
684 declare key
685 declare value
686 IFS="${kv_sep}" read -r key value <<<"${pair}" # split on kv_sep
687 map[$key]="${map[$key]+${map[$key]}${item_sep}}${value}" # append values from duplicate keys
688 done
689 # only output if there is a non-empty map
690 if [[ ${#map[@]} -gt 0 ]]; then
691 echo "${name}:"
692 for k in "${!map[@]}"; do
693 echo " ${k}:"
694 declare -a values
695 IFS="${item_sep}" read -ra values <<<"${map[$k]}"
696 for val in "${values[@]}"; do
697 # declare across two lines so errexit can catch failures
698 declare v
699 v=$(yaml-quote "${val}")
700 echo " - ${v}"
701 done
702 done
703 fi
704}
705
706# yaml-map-string-string converts the encoded structure to yaml format, and echoes the result
707# under the provided name. If the encoded structure is empty, echoes nothing.
708# 1: name to be output in yaml
709# 2: encoded map-string-string (no duplicate keys)
710# 3: bool, whether to yaml-quote the value string in the output (defaults to true)
711# 4: key-value separator (defaults to ':')
712# 5: item separator (defaults to ',')
713function yaml-map-string-string {
714 declare -r name="${1}"
715 declare -r encoded="${2}"
716 declare -r quote_val_string="${3:-true}"
717 declare -r kv_sep="${4:-:}"
718 declare -r item_sep="${5:-,}"
719
720 declare -a pairs # indexed array
721 declare -A map # associative array
722 IFS="${item_sep}" read -ra pairs <<<"${encoded}" # split on item_sep # TODO(mtaufen): try quoting this too
723 for pair in "${pairs[@]}"; do
724 declare key
725 declare value
726 IFS="${kv_sep}" read -r key value <<<"${pair}" # split on kv_sep
727 map[$key]="${value}" # add to associative array
728 done
729 # only output if there is a non-empty map
730 if [[ ${#map[@]} -gt 0 ]]; then
731 echo "${name}:"
732 for k in "${!map[@]}"; do
733 if [[ "${quote_val_string}" == "true" ]]; then
734 # declare across two lines so errexit can catch failures
735 declare v
736 v=$(yaml-quote "${map[$k]}")
737 echo " ${k}: ${v}"
738 else
739 echo " ${k}: ${map[$k]}"
740 fi
741 done
742 fi
743}
744
745# Returns kubelet flags used on both Linux and Windows nodes.
746function construct-common-kubelet-flags {
747 local flags="${KUBELET_TEST_LOG_LEVEL:-"--v=2"} ${KUBELET_TEST_ARGS:-}"
748 flags+=" --cloud-provider=${CLOUD_PROVIDER_FLAG:-external}"
749 # TODO(mtaufen): ROTATE_CERTIFICATES seems unused; delete it?
750 if [[ -n "${ROTATE_CERTIFICATES:-}" ]]; then
751 flags+=" --rotate-certificates=true"
752 fi
753 if [[ -n "${MAX_PODS_PER_NODE:-}" ]]; then
754 flags+=" --max-pods=${MAX_PODS_PER_NODE}"
755 fi
756 echo "$flags"
757}
758
759# Sets KUBELET_ARGS with the kubelet flags for Linux nodes.
760# $1: if 'true', we're rendering flags for a master, else a node
761function construct-linux-kubelet-flags {
762 local node_type="$1"
763 local flags
764 flags="$(construct-common-kubelet-flags)"
765 # Keep in sync with CONTAINERIZED_MOUNTER_HOME in configure-helper.sh
766 flags+=" --experimental-mounter-path=/home/kubernetes/containerized_mounter/mounter"
767 # Keep in sync with the mkdir command in configure-helper.sh (until the TODO is resolved)
768 flags+=" --cert-dir=/var/lib/kubelet/pki/"
769
770 # If ENABLE_AUTH_PROVIDER_GCP is set to true, kubelet is enabled to use out-of-tree auth
771 # credential provider instead of in-tree auth credential provider.
772 # https://kubernetes.io/docs/tasks/kubelet-credential-provider/kubelet-credential-provider
773 if [[ "${ENABLE_AUTH_PROVIDER_GCP:-true}" == "true" ]]; then
774 # Keep the values of --image-credential-provider-config and --image-credential-provider-bin-dir
775 # in sync with value of auth_config_file and auth_provider_dir set in install-auth-provider-gcp function
776 # in gci/configure.sh.
777 flags+=" --image-credential-provider-config=${AUTH_PROVIDER_GCP_LINUX_CONF_FILE}"
778 flags+=" --image-credential-provider-bin-dir=${AUTH_PROVIDER_GCP_LINUX_BIN_DIR}"
779 fi
780
781 if [[ "${node_type}" == "master" ]]; then
782 flags+=" ${MASTER_KUBELET_TEST_ARGS:-}"
783 if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then
784 #TODO(mikedanese): allow static pods to start before creating a client
785 #flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
786 #flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
787 flags+=" --register-with-taints=node-role.kubernetes.io/control-plane=:NoSchedule"
788 flags+=" --kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
789 flags+=" --register-schedulable=false"
790 fi
791 if [[ "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then
792 # Configure the file path for host dns configuration
793 # as ubuntu uses systemd-resolved
794 flags+=" --resolv-conf=/run/systemd/resolve/resolv.conf"
795 fi
796 else # For nodes
797 flags+=" ${NODE_KUBELET_TEST_ARGS:-}"
798 flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
799 flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
800 if [[ "${node_type}" == "heapster" ]]; then
801 flags+=" ${HEAPSTER_KUBELET_TEST_ARGS:-}"
802 fi
803 if [[ "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then
804 # Configure the file path for host dns configuration
805 # as ubuntu uses systemd-resolved
806 flags+=" --resolv-conf=/run/systemd/resolve/resolv.conf"
807 fi
808 fi
809 flags+=" --volume-plugin-dir=${VOLUME_PLUGIN_DIR}"
810 local node_labels
811 node_labels="$(build-linux-node-labels "${node_type}")"
812 if [[ -n "${node_labels:-}" ]]; then
813 flags+=" --node-labels=${node_labels}"
814 fi
815 if [[ -n "${NODE_TAINTS:-}" ]]; then
816 flags+=" --register-with-taints=${NODE_TAINTS}"
817 fi
818
819 CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-unix:///run/containerd/containerd.sock}
820 flags+=" --container-runtime-endpoint=${CONTAINER_RUNTIME_ENDPOINT}"
821
822 if [[ "${CONTAINER_RUNTIME_ENDPOINT}" =~ /containerd.sock$ ]]; then
823 flags+=" --runtime-cgroups=/system.slice/containerd.service"
824 fi
825
826 KUBELET_ARGS="${flags}"
827}
828
829# Sets KUBELET_ARGS with the kubelet flags for Windows nodes.
830# Note that to configure flags with explicit empty string values, we can't escape
831# double-quotes, because they still break sc.exe after expansion in the
832# binPath parameter, and single-quotes get parsed as characters instead of
833# string delimiters.
834function construct-windows-kubelet-flags {
835 local flags
836 flags="$(construct-common-kubelet-flags)"
837
838 # Note: NODE_KUBELET_TEST_ARGS is empty in typical kube-up runs.
839 flags+=" ${NODE_KUBELET_TEST_ARGS:-}"
840
841 local node_labels
842 node_labels="$(build-windows-node-labels)"
843 if [[ -n "${node_labels:-}" ]]; then
844 flags+=" --node-labels=${node_labels}"
845 fi
846
847 # Concatenate common and windows-only node taints and apply them.
848 local node_taints="${NODE_TAINTS:-}"
849 if [[ -n "${node_taints}" && -n "${WINDOWS_NODE_TAINTS:-}" ]]; then
850 node_taints+=":${WINDOWS_NODE_TAINTS}"
851 else
852 node_taints="${WINDOWS_NODE_TAINTS:-}"
853 fi
854 if [[ -n "${node_taints}" ]]; then
855 flags+=" --register-with-taints=${node_taints}"
856 fi
857
858 # Many of these flags were adapted from
859 # https://github.com/Microsoft/SDN/blob/master/Kubernetes/windows/start-kubelet.ps1.
860 flags+=" --config=${WINDOWS_KUBELET_CONFIG_FILE}"
861 flags+=" --kubeconfig=${WINDOWS_KUBECONFIG_FILE}"
862
863 # The directory where the TLS certs are located.
864 flags+=" --cert-dir=${WINDOWS_PKI_DIR}"
865 flags+=" --pod-manifest-path=${WINDOWS_MANIFESTS_DIR}"
866
867 # Configure kubelet to run as a windows service.
868 flags+=" --windows-service=true"
869
870 # Configure the file path for host dns configuration
871 flags+=" --resolv-conf=${WINDOWS_CNI_DIR}\hostdns.conf"
872
873 # Both --cgroups-per-qos and --enforce-node-allocatable should be disabled on
874 # windows; the latter requires the former to be enabled to work.
875 flags+=" --cgroups-per-qos=false --enforce-node-allocatable="
876
877 # Turn off kernel memory cgroup notification.
878 flags+=" --kernel-memcg-notification=false"
879
880 WINDOWS_CONTAINER_RUNTIME_ENDPOINT=${KUBE_WINDOWS_CONTAINER_RUNTIME_ENDPOINT:-npipe:////./pipe/containerd-containerd}
881 flags+=" --container-runtime-endpoint=${WINDOWS_CONTAINER_RUNTIME_ENDPOINT}"
882
883 # If ENABLE_AUTH_PROVIDER_GCP is set to true, kubelet is enabled to use out-of-tree auth
884 # credential provider. https://kubernetes.io/docs/tasks/kubelet-credential-provider/kubelet-credential-provider
885 if [[ "${ENABLE_AUTH_PROVIDER_GCP:-true}" == "true" ]]; then
886 flags+=" --image-credential-provider-config=${AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE}"
887 flags+=" --image-credential-provider-bin-dir=${AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR}"
888 fi
889
890 KUBELET_ARGS="${flags}"
891}
892
893function construct-windows-kubeproxy-flags {
894 local flags=""
895
896 # Use the same log level as the Kubelet during tests.
897 flags+=" ${KUBELET_TEST_LOG_LEVEL:-"--v=2"}"
898
899 # Windows uses kernelspace proxymode
900 flags+=" --proxy-mode=kernelspace"
901
902 # Configure kube-proxy to run as a windows service.
903 flags+=" --windows-service=true"
904
905 # Enabling Windows DSR mode unlocks newer network features and reduces
906 # port usage for services.
907 # https://techcommunity.microsoft.com/t5/networking-blog/direct-server-return-dsr-in-a-nutshell/ba-p/693710
908 if [[ "${WINDOWS_ENABLE_DSR:-}" == "true" ]]; then
909 flags+=" --feature-gates=WinDSR=true --enable-dsr=true "
910 fi
911
912 # Configure flags with explicit empty string values. We can't escape
913 # double-quotes, because they still break sc.exe after expansion in the
914 # binPath parameter, and single-quotes get parsed as characters instead
915 # of string delimiters.
916
917 KUBEPROXY_ARGS="${flags}"
918}
919
920# $1: if 'true', we're rendering config for a master, else a node
921function build-kubelet-config {
922 local master="$1"
923 local os="$2"
924 local file="$3"
925
926 rm -f "${file}"
927 {
928 print-common-kubelet-config
929 if [[ "${master}" == "true" ]]; then
930 print-master-kubelet-config
931 else
932 print-common-node-kubelet-config
933 if [[ "${os}" == "linux" ]]; then
934 print-linux-node-kubelet-config
935 elif [[ "${os}" == "windows" ]]; then
936 print-windows-node-kubelet-config
937 else
938 echo "Unknown OS ${os}" >&2
939 exit 1
940 fi
941 fi
942 } > "${file}"
943}
944
945# cat the Kubelet config yaml in common between masters, linux nodes, and
946# windows nodes
947function print-common-kubelet-config {
948 declare quoted_dns_server_ip
949 declare quoted_dns_domain
950 quoted_dns_server_ip=$(yaml-quote "${DNS_SERVER_IP}")
951 quoted_dns_domain=$(yaml-quote "${DNS_DOMAIN}")
952 cat <<EOF
953kind: KubeletConfiguration
954apiVersion: kubelet.config.k8s.io/v1beta1
955cgroupRoot: /
956clusterDNS:
957 - ${quoted_dns_server_ip}
958clusterDomain: ${quoted_dns_domain}
959readOnlyPort: 10255
960EOF
961
962 # Note: ENABLE_MANIFEST_URL is used by GKE.
963 # TODO(mtaufen): remove this since it's not used in kubernetes/kubernetes nor
964 # kubernetes/test-infra.
965 if [[ "${ENABLE_MANIFEST_URL:-}" == "true" ]]; then
966 declare quoted_manifest_url
967 quoted_manifest_url=$(yaml-quote "${MANIFEST_URL}")
968 cat <<EOF
969staticPodURL: ${quoted_manifest_url}
970EOF
971 yaml-map-string-stringarray 'staticPodURLHeader' "${MANIFEST_URL_HEADER}"
972 fi
973
974 if [[ -n "${EVICTION_HARD:-}" ]]; then
975 yaml-map-string-string 'evictionHard' "${EVICTION_HARD}" true '<'
976 fi
977
978 if [[ -n "${FEATURE_GATES:-}" ]]; then
979 yaml-map-string-string 'featureGates' "${FEATURE_GATES}" false '='
980 fi
981}
982
983# cat the Kubelet config yaml for masters
984function print-master-kubelet-config {
985 cat <<EOF
986enableDebuggingHandlers: ${MASTER_KUBELET_ENABLE_DEBUGGING_HANDLERS:-false}
987hairpinMode: none
988staticPodPath: /etc/kubernetes/manifests
989authentication:
990 webhook:
991 enabled: false
992 anonymous:
993 enabled: true
994authorization:
995 mode: AlwaysAllow
996EOF
997 if [[ "${REGISTER_MASTER_KUBELET:-false}" == "false" ]]; then
998 # Note: Standalone mode is used by GKE
999 declare quoted_master_ip_range
1000 quoted_master_ip_range=$(yaml-quote "${MASTER_IP_RANGE}")
1001 cat <<EOF
1002podCidr: ${quoted_master_ip_range}
1003EOF
1004 fi
1005}
1006
1007# cat the Kubelet config yaml in common between linux nodes and windows nodes
1008function print-common-node-kubelet-config {
1009 cat <<EOF
1010enableDebuggingHandlers: ${KUBELET_ENABLE_DEBUGGING_HANDLERS:-true}
1011EOF
1012 if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \
1013 [[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \
1014 [[ "${HAIRPIN_MODE:-}" == "none" ]]; then
1015 declare quoted_hairpin_mode
1016 quoted_hairpin_mode=$(yaml-quote "${HAIRPIN_MODE}")
1017 cat <<EOF
1018hairpinMode: ${quoted_hairpin_mode}
1019EOF
1020 fi
1021}
1022
1023# cat the Kubelet config yaml for linux nodes
1024function print-linux-node-kubelet-config {
1025 # Keep authentication.x509.clientCAFile in sync with CA_CERT_BUNDLE_PATH in configure-helper.sh
1026 cat <<EOF
1027staticPodPath: /etc/kubernetes/manifests
1028authentication:
1029 x509:
1030 clientCAFile: /etc/srv/kubernetes/pki/ca-certificates.crt
1031EOF
1032}
1033
1034# cat the Kubelet config yaml for windows nodes
1035function print-windows-node-kubelet-config {
1036 # Notes:
1037 # - We don't run any static pods on Windows nodes yet.
1038
1039 # TODO(mtaufen): Does it make any sense to set eviction thresholds for inodes
1040 # on Windows?
1041
1042 # TODO(pjh, mtaufen): It may make sense to use a different hairpin mode on
1043 # Windows. We're currently using hairpin-veth, but
1044 # https://github.com/Microsoft/SDN/blob/master/Kubernetes/windows/start-kubelet.ps1#L121
1045 # uses promiscuous-bridge.
1046
1047 # TODO(pjh, mtaufen): Does cgroupRoot make sense for Windows?
1048
1049 # Keep authentication.x509.clientCAFile in sync with CA_CERT_BUNDLE_PATH in
1050 # k8s-node-setup.psm1.
1051 cat <<EOF
1052authentication:
1053 x509:
1054 clientCAFile: '${WINDOWS_CA_FILE}'
1055EOF
1056}
1057
1058function build-kube-master-certs {
1059 local file=$1
1060 rm -f "$file"
1061 cat >"$file" <<EOF
1062KUBEAPISERVER_CERT: $(yaml-quote "${KUBEAPISERVER_CERT_BASE64:-}")
1063KUBEAPISERVER_KEY: $(yaml-quote "${KUBEAPISERVER_KEY_BASE64:-}")
1064CA_KEY: $(yaml-quote "${CA_KEY_BASE64:-}")
1065AGGREGATOR_CA_KEY: $(yaml-quote "${AGGREGATOR_CA_KEY_BASE64:-}")
1066REQUESTHEADER_CA_CERT: $(yaml-quote "${REQUESTHEADER_CA_CERT_BASE64:-}")
1067PROXY_CLIENT_CERT: $(yaml-quote "${PROXY_CLIENT_CERT_BASE64:-}")
1068PROXY_CLIENT_KEY: $(yaml-quote "${PROXY_CLIENT_KEY_BASE64:-}")
1069ETCD_APISERVER_CA_KEY: $(yaml-quote "${ETCD_APISERVER_CA_KEY_BASE64:-}")
1070ETCD_APISERVER_CA_CERT: $(yaml-quote "${ETCD_APISERVER_CA_CERT_BASE64:-}")
1071ETCD_APISERVER_SERVER_KEY: $(yaml-quote "${ETCD_APISERVER_SERVER_KEY_BASE64:-}")
1072ETCD_APISERVER_SERVER_CERT: $(yaml-quote "${ETCD_APISERVER_SERVER_CERT_BASE64:-}")
1073ETCD_APISERVER_CLIENT_KEY: $(yaml-quote "${ETCD_APISERVER_CLIENT_KEY_BASE64:-}")
1074ETCD_APISERVER_CLIENT_CERT: $(yaml-quote "${ETCD_APISERVER_CLIENT_CERT_BASE64:-}")
1075CLOUD_PVL_ADMISSION_CA_KEY: $(yaml-quote "${CLOUD_PVL_ADMISSION_CA_KEY_BASE64:-}")
1076CLOUD_PVL_ADMISSION_CA_CERT: $(yaml-quote "${CLOUD_PVL_ADMISSION_CA_CERT_BASE64:-}")
1077CLOUD_PVL_ADMISSION_CERT: $(yaml-quote "${CLOUD_PVL_ADMISSION_CERT_BASE64:-}")
1078CLOUD_PVL_ADMISSION_KEY: $(yaml-quote "${CLOUD_PVL_ADMISSION_KEY_BASE64:-}")
1079KONNECTIVITY_SERVER_CA_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CA_KEY_BASE64:-}")
1080KONNECTIVITY_SERVER_CA_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CA_CERT_BASE64:-}")
1081KONNECTIVITY_SERVER_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CERT_BASE64:-}")
1082KONNECTIVITY_SERVER_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_KEY_BASE64:-}")
1083KONNECTIVITY_SERVER_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CLIENT_CERT_BASE64:-}")
1084KONNECTIVITY_SERVER_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CLIENT_KEY_BASE64:-}")
1085KONNECTIVITY_AGENT_CA_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CA_KEY_BASE64:-}")
1086KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CA_CERT_BASE64:-}")
1087KONNECTIVITY_AGENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CERT_BASE64:-}")
1088KONNECTIVITY_AGENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_KEY_BASE64:-}")
1089KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-}")
1090KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-}")
1091EOF
1092}
1093
1094# $1: if 'true', we're building a master yaml, else a node
1095function build-linux-kube-env {
1096 local master="$1"
1097 local file="$2"
1098
1099 local server_binary_tar_url=$SERVER_BINARY_TAR_URL
1100 local kube_manifests_tar_url="${KUBE_MANIFESTS_TAR_URL:-}"
1101 if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]] || \
1102 [[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "ubuntu" || "${NODE_OS_DISTRIBUTION}" == "custom") ]]; then
1103 # TODO: Support fallback .tar.gz settings on Container Linux
1104 server_binary_tar_url=$(split_csv "${SERVER_BINARY_TAR_URL}")
1105 kube_manifests_tar_url=$(split_csv "${KUBE_MANIFESTS_TAR_URL}")
1106 fi
1107
1108 rm -f "$file"
1109 cat >"$file" <<EOF
1110CLUSTER_NAME: $(yaml-quote "${CLUSTER_NAME}")
1111ENV_TIMESTAMP: $(yaml-quote "$(date -u +%Y-%m-%dT%T%z)")
1112INSTANCE_PREFIX: $(yaml-quote "${INSTANCE_PREFIX}")
1113NODE_INSTANCE_PREFIX: $(yaml-quote "${NODE_INSTANCE_PREFIX}")
1114NODE_TAGS: $(yaml-quote "${NODE_TAGS:-}")
1115NODE_NETWORK: $(yaml-quote "${NETWORK:-}")
1116NODE_SUBNETWORK: $(yaml-quote "${SUBNETWORK:-}")
1117CLUSTER_IP_RANGE: $(yaml-quote "${CLUSTER_IP_RANGE:-10.244.0.0/16}")
1118SERVER_BINARY_TAR_URL: $(yaml-quote "${server_binary_tar_url}")
1119SERVER_BINARY_TAR_HASH: $(yaml-quote "${SERVER_BINARY_TAR_HASH}")
1120PROJECT_ID: $(yaml-quote "${PROJECT}")
1121NETWORK_PROJECT_ID: $(yaml-quote "${NETWORK_PROJECT}")
1122SERVICE_CLUSTER_IP_RANGE: $(yaml-quote "${SERVICE_CLUSTER_IP_RANGE}")
1123KUBERNETES_MASTER_NAME: $(yaml-quote "${KUBERNETES_MASTER_NAME}")
1124ALLOCATE_NODE_CIDRS: $(yaml-quote "${ALLOCATE_NODE_CIDRS:-false}")
1125ENABLE_METRICS_SERVER: $(yaml-quote "${ENABLE_METRICS_SERVER:-false}")
1126ENABLE_METADATA_AGENT: $(yaml-quote "${ENABLE_METADATA_AGENT:-none}")
1127METADATA_AGENT_CPU_REQUEST: $(yaml-quote "${METADATA_AGENT_CPU_REQUEST:-}")
1128METADATA_AGENT_MEMORY_REQUEST: $(yaml-quote "${METADATA_AGENT_MEMORY_REQUEST:-}")
1129METADATA_AGENT_CLUSTER_LEVEL_CPU_REQUEST: $(yaml-quote "${METADATA_AGENT_CLUSTER_LEVEL_CPU_REQUEST:-}")
1130METADATA_AGENT_CLUSTER_LEVEL_MEMORY_REQUEST: $(yaml-quote "${METADATA_AGENT_CLUSTER_LEVEL_MEMORY_REQUEST:-}")
1131DOCKER_REGISTRY_MIRROR_URL: $(yaml-quote "${DOCKER_REGISTRY_MIRROR_URL:-}")
1132ENABLE_L7_LOADBALANCING: $(yaml-quote "${ENABLE_L7_LOADBALANCING:-none}")
1133ENABLE_CLUSTER_LOGGING: $(yaml-quote "${ENABLE_CLUSTER_LOGGING:-false}")
1134ENABLE_AUTH_PROVIDER_GCP: $(yaml-quote "${ENABLE_AUTH_PROVIDER_GCP:-true}")
1135ENABLE_NODE_PROBLEM_DETECTOR: $(yaml-quote "${ENABLE_NODE_PROBLEM_DETECTOR:-none}")
1136NODE_PROBLEM_DETECTOR_VERSION: $(yaml-quote "${NODE_PROBLEM_DETECTOR_VERSION:-}")
1137NODE_PROBLEM_DETECTOR_TAR_HASH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TAR_HASH:-}")
1138NODE_PROBLEM_DETECTOR_RELEASE_PATH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_RELEASE_PATH:-}")
1139NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS: $(yaml-quote "${NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS:-}")
1140CNI_STORAGE_URL_BASE: $(yaml-quote "${CNI_STORAGE_URL_BASE:-}")
1141CNI_TAR_PREFIX: $(yaml-quote "${CNI_TAR_PREFIX:-}")
1142CNI_VERSION: $(yaml-quote "${CNI_VERSION:-}")
1143CNI_HASH: $(yaml-quote "${CNI_HASH:-}")
1144ENABLE_NODE_LOGGING: $(yaml-quote "${ENABLE_NODE_LOGGING:-false}")
1145LOGGING_DESTINATION: $(yaml-quote "${LOGGING_DESTINATION:-}")
1146ELASTICSEARCH_LOGGING_REPLICAS: $(yaml-quote "${ELASTICSEARCH_LOGGING_REPLICAS:-}")
1147ENABLE_CLUSTER_DNS: $(yaml-quote "${ENABLE_CLUSTER_DNS:-false}")
1148CLUSTER_DNS_CORE_DNS: $(yaml-quote "${CLUSTER_DNS_CORE_DNS:-true}")
1149ENABLE_NODELOCAL_DNS: $(yaml-quote "${ENABLE_NODELOCAL_DNS:-false}")
1150DNS_SERVER_IP: $(yaml-quote "${DNS_SERVER_IP:-}")
1151LOCAL_DNS_IP: $(yaml-quote "${LOCAL_DNS_IP:-}")
1152DNS_DOMAIN: $(yaml-quote "${DNS_DOMAIN:-}")
1153DNS_MEMORY_LIMIT: $(yaml-quote "${DNS_MEMORY_LIMIT:-}")
1154ENABLE_DNS_HORIZONTAL_AUTOSCALER: $(yaml-quote "${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}")
1155KUBE_PROXY_DAEMONSET: $(yaml-quote "${KUBE_PROXY_DAEMONSET:-false}")
1156KUBE_PROXY_TOKEN: $(yaml-quote "${KUBE_PROXY_TOKEN:-}")
1157KUBE_PROXY_MODE: $(yaml-quote "${KUBE_PROXY_MODE:-iptables}")
1158DETECT_LOCAL_MODE: $(yaml-quote "${DETECT_LOCAL_MODE:-}")
1159NODE_PROBLEM_DETECTOR_TOKEN: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TOKEN:-}")
1160ADMISSION_CONTROL: $(yaml-quote "${ADMISSION_CONTROL:-}")
1161MASTER_IP_RANGE: $(yaml-quote "${MASTER_IP_RANGE}")
1162RUNTIME_CONFIG: $(yaml-quote "${RUNTIME_CONFIG}")
1163CA_CERT: $(yaml-quote "${CA_CERT_BASE64:-}")
1164KUBELET_CERT: $(yaml-quote "${KUBELET_CERT_BASE64:-}")
1165KUBELET_KEY: $(yaml-quote "${KUBELET_KEY_BASE64:-}")
1166NETWORK_PROVIDER: $(yaml-quote "${NETWORK_PROVIDER:-}")
1167NETWORK_POLICY_PROVIDER: $(yaml-quote "${NETWORK_POLICY_PROVIDER:-}")
1168HAIRPIN_MODE: $(yaml-quote "${HAIRPIN_MODE:-}")
1169E2E_STORAGE_TEST_ENVIRONMENT: $(yaml-quote "${E2E_STORAGE_TEST_ENVIRONMENT:-}")
1170KUBE_DOCKER_REGISTRY: $(yaml-quote "${KUBE_DOCKER_REGISTRY:-}")
1171KUBE_ADDON_REGISTRY: $(yaml-quote "${KUBE_ADDON_REGISTRY:-}")
1172MULTIZONE: $(yaml-quote "${MULTIZONE:-}")
1173MULTIMASTER: $(yaml-quote "${MULTIMASTER:-}")
1174NON_MASQUERADE_CIDR: $(yaml-quote "${NON_MASQUERADE_CIDR:-}")
1175ENABLE_DEFAULT_STORAGE_CLASS: $(yaml-quote "${ENABLE_DEFAULT_STORAGE_CLASS:-}")
1176ENABLE_VOLUME_SNAPSHOTS: $(yaml-quote "${ENABLE_VOLUME_SNAPSHOTS:-}")
1177ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote "${ENABLE_APISERVER_ADVANCED_AUDIT:-}")
1178ENABLE_APISERVER_DYNAMIC_AUDIT: $(yaml-quote "${ENABLE_APISERVER_DYNAMIC_AUDIT:-}")
1179ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote "${ENABLE_CACHE_MUTATION_DETECTOR:-false}")
1180ENABLE_KUBE_WATCHLIST_INCONSISTENCY_DETECTOR: $(yaml-quote "${ENABLE_KUBE_WATCHLIST_INCONSISTENCY_DETECTOR:-false}")
1181ENABLE_PATCH_CONVERSION_DETECTOR: $(yaml-quote "${ENABLE_PATCH_CONVERSION_DETECTOR:-false}")
1182ADVANCED_AUDIT_POLICY: $(yaml-quote "${ADVANCED_AUDIT_POLICY:-}")
1183ADVANCED_AUDIT_BACKEND: $(yaml-quote "${ADVANCED_AUDIT_BACKEND:-log}")
1184ADVANCED_AUDIT_TRUNCATING_BACKEND: $(yaml-quote "${ADVANCED_AUDIT_TRUNCATING_BACKEND:-true}")
1185ADVANCED_AUDIT_LOG_MODE: $(yaml-quote "${ADVANCED_AUDIT_LOG_MODE:-}")
1186ADVANCED_AUDIT_LOG_BUFFER_SIZE: $(yaml-quote "${ADVANCED_AUDIT_LOG_BUFFER_SIZE:-}")
1187ADVANCED_AUDIT_LOG_MAX_BATCH_SIZE: $(yaml-quote "${ADVANCED_AUDIT_LOG_MAX_BATCH_SIZE:-}")
1188ADVANCED_AUDIT_LOG_MAX_BATCH_WAIT: $(yaml-quote "${ADVANCED_AUDIT_LOG_MAX_BATCH_WAIT:-}")
1189ADVANCED_AUDIT_LOG_THROTTLE_QPS: $(yaml-quote "${ADVANCED_AUDIT_LOG_THROTTLE_QPS:-}")
1190ADVANCED_AUDIT_LOG_THROTTLE_BURST: $(yaml-quote "${ADVANCED_AUDIT_LOG_THROTTLE_BURST:-}")
1191ADVANCED_AUDIT_LOG_INITIAL_BACKOFF: $(yaml-quote "${ADVANCED_AUDIT_LOG_INITIAL_BACKOFF:-}")
1192ADVANCED_AUDIT_WEBHOOK_MODE: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_MODE:-}")
1193ADVANCED_AUDIT_WEBHOOK_BUFFER_SIZE: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_BUFFER_SIZE:-}")
1194ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_SIZE: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_SIZE:-}")
1195ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_WAIT: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_WAIT:-}")
1196ADVANCED_AUDIT_WEBHOOK_THROTTLE_QPS: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_THROTTLE_QPS:-}")
1197ADVANCED_AUDIT_WEBHOOK_THROTTLE_BURST: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_THROTTLE_BURST:-}")
1198ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF:-}")
1199GCE_API_ENDPOINT: $(yaml-quote "${GCE_API_ENDPOINT:-}")
1200GCE_GLBC_IMAGE: $(yaml-quote "${GCE_GLBC_IMAGE:-}")
1201CUSTOM_INGRESS_YAML: |
1202${CUSTOM_INGRESS_YAML//\'/\'\'}
1203ENABLE_NODE_JOURNAL: $(yaml-quote "${ENABLE_NODE_JOURNAL:-false}")
1204PROMETHEUS_TO_SD_ENDPOINT: $(yaml-quote "${PROMETHEUS_TO_SD_ENDPOINT:-}")
1205PROMETHEUS_TO_SD_PREFIX: $(yaml-quote "${PROMETHEUS_TO_SD_PREFIX:-}")
1206ENABLE_PROMETHEUS_TO_SD: $(yaml-quote "${ENABLE_PROMETHEUS_TO_SD:-false}")
1207DISABLE_PROMETHEUS_TO_SD_IN_DS: $(yaml-quote "${DISABLE_PROMETHEUS_TO_SD_IN_DS:-false}")
1208CONTAINER_RUNTIME_ENDPOINT: $(yaml-quote "${CONTAINER_RUNTIME_ENDPOINT:-}")
1209CONTAINER_RUNTIME_NAME: $(yaml-quote "${CONTAINER_RUNTIME_NAME:-}")
1210CONTAINER_RUNTIME_TEST_HANDLER: $(yaml-quote "${CONTAINER_RUNTIME_TEST_HANDLER:-}")
1211CONTAINERD_INFRA_CONTAINER: $(yaml-quote "${CONTAINER_INFRA_CONTAINER:-}")
1212UBUNTU_INSTALL_CONTAINERD_VERSION: $(yaml-quote "${UBUNTU_INSTALL_CONTAINERD_VERSION:-}")
1213UBUNTU_INSTALL_RUNC_VERSION: $(yaml-quote "${UBUNTU_INSTALL_RUNC_VERSION:-}")
1214COS_INSTALL_CONTAINERD_VERSION: $(yaml-quote "${COS_INSTALL_CONTAINERD_VERSION:-}")
1215COS_INSTALL_RUNC_VERSION: $(yaml-quote "${COS_INSTALL_RUNC_VERSION:-}")
1216NODE_LOCAL_SSDS_EXT: $(yaml-quote "${NODE_LOCAL_SSDS_EXT:-}")
1217NODE_LOCAL_SSDS_EPHEMERAL: $(yaml-quote "${NODE_LOCAL_SSDS_EPHEMERAL:-}")
1218LOAD_IMAGE_COMMAND: $(yaml-quote "${LOAD_IMAGE_COMMAND:-}")
1219ZONE: $(yaml-quote "${ZONE}")
1220REGION: $(yaml-quote "${REGION}")
1221VOLUME_PLUGIN_DIR: $(yaml-quote "${VOLUME_PLUGIN_DIR}")
1222KUBELET_ARGS: $(yaml-quote "${KUBELET_ARGS}")
1223REQUIRE_METADATA_KUBELET_CONFIG_FILE: $(yaml-quote true)
1224ENABLE_NETD: $(yaml-quote "${ENABLE_NETD:-false}")
1225CUSTOM_NETD_YAML: |
1226${CUSTOM_NETD_YAML//\'/\'\'}
1227CUSTOM_CALICO_NODE_DAEMONSET_YAML: |
1228${CUSTOM_CALICO_NODE_DAEMONSET_YAML//\'/\'\'}
1229CUSTOM_TYPHA_DEPLOYMENT_YAML: |
1230${CUSTOM_TYPHA_DEPLOYMENT_YAML//\'/\'\'}
1231CONCURRENT_SERVICE_SYNCS: $(yaml-quote "${CONCURRENT_SERVICE_SYNCS:-}")
1232AUTH_PROVIDER_GCP_STORAGE_PATH: $(yaml-quote "${AUTH_PROVIDER_GCP_STORAGE_PATH}")
1233AUTH_PROVIDER_GCP_VERSION: $(yaml-quote "${AUTH_PROVIDER_GCP_VERSION}")
1234AUTH_PROVIDER_GCP_LINUX_BIN_DIR: $(yaml-quote "${AUTH_PROVIDER_GCP_LINUX_BIN_DIR}")
1235AUTH_PROVIDER_GCP_LINUX_CONF_FILE: $(yaml-quote "${AUTH_PROVIDER_GCP_LINUX_CONF_FILE}")
1236EOF
1237 if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "gci" ]] || \
1238 [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "gci" ]] || \
1239 [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "cos" ]] || \
1240 [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "cos" ]]; then
1241 cat >>"$file" <<EOF
1242REMOUNT_VOLUME_PLUGIN_DIR: $(yaml-quote "${REMOUNT_VOLUME_PLUGIN_DIR:-true}")
1243EOF
1244 fi
1245 if [[ "${master}" == "false" ]]; then
1246 cat >>"$file" <<EOF
1247KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CA_CERT_BASE64:-}")
1248KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-}")
1249KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-}")
1250EOF
1251 fi
1252 if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then
1253 cat >>"$file" <<EOF
1254KUBE_APISERVER_REQUEST_TIMEOUT: $(yaml-quote "${KUBE_APISERVER_REQUEST_TIMEOUT}")
1255EOF
1256 fi
1257 if [ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]; then
1258 cat >>"$file" <<EOF
1259TERMINATED_POD_GC_THRESHOLD: $(yaml-quote "${TERMINATED_POD_GC_THRESHOLD}")
1260EOF
1261 fi
1262 if [[ "${master}" == "true" && ("${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "ubuntu") ]] || \
1263 [[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" = "ubuntu" || "${NODE_OS_DISTRIBUTION}" = "custom") ]] ; then
1264 cat >>"$file" <<EOF
1265KUBE_MANIFESTS_TAR_URL: $(yaml-quote "${kube_manifests_tar_url}")
1266KUBE_MANIFESTS_TAR_HASH: $(yaml-quote "${KUBE_MANIFESTS_TAR_HASH}")
1267EOF
1268 fi
1269 if [ -n "${TEST_CLUSTER:-}" ]; then
1270 cat >>"$file" <<EOF
1271TEST_CLUSTER: $(yaml-quote "${TEST_CLUSTER}")
1272EOF
1273 fi
1274 if [ -n "${DOCKER_TEST_LOG_LEVEL:-}" ]; then
1275 cat >>"$file" <<EOF
1276DOCKER_TEST_LOG_LEVEL: $(yaml-quote "${DOCKER_TEST_LOG_LEVEL}")
1277EOF
1278 fi
1279 if [ -n "${DOCKER_LOG_DRIVER:-}" ]; then
1280 cat >>"$file" <<EOF
1281DOCKER_LOG_DRIVER: $(yaml-quote "${DOCKER_LOG_DRIVER}")
1282EOF
1283 fi
1284 if [ -n "${DOCKER_LOG_MAX_SIZE:-}" ]; then
1285 cat >>"$file" <<EOF
1286DOCKER_LOG_MAX_SIZE: $(yaml-quote "${DOCKER_LOG_MAX_SIZE}")
1287EOF
1288 fi
1289 if [ -n "${DOCKER_LOG_MAX_FILE:-}" ]; then
1290 cat >>"$file" <<EOF
1291DOCKER_LOG_MAX_FILE: $(yaml-quote "${DOCKER_LOG_MAX_FILE}")
1292EOF
1293 fi
1294 if [ -n "${CLOUD_PROVIDER_FLAG:-}" ]; then
1295 cat >>"$file" <<EOF
1296CLOUD_PROVIDER_FLAG: $(yaml-quote "${CLOUD_PROVIDER_FLAG}")
1297EOF
1298 fi
1299 if [ -n "${FEATURE_GATES:-}" ]; then
1300 cat >>"$file" <<EOF
1301FEATURE_GATES: $(yaml-quote "${FEATURE_GATES}")
1302EOF
1303 fi
1304 if [ -n "${RUN_CONTROLLERS:-}" ]; then
1305 cat >>"$file" <<EOF
1306RUN_CONTROLLERS: $(yaml-quote "${RUN_CONTROLLERS}")
1307EOF
1308 fi
1309 if [ -n "${RUN_CCM_CONTROLLERS:-}" ]; then
1310 cat >>"$file" <<EOF
1311RUN_CCM_CONTROLLERS: $(yaml-quote "${RUN_CCM_CONTROLLERS}")
1312EOF
1313 fi
1314 if [ -n "${PROVIDER_VARS:-}" ]; then
1315 local var_name
1316 local var_value
1317
1318 for var_name in ${PROVIDER_VARS}; do
1319 eval "local var_value=\$(yaml-quote \${${var_name}})"
1320 cat >>"$file" <<EOF
1321${var_name}: ${var_value}
1322EOF
1323 done
1324 fi
1325
1326 if [[ "${master}" == "true" ]]; then
1327 # Master-only env vars.
1328 cat >>"$file" <<EOF
1329KUBERNETES_MASTER: $(yaml-quote 'true')
1330KUBE_USER: $(yaml-quote "${KUBE_USER}")
1331KUBE_PASSWORD: $(yaml-quote "${KUBE_PASSWORD}")
1332KUBE_BEARER_TOKEN: $(yaml-quote "${KUBE_BEARER_TOKEN}")
1333MASTER_CERT: $(yaml-quote "${MASTER_CERT_BASE64:-}")
1334MASTER_KEY: $(yaml-quote "${MASTER_KEY_BASE64:-}")
1335KUBECFG_CERT: $(yaml-quote "${KUBECFG_CERT_BASE64:-}")
1336KUBECFG_KEY: $(yaml-quote "${KUBECFG_KEY_BASE64:-}")
1337KUBELET_APISERVER: $(yaml-quote "${KUBELET_APISERVER:-}")
1338NUM_NODES: $(yaml-quote "${NUM_NODES}")
1339STORAGE_BACKEND: $(yaml-quote "${STORAGE_BACKEND:-etcd3}")
1340STORAGE_MEDIA_TYPE: $(yaml-quote "${STORAGE_MEDIA_TYPE:-}")
1341ENABLE_GARBAGE_COLLECTOR: $(yaml-quote "${ENABLE_GARBAGE_COLLECTOR:-}")
1342ENABLE_LEGACY_ABAC: $(yaml-quote "${ENABLE_LEGACY_ABAC:-}")
1343MASTER_ADVERTISE_ADDRESS: $(yaml-quote "${MASTER_ADVERTISE_ADDRESS:-}")
1344ETCD_CA_KEY: $(yaml-quote "${ETCD_CA_KEY_BASE64:-}")
1345ETCD_CA_CERT: $(yaml-quote "${ETCD_CA_CERT_BASE64:-}")
1346ETCD_PEER_KEY: $(yaml-quote "${ETCD_PEER_KEY_BASE64:-}")
1347ETCD_PEER_CERT: $(yaml-quote "${ETCD_PEER_CERT_BASE64:-}")
1348SERVICEACCOUNT_ISSUER: $(yaml-quote "${SERVICEACCOUNT_ISSUER:-}")
1349KUBECTL_PRUNE_WHITELIST_OVERRIDE: $(yaml-quote "${KUBECTL_PRUNE_WHITELIST_OVERRIDE:-}")
1350CCM_FEATURE_GATES: $(yaml-quote "${CCM_FEATURE_GATES:-}")
1351KUBE_SCHEDULER_RUNASUSER: 2001
1352KUBE_SCHEDULER_RUNASGROUP: 2001
1353KUBE_ADDON_MANAGER_RUNASUSER: 2002
1354KUBE_ADDON_MANAGER_RUNASGROUP: 2002
1355KUBE_CONTROLLER_MANAGER_RUNASUSER: 2003
1356KUBE_CONTROLLER_MANAGER_RUNASGROUP: 2003
1357KUBE_API_SERVER_RUNASUSER: 2004
1358KUBE_API_SERVER_RUNASGROUP: 2004
1359KUBE_PKI_READERS_GROUP: 2005
1360ETCD_RUNASUSER: 2006
1361ETCD_RUNASGROUP: 2006
1362KUBE_POD_LOG_READERS_GROUP: 2007
1363KONNECTIVITY_SERVER_RUNASUSER: 2008
1364KONNECTIVITY_SERVER_RUNASGROUP: 2008
1365KONNECTIVITY_SERVER_SOCKET_WRITER_GROUP: 2008
1366CLOUD_CONTROLLER_MANAGER_RUNASUSER: 2009
1367CLOUD_CONTROLLER_MANAGER_RUNASGROUP: 2009
1368CLUSTER_AUTOSCALER_RUNASUSER: 2010
1369CLUSTER_AUTOSCALER_RUNASGROUP: 2010
1370
1371EOF
1372 # KUBE_APISERVER_REQUEST_TIMEOUT_SEC (if set) controls the --request-timeout
1373 # flag
1374 if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT_SEC:-}" ]; then
1375 cat >>"$file" <<EOF
1376KUBE_APISERVER_REQUEST_TIMEOUT_SEC: $(yaml-quote "${KUBE_APISERVER_REQUEST_TIMEOUT_SEC}")
1377EOF
1378 fi
1379 # KUBE_APISERVER_GODEBUG (if set) controls the value of GODEBUG env var for kube-apiserver.
1380 if [ -n "${KUBE_APISERVER_GODEBUG:-}" ]; then
1381 cat >>"$file" <<EOF
1382KUBE_APISERVER_GODEBUG: $(yaml-quote "${KUBE_APISERVER_GODEBUG}")
1383EOF
1384 fi
1385 # ETCD_IMAGE (if set) allows to use a custom etcd image.
1386 if [ -n "${ETCD_IMAGE:-}" ]; then
1387 cat >>"$file" <<EOF
1388ETCD_IMAGE: $(yaml-quote "${ETCD_IMAGE}")
1389EOF
1390 fi
1391 # ETCD_DOCKER_REPOSITORY (if set) allows to use a custom etcd docker repository to pull the etcd image from.
1392 if [ -n "${ETCD_DOCKER_REPOSITORY:-}" ]; then
1393 cat >>"$file" <<EOF
1394ETCD_DOCKER_REPOSITORY: $(yaml-quote "${ETCD_DOCKER_REPOSITORY}")
1395EOF
1396 fi
1397 # ETCD_VERSION (if set) allows you to use custom version of etcd.
1398 # The main purpose of using it may be rollback of etcd v3 API,
1399 # where we need 3.0.* image, but are rolling back to 2.3.7.
1400 if [ -n "${ETCD_VERSION:-}" ]; then
1401 cat >>"$file" <<EOF
1402ETCD_VERSION: $(yaml-quote "${ETCD_VERSION}")
1403EOF
1404 fi
1405 if [ -n "${ETCD_HOSTNAME:-}" ]; then
1406 cat >>"$file" <<EOF
1407ETCD_HOSTNAME: $(yaml-quote "${ETCD_HOSTNAME}")
1408EOF
1409 fi
1410 if [ -n "${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC:-}" ]; then
1411 cat >>"$file" <<EOF
1412ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC: $(yaml-quote "${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC}")
1413EOF
1414 fi
1415 if [ -n "${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC:-}" ]; then
1416 cat >>"$file" <<EOF
1417KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC: $(yaml-quote "${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC}")
1418EOF
1419 fi
1420 if [ -n "${ETCD_COMPACTION_INTERVAL_SEC:-}" ]; then
1421 cat >>"$file" <<EOF
1422ETCD_COMPACTION_INTERVAL_SEC: $(yaml-quote "${ETCD_COMPACTION_INTERVAL_SEC}")
1423EOF
1424 fi
1425 if [ -n "${ETCD_QUOTA_BACKEND_BYTES:-}" ]; then
1426 cat >>"$file" <<EOF
1427ETCD_QUOTA_BACKEND_BYTES: $(yaml-quote "${ETCD_QUOTA_BACKEND_BYTES}")
1428EOF
1429 fi
1430 if [ -n "${ETCD_EXTRA_ARGS:-}" ]; then
1431 cat >>"$file" <<EOF
1432ETCD_EXTRA_ARGS: $(yaml-quote "${ETCD_EXTRA_ARGS}")
1433EOF
1434 fi
1435 if [ -n "${ETCD_SERVERS:-}" ]; then
1436 cat >>"$file" <<EOF
1437ETCD_SERVERS: $(yaml-quote "${ETCD_SERVERS}")
1438EOF
1439 fi
1440 if [ -n "${ETCD_SERVERS_OVERRIDES:-}" ]; then
1441 cat >>"$file" <<EOF
1442ETCD_SERVERS_OVERRIDES: $(yaml-quote "${ETCD_SERVERS_OVERRIDES}")
1443EOF
1444 fi
1445 if [ -n "${APISERVER_TEST_ARGS:-}" ]; then
1446 cat >>"$file" <<EOF
1447APISERVER_TEST_ARGS: $(yaml-quote "${APISERVER_TEST_ARGS}")
1448EOF
1449 fi
1450 if [ -n "${CONTROLLER_MANAGER_TEST_ARGS:-}" ]; then
1451 cat >>"$file" <<EOF
1452CONTROLLER_MANAGER_TEST_ARGS: $(yaml-quote "${CONTROLLER_MANAGER_TEST_ARGS}")
1453EOF
1454 fi
1455 if [ -n "${KUBE_CONTROLLER_MANAGER_TEST_ARGS:-}" ]; then
1456 cat >>"$file" <<EOF
1457KUBE_CONTROLLER_MANAGER_TEST_ARGS: $(yaml-quote "${KUBE_CONTROLLER_MANAGER_TEST_ARGS}")
1458EOF
1459 fi
1460 if [ -n "${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-}" ]; then
1461 cat >>"$file" <<EOF
1462CONTROLLER_MANAGER_TEST_LOG_LEVEL: $(yaml-quote "${CONTROLLER_MANAGER_TEST_LOG_LEVEL}")
1463EOF
1464 fi
1465 if [ -n "${SCHEDULER_TEST_ARGS:-}" ]; then
1466 cat >>"$file" <<EOF
1467SCHEDULER_TEST_ARGS: $(yaml-quote "${SCHEDULER_TEST_ARGS}")
1468EOF
1469 fi
1470 if [ -n "${SCHEDULER_TEST_LOG_LEVEL:-}" ]; then
1471 cat >>"$file" <<EOF
1472SCHEDULER_TEST_LOG_LEVEL: $(yaml-quote "${SCHEDULER_TEST_LOG_LEVEL}")
1473EOF
1474 fi
1475 if [ -n "${INITIAL_ETCD_CLUSTER:-}" ]; then
1476 cat >>"$file" <<EOF
1477INITIAL_ETCD_CLUSTER: $(yaml-quote "${INITIAL_ETCD_CLUSTER}")
1478EOF
1479 fi
1480 if [ -n "${INITIAL_ETCD_CLUSTER_STATE:-}" ]; then
1481 cat >>"$file" <<EOF
1482INITIAL_ETCD_CLUSTER_STATE: $(yaml-quote "${INITIAL_ETCD_CLUSTER_STATE}")
1483EOF
1484 fi
1485 if [ -n "${CLUSTER_SIGNING_DURATION:-}" ]; then
1486 cat >>"$file" <<EOF
1487CLUSTER_SIGNING_DURATION: $(yaml-quote "${CLUSTER_SIGNING_DURATION}")
1488EOF
1489 fi
1490 if [[ "${NODE_ACCELERATORS:-}" == *"type=nvidia"* ]]; then
1491 cat >>"$file" <<EOF
1492ENABLE_NVIDIA_GPU_DEVICE_PLUGIN: $(yaml-quote "true")
1493EOF
1494 fi
1495 if [ -n "${ADDON_MANAGER_LEADER_ELECTION:-}" ]; then
1496 cat >>"$file" <<EOF
1497ADDON_MANAGER_LEADER_ELECTION: $(yaml-quote "${ADDON_MANAGER_LEADER_ELECTION}")
1498EOF
1499 fi
1500 if [ -n "${API_SERVER_TEST_LOG_LEVEL:-}" ]; then
1501 cat >>"$file" <<EOF
1502API_SERVER_TEST_LOG_LEVEL: $(yaml-quote "${API_SERVER_TEST_LOG_LEVEL}")
1503EOF
1504 fi
1505 if [ -n "${ETCD_LISTEN_CLIENT_IP:-}" ]; then
1506 cat >>"$file" <<EOF
1507ETCD_LISTEN_CLIENT_IP: $(yaml-quote "${ETCD_LISTEN_CLIENT_IP}")
1508EOF
1509 fi
1510 if [ -n "${ETCD_PROGRESS_NOTIFY_INTERVAL:-}" ]; then
1511 cat >>"$file" <<EOF
1512ETCD_PROGRESS_NOTIFY_INTERVAL: $(yaml-quote "${ETCD_PROGRESS_NOTIFY_INTERVAL}")
1513EOF
1514 fi
1515
1516 else
1517 # Node-only env vars.
1518 cat >>"$file" <<EOF
1519KUBERNETES_MASTER: $(yaml-quote "false")
1520EXTRA_DOCKER_OPTS: $(yaml-quote "${EXTRA_DOCKER_OPTS:-}")
1521EOF
1522 if [ -n "${KUBEPROXY_TEST_ARGS:-}" ]; then
1523 cat >>"$file" <<EOF
1524KUBEPROXY_TEST_ARGS: $(yaml-quote "${KUBEPROXY_TEST_ARGS}")
1525EOF
1526 fi
1527 if [ -n "${KUBEPROXY_TEST_LOG_LEVEL:-}" ]; then
1528 cat >>"$file" <<EOF
1529KUBEPROXY_TEST_LOG_LEVEL: $(yaml-quote "${KUBEPROXY_TEST_LOG_LEVEL}")
1530EOF
1531 fi
1532 fi
1533 if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then
1534 cat >>"$file" <<EOF
1535ENABLE_CLUSTER_AUTOSCALER: $(yaml-quote "${ENABLE_CLUSTER_AUTOSCALER}")
1536AUTOSCALER_MIG_CONFIG: $(yaml-quote "${AUTOSCALER_MIG_CONFIG}")
1537AUTOSCALER_EXPANDER_CONFIG: $(yaml-quote "${AUTOSCALER_EXPANDER_CONFIG}")
1538EOF
1539 if [[ "${master}" == "false" ]]; then
1540 # TODO(kubernetes/autoscaler#718): AUTOSCALER_ENV_VARS is a hotfix for cluster autoscaler,
1541 # which reads the kube-env to determine the shape of a node and was broken by #60020.
1542 # This should be removed as soon as a more reliable source of information is available!
1543 local node_labels
1544 local node_taints
1545 local autoscaler_env_vars
1546 node_labels="$(build-linux-node-labels node)"
1547 node_taints="${NODE_TAINTS:-}"
1548 autoscaler_env_vars="node_labels=${node_labels};node_taints=${node_taints}"
1549 cat >>"$file" <<EOF
1550AUTOSCALER_ENV_VARS: $(yaml-quote "${autoscaler_env_vars}")
1551EOF
1552 fi
1553 fi
1554 if [ -n "${SCHEDULING_ALGORITHM_PROVIDER:-}" ]; then
1555 cat >>"$file" <<EOF
1556SCHEDULING_ALGORITHM_PROVIDER: $(yaml-quote "${SCHEDULING_ALGORITHM_PROVIDER}")
1557EOF
1558 fi
1559 if [ -n "${MAX_PODS_PER_NODE:-}" ]; then
1560 cat >>"$file" <<EOF
1561MAX_PODS_PER_NODE: $(yaml-quote "${MAX_PODS_PER_NODE}")
1562EOF
1563 fi
1564 if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
1565 cat >>"$file" <<EOF
1566PREPARE_KONNECTIVITY_SERVICE: $(yaml-quote "${PREPARE_KONNECTIVITY_SERVICE}")
1567EOF
1568 fi
1569 if [[ "${EGRESS_VIA_KONNECTIVITY:-false}" == "true" ]]; then
1570 cat >>"$file" <<EOF
1571EGRESS_VIA_KONNECTIVITY: $(yaml-quote "${EGRESS_VIA_KONNECTIVITY}")
1572EOF
1573 fi
1574 if [[ "${RUN_KONNECTIVITY_PODS:-false}" == "true" ]]; then
1575 cat >>"$file" <<EOF
1576RUN_KONNECTIVITY_PODS: $(yaml-quote "${RUN_KONNECTIVITY_PODS}")
1577EOF
1578 fi
1579 if [[ -n "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-}" ]]; then
1580 cat >>"$file" <<EOF
1581KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE: $(yaml-quote "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE}")
1582EOF
1583 fi
1584}
1585
1586
1587function build-windows-kube-env {
1588 local file="$1"
1589 # For now the Windows kube-env is a superset of the Linux kube-env.
1590 build-linux-kube-env false "$file"
1591
1592 cat >>"$file" <<EOF
1593WINDOWS_NODE_INSTANCE_PREFIX: $(yaml-quote "${WINDOWS_NODE_INSTANCE_PREFIX}")
1594NODE_BINARY_TAR_URL: $(yaml-quote "${NODE_BINARY_TAR_URL}")
1595NODE_BINARY_TAR_HASH: $(yaml-quote "${NODE_BINARY_TAR_HASH}")
1596CSI_PROXY_STORAGE_PATH: $(yaml-quote "${CSI_PROXY_STORAGE_PATH}")
1597CSI_PROXY_VERSION: $(yaml-quote "${CSI_PROXY_VERSION}")
1598CSI_PROXY_FLAGS: $(yaml-quote "${CSI_PROXY_FLAGS}")
1599ENABLE_CSI_PROXY: $(yaml-quote "${ENABLE_CSI_PROXY}")
1600K8S_DIR: $(yaml-quote "${WINDOWS_K8S_DIR}")
1601NODE_DIR: $(yaml-quote "${WINDOWS_NODE_DIR}")
1602LOGS_DIR: $(yaml-quote "${WINDOWS_LOGS_DIR}")
1603CNI_DIR: $(yaml-quote "${WINDOWS_CNI_DIR}")
1604CNI_CONFIG_DIR: $(yaml-quote "${WINDOWS_CNI_CONFIG_DIR}")
1605WINDOWS_CNI_STORAGE_PATH: $(yaml-quote "${WINDOWS_CNI_STORAGE_PATH}")
1606WINDOWS_CNI_VERSION: $(yaml-quote "${WINDOWS_CNI_VERSION}")
1607WINDOWS_CONTAINER_RUNTIME: $(yaml-quote "${WINDOWS_CONTAINER_RUNTIME}")
1608WINDOWS_CONTAINER_RUNTIME_ENDPOINT: $(yaml-quote "${WINDOWS_CONTAINER_RUNTIME_ENDPOINT:-}")
1609MANIFESTS_DIR: $(yaml-quote "${WINDOWS_MANIFESTS_DIR}")
1610PKI_DIR: $(yaml-quote "${WINDOWS_PKI_DIR}")
1611CA_FILE_PATH: $(yaml-quote "${WINDOWS_CA_FILE}")
1612KUBELET_CONFIG_FILE: $(yaml-quote "${WINDOWS_KUBELET_CONFIG_FILE}")
1613KUBEPROXY_ARGS: $(yaml-quote "${KUBEPROXY_ARGS}")
1614KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_KUBECONFIG_FILE}")
1615BOOTSTRAP_KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_BOOTSTRAP_KUBECONFIG_FILE}")
1616KUBEPROXY_KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_KUBEPROXY_KUBECONFIG_FILE}")
1617WINDOWS_INFRA_CONTAINER: $(yaml-quote "${WINDOWS_INFRA_CONTAINER}")
1618WINDOWS_ENABLE_PIGZ: $(yaml-quote "${WINDOWS_ENABLE_PIGZ}")
1619WINDOWS_ENABLE_HYPERV: $(yaml-quote "${WINDOWS_ENABLE_HYPERV}")
1620ENABLE_AUTH_PROVIDER_GCP: $(yaml-quote "${ENABLE_AUTH_PROVIDER_GCP}")
1621ENABLE_NODE_PROBLEM_DETECTOR: $(yaml-quote "${WINDOWS_ENABLE_NODE_PROBLEM_DETECTOR}")
1622NODE_PROBLEM_DETECTOR_VERSION: $(yaml-quote "${NODE_PROBLEM_DETECTOR_VERSION}")
1623NODE_PROBLEM_DETECTOR_TAR_HASH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TAR_HASH}")
1624NODE_PROBLEM_DETECTOR_RELEASE_PATH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_RELEASE_PATH}")
1625NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS: $(yaml-quote "${WINDOWS_NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS}")
1626NODE_PROBLEM_DETECTOR_TOKEN: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TOKEN:-}")
1627WINDOWS_NODEPROBLEMDETECTOR_KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_NODEPROBLEMDETECTOR_KUBECONFIG_FILE}")
1628AUTH_PROVIDER_GCP_STORAGE_PATH: $(yaml-quote "${AUTH_PROVIDER_GCP_STORAGE_PATH}")
1629AUTH_PROVIDER_GCP_VERSION: $(yaml-quote "${AUTH_PROVIDER_GCP_VERSION}")
1630AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64: $(yaml-quote "${AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64}")
1631AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR: $(yaml-quote "${AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR}")
1632AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE: $(yaml-quote "${AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE}")
1633EOF
1634}
1635
1636function sha512sum-file() {
1637 local shasum
1638 if which sha512sum >/dev/null 2>&1; then
1639 shasum=$(sha512sum "$1")
1640 else
1641 shasum=$(shasum -a512 "$1")
1642 fi
1643 echo "${shasum%%[[:blank:]]*}"
1644}
1645
1646# Create certificate pairs for the cluster.
1647# $1: The public IP for the master.
1648#
1649# These are used for static cert distribution (e.g. static clustering) at
1650# cluster creation time. This will be obsoleted once we implement dynamic
1651# clustering.
1652#
1653# The following certificate pairs are created:
1654#
1655# - ca (the cluster's certificate authority)
1656# - server
1657# - kubelet
1658# - kubecfg (for kubectl)
1659#
1660# TODO(roberthbailey): Replace easyrsa with a simple Go program to generate
1661# the certs that we need.
1662#
1663# Assumed vars
1664# KUBE_TEMP
1665# MASTER_NAME
1666#
1667# Vars set:
1668# CERT_DIR
1669# CA_CERT_BASE64
1670# MASTER_CERT_BASE64
1671# MASTER_KEY_BASE64
1672# KUBELET_CERT_BASE64
1673# KUBELET_KEY_BASE64
1674# KUBECFG_CERT_BASE64
1675# KUBECFG_KEY_BASE64
1676function create-certs {
1677 local -r primary_cn="${1}"
1678
1679 # Determine extra certificate names for master
1680
1681 # Create service_ip by stripping the network mask part from
1682 # SERVICE_CLUSTER_IP_RANGE and incrementing the host part with 1
1683 service_ip=${SERVICE_CLUSTER_IP_RANGE%/*}
1684 service_ip="${service_ip%.*}.$((${service_ip##*.} + 1))"
1685 local sans=""
1686 for extra in "$@"; do
1687 if [[ -n "${extra}" ]]; then
1688 sans="${sans}IP:${extra},"
1689 fi
1690 done
1691 sans="${sans}IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${DNS_DOMAIN},DNS:${MASTER_NAME}"
1692
1693 echo "Generating certs for alternate-names: ${sans}"
1694
1695 setup-easyrsa
1696 PRIMARY_CN="${primary_cn}" SANS="${sans}" generate-certs
1697 AGGREGATOR_PRIMARY_CN="${primary_cn}" AGGREGATOR_SANS="${sans}" generate-aggregator-certs
1698 KONNECTIVITY_SERVER_PRIMARY_CN="${primary_cn}" KONNECTIVITY_SERVER_SANS="${sans}" generate-konnectivity-server-certs
1699 CLOUD_PVL_ADMISSION_PRIMARY_CN="${primary_cn}" CLOUD_PVL_ADMISSION_SANS="${sans}" generate-cloud-pvl-admission-certs
1700
1701 # By default, linux wraps base64 output every 76 cols, so we use 'tr -d' to remove whitespaces.
1702 # Note 'base64 -w0' doesn't work on Mac OS X, which has different flags.
1703 CA_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
1704 CA_CERT_BASE64=$(base64 "${CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
1705 MASTER_CERT_BASE64=$(base64 "${CERT_DIR}/pki/issued/${MASTER_NAME}.crt" | tr -d '\r\n')
1706 MASTER_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/${MASTER_NAME}.key" | tr -d '\r\n')
1707 KUBELET_CERT_BASE64=$(base64 "${CERT_DIR}/pki/issued/kubelet.crt" | tr -d '\r\n')
1708 KUBELET_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/kubelet.key" | tr -d '\r\n')
1709 KUBECFG_CERT_BASE64=$(base64 "${CERT_DIR}/pki/issued/kubecfg.crt" | tr -d '\r\n')
1710 KUBECFG_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/kubecfg.key" | tr -d '\r\n')
1711 KUBEAPISERVER_CERT_BASE64=$(base64 "${CERT_DIR}/pki/issued/kube-apiserver.crt" | tr -d '\r\n')
1712 KUBEAPISERVER_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/kube-apiserver.key" | tr -d '\r\n')
1713
1714 # Setting up an addition directory (beyond pki) as it is the simplest way to
1715 # ensure we get a different CA pair to sign the proxy-client certs and which
1716 # we can send CA public key to the user-apiserver to validate communication.
1717 AGGREGATOR_CA_KEY_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
1718 REQUESTHEADER_CA_CERT_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
1719 PROXY_CLIENT_CERT_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/issued/proxy-client.crt" | tr -d '\r\n')
1720 PROXY_CLIENT_KEY_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/private/proxy-client.key" | tr -d '\r\n')
1721
1722 # Setting up the Kubernetes API Server Konnectivity Server auth.
1723 # This includes certs for both API Server to Konnectivity Server and
1724 # Konnectivity Agent to Konnectivity Server.
1725 KONNECTIVITY_SERVER_CA_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
1726 KONNECTIVITY_SERVER_CA_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
1727 KONNECTIVITY_SERVER_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n')
1728 KONNECTIVITY_SERVER_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" | tr -d '\r\n')
1729 KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" | tr -d '\r\n')
1730 KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" | tr -d '\r\n')
1731 KONNECTIVITY_AGENT_CA_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
1732 KONNECTIVITY_AGENT_CA_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
1733 KONNECTIVITY_AGENT_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n')
1734 KONNECTIVITY_AGENT_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" | tr -d '\r\n')
1735 KONNECTIVITY_AGENT_CLIENT_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" | tr -d '\r\n')
1736 KONNECTIVITY_AGENT_CLIENT_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key" | tr -d '\r\n')
1737
1738 CLOUD_PVL_ADMISSION_CA_KEY_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
1739 CLOUD_PVL_ADMISSION_CA_CERT_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
1740 CLOUD_PVL_ADMISSION_CERT_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n')
1741 CLOUD_PVL_ADMISSION_KEY_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/server.key" | tr -d '\r\n')
1742}
1743
1744# Set up easy-rsa directory structure.
1745#
1746# Assumed vars
1747# KUBE_TEMP
1748#
1749# Vars set:
1750# CERT_DIR
1751# AGGREGATOR_CERT_DIR
1752function setup-easyrsa {
1753 local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
1754 # Note: This was heavily cribbed from make-ca-cert.sh
1755 (set -x
1756 cd "${KUBE_TEMP}"
1757 curl -L -O --connect-timeout 20 --retry 6 --retry-delay 2 https://dl.k8s.io/easy-rsa/easy-rsa.tar.gz
1758 tar xzf easy-rsa.tar.gz
1759 mkdir easy-rsa-master/kubelet
1760 cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/kubelet
1761 mkdir easy-rsa-master/aggregator
1762 cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator
1763 mkdir easy-rsa-master/cloud-pvl-admission
1764 cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/cloud-pvl-admission
1765 mkdir easy-rsa-master/konnectivity-server
1766 cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-server
1767 mkdir easy-rsa-master/konnectivity-agent
1768 cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-agent) &>"${cert_create_debug_output}" || true
1769 CERT_DIR="${KUBE_TEMP}/easy-rsa-master/easyrsa3"
1770 AGGREGATOR_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/aggregator"
1771 CLOUD_PVL_ADMISSION_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/cloud-pvl-admission"
1772 KONNECTIVITY_SERVER_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-server"
1773 KONNECTIVITY_AGENT_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-agent"
1774 if [ ! -x "${CERT_DIR}/easyrsa" ] || [ ! -x "${AGGREGATOR_CERT_DIR}/easyrsa" ]; then
1775 # TODO(roberthbailey,porridge): add better error handling here,
1776 # see https://github.com/kubernetes/kubernetes/issues/55229
1777 cat "${cert_create_debug_output}" >&2
1778 echo "=== Failed to setup easy-rsa: Aborting ===" >&2
1779 exit 2
1780 fi
1781}
1782
1783# Runs the easy RSA commands to generate certificate files.
1784# The generated files are IN ${CERT_DIR}
1785#
1786# Assumed vars (see shellcheck disable directives below)
1787# KUBE_TEMP
1788# MASTER_NAME
1789# CERT_DIR
1790# PRIMARY_CN: Primary canonical name
1791# SANS: Subject alternate names
1792#
1793#
1794function generate-certs {
1795 local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
1796 # Note: This was heavily cribbed from make-ca-cert.sh
1797 (set -x
1798 cd "${CERT_DIR}"
1799 ./easyrsa init-pki
1800 # this puts the cert into pki/ca.crt and the key into pki/private/ca.key
1801 # PRIMARY_CN (expected to be) defined by caller
1802 # shellcheck disable=SC2153
1803 ./easyrsa --batch "--req-cn=${PRIMARY_CN}@$(date +%s)" build-ca nopass
1804 # SANS (expected to be) defined by caller
1805 # shellcheck disable=SC2153
1806 ./easyrsa --subject-alt-name="${SANS}" build-server-full "${MASTER_NAME}" nopass
1807 ./easyrsa build-client-full kube-apiserver nopass
1808
1809 kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
1810
1811 # make the config for the signer
1812 echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json"
1813 # create the kubelet client cert with the correct groups
1814 echo '{"CN":"kubelet","names":[{"O":"system:nodes"}],"hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare kubelet
1815 mv "kubelet-key.pem" "pki/private/kubelet.key"
1816 mv "kubelet.pem" "pki/issued/kubelet.crt"
1817 rm -f "kubelet.csr"
1818
1819 # Make a superuser client cert with subject "O=system:masters, CN=kubecfg"
1820 ./easyrsa --dn-mode=org \
1821 --req-cn=kubecfg --req-org=system:masters \
1822 --req-c= --req-st= --req-city= --req-email= --req-ou= \
1823 build-client-full kubecfg nopass) &>"${cert_create_debug_output}" || true
1824 local output_file_missing=0
1825 local output_file
1826 for output_file in \
1827 "${CERT_DIR}/pki/private/ca.key" \
1828 "${CERT_DIR}/pki/ca.crt" \
1829 "${CERT_DIR}/pki/issued/${MASTER_NAME}.crt" \
1830 "${CERT_DIR}/pki/private/${MASTER_NAME}.key" \
1831 "${CERT_DIR}/pki/issued/kubelet.crt" \
1832 "${CERT_DIR}/pki/private/kubelet.key" \
1833 "${CERT_DIR}/pki/issued/kubecfg.crt" \
1834 "${CERT_DIR}/pki/private/kubecfg.key" \
1835 "${CERT_DIR}/pki/issued/kube-apiserver.crt" \
1836 "${CERT_DIR}/pki/private/kube-apiserver.key"
1837 do
1838 if [[ ! -s "${output_file}" ]]; then
1839 echo "Expected file ${output_file} not created" >&2
1840 output_file_missing=1
1841 fi
1842 done
1843 if [ $output_file_missing -ne 0 ]; then
1844 # TODO(roberthbailey,porridge): add better error handling here,
1845 # see https://github.com/kubernetes/kubernetes/issues/55229
1846 cat "${cert_create_debug_output}" >&2
1847 echo "=== Failed to generate master certificates: Aborting ===" >&2
1848 exit 2
1849 fi
1850}
1851
1852# Runs the easy RSA commands to generate aggregator certificate files.
1853# The generated files are in ${AGGREGATOR_CERT_DIR}
1854#
1855# Assumed vars
1856# KUBE_TEMP
1857# AGGREGATOR_MASTER_NAME
1858# AGGREGATOR_CERT_DIR
1859# AGGREGATOR_PRIMARY_CN: Primary canonical name
1860# AGGREGATOR_SANS: Subject alternate names
1861#
1862#
1863function generate-aggregator-certs {
1864 local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
1865 # Note: This was heavily cribbed from make-ca-cert.sh
1866 (set -x
1867 cd "${KUBE_TEMP}/easy-rsa-master/aggregator"
1868 ./easyrsa init-pki
1869 # this puts the cert into pki/ca.crt and the key into pki/private/ca.key
1870 ./easyrsa --batch "--req-cn=${AGGREGATOR_PRIMARY_CN}@$(date +%s)" build-ca nopass
1871 ./easyrsa --subject-alt-name="${AGGREGATOR_SANS}" build-server-full "${AGGREGATOR_MASTER_NAME}" nopass
1872 ./easyrsa build-client-full aggregator-apiserver nopass
1873
1874 kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
1875
1876 # make the config for the signer
1877 echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json"
1878 # create the aggregator client cert with the correct groups
1879 echo '{"CN":"aggregator","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare proxy-client
1880 mv "proxy-client-key.pem" "pki/private/proxy-client.key"
1881 mv "proxy-client.pem" "pki/issued/proxy-client.crt"
1882 rm -f "proxy-client.csr"
1883
1884 # Make a superuser client cert with subject "O=system:masters, CN=kubecfg"
1885 ./easyrsa --dn-mode=org \
1886 --req-cn=proxy-clientcfg --req-org=system:aggregator \
1887 --req-c= --req-st= --req-city= --req-email= --req-ou= \
1888 build-client-full proxy-clientcfg nopass) &>"${cert_create_debug_output}" || true
1889 local output_file_missing=0
1890 local output_file
1891 for output_file in \
1892 "${AGGREGATOR_CERT_DIR}/pki/private/ca.key" \
1893 "${AGGREGATOR_CERT_DIR}/pki/ca.crt" \
1894 "${AGGREGATOR_CERT_DIR}/pki/issued/proxy-client.crt" \
1895 "${AGGREGATOR_CERT_DIR}/pki/private/proxy-client.key"
1896 do
1897 if [[ ! -s "${output_file}" ]]; then
1898 echo "Expected file ${output_file} not created" >&2
1899 output_file_missing=1
1900 fi
1901 done
1902 if [ $output_file_missing -ne 0 ]; then
1903 # TODO(roberthbailey,porridge): add better error handling here,
1904 # see https://github.com/kubernetes/kubernetes/issues/55229
1905 cat "${cert_create_debug_output}" >&2
1906 echo "=== Failed to generate aggregator certificates: Aborting ===" >&2
1907 exit 2
1908 fi
1909}
1910
1911# Runs the easy RSA commands to generate server side certificate files
1912# for the konnectivity server. This includes both server side to both
1913# konnectivity-server and konnectivity-agent.
1914# The generated files are in ${KONNECTIVITY_SERVER_CERT_DIR} and
1915# ${KONNECTIVITY_AGENT_CERT_DIR}
1916#
1917# Assumed vars
1918# KUBE_TEMP
1919# KONNECTIVITY_SERVER_CERT_DIR
1920# KONNECTIVITY_SERVER_PRIMARY_CN: Primary canonical name
1921# KONNECTIVITY_SERVER_SANS: Subject alternate names
1922#
1923function generate-konnectivity-server-certs {
1924 local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
1925 # Note: This was heavily cribbed from make-ca-cert.sh
1926 (set -x
1927 # Make the client <-> konnectivity server side certificates.
1928 cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-server"
1929 ./easyrsa init-pki
1930 # this puts the cert into pki/ca.crt and the key into pki/private/ca.key
1931 ./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass
1932 ./easyrsa --subject-alt-name="IP:127.0.0.1,${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass
1933 ./easyrsa build-client-full client nopass
1934
1935 kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
1936
1937 # make the config for the signer
1938 echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json"
1939 # create the konnectivity server cert with the correct groups
1940 echo '{"CN":"konnectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-server
1941 rm -f "konnectivity-server.csr"
1942
1943 # Make the agent <-> konnectivity server side certificates.
1944 cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-agent"
1945 ./easyrsa init-pki
1946 # this puts the cert into pki/ca.crt and the key into pki/private/ca.key
1947 ./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass
1948 ./easyrsa --subject-alt-name="${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass
1949 ./easyrsa build-client-full client nopass
1950
1951 kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
1952
1953 # make the config for the signer
1954 echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json"
1955 # create the konnectivity server cert with the correct groups
1956 echo '{"CN":"koonectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent
1957 rm -f "konnectivity-agent.csr"
1958
1959 echo "completed main certificate section") &>"${cert_create_debug_output}" || true
1960
1961 local output_file_missing=0
1962 local output_file
1963 for output_file in \
1964 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" \
1965 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" \
1966 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" \
1967 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" \
1968 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" \
1969 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" \
1970 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" \
1971 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" \
1972 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" \
1973 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" \
1974 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" \
1975 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key"
1976 do
1977 if [[ ! -s "${output_file}" ]]; then
1978 echo "Expected file ${output_file} not created" >&2
1979 output_file_missing=1
1980 fi
1981 done
1982 if (( output_file_missing )); then
1983 # TODO(roberthbailey,porridge): add better error handling here,
1984 # see https://github.com/kubernetes/kubernetes/issues/55229
1985 cat "${cert_create_debug_output}" >&2
1986 echo "=== Failed to generate konnectivity-server certificates: Aborting ===" >&2
1987 exit 2
1988 fi
1989}
1990
1991# Runs the easy RSA commands to generate server side certificate files
1992# for the cloud-pvl-admission webhook.
1993# The generated files are in ${CLOUD_PVL_ADMISSION_CERT_DIR}
1994#
1995# Assumed vars
1996# KUBE_TEMP
1997# CLOUD_PVL_ADMISSION_CERT_DIR
1998# CLOUD_PVL_ADMISSION_PRIMARY_CN: Primary canonical name
1999# CLOUD_PVL_ADMISSION_SANS: Subject alternate names
2000#
2001function generate-cloud-pvl-admission-certs {
2002 local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
2003 # Note: This was heavily cribbed from make-ca-cert.sh
2004 (set -x
2005 # Make the client <-> cloud-pvl-admission server side certificates.
2006 cd "${KUBE_TEMP}/easy-rsa-master/cloud-pvl-admission"
2007 ./easyrsa init-pki
2008 # this puts the cert into pki/ca.crt and the key into pki/private/ca.key
2009 ./easyrsa --batch "--req-cn=${CLOUD_PVL_ADMISSION_PRIMARY_CN}@$(date +%s)" build-ca nopass
2010 ./easyrsa --subject-alt-name="IP:127.0.0.1,${CLOUD_PVL_ADMISSION_SANS}" build-server-full server nopass
2011 ./easyrsa build-client-full client nopass
2012
2013 kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
2014
2015 # make the config for the signer
2016 echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json"
2017 # create the cloud-pvl-admission cert with the correct groups
2018 echo '{"CN":"cloud-pvl-admission","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare cloud-pvl-admission
2019 rm -f "cloud-pvl-admission.csr"
2020
2021 # Make the cloud-pvl-admission server side certificates.
2022 cd "${KUBE_TEMP}/easy-rsa-master/cloud-pvl-admission"
2023 ./easyrsa init-pki
2024 # this puts the cert into pki/ca.crt and the key into pki/private/ca.key
2025 ./easyrsa --batch "--req-cn=${CLOUD_PVL_ADMISSION_PRIMARY_CN}@$(date +%s)" build-ca nopass
2026 ./easyrsa --subject-alt-name="${CLOUD_PVL_ADMISSION_SANS}" build-server-full server nopass
2027 ./easyrsa build-client-full client nopass
2028
2029 kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
2030
2031 # make the config for the signer
2032 echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json"
2033 # create the cloud-pvl-admission server cert with the correct groups
2034 echo '{"CN":"cloud-pvl-admission","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent
2035 rm -f "konnectivity-agent.csr"
2036
2037 echo "completed main certificate section") &>"${cert_create_debug_output}" || true
2038
2039 local output_file_missing=0
2040 local output_file
2041 for output_file in \
2042 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/ca.key" \
2043 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/ca.crt" \
2044 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/issued/server.crt" \
2045 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/server.key" \
2046 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/issued/client.crt" \
2047 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/client.key"
2048 do
2049 if [[ ! -s "${output_file}" ]]; then
2050 echo "Expected file ${output_file} not created" >&2
2051 output_file_missing=1
2052 fi
2053 done
2054 if (( output_file_missing )); then
2055 # TODO(roberthbailey,porridge): add better error handling here,
2056 # see https://github.com/kubernetes/kubernetes/issues/55229
2057 cat "${cert_create_debug_output}" >&2
2058 echo "=== Failed to generate cloud-pvl-admission certificates: Aborting ===" >&2
2059 exit 2
2060 fi
2061}
2062
2063# Using provided master env, extracts value from provided key.
2064#
2065# Args:
2066# $1 master env (kube-env of master; result of calling get-master-env)
2067# $2 env key to use
2068function get-env-val() {
2069 local match
2070 match=$( (echo "${1}" | grep -E "^${2}:") || echo '')
2071 if [[ -z "${match}" ]]; then
2072 echo ""
2073 fi
2074 echo "${match}" | cut -d : -f 2 | cut -d \' -f 2
2075}
2076
2077# Load the master env by calling get-master-env, and extract important values
2078function parse-master-env() {
2079 # Get required master env vars
2080 local master_env
2081 master_env=$(get-master-env)
2082 KUBE_PROXY_TOKEN=$(get-env-val "${master_env}" "KUBE_PROXY_TOKEN")
2083 NODE_PROBLEM_DETECTOR_TOKEN=$(get-env-val "${master_env}" "NODE_PROBLEM_DETECTOR_TOKEN")
2084 CA_CERT_BASE64=$(get-env-val "${master_env}" "CA_CERT")
2085 CA_KEY_BASE64=$(get-env-val "${master_env}" "CA_KEY")
2086 KUBEAPISERVER_CERT_BASE64=$(get-env-val "${master_env}" "KUBEAPISERVER_CERT")
2087 KUBEAPISERVER_KEY_BASE64=$(get-env-val "${master_env}" "KUBEAPISERVER_KEY")
2088 EXTRA_DOCKER_OPTS=$(get-env-val "${master_env}" "EXTRA_DOCKER_OPTS")
2089 KUBELET_CERT_BASE64=$(get-env-val "${master_env}" "KUBELET_CERT")
2090 KUBELET_KEY_BASE64=$(get-env-val "${master_env}" "KUBELET_KEY")
2091 MASTER_CERT_BASE64=$(get-env-val "${master_env}" "MASTER_CERT")
2092 MASTER_KEY_BASE64=$(get-env-val "${master_env}" "MASTER_KEY")
2093 AGGREGATOR_CA_KEY_BASE64=$(get-env-val "${master_env}" "AGGREGATOR_CA_KEY")
2094 REQUESTHEADER_CA_CERT_BASE64=$(get-env-val "${master_env}" "REQUESTHEADER_CA_CERT")
2095 PROXY_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "PROXY_CLIENT_CERT")
2096 PROXY_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "PROXY_CLIENT_KEY")
2097 ENABLE_LEGACY_ABAC=$(get-env-val "${master_env}" "ENABLE_LEGACY_ABAC")
2098 ETCD_APISERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CA_KEY")
2099 ETCD_APISERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CA_CERT")
2100 ETCD_APISERVER_SERVER_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_KEY")
2101 ETCD_APISERVER_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_CERT")
2102 ETCD_APISERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_KEY")
2103 ETCD_APISERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_CERT")
2104 CLOUD_PVL_ADMISSION_CA_KEY_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_CA_KEY")
2105 CLOUD_PVL_ADMISSION_CA_CERT_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_CA_CERT")
2106 CLOUD_PVL_ADMISSION_CERT_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_CERT")
2107 CLOUD_PVL_ADMISSION_KEY_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_KEY")
2108 KONNECTIVITY_SERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_KEY")
2109 KONNECTIVITY_SERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_CERT")
2110 KONNECTIVITY_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CERT")
2111 KONNECTIVITY_SERVER_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_KEY")
2112 KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_CERT")
2113 KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_KEY")
2114 KONNECTIVITY_AGENT_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_KEY")
2115 KONNECTIVITY_AGENT_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_CERT")
2116 KONNECTIVITY_AGENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CERT")
2117 KONNECTIVITY_AGENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_KEY")
2118}
2119
2120# Update or verify required gcloud components are installed
2121# at minimum required version.
2122# Assumed vars
2123# KUBE_PROMPT_FOR_UPDATE
2124function update-or-verify-gcloud() {
2125 local sudo_prefix=""
2126 if [ ! -w "$(dirname "$(which gcloud)")" ]; then
2127 sudo_prefix="sudo"
2128 fi
2129 # update and install components as needed
2130 # (deliberately word split $gcloud_prompt)
2131 # shellcheck disable=SC2086
2132 if [[ "${KUBE_PROMPT_FOR_UPDATE}" == "y" ]]; then
2133 ${sudo_prefix} gcloud ${gcloud_prompt:-} components install alpha
2134 ${sudo_prefix} gcloud ${gcloud_prompt:-} components install beta
2135 ${sudo_prefix} gcloud ${gcloud_prompt:-} components update
2136 else
2137 local version
2138 version=$(gcloud version --format=json)
2139 python3 -c"
2140import json,sys
2141from distutils import version
2142
2143minVersion = version.LooseVersion('1.3.0')
2144required = [ 'alpha', 'beta', 'core' ]
2145data = json.loads(sys.argv[1])
2146rel = data.get('Google Cloud SDK')
2147if 'CL @' in rel:
2148 print('Using dev version of gcloud: %s' %rel)
2149 exit(0)
2150if rel != 'HEAD' and version.LooseVersion(rel) < minVersion:
2151 print('gcloud version out of date ( < %s )' % minVersion)
2152 exit(1)
2153missing = []
2154for c in required:
2155 if not data.get(c):
2156 missing += [c]
2157if missing:
2158 for c in missing:
2159 print ('missing required gcloud component \"{0}\"'.format(c))
2160 print ('Try running \$(gcloud components install {0})'.format(c))
2161 exit(1)
2162 " "${version}"
2163 fi
2164}
2165
2166# Robustly try to create a static ip.
2167# $1: The name of the ip to create
2168# $2: The name of the region to create the ip in.
2169function create-static-ip() {
2170 detect-project
2171 local attempt=0
2172 local REGION="$2"
2173 while true; do
2174 if gcloud compute addresses create "$1" \
2175 --project "${PROJECT}" \
2176 --region "${REGION}" -q > /dev/null; then
2177 # successful operation - wait until it's visible
2178 start="$(date +%s)"
2179 while true; do
2180 now="$(date +%s)"
2181 # Timeout set to 15 minutes
2182 if [[ $((now - start)) -gt 900 ]]; then
2183 echo "Timeout while waiting for master IP visibility"
2184 exit 2
2185 fi
2186 if gcloud compute addresses describe "$1" --project "${PROJECT}" --region "${REGION}" >/dev/null 2>&1; then
2187 break
2188 fi
2189 echo "Master IP not visible yet. Waiting..."
2190 sleep 5
2191 done
2192 break
2193 fi
2194
2195 if gcloud compute addresses describe "$1" \
2196 --project "${PROJECT}" \
2197 --region "${REGION}" >/dev/null 2>&1; then
2198 # it exists - postcondition satisfied
2199 break
2200 fi
2201
2202 if (( attempt > 4 )); then
2203 echo -e "${color_red}Failed to create static ip $1 ${color_norm}" >&2
2204 exit 2
2205 fi
2206 attempt=$((attempt + 1))
2207 echo -e "${color_yellow:-}Attempt $attempt failed to create static ip $1. Retrying.${color_norm:-}" >&2
2208 sleep $((attempt * 5))
2209 done
2210}
2211
2212# Robustly try to create a firewall rule.
2213# $1: The name of firewall rule.
2214# $2: IP ranges.
2215# $3: Target tags for this firewall rule.
2216function create-firewall-rule() {
2217 detect-project
2218 local attempt=0
2219 while true; do
2220 if ! gcloud compute firewall-rules create "$1" \
2221 --project "${NETWORK_PROJECT}" \
2222 --network "${NETWORK}" \
2223 --source-ranges "$2" \
2224 --target-tags "$3" \
2225 --allow tcp,udp,icmp,esp,ah,sctp; then
2226 if (( attempt > 4 )); then
2227 echo -e "${color_red}Failed to create firewall rule $1 ${color_norm}" >&2
2228 exit 2
2229 fi
2230 echo -e "${color_yellow}Attempt $((attempt + 1)) failed to create firewall rule $1. Retrying.${color_norm}" >&2
2231 attempt=$((attempt + 1))
2232 sleep $((attempt * 5))
2233 else
2234 break
2235 fi
2236 done
2237}
2238
2239# Format the string argument for gcloud network.
2240function make-gcloud-network-argument() {
2241 local network_project="$1"
2242 local region="$2"
2243 local network="$3"
2244 local subnet="$4"
2245 local address="$5" # optional
2246 local enable_ip_alias="$6" # optional
2247 local alias_size="$7" # optional
2248
2249 local networkURL="projects/${network_project}/global/networks/${network}"
2250 local subnetURL="projects/${network_project}/regions/${region}/subnetworks/${subnet:-}"
2251
2252 local ret=""
2253
2254 if [[ "${enable_ip_alias}" == 'true' ]]; then
2255 ret="--network-interface"
2256 ret="${ret} network=${networkURL}"
2257 if [[ "${address:-}" == "no-address" ]]; then
2258 ret="${ret},no-address"
2259 else
2260 ret="${ret},address=${address:-}"
2261 fi
2262 ret="${ret},subnet=${subnetURL}"
2263 ret="${ret},aliases=pods-default:${alias_size}"
2264 ret="${ret} --no-can-ip-forward"
2265 else
2266 if [[ -n ${subnet:-} ]]; then
2267 ret="${ret} --subnet ${subnetURL}"
2268 else
2269 ret="${ret} --network ${networkURL}"
2270 fi
2271
2272 ret="${ret} --can-ip-forward"
2273 if [[ -n ${address:-} ]] && [[ "$address" != "no-address" ]]; then
2274 ret="${ret} --address ${address}"
2275 fi
2276 fi
2277
2278 echo "${ret}"
2279}
2280
2281# $1: version (required)
2282# $2: Prefix for the template name, i.e. NODE_INSTANCE_PREFIX or
2283# WINDOWS_NODE_INSTANCE_PREFIX.
2284function get-template-name-from-version() {
2285 local -r version=${1}
2286 local -r template_prefix=${2}
2287 # trim template name to pass gce name validation
2288 echo "${template_prefix}-template-${version}" | cut -c 1-63 | sed 's/[\.\+]/-/g;s/-*$//g'
2289}
2290
2291# validates the NODE_LOCAL_SSDS_EXT variable
2292function validate-node-local-ssds-ext(){
2293 ssdopts="${1}"
2294
2295 if [[ -z "${ssdopts[0]}" || -z "${ssdopts[1]}" || -z "${ssdopts[2]}" ]]; then
2296 echo -e "${color_red}Local SSD: NODE_LOCAL_SSDS_EXT is malformed, found ${ssdopts[0]-_},${ssdopts[1]-_},${ssdopts[2]-_} ${color_norm}" >&2
2297 exit 2
2298 fi
2299 if [[ "${ssdopts[1]}" != "scsi" && "${ssdopts[1]}" != "nvme" ]]; then
2300 echo -e "${color_red}Local SSD: Interface must be scsi or nvme, found: ${ssdopts[1]} ${color_norm}" >&2
2301 exit 2
2302 fi
2303 if [[ "${ssdopts[2]}" != "fs" && "${ssdopts[2]}" != "block" ]]; then
2304 echo -e "${color_red}Local SSD: Filesystem type must be fs or block, found: ${ssdopts[2]} ${color_norm}" >&2
2305 exit 2
2306 fi
2307 local_ssd_ext_count=$((local_ssd_ext_count+ssdopts[0]))
2308 if [[ "${local_ssd_ext_count}" -gt "${GCE_MAX_LOCAL_SSD}" || "${local_ssd_ext_count}" -lt 1 ]]; then
2309 echo -e "${color_red}Local SSD: Total number of local ssds must range from 1 to 8, found: ${local_ssd_ext_count} ${color_norm}" >&2
2310 exit 2
2311 fi
2312}
2313
2314# Robustly try to create an instance template.
2315# $1: The name of the instance template.
2316# $2: The scopes flag.
2317# $3: String of comma-separated metadata-from-file entries.
2318# $4: String of comma-separated metadata (key=value) entries.
2319# $5: the node OS ("linux" or "windows").
2320function create-node-template() {
2321 detect-project
2322 detect-subnetworks
2323 local template_name="$1"
2324 local metadata_values="$4"
2325 local os="$5"
2326 local machine_type="$6"
2327
2328 # First, ensure the template doesn't exist.
2329 # TODO(zmerlynn): To make this really robust, we need to parse the output and
2330 # add retries. Just relying on a non-zero exit code doesn't
2331 # distinguish an ephemeral failed call from a "not-exists".
2332 if gcloud compute instance-templates describe "${template_name}" --project "${PROJECT}" &>/dev/null; then
2333 echo "Instance template ${1} already exists; deleting." >&2
2334 if ! gcloud compute instance-templates delete "${template_name}" --project "${PROJECT}" --quiet &>/dev/null; then
2335 echo -e "${color_yellow}Failed to delete existing instance template${color_norm}" >&2
2336 exit 2
2337 fi
2338 fi
2339
2340 local gcloud="gcloud"
2341
2342 local accelerator_args=()
2343 # VMs with Accelerators cannot be live migrated.
2344 # More details here - https://cloud.google.com/compute/docs/gpus/add-gpus#create-new-gpu-instance
2345 if [[ -n "${NODE_ACCELERATORS}" ]]; then
2346 accelerator_args+=(--maintenance-policy TERMINATE --restart-on-failure --accelerator "${NODE_ACCELERATORS}")
2347 gcloud="gcloud beta"
2348 fi
2349
2350 local preemptible_minions=()
2351 if [[ "${PREEMPTIBLE_NODE}" == "true" ]]; then
2352 preemptible_minions+=(--preemptible --maintenance-policy TERMINATE)
2353 fi
2354
2355 local local_ssds=()
2356 local_ssd_ext_count=0
2357 if [[ -n "${NODE_LOCAL_SSDS_EXT:-}" ]]; then
2358 IFS=";" read -r -a ssdgroups <<< "${NODE_LOCAL_SSDS_EXT:-}"
2359 for ssdgroup in "${ssdgroups[@]}"
2360 do
2361 IFS="," read -r -a ssdopts <<< "${ssdgroup}"
2362 validate-node-local-ssds-ext "${ssdopts[@]}"
2363 for ((i=1; i<=ssdopts[0]; i++)); do
2364 local_ssds+=("--local-ssd=interface=${ssdopts[1]}")
2365 done
2366 done
2367 fi
2368
2369 if [[ -n ${NODE_LOCAL_SSDS+x} ]]; then
2370 # The NODE_LOCAL_SSDS check below fixes issue #49171
2371 for ((i=1; i<=NODE_LOCAL_SSDS; i++)); do
2372 local_ssds+=('--local-ssd=interface=SCSI')
2373 done
2374 fi
2375
2376 local address=""
2377 if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
2378 address="no-address"
2379 fi
2380
2381 local network
2382 network=$(make-gcloud-network-argument \
2383 "${NETWORK_PROJECT}" \
2384 "${REGION}" \
2385 "${NETWORK}" \
2386 "${SUBNETWORK:-}" \
2387 "${address}" \
2388 "${ENABLE_IP_ALIASES:-}" \
2389 "${IP_ALIAS_SIZE:-}")
2390
2391 local node_image_flags=()
2392 if [[ "${os}" == 'linux' ]]; then
2393 node_image_flags+=(--image-project "${NODE_IMAGE_PROJECT}" --image "${NODE_IMAGE}")
2394 elif [[ "${os}" == 'windows' ]]; then
2395 node_image_flags+=(--image-project "${WINDOWS_NODE_IMAGE_PROJECT}" --image "${WINDOWS_NODE_IMAGE}")
2396 else
2397 echo "Unknown OS ${os}" >&2
2398 exit 1
2399 fi
2400
2401 local metadata_flag="${metadata_values:+--metadata ${metadata_values}}"
2402
2403 local attempt=1
2404 while true; do
2405 echo "Attempt ${attempt} to create ${1}" >&2
2406 # Deliberately word split ${network}, $2 and ${metadata_flag}
2407 # shellcheck disable=SC2086
2408 if ! ${gcloud} compute instance-templates create \
2409 "${template_name}" \
2410 --project "${PROJECT}" \
2411 --machine-type "${machine_type}" \
2412 --boot-disk-type "${NODE_DISK_TYPE}" \
2413 --boot-disk-size "${NODE_DISK_SIZE}" \
2414 "${node_image_flags[@]}" \
2415 --service-account "${NODE_SERVICE_ACCOUNT}" \
2416 --tags "${NODE_TAG}" \
2417 "${accelerator_args[@]}" \
2418 "${local_ssds[@]}" \
2419 --region "${REGION}" \
2420 ${network} \
2421 "${preemptible_minions[@]}" \
2422 $2 \
2423 --metadata-from-file "$3" \
2424 ${metadata_flag} >&2; then
2425 if (( attempt > 5 )); then
2426 echo -e "${color_red}Failed to create instance template ${template_name} ${color_norm}" >&2
2427 exit 2
2428 fi
2429 echo -e "${color_yellow}Attempt ${attempt} failed to create instance template ${template_name}. Retrying.${color_norm}" >&2
2430 attempt=$((attempt + 1))
2431 sleep $((attempt * 5))
2432
2433 # In case the previous attempt failed with something like a
2434 # Backend Error and left the entry laying around, delete it
2435 # before we try again.
2436 gcloud compute instance-templates delete "${template_name}" --project "${PROJECT}" &>/dev/null || true
2437 else
2438 break
2439 fi
2440 done
2441}
2442
2443# Instantiate a kubernetes cluster
2444#
2445# Assumed vars
2446# KUBE_ROOT
2447# <Various vars set in config file>
2448function kube-up() {
2449 kube::util::ensure-temp-dir
2450 detect-project
2451
2452 load-or-gen-kube-basicauth
2453 load-or-gen-kube-bearertoken
2454
2455 # Make sure we have the tar files staged on Google Storage
2456 find-release-tars
2457 upload-tars
2458
2459 # ensure that environmental variables specifying number of migs to create
2460 set_num_migs
2461
2462 if [[ ${KUBE_USE_EXISTING_MASTER:-} == "true" ]]; then
2463 detect-master
2464 parse-master-env
2465 create-subnetworks
2466 detect-subnetworks
2467 # Windows nodes take longer to boot and setup so create them first.
2468 create-windows-nodes
2469 create-linux-nodes
2470 elif [[ ${KUBE_REPLICATE_EXISTING_MASTER:-} == "true" ]]; then
2471 detect-master
2472 if [[ "${MASTER_OS_DISTRIBUTION}" != "gci" && "${MASTER_OS_DISTRIBUTION}" != "ubuntu" ]]; then
2473 echo "Master replication supported only for gci and ubuntu"
2474 return 1
2475 fi
2476 if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
2477 create-internal-loadbalancer
2478 fi
2479 create-loadbalancer
2480 # If replication of master fails, we need to ensure that the replica is removed from etcd clusters.
2481 if ! replicate-master; then
2482 remove-replica-from-etcd 2379 true || true
2483 remove-replica-from-etcd 4002 false || true
2484 fi
2485 else
2486 check-existing
2487 create-network
2488 create-subnetworks
2489 detect-subnetworks
2490 create-cloud-nat-router
2491 write-cluster-location
2492 write-cluster-name
2493 create-autoscaler-config
2494 create-master
2495 create-nodes-firewall
2496 create-nodes-template
2497 if [[ "${KUBE_CREATE_NODES}" == "true" ]]; then
2498 # Windows nodes take longer to boot and setup so create them first.
2499 create-windows-nodes
2500 create-linux-nodes
2501 fi
2502 check-cluster
2503 fi
2504}
2505
2506function check-existing() {
2507 local running_in_terminal=false
2508 # May be false if tty is not allocated (for example with ssh -T).
2509 if [[ -t 1 ]]; then
2510 running_in_terminal=true
2511 fi
2512
2513 if [[ ${running_in_terminal} == "true" || ${KUBE_UP_AUTOMATIC_CLEANUP} == "true" ]]; then
2514 if ! check-resources; then
2515 local run_kube_down="n"
2516 echo "${KUBE_RESOURCE_FOUND} found." >&2
2517 # Get user input only if running in terminal.
2518 if [[ ${running_in_terminal} == "true" && ${KUBE_UP_AUTOMATIC_CLEANUP} == "false" ]]; then
2519 read -r -p "Would you like to shut down the old cluster (call kube-down)? [y/N] " run_kube_down
2520 fi
2521 if [[ ${run_kube_down} == "y" || ${run_kube_down} == "Y" || ${KUBE_UP_AUTOMATIC_CLEANUP} == "true" ]]; then
2522 echo "... calling kube-down" >&2
2523 kube-down
2524 fi
2525 fi
2526 fi
2527}
2528
2529function check-network-mode() {
2530 local mode
2531 mode=$(gcloud compute networks list --filter="name=('${NETWORK}')" --project "${NETWORK_PROJECT}" --format='value(x_gcloud_subnet_mode)' || true)
2532 # The deprecated field uses lower case. Convert to upper case for consistency.
2533 echo "$mode" | tr '[:lower:]' '[:upper:]'
2534}
2535
2536function create-network() {
2537 if ! gcloud compute networks --project "${NETWORK_PROJECT}" describe "${NETWORK}" &>/dev/null; then
2538 # The network needs to be created synchronously or we have a race. The
2539 # firewalls can be added concurrent with instance creation.
2540 local network_mode="auto"
2541 if [[ "${CREATE_CUSTOM_NETWORK:-}" == "true" ]]; then
2542 network_mode="custom"
2543 fi
2544 echo "Creating new ${network_mode} network: ${NETWORK}"
2545 gcloud compute networks create --project "${NETWORK_PROJECT}" "${NETWORK}" --subnet-mode="${network_mode}"
2546 else
2547 PREEXISTING_NETWORK=true
2548 PREEXISTING_NETWORK_MODE="$(check-network-mode)"
2549 echo "Found existing network ${NETWORK} in ${PREEXISTING_NETWORK_MODE} mode."
2550 fi
2551
2552 if ! gcloud compute firewall-rules --project "${NETWORK_PROJECT}" describe "${CLUSTER_NAME}-default-internal-master" &>/dev/null; then
2553 gcloud compute firewall-rules create "${CLUSTER_NAME}-default-internal-master" \
2554 --project "${NETWORK_PROJECT}" \
2555 --network "${NETWORK}" \
2556 --source-ranges "10.0.0.0/8" \
2557 --allow "tcp:1-2379,tcp:2382-65535,udp:1-65535,icmp" \
2558 --target-tags "${MASTER_TAG}"&
2559 fi
2560
2561 if ! gcloud compute firewall-rules --project "${NETWORK_PROJECT}" describe "${CLUSTER_NAME}-default-internal-node" &>/dev/null; then
2562 gcloud compute firewall-rules create "${CLUSTER_NAME}-default-internal-node" \
2563 --project "${NETWORK_PROJECT}" \
2564 --network "${NETWORK}" \
2565 --source-ranges "10.0.0.0/8" \
2566 --allow "tcp:1-65535,udp:1-65535,icmp" \
2567 --target-tags "${NODE_TAG}"&
2568 fi
2569
2570 if ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NETWORK}-default-ssh" &>/dev/null; then
2571 gcloud compute firewall-rules create "${NETWORK}-default-ssh" \
2572 --project "${NETWORK_PROJECT}" \
2573 --network "${NETWORK}" \
2574 --source-ranges "0.0.0.0/0" \
2575 --allow "tcp:22" &
2576 fi
2577
2578 # Open up TCP 3389 to allow RDP connections.
2579 if [[ ${NUM_WINDOWS_NODES} -gt 0 ]]; then
2580 if ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NETWORK}-default-rdp" &>/dev/null; then
2581 gcloud compute firewall-rules create "${NETWORK}-default-rdp" \
2582 --project "${NETWORK_PROJECT}" \
2583 --network "${NETWORK}" \
2584 --source-ranges "0.0.0.0/0" \
2585 --allow "tcp:3389" &
2586 fi
2587 fi
2588
2589 kube::util::wait-for-jobs || {
2590 code=$?
2591 echo -e "${color_red}Failed to create firewall rules.${color_norm}" >&2
2592 exit $code
2593 }
2594}
2595
2596function expand-default-subnetwork() {
2597 gcloud compute networks update "${NETWORK}" \
2598 --switch-to-custom-subnet-mode \
2599 --project "${NETWORK_PROJECT}" \
2600 --quiet || true
2601 gcloud compute networks subnets expand-ip-range "${NETWORK}" \
2602 --region="${REGION}" \
2603 --project "${NETWORK_PROJECT}" \
2604 --prefix-length=19 \
2605 --quiet
2606}
2607
2608function create-subnetworks() {
2609 case ${ENABLE_IP_ALIASES} in
2610 true) echo "IP aliases are enabled. Creating subnetworks.";;
2611 false)
2612 echo "IP aliases are disabled."
2613 if [[ "${ENABLE_BIG_CLUSTER_SUBNETS}" = "true" ]]; then
2614 if [[ "${PREEXISTING_NETWORK}" != "true" ]]; then
2615 expand-default-subnetwork
2616 else
2617 echo "${color_yellow}Using pre-existing network ${NETWORK}, subnets won't be expanded to /19!${color_norm}"
2618 fi
2619 elif [[ "${CREATE_CUSTOM_NETWORK:-}" == "true" && "${PREEXISTING_NETWORK}" != "true" ]]; then
2620 gcloud compute networks subnets create "${SUBNETWORK}" --project "${NETWORK_PROJECT}" --region "${REGION}" --network "${NETWORK}" --range "${NODE_IP_RANGE}"
2621 fi
2622 return;;
2623 *) echo "${color_red}Invalid argument to ENABLE_IP_ALIASES${color_norm}"
2624 exit 1;;
2625 esac
2626
2627 # Look for the alias subnet, it must exist and have a secondary
2628 # range configured.
2629 local subnet
2630 subnet=$(gcloud compute networks subnets describe \
2631 --project "${NETWORK_PROJECT}" \
2632 --region "${REGION}" \
2633 "${IP_ALIAS_SUBNETWORK}" 2>/dev/null || true)
2634 if [[ -z "${subnet}" ]]; then
2635 echo "Creating subnet ${NETWORK}:${IP_ALIAS_SUBNETWORK}"
2636 gcloud compute networks subnets create \
2637 "${IP_ALIAS_SUBNETWORK}" \
2638 --description "Automatically generated subnet for ${INSTANCE_PREFIX} cluster. This will be removed on cluster teardown." \
2639 --project "${NETWORK_PROJECT}" \
2640 --network "${NETWORK}" \
2641 --region "${REGION}" \
2642 --range "${NODE_IP_RANGE}" \
2643 --secondary-range "pods-default=${CLUSTER_IP_RANGE}" \
2644 --secondary-range "services-default=${SERVICE_CLUSTER_IP_RANGE}"
2645 echo "Created subnetwork ${IP_ALIAS_SUBNETWORK}"
2646 else
2647 if ! echo "${subnet}" | grep --quiet secondaryIpRanges; then
2648 echo "${color_red}Subnet ${IP_ALIAS_SUBNETWORK} does not have a secondary range${color_norm}"
2649 exit 1
2650 fi
2651 fi
2652}
2653
2654# detect-subnetworks sets the SUBNETWORK var if not already set
2655# Assumed vars:
2656# NETWORK
2657# REGION
2658# NETWORK_PROJECT
2659#
2660# Optional vars:
2661# SUBNETWORK
2662# IP_ALIAS_SUBNETWORK
2663function detect-subnetworks() {
2664 if [[ -n ${SUBNETWORK:-} ]]; then
2665 echo "Using subnet ${SUBNETWORK}"
2666 return 0
2667 fi
2668
2669 if [[ -n ${IP_ALIAS_SUBNETWORK:-} ]]; then
2670 SUBNETWORK=${IP_ALIAS_SUBNETWORK}
2671 echo "Using IP Alias subnet ${SUBNETWORK}"
2672 return 0
2673 fi
2674
2675 SUBNETWORK=$(gcloud compute networks subnets list \
2676 --network="${NETWORK}" \
2677 --regions="${REGION}" \
2678 --project="${NETWORK_PROJECT}" \
2679 --limit=1 \
2680 --format='value(name)' 2>/dev/null)
2681
2682 if [[ -n ${SUBNETWORK:-} ]]; then
2683 echo "Found subnet for region ${REGION} in network ${NETWORK}: ${SUBNETWORK}"
2684 return 0
2685 fi
2686
2687 echo "${color_red}Could not find subnetwork with region ${REGION}, network ${NETWORK}, and project ${NETWORK_PROJECT}"
2688}
2689
2690# Sets up Cloud NAT for the network.
2691# Assumed vars:
2692# NETWORK_PROJECT
2693# REGION
2694# NETWORK
2695function create-cloud-nat-router() {
2696 if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
2697 if gcloud compute routers describe "$NETWORK-nat-router" --project "$NETWORK_PROJECT" --region "$REGION" &>/dev/null; then
2698 echo "Cloud nat already exists"
2699 return 0
2700 fi
2701 gcloud compute routers create "$NETWORK-nat-router" \
2702 --project "$NETWORK_PROJECT" \
2703 --region "$REGION" \
2704 --network "$NETWORK"
2705 gcloud compute routers nats create "$NETWORK-nat-config" \
2706 --project "$NETWORK_PROJECT" \
2707 --router-region "$REGION" \
2708 --router "$NETWORK-nat-router" \
2709 --nat-primary-subnet-ip-ranges \
2710 --auto-allocate-nat-external-ips \
2711 ${GCE_PRIVATE_CLUSTER_PORTS_PER_VM:+--min-ports-per-vm ${GCE_PRIVATE_CLUSTER_PORTS_PER_VM}}
2712 fi
2713}
2714
2715function delete-all-firewall-rules() {
2716 local -a fws
2717 kube::util::read-array fws < <(gcloud compute firewall-rules list --project "${NETWORK_PROJECT}" --filter="network=${NETWORK}" --format="value(name)")
2718 if (( "${#fws[@]}" > 0 )); then
2719 echo "Deleting firewall rules remaining in network ${NETWORK}: ${fws[*]}"
2720 delete-firewall-rules "${fws[@]}"
2721 else
2722 echo "No firewall rules in network ${NETWORK}"
2723 fi
2724}
2725
2726# Ignores firewall rule arguments that do not exist in NETWORK_PROJECT.
2727function delete-firewall-rules() {
2728 for fw in "$@"; do
2729 if [[ -n $(gcloud compute firewall-rules --project "${NETWORK_PROJECT}" describe "${fw}" --format='value(name)' 2>/dev/null || true) ]]; then
2730 gcloud compute firewall-rules delete --project "${NETWORK_PROJECT}" --quiet "${fw}" &
2731 fi
2732 done
2733 kube::util::wait-for-jobs || {
2734 echo -e "${color_red}Failed to delete firewall rules.${color_norm}" >&2
2735 }
2736}
2737
2738function delete-network() {
2739 if [[ -n $(gcloud compute networks --project "${NETWORK_PROJECT}" describe "${NETWORK}" --format='value(name)' 2>/dev/null || true) ]]; then
2740 if ! gcloud compute networks delete --project "${NETWORK_PROJECT}" --quiet "${NETWORK}"; then
2741 echo "Failed to delete network '${NETWORK}'. Listing firewall-rules:"
2742 gcloud compute firewall-rules --project "${NETWORK_PROJECT}" list --filter="network=${NETWORK}"
2743 return 1
2744 fi
2745 fi
2746}
2747
2748function delete-cloud-nat-router() {
2749 if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
2750 if [[ -n $(gcloud compute routers describe --project "${NETWORK_PROJECT}" --region "${REGION}" "${NETWORK}-nat-router" --format='value(name)' 2>/dev/null || true) ]]; then
2751 echo "Deleting Cloud NAT router..."
2752 gcloud compute routers delete --project "${NETWORK_PROJECT}" --region "${REGION}" --quiet "${NETWORK}-nat-router"
2753 fi
2754 fi
2755}
2756
2757function delete-subnetworks() {
2758 # If running in custom mode network we need to delete subnets manually.
2759 mode="$(check-network-mode)"
2760 if [[ "${mode}" == "CUSTOM" ]]; then
2761 if [[ "${ENABLE_BIG_CLUSTER_SUBNETS}" = "true" ]]; then
2762 echo "Deleting default subnets..."
2763 # This value should be kept in sync with number of regions.
2764 local parallelism=9
2765 gcloud compute networks subnets list --network="${NETWORK}" --project "${NETWORK_PROJECT}" --format='value(region.basename())' | \
2766 xargs -I {} -P ${parallelism} gcloud --quiet compute networks subnets delete "${NETWORK}" --project "${NETWORK_PROJECT}" --region="{}" || true
2767 elif [[ "${CREATE_CUSTOM_NETWORK:-}" == "true" ]]; then
2768 echo "Deleting custom subnet..."
2769 gcloud --quiet compute networks subnets delete "${SUBNETWORK}" --project "${NETWORK_PROJECT}" --region="${REGION}" || true
2770 fi
2771 return
2772 fi
2773
2774 # If we reached here, it means we're not using custom network.
2775 # So the only thing we need to check is if IP-aliases was turned
2776 # on and we created a subnet for it. If so, we should delete it.
2777 if [[ ${ENABLE_IP_ALIASES:-} == "true" ]]; then
2778 # Only delete the subnet if we created it (i.e it's not pre-existing).
2779 if [[ -z "${KUBE_GCE_IP_ALIAS_SUBNETWORK:-}" ]]; then
2780 echo "Removing auto-created subnet ${NETWORK}:${IP_ALIAS_SUBNETWORK}"
2781 if [[ -n $(gcloud compute networks subnets describe \
2782 --project "${NETWORK_PROJECT}" \
2783 --region "${REGION}" \
2784 "${IP_ALIAS_SUBNETWORK}" 2>/dev/null) ]]; then
2785 gcloud --quiet compute networks subnets delete \
2786 --project "${NETWORK_PROJECT}" \
2787 --region "${REGION}" \
2788 "${IP_ALIAS_SUBNETWORK}"
2789 fi
2790 fi
2791 fi
2792}
2793
2794# Generates SSL certificates for etcd cluster peer to peer communication. Uses cfssl program.
2795#
2796# Assumed vars:
2797# KUBE_TEMP: temporary directory
2798#
2799# Args:
2800# $1: host name
2801# $2: CA certificate
2802# $3: CA key
2803#
2804# If CA cert/key is empty, the function will also generate certs for CA.
2805#
2806# Vars set:
2807# ETCD_CA_KEY_BASE64
2808# ETCD_CA_CERT_BASE64
2809# ETCD_PEER_KEY_BASE64
2810# ETCD_PEER_CERT_BASE64
2811#
2812function create-etcd-certs {
2813 local host=${1}
2814 local ca_cert=${2:-}
2815 local ca_key=${3:-}
2816
2817 GEN_ETCD_CA_CERT="${ca_cert}" GEN_ETCD_CA_KEY="${ca_key}" \
2818 generate-etcd-cert "${KUBE_TEMP}/cfssl" "${host}" "peer" "peer"
2819
2820 pushd "${KUBE_TEMP}/cfssl"
2821 ETCD_CA_KEY_BASE64=$(base64 "ca-key.pem" | tr -d '\r\n')
2822 ETCD_CA_CERT_BASE64=$(gzip -c "ca.pem" | base64 | tr -d '\r\n')
2823 ETCD_PEER_KEY_BASE64=$(base64 "peer-key.pem" | tr -d '\r\n')
2824 ETCD_PEER_CERT_BASE64=$(gzip -c "peer.pem" | base64 | tr -d '\r\n')
2825 popd
2826}
2827
2828# Generates SSL certificates for etcd-client and kube-apiserver communication. Uses cfssl program.
2829#
2830# Assumed vars:
2831# KUBE_TEMP: temporary directory
2832#
2833# Args:
2834# $1: host server name
2835# $2: host client name
2836# $3: CA certificate
2837# $4: CA key
2838#
2839# If CA cert/key is empty, the function will also generate certs for CA.
2840#
2841# Vars set:
2842# ETCD_APISERVER_CA_KEY_BASE64
2843# ETCD_APISERVER_CA_CERT_BASE64
2844# ETCD_APISERVER_SERVER_KEY_BASE64
2845# ETCD_APISERVER_SERVER_CERT_BASE64
2846# ETCD_APISERVER_CLIENT_KEY_BASE64
2847# ETCD_APISERVER_CLIENT_CERT_BASE64
2848#
2849function create-etcd-apiserver-certs {
2850 local hostServer=${1}
2851 local hostClient=${2}
2852 local etcd_apiserver_ca_cert=${3:-}
2853 local etcd_apiserver_ca_key=${4:-}
2854
2855 GEN_ETCD_CA_CERT="${etcd_apiserver_ca_cert}" GEN_ETCD_CA_KEY="${etcd_apiserver_ca_key}" \
2856 generate-etcd-cert "${KUBE_TEMP}/cfssl" "${hostServer}" "server" "etcd-apiserver-server"
2857 generate-etcd-cert "${KUBE_TEMP}/cfssl" "${hostClient}" "client" "etcd-apiserver-client"
2858
2859 pushd "${KUBE_TEMP}/cfssl"
2860 ETCD_APISERVER_CA_KEY_BASE64=$(base64 "ca-key.pem" | tr -d '\r\n')
2861 ETCD_APISERVER_CA_CERT_BASE64=$(gzip -c "ca.pem" | base64 | tr -d '\r\n')
2862 ETCD_APISERVER_SERVER_KEY_BASE64=$(base64 "etcd-apiserver-server-key.pem" | tr -d '\r\n')
2863 ETCD_APISERVER_SERVER_CERT_BASE64=$(gzip -c "etcd-apiserver-server.pem" | base64 | tr -d '\r\n')
2864 ETCD_APISERVER_CLIENT_KEY_BASE64=$(base64 "etcd-apiserver-client-key.pem" | tr -d '\r\n')
2865 ETCD_APISERVER_CLIENT_CERT_BASE64=$(gzip -c "etcd-apiserver-client.pem" | base64 | tr -d '\r\n')
2866 popd
2867}
2868
2869
2870function create-master() {
2871 echo "Starting master and configuring firewalls"
2872 gcloud compute firewall-rules create "${MASTER_NAME}-https" \
2873 --project "${NETWORK_PROJECT}" \
2874 --network "${NETWORK}" \
2875 --target-tags "${MASTER_TAG}" \
2876 --allow tcp:443 &
2877
2878 echo "Configuring firewall for apiserver konnectivity server"
2879 if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
2880 gcloud compute firewall-rules create "${MASTER_NAME}-konnectivity-server" \
2881 --project "${NETWORK_PROJECT}" \
2882 --network "${NETWORK}" \
2883 --target-tags "${MASTER_TAG}" \
2884 --allow tcp:8132 &
2885 fi
2886
2887 # We have to make sure the disk is created before creating the master VM, so
2888 # run this in the foreground.
2889 gcloud compute disks create "${MASTER_NAME}-pd" \
2890 --project "${PROJECT}" \
2891 --zone "${ZONE}" \
2892 --type "${MASTER_DISK_TYPE}" \
2893 --size "${MASTER_DISK_SIZE}"
2894
2895 # Create rule for accessing and securing etcd servers.
2896 if ! gcloud compute firewall-rules --project "${NETWORK_PROJECT}" describe "${MASTER_NAME}-etcd" &>/dev/null; then
2897 gcloud compute firewall-rules create "${MASTER_NAME}-etcd" \
2898 --project "${NETWORK_PROJECT}" \
2899 --network "${NETWORK}" \
2900 --source-tags "${MASTER_TAG}" \
2901 --allow "tcp:2380,tcp:2381" \
2902 --target-tags "${MASTER_TAG}" &
2903 fi
2904
2905 # Generate a bearer token for this cluster. We push this separately
2906 # from the other cluster variables so that the client (this
2907 # computer) can forget it later. This should disappear with
2908 # http://issue.k8s.io/3168
2909 KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
2910 if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
2911 NODE_PROBLEM_DETECTOR_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
2912 fi
2913
2914 # Reserve the master's IP so that it can later be transferred to another VM
2915 # without disrupting the kubelets.
2916 create-static-ip "${MASTER_NAME}-ip" "${REGION}"
2917 MASTER_RESERVED_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
2918 --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
2919
2920 if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" ]]; then
2921 KUBELET_APISERVER="${MASTER_RESERVED_IP}"
2922 fi
2923
2924 KUBERNETES_MASTER_NAME="${MASTER_RESERVED_IP}"
2925 MASTER_ADVERTISE_ADDRESS="${MASTER_RESERVED_IP}"
2926
2927 MASTER_INTERNAL_IP=""
2928 if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
2929 gcloud compute addresses create "${MASTER_NAME}-internal-ip" --project "${PROJECT}" --region "$REGION" --subnet "$SUBNETWORK"
2930 MASTER_INTERNAL_IP=$(gcloud compute addresses describe "${MASTER_NAME}-internal-ip" --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
2931 echo "Master internal ip is: $MASTER_INTERNAL_IP"
2932 KUBERNETES_MASTER_NAME="${MASTER_INTERNAL_IP}"
2933 MASTER_ADVERTISE_ADDRESS="${MASTER_INTERNAL_IP}"
2934 fi
2935
2936 create-certs "${MASTER_RESERVED_IP}" "${MASTER_INTERNAL_IP}"
2937 create-etcd-certs "${MASTER_NAME}"
2938 create-etcd-apiserver-certs "etcd-${MASTER_NAME}" "${MASTER_NAME}"
2939
2940 if [[ "$(get-num-nodes)" -ge "50" ]]; then
2941 # We block on master creation for large clusters to avoid doing too much
2942 # unnecessary work in case master start-up fails (like creation of nodes).
2943 create-master-instance "${MASTER_RESERVED_IP}" "${MASTER_INTERNAL_IP}"
2944 else
2945 create-master-instance "${MASTER_RESERVED_IP}" "${MASTER_INTERNAL_IP}" &
2946 fi
2947
2948}
2949
2950# Adds master replica to etcd cluster.
2951#
2952# Assumed vars:
2953# REPLICA_NAME
2954# PROJECT
2955# EXISTING_MASTER_NAME
2956# EXISTING_MASTER_ZONE
2957#
2958# $1: etcd client port
2959# $2: etcd internal port
2960# $3: whether etcd communication should use mtls
2961# returns the result of ssh command which adds replica
2962function add-replica-to-etcd() {
2963 local -r client_port="${1}"
2964 local -r internal_port="${2}"
2965 local -r use_mtls="${3}"
2966
2967 TLSARG=""
2968 PROTO="http://"
2969 if [[ "${use_mtls}" == "true" ]]; then
2970 # Keep in sync with ETCD_APISERVER_CA_CERT_PATH, ETCD_APISERVER_CLIENT_CERT_PATH and ETCD_APISERVER_CLIENT_KEY_PATH in configure-helper.sh.
2971 TLSARG="--cacert /etc/srv/kubernetes/pki/etcd-apiserver-ca.crt --cert /etc/srv/kubernetes/pki/etcd-apiserver-client.crt --key /etc/srv/kubernetes/pki/etcd-apiserver-client.key"
2972 PROTO="https://"
2973 fi
2974 run-gcloud-command "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}" "curl ${TLSARG} ${PROTO}127.0.0.1:${client_port}/v2/members -XPOST -H \"Content-Type: application/json\" -d '{\"peerURLs\":[\"https://${REPLICA_NAME}:${internal_port}\"]}' -s"
2975 return $?
2976}
2977
2978# Sets EXISTING_MASTER_NAME and EXISTING_MASTER_ZONE variables.
2979#
2980# Assumed vars:
2981# PROJECT
2982#
2983# NOTE: Must be in sync with get-replica-name-regexp
2984function set-existing-master() {
2985 local existing_master
2986 existing_master=$(gcloud compute instances list \
2987 --project "${PROJECT}" \
2988 --filter "name ~ '$(get-replica-name-regexp)'" \
2989 --format "value(name,zone)" | head -n1)
2990 EXISTING_MASTER_NAME="$(echo "${existing_master}" | cut -f1)"
2991 EXISTING_MASTER_ZONE="$(echo "${existing_master}" | cut -f2)"
2992}
2993
2994function replicate-master() {
2995 set-replica-name
2996 set-existing-master
2997
2998 echo "Experimental: replicating existing master ${EXISTING_MASTER_ZONE}/${EXISTING_MASTER_NAME} as ${ZONE}/${REPLICA_NAME}"
2999
3000 # Before we do anything else, we should configure etcd to expect more replicas.
3001 if ! add-replica-to-etcd 2379 2380 true; then
3002 echo "Failed to add master replica to etcd cluster."
3003 return 1
3004 fi
3005 if ! add-replica-to-etcd 4002 2381 false; then
3006 echo "Failed to add master replica to etcd events cluster."
3007 return 1
3008 fi
3009
3010 # We have to make sure the disk is created before creating the master VM, so
3011 # run this in the foreground.
3012 gcloud compute disks create "${REPLICA_NAME}-pd" \
3013 --project "${PROJECT}" \
3014 --zone "${ZONE}" \
3015 --type "${MASTER_DISK_TYPE}" \
3016 --size "${MASTER_DISK_SIZE}"
3017
3018 local existing_master_replicas
3019 existing_master_replicas="$(get-all-replica-names)"
3020 replicate-master-instance "${EXISTING_MASTER_ZONE}" "${EXISTING_MASTER_NAME}" "${existing_master_replicas}"
3021
3022 # Add new replica to the load balancer.
3023 gcloud compute target-pools add-instances "${MASTER_NAME}" \
3024 --project "${PROJECT}" \
3025 --zone "${ZONE}" \
3026 --instances "${REPLICA_NAME}"
3027
3028 if [[ "${GCE_PRIVATE_CLUSTER:-}" == "true" ]]; then
3029 add-to-internal-loadbalancer "${REPLICA_NAME}" "${ZONE}"
3030 fi
3031}
3032
3033# Detaches old and ataches new external IP to a VM.
3034#
3035# Arguments:
3036# $1 - VM name
3037# $2 - VM zone
3038# $3 - external static IP; if empty will use an ephemeral IP address.
3039function attach-external-ip() {
3040 local NAME=${1}
3041 local ZONE=${2}
3042 local IP_ADDR=${3:-}
3043 local ACCESS_CONFIG_NAME
3044 ACCESS_CONFIG_NAME=$(gcloud compute instances describe "${NAME}" \
3045 --project "${PROJECT}" --zone "${ZONE}" \
3046 --format="value(networkInterfaces[0].accessConfigs[0].name)")
3047 gcloud compute instances delete-access-config "${NAME}" \
3048 --project "${PROJECT}" --zone "${ZONE}" \
3049 --access-config-name "${ACCESS_CONFIG_NAME}"
3050 if [[ -z "${IP_ADDR}" ]]; then
3051 gcloud compute instances add-access-config "${NAME}" \
3052 --project "${PROJECT}" --zone "${ZONE}" \
3053 --access-config-name "${ACCESS_CONFIG_NAME}"
3054 else
3055 gcloud compute instances add-access-config "${NAME}" \
3056 --project "${PROJECT}" --zone "${ZONE}" \
3057 --access-config-name "${ACCESS_CONFIG_NAME}" \
3058 --address "${IP_ADDR}"
3059 fi
3060}
3061
3062# Creates load balancer in front of apiserver if it doesn't exists already. Assumes there's only one
3063# existing master replica.
3064#
3065# Assumes:
3066# PROJECT
3067# MASTER_NAME
3068# ZONE
3069# REGION
3070function create-loadbalancer() {
3071 # Step 0: Return early if LB is already configured.
3072 if gcloud compute forwarding-rules describe "${MASTER_NAME}" \
3073 --project "${PROJECT}" --region "${REGION}" > /dev/null 2>&1; then
3074 echo "Load balancer already exists"
3075 return
3076 fi
3077
3078 local EXISTING_MASTER_NAME
3079 local EXISTING_MASTER_ZONE
3080 EXISTING_MASTER_NAME="$(get-all-replica-names)"
3081 EXISTING_MASTER_ZONE=$(gcloud compute instances list "${EXISTING_MASTER_NAME}" \
3082 --project "${PROJECT}" --format='value(zone)')
3083
3084 echo "Creating load balancer in front of an already existing master in ${EXISTING_MASTER_ZONE}"
3085
3086 # Step 1: Detach master IP address and attach ephemeral address to the existing master
3087 attach-external-ip "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}"
3088
3089 # Step 2: Create target pool.
3090 gcloud compute target-pools create "${MASTER_NAME}" --project "${PROJECT}" --region "${REGION}"
3091 # TODO: We should also add master instances with suffixes
3092 gcloud compute target-pools add-instances "${MASTER_NAME}" --instances "${EXISTING_MASTER_NAME}" --project "${PROJECT}" --zone "${EXISTING_MASTER_ZONE}"
3093
3094 # Step 3: Create forwarding rule.
3095 # TODO: This step can take up to 20 min. We need to speed this up...
3096 gcloud compute forwarding-rules create "${MASTER_NAME}" \
3097 --project "${PROJECT}" --region "${REGION}" \
3098 --target-pool "${MASTER_NAME}" --address="${KUBE_MASTER_IP}" --ports=443
3099
3100 echo -n "Waiting for the load balancer configuration to propagate..."
3101 local counter=0
3102 until curl -k -m1 "https://${KUBE_MASTER_IP}" &> /dev/null; do
3103 counter=$((counter+1))
3104 echo -n .
3105 if [[ ${counter} -ge 1800 ]]; then
3106 echo -e "${color_red}TIMEOUT${color_norm}" >&2
3107 echo -e "${color_red}Load balancer failed to initialize within ${counter} seconds.${color_norm}" >&2
3108 exit 2
3109 fi
3110 done
3111 echo "DONE"
3112}
3113
3114
3115# attach-internal-master-ip attach internal ip to existing master.
3116#
3117# Assumes:
3118# * PROJECT
3119function attach-internal-master-ip() {
3120 local name="${1}"
3121 local zone="${2}"
3122 local ip="${3}"
3123
3124 local aliases
3125 aliases=$(gcloud compute instances describe "${name}" --project "${PROJECT}" --zone "${zone}" --flatten='networkInterfaces[0].aliasIpRanges[]' --format='value[separator=':'](networkInterfaces[0].aliasIpRanges.subnetworkRangeName,networkInterfaces[0].aliasIpRanges.ipCidrRange)' | sed 's/^://' | paste -s -d';' -)
3126 aliases="${aliases:+${aliases};}${ip}/32"
3127 echo "Setting ${name}'s aliases to '${aliases}' (added ${ip})"
3128 # Attach ${ip} to ${name}
3129 gcloud compute instances network-interfaces update "${name}" --project "${PROJECT}" --zone "${zone}" --aliases="${aliases}"
3130 gcloud compute instances add-metadata "${name}" --zone "${zone}" --metadata=kube-master-internal-ip="${ip}"
3131 run-gcloud-command "${name}" "${zone}" 'sudo /bin/bash /home/kubernetes/bin/kube-master-internal-route.sh' || true
3132 return $?
3133}
3134
3135
3136# detach-internal-master-ip detaches internal ip from existing master.
3137#
3138# Assumes:
3139# * PROJECT
3140function detach-internal-master-ip() {
3141 local name="${1}"
3142 local zone="${2}"
3143 local ip="${3}"
3144
3145 local aliases
3146 aliases=$(gcloud compute instances describe "${name}" --project "${PROJECT}" --zone "${zone}" --flatten='networkInterfaces[0].aliasIpRanges[]' --format='value[separator=':'](networkInterfaces[0].aliasIpRanges.subnetworkRangeName,networkInterfaces[0].aliasIpRanges.ipCidrRange)' | sed 's/^://' | grep -v "${ip}" | paste -s -d';' -)
3147 echo "Setting ${name}'s aliases to '${aliases}' (removed ${ip})"
3148 # Detach ${MASTER_NAME}-internal-ip from ${name}
3149 gcloud compute instances network-interfaces update "${name}" --project "${PROJECT}" --zone "${zone}" --aliases="${aliases}"
3150 gcloud compute instances remove-metadata "${name}" --zone "${zone}" --keys=kube-master-internal-ip
3151 # We want `ip route` to be run in the cloud and not this host
3152 run-gcloud-command "${name}" "${zone}" "sudo ip route del to local ${ip}/32 dev \$(ip route | grep default | while read -r _ _ _ _ dev _; do echo \$dev; done)" || true
3153 return $?
3154}
3155
3156# create-internal-loadbalancer creates an internal load balacer in front of existing master.
3157#
3158# Assumes:
3159# * MASTER_NAME
3160# * PROJECT
3161# * REGION
3162function create-internal-loadbalancer() {
3163 if gcloud compute forwarding-rules describe "${MASTER_NAME}-internal" \
3164 --project "${PROJECT}" --region "${REGION}" > /dev/null 2>&1; then
3165 echo "Load balancer already exists"
3166 return
3167 fi
3168
3169 local EXISTING_MASTER_NAME
3170 local EXISTING_MASTER_ZONE
3171 EXISTING_MASTER_NAME="$(get-all-replica-names)"
3172 EXISTING_MASTER_ZONE=$(gcloud compute instances list "${EXISTING_MASTER_NAME}" \
3173 --project "${PROJECT}" --format='value(zone)')
3174
3175 echo "Detaching ${KUBE_MASTER_INTERNAL_IP} from ${EXISTING_MASTER_NAME}/${EXISTING_MASTER_ZONE}"
3176 detach-internal-master-ip "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}" "${KUBE_MASTER_INTERNAL_IP}"
3177
3178 echo "Creating internal load balancer with IP: ${KUBE_MASTER_INTERNAL_IP}"
3179 gcloud compute health-checks --project "${PROJECT}" create tcp "${MASTER_NAME}-hc" --port=443
3180
3181 gcloud compute backend-services create "${MASTER_NAME}" \
3182 --project "${PROJECT}" \
3183 --region "${REGION}" \
3184 --protocol tcp \
3185 --region "${REGION}" \
3186 --load-balancing-scheme internal \
3187 --health-checks "${MASTER_NAME}-hc"
3188
3189 gcloud compute forwarding-rules create "${MASTER_NAME}-internal" \
3190 --project "${PROJECT}" \
3191 --region "${REGION}" \
3192 --load-balancing-scheme internal \
3193 --network "${NETWORK}" \
3194 --subnet "${SUBNETWORK}" \
3195 --address "${KUBE_MASTER_INTERNAL_IP}" \
3196 --ip-protocol TCP \
3197 --ports 443 \
3198 --backend-service "${MASTER_NAME}" \
3199 --backend-service-region "${REGION}"
3200
3201 echo "Adding ${EXISTING_MASTER_NAME}/${EXISTING_MASTER_ZONE} to the load balancer"
3202 add-to-internal-loadbalancer "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}"
3203}
3204
3205# add-to-internal-loadbalancer adds an instance to ILB.
3206# Assumes:
3207# * MASTER_NAME
3208# * PROJECT
3209# * REGION
3210function add-to-internal-loadbalancer() {
3211 local name="${1}"
3212 local zone="${2}"
3213
3214 gcloud compute instance-groups unmanaged create "${name}" --project "${PROJECT}" --zone "${zone}"
3215 gcloud compute instance-groups unmanaged add-instances "${name}" --project "${PROJECT}" --zone "${zone}" --instances "${name}"
3216 gcloud compute backend-services add-backend "${MASTER_NAME}" \
3217 --project "${PROJECT}" \
3218 --region "${REGION}" \
3219 --instance-group "${name}" \
3220 --instance-group-zone "${zone}"
3221}
3222
3223# remove-from-internal-loadbalancer removes an instance from ILB.
3224# Assumes:
3225# * MASTER_NAME
3226# * PROJECT
3227# * REGION
3228function remove-from-internal-loadbalancer() {
3229 local name="${1}"
3230 local zone="${2}"
3231
3232 if gcloud compute instance-groups unmanaged describe "${name}" --project "${PROJECT}" --zone "${zone}" &>/dev/null; then
3233 gcloud compute backend-services remove-backend "${MASTER_NAME}" \
3234 --project "${PROJECT}" \
3235 --region "${REGION}" \
3236 --instance-group "${name}" \
3237 --instance-group-zone "${zone}"
3238 gcloud compute instance-groups unmanaged delete "${name}" --project "${PROJECT}" --zone "${zone}" --quiet
3239 fi
3240}
3241
3242function delete-internal-loadbalancer() {
3243 if gcloud compute forwarding-rules describe "${MASTER_NAME}-internal" --project "${PROJECT}" --region "${REGION}" &>/dev/null; then
3244 gcloud compute forwarding-rules delete "${MASTER_NAME}-internal" --project "${PROJECT}" --region "${REGION}" --quiet
3245 fi
3246
3247 if gcloud compute backend-services describe "${MASTER_NAME}" --project "${PROJECT}" --region "${REGION}" &>/dev/null; then
3248 gcloud compute backend-services delete "${MASTER_NAME}" --project "${PROJECT}" --region "${REGION}" --quiet
3249 fi
3250 if gcloud compute health-checks describe "${MASTER_NAME}-gc" --project "${PROJECT}" &>/dev/null; then
3251 gcloud compute health-checks delete "${MASTER_NAME}-gc" --project "${PROJECT}" --quiet
3252 fi
3253}
3254
3255function create-nodes-firewall() {
3256 # Create a single firewall rule for all minions.
3257 create-firewall-rule "${NODE_TAG}-all" "${CLUSTER_IP_RANGE}" "${NODE_TAG}" &
3258
3259 # Report logging choice (if any).
3260 if [[ "${ENABLE_NODE_LOGGING-}" == "true" ]]; then
3261 echo "+++ Logging using Fluentd to ${LOGGING_DESTINATION:-unknown}"
3262 fi
3263
3264 # Wait for last batch of jobs
3265 kube::util::wait-for-jobs || {
3266 code=$?
3267 echo -e "${color_red}Failed to create firewall rule.${color_norm}" >&2
3268 exit $code
3269 }
3270}
3271
3272function get-scope-flags() {
3273 local scope_flags=
3274 if [[ -n "${NODE_SCOPES}" ]]; then
3275 scope_flags="--scopes ${NODE_SCOPES}"
3276 else
3277 scope_flags="--no-scopes"
3278 fi
3279 echo "${scope_flags}"
3280}
3281
3282function create-nodes-template() {
3283 echo "Creating nodes."
3284
3285 local scope_flags
3286 scope_flags=$(get-scope-flags)
3287
3288 write-linux-node-env
3289 write-windows-node-env
3290
3291 # NOTE: these template names and their format must match
3292 # create-[linux,windows]-nodes() as well as get-template()!
3293 local linux_template_name="${NODE_INSTANCE_PREFIX}-template"
3294 local windows_template_name="${WINDOWS_NODE_INSTANCE_PREFIX}-template"
3295 create-linux-node-instance-template "$linux_template_name"
3296 create-windows-node-instance-template "$windows_template_name" "${scope_flags[*]}"
3297 if [[ -n "${ADDITIONAL_MACHINE_TYPE:-}" ]]; then
3298 local linux_extra_template_name="${NODE_INSTANCE_PREFIX}-extra-template"
3299 create-linux-node-instance-template "$linux_extra_template_name" "${ADDITIONAL_MACHINE_TYPE}"
3300 fi
3301}
3302
3303# Assumes:
3304# - MAX_INSTANCES_PER_MIG
3305# - NUM_NODES
3306# - NUM_WINDOWS_NODES
3307# exports:
3308# - NUM_MIGS
3309# - NUM_WINDOWS_MIGS
3310function set_num_migs() {
3311 local defaulted_max_instances_per_mig=${MAX_INSTANCES_PER_MIG:-1000}
3312
3313 if [[ ${defaulted_max_instances_per_mig} -le "0" ]]; then
3314 echo "MAX_INSTANCES_PER_MIG cannot be negative. Assuming default 1000"
3315 defaulted_max_instances_per_mig=1000
3316 fi
3317 export NUM_MIGS=$(((NUM_NODES + defaulted_max_instances_per_mig - 1) / defaulted_max_instances_per_mig))
3318 export NUM_WINDOWS_MIGS=$(((NUM_WINDOWS_NODES + defaulted_max_instances_per_mig - 1) / defaulted_max_instances_per_mig))
3319}
3320
3321# Assumes:
3322# - NUM_MIGS
3323# - NODE_INSTANCE_PREFIX
3324# - NUM_NODES
3325# - PROJECT
3326# - ZONE
3327function create-linux-nodes() {
3328 local template_name="${NODE_INSTANCE_PREFIX}-template"
3329 local extra_template_name="${NODE_INSTANCE_PREFIX}-extra-template"
3330
3331 local nodes="${NUM_NODES}"
3332 if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then
3333 echo "Creating a special node for heapster with machine-type ${HEAPSTER_MACHINE_TYPE}"
3334 create-heapster-node
3335 nodes=$(( nodes - 1 ))
3336 fi
3337
3338 if [[ -n "${ADDITIONAL_MACHINE_TYPE:-}" && "${NUM_ADDITIONAL_NODES:-}" -gt 0 ]]; then
3339 local num_additional="${NUM_ADDITIONAL_NODES}"
3340 if [[ "${NUM_ADDITIONAL_NODES:-}" -gt "${nodes}" ]]; then
3341 echo "Capping NUM_ADDITIONAL_NODES to ${nodes}"
3342 num_additional="${nodes}"
3343 fi
3344 if [[ "${num_additional:-}" -gt 0 ]]; then
3345 echo "Creating ${num_additional} special nodes with machine-type ${ADDITIONAL_MACHINE_TYPE}"
3346 local extra_group_name="${NODE_INSTANCE_PREFIX}-extra"
3347 gcloud compute instance-groups managed \
3348 create "${extra_group_name}" \
3349 --project "${PROJECT}" \
3350 --zone "${ZONE}" \
3351 --base-instance-name "${extra_group_name}" \
3352 --size "${num_additional}" \
3353 --template "${extra_template_name}" || true;
3354 gcloud compute instance-groups managed wait-until --stable \
3355 "${extra_group_name}" \
3356 --zone "${ZONE}" \
3357 --project "${PROJECT}" \
3358 --timeout "${MIG_WAIT_UNTIL_STABLE_TIMEOUT}" || true
3359 nodes=$(( nodes - num_additional ))
3360 fi
3361 fi
3362
3363 local instances_left=${nodes}
3364
3365 for ((i=1; i<=NUM_MIGS; i++)); do
3366 local group_name="${NODE_INSTANCE_PREFIX}-group-$i"
3367 if [[ $i -eq ${NUM_MIGS} ]]; then
3368 # TODO: We don't add a suffix for the last group to keep backward compatibility when there's only one MIG.
3369 # We should change it at some point, but note #18545 when changing this.
3370 group_name="${NODE_INSTANCE_PREFIX}-group"
3371 fi
3372 # Spread the remaining number of nodes evenly
3373 this_mig_size=$((instances_left / (NUM_MIGS - i + 1)))
3374 instances_left=$((instances_left - this_mig_size))
3375
3376 # Run instance-groups creation in parallel.
3377 {
3378 gcloud compute instance-groups managed \
3379 create "${group_name}" \
3380 --project "${PROJECT}" \
3381 --zone "${ZONE}" \
3382 --base-instance-name "${group_name}" \
3383 --size "${this_mig_size}" \
3384 --template "${template_name}" || true;
3385 gcloud compute instance-groups managed wait-until --stable \
3386 "${group_name}" \
3387 --zone "${ZONE}" \
3388 --project "${PROJECT}" \
3389 --timeout "${MIG_WAIT_UNTIL_STABLE_TIMEOUT}" || true
3390 } &
3391 done
3392 wait
3393}
3394
3395# Assumes:
3396# - NUM_WINDOWS_MIGS
3397# - WINDOWS_NODE_INSTANCE_PREFIX
3398# - NUM_WINDOWS_NODES
3399# - PROJECT
3400# - ZONE
3401function create-windows-nodes() {
3402 local template_name="${WINDOWS_NODE_INSTANCE_PREFIX}-template"
3403
3404 local -r nodes="${NUM_WINDOWS_NODES}"
3405 local instances_left=${nodes}
3406
3407 for ((i=1; i <= NUM_WINDOWS_MIGS; i++)); do
3408 local group_name="${WINDOWS_NODE_INSTANCE_PREFIX}-group-$i"
3409 if [[ $i -eq ${NUM_WINDOWS_MIGS} ]]; then
3410 # TODO: We don't add a suffix for the last group to keep backward compatibility when there's only one MIG.
3411 # We should change it at some point, but note #18545 when changing this.
3412 group_name="${WINDOWS_NODE_INSTANCE_PREFIX}-group"
3413 fi
3414 # Spread the remaining number of nodes evenly
3415 this_mig_size=$((instances_left / (NUM_WINDOWS_MIGS - i + 1)))
3416 instances_left=$((instances_left - this_mig_size))
3417
3418 gcloud compute instance-groups managed \
3419 create "${group_name}" \
3420 --project "${PROJECT}" \
3421 --zone "${ZONE}" \
3422 --base-instance-name "${group_name}" \
3423 --size "${this_mig_size}" \
3424 --template "${template_name}" || true;
3425 gcloud compute instance-groups managed wait-until --stable \
3426 "${group_name}" \
3427 --zone "${ZONE}" \
3428 --project "${PROJECT}" \
3429 --timeout "${MIG_WAIT_UNTIL_STABLE_TIMEOUT}" || true;
3430 done
3431}
3432
3433# Assumes:
3434# - NODE_INSTANCE_PREFIX
3435# - PROJECT
3436# - NETWORK_PROJECT
3437# - REGION
3438# - ZONE
3439# - HEAPSTER_MACHINE_TYPE
3440# - NODE_DISK_TYPE
3441# - NODE_DISK_SIZE
3442# - NODE_IMAGE_PROJECT
3443# - NODE_IMAGE
3444# - NODE_SERVICE_ACCOUNT
3445# - NODE_TAG
3446# - NETWORK
3447# - ENABLE_IP_ALIASES
3448# - SUBNETWORK
3449# - IP_ALIAS_SIZE
3450function create-heapster-node() {
3451 local gcloud="gcloud"
3452
3453 local network
3454 network=$(make-gcloud-network-argument \
3455 "${NETWORK_PROJECT}" \
3456 "${REGION}" \
3457 "${NETWORK}" \
3458 "${SUBNETWORK:-}" \
3459 "" \
3460 "${ENABLE_IP_ALIASES:-}" \
3461 "${IP_ALIAS_SIZE:-}")
3462
3463 # Deliberately word split ${network} and $(get-scope-flags)
3464 # shellcheck disable=SC2086 disable=SC2046
3465 ${gcloud} compute instances \
3466 create "${NODE_INSTANCE_PREFIX}-heapster" \
3467 --project "${PROJECT}" \
3468 --zone "${ZONE}" \
3469 --machine-type="${HEAPSTER_MACHINE_TYPE}" \
3470 --boot-disk-type "${NODE_DISK_TYPE}" \
3471 --boot-disk-size "${NODE_DISK_SIZE}" \
3472 --image-project="${NODE_IMAGE_PROJECT}" \
3473 --image "${NODE_IMAGE}" \
3474 --service-account "${NODE_SERVICE_ACCOUNT}" \
3475 --tags "${NODE_TAG}" \
3476 ${network} \
3477 $(get-scope-flags) \
3478 --metadata-from-file "$(get-node-instance-metadata-from-file "heapster-kube-env")"
3479}
3480
3481# Assumes:
3482# - NUM_MIGS
3483# - NODE_INSTANCE_PREFIX
3484# - PROJECT
3485# - ZONE
3486# - AUTOSCALER_MAX_NODES
3487# - AUTOSCALER_MIN_NODES
3488# Exports
3489# - AUTOSCALER_MIG_CONFIG
3490function create-cluster-autoscaler-mig-config() {
3491
3492 # Each MIG must have at least one node, so the min number of nodes
3493 # must be greater or equal to the number of migs.
3494 if [[ ${AUTOSCALER_MIN_NODES} -lt 0 ]]; then
3495 echo "AUTOSCALER_MIN_NODES must be greater or equal 0"
3496 exit 2
3497 fi
3498
3499 # Each MIG must have at least one node, so the min number of nodes
3500 # must be greater or equal to the number of migs.
3501 if [[ ${AUTOSCALER_MAX_NODES} -lt ${NUM_MIGS} ]]; then
3502 echo "AUTOSCALER_MAX_NODES must be greater or equal ${NUM_MIGS}"
3503 exit 2
3504 fi
3505 if [[ ${NUM_WINDOWS_MIGS} -gt 0 ]]; then
3506 # TODO(pjh): implement Windows support in this function.
3507 echo "Not implemented yet: autoscaler config for Windows MIGs"
3508 exit 2
3509 fi
3510
3511 # The code assumes that the migs were created with create-nodes
3512 # function which tries to evenly spread nodes across the migs.
3513 AUTOSCALER_MIG_CONFIG=""
3514
3515 local left_min=${AUTOSCALER_MIN_NODES}
3516 local left_max=${AUTOSCALER_MAX_NODES}
3517
3518 for ((i=1; i <= NUM_MIGS; i++)); do
3519 local group_name="${NODE_INSTANCE_PREFIX}-group-$i"
3520 if [[ $i -eq ${NUM_MIGS} ]]; then
3521 # TODO: We don't add a suffix for the last group to keep backward compatibility when there's only one MIG.
3522 # We should change it at some point, but note #18545 when changing this.
3523 group_name="${NODE_INSTANCE_PREFIX}-group"
3524 fi
3525
3526 this_mig_min=$((left_min/(NUM_MIGS-i+1)))
3527 this_mig_max=$((left_max/(NUM_MIGS-i+1)))
3528 left_min=$((left_min-this_mig_min))
3529 left_max=$((left_max-this_mig_max))
3530
3531 local mig_url="https://www.googleapis.com/compute/v1/projects/${PROJECT}/zones/${ZONE}/instanceGroups/${group_name}"
3532 AUTOSCALER_MIG_CONFIG="${AUTOSCALER_MIG_CONFIG} --nodes=${this_mig_min}:${this_mig_max}:${mig_url}"
3533 done
3534
3535 AUTOSCALER_MIG_CONFIG="${AUTOSCALER_MIG_CONFIG} --scale-down-enabled=${AUTOSCALER_ENABLE_SCALE_DOWN}"
3536}
3537
3538# Assumes:
3539# - NUM_MIGS
3540# - NODE_INSTANCE_PREFIX
3541# - PROJECT
3542# - ZONE
3543# - ENABLE_CLUSTER_AUTOSCALER
3544# - AUTOSCALER_MAX_NODES
3545# - AUTOSCALER_MIN_NODES
3546function create-autoscaler-config() {
3547 # Create autoscaler for nodes configuration if requested
3548 if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then
3549 create-cluster-autoscaler-mig-config
3550 echo "Using autoscaler config: ${AUTOSCALER_MIG_CONFIG} ${AUTOSCALER_EXPANDER_CONFIG}"
3551 fi
3552}
3553
3554function check-cluster() {
3555 detect-node-names
3556 detect-master
3557
3558 echo "Waiting up to ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} seconds for cluster initialization."
3559 echo
3560 echo " This will continually check to see if the API for kubernetes is reachable."
3561 echo " This may time out if there was some uncaught error during start up."
3562 echo
3563
3564 # curl in mavericks is borked.
3565 secure=""
3566 if which sw_vers >& /dev/null; then
3567 if [[ $(sw_vers | grep ProductVersion | awk '{print $2}') = "10.9."* ]]; then
3568 secure="--insecure"
3569 fi
3570 fi
3571
3572 local start_time
3573 local curl_out
3574 start_time=$(date +%s)
3575 curl_out=$(mktemp)
3576 kube::util::trap_add "rm -f ${curl_out}" EXIT
3577 until curl -vsS --cacert "${CERT_DIR}/pki/ca.crt" \
3578 -H "Authorization: Bearer ${KUBE_BEARER_TOKEN}" \
3579 ${secure} \
3580 --max-time 5 --fail \
3581 "https://${KUBE_MASTER_IP}/api/v1/pods?limit=100" > "${curl_out}" 2>&1; do
3582 local elapsed
3583 elapsed=$(($(date +%s) - start_time))
3584 if [[ ${elapsed} -gt ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} ]]; then
3585 echo -e "${color_red}Cluster failed to initialize within ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} seconds.${color_norm}" >&2
3586 echo "Last output from querying API server follows:" >&2
3587 echo "-----------------------------------------------------" >&2
3588 cat "${curl_out}" >&2
3589 echo "-----------------------------------------------------" >&2
3590 exit 2
3591 fi
3592 printf "."
3593 sleep 2
3594 done
3595
3596 echo "Kubernetes cluster created."
3597
3598 export KUBE_CERT="${CERT_DIR}/pki/issued/kubecfg.crt"
3599 export KUBE_KEY="${CERT_DIR}/pki/private/kubecfg.key"
3600 export CA_CERT="${CERT_DIR}/pki/ca.crt"
3601 export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}"
3602 (
3603 umask 077
3604
3605 # Update the user's kubeconfig to include credentials for this apiserver.
3606 create-kubeconfig
3607 )
3608
3609 # ensures KUBECONFIG is set
3610 get-kubeconfig-basicauth
3611
3612 if [[ ${GCE_UPLOAD_KUBCONFIG_TO_MASTER_METADATA:-} == "true" ]]; then
3613 gcloud compute instances add-metadata "${MASTER_NAME}" --project="${PROJECT}" --zone="${ZONE}" --metadata-from-file="kubeconfig=${KUBECONFIG}" || true
3614 fi
3615
3616 echo
3617 echo -e "${color_green:-}Kubernetes cluster is running. The master is running at:"
3618 echo
3619 echo -e "${color_yellow} https://${KUBE_MASTER_IP}"
3620 echo
3621 echo -e "${color_green}The user name and password to use is located in ${KUBECONFIG}.${color_norm}"
3622 echo
3623
3624}
3625
3626# Removes master replica from etcd cluster.
3627#
3628# Assumed vars:
3629# REPLICA_NAME
3630# PROJECT
3631# EXISTING_MASTER_NAME
3632# EXISTING_MASTER_ZONE
3633#
3634# $1: etcd client port
3635# $2: whether etcd communication should use mtls
3636# returns the result of ssh command which removes replica
3637function remove-replica-from-etcd() {
3638 local -r port="${1}"
3639 local -r use_mtls="${2}"
3640
3641 TLSARG=""
3642 PROTO="http://"
3643 if [[ "${use_mtls}" == "true" ]]; then
3644 # Keep in sync with ETCD_APISERVER_CA_CERT_PATH, ETCD_APISERVER_CLIENT_CERT_PATH and ETCD_APISERVER_CLIENT_KEY_PATH in configure-helper.sh.
3645 TLSARG="--cacert /etc/srv/kubernetes/pki/etcd-apiserver-ca.crt --cert /etc/srv/kubernetes/pki/etcd-apiserver-client.crt --key /etc/srv/kubernetes/pki/etcd-apiserver-client.key"
3646 PROTO="https://"
3647 fi
3648 [[ -n "${EXISTING_MASTER_NAME}" ]] || return
3649 run-gcloud-command "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}" "curl -s ${TLSARG} ${PROTO}127.0.0.1:${port}/v2/members/\$(curl -s ${TLSARG} ${PROTO}127.0.0.1:${port}/v2/members -XGET | sed 's/{\\\"id/\n/g' | grep ${REPLICA_NAME}\\\" | cut -f 3 -d \\\") -XDELETE -L 2>/dev/null"
3650 local -r res=$?
3651 echo "Removing etcd replica, name: ${REPLICA_NAME}, port: ${port}, result: ${res}"
3652 return "${res}"
3653}
3654
3655# Delete a kubernetes cluster. This is called from test-teardown.
3656#
3657# Assumed vars:
3658# MASTER_NAME
3659# NODE_INSTANCE_PREFIX
3660# WINDOWS_NODE_INSTANCE_PREFIX
3661# ZONE
3662# This function tears down cluster resources 10 at a time to avoid issuing too many
3663# API calls and exceeding API quota. It is important to bring down the instances before bringing
3664# down the firewall rules and routes.
3665function kube-down() {
3666 local -r batch=200
3667
3668 detect-project
3669 detect-node-names # For INSTANCE_GROUPS and WINDOWS_INSTANCE_GROUPS
3670
3671 echo "Bringing down cluster"
3672 set +e # Do not stop on error
3673
3674 if [[ "${KUBE_DELETE_NODES:-}" != "false" ]]; then
3675 # Get the name of the managed instance group template before we delete the
3676 # managed instance group. (The name of the managed instance group template may
3677 # change during a cluster upgrade.)
3678 local templates
3679 templates=$(get-template "${PROJECT}")
3680
3681 # Deliberately allow globbing, do not change unless a bug is found
3682 # shellcheck disable=SC2206
3683 local all_instance_groups=(${INSTANCE_GROUPS[@]:-} ${WINDOWS_INSTANCE_GROUPS[@]:-})
3684 # Deliberately do not quote, do not change unless a bug is found
3685 # shellcheck disable=SC2068
3686 for group in ${all_instance_groups[@]:-}; do
3687 {
3688 if gcloud compute instance-groups managed describe "${group}" --project "${PROJECT}" --zone "${ZONE}" &>/dev/null; then
3689 gcloud compute instance-groups managed delete \
3690 --project "${PROJECT}" \
3691 --quiet \
3692 --zone "${ZONE}" \
3693 "${group}"
3694 fi
3695 } &
3696 done
3697
3698 # Wait for last batch of jobs
3699 kube::util::wait-for-jobs || {
3700 echo -e "Failed to delete instance group(s)." >&2
3701 }
3702
3703 # Deliberately do not quote, do not change unless a bug is found
3704 # shellcheck disable=SC2068
3705 for template in ${templates[@]:-}; do
3706 {
3707 if gcloud compute instance-templates describe --project "${PROJECT}" "${template}" &>/dev/null; then
3708 gcloud compute instance-templates delete \
3709 --project "${PROJECT}" \
3710 --quiet \
3711 "${template}"
3712 fi
3713 } &
3714 done
3715
3716 # Wait for last batch of jobs
3717 kube::util::wait-for-jobs || {
3718 echo -e "Failed to delete instance template(s)." >&2
3719 }
3720
3721 # Delete the special heapster node (if it exists).
3722 if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then
3723 local -r heapster_machine_name="${NODE_INSTANCE_PREFIX}-heapster"
3724 if gcloud compute instances describe "${heapster_machine_name}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
3725 # Now we can safely delete the VM.
3726 gcloud compute instances delete \
3727 --project "${PROJECT}" \
3728 --quiet \
3729 --delete-disks all \
3730 --zone "${ZONE}" \
3731 "${heapster_machine_name}"
3732 fi
3733 fi
3734 fi
3735
3736 local -r REPLICA_NAME="${KUBE_REPLICA_NAME:-$(get-replica-name)}"
3737
3738 set-existing-master
3739
3740 # Un-register the master replica from etcd and events etcd.
3741 remove-replica-from-etcd 2379 true
3742 remove-replica-from-etcd 4002 false
3743
3744 # Delete the master replica (if it exists).
3745 if gcloud compute instances describe "${REPLICA_NAME}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
3746 # If there is a load balancer in front of apiservers we need to first update its configuration.
3747 if gcloud compute target-pools describe "${MASTER_NAME}" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
3748 gcloud compute target-pools remove-instances "${MASTER_NAME}" \
3749 --project "${PROJECT}" \
3750 --zone "${ZONE}" \
3751 --instances "${REPLICA_NAME}"
3752 fi
3753 # Detach replica from LB if needed.
3754 if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
3755 remove-from-internal-loadbalancer "${REPLICA_NAME}" "${ZONE}"
3756 fi
3757 # Now we can safely delete the VM.
3758 gcloud compute instances delete \
3759 --project "${PROJECT}" \
3760 --quiet \
3761 --delete-disks all \
3762 --zone "${ZONE}" \
3763 "${REPLICA_NAME}"
3764 fi
3765
3766 # Delete the master replica pd (possibly leaked by kube-up if master create failed).
3767 # TODO(jszczepkowski): remove also possibly leaked replicas' pds
3768 local -r replica_pd="${REPLICA_NAME:-${MASTER_NAME}}-pd"
3769 if gcloud compute disks describe "${replica_pd}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
3770 gcloud compute disks delete \
3771 --project "${PROJECT}" \
3772 --quiet \
3773 --zone "${ZONE}" \
3774 "${replica_pd}"
3775 fi
3776
3777 # Check if this are any remaining master replicas.
3778 local REMAINING_MASTER_COUNT
3779 REMAINING_MASTER_COUNT=$(gcloud compute instances list \
3780 --project "${PROJECT}" \
3781 --filter="name ~ '$(get-replica-name-regexp)'" \
3782 --format "value(zone)" | wc -l)
3783
3784 # In the replicated scenario, if there's only a single master left, we should also delete load balancer in front of it.
3785 if [[ "${REMAINING_MASTER_COUNT}" -eq 1 ]]; then
3786 detect-master
3787 local REMAINING_REPLICA_NAME
3788 local REMAINING_REPLICA_ZONE
3789 REMAINING_REPLICA_NAME="$(get-all-replica-names)"
3790 REMAINING_REPLICA_ZONE=$(gcloud compute instances list "${REMAINING_REPLICA_NAME}" \
3791 --project "${PROJECT}" --format='value(zone)')
3792 if gcloud compute forwarding-rules describe "${MASTER_NAME}" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
3793 gcloud compute forwarding-rules delete \
3794 --project "${PROJECT}" \
3795 --region "${REGION}" \
3796 --quiet \
3797 "${MASTER_NAME}"
3798 attach-external-ip "${REMAINING_REPLICA_NAME}" "${REMAINING_REPLICA_ZONE}" "${KUBE_MASTER_IP}"
3799 gcloud compute target-pools delete \
3800 --project "${PROJECT}" \
3801 --region "${REGION}" \
3802 --quiet \
3803 "${MASTER_NAME}"
3804 fi
3805
3806 if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
3807 remove-from-internal-loadbalancer "${REMAINING_REPLICA_NAME}" "${REMAINING_REPLICA_ZONE}"
3808 delete-internal-loadbalancer
3809 attach-internal-master-ip "${REMAINING_REPLICA_NAME}" "${REMAINING_REPLICA_ZONE}" "${KUBE_MASTER_INTERNAL_IP}"
3810 fi
3811 fi
3812
3813 # If there are no more remaining master replicas, we should delete all remaining network resources.
3814 if [[ "${REMAINING_MASTER_COUNT}" -eq 0 ]]; then
3815 # Delete firewall rule for the master, etcd servers, and nodes.
3816 delete-firewall-rules "${MASTER_NAME}-https" "${MASTER_NAME}-etcd" "${NODE_TAG}-all" "${MASTER_NAME}-konnectivity-server"
3817 # Delete the master's reserved IP
3818 if gcloud compute addresses describe "${MASTER_NAME}-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
3819 gcloud compute addresses delete \
3820 --project "${PROJECT}" \
3821 --region "${REGION}" \
3822 --quiet \
3823 "${MASTER_NAME}-ip"
3824 fi
3825
3826 if gcloud compute addresses describe "${MASTER_NAME}-internal-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
3827 gcloud compute addresses delete \
3828 --project "${PROJECT}" \
3829 --region "${REGION}" \
3830 --quiet \
3831 "${MASTER_NAME}-internal-ip"
3832 fi
3833 fi
3834
3835 if [[ "${KUBE_DELETE_NODES:-}" != "false" ]]; then
3836 # Find out what minions are running.
3837 local -a minions
3838 kube::util::read-array minions < <(gcloud compute instances list \
3839 --project "${PROJECT}" \
3840 --filter="(name ~ '${NODE_INSTANCE_PREFIX}-.+' OR name ~ '${WINDOWS_NODE_INSTANCE_PREFIX}-.+') AND zone:(${ZONE})" \
3841 --format='value(name)')
3842 # If any minions are running, delete them in batches.
3843 while (( "${#minions[@]}" > 0 )); do
3844 echo Deleting nodes "${minions[*]::${batch}}"
3845 gcloud compute instances delete \
3846 --project "${PROJECT}" \
3847 --quiet \
3848 --delete-disks boot \
3849 --zone "${ZONE}" \
3850 "${minions[@]::${batch}}"
3851 minions=( "${minions[@]:${batch}}" )
3852 done
3853 fi
3854
3855 # If there are no more remaining master replicas: delete routes, pd for influxdb and update kubeconfig
3856 if [[ "${REMAINING_MASTER_COUNT}" -eq 0 ]]; then
3857 # Delete routes.
3858 local -a routes
3859 # Clean up all routes w/ names like "<cluster-name>-<node-GUID>"
3860 # e.g. "kubernetes-12345678-90ab-cdef-1234-567890abcdef". The name is
3861 # determined by the node controller on the master.
3862 # Note that this is currently a noop, as synchronously deleting the node MIG
3863 # first allows the master to cleanup routes itself.
3864 local TRUNCATED_PREFIX="${INSTANCE_PREFIX:0:26}"
3865 kube::util::read-array routes < <(gcloud compute routes list --project "${NETWORK_PROJECT}" \
3866 --filter="name ~ '${TRUNCATED_PREFIX}-.{8}-.{4}-.{4}-.{4}-.{12}'" \
3867 --format='value(name)')
3868 while (( "${#routes[@]}" > 0 )); do
3869 echo Deleting routes "${routes[*]::${batch}}"
3870 gcloud compute routes delete \
3871 --project "${NETWORK_PROJECT}" \
3872 --quiet \
3873 "${routes[@]::${batch}}"
3874 routes=( "${routes[@]:${batch}}" )
3875 done
3876
3877 # Delete persistent disk for influx-db.
3878 if gcloud compute disks describe "${INSTANCE_PREFIX}"-influxdb-pd --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
3879 gcloud compute disks delete \
3880 --project "${PROJECT}" \
3881 --quiet \
3882 --zone "${ZONE}" \
3883 "${INSTANCE_PREFIX}"-influxdb-pd
3884 fi
3885
3886 # Delete all remaining firewall rules and network.
3887 delete-firewall-rules \
3888 "${CLUSTER_NAME}-default-internal-master" \
3889 "${CLUSTER_NAME}-default-internal-node"
3890
3891 if [[ "${KUBE_DELETE_NETWORK}" == "true" ]]; then
3892 delete-firewall-rules \
3893 "${NETWORK}-default-ssh" \
3894 "${NETWORK}-default-rdp" \
3895 "${NETWORK}-default-internal" # Pre-1.5 clusters
3896 delete-cloud-nat-router
3897 # Delete all remaining firewall rules in the network.
3898 delete-all-firewall-rules || true
3899 delete-subnetworks || true
3900 delete-network || true # might fail if there are leaked resources that reference the network
3901 fi
3902
3903 # If there are no more remaining master replicas, we should update kubeconfig.
3904 export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}"
3905 clear-kubeconfig
3906 else
3907 # If some master replicas remain: cluster has been changed, we need to re-validate it.
3908 echo "... calling validate-cluster" >&2
3909 # Override errexit
3910 (validate-cluster) && validate_result="$?" || validate_result="$?"
3911
3912 # We have two different failure modes from validate cluster:
3913 # - 1: fatal error - cluster won't be working correctly
3914 # - 2: weak error - something went wrong, but cluster probably will be working correctly
3915 # We just print an error message in case 2).
3916 if [[ "${validate_result}" -eq 1 ]]; then
3917 exit 1
3918 elif [[ "${validate_result}" -eq 2 ]]; then
3919 echo "...ignoring non-fatal errors in validate-cluster" >&2
3920 fi
3921 fi
3922 set -e
3923}
3924
3925# Prints name of one of the master replicas in the current zone. It will be either
3926# just MASTER_NAME or MASTER_NAME with a suffix for a replica (see get-replica-name-regexp).
3927#
3928# Assumed vars:
3929# PROJECT
3930# ZONE
3931# MASTER_NAME
3932#
3933# NOTE: Must be in sync with get-replica-name-regexp and set-replica-name.
3934function get-replica-name() {
3935 # Ignore if gcloud compute fails and successfully echo any outcome
3936 # shellcheck disable=SC2005
3937 echo "$(gcloud compute instances list \
3938 --project "${PROJECT}" \
3939 --filter="name ~ '$(get-replica-name-regexp)' AND zone:(${ZONE})" \
3940 --format "value(name)" | head -n1)"
3941}
3942
3943# Prints comma-separated names of all of the master replicas in all zones.
3944#
3945# Assumed vars:
3946# PROJECT
3947# MASTER_NAME
3948#
3949# NOTE: Must be in sync with get-replica-name-regexp and set-replica-name.
3950function get-all-replica-names() {
3951 # Ignore if gcloud compute fails and successfully echo any outcome
3952 # shellcheck disable=SC2005
3953 echo "$(gcloud compute instances list \
3954 --project "${PROJECT}" \
3955 --filter="name ~ '$(get-replica-name-regexp)'" \
3956 --format "value(name)" | tr "\n" "," | sed 's/,$//')"
3957}
3958
3959# Prints the number of all of the master replicas in all zones.
3960#
3961# Assumed vars:
3962# MASTER_NAME
3963function get-master-replicas-count() {
3964 detect-project
3965 local num_masters
3966 num_masters=$(gcloud compute instances list \
3967 --project "${PROJECT}" \
3968 --filter="name ~ '$(get-replica-name-regexp)'" \
3969 --format "value(zone)" | wc -l)
3970 echo -n "${num_masters}"
3971}
3972
3973# Prints regexp for full master machine name. In a cluster with replicated master,
3974# VM names may either be MASTER_NAME or MASTER_NAME with a suffix for a replica.
3975function get-replica-name-regexp() {
3976 echo "^${MASTER_NAME}(-...)?"
3977}
3978
3979# Sets REPLICA_NAME to a unique name for a master replica that will match
3980# expected regexp (see get-replica-name-regexp).
3981#
3982# Assumed vars:
3983# PROJECT
3984# ZONE
3985# MASTER_NAME
3986#
3987# Sets:
3988# REPLICA_NAME
3989function set-replica-name() {
3990 local instances
3991 instances=$(gcloud compute instances list \
3992 --project "${PROJECT}" \
3993 --filter="name ~ '$(get-replica-name-regexp)'" \
3994 --format "value(name)")
3995
3996 suffix=""
3997 while echo "${instances}" | grep "${suffix}" &>/dev/null; do
3998 suffix="$(date | md5sum | head -c3)"
3999 done
4000 REPLICA_NAME="${MASTER_NAME}-${suffix}"
4001}
4002
4003# Gets the instance templates in use by the cluster. It echos the template names
4004# so that the function output can be used.
4005# Assumed vars:
4006# NODE_INSTANCE_PREFIX
4007# WINDOWS_NODE_INSTANCE_PREFIX
4008#
4009# $1: project
4010function get-template() {
4011 local linux_filter="${NODE_INSTANCE_PREFIX}-(extra-)?template(-(${KUBE_RELEASE_VERSION_DASHED_REGEX}|${KUBE_CI_VERSION_DASHED_REGEX}))?"
4012 local windows_filter="${WINDOWS_NODE_INSTANCE_PREFIX}-template(-(${KUBE_RELEASE_VERSION_DASHED_REGEX}|${KUBE_CI_VERSION_DASHED_REGEX}))?"
4013
4014 gcloud compute instance-templates list \
4015 --filter="name ~ '${linux_filter}' OR name ~ '${windows_filter}'" \
4016 --project="${1}" --format='value(name)'
4017}
4018
4019# Checks if there are any present resources related kubernetes cluster.
4020#
4021# Assumed vars:
4022# MASTER_NAME
4023# NODE_INSTANCE_PREFIX
4024# WINDOWS_NODE_INSTANCE_PREFIX
4025# ZONE
4026# REGION
4027# Vars set:
4028# KUBE_RESOURCE_FOUND
4029function check-resources() {
4030 detect-project
4031 detect-node-names
4032
4033 echo "Looking for already existing resources"
4034 KUBE_RESOURCE_FOUND=""
4035
4036 if [[ -n "${INSTANCE_GROUPS[*]:-}" ]]; then
4037 KUBE_RESOURCE_FOUND="Managed instance groups ${INSTANCE_GROUPS[*]}"
4038 return 1
4039 fi
4040 if [[ -n "${WINDOWS_INSTANCE_GROUPS[*]:-}" ]]; then
4041 KUBE_RESOURCE_FOUND="Managed instance groups ${WINDOWS_INSTANCE_GROUPS[*]}"
4042 return 1
4043 fi
4044
4045 if gcloud compute instance-templates describe --project "${PROJECT}" "${NODE_INSTANCE_PREFIX}-template" &>/dev/null; then
4046 KUBE_RESOURCE_FOUND="Instance template ${NODE_INSTANCE_PREFIX}-template"
4047 return 1
4048 fi
4049 if gcloud compute instance-templates describe --project "${PROJECT}" "${WINDOWS_NODE_INSTANCE_PREFIX}-template" &>/dev/null; then
4050 KUBE_RESOURCE_FOUND="Instance template ${WINDOWS_NODE_INSTANCE_PREFIX}-template"
4051 return 1
4052 fi
4053
4054 if gcloud compute instances describe --project "${PROJECT}" "${MASTER_NAME}" --zone "${ZONE}" &>/dev/null; then
4055 KUBE_RESOURCE_FOUND="Kubernetes master ${MASTER_NAME}"
4056 return 1
4057 fi
4058
4059 if gcloud compute disks describe --project "${PROJECT}" "${MASTER_NAME}"-pd --zone "${ZONE}" &>/dev/null; then
4060 KUBE_RESOURCE_FOUND="Persistent disk ${MASTER_NAME}-pd"
4061 return 1
4062 fi
4063
4064 # Find out what minions are running.
4065 local -a minions
4066 kube::util::read-array minions < <(gcloud compute instances list \
4067 --project "${PROJECT}" \
4068 --filter="(name ~ '${NODE_INSTANCE_PREFIX}-.+' OR name ~ '${WINDOWS_NODE_INSTANCE_PREFIX}-.+') AND zone:(${ZONE})" \
4069 --format='value(name)')
4070 if (( "${#minions[@]}" > 0 )); then
4071 KUBE_RESOURCE_FOUND="${#minions[@]} matching ${NODE_INSTANCE_PREFIX}-.+ or ${WINDOWS_NODE_INSTANCE_PREFIX}-.+"
4072 return 1
4073 fi
4074
4075 if gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${MASTER_NAME}-https" &>/dev/null; then
4076 KUBE_RESOURCE_FOUND="Firewall rules for ${MASTER_NAME}-https"
4077 return 1
4078 fi
4079
4080 if gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NODE_TAG}-all" &>/dev/null; then
4081 KUBE_RESOURCE_FOUND="Firewall rules for ${MASTER_NAME}-all"
4082 return 1
4083 fi
4084
4085 local -a routes
4086 kube::util::read-array routes < <(gcloud compute routes list --project "${NETWORK_PROJECT}" \
4087 --filter="name ~ '${INSTANCE_PREFIX}-minion-.{4}'" --format='value(name)')
4088 if (( "${#routes[@]}" > 0 )); then
4089 KUBE_RESOURCE_FOUND="${#routes[@]} routes matching ${INSTANCE_PREFIX}-minion-.{4}"
4090 return 1
4091 fi
4092
4093 if gcloud compute addresses describe --project "${PROJECT}" "${MASTER_NAME}-ip" --region "${REGION}" &>/dev/null; then
4094 KUBE_RESOURCE_FOUND="Master's reserved IP"
4095 return 1
4096 fi
4097
4098 # No resources found.
4099 return 0
4100}
4101
4102# -----------------------------------------------------------------------------
4103# Cluster specific test helpers
4104
4105# Execute prior to running tests to build a release if required for env.
4106#
4107# Assumed Vars:
4108# KUBE_ROOT
4109function test-build-release() {
4110 # Make a release
4111 "${KUBE_ROOT}/build/release.sh"
4112}
4113
4114# Execute prior to running tests to initialize required structure.
4115#
4116# Assumed vars:
4117# Variables from config.sh
4118function test-setup() {
4119 # Detect the project into $PROJECT if it isn't set
4120 detect-project
4121
4122 if [[ ${MULTIZONE:-} == "true" && -n ${E2E_ZONES:-} ]]; then
4123 for KUBE_GCE_ZONE in ${E2E_ZONES}; do
4124 KUBE_GCE_ZONE="${KUBE_GCE_ZONE}" KUBE_USE_EXISTING_MASTER="${KUBE_USE_EXISTING_MASTER:-}" "${KUBE_ROOT}/cluster/kube-up.sh"
4125 KUBE_USE_EXISTING_MASTER="true" # For subsequent zones we use the existing master
4126 done
4127 else
4128 "${KUBE_ROOT}/cluster/kube-up.sh"
4129 fi
4130
4131 # Open up port 80 & 8080 so common containers on minions can be reached
4132 # TODO(roberthbailey): Remove this once we are no longer relying on hostPorts.
4133 local start
4134 start=$(date +%s)
4135 gcloud compute firewall-rules create \
4136 --project "${NETWORK_PROJECT}" \
4137 --target-tags "${NODE_TAG}" \
4138 --allow tcp:80,tcp:8080 \
4139 --network "${NETWORK}" \
4140 "${NODE_TAG}-http-alt" 2> /dev/null || true
4141 # As there is no simple way to wait longer for this operation we need to manually
4142 # wait some additional time (20 minutes altogether).
4143 while ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NODE_TAG}-http-alt" 2> /dev/null; do
4144 if [[ $((start + 1200)) -lt $(date +%s) ]]; then
4145 echo -e "${color_red:-}Failed to create firewall ${NODE_TAG}-http-alt in ${NETWORK_PROJECT}" >&2
4146 exit 1
4147 fi
4148 sleep 5
4149 done
4150
4151 # Open up the NodePort range
4152 # TODO(justinsb): Move to main setup, if we decide whether we want to do this by default.
4153 start=$(date +%s)
4154 gcloud compute firewall-rules create \
4155 --project "${NETWORK_PROJECT}" \
4156 --target-tags "${NODE_TAG}" \
4157 --allow tcp:30000-32767,udp:30000-32767 \
4158 --network "${NETWORK}" \
4159 "${NODE_TAG}-nodeports" 2> /dev/null || true
4160 # As there is no simple way to wait longer for this operation we need to manually
4161 # wait some additional time (20 minutes altogether).
4162 while ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NODE_TAG}-nodeports" 2> /dev/null; do
4163 if [[ $((start + 1200)) -lt $(date +%s) ]]; then
4164 echo -e "${color_red}Failed to create firewall ${NODE_TAG}-nodeports in ${PROJECT}" >&2
4165 exit 1
4166 fi
4167 sleep 5
4168 done
4169}
4170
4171# Execute after running tests to perform any required clean-up.
4172function test-teardown() {
4173 detect-project
4174 echo "Shutting down test cluster in background."
4175 delete-firewall-rules \
4176 "${NODE_TAG}-http-alt" \
4177 "${NODE_TAG}-nodeports"
4178 if [[ ${MULTIZONE:-} == "true" && -n ${E2E_ZONES:-} ]]; then
4179 local zones
4180 read -r -a zones <<< "${E2E_ZONES}"
4181 # tear them down in reverse order, finally tearing down the master too.
4182 for ((zone_num=${#zones[@]}-1; zone_num>0; zone_num--)); do
4183 KUBE_GCE_ZONE="${zones[zone_num]}" KUBE_USE_EXISTING_MASTER="true" "${KUBE_ROOT}/cluster/kube-down.sh"
4184 done
4185 KUBE_GCE_ZONE="${zones[0]}" KUBE_USE_EXISTING_MASTER="false" "${KUBE_ROOT}/cluster/kube-down.sh"
4186 else
4187 "${KUBE_ROOT}/cluster/kube-down.sh"
4188 fi
4189}
4190
4191# SSH to a node by name ($1) and run a command ($2).
4192function ssh-to-node() {
4193 local node="$1"
4194 local cmd="$2"
4195 # Loop until we can successfully ssh into the box
4196 for (( i=0; i<5; i++)); do
4197 if gcloud compute ssh --ssh-flag='-o LogLevel=quiet' --ssh-flag='-o ConnectTimeout=30' --project "${PROJECT}" --zone="${ZONE}" "${node}" --command 'echo test > /dev/null'; then
4198 break
4199 fi
4200 sleep 5
4201 done
4202 # Then actually try the command.
4203 gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --ssh-flag="-o ConnectTimeout=30" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "${cmd}"
4204}
4205
4206# Perform preparations required to run e2e tests
4207function prepare-e2e() {
4208 detect-project
4209}
4210
4211# Delete the image given by $1.
4212function delete-image() {
4213 gcloud container images delete --quiet "$1"
4214}
View as plain text