...

Text file src/k8s.io/kubernetes/cluster/gce/util.sh

Documentation: k8s.io/kubernetes/cluster/gce

     1#!/usr/bin/env bash
     2
     3# Copyright 2017 The Kubernetes Authors.
     4#
     5# Licensed under the Apache License, Version 2.0 (the "License");
     6# you may not use this file except in compliance with the License.
     7# You may obtain a copy of the License at
     8#
     9#     http://www.apache.org/licenses/LICENSE-2.0
    10#
    11# Unless required by applicable law or agreed to in writing, software
    12# distributed under the License is distributed on an "AS IS" BASIS,
    13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14# See the License for the specific language governing permissions and
    15# limitations under the License.
    16
    17# A library of helper functions and constant for the local config.
    18
    19# Use the config file specified in $KUBE_CONFIG_FILE, or default to
    20# config-default.sh.
    21readonly GCE_MAX_LOCAL_SSD=8
    22
    23KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
    24source "${KUBE_ROOT}/cluster/gce/${KUBE_CONFIG_FILE-"config-default.sh"}"
    25source "${KUBE_ROOT}/cluster/common.sh"
    26source "${KUBE_ROOT}/hack/lib/util.sh"
    27
    28if [[ "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "ubuntu" || "${NODE_OS_DISTRIBUTION}" == "custom" ]]; then
    29  source "${KUBE_ROOT}/cluster/gce/${NODE_OS_DISTRIBUTION}/node-helper.sh"
    30else
    31  echo "Cannot operate on cluster using node os distro: ${NODE_OS_DISTRIBUTION}" >&2
    32  exit 1
    33fi
    34
    35source "${KUBE_ROOT}/cluster/gce/windows/node-helper.sh"
    36
    37if [[ "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then
    38  source "${KUBE_ROOT}/cluster/gce/${MASTER_OS_DISTRIBUTION}/master-helper.sh"
    39else
    40  echo "Cannot operate on cluster using master os distro: ${MASTER_OS_DISTRIBUTION}" >&2
    41  exit 1
    42fi
    43
    44if [[ ${NODE_LOCAL_SSDS:-} -ge 1 ]] && [[ -n ${NODE_LOCAL_SSDS_EXT:-} ]] ; then
    45  echo -e "${color_red:-}Local SSD: Only one of NODE_LOCAL_SSDS and NODE_LOCAL_SSDS_EXT can be specified at once${color_norm:-}" >&2
    46  exit 2
    47fi
    48
    49if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then
    50    DEFAULT_GCI_PROJECT=google-containers
    51    if [[ "${GCI_VERSION}" == "cos"* ]] || [[ "${MASTER_IMAGE_FAMILY}" == "cos"* ]]; then
    52        DEFAULT_GCI_PROJECT=cos-cloud
    53    fi
    54    export MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-${DEFAULT_GCI_PROJECT}}
    55
    56    # If the master image is not set, we use the latest image based on image
    57    # family.
    58    kube_master_image="${KUBE_GCE_MASTER_IMAGE:-${GCI_VERSION}}"
    59    if [[ -z "${kube_master_image}" ]]; then
    60      kube_master_image=$(gcloud compute images list --project="${MASTER_IMAGE_PROJECT}" --no-standard-images --filter="family:${MASTER_IMAGE_FAMILY}" --format 'value(name)')
    61    fi
    62
    63    echo "Using image: ${kube_master_image} from project: ${MASTER_IMAGE_PROJECT} as master image" >&2
    64    export MASTER_IMAGE="${kube_master_image}"
    65fi
    66
    67# Sets node image based on the specified os distro. Currently this function only
    68# supports gci and debian.
    69#
    70# Requires:
    71#   NODE_OS_DISTRIBUTION
    72# Sets:
    73#   DEFAULT_GCI_PROJECT
    74#   NODE_IMAGE
    75#   NODE_IMAGE_PROJECT
    76function set-linux-node-image() {
    77  if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then
    78    DEFAULT_GCI_PROJECT=google-containers
    79    if [[ "${GCI_VERSION}" == "cos"* ]] || [[ "${NODE_IMAGE_FAMILY}" == "cos"* ]]; then
    80      DEFAULT_GCI_PROJECT=cos-cloud
    81    fi
    82
    83    # If the node image is not set, we use the latest image based on image
    84    # family.
    85    # Otherwise, we respect whatever is set by the user.
    86    NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-${DEFAULT_GCI_PROJECT}}
    87    local kube_node_image
    88
    89    kube_node_image="${KUBE_GCE_NODE_IMAGE:-${GCI_VERSION}}"
    90    if [[ -z "${kube_node_image}" ]]; then
    91      kube_node_image=$(gcloud compute images list --project="${NODE_IMAGE_PROJECT}" --no-standard-images --filter="family:${NODE_IMAGE_FAMILY}" --format 'value(name)')
    92    fi
    93
    94    echo "Using image: ${kube_node_image} from project: ${NODE_IMAGE_PROJECT} as node image" >&2
    95    export NODE_IMAGE="${kube_node_image}"
    96  fi
    97}
    98
    99# Requires:
   100#   WINDOWS_NODE_OS_DISTRIBUTION
   101# Sets:
   102#   WINDOWS_NODE_IMAGE_PROJECT
   103#   WINDOWS_NODE_IMAGE
   104function set-windows-node-image() {
   105  WINDOWS_NODE_IMAGE_PROJECT="windows-cloud"
   106  if [[ "${WINDOWS_NODE_OS_DISTRIBUTION}" == "win2019" ]]; then
   107    WINDOWS_NODE_IMAGE="windows-server-2019-dc-core-v20210914"
   108  elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION}" == "win1909" ]]; then
   109    WINDOWS_NODE_IMAGE="windows-server-1909-dc-core-v20210413"
   110  elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION}" == "win2004" ]]; then
   111    WINDOWS_NODE_IMAGE="windows-server-2004-dc-core-v20210914"
   112  elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION,,}" == "win20h2" ]]; then
   113    WINDOWS_NODE_IMAGE="windows-server-20h2-dc-core-v20210914"
   114  elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION,,}" == "win2022" ]]; then
   115    WINDOWS_NODE_IMAGE="windows-server-2022-dc-core-v20220513"
   116  else
   117    echo "Unknown WINDOWS_NODE_OS_DISTRIBUTION ${WINDOWS_NODE_OS_DISTRIBUTION}" >&2
   118    exit 1
   119  fi
   120}
   121
   122set-linux-node-image
   123set-windows-node-image
   124
   125# Verify cluster autoscaler configuration.
   126if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then
   127  if [[ -z $AUTOSCALER_MIN_NODES ]]; then
   128    echo "AUTOSCALER_MIN_NODES not set."
   129    exit 1
   130  fi
   131  if [[ -z $AUTOSCALER_MAX_NODES ]]; then
   132    echo "AUTOSCALER_MAX_NODES not set."
   133    exit 1
   134  fi
   135fi
   136
   137# These prefixes must not be prefixes of each other, so that they can be used to
   138# detect mutually exclusive sets of nodes.
   139NODE_INSTANCE_PREFIX=${NODE_INSTANCE_PREFIX:-"${INSTANCE_PREFIX}-minion"}
   140WINDOWS_NODE_INSTANCE_PREFIX=${WINDOWS_NODE_INSTANCE_PREFIX:-"${INSTANCE_PREFIX}-windows-node"}
   141
   142# NODE_TAG (expected to be) defined by caller
   143# shellcheck disable=SC2153
   144NODE_TAGS="${NODE_TAG}"
   145
   146ALLOCATE_NODE_CIDRS=true
   147PREEXISTING_NETWORK=false
   148PREEXISTING_NETWORK_MODE=""
   149
   150KUBE_PROMPT_FOR_UPDATE=${KUBE_PROMPT_FOR_UPDATE:-"n"}
   151# How long (in seconds) to wait for cluster initialization.
   152KUBE_CLUSTER_INITIALIZATION_TIMEOUT=${KUBE_CLUSTER_INITIALIZATION_TIMEOUT:-300}
   153
   154function join_csv() {
   155  local IFS=','; echo "$*";
   156}
   157
   158# This function returns the first string before the comma
   159function split_csv() {
   160  echo "$*" | cut -d',' -f1
   161}
   162
   163# Verify prereqs
   164function verify-prereqs() {
   165  local cmd
   166
   167  # we use openssl to generate certs
   168  kube::util::test_openssl_installed
   169
   170  # ensure a version supported by easyrsa is installed
   171  if [ "$(openssl version | cut -d\  -f1)" == "LibreSSL" ]; then
   172    echo "LibreSSL is not supported. Please ensure openssl points to an OpenSSL binary"
   173    if [ "$(uname -s)" == "Darwin" ]; then
   174      # We want this print just the way it is
   175      # shellcheck disable=SC2016
   176      echo 'On macOS we recommend using homebrew and adding "$(brew --prefix openssl)/bin" to your PATH'
   177    fi
   178    exit 1
   179  fi
   180
   181  # we use gcloud to create the cluster, gsutil to stage binaries and data
   182  for cmd in gcloud gsutil; do
   183    if ! which "${cmd}" >/dev/null; then
   184      echo "Can't find ${cmd} in PATH, please fix and retry. The Google Cloud " >&2
   185      echo "SDK can be downloaded from https://cloud.google.com/sdk/." >&2
   186      exit 1
   187    fi
   188  done
   189  update-or-verify-gcloud
   190}
   191
   192# Use the gcloud defaults to find the project.  If it is already set in the
   193# environment then go with that.
   194#
   195# Vars set:
   196#   PROJECT
   197#   NETWORK_PROJECT
   198#   PROJECT_REPORTED
   199function detect-project() {
   200  if [[ -z "${PROJECT-}" ]]; then
   201    PROJECT=$(gcloud config list project --format 'value(core.project)')
   202  fi
   203
   204  NETWORK_PROJECT=${NETWORK_PROJECT:-${PROJECT}}
   205
   206  if [[ -z "${PROJECT-}" ]]; then
   207    echo "Could not detect Google Cloud Platform project.  Set the default project using " >&2
   208    echo "'gcloud config set project <PROJECT>'" >&2
   209    exit 1
   210  fi
   211  if [[ -z "${PROJECT_REPORTED-}" ]]; then
   212    echo "Project: ${PROJECT}" >&2
   213    echo "Network Project: ${NETWORK_PROJECT}" >&2
   214    echo "Zone: ${ZONE}" >&2
   215    PROJECT_REPORTED=true
   216  fi
   217}
   218
   219# Use gsutil to get the md5 hash for a particular tar
   220function gsutil_get_tar_md5() {
   221  # location_tar could be local or in the cloud
   222  # local tar_location example ./_output/release-tars/kubernetes-server-linux-amd64.tar.gz
   223  # cloud tar_location example gs://kubernetes-staging-PROJECT/kubernetes-devel/kubernetes-server-linux-amd64.tar.gz
   224  local -r tar_location=$1
   225  #parse the output and return the md5 hash
   226  #the sed command at the end removes whitespace
   227  local -r tar_md5=$(gsutil hash -h -m "${tar_location}" 2>/dev/null | grep "Hash (md5):" | awk -F ':' '{print $2}' | sed 's/^[[:space:]]*//g')
   228  echo "${tar_md5}"
   229}
   230
   231# Copy a release tar and its accompanying hash.
   232function copy-to-staging() {
   233  local -r staging_path=$1
   234  local -r gs_url=$2
   235  local -r tar=$3
   236  local -r hash=$4
   237  local -r basename_tar=$(basename "${tar}")
   238
   239  #check whether this tar alread exists and has the same hash
   240  #if it matches, then don't bother uploading it again
   241
   242  #remote_tar_md5 checks the remote location for the existing tarball and its md5
   243  #staging_path example gs://kubernetes-staging-PROJECT/kubernetes-devel
   244  #basename_tar example kubernetes-server-linux-amd64.tar.gz
   245  local -r remote_tar_md5=$(gsutil_get_tar_md5 "${staging_path}/${basename_tar}")
   246  if [[ -n ${remote_tar_md5} ]]; then
   247    #local_tar_md5 checks the remote location for the existing tarball and its md5 hash
   248    #tar example ./_output/release-tars/kubernetes-server-linux-amd64.tar.gz
   249    local -r local_tar_md5=$(gsutil_get_tar_md5 "${tar}")
   250    if [[ "${remote_tar_md5}" == "${local_tar_md5}" ]]; then
   251      echo "+++ ${basename_tar} uploaded earlier, cloud and local file md5 match (md5 = ${local_tar_md5})"
   252      return 0
   253    fi
   254  fi
   255
   256  echo "${hash}" > "${tar}.sha512"
   257  gsutil -m -q -h "Cache-Control:private, max-age=0" cp "${tar}" "${tar}.sha512" "${staging_path}"
   258  gsutil -m acl ch -g all:R "${gs_url}" "${gs_url}.sha512" >/dev/null 2>&1 || true
   259  echo "+++ ${basename_tar} uploaded (sha512 = ${hash})"
   260}
   261
   262
   263# Given the cluster zone, return the list of regional GCS release
   264# bucket suffixes for the release in preference order. GCS doesn't
   265# give us an API for this, so we hardcode it.
   266#
   267# Assumed vars:
   268#   RELEASE_REGION_FALLBACK
   269#   REGIONAL_KUBE_ADDONS
   270#   ZONE
   271# Vars set:
   272#   PREFERRED_REGION
   273function set-preferred-region() {
   274  case ${ZONE} in
   275    asia-*)
   276      PREFERRED_REGION=("asia-northeast1" "us-central1" "europe-west6")
   277      ;;
   278    europe-*)
   279      PREFERRED_REGION=("europe-west6" "us-central1" "asia-northeast1")
   280      ;;
   281    *)
   282      PREFERRED_REGION=("us-central1" "europe-west6" "asia-northeast1")
   283      ;;
   284  esac
   285
   286  if [[ "${RELEASE_REGION_FALLBACK}" != "true" ]]; then
   287    PREFERRED_REGION=( "${PREFERRED_REGION[0]}" )
   288  fi
   289}
   290
   291# Take the local tar files and upload them to Google Storage.  They will then be
   292# downloaded by the master as part of the start up script for the master.
   293#
   294# Assumed vars:
   295#   PROJECT
   296#   SERVER_BINARY_TAR
   297#   KUBE_MANIFESTS_TAR
   298#   ZONE
   299# Vars set:
   300#   SERVER_BINARY_TAR_URL
   301#   SERVER_BINARY_TAR_HASH
   302#   NODE_BINARY_TAR_URL
   303#   NODE_BINARY_TAR_HASH
   304#   KUBE_MANIFESTS_TAR_URL
   305#   KUBE_MANIFESTS_TAR_HASH
   306function upload-tars() {
   307  SERVER_BINARY_TAR_URL=
   308  SERVER_BINARY_TAR_HASH=
   309  NODE_BINARY_TAR_URL=
   310  NODE_BINARY_TAR_HASH=
   311  KUBE_MANIFESTS_TAR_URL=
   312  KUBE_MANIFESTS_TAR_HASH=
   313
   314  local project_hash
   315  if which md5 > /dev/null 2>&1; then
   316    project_hash=$(md5 -q -s "$PROJECT")
   317  else
   318    project_hash=$(echo -n "$PROJECT" | md5sum)
   319    project_hash=${project_hash%%[[:blank:]]*}
   320  fi
   321
   322  # This requires 1 million projects before the probability of collision is 50%
   323  # that's probably good enough for now :P
   324  project_hash=${project_hash:0:10}
   325
   326  set-preferred-region
   327
   328  if [[ "${ENABLE_DOCKER_REGISTRY_CACHE:-}" == "true" ]]; then
   329    DOCKER_REGISTRY_MIRROR_URL="https://mirror.gcr.io"
   330  fi
   331
   332  SERVER_BINARY_TAR_HASH=$(sha512sum-file "${SERVER_BINARY_TAR}")
   333
   334  if [[ -n "${NODE_BINARY_TAR:-}" ]]; then
   335    NODE_BINARY_TAR_HASH=$(sha512sum-file "${NODE_BINARY_TAR}")
   336  fi
   337  if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then
   338    KUBE_MANIFESTS_TAR_HASH=$(sha512sum-file "${KUBE_MANIFESTS_TAR}")
   339  fi
   340
   341  local server_binary_tar_urls=()
   342  local node_binary_tar_urls=()
   343
   344  for region in "${PREFERRED_REGION[@]}"; do
   345    suffix="-${region}"
   346    local staging_bucket="gs://kubernetes-staging-${project_hash}${suffix}"
   347
   348    # Ensure the buckets are created
   349    if ! gsutil ls "${staging_bucket}" >/dev/null; then
   350      echo "Creating ${staging_bucket}"
   351      gsutil mb -l "${region}" -p "${PROJECT}" "${staging_bucket}"
   352    fi
   353
   354    local staging_path="${staging_bucket}/${INSTANCE_PREFIX}-devel"
   355
   356    echo "+++ Staging tars to Google Storage: ${staging_path}"
   357    local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}"
   358    copy-to-staging "${staging_path}" "${server_binary_gs_url}" "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_HASH}"
   359
   360    if [[ -n "${NODE_BINARY_TAR:-}" ]]; then
   361      local node_binary_gs_url="${staging_path}/${NODE_BINARY_TAR##*/}"
   362      copy-to-staging "${staging_path}" "${node_binary_gs_url}" "${NODE_BINARY_TAR}" "${NODE_BINARY_TAR_HASH}"
   363    fi
   364
   365    # Convert from gs:// URL to an https:// URL
   366    server_binary_tar_urls+=("${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}")
   367    if [[ -n "${NODE_BINARY_TAR:-}" ]]; then
   368      node_binary_tar_urls+=("${node_binary_gs_url/gs:\/\//https://storage.googleapis.com/}")
   369    fi
   370    if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then
   371      local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}"
   372      copy-to-staging "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}"
   373      # Convert from gs:// URL to an https:// URL
   374      kube_manifests_tar_urls+=("${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}")
   375    fi
   376  done
   377
   378  SERVER_BINARY_TAR_URL=$(join_csv "${server_binary_tar_urls[@]}")
   379  if [[ -n "${NODE_BINARY_TAR:-}" ]]; then
   380    NODE_BINARY_TAR_URL=$(join_csv "${node_binary_tar_urls[@]}")
   381  fi
   382  if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then
   383    KUBE_MANIFESTS_TAR_URL=$(join_csv "${kube_manifests_tar_urls[@]}")
   384  fi
   385}
   386
   387# Detect Linux and Windows nodes created in the instance group.
   388#
   389# Assumed vars:
   390#   NODE_INSTANCE_PREFIX
   391#   WINDOWS_NODE_INSTANCE_PREFIX
   392# Vars set:
   393#   NODE_NAMES
   394#   INSTANCE_GROUPS
   395#   WINDOWS_NODE_NAMES
   396#   WINDOWS_INSTANCE_GROUPS
   397function detect-node-names() {
   398  detect-project
   399  INSTANCE_GROUPS=()
   400  kube::util::read-array INSTANCE_GROUPS < <(gcloud compute instance-groups managed list \
   401    --project "${PROJECT}" \
   402    --filter "name ~ '${NODE_INSTANCE_PREFIX}-.+' AND zone:(${ZONE})" \
   403    --format='value(name)' || true)
   404  WINDOWS_INSTANCE_GROUPS=()
   405  kube::util::read-array WINDOWS_INSTANCE_GROUPS < <(gcloud compute instance-groups managed list \
   406    --project "${PROJECT}" \
   407    --filter "name ~ '${WINDOWS_NODE_INSTANCE_PREFIX}-.+' AND zone:(${ZONE})" \
   408    --format='value(name)' || true)
   409
   410  NODE_NAMES=()
   411  if [[ -n "${INSTANCE_GROUPS[*]:-}" ]]; then
   412    for group in "${INSTANCE_GROUPS[@]}"; do
   413      kube::util::read-array NODE_NAMES < <(gcloud compute instance-groups managed list-instances \
   414        "${group}" --zone "${ZONE}" --project "${PROJECT}" \
   415        --format='value(name)')
   416    done
   417  fi
   418  # Add heapster node name to the list too (if it exists).
   419  if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then
   420    NODE_NAMES+=("${NODE_INSTANCE_PREFIX}-heapster")
   421  fi
   422  export NODE_NAMES
   423  WINDOWS_NODE_NAMES=()
   424  if [[ -n "${WINDOWS_INSTANCE_GROUPS[*]:-}" ]]; then
   425    for group in "${WINDOWS_INSTANCE_GROUPS[@]}"; do
   426      kube::util::read-array WINDOWS_NODE_NAMES < <(gcloud compute instance-groups managed \
   427        list-instances "${group}" --zone "${ZONE}" --project "${PROJECT}" \
   428        --format='value(name)')
   429    done
   430  fi
   431  export WINDOWS_NODE_NAMES
   432
   433  echo "INSTANCE_GROUPS=${INSTANCE_GROUPS[*]:-}" >&2
   434  echo "NODE_NAMES=${NODE_NAMES[*]:-}" >&2
   435}
   436
   437# Detect the information about the minions
   438#
   439# Assumed vars:
   440#   ZONE
   441# Vars set:
   442#   NODE_NAMES
   443#   KUBE_NODE_IP_ADDRESSES (array)
   444function detect-nodes() {
   445  detect-project
   446  detect-node-names
   447  KUBE_NODE_IP_ADDRESSES=()
   448  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
   449    local node_ip
   450    node_ip=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \
   451      "${NODE_NAMES[$i]}" --format='value(networkInterfaces[0].accessConfigs[0].natIP)')
   452    if [[ -z "${node_ip-}" ]] ; then
   453      echo "Did not find ${NODE_NAMES[$i]}" >&2
   454    else
   455      echo "Found ${NODE_NAMES[$i]} at ${node_ip}"
   456      KUBE_NODE_IP_ADDRESSES+=("${node_ip}")
   457    fi
   458  done
   459  if [[ -z "${KUBE_NODE_IP_ADDRESSES-}" ]]; then
   460    echo "Could not detect Kubernetes minion nodes.  Make sure you've launched a cluster with 'kube-up.sh'" >&2
   461    exit 1
   462  fi
   463}
   464
   465# Detect the IP for the master
   466#
   467# Assumed vars:
   468#   MASTER_NAME
   469#   ZONE
   470#   REGION
   471# Vars set:
   472#   KUBE_MASTER
   473#   KUBE_MASTER_IP
   474function detect-master() {
   475  detect-project
   476  KUBE_MASTER=${MASTER_NAME}
   477  echo "Trying to find master named '${MASTER_NAME}'" >&2
   478  if [[ -z "${KUBE_MASTER_IP-}" ]]; then
   479    local master_address_name="${MASTER_NAME}-ip"
   480    echo "Looking for address '${master_address_name}'" >&2
   481    if ! KUBE_MASTER_IP=$(gcloud compute addresses describe "${master_address_name}" \
   482      --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') || \
   483      [[ -z "${KUBE_MASTER_IP-}" ]]; then
   484      echo "Could not detect Kubernetes master node.  Make sure you've launched a cluster with 'kube-up.sh'" >&2
   485      exit 1
   486    fi
   487  fi
   488  if [[ -z "${KUBE_MASTER_INTERNAL_IP-}" ]] && [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
   489      local master_address_name="${MASTER_NAME}-internal-ip"
   490      echo "Looking for address '${master_address_name}'" >&2
   491      if ! KUBE_MASTER_INTERNAL_IP=$(gcloud compute addresses describe "${master_address_name}" \
   492        --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') || \
   493        [[ -z "${KUBE_MASTER_INTERNAL_IP-}" ]]; then
   494        echo "Could not detect Kubernetes master node.  Make sure you've launched a cluster with 'kube-up.sh'" >&2
   495        exit 1
   496      fi
   497  fi
   498  echo "Using master: $KUBE_MASTER (external IP: $KUBE_MASTER_IP; internal IP: ${KUBE_MASTER_INTERNAL_IP:-(not set)})" >&2
   499}
   500
   501function load-or-gen-kube-bearertoken() {
   502  if [[ -n "${KUBE_CONTEXT:-}" ]]; then
   503    get-kubeconfig-bearertoken
   504  fi
   505  if [[ -z "${KUBE_BEARER_TOKEN:-}" ]]; then
   506    gen-kube-bearertoken
   507  fi
   508}
   509
   510# Figure out which binary use on the server and assure it is available.
   511# If KUBE_VERSION is specified use binaries specified by it, otherwise
   512# use local dev binaries.
   513#
   514# Assumed vars:
   515#   KUBE_VERSION
   516#   KUBE_RELEASE_VERSION_REGEX
   517#   KUBE_CI_VERSION_REGEX
   518# Vars set:
   519#   KUBE_TAR_HASH
   520#   SERVER_BINARY_TAR_URL
   521#   SERVER_BINARY_TAR_HASH
   522function tars_from_version() {
   523  local sha512sum=""
   524  if which sha512sum >/dev/null 2>&1; then
   525    sha512sum="sha512sum"
   526  else
   527    sha512sum="shasum -a512"
   528  fi
   529
   530  if [[ -z "${KUBE_VERSION-}" ]]; then
   531    find-release-tars
   532    upload-tars
   533  elif [[ ${KUBE_VERSION} =~ ${KUBE_RELEASE_VERSION_REGEX} ]]; then
   534    SERVER_BINARY_TAR_URL="https://dl.k8s.io/release/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz"
   535    # TODO: Clean this up.
   536    KUBE_MANIFESTS_TAR_URL="${SERVER_BINARY_TAR_URL/server-linux-amd64/manifests}"
   537    KUBE_MANIFESTS_TAR_HASH=$(curl -L "${KUBE_MANIFESTS_TAR_URL}" --silent --show-error | ${sha512sum})
   538    KUBE_MANIFESTS_TAR_HASH=${KUBE_MANIFESTS_TAR_HASH%%[[:blank:]]*}
   539  elif [[ ${KUBE_VERSION} =~ ${KUBE_CI_VERSION_REGEX} ]]; then
   540    SERVER_BINARY_TAR_URL="https://storage.googleapis.com/k8s-release-dev/ci/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz"
   541    # TODO: Clean this up.
   542    KUBE_MANIFESTS_TAR_URL="${SERVER_BINARY_TAR_URL/server-linux-amd64/manifests}"
   543    KUBE_MANIFESTS_TAR_HASH=$(curl "${KUBE_MANIFESTS_TAR_URL}" --silent --show-error | ${sha512sum})
   544    KUBE_MANIFESTS_TAR_HASH=${KUBE_MANIFESTS_TAR_HASH%%[[:blank:]]*}
   545  else
   546    echo "Version doesn't match regexp" >&2
   547    exit 1
   548  fi
   549  if ! SERVER_BINARY_TAR_HASH=$(curl -Ss --fail "${SERVER_BINARY_TAR_URL}.sha512"); then
   550    echo "Failure trying to curl release .sha512"
   551  fi
   552
   553  if ! curl -Ss --head "${SERVER_BINARY_TAR_URL}" >&/dev/null; then
   554    echo "Can't find release at ${SERVER_BINARY_TAR_URL}" >&2
   555    exit 1
   556  fi
   557}
   558
   559# Reads kube-env metadata from master
   560#
   561# Assumed vars:
   562#   KUBE_MASTER
   563#   PROJECT
   564#   ZONE
   565function get-master-env() {
   566  # TODO(zmerlynn): Make this more reliable with retries.
   567  gcloud compute --project "${PROJECT}" ssh --zone "${ZONE}" "${KUBE_MASTER}" --command \
   568    "curl --fail --silent -H 'Metadata-Flavor: Google' \
   569      'http://metadata/computeMetadata/v1/instance/attributes/kube-env'" 2>/dev/null
   570  gcloud compute --project "${PROJECT}" ssh --zone "${ZONE}" "${KUBE_MASTER}" --command \
   571    "curl --fail --silent -H 'Metadata-Flavor: Google' \
   572      'http://metadata/computeMetadata/v1/instance/attributes/kube-master-certs'" 2>/dev/null
   573}
   574
   575# Quote something appropriate for a yaml string.
   576#
   577# TODO(zmerlynn): Note that this function doesn't so much "quote" as
   578# "strip out quotes", and we really should be using a YAML library for
   579# this, but PyYAML isn't shipped by default, and *rant rant rant ... SIGH*
   580function yaml-quote {
   581  echo "${@:-}" | sed -e "s/'/''/g;s/^/'/i;s/$/'/i"
   582}
   583
   584# Writes the cluster location into a temporary file.
   585# Assumed vars
   586#   ZONE
   587function write-cluster-location {
   588  cat >"${KUBE_TEMP}/cluster-location.txt" << EOF
   589${ZONE}
   590EOF
   591}
   592
   593# Writes the cluster name into a temporary file.
   594# Assumed vars
   595#   CLUSTER_NAME
   596function write-cluster-name {
   597  cat >"${KUBE_TEMP}/cluster-name.txt" << EOF
   598${CLUSTER_NAME}
   599EOF
   600}
   601
   602function write-master-env {
   603  # If the user requested that the master be part of the cluster, set the
   604  # environment variable to program the master kubelet to register itself.
   605  if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" && -z "${KUBELET_APISERVER:-}" ]]; then
   606    KUBELET_APISERVER="${MASTER_NAME}"
   607  fi
   608  if [[ -z "${KUBERNETES_MASTER_NAME:-}" ]]; then
   609    KUBERNETES_MASTER_NAME="${MASTER_NAME}"
   610  fi
   611
   612  construct-linux-kubelet-flags "master"
   613  build-linux-kube-env true "${KUBE_TEMP}/master-kube-env.yaml"
   614  build-kubelet-config true "linux" "${KUBE_TEMP}/master-kubelet-config.yaml"
   615  build-kube-master-certs "${KUBE_TEMP}/kube-master-certs.yaml"
   616}
   617
   618function write-linux-node-env {
   619  if [[ -z "${KUBERNETES_MASTER_NAME:-}" ]]; then
   620    KUBERNETES_MASTER_NAME="${MASTER_NAME}"
   621  fi
   622
   623  construct-linux-kubelet-flags "heapster"
   624  build-linux-kube-env false "${KUBE_TEMP}/heapster-kube-env.yaml"
   625  construct-linux-kubelet-flags "node"
   626  build-linux-kube-env false "${KUBE_TEMP}/node-kube-env.yaml"
   627  build-kubelet-config false "linux" "${KUBE_TEMP}/node-kubelet-config.yaml"
   628}
   629
   630function write-windows-node-env {
   631  construct-windows-kubelet-flags
   632  construct-windows-kubeproxy-flags
   633  build-windows-kube-env "${KUBE_TEMP}/windows-node-kube-env.yaml"
   634  build-kubelet-config false "windows" "${KUBE_TEMP}/windows-node-kubelet-config.yaml"
   635}
   636
   637function build-linux-node-labels {
   638  local node_type=$1
   639  local node_labels=""
   640  if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${node_type}" != "master" ]]; then
   641    # Add kube-proxy daemonset label to node to avoid situation during cluster
   642    # upgrade/downgrade when there are two instances of kube-proxy running on a node.
   643    node_labels="node.kubernetes.io/kube-proxy-ds-ready=true"
   644  fi
   645  if [[ -n "${NODE_LABELS:-}" ]]; then
   646    node_labels="${node_labels:+${node_labels},}${NODE_LABELS}"
   647  fi
   648  if [[ -n "${NON_MASTER_NODE_LABELS:-}" && "${node_type}" != "master" ]]; then
   649    node_labels="${node_labels:+${node_labels},}${NON_MASTER_NODE_LABELS}"
   650  fi
   651  if [[ -n "${MASTER_NODE_LABELS:-}" && "${node_type}" == "master" ]]; then
   652    node_labels="${node_labels:+${node_labels},}${MASTER_NODE_LABELS}"
   653  fi
   654  echo "$node_labels"
   655}
   656
   657function build-windows-node-labels {
   658  local node_labels=""
   659  if [[ -n "${WINDOWS_NODE_LABELS:-}" ]]; then
   660    node_labels="${node_labels:+${node_labels},}${WINDOWS_NODE_LABELS}"
   661  fi
   662  if [[ -n "${WINDOWS_NON_MASTER_NODE_LABELS:-}" ]]; then
   663    node_labels="${node_labels:+${node_labels},}${WINDOWS_NON_MASTER_NODE_LABELS}"
   664  fi
   665  echo "$node_labels"
   666}
   667
   668# yaml-map-string-stringarray converts the encoded structure to yaml format, and echoes the result
   669# under the provided name. If the encoded structure is empty, echoes nothing.
   670# 1: name to be output in yaml
   671# 2: encoded map-string-string (which may contain duplicate keys - resulting in map-string-stringarray)
   672# 3: key-value separator (defaults to ':')
   673# 4: item separator (defaults to ',')
   674function yaml-map-string-stringarray {
   675  declare -r name="${1}"
   676  declare -r encoded="${2}"
   677  declare -r kv_sep="${3:-:}"
   678  declare -r item_sep="${4:-,}"
   679
   680  declare -a pairs # indexed array
   681  declare -A map # associative array
   682  IFS="${item_sep}" read -ra pairs <<<"${encoded}" # split on item_sep
   683  for pair in "${pairs[@]}"; do
   684    declare key
   685    declare value
   686    IFS="${kv_sep}" read -r key value <<<"${pair}" # split on kv_sep
   687    map[$key]="${map[$key]+${map[$key]}${item_sep}}${value}" # append values from duplicate keys
   688  done
   689  # only output if there is a non-empty map
   690  if [[ ${#map[@]} -gt 0 ]]; then
   691    echo "${name}:"
   692    for k in "${!map[@]}"; do
   693      echo "  ${k}:"
   694      declare -a values
   695      IFS="${item_sep}" read -ra values <<<"${map[$k]}"
   696      for val in "${values[@]}"; do
   697        # declare across two lines so errexit can catch failures
   698        declare v
   699        v=$(yaml-quote "${val}")
   700        echo "    - ${v}"
   701      done
   702    done
   703  fi
   704}
   705
   706# yaml-map-string-string converts the encoded structure to yaml format, and echoes the result
   707# under the provided name. If the encoded structure is empty, echoes nothing.
   708# 1: name to be output in yaml
   709# 2: encoded map-string-string (no duplicate keys)
   710# 3: bool, whether to yaml-quote the value string in the output (defaults to true)
   711# 4: key-value separator (defaults to ':')
   712# 5: item separator (defaults to ',')
   713function yaml-map-string-string {
   714  declare -r name="${1}"
   715  declare -r encoded="${2}"
   716  declare -r quote_val_string="${3:-true}"
   717  declare -r kv_sep="${4:-:}"
   718  declare -r item_sep="${5:-,}"
   719
   720  declare -a pairs # indexed array
   721  declare -A map # associative array
   722  IFS="${item_sep}" read -ra pairs <<<"${encoded}" # split on item_sep # TODO(mtaufen): try quoting this too
   723  for pair in "${pairs[@]}"; do
   724    declare key
   725    declare value
   726    IFS="${kv_sep}" read -r key value <<<"${pair}" # split on kv_sep
   727    map[$key]="${value}" # add to associative array
   728  done
   729  # only output if there is a non-empty map
   730  if [[ ${#map[@]} -gt 0 ]]; then
   731    echo "${name}:"
   732    for k in "${!map[@]}"; do
   733      if [[ "${quote_val_string}" == "true" ]]; then
   734        # declare across two lines so errexit can catch failures
   735        declare v
   736        v=$(yaml-quote "${map[$k]}")
   737        echo "  ${k}: ${v}"
   738      else
   739        echo "  ${k}: ${map[$k]}"
   740      fi
   741    done
   742  fi
   743}
   744
   745# Returns kubelet flags used on both Linux and Windows nodes.
   746function construct-common-kubelet-flags {
   747  local flags="${KUBELET_TEST_LOG_LEVEL:-"--v=2"} ${KUBELET_TEST_ARGS:-}"
   748  flags+=" --cloud-provider=${CLOUD_PROVIDER_FLAG:-external}"
   749  # TODO(mtaufen): ROTATE_CERTIFICATES seems unused; delete it?
   750  if [[ -n "${ROTATE_CERTIFICATES:-}" ]]; then
   751    flags+=" --rotate-certificates=true"
   752  fi
   753  if [[ -n "${MAX_PODS_PER_NODE:-}" ]]; then
   754    flags+=" --max-pods=${MAX_PODS_PER_NODE}"
   755  fi
   756  echo "$flags"
   757}
   758
   759# Sets KUBELET_ARGS with the kubelet flags for Linux nodes.
   760# $1: if 'true', we're rendering flags for a master, else a node
   761function construct-linux-kubelet-flags {
   762  local node_type="$1"
   763  local flags
   764  flags="$(construct-common-kubelet-flags)"
   765  # Keep in sync with CONTAINERIZED_MOUNTER_HOME in configure-helper.sh
   766  flags+=" --experimental-mounter-path=/home/kubernetes/containerized_mounter/mounter"
   767  # Keep in sync with the mkdir command in configure-helper.sh (until the TODO is resolved)
   768  flags+=" --cert-dir=/var/lib/kubelet/pki/"
   769
   770  # If ENABLE_AUTH_PROVIDER_GCP is set to true, kubelet is enabled to use out-of-tree auth 
   771  # credential provider instead of in-tree auth credential provider.
   772  # https://kubernetes.io/docs/tasks/kubelet-credential-provider/kubelet-credential-provider
   773  if [[ "${ENABLE_AUTH_PROVIDER_GCP:-true}" == "true" ]]; then
   774    # Keep the values of --image-credential-provider-config and --image-credential-provider-bin-dir
   775    # in sync with value of auth_config_file and auth_provider_dir set in install-auth-provider-gcp function
   776    # in gci/configure.sh.
   777    flags+="  --image-credential-provider-config=${AUTH_PROVIDER_GCP_LINUX_CONF_FILE}"
   778    flags+="  --image-credential-provider-bin-dir=${AUTH_PROVIDER_GCP_LINUX_BIN_DIR}"
   779  fi
   780
   781  if [[ "${node_type}" == "master" ]]; then
   782    flags+=" ${MASTER_KUBELET_TEST_ARGS:-}"
   783    if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then
   784      #TODO(mikedanese): allow static pods to start before creating a client
   785      #flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
   786      #flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
   787      flags+=" --register-with-taints=node-role.kubernetes.io/control-plane=:NoSchedule"
   788      flags+=" --kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
   789      flags+=" --register-schedulable=false"
   790    fi
   791    if [[ "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then
   792      # Configure the file path for host dns configuration
   793      # as ubuntu uses systemd-resolved
   794      flags+=" --resolv-conf=/run/systemd/resolve/resolv.conf"
   795    fi
   796  else # For nodes
   797    flags+=" ${NODE_KUBELET_TEST_ARGS:-}"
   798    flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
   799    flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
   800    if [[ "${node_type}" == "heapster" ]]; then
   801        flags+=" ${HEAPSTER_KUBELET_TEST_ARGS:-}"
   802    fi
   803    if [[ "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then
   804      # Configure the file path for host dns configuration
   805      # as ubuntu uses systemd-resolved
   806      flags+=" --resolv-conf=/run/systemd/resolve/resolv.conf"
   807    fi
   808  fi
   809  flags+=" --volume-plugin-dir=${VOLUME_PLUGIN_DIR}"
   810  local node_labels
   811  node_labels="$(build-linux-node-labels "${node_type}")"
   812  if [[ -n "${node_labels:-}" ]]; then
   813    flags+=" --node-labels=${node_labels}"
   814  fi
   815  if [[ -n "${NODE_TAINTS:-}" ]]; then
   816    flags+=" --register-with-taints=${NODE_TAINTS}"
   817  fi
   818
   819  CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-unix:///run/containerd/containerd.sock}
   820  flags+=" --container-runtime-endpoint=${CONTAINER_RUNTIME_ENDPOINT}"
   821
   822  if [[ "${CONTAINER_RUNTIME_ENDPOINT}" =~ /containerd.sock$ ]]; then
   823    flags+=" --runtime-cgroups=/system.slice/containerd.service"
   824  fi
   825
   826  KUBELET_ARGS="${flags}"
   827}
   828
   829# Sets KUBELET_ARGS with the kubelet flags for Windows nodes.
   830# Note that to configure flags with explicit empty string values, we can't escape
   831# double-quotes, because they still break sc.exe after expansion in the
   832# binPath parameter, and single-quotes get parsed as characters instead of
   833# string delimiters.
   834function construct-windows-kubelet-flags {
   835  local flags
   836  flags="$(construct-common-kubelet-flags)"
   837
   838  # Note: NODE_KUBELET_TEST_ARGS is empty in typical kube-up runs.
   839  flags+=" ${NODE_KUBELET_TEST_ARGS:-}"
   840
   841  local node_labels
   842  node_labels="$(build-windows-node-labels)"
   843  if [[ -n "${node_labels:-}" ]]; then
   844    flags+=" --node-labels=${node_labels}"
   845  fi
   846
   847  # Concatenate common and windows-only node taints and apply them.
   848  local node_taints="${NODE_TAINTS:-}"
   849  if [[ -n "${node_taints}" && -n "${WINDOWS_NODE_TAINTS:-}" ]]; then
   850    node_taints+=":${WINDOWS_NODE_TAINTS}"
   851  else
   852    node_taints="${WINDOWS_NODE_TAINTS:-}"
   853  fi
   854  if [[ -n "${node_taints}" ]]; then
   855    flags+=" --register-with-taints=${node_taints}"
   856  fi
   857
   858  # Many of these flags were adapted from
   859  # https://github.com/Microsoft/SDN/blob/master/Kubernetes/windows/start-kubelet.ps1.
   860  flags+=" --config=${WINDOWS_KUBELET_CONFIG_FILE}"
   861  flags+=" --kubeconfig=${WINDOWS_KUBECONFIG_FILE}"
   862
   863  # The directory where the TLS certs are located.
   864  flags+=" --cert-dir=${WINDOWS_PKI_DIR}"
   865  flags+=" --pod-manifest-path=${WINDOWS_MANIFESTS_DIR}"
   866
   867  # Configure kubelet to run as a windows service.
   868  flags+=" --windows-service=true"
   869
   870  # Configure the file path for host dns configuration
   871  flags+=" --resolv-conf=${WINDOWS_CNI_DIR}\hostdns.conf"
   872
   873  # Both --cgroups-per-qos and --enforce-node-allocatable should be disabled on
   874  # windows; the latter requires the former to be enabled to work.
   875  flags+=" --cgroups-per-qos=false --enforce-node-allocatable="
   876
   877  # Turn off kernel memory cgroup notification.
   878  flags+=" --kernel-memcg-notification=false"
   879
   880  WINDOWS_CONTAINER_RUNTIME_ENDPOINT=${KUBE_WINDOWS_CONTAINER_RUNTIME_ENDPOINT:-npipe:////./pipe/containerd-containerd}
   881  flags+=" --container-runtime-endpoint=${WINDOWS_CONTAINER_RUNTIME_ENDPOINT}"
   882
   883  # If ENABLE_AUTH_PROVIDER_GCP is set to true, kubelet is enabled to use out-of-tree auth
   884  # credential provider. https://kubernetes.io/docs/tasks/kubelet-credential-provider/kubelet-credential-provider
   885  if [[ "${ENABLE_AUTH_PROVIDER_GCP:-true}" == "true" ]]; then
   886    flags+="  --image-credential-provider-config=${AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE}"
   887    flags+="  --image-credential-provider-bin-dir=${AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR}"
   888  fi
   889
   890  KUBELET_ARGS="${flags}"
   891}
   892
   893function construct-windows-kubeproxy-flags {
   894  local flags=""
   895
   896  # Use the same log level as the Kubelet during tests.
   897  flags+=" ${KUBELET_TEST_LOG_LEVEL:-"--v=2"}"
   898
   899  # Windows uses kernelspace proxymode
   900  flags+=" --proxy-mode=kernelspace"
   901
   902  # Configure kube-proxy to run as a windows service.
   903  flags+=" --windows-service=true"
   904
   905  # Enabling Windows DSR mode unlocks newer network features and reduces
   906  # port usage for services.
   907  # https://techcommunity.microsoft.com/t5/networking-blog/direct-server-return-dsr-in-a-nutshell/ba-p/693710
   908  if [[ "${WINDOWS_ENABLE_DSR:-}" == "true" ]]; then
   909    flags+=" --feature-gates=WinDSR=true --enable-dsr=true "
   910  fi
   911
   912  # Configure flags with explicit empty string values. We can't escape
   913  # double-quotes, because they still break sc.exe after expansion in the
   914  # binPath parameter, and single-quotes get parsed as characters instead
   915  # of string delimiters.
   916
   917  KUBEPROXY_ARGS="${flags}"
   918}
   919
   920# $1: if 'true', we're rendering config for a master, else a node
   921function build-kubelet-config {
   922  local master="$1"
   923  local os="$2"
   924  local file="$3"
   925
   926  rm -f "${file}"
   927  {
   928    print-common-kubelet-config
   929    if [[ "${master}" == "true" ]]; then
   930      print-master-kubelet-config
   931    else
   932      print-common-node-kubelet-config
   933      if [[ "${os}" == "linux" ]]; then
   934        print-linux-node-kubelet-config
   935      elif [[ "${os}" == "windows" ]]; then
   936        print-windows-node-kubelet-config
   937      else
   938        echo "Unknown OS ${os}" >&2
   939        exit 1
   940      fi
   941    fi
   942  } > "${file}"
   943}
   944
   945# cat the Kubelet config yaml in common between masters, linux nodes, and
   946# windows nodes
   947function print-common-kubelet-config {
   948  declare quoted_dns_server_ip
   949  declare quoted_dns_domain
   950  quoted_dns_server_ip=$(yaml-quote "${DNS_SERVER_IP}")
   951  quoted_dns_domain=$(yaml-quote "${DNS_DOMAIN}")
   952  cat <<EOF
   953kind: KubeletConfiguration
   954apiVersion: kubelet.config.k8s.io/v1beta1
   955cgroupRoot: /
   956clusterDNS:
   957  - ${quoted_dns_server_ip}
   958clusterDomain: ${quoted_dns_domain}
   959readOnlyPort: 10255
   960EOF
   961
   962  # Note: ENABLE_MANIFEST_URL is used by GKE.
   963  # TODO(mtaufen): remove this since it's not used in kubernetes/kubernetes nor
   964  # kubernetes/test-infra.
   965  if [[ "${ENABLE_MANIFEST_URL:-}" == "true" ]]; then
   966    declare quoted_manifest_url
   967    quoted_manifest_url=$(yaml-quote "${MANIFEST_URL}")
   968    cat <<EOF
   969staticPodURL: ${quoted_manifest_url}
   970EOF
   971    yaml-map-string-stringarray 'staticPodURLHeader' "${MANIFEST_URL_HEADER}"
   972  fi
   973
   974  if [[ -n "${EVICTION_HARD:-}" ]]; then
   975    yaml-map-string-string 'evictionHard' "${EVICTION_HARD}" true '<'
   976  fi
   977
   978  if [[ -n "${FEATURE_GATES:-}" ]]; then
   979    yaml-map-string-string 'featureGates' "${FEATURE_GATES}" false '='
   980  fi
   981}
   982
   983# cat the Kubelet config yaml for masters
   984function print-master-kubelet-config {
   985  cat <<EOF
   986enableDebuggingHandlers: ${MASTER_KUBELET_ENABLE_DEBUGGING_HANDLERS:-false}
   987hairpinMode: none
   988staticPodPath: /etc/kubernetes/manifests
   989authentication:
   990  webhook:
   991    enabled: false
   992  anonymous:
   993    enabled: true
   994authorization:
   995  mode: AlwaysAllow
   996EOF
   997  if [[ "${REGISTER_MASTER_KUBELET:-false}" == "false" ]]; then
   998     # Note: Standalone mode is used by GKE
   999    declare quoted_master_ip_range
  1000    quoted_master_ip_range=$(yaml-quote "${MASTER_IP_RANGE}")
  1001     cat <<EOF
  1002podCidr: ${quoted_master_ip_range}
  1003EOF
  1004  fi
  1005}
  1006
  1007# cat the Kubelet config yaml in common between linux nodes and windows nodes
  1008function print-common-node-kubelet-config {
  1009  cat <<EOF
  1010enableDebuggingHandlers: ${KUBELET_ENABLE_DEBUGGING_HANDLERS:-true}
  1011EOF
  1012  if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \
  1013     [[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \
  1014     [[ "${HAIRPIN_MODE:-}" == "none" ]]; then
  1015      declare quoted_hairpin_mode
  1016      quoted_hairpin_mode=$(yaml-quote "${HAIRPIN_MODE}")
  1017      cat <<EOF
  1018hairpinMode: ${quoted_hairpin_mode}
  1019EOF
  1020  fi
  1021}
  1022
  1023# cat the Kubelet config yaml for linux nodes
  1024function print-linux-node-kubelet-config {
  1025  # Keep authentication.x509.clientCAFile in sync with CA_CERT_BUNDLE_PATH in configure-helper.sh
  1026  cat <<EOF
  1027staticPodPath: /etc/kubernetes/manifests
  1028authentication:
  1029  x509:
  1030    clientCAFile: /etc/srv/kubernetes/pki/ca-certificates.crt
  1031EOF
  1032}
  1033
  1034# cat the Kubelet config yaml for windows nodes
  1035function print-windows-node-kubelet-config {
  1036  # Notes:
  1037  # - We don't run any static pods on Windows nodes yet.
  1038
  1039  # TODO(mtaufen): Does it make any sense to set eviction thresholds for inodes
  1040  # on Windows?
  1041
  1042  # TODO(pjh, mtaufen): It may make sense to use a different hairpin mode on
  1043  # Windows. We're currently using hairpin-veth, but
  1044  # https://github.com/Microsoft/SDN/blob/master/Kubernetes/windows/start-kubelet.ps1#L121
  1045  # uses promiscuous-bridge.
  1046
  1047  # TODO(pjh, mtaufen): Does cgroupRoot make sense for Windows?
  1048
  1049  # Keep authentication.x509.clientCAFile in sync with CA_CERT_BUNDLE_PATH in
  1050  # k8s-node-setup.psm1.
  1051  cat <<EOF
  1052authentication:
  1053  x509:
  1054    clientCAFile: '${WINDOWS_CA_FILE}'
  1055EOF
  1056}
  1057
  1058function build-kube-master-certs {
  1059  local file=$1
  1060  rm -f "$file"
  1061  cat >"$file" <<EOF
  1062KUBEAPISERVER_CERT: $(yaml-quote "${KUBEAPISERVER_CERT_BASE64:-}")
  1063KUBEAPISERVER_KEY: $(yaml-quote "${KUBEAPISERVER_KEY_BASE64:-}")
  1064CA_KEY: $(yaml-quote "${CA_KEY_BASE64:-}")
  1065AGGREGATOR_CA_KEY: $(yaml-quote "${AGGREGATOR_CA_KEY_BASE64:-}")
  1066REQUESTHEADER_CA_CERT: $(yaml-quote "${REQUESTHEADER_CA_CERT_BASE64:-}")
  1067PROXY_CLIENT_CERT: $(yaml-quote "${PROXY_CLIENT_CERT_BASE64:-}")
  1068PROXY_CLIENT_KEY: $(yaml-quote "${PROXY_CLIENT_KEY_BASE64:-}")
  1069ETCD_APISERVER_CA_KEY: $(yaml-quote "${ETCD_APISERVER_CA_KEY_BASE64:-}")
  1070ETCD_APISERVER_CA_CERT: $(yaml-quote "${ETCD_APISERVER_CA_CERT_BASE64:-}")
  1071ETCD_APISERVER_SERVER_KEY: $(yaml-quote "${ETCD_APISERVER_SERVER_KEY_BASE64:-}")
  1072ETCD_APISERVER_SERVER_CERT: $(yaml-quote "${ETCD_APISERVER_SERVER_CERT_BASE64:-}")
  1073ETCD_APISERVER_CLIENT_KEY: $(yaml-quote "${ETCD_APISERVER_CLIENT_KEY_BASE64:-}")
  1074ETCD_APISERVER_CLIENT_CERT: $(yaml-quote "${ETCD_APISERVER_CLIENT_CERT_BASE64:-}")
  1075CLOUD_PVL_ADMISSION_CA_KEY: $(yaml-quote "${CLOUD_PVL_ADMISSION_CA_KEY_BASE64:-}")
  1076CLOUD_PVL_ADMISSION_CA_CERT: $(yaml-quote "${CLOUD_PVL_ADMISSION_CA_CERT_BASE64:-}")
  1077CLOUD_PVL_ADMISSION_CERT: $(yaml-quote "${CLOUD_PVL_ADMISSION_CERT_BASE64:-}")
  1078CLOUD_PVL_ADMISSION_KEY: $(yaml-quote "${CLOUD_PVL_ADMISSION_KEY_BASE64:-}")
  1079KONNECTIVITY_SERVER_CA_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CA_KEY_BASE64:-}")
  1080KONNECTIVITY_SERVER_CA_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CA_CERT_BASE64:-}")
  1081KONNECTIVITY_SERVER_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CERT_BASE64:-}")
  1082KONNECTIVITY_SERVER_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_KEY_BASE64:-}")
  1083KONNECTIVITY_SERVER_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CLIENT_CERT_BASE64:-}")
  1084KONNECTIVITY_SERVER_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CLIENT_KEY_BASE64:-}")
  1085KONNECTIVITY_AGENT_CA_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CA_KEY_BASE64:-}")
  1086KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CA_CERT_BASE64:-}")
  1087KONNECTIVITY_AGENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CERT_BASE64:-}")
  1088KONNECTIVITY_AGENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_KEY_BASE64:-}")
  1089KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-}")
  1090KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-}")
  1091EOF
  1092}
  1093
  1094# $1: if 'true', we're building a master yaml, else a node
  1095function build-linux-kube-env {
  1096  local master="$1"
  1097  local file="$2"
  1098
  1099  local server_binary_tar_url=$SERVER_BINARY_TAR_URL
  1100  local kube_manifests_tar_url="${KUBE_MANIFESTS_TAR_URL:-}"
  1101  if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]] || \
  1102     [[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "ubuntu" || "${NODE_OS_DISTRIBUTION}" == "custom") ]]; then
  1103    # TODO: Support fallback .tar.gz settings on Container Linux
  1104    server_binary_tar_url=$(split_csv "${SERVER_BINARY_TAR_URL}")
  1105    kube_manifests_tar_url=$(split_csv "${KUBE_MANIFESTS_TAR_URL}")
  1106  fi
  1107
  1108  rm -f "$file"
  1109  cat >"$file" <<EOF
  1110CLUSTER_NAME: $(yaml-quote "${CLUSTER_NAME}")
  1111ENV_TIMESTAMP: $(yaml-quote "$(date -u +%Y-%m-%dT%T%z)")
  1112INSTANCE_PREFIX: $(yaml-quote "${INSTANCE_PREFIX}")
  1113NODE_INSTANCE_PREFIX: $(yaml-quote "${NODE_INSTANCE_PREFIX}")
  1114NODE_TAGS: $(yaml-quote "${NODE_TAGS:-}")
  1115NODE_NETWORK: $(yaml-quote "${NETWORK:-}")
  1116NODE_SUBNETWORK: $(yaml-quote "${SUBNETWORK:-}")
  1117CLUSTER_IP_RANGE: $(yaml-quote "${CLUSTER_IP_RANGE:-10.244.0.0/16}")
  1118SERVER_BINARY_TAR_URL: $(yaml-quote "${server_binary_tar_url}")
  1119SERVER_BINARY_TAR_HASH: $(yaml-quote "${SERVER_BINARY_TAR_HASH}")
  1120PROJECT_ID: $(yaml-quote "${PROJECT}")
  1121NETWORK_PROJECT_ID: $(yaml-quote "${NETWORK_PROJECT}")
  1122SERVICE_CLUSTER_IP_RANGE: $(yaml-quote "${SERVICE_CLUSTER_IP_RANGE}")
  1123KUBERNETES_MASTER_NAME: $(yaml-quote "${KUBERNETES_MASTER_NAME}")
  1124ALLOCATE_NODE_CIDRS: $(yaml-quote "${ALLOCATE_NODE_CIDRS:-false}")
  1125ENABLE_METRICS_SERVER: $(yaml-quote "${ENABLE_METRICS_SERVER:-false}")
  1126ENABLE_METADATA_AGENT: $(yaml-quote "${ENABLE_METADATA_AGENT:-none}")
  1127METADATA_AGENT_CPU_REQUEST: $(yaml-quote "${METADATA_AGENT_CPU_REQUEST:-}")
  1128METADATA_AGENT_MEMORY_REQUEST: $(yaml-quote "${METADATA_AGENT_MEMORY_REQUEST:-}")
  1129METADATA_AGENT_CLUSTER_LEVEL_CPU_REQUEST: $(yaml-quote "${METADATA_AGENT_CLUSTER_LEVEL_CPU_REQUEST:-}")
  1130METADATA_AGENT_CLUSTER_LEVEL_MEMORY_REQUEST: $(yaml-quote "${METADATA_AGENT_CLUSTER_LEVEL_MEMORY_REQUEST:-}")
  1131DOCKER_REGISTRY_MIRROR_URL: $(yaml-quote "${DOCKER_REGISTRY_MIRROR_URL:-}")
  1132ENABLE_L7_LOADBALANCING: $(yaml-quote "${ENABLE_L7_LOADBALANCING:-none}")
  1133ENABLE_CLUSTER_LOGGING: $(yaml-quote "${ENABLE_CLUSTER_LOGGING:-false}")
  1134ENABLE_AUTH_PROVIDER_GCP: $(yaml-quote "${ENABLE_AUTH_PROVIDER_GCP:-true}")
  1135ENABLE_NODE_PROBLEM_DETECTOR: $(yaml-quote "${ENABLE_NODE_PROBLEM_DETECTOR:-none}")
  1136NODE_PROBLEM_DETECTOR_VERSION: $(yaml-quote "${NODE_PROBLEM_DETECTOR_VERSION:-}")
  1137NODE_PROBLEM_DETECTOR_TAR_HASH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TAR_HASH:-}")
  1138NODE_PROBLEM_DETECTOR_RELEASE_PATH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_RELEASE_PATH:-}")
  1139NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS: $(yaml-quote "${NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS:-}")
  1140CNI_STORAGE_URL_BASE: $(yaml-quote "${CNI_STORAGE_URL_BASE:-}")
  1141CNI_TAR_PREFIX: $(yaml-quote "${CNI_TAR_PREFIX:-}")
  1142CNI_VERSION: $(yaml-quote "${CNI_VERSION:-}")
  1143CNI_HASH: $(yaml-quote "${CNI_HASH:-}")
  1144ENABLE_NODE_LOGGING: $(yaml-quote "${ENABLE_NODE_LOGGING:-false}")
  1145LOGGING_DESTINATION: $(yaml-quote "${LOGGING_DESTINATION:-}")
  1146ELASTICSEARCH_LOGGING_REPLICAS: $(yaml-quote "${ELASTICSEARCH_LOGGING_REPLICAS:-}")
  1147ENABLE_CLUSTER_DNS: $(yaml-quote "${ENABLE_CLUSTER_DNS:-false}")
  1148CLUSTER_DNS_CORE_DNS: $(yaml-quote "${CLUSTER_DNS_CORE_DNS:-true}")
  1149ENABLE_NODELOCAL_DNS: $(yaml-quote "${ENABLE_NODELOCAL_DNS:-false}")
  1150DNS_SERVER_IP: $(yaml-quote "${DNS_SERVER_IP:-}")
  1151LOCAL_DNS_IP: $(yaml-quote "${LOCAL_DNS_IP:-}")
  1152DNS_DOMAIN: $(yaml-quote "${DNS_DOMAIN:-}")
  1153DNS_MEMORY_LIMIT: $(yaml-quote "${DNS_MEMORY_LIMIT:-}")
  1154ENABLE_DNS_HORIZONTAL_AUTOSCALER: $(yaml-quote "${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}")
  1155KUBE_PROXY_DAEMONSET: $(yaml-quote "${KUBE_PROXY_DAEMONSET:-false}")
  1156KUBE_PROXY_TOKEN: $(yaml-quote "${KUBE_PROXY_TOKEN:-}")
  1157KUBE_PROXY_MODE: $(yaml-quote "${KUBE_PROXY_MODE:-iptables}")
  1158DETECT_LOCAL_MODE: $(yaml-quote "${DETECT_LOCAL_MODE:-}")
  1159NODE_PROBLEM_DETECTOR_TOKEN: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TOKEN:-}")
  1160ADMISSION_CONTROL: $(yaml-quote "${ADMISSION_CONTROL:-}")
  1161MASTER_IP_RANGE: $(yaml-quote "${MASTER_IP_RANGE}")
  1162RUNTIME_CONFIG: $(yaml-quote "${RUNTIME_CONFIG}")
  1163CA_CERT: $(yaml-quote "${CA_CERT_BASE64:-}")
  1164KUBELET_CERT: $(yaml-quote "${KUBELET_CERT_BASE64:-}")
  1165KUBELET_KEY: $(yaml-quote "${KUBELET_KEY_BASE64:-}")
  1166NETWORK_PROVIDER: $(yaml-quote "${NETWORK_PROVIDER:-}")
  1167NETWORK_POLICY_PROVIDER: $(yaml-quote "${NETWORK_POLICY_PROVIDER:-}")
  1168HAIRPIN_MODE: $(yaml-quote "${HAIRPIN_MODE:-}")
  1169E2E_STORAGE_TEST_ENVIRONMENT: $(yaml-quote "${E2E_STORAGE_TEST_ENVIRONMENT:-}")
  1170KUBE_DOCKER_REGISTRY: $(yaml-quote "${KUBE_DOCKER_REGISTRY:-}")
  1171KUBE_ADDON_REGISTRY: $(yaml-quote "${KUBE_ADDON_REGISTRY:-}")
  1172MULTIZONE: $(yaml-quote "${MULTIZONE:-}")
  1173MULTIMASTER: $(yaml-quote "${MULTIMASTER:-}")
  1174NON_MASQUERADE_CIDR: $(yaml-quote "${NON_MASQUERADE_CIDR:-}")
  1175ENABLE_DEFAULT_STORAGE_CLASS: $(yaml-quote "${ENABLE_DEFAULT_STORAGE_CLASS:-}")
  1176ENABLE_VOLUME_SNAPSHOTS: $(yaml-quote "${ENABLE_VOLUME_SNAPSHOTS:-}")
  1177ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote "${ENABLE_APISERVER_ADVANCED_AUDIT:-}")
  1178ENABLE_APISERVER_DYNAMIC_AUDIT: $(yaml-quote "${ENABLE_APISERVER_DYNAMIC_AUDIT:-}")
  1179ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote "${ENABLE_CACHE_MUTATION_DETECTOR:-false}")
  1180ENABLE_KUBE_WATCHLIST_INCONSISTENCY_DETECTOR: $(yaml-quote "${ENABLE_KUBE_WATCHLIST_INCONSISTENCY_DETECTOR:-false}")
  1181ENABLE_PATCH_CONVERSION_DETECTOR: $(yaml-quote "${ENABLE_PATCH_CONVERSION_DETECTOR:-false}")
  1182ADVANCED_AUDIT_POLICY: $(yaml-quote "${ADVANCED_AUDIT_POLICY:-}")
  1183ADVANCED_AUDIT_BACKEND: $(yaml-quote "${ADVANCED_AUDIT_BACKEND:-log}")
  1184ADVANCED_AUDIT_TRUNCATING_BACKEND: $(yaml-quote "${ADVANCED_AUDIT_TRUNCATING_BACKEND:-true}")
  1185ADVANCED_AUDIT_LOG_MODE: $(yaml-quote "${ADVANCED_AUDIT_LOG_MODE:-}")
  1186ADVANCED_AUDIT_LOG_BUFFER_SIZE: $(yaml-quote "${ADVANCED_AUDIT_LOG_BUFFER_SIZE:-}")
  1187ADVANCED_AUDIT_LOG_MAX_BATCH_SIZE: $(yaml-quote "${ADVANCED_AUDIT_LOG_MAX_BATCH_SIZE:-}")
  1188ADVANCED_AUDIT_LOG_MAX_BATCH_WAIT: $(yaml-quote "${ADVANCED_AUDIT_LOG_MAX_BATCH_WAIT:-}")
  1189ADVANCED_AUDIT_LOG_THROTTLE_QPS: $(yaml-quote "${ADVANCED_AUDIT_LOG_THROTTLE_QPS:-}")
  1190ADVANCED_AUDIT_LOG_THROTTLE_BURST: $(yaml-quote "${ADVANCED_AUDIT_LOG_THROTTLE_BURST:-}")
  1191ADVANCED_AUDIT_LOG_INITIAL_BACKOFF: $(yaml-quote "${ADVANCED_AUDIT_LOG_INITIAL_BACKOFF:-}")
  1192ADVANCED_AUDIT_WEBHOOK_MODE: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_MODE:-}")
  1193ADVANCED_AUDIT_WEBHOOK_BUFFER_SIZE: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_BUFFER_SIZE:-}")
  1194ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_SIZE: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_SIZE:-}")
  1195ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_WAIT: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_WAIT:-}")
  1196ADVANCED_AUDIT_WEBHOOK_THROTTLE_QPS: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_THROTTLE_QPS:-}")
  1197ADVANCED_AUDIT_WEBHOOK_THROTTLE_BURST: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_THROTTLE_BURST:-}")
  1198ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF:-}")
  1199GCE_API_ENDPOINT: $(yaml-quote "${GCE_API_ENDPOINT:-}")
  1200GCE_GLBC_IMAGE: $(yaml-quote "${GCE_GLBC_IMAGE:-}")
  1201CUSTOM_INGRESS_YAML: |
  1202${CUSTOM_INGRESS_YAML//\'/\'\'}
  1203ENABLE_NODE_JOURNAL: $(yaml-quote "${ENABLE_NODE_JOURNAL:-false}")
  1204PROMETHEUS_TO_SD_ENDPOINT: $(yaml-quote "${PROMETHEUS_TO_SD_ENDPOINT:-}")
  1205PROMETHEUS_TO_SD_PREFIX: $(yaml-quote "${PROMETHEUS_TO_SD_PREFIX:-}")
  1206ENABLE_PROMETHEUS_TO_SD: $(yaml-quote "${ENABLE_PROMETHEUS_TO_SD:-false}")
  1207DISABLE_PROMETHEUS_TO_SD_IN_DS: $(yaml-quote "${DISABLE_PROMETHEUS_TO_SD_IN_DS:-false}")
  1208CONTAINER_RUNTIME_ENDPOINT: $(yaml-quote "${CONTAINER_RUNTIME_ENDPOINT:-}")
  1209CONTAINER_RUNTIME_NAME: $(yaml-quote "${CONTAINER_RUNTIME_NAME:-}")
  1210CONTAINER_RUNTIME_TEST_HANDLER: $(yaml-quote "${CONTAINER_RUNTIME_TEST_HANDLER:-}")
  1211CONTAINERD_INFRA_CONTAINER: $(yaml-quote "${CONTAINER_INFRA_CONTAINER:-}")
  1212UBUNTU_INSTALL_CONTAINERD_VERSION: $(yaml-quote "${UBUNTU_INSTALL_CONTAINERD_VERSION:-}")
  1213UBUNTU_INSTALL_RUNC_VERSION: $(yaml-quote "${UBUNTU_INSTALL_RUNC_VERSION:-}")
  1214COS_INSTALL_CONTAINERD_VERSION: $(yaml-quote "${COS_INSTALL_CONTAINERD_VERSION:-}")
  1215COS_INSTALL_RUNC_VERSION: $(yaml-quote "${COS_INSTALL_RUNC_VERSION:-}")
  1216NODE_LOCAL_SSDS_EXT: $(yaml-quote "${NODE_LOCAL_SSDS_EXT:-}")
  1217NODE_LOCAL_SSDS_EPHEMERAL: $(yaml-quote "${NODE_LOCAL_SSDS_EPHEMERAL:-}")
  1218LOAD_IMAGE_COMMAND: $(yaml-quote "${LOAD_IMAGE_COMMAND:-}")
  1219ZONE: $(yaml-quote "${ZONE}")
  1220REGION: $(yaml-quote "${REGION}")
  1221VOLUME_PLUGIN_DIR: $(yaml-quote "${VOLUME_PLUGIN_DIR}")
  1222KUBELET_ARGS: $(yaml-quote "${KUBELET_ARGS}")
  1223REQUIRE_METADATA_KUBELET_CONFIG_FILE: $(yaml-quote true)
  1224ENABLE_NETD: $(yaml-quote "${ENABLE_NETD:-false}")
  1225CUSTOM_NETD_YAML: |
  1226${CUSTOM_NETD_YAML//\'/\'\'}
  1227CUSTOM_CALICO_NODE_DAEMONSET_YAML: |
  1228${CUSTOM_CALICO_NODE_DAEMONSET_YAML//\'/\'\'}
  1229CUSTOM_TYPHA_DEPLOYMENT_YAML: |
  1230${CUSTOM_TYPHA_DEPLOYMENT_YAML//\'/\'\'}
  1231CONCURRENT_SERVICE_SYNCS: $(yaml-quote "${CONCURRENT_SERVICE_SYNCS:-}")
  1232AUTH_PROVIDER_GCP_STORAGE_PATH: $(yaml-quote "${AUTH_PROVIDER_GCP_STORAGE_PATH}")
  1233AUTH_PROVIDER_GCP_VERSION: $(yaml-quote "${AUTH_PROVIDER_GCP_VERSION}")
  1234AUTH_PROVIDER_GCP_LINUX_BIN_DIR: $(yaml-quote "${AUTH_PROVIDER_GCP_LINUX_BIN_DIR}")
  1235AUTH_PROVIDER_GCP_LINUX_CONF_FILE: $(yaml-quote "${AUTH_PROVIDER_GCP_LINUX_CONF_FILE}")
  1236EOF
  1237  if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "gci" ]] || \
  1238     [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "gci" ]]  || \
  1239     [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "cos" ]] || \
  1240     [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "cos" ]]; then
  1241    cat >>"$file" <<EOF
  1242REMOUNT_VOLUME_PLUGIN_DIR: $(yaml-quote "${REMOUNT_VOLUME_PLUGIN_DIR:-true}")
  1243EOF
  1244  fi
  1245  if [[ "${master}" == "false" ]]; then
  1246    cat >>"$file" <<EOF
  1247KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CA_CERT_BASE64:-}")
  1248KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-}")
  1249KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-}")
  1250EOF
  1251  fi
  1252  if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then
  1253    cat >>"$file" <<EOF
  1254KUBE_APISERVER_REQUEST_TIMEOUT: $(yaml-quote "${KUBE_APISERVER_REQUEST_TIMEOUT}")
  1255EOF
  1256  fi
  1257  if [ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]; then
  1258    cat >>"$file" <<EOF
  1259TERMINATED_POD_GC_THRESHOLD: $(yaml-quote "${TERMINATED_POD_GC_THRESHOLD}")
  1260EOF
  1261  fi
  1262  if [[ "${master}" == "true" && ("${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "ubuntu") ]] || \
  1263     [[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" = "ubuntu" || "${NODE_OS_DISTRIBUTION}" = "custom") ]] ; then
  1264    cat >>"$file" <<EOF
  1265KUBE_MANIFESTS_TAR_URL: $(yaml-quote "${kube_manifests_tar_url}")
  1266KUBE_MANIFESTS_TAR_HASH: $(yaml-quote "${KUBE_MANIFESTS_TAR_HASH}")
  1267EOF
  1268  fi
  1269  if [ -n "${TEST_CLUSTER:-}" ]; then
  1270    cat >>"$file" <<EOF
  1271TEST_CLUSTER: $(yaml-quote "${TEST_CLUSTER}")
  1272EOF
  1273  fi
  1274  if [ -n "${DOCKER_TEST_LOG_LEVEL:-}" ]; then
  1275      cat >>"$file" <<EOF
  1276DOCKER_TEST_LOG_LEVEL: $(yaml-quote "${DOCKER_TEST_LOG_LEVEL}")
  1277EOF
  1278  fi
  1279  if [ -n "${DOCKER_LOG_DRIVER:-}" ]; then
  1280      cat >>"$file" <<EOF
  1281DOCKER_LOG_DRIVER: $(yaml-quote "${DOCKER_LOG_DRIVER}")
  1282EOF
  1283  fi
  1284  if [ -n "${DOCKER_LOG_MAX_SIZE:-}" ]; then
  1285      cat >>"$file" <<EOF
  1286DOCKER_LOG_MAX_SIZE: $(yaml-quote "${DOCKER_LOG_MAX_SIZE}")
  1287EOF
  1288  fi
  1289  if [ -n "${DOCKER_LOG_MAX_FILE:-}" ]; then
  1290      cat >>"$file" <<EOF
  1291DOCKER_LOG_MAX_FILE: $(yaml-quote "${DOCKER_LOG_MAX_FILE}")
  1292EOF
  1293  fi
  1294  if [ -n "${CLOUD_PROVIDER_FLAG:-}" ]; then
  1295    cat >>"$file" <<EOF
  1296CLOUD_PROVIDER_FLAG: $(yaml-quote "${CLOUD_PROVIDER_FLAG}")
  1297EOF
  1298  fi
  1299  if [ -n "${FEATURE_GATES:-}" ]; then
  1300    cat >>"$file" <<EOF
  1301FEATURE_GATES: $(yaml-quote "${FEATURE_GATES}")
  1302EOF
  1303  fi
  1304  if [ -n "${RUN_CONTROLLERS:-}" ]; then
  1305    cat >>"$file" <<EOF
  1306RUN_CONTROLLERS: $(yaml-quote "${RUN_CONTROLLERS}")
  1307EOF
  1308  fi
  1309  if [ -n "${RUN_CCM_CONTROLLERS:-}" ]; then
  1310    cat >>"$file" <<EOF
  1311RUN_CCM_CONTROLLERS: $(yaml-quote "${RUN_CCM_CONTROLLERS}")
  1312EOF
  1313  fi
  1314  if [ -n "${PROVIDER_VARS:-}" ]; then
  1315    local var_name
  1316    local var_value
  1317
  1318    for var_name in ${PROVIDER_VARS}; do
  1319      eval "local var_value=\$(yaml-quote \${${var_name}})"
  1320      cat >>"$file" <<EOF
  1321${var_name}: ${var_value}
  1322EOF
  1323    done
  1324  fi
  1325
  1326  if [[ "${master}" == "true" ]]; then
  1327    # Master-only env vars.
  1328    cat >>"$file" <<EOF
  1329KUBERNETES_MASTER: $(yaml-quote 'true')
  1330KUBE_USER: $(yaml-quote "${KUBE_USER}")
  1331KUBE_PASSWORD: $(yaml-quote "${KUBE_PASSWORD}")
  1332KUBE_BEARER_TOKEN: $(yaml-quote "${KUBE_BEARER_TOKEN}")
  1333MASTER_CERT: $(yaml-quote "${MASTER_CERT_BASE64:-}")
  1334MASTER_KEY: $(yaml-quote "${MASTER_KEY_BASE64:-}")
  1335KUBECFG_CERT: $(yaml-quote "${KUBECFG_CERT_BASE64:-}")
  1336KUBECFG_KEY: $(yaml-quote "${KUBECFG_KEY_BASE64:-}")
  1337KUBELET_APISERVER: $(yaml-quote "${KUBELET_APISERVER:-}")
  1338NUM_NODES: $(yaml-quote "${NUM_NODES}")
  1339STORAGE_BACKEND: $(yaml-quote "${STORAGE_BACKEND:-etcd3}")
  1340STORAGE_MEDIA_TYPE: $(yaml-quote "${STORAGE_MEDIA_TYPE:-}")
  1341ENABLE_GARBAGE_COLLECTOR: $(yaml-quote "${ENABLE_GARBAGE_COLLECTOR:-}")
  1342ENABLE_LEGACY_ABAC: $(yaml-quote "${ENABLE_LEGACY_ABAC:-}")
  1343MASTER_ADVERTISE_ADDRESS: $(yaml-quote "${MASTER_ADVERTISE_ADDRESS:-}")
  1344ETCD_CA_KEY: $(yaml-quote "${ETCD_CA_KEY_BASE64:-}")
  1345ETCD_CA_CERT: $(yaml-quote "${ETCD_CA_CERT_BASE64:-}")
  1346ETCD_PEER_KEY: $(yaml-quote "${ETCD_PEER_KEY_BASE64:-}")
  1347ETCD_PEER_CERT: $(yaml-quote "${ETCD_PEER_CERT_BASE64:-}")
  1348SERVICEACCOUNT_ISSUER: $(yaml-quote "${SERVICEACCOUNT_ISSUER:-}")
  1349KUBECTL_PRUNE_WHITELIST_OVERRIDE: $(yaml-quote "${KUBECTL_PRUNE_WHITELIST_OVERRIDE:-}")
  1350CCM_FEATURE_GATES:  $(yaml-quote "${CCM_FEATURE_GATES:-}")
  1351KUBE_SCHEDULER_RUNASUSER: 2001
  1352KUBE_SCHEDULER_RUNASGROUP: 2001
  1353KUBE_ADDON_MANAGER_RUNASUSER: 2002
  1354KUBE_ADDON_MANAGER_RUNASGROUP: 2002
  1355KUBE_CONTROLLER_MANAGER_RUNASUSER: 2003
  1356KUBE_CONTROLLER_MANAGER_RUNASGROUP: 2003
  1357KUBE_API_SERVER_RUNASUSER: 2004
  1358KUBE_API_SERVER_RUNASGROUP: 2004
  1359KUBE_PKI_READERS_GROUP: 2005
  1360ETCD_RUNASUSER: 2006
  1361ETCD_RUNASGROUP: 2006
  1362KUBE_POD_LOG_READERS_GROUP: 2007
  1363KONNECTIVITY_SERVER_RUNASUSER: 2008
  1364KONNECTIVITY_SERVER_RUNASGROUP: 2008
  1365KONNECTIVITY_SERVER_SOCKET_WRITER_GROUP: 2008
  1366CLOUD_CONTROLLER_MANAGER_RUNASUSER: 2009
  1367CLOUD_CONTROLLER_MANAGER_RUNASGROUP: 2009
  1368CLUSTER_AUTOSCALER_RUNASUSER: 2010
  1369CLUSTER_AUTOSCALER_RUNASGROUP: 2010
  1370
  1371EOF
  1372    # KUBE_APISERVER_REQUEST_TIMEOUT_SEC (if set) controls the --request-timeout
  1373    # flag
  1374    if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT_SEC:-}" ]; then
  1375      cat >>"$file" <<EOF
  1376KUBE_APISERVER_REQUEST_TIMEOUT_SEC: $(yaml-quote "${KUBE_APISERVER_REQUEST_TIMEOUT_SEC}")
  1377EOF
  1378    fi
  1379    # KUBE_APISERVER_GODEBUG (if set) controls the value of GODEBUG env var for kube-apiserver.
  1380    if [ -n "${KUBE_APISERVER_GODEBUG:-}" ]; then
  1381      cat >>"$file" <<EOF
  1382KUBE_APISERVER_GODEBUG: $(yaml-quote "${KUBE_APISERVER_GODEBUG}")
  1383EOF
  1384    fi
  1385    # ETCD_IMAGE (if set) allows to use a custom etcd image.
  1386    if [ -n "${ETCD_IMAGE:-}" ]; then
  1387      cat >>"$file" <<EOF
  1388ETCD_IMAGE: $(yaml-quote "${ETCD_IMAGE}")
  1389EOF
  1390    fi
  1391    # ETCD_DOCKER_REPOSITORY (if set) allows to use a custom etcd docker repository to pull the etcd image from.
  1392    if [ -n "${ETCD_DOCKER_REPOSITORY:-}" ]; then
  1393      cat >>"$file" <<EOF
  1394ETCD_DOCKER_REPOSITORY: $(yaml-quote "${ETCD_DOCKER_REPOSITORY}")
  1395EOF
  1396    fi
  1397    # ETCD_VERSION (if set) allows you to use custom version of etcd.
  1398    # The main purpose of using it may be rollback of etcd v3 API,
  1399    # where we need 3.0.* image, but are rolling back to 2.3.7.
  1400    if [ -n "${ETCD_VERSION:-}" ]; then
  1401      cat >>"$file" <<EOF
  1402ETCD_VERSION: $(yaml-quote "${ETCD_VERSION}")
  1403EOF
  1404    fi
  1405    if [ -n "${ETCD_HOSTNAME:-}" ]; then
  1406      cat >>"$file" <<EOF
  1407ETCD_HOSTNAME: $(yaml-quote "${ETCD_HOSTNAME}")
  1408EOF
  1409    fi
  1410    if [ -n "${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC:-}" ]; then
  1411      cat >>"$file" <<EOF
  1412ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC: $(yaml-quote "${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC}")
  1413EOF
  1414    fi
  1415    if [ -n "${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC:-}" ]; then
  1416      cat >>"$file" <<EOF
  1417KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC: $(yaml-quote "${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC}")
  1418EOF
  1419    fi
  1420    if [ -n "${ETCD_COMPACTION_INTERVAL_SEC:-}" ]; then
  1421      cat >>"$file" <<EOF
  1422ETCD_COMPACTION_INTERVAL_SEC: $(yaml-quote "${ETCD_COMPACTION_INTERVAL_SEC}")
  1423EOF
  1424    fi
  1425    if [ -n "${ETCD_QUOTA_BACKEND_BYTES:-}" ]; then
  1426      cat >>"$file" <<EOF
  1427ETCD_QUOTA_BACKEND_BYTES: $(yaml-quote "${ETCD_QUOTA_BACKEND_BYTES}")
  1428EOF
  1429    fi
  1430    if [ -n "${ETCD_EXTRA_ARGS:-}" ]; then
  1431    cat >>"$file" <<EOF
  1432ETCD_EXTRA_ARGS: $(yaml-quote "${ETCD_EXTRA_ARGS}")
  1433EOF
  1434    fi
  1435    if [ -n "${ETCD_SERVERS:-}" ]; then
  1436    cat >>"$file" <<EOF
  1437ETCD_SERVERS: $(yaml-quote "${ETCD_SERVERS}")
  1438EOF
  1439    fi
  1440    if [ -n "${ETCD_SERVERS_OVERRIDES:-}" ]; then
  1441    cat >>"$file" <<EOF
  1442ETCD_SERVERS_OVERRIDES: $(yaml-quote "${ETCD_SERVERS_OVERRIDES}")
  1443EOF
  1444    fi
  1445    if [ -n "${APISERVER_TEST_ARGS:-}" ]; then
  1446      cat >>"$file" <<EOF
  1447APISERVER_TEST_ARGS: $(yaml-quote "${APISERVER_TEST_ARGS}")
  1448EOF
  1449    fi
  1450    if [ -n "${CONTROLLER_MANAGER_TEST_ARGS:-}" ]; then
  1451      cat >>"$file" <<EOF
  1452CONTROLLER_MANAGER_TEST_ARGS: $(yaml-quote "${CONTROLLER_MANAGER_TEST_ARGS}")
  1453EOF
  1454    fi
  1455    if [ -n "${KUBE_CONTROLLER_MANAGER_TEST_ARGS:-}" ]; then
  1456      cat >>"$file" <<EOF
  1457KUBE_CONTROLLER_MANAGER_TEST_ARGS: $(yaml-quote "${KUBE_CONTROLLER_MANAGER_TEST_ARGS}")
  1458EOF
  1459    fi
  1460    if [ -n "${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-}" ]; then
  1461      cat >>"$file" <<EOF
  1462CONTROLLER_MANAGER_TEST_LOG_LEVEL: $(yaml-quote "${CONTROLLER_MANAGER_TEST_LOG_LEVEL}")
  1463EOF
  1464    fi
  1465    if [ -n "${SCHEDULER_TEST_ARGS:-}" ]; then
  1466      cat >>"$file" <<EOF
  1467SCHEDULER_TEST_ARGS: $(yaml-quote "${SCHEDULER_TEST_ARGS}")
  1468EOF
  1469    fi
  1470    if [ -n "${SCHEDULER_TEST_LOG_LEVEL:-}" ]; then
  1471      cat >>"$file" <<EOF
  1472SCHEDULER_TEST_LOG_LEVEL: $(yaml-quote "${SCHEDULER_TEST_LOG_LEVEL}")
  1473EOF
  1474    fi
  1475    if [ -n "${INITIAL_ETCD_CLUSTER:-}" ]; then
  1476      cat >>"$file" <<EOF
  1477INITIAL_ETCD_CLUSTER: $(yaml-quote "${INITIAL_ETCD_CLUSTER}")
  1478EOF
  1479    fi
  1480    if [ -n "${INITIAL_ETCD_CLUSTER_STATE:-}" ]; then
  1481      cat >>"$file" <<EOF
  1482INITIAL_ETCD_CLUSTER_STATE: $(yaml-quote "${INITIAL_ETCD_CLUSTER_STATE}")
  1483EOF
  1484    fi
  1485    if [ -n "${CLUSTER_SIGNING_DURATION:-}" ]; then
  1486      cat >>"$file" <<EOF
  1487CLUSTER_SIGNING_DURATION: $(yaml-quote "${CLUSTER_SIGNING_DURATION}")
  1488EOF
  1489    fi
  1490    if [[ "${NODE_ACCELERATORS:-}" == *"type=nvidia"* ]]; then
  1491      cat >>"$file" <<EOF
  1492ENABLE_NVIDIA_GPU_DEVICE_PLUGIN: $(yaml-quote "true")
  1493EOF
  1494    fi
  1495    if [ -n "${ADDON_MANAGER_LEADER_ELECTION:-}" ]; then
  1496      cat >>"$file" <<EOF
  1497ADDON_MANAGER_LEADER_ELECTION: $(yaml-quote "${ADDON_MANAGER_LEADER_ELECTION}")
  1498EOF
  1499    fi
  1500    if [ -n "${API_SERVER_TEST_LOG_LEVEL:-}" ]; then
  1501      cat >>"$file" <<EOF
  1502API_SERVER_TEST_LOG_LEVEL: $(yaml-quote "${API_SERVER_TEST_LOG_LEVEL}")
  1503EOF
  1504    fi
  1505    if [ -n "${ETCD_LISTEN_CLIENT_IP:-}" ]; then
  1506      cat >>"$file" <<EOF
  1507ETCD_LISTEN_CLIENT_IP: $(yaml-quote "${ETCD_LISTEN_CLIENT_IP}")
  1508EOF
  1509    fi
  1510    if [ -n "${ETCD_PROGRESS_NOTIFY_INTERVAL:-}" ]; then
  1511      cat >>"$file" <<EOF
  1512ETCD_PROGRESS_NOTIFY_INTERVAL: $(yaml-quote "${ETCD_PROGRESS_NOTIFY_INTERVAL}")
  1513EOF
  1514    fi
  1515
  1516  else
  1517    # Node-only env vars.
  1518    cat >>"$file" <<EOF
  1519KUBERNETES_MASTER: $(yaml-quote "false")
  1520EXTRA_DOCKER_OPTS: $(yaml-quote "${EXTRA_DOCKER_OPTS:-}")
  1521EOF
  1522    if [ -n "${KUBEPROXY_TEST_ARGS:-}" ]; then
  1523      cat >>"$file" <<EOF
  1524KUBEPROXY_TEST_ARGS: $(yaml-quote "${KUBEPROXY_TEST_ARGS}")
  1525EOF
  1526    fi
  1527    if [ -n "${KUBEPROXY_TEST_LOG_LEVEL:-}" ]; then
  1528      cat >>"$file" <<EOF
  1529KUBEPROXY_TEST_LOG_LEVEL: $(yaml-quote "${KUBEPROXY_TEST_LOG_LEVEL}")
  1530EOF
  1531    fi
  1532  fi
  1533  if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then
  1534      cat >>"$file" <<EOF
  1535ENABLE_CLUSTER_AUTOSCALER: $(yaml-quote "${ENABLE_CLUSTER_AUTOSCALER}")
  1536AUTOSCALER_MIG_CONFIG: $(yaml-quote "${AUTOSCALER_MIG_CONFIG}")
  1537AUTOSCALER_EXPANDER_CONFIG: $(yaml-quote "${AUTOSCALER_EXPANDER_CONFIG}")
  1538EOF
  1539      if [[ "${master}" == "false" ]]; then
  1540          # TODO(kubernetes/autoscaler#718): AUTOSCALER_ENV_VARS is a hotfix for cluster autoscaler,
  1541          # which reads the kube-env to determine the shape of a node and was broken by #60020.
  1542          # This should be removed as soon as a more reliable source of information is available!
  1543          local node_labels
  1544          local node_taints
  1545          local autoscaler_env_vars
  1546          node_labels="$(build-linux-node-labels node)"
  1547          node_taints="${NODE_TAINTS:-}"
  1548          autoscaler_env_vars="node_labels=${node_labels};node_taints=${node_taints}"
  1549          cat >>"$file" <<EOF
  1550AUTOSCALER_ENV_VARS: $(yaml-quote "${autoscaler_env_vars}")
  1551EOF
  1552      fi
  1553  fi
  1554  if [ -n "${SCHEDULING_ALGORITHM_PROVIDER:-}" ]; then
  1555    cat >>"$file" <<EOF
  1556SCHEDULING_ALGORITHM_PROVIDER: $(yaml-quote "${SCHEDULING_ALGORITHM_PROVIDER}")
  1557EOF
  1558  fi
  1559  if [ -n "${MAX_PODS_PER_NODE:-}" ]; then
  1560    cat >>"$file" <<EOF
  1561MAX_PODS_PER_NODE: $(yaml-quote "${MAX_PODS_PER_NODE}")
  1562EOF
  1563  fi
  1564  if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
  1565      cat >>"$file" <<EOF
  1566PREPARE_KONNECTIVITY_SERVICE: $(yaml-quote "${PREPARE_KONNECTIVITY_SERVICE}")
  1567EOF
  1568  fi
  1569  if [[ "${EGRESS_VIA_KONNECTIVITY:-false}" == "true" ]]; then
  1570      cat >>"$file" <<EOF
  1571EGRESS_VIA_KONNECTIVITY: $(yaml-quote "${EGRESS_VIA_KONNECTIVITY}")
  1572EOF
  1573  fi
  1574  if [[ "${RUN_KONNECTIVITY_PODS:-false}" == "true" ]]; then
  1575      cat >>"$file" <<EOF
  1576RUN_KONNECTIVITY_PODS: $(yaml-quote "${RUN_KONNECTIVITY_PODS}")
  1577EOF
  1578  fi
  1579  if [[ -n "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-}" ]]; then
  1580      cat >>"$file" <<EOF
  1581KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE: $(yaml-quote "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE}")
  1582EOF
  1583  fi
  1584}
  1585
  1586
  1587function build-windows-kube-env {
  1588  local file="$1"
  1589  # For now the Windows kube-env is a superset of the Linux kube-env.
  1590  build-linux-kube-env false "$file"
  1591
  1592  cat >>"$file" <<EOF
  1593WINDOWS_NODE_INSTANCE_PREFIX: $(yaml-quote "${WINDOWS_NODE_INSTANCE_PREFIX}")
  1594NODE_BINARY_TAR_URL: $(yaml-quote "${NODE_BINARY_TAR_URL}")
  1595NODE_BINARY_TAR_HASH: $(yaml-quote "${NODE_BINARY_TAR_HASH}")
  1596CSI_PROXY_STORAGE_PATH: $(yaml-quote "${CSI_PROXY_STORAGE_PATH}")
  1597CSI_PROXY_VERSION: $(yaml-quote "${CSI_PROXY_VERSION}")
  1598CSI_PROXY_FLAGS: $(yaml-quote "${CSI_PROXY_FLAGS}")
  1599ENABLE_CSI_PROXY: $(yaml-quote "${ENABLE_CSI_PROXY}")
  1600K8S_DIR: $(yaml-quote "${WINDOWS_K8S_DIR}")
  1601NODE_DIR: $(yaml-quote "${WINDOWS_NODE_DIR}")
  1602LOGS_DIR: $(yaml-quote "${WINDOWS_LOGS_DIR}")
  1603CNI_DIR: $(yaml-quote "${WINDOWS_CNI_DIR}")
  1604CNI_CONFIG_DIR: $(yaml-quote "${WINDOWS_CNI_CONFIG_DIR}")
  1605WINDOWS_CNI_STORAGE_PATH: $(yaml-quote "${WINDOWS_CNI_STORAGE_PATH}")
  1606WINDOWS_CNI_VERSION: $(yaml-quote "${WINDOWS_CNI_VERSION}")
  1607WINDOWS_CONTAINER_RUNTIME: $(yaml-quote "${WINDOWS_CONTAINER_RUNTIME}")
  1608WINDOWS_CONTAINER_RUNTIME_ENDPOINT: $(yaml-quote "${WINDOWS_CONTAINER_RUNTIME_ENDPOINT:-}")
  1609MANIFESTS_DIR: $(yaml-quote "${WINDOWS_MANIFESTS_DIR}")
  1610PKI_DIR: $(yaml-quote "${WINDOWS_PKI_DIR}")
  1611CA_FILE_PATH: $(yaml-quote "${WINDOWS_CA_FILE}")
  1612KUBELET_CONFIG_FILE: $(yaml-quote "${WINDOWS_KUBELET_CONFIG_FILE}")
  1613KUBEPROXY_ARGS: $(yaml-quote "${KUBEPROXY_ARGS}")
  1614KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_KUBECONFIG_FILE}")
  1615BOOTSTRAP_KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_BOOTSTRAP_KUBECONFIG_FILE}")
  1616KUBEPROXY_KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_KUBEPROXY_KUBECONFIG_FILE}")
  1617WINDOWS_INFRA_CONTAINER: $(yaml-quote "${WINDOWS_INFRA_CONTAINER}")
  1618WINDOWS_ENABLE_PIGZ: $(yaml-quote "${WINDOWS_ENABLE_PIGZ}")
  1619WINDOWS_ENABLE_HYPERV: $(yaml-quote "${WINDOWS_ENABLE_HYPERV}")
  1620ENABLE_AUTH_PROVIDER_GCP: $(yaml-quote "${ENABLE_AUTH_PROVIDER_GCP}")
  1621ENABLE_NODE_PROBLEM_DETECTOR: $(yaml-quote "${WINDOWS_ENABLE_NODE_PROBLEM_DETECTOR}")
  1622NODE_PROBLEM_DETECTOR_VERSION: $(yaml-quote "${NODE_PROBLEM_DETECTOR_VERSION}")
  1623NODE_PROBLEM_DETECTOR_TAR_HASH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TAR_HASH}")
  1624NODE_PROBLEM_DETECTOR_RELEASE_PATH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_RELEASE_PATH}")
  1625NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS: $(yaml-quote "${WINDOWS_NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS}")
  1626NODE_PROBLEM_DETECTOR_TOKEN: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TOKEN:-}")
  1627WINDOWS_NODEPROBLEMDETECTOR_KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_NODEPROBLEMDETECTOR_KUBECONFIG_FILE}")
  1628AUTH_PROVIDER_GCP_STORAGE_PATH: $(yaml-quote "${AUTH_PROVIDER_GCP_STORAGE_PATH}")
  1629AUTH_PROVIDER_GCP_VERSION: $(yaml-quote "${AUTH_PROVIDER_GCP_VERSION}")
  1630AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64: $(yaml-quote "${AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64}")
  1631AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR: $(yaml-quote "${AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR}")
  1632AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE: $(yaml-quote "${AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE}")
  1633EOF
  1634}
  1635
  1636function sha512sum-file() {
  1637  local shasum
  1638  if which sha512sum >/dev/null 2>&1; then
  1639    shasum=$(sha512sum "$1")
  1640  else
  1641    shasum=$(shasum -a512 "$1")
  1642  fi
  1643  echo "${shasum%%[[:blank:]]*}"
  1644}
  1645
  1646# Create certificate pairs for the cluster.
  1647# $1: The public IP for the master.
  1648#
  1649# These are used for static cert distribution (e.g. static clustering) at
  1650# cluster creation time. This will be obsoleted once we implement dynamic
  1651# clustering.
  1652#
  1653# The following certificate pairs are created:
  1654#
  1655#  - ca (the cluster's certificate authority)
  1656#  - server
  1657#  - kubelet
  1658#  - kubecfg (for kubectl)
  1659#
  1660# TODO(roberthbailey): Replace easyrsa with a simple Go program to generate
  1661# the certs that we need.
  1662#
  1663# Assumed vars
  1664#   KUBE_TEMP
  1665#   MASTER_NAME
  1666#
  1667# Vars set:
  1668#   CERT_DIR
  1669#   CA_CERT_BASE64
  1670#   MASTER_CERT_BASE64
  1671#   MASTER_KEY_BASE64
  1672#   KUBELET_CERT_BASE64
  1673#   KUBELET_KEY_BASE64
  1674#   KUBECFG_CERT_BASE64
  1675#   KUBECFG_KEY_BASE64
  1676function create-certs {
  1677  local -r primary_cn="${1}"
  1678
  1679  # Determine extra certificate names for master
  1680
  1681  # Create service_ip by stripping the network mask part from
  1682  # SERVICE_CLUSTER_IP_RANGE and incrementing the host part with 1
  1683  service_ip=${SERVICE_CLUSTER_IP_RANGE%/*}
  1684  service_ip="${service_ip%.*}.$((${service_ip##*.} + 1))"
  1685  local sans=""
  1686  for extra in "$@"; do
  1687    if [[ -n "${extra}" ]]; then
  1688      sans="${sans}IP:${extra},"
  1689    fi
  1690  done
  1691  sans="${sans}IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${DNS_DOMAIN},DNS:${MASTER_NAME}"
  1692
  1693  echo "Generating certs for alternate-names: ${sans}"
  1694
  1695  setup-easyrsa
  1696  PRIMARY_CN="${primary_cn}" SANS="${sans}" generate-certs
  1697  AGGREGATOR_PRIMARY_CN="${primary_cn}" AGGREGATOR_SANS="${sans}" generate-aggregator-certs
  1698  KONNECTIVITY_SERVER_PRIMARY_CN="${primary_cn}" KONNECTIVITY_SERVER_SANS="${sans}" generate-konnectivity-server-certs
  1699  CLOUD_PVL_ADMISSION_PRIMARY_CN="${primary_cn}" CLOUD_PVL_ADMISSION_SANS="${sans}" generate-cloud-pvl-admission-certs
  1700
  1701  # By default, linux wraps base64 output every 76 cols, so we use 'tr -d' to remove whitespaces.
  1702  # Note 'base64 -w0' doesn't work on Mac OS X, which has different flags.
  1703  CA_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
  1704  CA_CERT_BASE64=$(base64 "${CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
  1705  MASTER_CERT_BASE64=$(base64 "${CERT_DIR}/pki/issued/${MASTER_NAME}.crt" | tr -d '\r\n')
  1706  MASTER_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/${MASTER_NAME}.key" | tr -d '\r\n')
  1707  KUBELET_CERT_BASE64=$(base64 "${CERT_DIR}/pki/issued/kubelet.crt" | tr -d '\r\n')
  1708  KUBELET_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/kubelet.key" | tr -d '\r\n')
  1709  KUBECFG_CERT_BASE64=$(base64 "${CERT_DIR}/pki/issued/kubecfg.crt" | tr -d '\r\n')
  1710  KUBECFG_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/kubecfg.key" | tr -d '\r\n')
  1711  KUBEAPISERVER_CERT_BASE64=$(base64 "${CERT_DIR}/pki/issued/kube-apiserver.crt" | tr -d '\r\n')
  1712  KUBEAPISERVER_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/kube-apiserver.key" | tr -d '\r\n')
  1713
  1714  # Setting up an addition directory (beyond pki) as it is the simplest way to
  1715  # ensure we get a different CA pair to sign the proxy-client certs and which
  1716  # we can send CA public key to the user-apiserver to validate communication.
  1717  AGGREGATOR_CA_KEY_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
  1718  REQUESTHEADER_CA_CERT_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
  1719  PROXY_CLIENT_CERT_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/issued/proxy-client.crt" | tr -d '\r\n')
  1720  PROXY_CLIENT_KEY_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/private/proxy-client.key" | tr -d '\r\n')
  1721
  1722  # Setting up the Kubernetes API Server Konnectivity Server auth.
  1723  # This includes certs for both API Server to Konnectivity Server and
  1724  # Konnectivity Agent to Konnectivity Server.
  1725  KONNECTIVITY_SERVER_CA_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
  1726  KONNECTIVITY_SERVER_CA_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
  1727  KONNECTIVITY_SERVER_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n')
  1728  KONNECTIVITY_SERVER_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" | tr -d '\r\n')
  1729  KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" | tr -d '\r\n')
  1730  KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" | tr -d '\r\n')
  1731  KONNECTIVITY_AGENT_CA_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
  1732  KONNECTIVITY_AGENT_CA_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
  1733  KONNECTIVITY_AGENT_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n')
  1734  KONNECTIVITY_AGENT_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" | tr -d '\r\n')
  1735  KONNECTIVITY_AGENT_CLIENT_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" | tr -d '\r\n')
  1736  KONNECTIVITY_AGENT_CLIENT_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key" | tr -d '\r\n')
  1737
  1738  CLOUD_PVL_ADMISSION_CA_KEY_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
  1739  CLOUD_PVL_ADMISSION_CA_CERT_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
  1740  CLOUD_PVL_ADMISSION_CERT_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n')
  1741  CLOUD_PVL_ADMISSION_KEY_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/server.key" | tr -d '\r\n')
  1742}
  1743
  1744# Set up easy-rsa directory structure.
  1745#
  1746# Assumed vars
  1747#   KUBE_TEMP
  1748#
  1749# Vars set:
  1750#   CERT_DIR
  1751#   AGGREGATOR_CERT_DIR
  1752function setup-easyrsa {
  1753  local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
  1754  # Note: This was heavily cribbed from make-ca-cert.sh
  1755  (set -x
  1756    cd "${KUBE_TEMP}"
  1757    curl -L -O --connect-timeout 20 --retry 6 --retry-delay 2 https://dl.k8s.io/easy-rsa/easy-rsa.tar.gz
  1758    tar xzf easy-rsa.tar.gz
  1759    mkdir easy-rsa-master/kubelet
  1760    cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/kubelet
  1761    mkdir easy-rsa-master/aggregator
  1762    cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator
  1763    mkdir easy-rsa-master/cloud-pvl-admission
  1764    cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/cloud-pvl-admission
  1765    mkdir easy-rsa-master/konnectivity-server
  1766    cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-server
  1767    mkdir easy-rsa-master/konnectivity-agent
  1768    cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-agent) &>"${cert_create_debug_output}" || true
  1769  CERT_DIR="${KUBE_TEMP}/easy-rsa-master/easyrsa3"
  1770  AGGREGATOR_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/aggregator"
  1771  CLOUD_PVL_ADMISSION_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/cloud-pvl-admission"
  1772  KONNECTIVITY_SERVER_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-server"
  1773  KONNECTIVITY_AGENT_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-agent"
  1774  if [ ! -x "${CERT_DIR}/easyrsa" ] || [ ! -x "${AGGREGATOR_CERT_DIR}/easyrsa" ]; then
  1775    # TODO(roberthbailey,porridge): add better error handling here,
  1776    # see https://github.com/kubernetes/kubernetes/issues/55229
  1777    cat "${cert_create_debug_output}" >&2
  1778    echo "=== Failed to setup easy-rsa: Aborting ===" >&2
  1779    exit 2
  1780  fi
  1781}
  1782
  1783# Runs the easy RSA commands to generate certificate files.
  1784# The generated files are IN ${CERT_DIR}
  1785#
  1786# Assumed vars (see shellcheck disable directives below)
  1787#   KUBE_TEMP
  1788#   MASTER_NAME
  1789#   CERT_DIR
  1790#   PRIMARY_CN: Primary canonical name
  1791#   SANS: Subject alternate names
  1792#
  1793#
  1794function generate-certs {
  1795  local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
  1796  # Note: This was heavily cribbed from make-ca-cert.sh
  1797  (set -x
  1798    cd "${CERT_DIR}"
  1799    ./easyrsa init-pki
  1800    # this puts the cert into pki/ca.crt and the key into pki/private/ca.key
  1801    # PRIMARY_CN (expected to be) defined by caller
  1802    # shellcheck disable=SC2153
  1803    ./easyrsa --batch "--req-cn=${PRIMARY_CN}@$(date +%s)" build-ca nopass
  1804    # SANS (expected to be) defined by caller
  1805    # shellcheck disable=SC2153
  1806    ./easyrsa --subject-alt-name="${SANS}" build-server-full "${MASTER_NAME}" nopass
  1807    ./easyrsa build-client-full kube-apiserver nopass
  1808
  1809    kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
  1810
  1811    # make the config for the signer
  1812    echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json"
  1813    # create the kubelet client cert with the correct groups
  1814    echo '{"CN":"kubelet","names":[{"O":"system:nodes"}],"hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare kubelet
  1815    mv "kubelet-key.pem" "pki/private/kubelet.key"
  1816    mv "kubelet.pem" "pki/issued/kubelet.crt"
  1817    rm -f "kubelet.csr"
  1818
  1819    # Make a superuser client cert with subject "O=system:masters, CN=kubecfg"
  1820    ./easyrsa --dn-mode=org \
  1821      --req-cn=kubecfg --req-org=system:masters \
  1822      --req-c= --req-st= --req-city= --req-email= --req-ou= \
  1823      build-client-full kubecfg nopass) &>"${cert_create_debug_output}" || true
  1824  local output_file_missing=0
  1825  local output_file
  1826  for output_file in \
  1827    "${CERT_DIR}/pki/private/ca.key" \
  1828    "${CERT_DIR}/pki/ca.crt" \
  1829    "${CERT_DIR}/pki/issued/${MASTER_NAME}.crt" \
  1830    "${CERT_DIR}/pki/private/${MASTER_NAME}.key" \
  1831    "${CERT_DIR}/pki/issued/kubelet.crt" \
  1832    "${CERT_DIR}/pki/private/kubelet.key" \
  1833    "${CERT_DIR}/pki/issued/kubecfg.crt" \
  1834    "${CERT_DIR}/pki/private/kubecfg.key" \
  1835    "${CERT_DIR}/pki/issued/kube-apiserver.crt" \
  1836    "${CERT_DIR}/pki/private/kube-apiserver.key"
  1837  do
  1838    if [[ ! -s "${output_file}" ]]; then
  1839      echo "Expected file ${output_file} not created" >&2
  1840      output_file_missing=1
  1841    fi
  1842  done
  1843  if [ $output_file_missing -ne 0 ]; then
  1844    # TODO(roberthbailey,porridge): add better error handling here,
  1845    # see https://github.com/kubernetes/kubernetes/issues/55229
  1846    cat "${cert_create_debug_output}" >&2
  1847    echo "=== Failed to generate master certificates: Aborting ===" >&2
  1848    exit 2
  1849  fi
  1850}
  1851
  1852# Runs the easy RSA commands to generate aggregator certificate files.
  1853# The generated files are in ${AGGREGATOR_CERT_DIR}
  1854#
  1855# Assumed vars
  1856#   KUBE_TEMP
  1857#   AGGREGATOR_MASTER_NAME
  1858#   AGGREGATOR_CERT_DIR
  1859#   AGGREGATOR_PRIMARY_CN: Primary canonical name
  1860#   AGGREGATOR_SANS: Subject alternate names
  1861#
  1862#
  1863function generate-aggregator-certs {
  1864  local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
  1865  # Note: This was heavily cribbed from make-ca-cert.sh
  1866  (set -x
  1867    cd "${KUBE_TEMP}/easy-rsa-master/aggregator"
  1868    ./easyrsa init-pki
  1869    # this puts the cert into pki/ca.crt and the key into pki/private/ca.key
  1870    ./easyrsa --batch "--req-cn=${AGGREGATOR_PRIMARY_CN}@$(date +%s)" build-ca nopass
  1871    ./easyrsa --subject-alt-name="${AGGREGATOR_SANS}" build-server-full "${AGGREGATOR_MASTER_NAME}" nopass
  1872    ./easyrsa build-client-full aggregator-apiserver nopass
  1873
  1874    kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
  1875
  1876    # make the config for the signer
  1877    echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json"
  1878    # create the aggregator client cert with the correct groups
  1879    echo '{"CN":"aggregator","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare proxy-client
  1880    mv "proxy-client-key.pem" "pki/private/proxy-client.key"
  1881    mv "proxy-client.pem" "pki/issued/proxy-client.crt"
  1882    rm -f "proxy-client.csr"
  1883
  1884    # Make a superuser client cert with subject "O=system:masters, CN=kubecfg"
  1885    ./easyrsa --dn-mode=org \
  1886      --req-cn=proxy-clientcfg --req-org=system:aggregator \
  1887      --req-c= --req-st= --req-city= --req-email= --req-ou= \
  1888      build-client-full proxy-clientcfg nopass) &>"${cert_create_debug_output}" || true
  1889  local output_file_missing=0
  1890  local output_file
  1891  for output_file in \
  1892    "${AGGREGATOR_CERT_DIR}/pki/private/ca.key" \
  1893    "${AGGREGATOR_CERT_DIR}/pki/ca.crt" \
  1894    "${AGGREGATOR_CERT_DIR}/pki/issued/proxy-client.crt" \
  1895    "${AGGREGATOR_CERT_DIR}/pki/private/proxy-client.key"
  1896  do
  1897    if [[ ! -s "${output_file}" ]]; then
  1898      echo "Expected file ${output_file} not created" >&2
  1899      output_file_missing=1
  1900    fi
  1901  done
  1902  if [ $output_file_missing -ne 0 ]; then
  1903    # TODO(roberthbailey,porridge): add better error handling here,
  1904    # see https://github.com/kubernetes/kubernetes/issues/55229
  1905    cat "${cert_create_debug_output}" >&2
  1906    echo "=== Failed to generate aggregator certificates: Aborting ===" >&2
  1907    exit 2
  1908  fi
  1909}
  1910
  1911# Runs the easy RSA commands to generate server side certificate files
  1912# for the konnectivity server. This includes both server side to both
  1913# konnectivity-server and konnectivity-agent.
  1914# The generated files are in ${KONNECTIVITY_SERVER_CERT_DIR} and
  1915# ${KONNECTIVITY_AGENT_CERT_DIR}
  1916#
  1917# Assumed vars
  1918#   KUBE_TEMP
  1919#   KONNECTIVITY_SERVER_CERT_DIR
  1920#   KONNECTIVITY_SERVER_PRIMARY_CN: Primary canonical name
  1921#   KONNECTIVITY_SERVER_SANS: Subject alternate names
  1922#
  1923function generate-konnectivity-server-certs {
  1924  local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
  1925  # Note: This was heavily cribbed from make-ca-cert.sh
  1926  (set -x
  1927    # Make the client <-> konnectivity server side certificates.
  1928    cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-server"
  1929    ./easyrsa init-pki
  1930    # this puts the cert into pki/ca.crt and the key into pki/private/ca.key
  1931    ./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass
  1932    ./easyrsa --subject-alt-name="IP:127.0.0.1,${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass
  1933    ./easyrsa build-client-full client nopass
  1934
  1935    kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
  1936
  1937    # make the config for the signer
  1938    echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json"
  1939    # create the konnectivity server cert with the correct groups
  1940    echo '{"CN":"konnectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-server
  1941    rm -f "konnectivity-server.csr"
  1942
  1943    # Make the agent <-> konnectivity server side certificates.
  1944    cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-agent"
  1945    ./easyrsa init-pki
  1946    # this puts the cert into pki/ca.crt and the key into pki/private/ca.key
  1947    ./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass
  1948    ./easyrsa --subject-alt-name="${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass
  1949    ./easyrsa build-client-full client nopass
  1950
  1951    kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
  1952
  1953    # make the config for the signer
  1954    echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json"
  1955    # create the konnectivity server cert with the correct groups
  1956    echo '{"CN":"koonectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent
  1957    rm -f "konnectivity-agent.csr"
  1958
  1959    echo "completed main certificate section") &>"${cert_create_debug_output}" || true
  1960
  1961  local output_file_missing=0
  1962  local output_file
  1963  for output_file in \
  1964    "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" \
  1965    "${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" \
  1966    "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" \
  1967    "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" \
  1968    "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" \
  1969    "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" \
  1970    "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" \
  1971    "${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" \
  1972    "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" \
  1973    "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" \
  1974    "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" \
  1975    "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key"
  1976  do
  1977    if [[ ! -s "${output_file}" ]]; then
  1978      echo "Expected file ${output_file} not created" >&2
  1979      output_file_missing=1
  1980    fi
  1981  done
  1982  if (( output_file_missing )); then
  1983    # TODO(roberthbailey,porridge): add better error handling here,
  1984    # see https://github.com/kubernetes/kubernetes/issues/55229
  1985    cat "${cert_create_debug_output}" >&2
  1986    echo "=== Failed to generate konnectivity-server certificates: Aborting ===" >&2
  1987    exit 2
  1988  fi
  1989}
  1990
  1991# Runs the easy RSA commands to generate server side certificate files
  1992# for the cloud-pvl-admission webhook.
  1993# The generated files are in ${CLOUD_PVL_ADMISSION_CERT_DIR}
  1994#
  1995# Assumed vars
  1996#   KUBE_TEMP
  1997#   CLOUD_PVL_ADMISSION_CERT_DIR
  1998#   CLOUD_PVL_ADMISSION_PRIMARY_CN: Primary canonical name
  1999#   CLOUD_PVL_ADMISSION_SANS: Subject alternate names
  2000#
  2001function generate-cloud-pvl-admission-certs {
  2002  local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
  2003  # Note: This was heavily cribbed from make-ca-cert.sh
  2004  (set -x
  2005    # Make the client <-> cloud-pvl-admission server side certificates.
  2006    cd "${KUBE_TEMP}/easy-rsa-master/cloud-pvl-admission"
  2007    ./easyrsa init-pki
  2008    # this puts the cert into pki/ca.crt and the key into pki/private/ca.key
  2009    ./easyrsa --batch "--req-cn=${CLOUD_PVL_ADMISSION_PRIMARY_CN}@$(date +%s)" build-ca nopass
  2010    ./easyrsa --subject-alt-name="IP:127.0.0.1,${CLOUD_PVL_ADMISSION_SANS}" build-server-full server nopass
  2011    ./easyrsa build-client-full client nopass
  2012
  2013    kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
  2014
  2015    # make the config for the signer
  2016    echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json"
  2017    # create the cloud-pvl-admission cert with the correct groups
  2018    echo '{"CN":"cloud-pvl-admission","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare cloud-pvl-admission
  2019    rm -f "cloud-pvl-admission.csr"
  2020
  2021    # Make the cloud-pvl-admission server side certificates.
  2022    cd "${KUBE_TEMP}/easy-rsa-master/cloud-pvl-admission"
  2023    ./easyrsa init-pki
  2024    # this puts the cert into pki/ca.crt and the key into pki/private/ca.key
  2025    ./easyrsa --batch "--req-cn=${CLOUD_PVL_ADMISSION_PRIMARY_CN}@$(date +%s)" build-ca nopass
  2026    ./easyrsa --subject-alt-name="${CLOUD_PVL_ADMISSION_SANS}" build-server-full server nopass
  2027    ./easyrsa build-client-full client nopass
  2028
  2029    kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
  2030
  2031    # make the config for the signer
  2032    echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json"
  2033    # create the cloud-pvl-admission server cert with the correct groups
  2034    echo '{"CN":"cloud-pvl-admission","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent
  2035    rm -f "konnectivity-agent.csr"
  2036
  2037    echo "completed main certificate section") &>"${cert_create_debug_output}" || true
  2038
  2039  local output_file_missing=0
  2040  local output_file
  2041  for output_file in \
  2042    "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/ca.key" \
  2043    "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/ca.crt" \
  2044    "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/issued/server.crt" \
  2045    "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/server.key" \
  2046    "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/issued/client.crt" \
  2047    "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/client.key"
  2048  do
  2049    if [[ ! -s "${output_file}" ]]; then
  2050      echo "Expected file ${output_file} not created" >&2
  2051      output_file_missing=1
  2052    fi
  2053  done
  2054  if (( output_file_missing )); then
  2055    # TODO(roberthbailey,porridge): add better error handling here,
  2056    # see https://github.com/kubernetes/kubernetes/issues/55229
  2057    cat "${cert_create_debug_output}" >&2
  2058    echo "=== Failed to generate cloud-pvl-admission certificates: Aborting ===" >&2
  2059    exit 2
  2060  fi
  2061}
  2062
  2063# Using provided master env, extracts value from provided key.
  2064#
  2065# Args:
  2066# $1 master env (kube-env of master; result of calling get-master-env)
  2067# $2 env key to use
  2068function get-env-val() {
  2069  local match
  2070  match=$( (echo "${1}" | grep -E "^${2}:") || echo '')
  2071  if [[ -z "${match}" ]]; then
  2072    echo ""
  2073  fi
  2074  echo "${match}" | cut -d : -f 2 | cut -d \' -f 2
  2075}
  2076
  2077# Load the master env by calling get-master-env, and extract important values
  2078function parse-master-env() {
  2079  # Get required master env vars
  2080  local master_env
  2081  master_env=$(get-master-env)
  2082  KUBE_PROXY_TOKEN=$(get-env-val "${master_env}" "KUBE_PROXY_TOKEN")
  2083  NODE_PROBLEM_DETECTOR_TOKEN=$(get-env-val "${master_env}" "NODE_PROBLEM_DETECTOR_TOKEN")
  2084  CA_CERT_BASE64=$(get-env-val "${master_env}" "CA_CERT")
  2085  CA_KEY_BASE64=$(get-env-val "${master_env}" "CA_KEY")
  2086  KUBEAPISERVER_CERT_BASE64=$(get-env-val "${master_env}" "KUBEAPISERVER_CERT")
  2087  KUBEAPISERVER_KEY_BASE64=$(get-env-val "${master_env}" "KUBEAPISERVER_KEY")
  2088  EXTRA_DOCKER_OPTS=$(get-env-val "${master_env}" "EXTRA_DOCKER_OPTS")
  2089  KUBELET_CERT_BASE64=$(get-env-val "${master_env}" "KUBELET_CERT")
  2090  KUBELET_KEY_BASE64=$(get-env-val "${master_env}" "KUBELET_KEY")
  2091  MASTER_CERT_BASE64=$(get-env-val "${master_env}" "MASTER_CERT")
  2092  MASTER_KEY_BASE64=$(get-env-val "${master_env}" "MASTER_KEY")
  2093  AGGREGATOR_CA_KEY_BASE64=$(get-env-val "${master_env}" "AGGREGATOR_CA_KEY")
  2094  REQUESTHEADER_CA_CERT_BASE64=$(get-env-val "${master_env}" "REQUESTHEADER_CA_CERT")
  2095  PROXY_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "PROXY_CLIENT_CERT")
  2096  PROXY_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "PROXY_CLIENT_KEY")
  2097  ENABLE_LEGACY_ABAC=$(get-env-val "${master_env}" "ENABLE_LEGACY_ABAC")
  2098  ETCD_APISERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CA_KEY")
  2099  ETCD_APISERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CA_CERT")
  2100  ETCD_APISERVER_SERVER_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_KEY")
  2101  ETCD_APISERVER_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_CERT")
  2102  ETCD_APISERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_KEY")
  2103  ETCD_APISERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_CERT")
  2104  CLOUD_PVL_ADMISSION_CA_KEY_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_CA_KEY")
  2105  CLOUD_PVL_ADMISSION_CA_CERT_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_CA_CERT")
  2106  CLOUD_PVL_ADMISSION_CERT_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_CERT")
  2107  CLOUD_PVL_ADMISSION_KEY_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_KEY")
  2108  KONNECTIVITY_SERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_KEY")
  2109  KONNECTIVITY_SERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_CERT")
  2110  KONNECTIVITY_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CERT")
  2111  KONNECTIVITY_SERVER_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_KEY")
  2112  KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_CERT")
  2113  KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_KEY")
  2114  KONNECTIVITY_AGENT_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_KEY")
  2115  KONNECTIVITY_AGENT_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_CERT")
  2116  KONNECTIVITY_AGENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CERT")
  2117  KONNECTIVITY_AGENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_KEY")
  2118}
  2119
  2120# Update or verify required gcloud components are installed
  2121# at minimum required version.
  2122# Assumed vars
  2123#   KUBE_PROMPT_FOR_UPDATE
  2124function update-or-verify-gcloud() {
  2125  local sudo_prefix=""
  2126  if [ ! -w "$(dirname "$(which gcloud)")" ]; then
  2127    sudo_prefix="sudo"
  2128  fi
  2129  # update and install components as needed
  2130  # (deliberately word split $gcloud_prompt)
  2131  # shellcheck disable=SC2086
  2132  if [[ "${KUBE_PROMPT_FOR_UPDATE}" == "y" ]]; then
  2133    ${sudo_prefix} gcloud ${gcloud_prompt:-} components install alpha
  2134    ${sudo_prefix} gcloud ${gcloud_prompt:-} components install beta
  2135    ${sudo_prefix} gcloud ${gcloud_prompt:-} components update
  2136  else
  2137    local version
  2138    version=$(gcloud version --format=json)
  2139    python3 -c"
  2140import json,sys
  2141from distutils import version
  2142
  2143minVersion = version.LooseVersion('1.3.0')
  2144required = [ 'alpha', 'beta', 'core' ]
  2145data = json.loads(sys.argv[1])
  2146rel = data.get('Google Cloud SDK')
  2147if 'CL @' in rel:
  2148  print('Using dev version of gcloud: %s' %rel)
  2149  exit(0)
  2150if rel != 'HEAD' and version.LooseVersion(rel) < minVersion:
  2151  print('gcloud version out of date ( < %s )' % minVersion)
  2152  exit(1)
  2153missing = []
  2154for c in required:
  2155  if not data.get(c):
  2156    missing += [c]
  2157if missing:
  2158  for c in missing:
  2159    print ('missing required gcloud component \"{0}\"'.format(c))
  2160    print ('Try running \$(gcloud components install {0})'.format(c))
  2161  exit(1)
  2162    " "${version}"
  2163  fi
  2164}
  2165
  2166# Robustly try to create a static ip.
  2167# $1: The name of the ip to create
  2168# $2: The name of the region to create the ip in.
  2169function create-static-ip() {
  2170  detect-project
  2171  local attempt=0
  2172  local REGION="$2"
  2173  while true; do
  2174    if gcloud compute addresses create "$1" \
  2175      --project "${PROJECT}" \
  2176      --region "${REGION}" -q > /dev/null; then
  2177      # successful operation - wait until it's visible
  2178      start="$(date +%s)"
  2179      while true; do
  2180        now="$(date +%s)"
  2181        # Timeout set to 15 minutes
  2182        if [[ $((now - start)) -gt 900 ]]; then
  2183          echo "Timeout while waiting for master IP visibility"
  2184          exit 2
  2185        fi
  2186        if gcloud compute addresses describe "$1" --project "${PROJECT}" --region "${REGION}" >/dev/null 2>&1; then
  2187          break
  2188        fi
  2189        echo "Master IP not visible yet. Waiting..."
  2190        sleep 5
  2191      done
  2192      break
  2193    fi
  2194
  2195    if gcloud compute addresses describe "$1" \
  2196      --project "${PROJECT}" \
  2197      --region "${REGION}" >/dev/null 2>&1; then
  2198      # it exists - postcondition satisfied
  2199      break
  2200    fi
  2201
  2202    if (( attempt > 4 )); then
  2203      echo -e "${color_red}Failed to create static ip $1 ${color_norm}" >&2
  2204      exit 2
  2205    fi
  2206    attempt=$((attempt + 1))
  2207    echo -e "${color_yellow:-}Attempt $attempt failed to create static ip $1. Retrying.${color_norm:-}" >&2
  2208    sleep $((attempt * 5))
  2209  done
  2210}
  2211
  2212# Robustly try to create a firewall rule.
  2213# $1: The name of firewall rule.
  2214# $2: IP ranges.
  2215# $3: Target tags for this firewall rule.
  2216function create-firewall-rule() {
  2217  detect-project
  2218  local attempt=0
  2219  while true; do
  2220    if ! gcloud compute firewall-rules create "$1" \
  2221      --project "${NETWORK_PROJECT}" \
  2222      --network "${NETWORK}" \
  2223      --source-ranges "$2" \
  2224      --target-tags "$3" \
  2225      --allow tcp,udp,icmp,esp,ah,sctp; then
  2226      if (( attempt > 4 )); then
  2227        echo -e "${color_red}Failed to create firewall rule $1 ${color_norm}" >&2
  2228        exit 2
  2229      fi
  2230      echo -e "${color_yellow}Attempt $((attempt + 1)) failed to create firewall rule $1. Retrying.${color_norm}" >&2
  2231      attempt=$((attempt + 1))
  2232      sleep $((attempt * 5))
  2233    else
  2234        break
  2235    fi
  2236  done
  2237}
  2238
  2239# Format the string argument for gcloud network.
  2240function make-gcloud-network-argument() {
  2241  local network_project="$1"
  2242  local region="$2"
  2243  local network="$3"
  2244  local subnet="$4"
  2245  local address="$5"          # optional
  2246  local enable_ip_alias="$6"  # optional
  2247  local alias_size="$7"       # optional
  2248
  2249  local networkURL="projects/${network_project}/global/networks/${network}"
  2250  local subnetURL="projects/${network_project}/regions/${region}/subnetworks/${subnet:-}"
  2251
  2252  local ret=""
  2253
  2254  if [[ "${enable_ip_alias}" == 'true' ]]; then
  2255    ret="--network-interface"
  2256    ret="${ret} network=${networkURL}"
  2257    if [[ "${address:-}" == "no-address" ]]; then
  2258      ret="${ret},no-address"
  2259    else
  2260      ret="${ret},address=${address:-}"
  2261    fi
  2262    ret="${ret},subnet=${subnetURL}"
  2263    ret="${ret},aliases=pods-default:${alias_size}"
  2264    ret="${ret} --no-can-ip-forward"
  2265  else
  2266    if [[ -n ${subnet:-} ]]; then
  2267      ret="${ret} --subnet ${subnetURL}"
  2268    else
  2269      ret="${ret} --network ${networkURL}"
  2270    fi
  2271
  2272    ret="${ret} --can-ip-forward"
  2273    if [[ -n ${address:-} ]] && [[ "$address" != "no-address" ]]; then
  2274      ret="${ret} --address ${address}"
  2275    fi
  2276  fi
  2277
  2278  echo "${ret}"
  2279}
  2280
  2281# $1: version (required)
  2282# $2: Prefix for the template name, i.e. NODE_INSTANCE_PREFIX or
  2283#     WINDOWS_NODE_INSTANCE_PREFIX.
  2284function get-template-name-from-version() {
  2285  local -r version=${1}
  2286  local -r template_prefix=${2}
  2287  # trim template name to pass gce name validation
  2288  echo "${template_prefix}-template-${version}" | cut -c 1-63 | sed 's/[\.\+]/-/g;s/-*$//g'
  2289}
  2290
  2291# validates the NODE_LOCAL_SSDS_EXT variable
  2292function validate-node-local-ssds-ext(){
  2293  ssdopts="${1}"
  2294
  2295  if [[ -z "${ssdopts[0]}" || -z "${ssdopts[1]}" || -z "${ssdopts[2]}" ]]; then
  2296    echo -e "${color_red}Local SSD: NODE_LOCAL_SSDS_EXT is malformed, found ${ssdopts[0]-_},${ssdopts[1]-_},${ssdopts[2]-_} ${color_norm}" >&2
  2297    exit 2
  2298  fi
  2299  if [[ "${ssdopts[1]}" != "scsi" && "${ssdopts[1]}" != "nvme" ]]; then
  2300    echo -e "${color_red}Local SSD: Interface must be scsi or nvme, found: ${ssdopts[1]} ${color_norm}" >&2
  2301    exit 2
  2302  fi
  2303  if [[ "${ssdopts[2]}" != "fs" && "${ssdopts[2]}" != "block" ]]; then
  2304    echo -e "${color_red}Local SSD: Filesystem type must be fs or block, found: ${ssdopts[2]} ${color_norm}"  >&2
  2305    exit 2
  2306  fi
  2307  local_ssd_ext_count=$((local_ssd_ext_count+ssdopts[0]))
  2308  if [[ "${local_ssd_ext_count}" -gt "${GCE_MAX_LOCAL_SSD}" || "${local_ssd_ext_count}" -lt 1 ]]; then
  2309    echo -e "${color_red}Local SSD: Total number of local ssds must range from 1 to 8, found: ${local_ssd_ext_count} ${color_norm}" >&2
  2310    exit 2
  2311  fi
  2312}
  2313
  2314# Robustly try to create an instance template.
  2315# $1: The name of the instance template.
  2316# $2: The scopes flag.
  2317# $3: String of comma-separated metadata-from-file entries.
  2318# $4: String of comma-separated metadata (key=value) entries.
  2319# $5: the node OS ("linux" or "windows").
  2320function create-node-template() {
  2321  detect-project
  2322  detect-subnetworks
  2323  local template_name="$1"
  2324  local metadata_values="$4"
  2325  local os="$5"
  2326  local machine_type="$6"
  2327
  2328  # First, ensure the template doesn't exist.
  2329  # TODO(zmerlynn): To make this really robust, we need to parse the output and
  2330  #                 add retries. Just relying on a non-zero exit code doesn't
  2331  #                 distinguish an ephemeral failed call from a "not-exists".
  2332  if gcloud compute instance-templates describe "${template_name}" --project "${PROJECT}" &>/dev/null; then
  2333    echo "Instance template ${1} already exists; deleting." >&2
  2334    if ! gcloud compute instance-templates delete "${template_name}" --project "${PROJECT}" --quiet &>/dev/null; then
  2335      echo -e "${color_yellow}Failed to delete existing instance template${color_norm}" >&2
  2336      exit 2
  2337    fi
  2338  fi
  2339
  2340  local gcloud="gcloud"
  2341
  2342  local accelerator_args=()
  2343  # VMs with Accelerators cannot be live migrated.
  2344  # More details here - https://cloud.google.com/compute/docs/gpus/add-gpus#create-new-gpu-instance
  2345  if [[ -n "${NODE_ACCELERATORS}" ]]; then
  2346    accelerator_args+=(--maintenance-policy TERMINATE --restart-on-failure --accelerator "${NODE_ACCELERATORS}")
  2347    gcloud="gcloud beta"
  2348  fi
  2349
  2350  local preemptible_minions=()
  2351  if [[ "${PREEMPTIBLE_NODE}" == "true" ]]; then
  2352    preemptible_minions+=(--preemptible --maintenance-policy TERMINATE)
  2353  fi
  2354
  2355  local local_ssds=()
  2356  local_ssd_ext_count=0
  2357  if [[ -n "${NODE_LOCAL_SSDS_EXT:-}" ]]; then
  2358    IFS=";" read -r -a ssdgroups <<< "${NODE_LOCAL_SSDS_EXT:-}"
  2359    for ssdgroup in "${ssdgroups[@]}"
  2360    do
  2361      IFS="," read -r -a ssdopts <<< "${ssdgroup}"
  2362      validate-node-local-ssds-ext "${ssdopts[@]}"
  2363      for ((i=1; i<=ssdopts[0]; i++)); do
  2364        local_ssds+=("--local-ssd=interface=${ssdopts[1]}")
  2365      done
  2366    done
  2367  fi
  2368
  2369  if [[ -n ${NODE_LOCAL_SSDS+x} ]]; then
  2370    # The NODE_LOCAL_SSDS check below fixes issue #49171
  2371    for ((i=1; i<=NODE_LOCAL_SSDS; i++)); do
  2372      local_ssds+=('--local-ssd=interface=SCSI')
  2373    done
  2374  fi
  2375
  2376  local address=""
  2377  if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
  2378    address="no-address"
  2379  fi
  2380
  2381  local network
  2382  network=$(make-gcloud-network-argument \
  2383    "${NETWORK_PROJECT}" \
  2384    "${REGION}" \
  2385    "${NETWORK}" \
  2386    "${SUBNETWORK:-}" \
  2387    "${address}" \
  2388    "${ENABLE_IP_ALIASES:-}" \
  2389    "${IP_ALIAS_SIZE:-}")
  2390
  2391  local node_image_flags=()
  2392  if [[ "${os}" == 'linux' ]]; then
  2393      node_image_flags+=(--image-project "${NODE_IMAGE_PROJECT}" --image "${NODE_IMAGE}")
  2394  elif [[ "${os}" == 'windows' ]]; then
  2395      node_image_flags+=(--image-project "${WINDOWS_NODE_IMAGE_PROJECT}" --image "${WINDOWS_NODE_IMAGE}")
  2396  else
  2397      echo "Unknown OS ${os}" >&2
  2398      exit 1
  2399  fi
  2400
  2401  local metadata_flag="${metadata_values:+--metadata ${metadata_values}}"
  2402
  2403  local attempt=1
  2404  while true; do
  2405    echo "Attempt ${attempt} to create ${1}" >&2
  2406    # Deliberately word split ${network}, $2 and ${metadata_flag}
  2407    # shellcheck disable=SC2086
  2408    if ! ${gcloud} compute instance-templates create \
  2409      "${template_name}" \
  2410      --project "${PROJECT}" \
  2411      --machine-type "${machine_type}" \
  2412      --boot-disk-type "${NODE_DISK_TYPE}" \
  2413      --boot-disk-size "${NODE_DISK_SIZE}" \
  2414      "${node_image_flags[@]}" \
  2415      --service-account "${NODE_SERVICE_ACCOUNT}" \
  2416      --tags "${NODE_TAG}" \
  2417      "${accelerator_args[@]}" \
  2418      "${local_ssds[@]}" \
  2419      --region "${REGION}" \
  2420      ${network} \
  2421      "${preemptible_minions[@]}" \
  2422      $2 \
  2423      --metadata-from-file "$3" \
  2424      ${metadata_flag} >&2; then
  2425        if (( attempt > 5 )); then
  2426          echo -e "${color_red}Failed to create instance template ${template_name} ${color_norm}" >&2
  2427          exit 2
  2428        fi
  2429        echo -e "${color_yellow}Attempt ${attempt} failed to create instance template ${template_name}. Retrying.${color_norm}" >&2
  2430        attempt=$((attempt + 1))
  2431        sleep $((attempt * 5))
  2432
  2433        # In case the previous attempt failed with something like a
  2434        # Backend Error and left the entry laying around, delete it
  2435        # before we try again.
  2436        gcloud compute instance-templates delete "${template_name}" --project "${PROJECT}" &>/dev/null || true
  2437    else
  2438        break
  2439    fi
  2440  done
  2441}
  2442
  2443# Instantiate a kubernetes cluster
  2444#
  2445# Assumed vars
  2446#   KUBE_ROOT
  2447#   <Various vars set in config file>
  2448function kube-up() {
  2449  kube::util::ensure-temp-dir
  2450  detect-project
  2451
  2452  load-or-gen-kube-basicauth
  2453  load-or-gen-kube-bearertoken
  2454
  2455  # Make sure we have the tar files staged on Google Storage
  2456  find-release-tars
  2457  upload-tars
  2458
  2459  # ensure that environmental variables specifying number of migs to create
  2460  set_num_migs
  2461
  2462  if [[ ${KUBE_USE_EXISTING_MASTER:-} == "true" ]]; then
  2463    detect-master
  2464    parse-master-env
  2465    create-subnetworks
  2466    detect-subnetworks
  2467    # Windows nodes take longer to boot and setup so create them first.
  2468    create-windows-nodes
  2469    create-linux-nodes
  2470  elif [[ ${KUBE_REPLICATE_EXISTING_MASTER:-} == "true" ]]; then
  2471    detect-master
  2472    if  [[ "${MASTER_OS_DISTRIBUTION}" != "gci" && "${MASTER_OS_DISTRIBUTION}" != "ubuntu" ]]; then
  2473      echo "Master replication supported only for gci and ubuntu"
  2474      return 1
  2475    fi
  2476    if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
  2477      create-internal-loadbalancer
  2478    fi
  2479    create-loadbalancer
  2480    # If replication of master fails, we need to ensure that the replica is removed from etcd clusters.
  2481    if ! replicate-master; then
  2482      remove-replica-from-etcd 2379 true || true
  2483      remove-replica-from-etcd 4002 false || true
  2484    fi
  2485  else
  2486    check-existing
  2487    create-network
  2488    create-subnetworks
  2489    detect-subnetworks
  2490    create-cloud-nat-router
  2491    write-cluster-location
  2492    write-cluster-name
  2493    create-autoscaler-config
  2494    create-master
  2495    create-nodes-firewall
  2496    create-nodes-template
  2497    if [[ "${KUBE_CREATE_NODES}" == "true" ]]; then
  2498      # Windows nodes take longer to boot and setup so create them first.
  2499      create-windows-nodes
  2500      create-linux-nodes
  2501    fi
  2502    check-cluster
  2503  fi
  2504}
  2505
  2506function check-existing() {
  2507  local running_in_terminal=false
  2508  # May be false if tty is not allocated (for example with ssh -T).
  2509  if [[ -t 1 ]]; then
  2510    running_in_terminal=true
  2511  fi
  2512
  2513  if [[ ${running_in_terminal} == "true" || ${KUBE_UP_AUTOMATIC_CLEANUP} == "true" ]]; then
  2514    if ! check-resources; then
  2515      local run_kube_down="n"
  2516      echo "${KUBE_RESOURCE_FOUND} found." >&2
  2517      # Get user input only if running in terminal.
  2518      if [[ ${running_in_terminal} == "true" && ${KUBE_UP_AUTOMATIC_CLEANUP} == "false" ]]; then
  2519        read -r -p "Would you like to shut down the old cluster (call kube-down)? [y/N] " run_kube_down
  2520      fi
  2521      if [[ ${run_kube_down} == "y" || ${run_kube_down} == "Y" || ${KUBE_UP_AUTOMATIC_CLEANUP} == "true" ]]; then
  2522        echo "... calling kube-down" >&2
  2523        kube-down
  2524      fi
  2525    fi
  2526  fi
  2527}
  2528
  2529function check-network-mode() {
  2530  local mode
  2531  mode=$(gcloud compute networks list --filter="name=('${NETWORK}')" --project "${NETWORK_PROJECT}" --format='value(x_gcloud_subnet_mode)' || true)
  2532  # The deprecated field uses lower case. Convert to upper case for consistency.
  2533  echo "$mode" | tr '[:lower:]' '[:upper:]'
  2534}
  2535
  2536function create-network() {
  2537  if ! gcloud compute networks --project "${NETWORK_PROJECT}" describe "${NETWORK}" &>/dev/null; then
  2538    # The network needs to be created synchronously or we have a race. The
  2539    # firewalls can be added concurrent with instance creation.
  2540    local network_mode="auto"
  2541    if [[ "${CREATE_CUSTOM_NETWORK:-}" == "true" ]]; then
  2542      network_mode="custom"
  2543    fi
  2544    echo "Creating new ${network_mode} network: ${NETWORK}"
  2545    gcloud compute networks create --project "${NETWORK_PROJECT}" "${NETWORK}" --subnet-mode="${network_mode}"
  2546  else
  2547    PREEXISTING_NETWORK=true
  2548    PREEXISTING_NETWORK_MODE="$(check-network-mode)"
  2549    echo "Found existing network ${NETWORK} in ${PREEXISTING_NETWORK_MODE} mode."
  2550  fi
  2551
  2552  if ! gcloud compute firewall-rules --project "${NETWORK_PROJECT}" describe "${CLUSTER_NAME}-default-internal-master" &>/dev/null; then
  2553    gcloud compute firewall-rules create "${CLUSTER_NAME}-default-internal-master" \
  2554      --project "${NETWORK_PROJECT}" \
  2555      --network "${NETWORK}" \
  2556      --source-ranges "10.0.0.0/8" \
  2557      --allow "tcp:1-2379,tcp:2382-65535,udp:1-65535,icmp" \
  2558      --target-tags "${MASTER_TAG}"&
  2559  fi
  2560
  2561  if ! gcloud compute firewall-rules --project "${NETWORK_PROJECT}" describe "${CLUSTER_NAME}-default-internal-node" &>/dev/null; then
  2562    gcloud compute firewall-rules create "${CLUSTER_NAME}-default-internal-node" \
  2563      --project "${NETWORK_PROJECT}" \
  2564      --network "${NETWORK}" \
  2565      --source-ranges "10.0.0.0/8" \
  2566      --allow "tcp:1-65535,udp:1-65535,icmp" \
  2567      --target-tags "${NODE_TAG}"&
  2568  fi
  2569
  2570  if ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NETWORK}-default-ssh" &>/dev/null; then
  2571    gcloud compute firewall-rules create "${NETWORK}-default-ssh" \
  2572      --project "${NETWORK_PROJECT}" \
  2573      --network "${NETWORK}" \
  2574      --source-ranges "0.0.0.0/0" \
  2575      --allow "tcp:22" &
  2576  fi
  2577
  2578  # Open up TCP 3389 to allow RDP connections.
  2579  if [[ ${NUM_WINDOWS_NODES} -gt 0 ]]; then
  2580    if ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NETWORK}-default-rdp" &>/dev/null; then
  2581      gcloud compute firewall-rules create "${NETWORK}-default-rdp" \
  2582        --project "${NETWORK_PROJECT}" \
  2583        --network "${NETWORK}" \
  2584        --source-ranges "0.0.0.0/0" \
  2585        --allow "tcp:3389" &
  2586    fi
  2587  fi
  2588
  2589  kube::util::wait-for-jobs || {
  2590    code=$?
  2591    echo -e "${color_red}Failed to create firewall rules.${color_norm}" >&2
  2592    exit $code
  2593  }
  2594}
  2595
  2596function expand-default-subnetwork() {
  2597  gcloud compute networks update "${NETWORK}" \
  2598    --switch-to-custom-subnet-mode \
  2599    --project "${NETWORK_PROJECT}" \
  2600    --quiet || true
  2601  gcloud compute networks subnets expand-ip-range "${NETWORK}" \
  2602    --region="${REGION}" \
  2603    --project "${NETWORK_PROJECT}" \
  2604    --prefix-length=19 \
  2605    --quiet
  2606}
  2607
  2608function create-subnetworks() {
  2609  case ${ENABLE_IP_ALIASES} in
  2610    true) echo "IP aliases are enabled. Creating subnetworks.";;
  2611    false)
  2612      echo "IP aliases are disabled."
  2613      if [[ "${ENABLE_BIG_CLUSTER_SUBNETS}" = "true" ]]; then
  2614        if [[  "${PREEXISTING_NETWORK}" != "true" ]]; then
  2615          expand-default-subnetwork
  2616        else
  2617          echo "${color_yellow}Using pre-existing network ${NETWORK}, subnets won't be expanded to /19!${color_norm}"
  2618        fi
  2619      elif [[ "${CREATE_CUSTOM_NETWORK:-}" == "true" && "${PREEXISTING_NETWORK}" != "true" ]]; then
  2620          gcloud compute networks subnets create "${SUBNETWORK}" --project "${NETWORK_PROJECT}" --region "${REGION}" --network "${NETWORK}" --range "${NODE_IP_RANGE}"
  2621      fi
  2622      return;;
  2623    *) echo "${color_red}Invalid argument to ENABLE_IP_ALIASES${color_norm}"
  2624       exit 1;;
  2625  esac
  2626
  2627  # Look for the alias subnet, it must exist and have a secondary
  2628  # range configured.
  2629  local subnet
  2630  subnet=$(gcloud compute networks subnets describe \
  2631    --project "${NETWORK_PROJECT}" \
  2632    --region "${REGION}" \
  2633    "${IP_ALIAS_SUBNETWORK}" 2>/dev/null || true)
  2634  if [[ -z "${subnet}" ]]; then
  2635    echo "Creating subnet ${NETWORK}:${IP_ALIAS_SUBNETWORK}"
  2636    gcloud compute networks subnets create \
  2637      "${IP_ALIAS_SUBNETWORK}" \
  2638      --description "Automatically generated subnet for ${INSTANCE_PREFIX} cluster. This will be removed on cluster teardown." \
  2639      --project "${NETWORK_PROJECT}" \
  2640      --network "${NETWORK}" \
  2641      --region "${REGION}" \
  2642      --range "${NODE_IP_RANGE}" \
  2643      --secondary-range "pods-default=${CLUSTER_IP_RANGE}" \
  2644      --secondary-range "services-default=${SERVICE_CLUSTER_IP_RANGE}"
  2645    echo "Created subnetwork ${IP_ALIAS_SUBNETWORK}"
  2646  else
  2647    if ! echo "${subnet}" | grep --quiet secondaryIpRanges; then
  2648      echo "${color_red}Subnet ${IP_ALIAS_SUBNETWORK} does not have a secondary range${color_norm}"
  2649      exit 1
  2650    fi
  2651  fi
  2652}
  2653
  2654# detect-subnetworks sets the SUBNETWORK var if not already set
  2655# Assumed vars:
  2656#   NETWORK
  2657#   REGION
  2658#   NETWORK_PROJECT
  2659#
  2660# Optional vars:
  2661#   SUBNETWORK
  2662#   IP_ALIAS_SUBNETWORK
  2663function detect-subnetworks() {
  2664  if [[ -n ${SUBNETWORK:-} ]]; then
  2665    echo "Using subnet ${SUBNETWORK}"
  2666    return 0
  2667  fi
  2668
  2669  if [[ -n ${IP_ALIAS_SUBNETWORK:-} ]]; then
  2670    SUBNETWORK=${IP_ALIAS_SUBNETWORK}
  2671    echo "Using IP Alias subnet ${SUBNETWORK}"
  2672    return 0
  2673  fi
  2674
  2675  SUBNETWORK=$(gcloud compute networks subnets list \
  2676    --network="${NETWORK}" \
  2677    --regions="${REGION}" \
  2678    --project="${NETWORK_PROJECT}" \
  2679    --limit=1 \
  2680    --format='value(name)' 2>/dev/null)
  2681
  2682  if [[ -n ${SUBNETWORK:-} ]]; then
  2683    echo "Found subnet for region ${REGION} in network ${NETWORK}: ${SUBNETWORK}"
  2684    return 0
  2685  fi
  2686
  2687  echo "${color_red}Could not find subnetwork with region ${REGION}, network ${NETWORK}, and project ${NETWORK_PROJECT}"
  2688}
  2689
  2690# Sets up Cloud NAT for the network.
  2691# Assumed vars:
  2692#   NETWORK_PROJECT
  2693#   REGION
  2694#   NETWORK
  2695function create-cloud-nat-router() {
  2696  if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
  2697    if gcloud compute routers describe "$NETWORK-nat-router" --project "$NETWORK_PROJECT" --region "$REGION" &>/dev/null; then
  2698      echo "Cloud nat already exists"
  2699      return 0
  2700    fi
  2701    gcloud compute routers create "$NETWORK-nat-router" \
  2702      --project "$NETWORK_PROJECT" \
  2703      --region "$REGION" \
  2704      --network "$NETWORK"
  2705    gcloud compute routers nats create "$NETWORK-nat-config" \
  2706      --project "$NETWORK_PROJECT" \
  2707      --router-region "$REGION" \
  2708      --router "$NETWORK-nat-router" \
  2709      --nat-primary-subnet-ip-ranges \
  2710      --auto-allocate-nat-external-ips \
  2711      ${GCE_PRIVATE_CLUSTER_PORTS_PER_VM:+--min-ports-per-vm ${GCE_PRIVATE_CLUSTER_PORTS_PER_VM}}
  2712  fi
  2713}
  2714
  2715function delete-all-firewall-rules() {
  2716  local -a fws
  2717  kube::util::read-array fws < <(gcloud compute firewall-rules list --project "${NETWORK_PROJECT}" --filter="network=${NETWORK}" --format="value(name)")
  2718  if (( "${#fws[@]}" > 0 )); then
  2719    echo "Deleting firewall rules remaining in network ${NETWORK}: ${fws[*]}"
  2720    delete-firewall-rules "${fws[@]}"
  2721  else
  2722    echo "No firewall rules in network ${NETWORK}"
  2723  fi
  2724}
  2725
  2726# Ignores firewall rule arguments that do not exist in NETWORK_PROJECT.
  2727function delete-firewall-rules() {
  2728  for fw in "$@"; do
  2729    if [[ -n $(gcloud compute firewall-rules --project "${NETWORK_PROJECT}" describe "${fw}" --format='value(name)' 2>/dev/null || true) ]]; then
  2730      gcloud compute firewall-rules delete --project "${NETWORK_PROJECT}" --quiet "${fw}" &
  2731    fi
  2732  done
  2733  kube::util::wait-for-jobs || {
  2734    echo -e "${color_red}Failed to delete firewall rules.${color_norm}" >&2
  2735  }
  2736}
  2737
  2738function delete-network() {
  2739  if [[ -n $(gcloud compute networks --project "${NETWORK_PROJECT}" describe "${NETWORK}" --format='value(name)' 2>/dev/null || true) ]]; then
  2740    if ! gcloud compute networks delete --project "${NETWORK_PROJECT}" --quiet "${NETWORK}"; then
  2741      echo "Failed to delete network '${NETWORK}'. Listing firewall-rules:"
  2742      gcloud compute firewall-rules --project "${NETWORK_PROJECT}" list --filter="network=${NETWORK}"
  2743      return 1
  2744    fi
  2745  fi
  2746}
  2747
  2748function delete-cloud-nat-router() {
  2749  if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
  2750    if [[ -n $(gcloud compute routers describe --project "${NETWORK_PROJECT}" --region "${REGION}" "${NETWORK}-nat-router" --format='value(name)' 2>/dev/null || true) ]]; then
  2751      echo "Deleting Cloud NAT router..."
  2752      gcloud compute routers delete --project "${NETWORK_PROJECT}" --region "${REGION}" --quiet "${NETWORK}-nat-router"
  2753    fi
  2754  fi
  2755}
  2756
  2757function delete-subnetworks() {
  2758  # If running in custom mode network we need to delete subnets manually.
  2759  mode="$(check-network-mode)"
  2760  if [[ "${mode}" == "CUSTOM" ]]; then
  2761    if [[ "${ENABLE_BIG_CLUSTER_SUBNETS}" = "true" ]]; then
  2762      echo "Deleting default subnets..."
  2763      # This value should be kept in sync with number of regions.
  2764      local parallelism=9
  2765      gcloud compute networks subnets list --network="${NETWORK}" --project "${NETWORK_PROJECT}" --format='value(region.basename())' | \
  2766        xargs -I {} -P ${parallelism} gcloud --quiet compute networks subnets delete "${NETWORK}" --project "${NETWORK_PROJECT}" --region="{}" || true
  2767    elif [[ "${CREATE_CUSTOM_NETWORK:-}" == "true" ]]; then
  2768      echo "Deleting custom subnet..."
  2769      gcloud --quiet compute networks subnets delete "${SUBNETWORK}" --project "${NETWORK_PROJECT}" --region="${REGION}" || true
  2770    fi
  2771    return
  2772  fi
  2773
  2774  # If we reached here, it means we're not using custom network.
  2775  # So the only thing we need to check is if IP-aliases was turned
  2776  # on and we created a subnet for it. If so, we should delete it.
  2777  if [[ ${ENABLE_IP_ALIASES:-} == "true" ]]; then
  2778    # Only delete the subnet if we created it (i.e it's not pre-existing).
  2779    if [[ -z "${KUBE_GCE_IP_ALIAS_SUBNETWORK:-}" ]]; then
  2780      echo "Removing auto-created subnet ${NETWORK}:${IP_ALIAS_SUBNETWORK}"
  2781      if [[ -n $(gcloud compute networks subnets describe \
  2782            --project "${NETWORK_PROJECT}" \
  2783            --region "${REGION}" \
  2784            "${IP_ALIAS_SUBNETWORK}" 2>/dev/null) ]]; then
  2785        gcloud --quiet compute networks subnets delete \
  2786          --project "${NETWORK_PROJECT}" \
  2787          --region "${REGION}" \
  2788          "${IP_ALIAS_SUBNETWORK}"
  2789      fi
  2790    fi
  2791  fi
  2792}
  2793
  2794# Generates SSL certificates for etcd cluster peer to peer communication. Uses cfssl program.
  2795#
  2796# Assumed vars:
  2797#   KUBE_TEMP: temporary directory
  2798#
  2799# Args:
  2800#  $1: host name
  2801#  $2: CA certificate
  2802#  $3: CA key
  2803#
  2804# If CA cert/key is empty, the function will also generate certs for CA.
  2805#
  2806# Vars set:
  2807#   ETCD_CA_KEY_BASE64
  2808#   ETCD_CA_CERT_BASE64
  2809#   ETCD_PEER_KEY_BASE64
  2810#   ETCD_PEER_CERT_BASE64
  2811#
  2812function create-etcd-certs {
  2813  local host=${1}
  2814  local ca_cert=${2:-}
  2815  local ca_key=${3:-}
  2816
  2817  GEN_ETCD_CA_CERT="${ca_cert}" GEN_ETCD_CA_KEY="${ca_key}" \
  2818    generate-etcd-cert "${KUBE_TEMP}/cfssl" "${host}" "peer" "peer"
  2819
  2820  pushd "${KUBE_TEMP}/cfssl"
  2821  ETCD_CA_KEY_BASE64=$(base64 "ca-key.pem" | tr -d '\r\n')
  2822  ETCD_CA_CERT_BASE64=$(gzip -c "ca.pem" | base64 | tr -d '\r\n')
  2823  ETCD_PEER_KEY_BASE64=$(base64 "peer-key.pem" | tr -d '\r\n')
  2824  ETCD_PEER_CERT_BASE64=$(gzip -c "peer.pem" | base64 | tr -d '\r\n')
  2825  popd
  2826}
  2827
  2828# Generates SSL certificates for etcd-client and kube-apiserver communication. Uses cfssl program.
  2829#
  2830# Assumed vars:
  2831#   KUBE_TEMP: temporary directory
  2832#
  2833# Args:
  2834#  $1: host server name
  2835#  $2: host client name
  2836#  $3: CA certificate
  2837#  $4: CA key
  2838#
  2839# If CA cert/key is empty, the function will also generate certs for CA.
  2840#
  2841# Vars set:
  2842#   ETCD_APISERVER_CA_KEY_BASE64
  2843#   ETCD_APISERVER_CA_CERT_BASE64
  2844#   ETCD_APISERVER_SERVER_KEY_BASE64
  2845#   ETCD_APISERVER_SERVER_CERT_BASE64
  2846#   ETCD_APISERVER_CLIENT_KEY_BASE64
  2847#   ETCD_APISERVER_CLIENT_CERT_BASE64
  2848#
  2849function create-etcd-apiserver-certs {
  2850  local hostServer=${1}
  2851  local hostClient=${2}
  2852  local etcd_apiserver_ca_cert=${3:-}
  2853  local etcd_apiserver_ca_key=${4:-}
  2854
  2855  GEN_ETCD_CA_CERT="${etcd_apiserver_ca_cert}" GEN_ETCD_CA_KEY="${etcd_apiserver_ca_key}" \
  2856    generate-etcd-cert "${KUBE_TEMP}/cfssl" "${hostServer}" "server" "etcd-apiserver-server"
  2857    generate-etcd-cert "${KUBE_TEMP}/cfssl" "${hostClient}" "client" "etcd-apiserver-client"
  2858
  2859  pushd "${KUBE_TEMP}/cfssl"
  2860  ETCD_APISERVER_CA_KEY_BASE64=$(base64 "ca-key.pem" | tr -d '\r\n')
  2861  ETCD_APISERVER_CA_CERT_BASE64=$(gzip -c "ca.pem" | base64 | tr -d '\r\n')
  2862  ETCD_APISERVER_SERVER_KEY_BASE64=$(base64 "etcd-apiserver-server-key.pem" | tr -d '\r\n')
  2863  ETCD_APISERVER_SERVER_CERT_BASE64=$(gzip -c "etcd-apiserver-server.pem" | base64 | tr -d '\r\n')
  2864  ETCD_APISERVER_CLIENT_KEY_BASE64=$(base64 "etcd-apiserver-client-key.pem" | tr -d '\r\n')
  2865  ETCD_APISERVER_CLIENT_CERT_BASE64=$(gzip -c "etcd-apiserver-client.pem" | base64 | tr -d '\r\n')
  2866  popd
  2867}
  2868
  2869
  2870function create-master() {
  2871  echo "Starting master and configuring firewalls"
  2872  gcloud compute firewall-rules create "${MASTER_NAME}-https" \
  2873    --project "${NETWORK_PROJECT}" \
  2874    --network "${NETWORK}" \
  2875    --target-tags "${MASTER_TAG}" \
  2876    --allow tcp:443 &
  2877
  2878  echo "Configuring firewall for apiserver konnectivity server"
  2879  if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
  2880    gcloud compute firewall-rules create "${MASTER_NAME}-konnectivity-server" \
  2881      --project "${NETWORK_PROJECT}" \
  2882      --network "${NETWORK}" \
  2883      --target-tags "${MASTER_TAG}" \
  2884      --allow tcp:8132 &
  2885  fi
  2886
  2887  # We have to make sure the disk is created before creating the master VM, so
  2888  # run this in the foreground.
  2889  gcloud compute disks create "${MASTER_NAME}-pd" \
  2890    --project "${PROJECT}" \
  2891    --zone "${ZONE}" \
  2892    --type "${MASTER_DISK_TYPE}" \
  2893    --size "${MASTER_DISK_SIZE}"
  2894
  2895  # Create rule for accessing and securing etcd servers.
  2896  if ! gcloud compute firewall-rules --project "${NETWORK_PROJECT}" describe "${MASTER_NAME}-etcd" &>/dev/null; then
  2897    gcloud compute firewall-rules create "${MASTER_NAME}-etcd" \
  2898      --project "${NETWORK_PROJECT}" \
  2899      --network "${NETWORK}" \
  2900      --source-tags "${MASTER_TAG}" \
  2901      --allow "tcp:2380,tcp:2381" \
  2902      --target-tags "${MASTER_TAG}" &
  2903  fi
  2904
  2905  # Generate a bearer token for this cluster. We push this separately
  2906  # from the other cluster variables so that the client (this
  2907  # computer) can forget it later. This should disappear with
  2908  # http://issue.k8s.io/3168
  2909  KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
  2910  if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
  2911    NODE_PROBLEM_DETECTOR_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
  2912  fi
  2913
  2914  # Reserve the master's IP so that it can later be transferred to another VM
  2915  # without disrupting the kubelets.
  2916  create-static-ip "${MASTER_NAME}-ip" "${REGION}"
  2917  MASTER_RESERVED_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
  2918    --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
  2919
  2920  if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" ]]; then
  2921    KUBELET_APISERVER="${MASTER_RESERVED_IP}"
  2922  fi
  2923
  2924  KUBERNETES_MASTER_NAME="${MASTER_RESERVED_IP}"
  2925  MASTER_ADVERTISE_ADDRESS="${MASTER_RESERVED_IP}"
  2926
  2927  MASTER_INTERNAL_IP=""
  2928  if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
  2929    gcloud compute addresses create "${MASTER_NAME}-internal-ip" --project "${PROJECT}" --region "$REGION" --subnet "$SUBNETWORK"
  2930    MASTER_INTERNAL_IP=$(gcloud compute addresses describe "${MASTER_NAME}-internal-ip" --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
  2931    echo "Master internal ip is: $MASTER_INTERNAL_IP"
  2932    KUBERNETES_MASTER_NAME="${MASTER_INTERNAL_IP}"
  2933    MASTER_ADVERTISE_ADDRESS="${MASTER_INTERNAL_IP}"
  2934  fi
  2935
  2936  create-certs "${MASTER_RESERVED_IP}" "${MASTER_INTERNAL_IP}"
  2937  create-etcd-certs "${MASTER_NAME}"
  2938  create-etcd-apiserver-certs "etcd-${MASTER_NAME}" "${MASTER_NAME}"
  2939
  2940  if [[ "$(get-num-nodes)" -ge "50" ]]; then
  2941    # We block on master creation for large clusters to avoid doing too much
  2942    # unnecessary work in case master start-up fails (like creation of nodes).
  2943    create-master-instance "${MASTER_RESERVED_IP}" "${MASTER_INTERNAL_IP}"
  2944  else
  2945    create-master-instance "${MASTER_RESERVED_IP}" "${MASTER_INTERNAL_IP}" &
  2946  fi
  2947
  2948}
  2949
  2950# Adds master replica to etcd cluster.
  2951#
  2952# Assumed vars:
  2953#   REPLICA_NAME
  2954#   PROJECT
  2955#   EXISTING_MASTER_NAME
  2956#   EXISTING_MASTER_ZONE
  2957#
  2958# $1: etcd client port
  2959# $2: etcd internal port
  2960# $3: whether etcd communication should use mtls
  2961# returns the result of ssh command which adds replica
  2962function add-replica-to-etcd() {
  2963  local -r client_port="${1}"
  2964  local -r internal_port="${2}"
  2965  local -r use_mtls="${3}"
  2966
  2967  TLSARG=""
  2968  PROTO="http://"
  2969  if [[ "${use_mtls}" == "true" ]]; then
  2970    # Keep in sync with ETCD_APISERVER_CA_CERT_PATH, ETCD_APISERVER_CLIENT_CERT_PATH and ETCD_APISERVER_CLIENT_KEY_PATH in configure-helper.sh.
  2971    TLSARG="--cacert /etc/srv/kubernetes/pki/etcd-apiserver-ca.crt --cert /etc/srv/kubernetes/pki/etcd-apiserver-client.crt --key /etc/srv/kubernetes/pki/etcd-apiserver-client.key"
  2972    PROTO="https://"
  2973  fi
  2974  run-gcloud-command "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}" "curl ${TLSARG} ${PROTO}127.0.0.1:${client_port}/v2/members -XPOST -H \"Content-Type: application/json\" -d '{\"peerURLs\":[\"https://${REPLICA_NAME}:${internal_port}\"]}' -s"
  2975  return $?
  2976}
  2977
  2978# Sets EXISTING_MASTER_NAME and EXISTING_MASTER_ZONE variables.
  2979#
  2980# Assumed vars:
  2981#   PROJECT
  2982#
  2983# NOTE: Must be in sync with get-replica-name-regexp
  2984function set-existing-master() {
  2985  local existing_master
  2986  existing_master=$(gcloud compute instances list \
  2987    --project "${PROJECT}" \
  2988    --filter "name ~ '$(get-replica-name-regexp)'" \
  2989    --format "value(name,zone)" | head -n1)
  2990  EXISTING_MASTER_NAME="$(echo "${existing_master}" | cut -f1)"
  2991  EXISTING_MASTER_ZONE="$(echo "${existing_master}" | cut -f2)"
  2992}
  2993
  2994function replicate-master() {
  2995  set-replica-name
  2996  set-existing-master
  2997
  2998  echo "Experimental: replicating existing master ${EXISTING_MASTER_ZONE}/${EXISTING_MASTER_NAME} as ${ZONE}/${REPLICA_NAME}"
  2999
  3000  # Before we do anything else, we should configure etcd to expect more replicas.
  3001  if ! add-replica-to-etcd 2379 2380 true; then
  3002    echo "Failed to add master replica to etcd cluster."
  3003    return 1
  3004  fi
  3005  if ! add-replica-to-etcd 4002 2381 false; then
  3006    echo "Failed to add master replica to etcd events cluster."
  3007    return 1
  3008  fi
  3009
  3010  # We have to make sure the disk is created before creating the master VM, so
  3011  # run this in the foreground.
  3012  gcloud compute disks create "${REPLICA_NAME}-pd" \
  3013    --project "${PROJECT}" \
  3014    --zone "${ZONE}" \
  3015    --type "${MASTER_DISK_TYPE}" \
  3016    --size "${MASTER_DISK_SIZE}"
  3017
  3018  local existing_master_replicas
  3019  existing_master_replicas="$(get-all-replica-names)"
  3020  replicate-master-instance "${EXISTING_MASTER_ZONE}" "${EXISTING_MASTER_NAME}" "${existing_master_replicas}"
  3021
  3022  # Add new replica to the load balancer.
  3023  gcloud compute target-pools add-instances "${MASTER_NAME}" \
  3024    --project "${PROJECT}" \
  3025    --zone "${ZONE}" \
  3026    --instances "${REPLICA_NAME}"
  3027
  3028  if [[ "${GCE_PRIVATE_CLUSTER:-}" == "true" ]]; then
  3029    add-to-internal-loadbalancer "${REPLICA_NAME}" "${ZONE}"
  3030  fi
  3031}
  3032
  3033# Detaches old and ataches new external IP to a VM.
  3034#
  3035# Arguments:
  3036#   $1 - VM name
  3037#   $2 - VM zone
  3038#   $3 - external static IP; if empty will use an ephemeral IP address.
  3039function attach-external-ip() {
  3040  local NAME=${1}
  3041  local ZONE=${2}
  3042  local IP_ADDR=${3:-}
  3043  local ACCESS_CONFIG_NAME
  3044  ACCESS_CONFIG_NAME=$(gcloud compute instances describe "${NAME}" \
  3045    --project "${PROJECT}" --zone "${ZONE}" \
  3046    --format="value(networkInterfaces[0].accessConfigs[0].name)")
  3047  gcloud compute instances delete-access-config "${NAME}" \
  3048    --project "${PROJECT}" --zone "${ZONE}" \
  3049    --access-config-name "${ACCESS_CONFIG_NAME}"
  3050  if [[ -z "${IP_ADDR}" ]]; then
  3051    gcloud compute instances add-access-config "${NAME}" \
  3052      --project "${PROJECT}" --zone "${ZONE}" \
  3053      --access-config-name "${ACCESS_CONFIG_NAME}"
  3054  else
  3055    gcloud compute instances add-access-config "${NAME}" \
  3056      --project "${PROJECT}" --zone "${ZONE}" \
  3057      --access-config-name "${ACCESS_CONFIG_NAME}" \
  3058      --address "${IP_ADDR}"
  3059  fi
  3060}
  3061
  3062# Creates load balancer in front of apiserver if it doesn't exists already. Assumes there's only one
  3063# existing master replica.
  3064#
  3065# Assumes:
  3066#   PROJECT
  3067#   MASTER_NAME
  3068#   ZONE
  3069#   REGION
  3070function create-loadbalancer() {
  3071  # Step 0: Return early if LB is already configured.
  3072  if gcloud compute forwarding-rules describe "${MASTER_NAME}" \
  3073    --project "${PROJECT}" --region "${REGION}" > /dev/null 2>&1; then
  3074    echo "Load balancer already exists"
  3075    return
  3076  fi
  3077
  3078  local EXISTING_MASTER_NAME
  3079  local EXISTING_MASTER_ZONE
  3080  EXISTING_MASTER_NAME="$(get-all-replica-names)"
  3081  EXISTING_MASTER_ZONE=$(gcloud compute instances list "${EXISTING_MASTER_NAME}" \
  3082    --project "${PROJECT}" --format='value(zone)')
  3083
  3084  echo "Creating load balancer in front of an already existing master in ${EXISTING_MASTER_ZONE}"
  3085
  3086  # Step 1: Detach master IP address and attach ephemeral address to the existing master
  3087  attach-external-ip "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}"
  3088
  3089  # Step 2: Create target pool.
  3090  gcloud compute target-pools create "${MASTER_NAME}" --project "${PROJECT}" --region "${REGION}"
  3091  # TODO: We should also add master instances with suffixes
  3092  gcloud compute target-pools add-instances "${MASTER_NAME}" --instances "${EXISTING_MASTER_NAME}" --project "${PROJECT}" --zone "${EXISTING_MASTER_ZONE}"
  3093
  3094  # Step 3: Create forwarding rule.
  3095  # TODO: This step can take up to 20 min. We need to speed this up...
  3096  gcloud compute forwarding-rules create "${MASTER_NAME}" \
  3097    --project "${PROJECT}" --region "${REGION}" \
  3098    --target-pool "${MASTER_NAME}" --address="${KUBE_MASTER_IP}" --ports=443
  3099
  3100  echo -n "Waiting for the load balancer configuration to propagate..."
  3101  local counter=0
  3102  until curl -k -m1 "https://${KUBE_MASTER_IP}" &> /dev/null; do
  3103    counter=$((counter+1))
  3104    echo -n .
  3105    if [[ ${counter} -ge 1800 ]]; then
  3106      echo -e "${color_red}TIMEOUT${color_norm}" >&2
  3107      echo -e "${color_red}Load balancer failed to initialize within ${counter} seconds.${color_norm}" >&2
  3108      exit 2
  3109    fi
  3110  done
  3111  echo "DONE"
  3112}
  3113
  3114
  3115# attach-internal-master-ip attach internal ip to existing master.
  3116#
  3117# Assumes:
  3118# * PROJECT
  3119function attach-internal-master-ip() {
  3120  local name="${1}"
  3121  local zone="${2}"
  3122  local ip="${3}"
  3123
  3124  local aliases
  3125  aliases=$(gcloud compute instances describe "${name}" --project "${PROJECT}" --zone "${zone}" --flatten='networkInterfaces[0].aliasIpRanges[]' --format='value[separator=':'](networkInterfaces[0].aliasIpRanges.subnetworkRangeName,networkInterfaces[0].aliasIpRanges.ipCidrRange)' | sed 's/^://' | paste -s -d';' -)
  3126  aliases="${aliases:+${aliases};}${ip}/32"
  3127  echo "Setting ${name}'s aliases to '${aliases}' (added ${ip})"
  3128  # Attach ${ip} to ${name}
  3129  gcloud compute instances network-interfaces update "${name}" --project "${PROJECT}" --zone "${zone}" --aliases="${aliases}"
  3130  gcloud compute instances add-metadata "${name}" --zone "${zone}" --metadata=kube-master-internal-ip="${ip}"
  3131  run-gcloud-command "${name}" "${zone}" 'sudo /bin/bash /home/kubernetes/bin/kube-master-internal-route.sh' || true
  3132  return $?
  3133}
  3134
  3135
  3136# detach-internal-master-ip detaches internal ip from existing master.
  3137#
  3138# Assumes:
  3139# * PROJECT
  3140function detach-internal-master-ip() {
  3141  local name="${1}"
  3142  local zone="${2}"
  3143  local ip="${3}"
  3144
  3145  local aliases
  3146  aliases=$(gcloud compute instances describe "${name}" --project "${PROJECT}" --zone "${zone}" --flatten='networkInterfaces[0].aliasIpRanges[]' --format='value[separator=':'](networkInterfaces[0].aliasIpRanges.subnetworkRangeName,networkInterfaces[0].aliasIpRanges.ipCidrRange)' | sed 's/^://' | grep -v "${ip}" | paste -s -d';' -)
  3147  echo "Setting ${name}'s aliases to '${aliases}' (removed ${ip})"
  3148  # Detach ${MASTER_NAME}-internal-ip from ${name}
  3149  gcloud compute instances network-interfaces update "${name}" --project "${PROJECT}" --zone "${zone}" --aliases="${aliases}"
  3150  gcloud compute instances remove-metadata "${name}" --zone "${zone}" --keys=kube-master-internal-ip
  3151  # We want `ip route` to be run in the cloud and not this host
  3152  run-gcloud-command "${name}" "${zone}" "sudo ip route del to local ${ip}/32 dev \$(ip route | grep default | while read -r _ _ _ _ dev _; do echo \$dev; done)" || true
  3153  return $?
  3154}
  3155
  3156# create-internal-loadbalancer creates an internal load balacer in front of existing master.
  3157#
  3158# Assumes:
  3159# * MASTER_NAME
  3160# * PROJECT
  3161# * REGION
  3162function create-internal-loadbalancer() {
  3163  if gcloud compute forwarding-rules describe "${MASTER_NAME}-internal" \
  3164    --project "${PROJECT}" --region "${REGION}" > /dev/null 2>&1; then
  3165    echo "Load balancer already exists"
  3166    return
  3167  fi
  3168
  3169  local EXISTING_MASTER_NAME
  3170  local EXISTING_MASTER_ZONE
  3171  EXISTING_MASTER_NAME="$(get-all-replica-names)"
  3172  EXISTING_MASTER_ZONE=$(gcloud compute instances list "${EXISTING_MASTER_NAME}" \
  3173    --project "${PROJECT}" --format='value(zone)')
  3174
  3175  echo "Detaching ${KUBE_MASTER_INTERNAL_IP} from ${EXISTING_MASTER_NAME}/${EXISTING_MASTER_ZONE}"
  3176  detach-internal-master-ip "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}" "${KUBE_MASTER_INTERNAL_IP}"
  3177
  3178  echo "Creating internal load balancer with IP: ${KUBE_MASTER_INTERNAL_IP}"
  3179  gcloud compute health-checks --project "${PROJECT}" create tcp "${MASTER_NAME}-hc" --port=443
  3180
  3181  gcloud compute backend-services create "${MASTER_NAME}" \
  3182    --project "${PROJECT}" \
  3183    --region "${REGION}" \
  3184    --protocol tcp \
  3185    --region "${REGION}" \
  3186    --load-balancing-scheme internal \
  3187    --health-checks "${MASTER_NAME}-hc"
  3188
  3189  gcloud compute forwarding-rules create "${MASTER_NAME}-internal" \
  3190    --project "${PROJECT}" \
  3191    --region "${REGION}" \
  3192    --load-balancing-scheme internal \
  3193    --network "${NETWORK}" \
  3194    --subnet "${SUBNETWORK}" \
  3195    --address "${KUBE_MASTER_INTERNAL_IP}" \
  3196    --ip-protocol TCP \
  3197    --ports 443 \
  3198    --backend-service "${MASTER_NAME}" \
  3199    --backend-service-region "${REGION}"
  3200
  3201  echo "Adding ${EXISTING_MASTER_NAME}/${EXISTING_MASTER_ZONE} to the load balancer"
  3202  add-to-internal-loadbalancer "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}"
  3203}
  3204
  3205# add-to-internal-loadbalancer adds an instance to ILB.
  3206# Assumes:
  3207# * MASTER_NAME
  3208# * PROJECT
  3209# * REGION
  3210function add-to-internal-loadbalancer() {
  3211  local name="${1}"
  3212  local zone="${2}"
  3213
  3214  gcloud compute instance-groups unmanaged create "${name}" --project "${PROJECT}" --zone "${zone}"
  3215  gcloud compute instance-groups unmanaged add-instances "${name}" --project "${PROJECT}" --zone "${zone}" --instances "${name}"
  3216  gcloud compute backend-services add-backend "${MASTER_NAME}" \
  3217    --project "${PROJECT}" \
  3218    --region "${REGION}" \
  3219    --instance-group "${name}" \
  3220    --instance-group-zone "${zone}"
  3221}
  3222
  3223# remove-from-internal-loadbalancer removes an instance from ILB.
  3224# Assumes:
  3225# * MASTER_NAME
  3226# * PROJECT
  3227# * REGION
  3228function remove-from-internal-loadbalancer() {
  3229  local name="${1}"
  3230  local zone="${2}"
  3231
  3232  if gcloud compute instance-groups unmanaged describe "${name}" --project "${PROJECT}" --zone "${zone}" &>/dev/null; then
  3233    gcloud compute backend-services remove-backend "${MASTER_NAME}" \
  3234          --project "${PROJECT}" \
  3235          --region "${REGION}" \
  3236          --instance-group "${name}" \
  3237          --instance-group-zone "${zone}"
  3238    gcloud compute instance-groups unmanaged delete "${name}" --project "${PROJECT}" --zone "${zone}" --quiet
  3239  fi
  3240}
  3241
  3242function delete-internal-loadbalancer() {
  3243  if gcloud compute forwarding-rules describe "${MASTER_NAME}-internal" --project "${PROJECT}" --region "${REGION}" &>/dev/null; then
  3244    gcloud compute forwarding-rules delete "${MASTER_NAME}-internal" --project "${PROJECT}" --region "${REGION}" --quiet
  3245  fi
  3246
  3247  if gcloud compute backend-services describe "${MASTER_NAME}" --project "${PROJECT}" --region "${REGION}" &>/dev/null; then
  3248    gcloud compute backend-services delete "${MASTER_NAME}" --project "${PROJECT}" --region "${REGION}" --quiet
  3249  fi
  3250  if gcloud compute health-checks describe "${MASTER_NAME}-gc" --project "${PROJECT}" &>/dev/null; then
  3251    gcloud compute health-checks delete "${MASTER_NAME}-gc" --project "${PROJECT}" --quiet
  3252  fi
  3253}
  3254
  3255function create-nodes-firewall() {
  3256  # Create a single firewall rule for all minions.
  3257  create-firewall-rule "${NODE_TAG}-all" "${CLUSTER_IP_RANGE}" "${NODE_TAG}" &
  3258
  3259  # Report logging choice (if any).
  3260  if [[ "${ENABLE_NODE_LOGGING-}" == "true" ]]; then
  3261    echo "+++ Logging using Fluentd to ${LOGGING_DESTINATION:-unknown}"
  3262  fi
  3263
  3264  # Wait for last batch of jobs
  3265  kube::util::wait-for-jobs || {
  3266    code=$?
  3267    echo -e "${color_red}Failed to create firewall rule.${color_norm}" >&2
  3268    exit $code
  3269  }
  3270}
  3271
  3272function get-scope-flags() {
  3273  local scope_flags=
  3274  if [[ -n "${NODE_SCOPES}" ]]; then
  3275    scope_flags="--scopes ${NODE_SCOPES}"
  3276  else
  3277    scope_flags="--no-scopes"
  3278  fi
  3279  echo "${scope_flags}"
  3280}
  3281
  3282function create-nodes-template() {
  3283  echo "Creating nodes."
  3284
  3285  local scope_flags
  3286  scope_flags=$(get-scope-flags)
  3287
  3288  write-linux-node-env
  3289  write-windows-node-env
  3290
  3291  # NOTE: these template names and their format must match
  3292  # create-[linux,windows]-nodes() as well as get-template()!
  3293  local linux_template_name="${NODE_INSTANCE_PREFIX}-template"
  3294  local windows_template_name="${WINDOWS_NODE_INSTANCE_PREFIX}-template"
  3295  create-linux-node-instance-template "$linux_template_name"
  3296  create-windows-node-instance-template "$windows_template_name" "${scope_flags[*]}"
  3297  if [[ -n "${ADDITIONAL_MACHINE_TYPE:-}" ]]; then
  3298    local linux_extra_template_name="${NODE_INSTANCE_PREFIX}-extra-template"
  3299    create-linux-node-instance-template "$linux_extra_template_name" "${ADDITIONAL_MACHINE_TYPE}"
  3300  fi
  3301}
  3302
  3303# Assumes:
  3304# - MAX_INSTANCES_PER_MIG
  3305# - NUM_NODES
  3306# - NUM_WINDOWS_NODES
  3307# exports:
  3308# - NUM_MIGS
  3309# - NUM_WINDOWS_MIGS
  3310function set_num_migs() {
  3311  local defaulted_max_instances_per_mig=${MAX_INSTANCES_PER_MIG:-1000}
  3312
  3313  if [[ ${defaulted_max_instances_per_mig} -le "0" ]]; then
  3314    echo "MAX_INSTANCES_PER_MIG cannot be negative. Assuming default 1000"
  3315    defaulted_max_instances_per_mig=1000
  3316  fi
  3317  export NUM_MIGS=$(((NUM_NODES + defaulted_max_instances_per_mig - 1) / defaulted_max_instances_per_mig))
  3318  export NUM_WINDOWS_MIGS=$(((NUM_WINDOWS_NODES + defaulted_max_instances_per_mig - 1) / defaulted_max_instances_per_mig))
  3319}
  3320
  3321# Assumes:
  3322# - NUM_MIGS
  3323# - NODE_INSTANCE_PREFIX
  3324# - NUM_NODES
  3325# - PROJECT
  3326# - ZONE
  3327function create-linux-nodes() {
  3328  local template_name="${NODE_INSTANCE_PREFIX}-template"
  3329  local extra_template_name="${NODE_INSTANCE_PREFIX}-extra-template"
  3330
  3331  local nodes="${NUM_NODES}"
  3332  if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then
  3333    echo "Creating a special node for heapster with machine-type ${HEAPSTER_MACHINE_TYPE}"
  3334    create-heapster-node
  3335    nodes=$(( nodes - 1 ))
  3336  fi
  3337
  3338  if [[ -n "${ADDITIONAL_MACHINE_TYPE:-}" && "${NUM_ADDITIONAL_NODES:-}" -gt 0 ]]; then
  3339    local num_additional="${NUM_ADDITIONAL_NODES}"
  3340    if [[ "${NUM_ADDITIONAL_NODES:-}" -gt "${nodes}" ]]; then
  3341      echo "Capping NUM_ADDITIONAL_NODES to ${nodes}"
  3342      num_additional="${nodes}"
  3343    fi
  3344    if [[ "${num_additional:-}" -gt 0 ]]; then
  3345      echo "Creating ${num_additional} special nodes with machine-type ${ADDITIONAL_MACHINE_TYPE}"
  3346      local extra_group_name="${NODE_INSTANCE_PREFIX}-extra"
  3347      gcloud compute instance-groups managed \
  3348          create "${extra_group_name}" \
  3349          --project "${PROJECT}" \
  3350          --zone "${ZONE}" \
  3351          --base-instance-name "${extra_group_name}" \
  3352          --size "${num_additional}" \
  3353          --template "${extra_template_name}" || true;
  3354      gcloud compute instance-groups managed wait-until --stable \
  3355          "${extra_group_name}" \
  3356          --zone "${ZONE}" \
  3357          --project "${PROJECT}" \
  3358          --timeout "${MIG_WAIT_UNTIL_STABLE_TIMEOUT}" || true
  3359      nodes=$(( nodes - num_additional ))
  3360    fi
  3361  fi
  3362
  3363  local instances_left=${nodes}
  3364
  3365  for ((i=1; i<=NUM_MIGS; i++)); do
  3366    local group_name="${NODE_INSTANCE_PREFIX}-group-$i"
  3367    if [[ $i -eq ${NUM_MIGS} ]]; then
  3368      # TODO: We don't add a suffix for the last group to keep backward compatibility when there's only one MIG.
  3369      # We should change it at some point, but note #18545 when changing this.
  3370      group_name="${NODE_INSTANCE_PREFIX}-group"
  3371    fi
  3372    # Spread the remaining number of nodes evenly
  3373    this_mig_size=$((instances_left / (NUM_MIGS - i + 1)))
  3374    instances_left=$((instances_left - this_mig_size))
  3375
  3376    # Run instance-groups creation in parallel.
  3377    {
  3378      gcloud compute instance-groups managed \
  3379          create "${group_name}" \
  3380          --project "${PROJECT}" \
  3381          --zone "${ZONE}" \
  3382          --base-instance-name "${group_name}" \
  3383          --size "${this_mig_size}" \
  3384          --template "${template_name}" || true;
  3385      gcloud compute instance-groups managed wait-until --stable \
  3386          "${group_name}" \
  3387          --zone "${ZONE}" \
  3388          --project "${PROJECT}" \
  3389          --timeout "${MIG_WAIT_UNTIL_STABLE_TIMEOUT}" || true
  3390    } &
  3391  done
  3392  wait
  3393}
  3394
  3395# Assumes:
  3396# - NUM_WINDOWS_MIGS
  3397# - WINDOWS_NODE_INSTANCE_PREFIX
  3398# - NUM_WINDOWS_NODES
  3399# - PROJECT
  3400# - ZONE
  3401function create-windows-nodes() {
  3402  local template_name="${WINDOWS_NODE_INSTANCE_PREFIX}-template"
  3403
  3404  local -r nodes="${NUM_WINDOWS_NODES}"
  3405  local instances_left=${nodes}
  3406
  3407  for ((i=1; i <= NUM_WINDOWS_MIGS; i++)); do
  3408    local group_name="${WINDOWS_NODE_INSTANCE_PREFIX}-group-$i"
  3409    if [[ $i -eq ${NUM_WINDOWS_MIGS} ]]; then
  3410      # TODO: We don't add a suffix for the last group to keep backward compatibility when there's only one MIG.
  3411      # We should change it at some point, but note #18545 when changing this.
  3412      group_name="${WINDOWS_NODE_INSTANCE_PREFIX}-group"
  3413    fi
  3414    # Spread the remaining number of nodes evenly
  3415    this_mig_size=$((instances_left / (NUM_WINDOWS_MIGS - i + 1)))
  3416    instances_left=$((instances_left - this_mig_size))
  3417
  3418    gcloud compute instance-groups managed \
  3419        create "${group_name}" \
  3420        --project "${PROJECT}" \
  3421        --zone "${ZONE}" \
  3422        --base-instance-name "${group_name}" \
  3423        --size "${this_mig_size}" \
  3424        --template "${template_name}" || true;
  3425    gcloud compute instance-groups managed wait-until --stable \
  3426        "${group_name}" \
  3427        --zone "${ZONE}" \
  3428        --project "${PROJECT}" \
  3429        --timeout "${MIG_WAIT_UNTIL_STABLE_TIMEOUT}" || true;
  3430  done
  3431}
  3432
  3433# Assumes:
  3434# - NODE_INSTANCE_PREFIX
  3435# - PROJECT
  3436# - NETWORK_PROJECT
  3437# - REGION
  3438# - ZONE
  3439# - HEAPSTER_MACHINE_TYPE
  3440# - NODE_DISK_TYPE
  3441# - NODE_DISK_SIZE
  3442# - NODE_IMAGE_PROJECT
  3443# - NODE_IMAGE
  3444# - NODE_SERVICE_ACCOUNT
  3445# - NODE_TAG
  3446# - NETWORK
  3447# - ENABLE_IP_ALIASES
  3448# - SUBNETWORK
  3449# - IP_ALIAS_SIZE
  3450function create-heapster-node() {
  3451  local gcloud="gcloud"
  3452
  3453  local network
  3454  network=$(make-gcloud-network-argument \
  3455      "${NETWORK_PROJECT}" \
  3456      "${REGION}" \
  3457      "${NETWORK}" \
  3458      "${SUBNETWORK:-}" \
  3459      "" \
  3460      "${ENABLE_IP_ALIASES:-}" \
  3461      "${IP_ALIAS_SIZE:-}")
  3462
  3463  # Deliberately word split ${network} and $(get-scope-flags)
  3464  # shellcheck disable=SC2086 disable=SC2046
  3465  ${gcloud} compute instances \
  3466      create "${NODE_INSTANCE_PREFIX}-heapster" \
  3467      --project "${PROJECT}" \
  3468      --zone "${ZONE}" \
  3469      --machine-type="${HEAPSTER_MACHINE_TYPE}" \
  3470      --boot-disk-type "${NODE_DISK_TYPE}" \
  3471      --boot-disk-size "${NODE_DISK_SIZE}" \
  3472      --image-project="${NODE_IMAGE_PROJECT}" \
  3473      --image "${NODE_IMAGE}" \
  3474      --service-account "${NODE_SERVICE_ACCOUNT}" \
  3475      --tags "${NODE_TAG}" \
  3476      ${network} \
  3477      $(get-scope-flags) \
  3478      --metadata-from-file "$(get-node-instance-metadata-from-file "heapster-kube-env")"
  3479}
  3480
  3481# Assumes:
  3482# - NUM_MIGS
  3483# - NODE_INSTANCE_PREFIX
  3484# - PROJECT
  3485# - ZONE
  3486# - AUTOSCALER_MAX_NODES
  3487# - AUTOSCALER_MIN_NODES
  3488# Exports
  3489# - AUTOSCALER_MIG_CONFIG
  3490function create-cluster-autoscaler-mig-config() {
  3491
  3492  # Each MIG must have at least one node, so the min number of nodes
  3493  # must be greater or equal to the number of migs.
  3494  if [[ ${AUTOSCALER_MIN_NODES} -lt 0 ]]; then
  3495    echo "AUTOSCALER_MIN_NODES must be greater or equal 0"
  3496    exit 2
  3497  fi
  3498
  3499  # Each MIG must have at least one node, so the min number of nodes
  3500  # must be greater or equal to the number of migs.
  3501  if [[ ${AUTOSCALER_MAX_NODES} -lt ${NUM_MIGS} ]]; then
  3502    echo "AUTOSCALER_MAX_NODES must be greater or equal ${NUM_MIGS}"
  3503    exit 2
  3504  fi
  3505  if [[ ${NUM_WINDOWS_MIGS} -gt 0 ]]; then
  3506    # TODO(pjh): implement Windows support in this function.
  3507    echo "Not implemented yet: autoscaler config for Windows MIGs"
  3508    exit 2
  3509  fi
  3510
  3511  # The code assumes that the migs were created with create-nodes
  3512  # function which tries to evenly spread nodes across the migs.
  3513  AUTOSCALER_MIG_CONFIG=""
  3514
  3515  local left_min=${AUTOSCALER_MIN_NODES}
  3516  local left_max=${AUTOSCALER_MAX_NODES}
  3517
  3518  for ((i=1; i <= NUM_MIGS; i++)); do
  3519    local group_name="${NODE_INSTANCE_PREFIX}-group-$i"
  3520    if [[ $i -eq ${NUM_MIGS} ]]; then
  3521      # TODO: We don't add a suffix for the last group to keep backward compatibility when there's only one MIG.
  3522      # We should change it at some point, but note #18545 when changing this.
  3523      group_name="${NODE_INSTANCE_PREFIX}-group"
  3524    fi
  3525
  3526    this_mig_min=$((left_min/(NUM_MIGS-i+1)))
  3527    this_mig_max=$((left_max/(NUM_MIGS-i+1)))
  3528    left_min=$((left_min-this_mig_min))
  3529    left_max=$((left_max-this_mig_max))
  3530
  3531    local mig_url="https://www.googleapis.com/compute/v1/projects/${PROJECT}/zones/${ZONE}/instanceGroups/${group_name}"
  3532    AUTOSCALER_MIG_CONFIG="${AUTOSCALER_MIG_CONFIG} --nodes=${this_mig_min}:${this_mig_max}:${mig_url}"
  3533  done
  3534
  3535  AUTOSCALER_MIG_CONFIG="${AUTOSCALER_MIG_CONFIG} --scale-down-enabled=${AUTOSCALER_ENABLE_SCALE_DOWN}"
  3536}
  3537
  3538# Assumes:
  3539# - NUM_MIGS
  3540# - NODE_INSTANCE_PREFIX
  3541# - PROJECT
  3542# - ZONE
  3543# - ENABLE_CLUSTER_AUTOSCALER
  3544# - AUTOSCALER_MAX_NODES
  3545# - AUTOSCALER_MIN_NODES
  3546function create-autoscaler-config() {
  3547  # Create autoscaler for nodes configuration if requested
  3548  if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then
  3549    create-cluster-autoscaler-mig-config
  3550    echo "Using autoscaler config: ${AUTOSCALER_MIG_CONFIG} ${AUTOSCALER_EXPANDER_CONFIG}"
  3551  fi
  3552}
  3553
  3554function check-cluster() {
  3555  detect-node-names
  3556  detect-master
  3557
  3558  echo "Waiting up to ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} seconds for cluster initialization."
  3559  echo
  3560  echo "  This will continually check to see if the API for kubernetes is reachable."
  3561  echo "  This may time out if there was some uncaught error during start up."
  3562  echo
  3563
  3564  # curl in mavericks is borked.
  3565  secure=""
  3566  if which sw_vers >& /dev/null; then
  3567    if [[ $(sw_vers | grep ProductVersion | awk '{print $2}') = "10.9."* ]]; then
  3568      secure="--insecure"
  3569    fi
  3570  fi
  3571
  3572  local start_time
  3573  local curl_out
  3574  start_time=$(date +%s)
  3575  curl_out=$(mktemp)
  3576  kube::util::trap_add "rm -f ${curl_out}" EXIT
  3577  until curl -vsS --cacert "${CERT_DIR}/pki/ca.crt" \
  3578          -H "Authorization: Bearer ${KUBE_BEARER_TOKEN}" \
  3579          ${secure} \
  3580          --max-time 5 --fail \
  3581          "https://${KUBE_MASTER_IP}/api/v1/pods?limit=100" > "${curl_out}" 2>&1; do
  3582      local elapsed
  3583      elapsed=$(($(date +%s) - start_time))
  3584      if [[ ${elapsed} -gt ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} ]]; then
  3585          echo -e "${color_red}Cluster failed to initialize within ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} seconds.${color_norm}" >&2
  3586          echo "Last output from querying API server follows:" >&2
  3587          echo "-----------------------------------------------------" >&2
  3588          cat "${curl_out}" >&2
  3589          echo "-----------------------------------------------------" >&2
  3590          exit 2
  3591      fi
  3592      printf "."
  3593      sleep 2
  3594  done
  3595
  3596  echo "Kubernetes cluster created."
  3597
  3598  export KUBE_CERT="${CERT_DIR}/pki/issued/kubecfg.crt"
  3599  export KUBE_KEY="${CERT_DIR}/pki/private/kubecfg.key"
  3600  export CA_CERT="${CERT_DIR}/pki/ca.crt"
  3601  export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}"
  3602  (
  3603   umask 077
  3604
  3605   # Update the user's kubeconfig to include credentials for this apiserver.
  3606   create-kubeconfig
  3607  )
  3608
  3609  # ensures KUBECONFIG is set
  3610  get-kubeconfig-basicauth
  3611
  3612  if [[ ${GCE_UPLOAD_KUBCONFIG_TO_MASTER_METADATA:-} == "true" ]]; then
  3613    gcloud compute instances add-metadata "${MASTER_NAME}" --project="${PROJECT}" --zone="${ZONE}"  --metadata-from-file="kubeconfig=${KUBECONFIG}" || true
  3614  fi
  3615
  3616  echo
  3617  echo -e "${color_green:-}Kubernetes cluster is running.  The master is running at:"
  3618  echo
  3619  echo -e "${color_yellow}  https://${KUBE_MASTER_IP}"
  3620  echo
  3621  echo -e "${color_green}The user name and password to use is located in ${KUBECONFIG}.${color_norm}"
  3622  echo
  3623
  3624}
  3625
  3626# Removes master replica from etcd cluster.
  3627#
  3628# Assumed vars:
  3629#   REPLICA_NAME
  3630#   PROJECT
  3631#   EXISTING_MASTER_NAME
  3632#   EXISTING_MASTER_ZONE
  3633#
  3634# $1: etcd client port
  3635# $2: whether etcd communication should use mtls
  3636# returns the result of ssh command which removes replica
  3637function remove-replica-from-etcd() {
  3638  local -r port="${1}"
  3639  local -r use_mtls="${2}"
  3640
  3641  TLSARG=""
  3642  PROTO="http://"
  3643  if [[ "${use_mtls}" == "true" ]]; then
  3644    # Keep in sync with ETCD_APISERVER_CA_CERT_PATH, ETCD_APISERVER_CLIENT_CERT_PATH and ETCD_APISERVER_CLIENT_KEY_PATH in configure-helper.sh.
  3645    TLSARG="--cacert /etc/srv/kubernetes/pki/etcd-apiserver-ca.crt --cert /etc/srv/kubernetes/pki/etcd-apiserver-client.crt --key /etc/srv/kubernetes/pki/etcd-apiserver-client.key"
  3646    PROTO="https://"
  3647  fi
  3648  [[ -n "${EXISTING_MASTER_NAME}" ]] || return
  3649  run-gcloud-command "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}" "curl -s ${TLSARG} ${PROTO}127.0.0.1:${port}/v2/members/\$(curl -s ${TLSARG} ${PROTO}127.0.0.1:${port}/v2/members -XGET | sed 's/{\\\"id/\n/g' | grep ${REPLICA_NAME}\\\" | cut -f 3 -d \\\") -XDELETE -L 2>/dev/null"
  3650  local -r res=$?
  3651  echo "Removing etcd replica, name: ${REPLICA_NAME}, port: ${port}, result: ${res}"
  3652  return "${res}"
  3653}
  3654
  3655# Delete a kubernetes cluster. This is called from test-teardown.
  3656#
  3657# Assumed vars:
  3658#   MASTER_NAME
  3659#   NODE_INSTANCE_PREFIX
  3660#   WINDOWS_NODE_INSTANCE_PREFIX
  3661#   ZONE
  3662# This function tears down cluster resources 10 at a time to avoid issuing too many
  3663# API calls and exceeding API quota. It is important to bring down the instances before bringing
  3664# down the firewall rules and routes.
  3665function kube-down() {
  3666  local -r batch=200
  3667
  3668  detect-project
  3669  detect-node-names # For INSTANCE_GROUPS and WINDOWS_INSTANCE_GROUPS
  3670
  3671  echo "Bringing down cluster"
  3672  set +e  # Do not stop on error
  3673
  3674  if [[ "${KUBE_DELETE_NODES:-}" != "false" ]]; then
  3675    # Get the name of the managed instance group template before we delete the
  3676    # managed instance group. (The name of the managed instance group template may
  3677    # change during a cluster upgrade.)
  3678    local templates
  3679    templates=$(get-template "${PROJECT}")
  3680
  3681    # Deliberately allow globbing, do not change unless a bug is found
  3682    # shellcheck disable=SC2206
  3683    local all_instance_groups=(${INSTANCE_GROUPS[@]:-} ${WINDOWS_INSTANCE_GROUPS[@]:-})
  3684    # Deliberately do not quote, do not change unless a bug is found
  3685    # shellcheck disable=SC2068
  3686    for group in ${all_instance_groups[@]:-}; do
  3687      {
  3688        if gcloud compute instance-groups managed describe "${group}" --project "${PROJECT}" --zone "${ZONE}" &>/dev/null; then
  3689          gcloud compute instance-groups managed delete \
  3690            --project "${PROJECT}" \
  3691            --quiet \
  3692            --zone "${ZONE}" \
  3693            "${group}"
  3694        fi
  3695      } &
  3696    done
  3697
  3698    # Wait for last batch of jobs
  3699    kube::util::wait-for-jobs || {
  3700      echo -e "Failed to delete instance group(s)." >&2
  3701    }
  3702
  3703    # Deliberately do not quote, do not change unless a bug is found
  3704    # shellcheck disable=SC2068
  3705    for template in ${templates[@]:-}; do
  3706      {
  3707        if gcloud compute instance-templates describe --project "${PROJECT}" "${template}" &>/dev/null; then
  3708          gcloud compute instance-templates delete \
  3709            --project "${PROJECT}" \
  3710            --quiet \
  3711            "${template}"
  3712        fi
  3713      } &
  3714    done
  3715
  3716    # Wait for last batch of jobs
  3717    kube::util::wait-for-jobs || {
  3718      echo -e "Failed to delete instance template(s)." >&2
  3719    }
  3720
  3721    # Delete the special heapster node (if it exists).
  3722    if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then
  3723      local -r heapster_machine_name="${NODE_INSTANCE_PREFIX}-heapster"
  3724      if gcloud compute instances describe "${heapster_machine_name}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
  3725        # Now we can safely delete the VM.
  3726        gcloud compute instances delete \
  3727          --project "${PROJECT}" \
  3728          --quiet \
  3729          --delete-disks all \
  3730          --zone "${ZONE}" \
  3731          "${heapster_machine_name}"
  3732      fi
  3733    fi
  3734  fi
  3735
  3736  local -r REPLICA_NAME="${KUBE_REPLICA_NAME:-$(get-replica-name)}"
  3737
  3738  set-existing-master
  3739
  3740  # Un-register the master replica from etcd and events etcd.
  3741  remove-replica-from-etcd 2379 true
  3742  remove-replica-from-etcd 4002 false
  3743
  3744  # Delete the master replica (if it exists).
  3745  if gcloud compute instances describe "${REPLICA_NAME}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
  3746    # If there is a load balancer in front of apiservers we need to first update its configuration.
  3747    if gcloud compute target-pools describe "${MASTER_NAME}" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
  3748      gcloud compute target-pools remove-instances "${MASTER_NAME}" \
  3749        --project "${PROJECT}" \
  3750        --zone "${ZONE}" \
  3751        --instances "${REPLICA_NAME}"
  3752    fi
  3753    # Detach replica from LB if needed.
  3754    if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
  3755      remove-from-internal-loadbalancer "${REPLICA_NAME}" "${ZONE}"
  3756    fi
  3757    # Now we can safely delete the VM.
  3758    gcloud compute instances delete \
  3759      --project "${PROJECT}" \
  3760      --quiet \
  3761      --delete-disks all \
  3762      --zone "${ZONE}" \
  3763      "${REPLICA_NAME}"
  3764  fi
  3765
  3766  # Delete the master replica pd (possibly leaked by kube-up if master create failed).
  3767  # TODO(jszczepkowski): remove also possibly leaked replicas' pds
  3768  local -r replica_pd="${REPLICA_NAME:-${MASTER_NAME}}-pd"
  3769  if gcloud compute disks describe "${replica_pd}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
  3770    gcloud compute disks delete \
  3771      --project "${PROJECT}" \
  3772      --quiet \
  3773      --zone "${ZONE}" \
  3774      "${replica_pd}"
  3775  fi
  3776
  3777  # Check if this are any remaining master replicas.
  3778  local REMAINING_MASTER_COUNT
  3779  REMAINING_MASTER_COUNT=$(gcloud compute instances list \
  3780    --project "${PROJECT}" \
  3781    --filter="name ~ '$(get-replica-name-regexp)'" \
  3782    --format "value(zone)" | wc -l)
  3783
  3784  # In the replicated scenario, if there's only a single master left, we should also delete load balancer in front of it.
  3785  if [[ "${REMAINING_MASTER_COUNT}" -eq 1 ]]; then
  3786    detect-master
  3787    local REMAINING_REPLICA_NAME
  3788    local REMAINING_REPLICA_ZONE
  3789    REMAINING_REPLICA_NAME="$(get-all-replica-names)"
  3790    REMAINING_REPLICA_ZONE=$(gcloud compute instances list "${REMAINING_REPLICA_NAME}" \
  3791      --project "${PROJECT}" --format='value(zone)')
  3792    if gcloud compute forwarding-rules describe "${MASTER_NAME}" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
  3793      gcloud compute forwarding-rules delete \
  3794        --project "${PROJECT}" \
  3795        --region "${REGION}" \
  3796        --quiet \
  3797        "${MASTER_NAME}"
  3798      attach-external-ip "${REMAINING_REPLICA_NAME}" "${REMAINING_REPLICA_ZONE}" "${KUBE_MASTER_IP}"
  3799      gcloud compute target-pools delete \
  3800        --project "${PROJECT}" \
  3801        --region "${REGION}" \
  3802        --quiet \
  3803        "${MASTER_NAME}"
  3804    fi
  3805
  3806    if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
  3807      remove-from-internal-loadbalancer "${REMAINING_REPLICA_NAME}" "${REMAINING_REPLICA_ZONE}"
  3808      delete-internal-loadbalancer
  3809      attach-internal-master-ip "${REMAINING_REPLICA_NAME}" "${REMAINING_REPLICA_ZONE}" "${KUBE_MASTER_INTERNAL_IP}"
  3810    fi
  3811  fi
  3812
  3813  # If there are no more remaining master replicas, we should delete all remaining network resources.
  3814  if [[ "${REMAINING_MASTER_COUNT}" -eq 0 ]]; then
  3815    # Delete firewall rule for the master, etcd servers, and nodes.
  3816    delete-firewall-rules "${MASTER_NAME}-https" "${MASTER_NAME}-etcd" "${NODE_TAG}-all" "${MASTER_NAME}-konnectivity-server"
  3817    # Delete the master's reserved IP
  3818    if gcloud compute addresses describe "${MASTER_NAME}-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
  3819      gcloud compute addresses delete \
  3820        --project "${PROJECT}" \
  3821        --region "${REGION}" \
  3822        --quiet \
  3823        "${MASTER_NAME}-ip"
  3824    fi
  3825
  3826    if gcloud compute addresses describe "${MASTER_NAME}-internal-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
  3827      gcloud compute addresses delete \
  3828        --project "${PROJECT}" \
  3829        --region "${REGION}" \
  3830        --quiet \
  3831        "${MASTER_NAME}-internal-ip"
  3832    fi
  3833  fi
  3834
  3835  if [[ "${KUBE_DELETE_NODES:-}" != "false" ]]; then
  3836    # Find out what minions are running.
  3837    local -a minions
  3838    kube::util::read-array minions < <(gcloud compute instances list \
  3839      --project "${PROJECT}" \
  3840      --filter="(name ~ '${NODE_INSTANCE_PREFIX}-.+' OR name ~ '${WINDOWS_NODE_INSTANCE_PREFIX}-.+') AND zone:(${ZONE})" \
  3841      --format='value(name)')
  3842    # If any minions are running, delete them in batches.
  3843    while (( "${#minions[@]}" > 0 )); do
  3844      echo Deleting nodes "${minions[*]::${batch}}"
  3845      gcloud compute instances delete \
  3846        --project "${PROJECT}" \
  3847        --quiet \
  3848        --delete-disks boot \
  3849        --zone "${ZONE}" \
  3850        "${minions[@]::${batch}}"
  3851      minions=( "${minions[@]:${batch}}" )
  3852    done
  3853  fi
  3854
  3855  # If there are no more remaining master replicas: delete routes, pd for influxdb and update kubeconfig
  3856  if [[ "${REMAINING_MASTER_COUNT}" -eq 0 ]]; then
  3857    # Delete routes.
  3858    local -a routes
  3859    # Clean up all routes w/ names like "<cluster-name>-<node-GUID>"
  3860    # e.g. "kubernetes-12345678-90ab-cdef-1234-567890abcdef". The name is
  3861    # determined by the node controller on the master.
  3862    # Note that this is currently a noop, as synchronously deleting the node MIG
  3863    # first allows the master to cleanup routes itself.
  3864    local TRUNCATED_PREFIX="${INSTANCE_PREFIX:0:26}"
  3865    kube::util::read-array routes < <(gcloud compute routes list --project "${NETWORK_PROJECT}" \
  3866      --filter="name ~ '${TRUNCATED_PREFIX}-.{8}-.{4}-.{4}-.{4}-.{12}'" \
  3867      --format='value(name)')
  3868    while (( "${#routes[@]}" > 0 )); do
  3869      echo Deleting routes "${routes[*]::${batch}}"
  3870      gcloud compute routes delete \
  3871        --project "${NETWORK_PROJECT}" \
  3872        --quiet \
  3873        "${routes[@]::${batch}}"
  3874      routes=( "${routes[@]:${batch}}" )
  3875    done
  3876
  3877    # Delete persistent disk for influx-db.
  3878    if gcloud compute disks describe "${INSTANCE_PREFIX}"-influxdb-pd --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
  3879      gcloud compute disks delete \
  3880        --project "${PROJECT}" \
  3881        --quiet \
  3882        --zone "${ZONE}" \
  3883        "${INSTANCE_PREFIX}"-influxdb-pd
  3884    fi
  3885
  3886    # Delete all remaining firewall rules and network.
  3887    delete-firewall-rules \
  3888      "${CLUSTER_NAME}-default-internal-master" \
  3889      "${CLUSTER_NAME}-default-internal-node"
  3890
  3891    if [[ "${KUBE_DELETE_NETWORK}" == "true" ]]; then
  3892      delete-firewall-rules \
  3893      "${NETWORK}-default-ssh" \
  3894      "${NETWORK}-default-rdp" \
  3895      "${NETWORK}-default-internal"  # Pre-1.5 clusters
  3896      delete-cloud-nat-router
  3897      # Delete all remaining firewall rules in the network.
  3898      delete-all-firewall-rules || true
  3899      delete-subnetworks || true
  3900      delete-network || true  # might fail if there are leaked resources that reference the network
  3901    fi
  3902
  3903    # If there are no more remaining master replicas, we should update kubeconfig.
  3904    export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}"
  3905    clear-kubeconfig
  3906  else
  3907  # If some master replicas remain: cluster has been changed, we need to re-validate it.
  3908    echo "... calling validate-cluster" >&2
  3909    # Override errexit
  3910    (validate-cluster) && validate_result="$?" || validate_result="$?"
  3911
  3912    # We have two different failure modes from validate cluster:
  3913    # - 1: fatal error - cluster won't be working correctly
  3914    # - 2: weak error - something went wrong, but cluster probably will be working correctly
  3915    # We just print an error message in case 2).
  3916    if [[ "${validate_result}" -eq 1 ]]; then
  3917      exit 1
  3918    elif [[ "${validate_result}" -eq 2 ]]; then
  3919      echo "...ignoring non-fatal errors in validate-cluster" >&2
  3920    fi
  3921  fi
  3922  set -e
  3923}
  3924
  3925# Prints name of one of the master replicas in the current zone. It will be either
  3926# just MASTER_NAME or MASTER_NAME with a suffix for a replica (see get-replica-name-regexp).
  3927#
  3928# Assumed vars:
  3929#   PROJECT
  3930#   ZONE
  3931#   MASTER_NAME
  3932#
  3933# NOTE: Must be in sync with get-replica-name-regexp and set-replica-name.
  3934function get-replica-name() {
  3935  # Ignore if gcloud compute fails and successfully echo any outcome
  3936  # shellcheck disable=SC2005
  3937  echo "$(gcloud compute instances list \
  3938    --project "${PROJECT}" \
  3939    --filter="name ~ '$(get-replica-name-regexp)' AND zone:(${ZONE})" \
  3940    --format "value(name)" | head -n1)"
  3941}
  3942
  3943# Prints comma-separated names of all of the master replicas in all zones.
  3944#
  3945# Assumed vars:
  3946#   PROJECT
  3947#   MASTER_NAME
  3948#
  3949# NOTE: Must be in sync with get-replica-name-regexp and set-replica-name.
  3950function get-all-replica-names() {
  3951  # Ignore if gcloud compute fails and successfully echo any outcome
  3952  # shellcheck disable=SC2005
  3953  echo "$(gcloud compute instances list \
  3954    --project "${PROJECT}" \
  3955    --filter="name ~ '$(get-replica-name-regexp)'" \
  3956    --format "value(name)" | tr "\n" "," | sed 's/,$//')"
  3957}
  3958
  3959# Prints the number of all of the master replicas in all zones.
  3960#
  3961# Assumed vars:
  3962#   MASTER_NAME
  3963function get-master-replicas-count() {
  3964  detect-project
  3965  local num_masters
  3966  num_masters=$(gcloud compute instances list \
  3967    --project "${PROJECT}" \
  3968    --filter="name ~ '$(get-replica-name-regexp)'" \
  3969    --format "value(zone)" | wc -l)
  3970  echo -n "${num_masters}"
  3971}
  3972
  3973# Prints regexp for full master machine name. In a cluster with replicated master,
  3974# VM names may either be MASTER_NAME or MASTER_NAME with a suffix for a replica.
  3975function get-replica-name-regexp() {
  3976  echo "^${MASTER_NAME}(-...)?"
  3977}
  3978
  3979# Sets REPLICA_NAME to a unique name for a master replica that will match
  3980# expected regexp (see get-replica-name-regexp).
  3981#
  3982# Assumed vars:
  3983#   PROJECT
  3984#   ZONE
  3985#   MASTER_NAME
  3986#
  3987# Sets:
  3988#   REPLICA_NAME
  3989function set-replica-name() {
  3990  local instances
  3991  instances=$(gcloud compute instances list \
  3992    --project "${PROJECT}" \
  3993    --filter="name ~ '$(get-replica-name-regexp)'" \
  3994    --format "value(name)")
  3995
  3996  suffix=""
  3997  while echo "${instances}" | grep "${suffix}" &>/dev/null; do
  3998    suffix="$(date | md5sum | head -c3)"
  3999  done
  4000  REPLICA_NAME="${MASTER_NAME}-${suffix}"
  4001}
  4002
  4003# Gets the instance templates in use by the cluster. It echos the template names
  4004# so that the function output can be used.
  4005# Assumed vars:
  4006#   NODE_INSTANCE_PREFIX
  4007#   WINDOWS_NODE_INSTANCE_PREFIX
  4008#
  4009# $1: project
  4010function get-template() {
  4011  local linux_filter="${NODE_INSTANCE_PREFIX}-(extra-)?template(-(${KUBE_RELEASE_VERSION_DASHED_REGEX}|${KUBE_CI_VERSION_DASHED_REGEX}))?"
  4012  local windows_filter="${WINDOWS_NODE_INSTANCE_PREFIX}-template(-(${KUBE_RELEASE_VERSION_DASHED_REGEX}|${KUBE_CI_VERSION_DASHED_REGEX}))?"
  4013
  4014  gcloud compute instance-templates list \
  4015    --filter="name ~ '${linux_filter}' OR name ~ '${windows_filter}'" \
  4016    --project="${1}" --format='value(name)'
  4017}
  4018
  4019# Checks if there are any present resources related kubernetes cluster.
  4020#
  4021# Assumed vars:
  4022#   MASTER_NAME
  4023#   NODE_INSTANCE_PREFIX
  4024#   WINDOWS_NODE_INSTANCE_PREFIX
  4025#   ZONE
  4026#   REGION
  4027# Vars set:
  4028#   KUBE_RESOURCE_FOUND
  4029function check-resources() {
  4030  detect-project
  4031  detect-node-names
  4032
  4033  echo "Looking for already existing resources"
  4034  KUBE_RESOURCE_FOUND=""
  4035
  4036  if [[ -n "${INSTANCE_GROUPS[*]:-}" ]]; then
  4037    KUBE_RESOURCE_FOUND="Managed instance groups ${INSTANCE_GROUPS[*]}"
  4038    return 1
  4039  fi
  4040  if [[ -n "${WINDOWS_INSTANCE_GROUPS[*]:-}" ]]; then
  4041    KUBE_RESOURCE_FOUND="Managed instance groups ${WINDOWS_INSTANCE_GROUPS[*]}"
  4042    return 1
  4043  fi
  4044
  4045  if gcloud compute instance-templates describe --project "${PROJECT}" "${NODE_INSTANCE_PREFIX}-template" &>/dev/null; then
  4046    KUBE_RESOURCE_FOUND="Instance template ${NODE_INSTANCE_PREFIX}-template"
  4047    return 1
  4048  fi
  4049  if gcloud compute instance-templates describe --project "${PROJECT}" "${WINDOWS_NODE_INSTANCE_PREFIX}-template" &>/dev/null; then
  4050    KUBE_RESOURCE_FOUND="Instance template ${WINDOWS_NODE_INSTANCE_PREFIX}-template"
  4051    return 1
  4052  fi
  4053
  4054  if gcloud compute instances describe --project "${PROJECT}" "${MASTER_NAME}" --zone "${ZONE}" &>/dev/null; then
  4055    KUBE_RESOURCE_FOUND="Kubernetes master ${MASTER_NAME}"
  4056    return 1
  4057  fi
  4058
  4059  if gcloud compute disks describe --project "${PROJECT}" "${MASTER_NAME}"-pd --zone "${ZONE}" &>/dev/null; then
  4060    KUBE_RESOURCE_FOUND="Persistent disk ${MASTER_NAME}-pd"
  4061    return 1
  4062  fi
  4063
  4064  # Find out what minions are running.
  4065  local -a minions
  4066  kube::util::read-array minions < <(gcloud compute instances list \
  4067    --project "${PROJECT}" \
  4068    --filter="(name ~ '${NODE_INSTANCE_PREFIX}-.+' OR name ~ '${WINDOWS_NODE_INSTANCE_PREFIX}-.+') AND zone:(${ZONE})" \
  4069    --format='value(name)')
  4070  if (( "${#minions[@]}" > 0 )); then
  4071    KUBE_RESOURCE_FOUND="${#minions[@]} matching ${NODE_INSTANCE_PREFIX}-.+ or ${WINDOWS_NODE_INSTANCE_PREFIX}-.+"
  4072    return 1
  4073  fi
  4074
  4075  if gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${MASTER_NAME}-https" &>/dev/null; then
  4076    KUBE_RESOURCE_FOUND="Firewall rules for ${MASTER_NAME}-https"
  4077    return 1
  4078  fi
  4079
  4080  if gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NODE_TAG}-all" &>/dev/null; then
  4081    KUBE_RESOURCE_FOUND="Firewall rules for ${MASTER_NAME}-all"
  4082    return 1
  4083  fi
  4084
  4085  local -a routes
  4086  kube::util::read-array routes < <(gcloud compute routes list --project "${NETWORK_PROJECT}" \
  4087    --filter="name ~ '${INSTANCE_PREFIX}-minion-.{4}'" --format='value(name)')
  4088  if (( "${#routes[@]}" > 0 )); then
  4089    KUBE_RESOURCE_FOUND="${#routes[@]} routes matching ${INSTANCE_PREFIX}-minion-.{4}"
  4090    return 1
  4091  fi
  4092
  4093  if gcloud compute addresses describe --project "${PROJECT}" "${MASTER_NAME}-ip" --region "${REGION}" &>/dev/null; then
  4094    KUBE_RESOURCE_FOUND="Master's reserved IP"
  4095    return 1
  4096  fi
  4097
  4098  # No resources found.
  4099  return 0
  4100}
  4101
  4102# -----------------------------------------------------------------------------
  4103# Cluster specific test helpers
  4104
  4105# Execute prior to running tests to build a release if required for env.
  4106#
  4107# Assumed Vars:
  4108#   KUBE_ROOT
  4109function test-build-release() {
  4110  # Make a release
  4111  "${KUBE_ROOT}/build/release.sh"
  4112}
  4113
  4114# Execute prior to running tests to initialize required structure.
  4115#
  4116# Assumed vars:
  4117#   Variables from config.sh
  4118function test-setup() {
  4119  # Detect the project into $PROJECT if it isn't set
  4120  detect-project
  4121
  4122  if [[ ${MULTIZONE:-} == "true" && -n ${E2E_ZONES:-} ]]; then
  4123    for KUBE_GCE_ZONE in ${E2E_ZONES}; do
  4124      KUBE_GCE_ZONE="${KUBE_GCE_ZONE}" KUBE_USE_EXISTING_MASTER="${KUBE_USE_EXISTING_MASTER:-}" "${KUBE_ROOT}/cluster/kube-up.sh"
  4125      KUBE_USE_EXISTING_MASTER="true" # For subsequent zones we use the existing master
  4126    done
  4127  else
  4128    "${KUBE_ROOT}/cluster/kube-up.sh"
  4129  fi
  4130
  4131  # Open up port 80 & 8080 so common containers on minions can be reached
  4132  # TODO(roberthbailey): Remove this once we are no longer relying on hostPorts.
  4133  local start
  4134  start=$(date +%s)
  4135  gcloud compute firewall-rules create \
  4136    --project "${NETWORK_PROJECT}" \
  4137    --target-tags "${NODE_TAG}" \
  4138    --allow tcp:80,tcp:8080 \
  4139    --network "${NETWORK}" \
  4140    "${NODE_TAG}-http-alt" 2> /dev/null || true
  4141  # As there is no simple way to wait longer for this operation we need to manually
  4142  # wait some additional time (20 minutes altogether).
  4143  while ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NODE_TAG}-http-alt" 2> /dev/null; do
  4144    if [[ $((start + 1200)) -lt $(date +%s) ]]; then
  4145      echo -e "${color_red:-}Failed to create firewall ${NODE_TAG}-http-alt in ${NETWORK_PROJECT}" >&2
  4146      exit 1
  4147    fi
  4148    sleep 5
  4149  done
  4150
  4151  # Open up the NodePort range
  4152  # TODO(justinsb): Move to main setup, if we decide whether we want to do this by default.
  4153  start=$(date +%s)
  4154  gcloud compute firewall-rules create \
  4155    --project "${NETWORK_PROJECT}" \
  4156    --target-tags "${NODE_TAG}" \
  4157    --allow tcp:30000-32767,udp:30000-32767 \
  4158    --network "${NETWORK}" \
  4159    "${NODE_TAG}-nodeports" 2> /dev/null || true
  4160  # As there is no simple way to wait longer for this operation we need to manually
  4161  # wait some additional time (20 minutes altogether).
  4162  while ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NODE_TAG}-nodeports" 2> /dev/null; do
  4163    if [[ $((start + 1200)) -lt $(date +%s) ]]; then
  4164      echo -e "${color_red}Failed to create firewall ${NODE_TAG}-nodeports in ${PROJECT}" >&2
  4165      exit 1
  4166    fi
  4167    sleep 5
  4168  done
  4169}
  4170
  4171# Execute after running tests to perform any required clean-up.
  4172function test-teardown() {
  4173  detect-project
  4174  echo "Shutting down test cluster in background."
  4175  delete-firewall-rules \
  4176    "${NODE_TAG}-http-alt" \
  4177    "${NODE_TAG}-nodeports"
  4178  if [[ ${MULTIZONE:-} == "true" && -n ${E2E_ZONES:-} ]]; then
  4179    local zones
  4180    read -r -a zones <<< "${E2E_ZONES}"
  4181    # tear them down in reverse order, finally tearing down the master too.
  4182    for ((zone_num=${#zones[@]}-1; zone_num>0; zone_num--)); do
  4183      KUBE_GCE_ZONE="${zones[zone_num]}" KUBE_USE_EXISTING_MASTER="true" "${KUBE_ROOT}/cluster/kube-down.sh"
  4184    done
  4185    KUBE_GCE_ZONE="${zones[0]}" KUBE_USE_EXISTING_MASTER="false" "${KUBE_ROOT}/cluster/kube-down.sh"
  4186  else
  4187    "${KUBE_ROOT}/cluster/kube-down.sh"
  4188  fi
  4189}
  4190
  4191# SSH to a node by name ($1) and run a command ($2).
  4192function ssh-to-node() {
  4193  local node="$1"
  4194  local cmd="$2"
  4195  # Loop until we can successfully ssh into the box
  4196  for (( i=0; i<5; i++)); do
  4197    if gcloud compute ssh --ssh-flag='-o LogLevel=quiet' --ssh-flag='-o ConnectTimeout=30' --project "${PROJECT}" --zone="${ZONE}" "${node}" --command 'echo test > /dev/null'; then
  4198      break
  4199    fi
  4200    sleep 5
  4201  done
  4202  # Then actually try the command.
  4203  gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --ssh-flag="-o ConnectTimeout=30" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "${cmd}"
  4204}
  4205
  4206# Perform preparations required to run e2e tests
  4207function prepare-e2e() {
  4208  detect-project
  4209}
  4210
  4211# Delete the image given by $1.
  4212function delete-image() {
  4213  gcloud container images delete --quiet "$1"
  4214}

View as plain text