...

Text file src/k8s.io/kubernetes/cluster/gce/config-test.sh

Documentation: k8s.io/kubernetes/cluster/gce

     1#!/usr/bin/env bash
     2
     3# Copyright 2014 The Kubernetes Authors.
     4#
     5# Licensed under the Apache License, Version 2.0 (the "License");
     6# you may not use this file except in compliance with the License.
     7# You may obtain a copy of the License at
     8#
     9#     http://www.apache.org/licenses/LICENSE-2.0
    10#
    11# Unless required by applicable law or agreed to in writing, software
    12# distributed under the License is distributed on an "AS IS" BASIS,
    13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14# See the License for the specific language governing permissions and
    15# limitations under the License.
    16
    17# TODO(jbeda): Provide a way to override project
    18# gcloud multiplexing for shared GCE/GKE tests.
    19KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
    20source "${KUBE_ROOT}/cluster/gce/config-common.sh"
    21
    22# Specifying KUBE_GCE_API_ENDPOINT will override the default GCE Compute API endpoint (https://www.googleapis.com/compute/v1/).
    23# This endpoint has to be pointing to v1 api. For example, https://www.googleapis.com/compute/staging_v1/
    24export GCE_API_ENDPOINT=${KUBE_GCE_API_ENDPOINT:-}
    25ZONE=${KUBE_GCE_ZONE:-us-central1-b}
    26export REGION=${ZONE%-*}
    27RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false}
    28REGIONAL_KUBE_ADDONS=${REGIONAL_KUBE_ADDONS:-true}
    29NODE_SIZE=${NODE_SIZE:-e2-standard-2}
    30NUM_NODES=${NUM_NODES:-3}
    31NUM_WINDOWS_NODES=${NUM_WINDOWS_NODES:-0}
    32MASTER_SIZE=${MASTER_SIZE:-e2-standard-$(get-master-size)}
    33MASTER_MIN_CPU_ARCHITECTURE=${MASTER_MIN_CPU_ARCHITECTURE:-} # To allow choosing better architectures.
    34export MASTER_DISK_TYPE=pd-ssd
    35MASTER_DISK_SIZE=${MASTER_DISK_SIZE:-$(get-master-disk-size)}
    36MASTER_ROOT_DISK_SIZE=${MASTER_ROOT_DISK_SIZE:-$(get-master-root-disk-size)}
    37NODE_DISK_TYPE=${NODE_DISK_TYPE:-pd-standard}
    38NODE_DISK_SIZE=${NODE_DISK_SIZE:-100GB}
    39NODE_LOCAL_SSDS=${NODE_LOCAL_SSDS:-0}
    40NODE_LABELS=${KUBE_NODE_LABELS:-}
    41WINDOWS_NODE_LABELS=${WINDOWS_NODE_LABELS:-}
    42NODE_LOCAL_SSDS_EPHEMERAL=${NODE_LOCAL_SSDS_EPHEMERAL:-}
    43# Turning GRPC based Konnectivity testing on id advance of
    44# removing the SSHTunnel code.
    45export KUBE_ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE=true
    46export PREPARE_KONNECTIVITY_SERVICE="${KUBE_ENABLE_KONNECTIVITY_SERVICE:-true}"
    47export EGRESS_VIA_KONNECTIVITY="${KUBE_ENABLE_KONNECTIVITY_SERVICE:-true}"
    48export RUN_KONNECTIVITY_PODS="${KUBE_ENABLE_KONNECTIVITY_SERVICE:-true}"
    49export KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE="${KUBE_KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}"
    50
    51# KUBE_CREATE_NODES can be used to avoid creating nodes, while master will be sized for NUM_NODES nodes.
    52# Firewalls and node templates are still created.
    53KUBE_CREATE_NODES=${KUBE_CREATE_NODES:-true}
    54
    55# An extension to local SSDs allowing users to specify block/fs and SCSI/NVMe devices
    56# Format of this variable will be "#,scsi/nvme,block/fs" you can specify multiple
    57# configurations by separating them by a semi-colon ex. "2,scsi,fs;1,nvme,block"
    58# is a request for 2 SCSI formatted and mounted SSDs and 1 NVMe block device SSD.
    59NODE_LOCAL_SSDS_EXT=${NODE_LOCAL_SSDS_EXT:-}
    60NODE_ACCELERATORS=${NODE_ACCELERATORS:-''}
    61export REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-true}
    62export KUBE_APISERVER_REQUEST_TIMEOUT=300
    63# Increase initial delay for the apiserver liveness probe, to avoid prematurely tearing it down
    64KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC=${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC:-45}
    65# Also increase the initial delay for etcd just to be safe
    66ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC=${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC:-45}
    67PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false}
    68PREEMPTIBLE_MASTER=${PREEMPTIBLE_MASTER:-false}
    69KUBE_DELETE_NODES=${KUBE_DELETE_NODES:-true}
    70KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-true}
    71CREATE_CUSTOM_NETWORK=${CREATE_CUSTOM_NETWORK:-false}
    72MIG_WAIT_UNTIL_STABLE_TIMEOUT=${MIG_WAIT_UNTIL_STABLE_TIMEOUT:-1800}
    73
    74MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
    75NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
    76WINDOWS_NODE_OS_DISTRIBUTION=${WINDOWS_NODE_OS_DISTRIBUTION:-win2019}
    77
    78if [[ "${MASTER_OS_DISTRIBUTION}" = 'cos' ]]; then
    79  MASTER_OS_DISTRIBUTION='gci'
    80fi
    81
    82if [[ "${NODE_OS_DISTRIBUTION}" = 'cos' ]]; then
    83  NODE_OS_DISTRIBUTION='gci'
    84fi
    85
    86# GPUs supported in GCE do not have compatible drivers in Debian 7.
    87if [[ "${NODE_OS_DISTRIBUTION}" = 'debian' ]]; then
    88  NODE_ACCELERATORS=''
    89fi
    90
    91# To avoid failing large tests due to some flakes in starting nodes, allow
    92# for a small percentage of nodes to not start during cluster startup.
    93ALLOWED_NOTREADY_NODES=${ALLOWED_NOTREADY_NODES:-$(($(get-num-nodes) / 100))}
    94
    95# By default a cluster will be started with the master and nodes
    96# on Container-optimized OS (cos, previously known as gci). If
    97# you are updating the os image versions, update this variable.
    98# Also please update corresponding image for node e2e at:
    99# https://github.com/kubernetes/kubernetes/blob/master/test/e2e_node/jenkins/image-config.yaml
   100#
   101# By default, the latest image from the image family will be used unless an
   102# explicit image will be set.
   103GCI_VERSION=${KUBE_GCI_VERSION:-}
   104IMAGE_FAMILY=${KUBE_IMAGE_FAMILY:-cos-109-lts}
   105export MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-}
   106export MASTER_IMAGE_FAMILY=${KUBE_GCE_MASTER_IMAGE_FAMILY:-${IMAGE_FAMILY}}
   107export MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-cos-cloud}
   108export NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${GCI_VERSION}}
   109export NODE_IMAGE_FAMILY=${KUBE_GCE_NODE_IMAGE_FAMILY:-${IMAGE_FAMILY}}
   110export NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-cos-cloud}
   111export NODE_SERVICE_ACCOUNT=${KUBE_GCE_NODE_SERVICE_ACCOUNT:-default}
   112
   113export CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-unix:///run/containerd/containerd.sock}
   114export CONTAINER_RUNTIME_NAME=${KUBE_CONTAINER_RUNTIME_NAME:-containerd}
   115export LOAD_IMAGE_COMMAND=${KUBE_LOAD_IMAGE_COMMAND:-ctr -n=k8s.io images import}
   116export LOG_DUMP_SYSTEMD_SERVICES=${LOG_DUMP_SYSTEMD_SERVICES:-containerd}
   117export CONTAINER_RUNTIME_TEST_HANDLER="true"
   118
   119export GCI_DOCKER_VERSION=${KUBE_GCI_DOCKER_VERSION:-}
   120
   121# Ability to inject custom versions (Ubuntu OS images ONLY)
   122# if KUBE_UBUNTU_INSTALL_CONTAINERD_VERSION or KUBE_UBUNTU_INSTALL_RUNC_VERSION
   123# is set to empty then we do not override the version(s) and just
   124# use whatever is in the default installation of containerd package
   125export UBUNTU_INSTALL_CONTAINERD_VERSION=${KUBE_UBUNTU_INSTALL_CONTAINERD_VERSION:-}
   126export UBUNTU_INSTALL_RUNC_VERSION=${KUBE_UBUNTU_INSTALL_RUNC_VERSION:-}
   127
   128# Ability to inject custom versions (COS images ONLY)
   129# if KUBE_COS_INSTALL_CONTAINERD_VERSION or KUBE_COS_INSTALL_RUNC_VERSION
   130# is set to empty then we do not override the version(s) and just
   131# use whatever is in the default installation of containerd package
   132export COS_INSTALL_CONTAINERD_VERSION=${KUBE_COS_INSTALL_CONTAINERD_VERSION:-}
   133export COS_INSTALL_RUNC_VERSION=${KUBE_COS_INSTALL_RUNC_VERSION:-}
   134
   135# MASTER_EXTRA_METADATA is the extra instance metadata on master instance separated by commas.
   136export MASTER_EXTRA_METADATA=${KUBE_MASTER_EXTRA_METADATA:-${KUBE_EXTRA_METADATA:-}}
   137# MASTER_EXTRA_METADATA is the extra instance metadata on node instance separated by commas.
   138export NODE_EXTRA_METADATA=${KUBE_NODE_EXTRA_METADATA:-${KUBE_EXTRA_METADATA:-}}
   139
   140NETWORK=${KUBE_GCE_NETWORK:-e2e-test-${USER}}
   141if [[ "${CREATE_CUSTOM_NETWORK}" = true ]]; then
   142  SUBNETWORK=${SUBNETWORK:-${NETWORK}-custom-subnet}
   143fi
   144INSTANCE_PREFIX=${KUBE_GCE_INSTANCE_PREFIX:-e2e-test-${USER}}
   145CLUSTER_NAME=${CLUSTER_NAME:-${INSTANCE_PREFIX}}
   146MASTER_NAME="${INSTANCE_PREFIX}-master"
   147export AGGREGATOR_MASTER_NAME="${INSTANCE_PREFIX}-aggregator"
   148export INITIAL_ETCD_CLUSTER=${MASTER_NAME}
   149export MASTER_TAG="${INSTANCE_PREFIX}-master"
   150export NODE_TAG="${INSTANCE_PREFIX}-minion"
   151
   152CLUSTER_IP_RANGE=${CLUSTER_IP_RANGE:-$(get-cluster-ip-range)}
   153MASTER_IP_RANGE=${MASTER_IP_RANGE:-10.246.0.0/24}
   154# NODE_IP_RANGE is used when ENABLE_IP_ALIASES=true or CREATE_CUSTOM_NETWORK=true.
   155# It is the primary range in the subnet and is the range used for node instance IPs.
   156NODE_IP_RANGE=$(get-node-ip-range)
   157export NODE_IP_RANGE
   158
   159export RUNTIME_CONFIG=${KUBE_RUNTIME_CONFIG:-}
   160
   161if [[ "${KUBE_FEATURE_GATES:-}" = 'AllAlpha=true' ]]; then
   162  RUNTIME_CONFIG=${KUBE_RUNTIME_CONFIG:-api/all=true}
   163fi
   164
   165# By default disable gkenetworkparamset controller in CCM
   166RUN_CCM_CONTROLLERS="${RUN_CCM_CONTROLLERS:-*,-gkenetworkparamset}"
   167
   168# Optional: set feature gates
   169# shellcheck disable=SC2034 # Variables sourced in other scripts.
   170FEATURE_GATES=${KUBE_FEATURE_GATES:-}
   171
   172TERMINATED_POD_GC_THRESHOLD=${TERMINATED_POD_GC_THRESHOLD:-100}
   173
   174# Extra docker options for nodes.
   175EXTRA_DOCKER_OPTS=${EXTRA_DOCKER_OPTS:-}
   176
   177# Enable the docker debug mode.
   178EXTRA_DOCKER_OPTS="${EXTRA_DOCKER_OPTS} --debug"
   179
   180export SERVICE_CLUSTER_IP_RANGE='10.0.0.0/16'  # formerly PORTAL_NET
   181
   182# When set to true, Docker Cache is enabled by default as part of the cluster bring up.
   183export ENABLE_DOCKER_REGISTRY_CACHE=true
   184
   185# Optional: Deploy a L7 loadbalancer controller to fulfill Ingress requests:
   186#   glbc           - CE L7 Load Balancer Controller
   187export ENABLE_L7_LOADBALANCING=${KUBE_ENABLE_L7_LOADBALANCING:-glbc}
   188
   189# Optional: Enable Metrics Server. Metrics Server should be enable everywhere,
   190# since it's a critical component, but in the first release we need a way to disable
   191# this in case of stability issues.
   192# TODO(piosz) remove this option once Metrics Server became a stable thing.
   193export ENABLE_METRICS_SERVER=${KUBE_ENABLE_METRICS_SERVER:-true}
   194
   195# Optional: Metadata agent to setup as part of the cluster bring up:
   196#   none        - No metadata agent
   197#   stackdriver - Stackdriver metadata agent
   198# Metadata agent is a daemon set that provides metadata of kubernetes objects
   199# running on the same node for exporting metrics and logs.
   200export ENABLE_METADATA_AGENT=${KUBE_ENABLE_METADATA_AGENT:-none}
   201
   202# One special node out of NUM_NODES would be created of this type if specified.
   203# Useful for scheduling heapster in large clusters with nodes of small size.
   204HEAPSTER_MACHINE_TYPE=${HEAPSTER_MACHINE_TYPE:-}
   205
   206# Optional: Additional nodes would be created if their type and number is specified.
   207# NUM_NODES would be lowered respectively.
   208# Useful for running cluster-level addons that needs more resources than would fit
   209# on small nodes, like network plugins.
   210NUM_ADDITIONAL_NODES=${NUM_ADDITIONAL_NODES:-}
   211ADDITIONAL_MACHINE_TYPE=${ADDITIONAL_MACHINE_TYPE:-}
   212
   213# Set etcd image (e.g. registry.k8s.io/etcd) and version (e.g. v3.5.1-0) if you need
   214# non-default version.
   215export ETCD_IMAGE=${TEST_ETCD_IMAGE:-}
   216export ETCD_DOCKER_REPOSITORY=${TEST_ETCD_DOCKER_REPOSITORY:-}
   217export ETCD_VERSION=${TEST_ETCD_VERSION:-}
   218
   219# Default Log level for all components in test clusters and variables to override it in specific components.
   220TEST_CLUSTER_LOG_LEVEL=${TEST_CLUSTER_LOG_LEVEL:---v=4}
   221KUBELET_TEST_LOG_LEVEL=${KUBELET_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}
   222DOCKER_TEST_LOG_LEVEL=${DOCKER_TEST_LOG_LEVEL:---log-level=info}
   223API_SERVER_TEST_LOG_LEVEL=${API_SERVER_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}
   224CONTROLLER_MANAGER_TEST_LOG_LEVEL=${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}
   225SCHEDULER_TEST_LOG_LEVEL=${SCHEDULER_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}
   226KUBEPROXY_TEST_LOG_LEVEL=${KUBEPROXY_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}
   227
   228VOLUME_PLUGIN_DIR=${VOLUME_PLUGIN_DIR:-/home/kubernetes/flexvolume}
   229
   230TEST_CLUSTER_DELETE_COLLECTION_WORKERS=${TEST_CLUSTER_DELETE_COLLECTION_WORKERS:---delete-collection-workers=1}
   231TEST_CLUSTER_MAX_REQUESTS_INFLIGHT=${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT:-}
   232TEST_CLUSTER_RESYNC_PERIOD=${TEST_CLUSTER_RESYNC_PERIOD:---min-resync-period=3m}
   233
   234# ContentType used by all components to communicate with apiserver.
   235TEST_CLUSTER_API_CONTENT_TYPE=${TEST_CLUSTER_API_CONTENT_TYPE:-}
   236
   237# Enable debug handlers (port forwarding, exec, container logs, etc.).
   238KUBELET_ENABLE_DEBUGGING_HANDLERS=${KUBELET_ENABLE_DEBUGGING_HANDLERS:-true}
   239MASTER_KUBELET_ENABLE_DEBUGGING_HANDLERS=${MASTER_KUBELET_ENABLE_DEBUGGING_HANDLERS:-${KUBELET_ENABLE_DEBUGGING_HANDLERS}}
   240
   241KUBELET_TEST_ARGS="${KUBELET_TEST_ARGS:-} --serialize-image-pulls=false ${TEST_CLUSTER_API_CONTENT_TYPE}"
   242if [[ "${NODE_OS_DISTRIBUTION}" = 'gci' ]] || [[ "${NODE_OS_DISTRIBUTION}" = 'ubuntu' ]] || [[ "${NODE_OS_DISTRIBUTION}" = 'custom' ]]; then
   243  NODE_KUBELET_TEST_ARGS="${NODE_KUBELET_TEST_ARGS:-} --kernel-memcg-notification=true"
   244fi
   245if [[ "${MASTER_OS_DISTRIBUTION}" = 'gci' ]] || [[ "${MASTER_OS_DISTRIBUTION}" = 'ubuntu' ]]; then
   246  MASTER_KUBELET_TEST_ARGS="${MASTER_KUBELET_TEST_ARGS:-} --kernel-memcg-notification=true"
   247fi
   248APISERVER_TEST_ARGS="${APISERVER_TEST_ARGS:-} --runtime-config=extensions/v1beta1,scheduling.k8s.io/v1alpha1 ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT}"
   249CONTROLLER_MANAGER_TEST_ARGS="${CONTROLLER_MANAGER_TEST_ARGS:-} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE}"
   250SCHEDULER_TEST_ARGS="${SCHEDULER_TEST_ARGS:-} ${TEST_CLUSTER_API_CONTENT_TYPE}"
   251KUBEPROXY_TEST_ARGS="${KUBEPROXY_TEST_ARGS:-} ${TEST_CLUSTER_API_CONTENT_TYPE}"
   252
   253export MASTER_NODE_LABELS=${KUBE_MASTER_NODE_LABELS:-}
   254# NON_MASTER_NODE_LABELS are labels will only be applied on non-master nodes.
   255NON_MASTER_NODE_LABELS=${KUBE_NON_MASTER_NODE_LABELS:-}
   256WINDOWS_NON_MASTER_NODE_LABELS=${WINDOWS_NON_MASTER_NODE_LABELS:-}
   257
   258if [[ "${PREEMPTIBLE_MASTER}" = 'true' ]]; then
   259  NODE_LABELS="${NODE_LABELS},cloud.google.com/gke-preemptible=true"
   260  WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS},cloud.google.com/gke-preemptible=true"
   261elif [[ "${PREEMPTIBLE_NODE}" = 'true' ]]; then
   262  NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS},cloud.google.com/gke-preemptible=true"
   263  WINDOWS_NON_MASTER_NODE_LABELS="${WINDOWS_NON_MASTER_NODE_LABELS},cloud.google.com/gke-preemptible=true"
   264fi
   265
   266# Optional: Enable netd.
   267ENABLE_NETD=${KUBE_ENABLE_NETD:-false}
   268export CUSTOM_NETD_YAML=${KUBE_CUSTOM_NETD_YAML:-}
   269export CUSTOM_CALICO_NODE_DAEMONSET_YAML=${KUBE_CUSTOM_CALICO_NODE_DAEMONSET_YAML:-}
   270export CUSTOM_TYPHA_DEPLOYMENT_YAML=${KUBE_CUSTOM_TYPHA_DEPLOYMENT_YAML:-}
   271
   272# To avoid running netd on a node that is not configured appropriately,
   273# label each Node so that the DaemonSet can run the Pods only on ready Nodes.
   274# Windows nodes do not support netd.
   275if [[ ${ENABLE_NETD:-} = 'true' ]]; then
   276  NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}cloud.google.com/gke-netd-ready=true"
   277fi
   278
   279export ENABLE_NODELOCAL_DNS=${KUBE_ENABLE_NODELOCAL_DNS:-false}
   280
   281# To avoid running Calico on a node that is not configured appropriately,
   282# label each Node so that the DaemonSet can run the Pods only on ready Nodes.
   283# Windows nodes do not support Calico.
   284if [[ ${NETWORK_POLICY_PROVIDER:-} = 'calico' ]]; then
   285  NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}projectcalico.org/ds-ready=true"
   286fi
   287
   288# Enable metadata concealment by firewalling pod traffic to the metadata server
   289# and run a proxy daemonset on nodes.
   290ENABLE_METADATA_CONCEALMENT=${ENABLE_METADATA_CONCEALMENT:-true} # true, false
   291METADATA_CONCEALMENT_NO_FIREWALL=${METADATA_CONCEALMENT_NO_FIREWALL:-false} # true, false
   292if [[ ${ENABLE_METADATA_CONCEALMENT:-} = 'true' ]]; then
   293  # Put the necessary label on the node so the daemonset gets scheduled.
   294  NODE_LABELS="${NODE_LABELS},cloud.google.com/metadata-proxy-ready=true"
   295  # Add to the provider custom variables.
   296  PROVIDER_VARS="${PROVIDER_VARS:-} ENABLE_METADATA_CONCEALMENT METADATA_CONCEALMENT_NO_FIREWALL"
   297fi
   298
   299# Optional: Enable node logging.
   300export ENABLE_NODE_LOGGING=${KUBE_ENABLE_NODE_LOGGING:-true}
   301export LOGGING_DESTINATION=${KUBE_LOGGING_DESTINATION:-gcp} # options: gcp
   302
   303# Optional: When set to true, Elasticsearch and Kibana will be setup as part of the cluster bring up.
   304export ENABLE_CLUSTER_LOGGING=${KUBE_ENABLE_CLUSTER_LOGGING:-true}
   305export ELASTICSEARCH_LOGGING_REPLICAS=1
   306
   307# Optional: Don't require https for registries in our local RFC1918 network
   308if [[ ${KUBE_ENABLE_INSECURE_REGISTRY:-false} = 'true' ]]; then
   309  EXTRA_DOCKER_OPTS="${EXTRA_DOCKER_OPTS} --insecure-registry 10.0.0.0/8"
   310fi
   311
   312if [[ -n "${NODE_ACCELERATORS}" ]]; then
   313    if [[ "${NODE_ACCELERATORS}" =~ .*type=([a-zA-Z0-9-]+).* ]]; then
   314        NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS},cloud.google.com/gke-accelerator=${BASH_REMATCH[1]}"
   315    fi
   316fi
   317
   318# List of the set of feature gates recognized by the GCP CCM
   319export CCM_FEATURE_GATES="APIPriorityAndFairness,APIResponseCompression,APIServerIdentity,APIServerTracing,AllAlpha,AllBeta,CustomResourceValidationExpressions,KMSv2,OpenAPIEnums,OpenAPIV3,ServerSideFieldValidation,StorageVersionAPI,StorageVersionHash"
   320
   321# Optional: Install cluster DNS.
   322# Set CLUSTER_DNS_CORE_DNS to 'false' to install kube-dns instead of CoreDNS.
   323CLUSTER_DNS_CORE_DNS=${CLUSTER_DNS_CORE_DNS:-true}
   324export ENABLE_CLUSTER_DNS=${KUBE_ENABLE_CLUSTER_DNS:-true}
   325export DNS_SERVER_IP='10.0.0.10'
   326export LOCAL_DNS_IP=${KUBE_LOCAL_DNS_IP:-169.254.20.10}
   327export DNS_DOMAIN='cluster.local'
   328export DNS_MEMORY_LIMIT=${KUBE_DNS_MEMORY_LIMIT:-170Mi}
   329
   330# Optional: Enable DNS horizontal autoscaler
   331export ENABLE_DNS_HORIZONTAL_AUTOSCALER=${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-true}
   332
   333# Optional: Install node problem detector.
   334#   none           - Not run node problem detector.
   335#   daemonset      - Run node problem detector as daemonset.
   336#   standalone     - Run node problem detector as standalone system daemon.
   337export ENABLE_NODE_PROBLEM_DETECTOR=${KUBE_ENABLE_NODE_PROBLEM_DETECTOR:-daemonset}
   338NODE_PROBLEM_DETECTOR_VERSION=${NODE_PROBLEM_DETECTOR_VERSION:-}
   339NODE_PROBLEM_DETECTOR_TAR_HASH=${NODE_PROBLEM_DETECTOR_TAR_HASH:-}
   340NODE_PROBLEM_DETECTOR_RELEASE_PATH=${NODE_PROBLEM_DETECTOR_RELEASE_PATH:-}
   341NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS=${NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS:-}
   342
   343CNI_HASH=${CNI_HASH:-}
   344CNI_TAR_PREFIX=${CNI_TAR_PREFIX:-cni-plugins-linux-amd64-}
   345CNI_STORAGE_URL_BASE=${CNI_STORAGE_URL_BASE:-https://storage.googleapis.com/k8s-artifacts-cni/release}
   346
   347# Optional: Create autoscaler for cluster's nodes.
   348export ENABLE_CLUSTER_AUTOSCALER=${KUBE_ENABLE_CLUSTER_AUTOSCALER:-false}
   349if [[ "${ENABLE_CLUSTER_AUTOSCALER}" = 'true' ]]; then
   350  export AUTOSCALER_MIN_NODES=${KUBE_AUTOSCALER_MIN_NODES:-}
   351  export AUTOSCALER_MAX_NODES=${KUBE_AUTOSCALER_MAX_NODES:-}
   352  export AUTOSCALER_ENABLE_SCALE_DOWN=${KUBE_AUTOSCALER_ENABLE_SCALE_DOWN:-false}
   353  export AUTOSCALER_EXPANDER_CONFIG=${KUBE_AUTOSCALER_EXPANDER_CONFIG:---expander=price}
   354fi
   355
   356# Optional: Enable allocation of pod IPs using IP aliases.
   357#
   358# BETA FEATURE.
   359#
   360# IP_ALIAS_SIZE is the size of the podCIDR allocated to a node.
   361# IP_ALIAS_SUBNETWORK is the subnetwork to allocate from. If empty, a
   362#   new subnetwork will be created for the cluster.
   363ENABLE_IP_ALIASES=${KUBE_GCE_ENABLE_IP_ALIASES:-true}
   364export NODE_IPAM_MODE=${KUBE_GCE_NODE_IPAM_MODE:-RangeAllocator}
   365if [ "${ENABLE_IP_ALIASES}" = true ]; then
   366  # Number of Pods that can run on this node.
   367  MAX_PODS_PER_NODE=${MAX_PODS_PER_NODE:-110}
   368  # Size of ranges allocated to each node.
   369  IP_ALIAS_SIZE="/$(get-alias-range-size "${MAX_PODS_PER_NODE}")"
   370  IP_ALIAS_SUBNETWORK=${KUBE_GCE_IP_ALIAS_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-default}
   371  # If we're using custom network, use the subnet we already create for it as the one for ip-alias.
   372  # Note that this means SUBNETWORK would override KUBE_GCE_IP_ALIAS_SUBNETWORK in case of custom network.
   373  if [[ "${CREATE_CUSTOM_NETWORK}" = true ]]; then
   374    IP_ALIAS_SUBNETWORK=${SUBNETWORK}
   375  fi
   376  export IP_ALIAS_SIZE
   377  export IP_ALIAS_SUBNETWORK
   378  # Reserve the services IP space to avoid being allocated for other GCP resources.
   379  export SERVICE_CLUSTER_IP_SUBNETWORK=${KUBE_GCE_SERVICE_CLUSTER_IP_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-services}
   380  NODE_IPAM_MODE=${KUBE_GCE_NODE_IPAM_MODE:-CloudAllocator}
   381  SECONDARY_RANGE_NAME=${SECONDARY_RANGE_NAME:-}
   382  # Add to the provider custom variables.
   383  PROVIDER_VARS="${PROVIDER_VARS:-} ENABLE_IP_ALIASES"
   384  PROVIDER_VARS="${PROVIDER_VARS:-} NODE_IPAM_MODE"
   385  PROVIDER_VARS="${PROVIDER_VARS:-} SECONDARY_RANGE_NAME"
   386else
   387  if [[ -n "${MAX_PODS_PER_NODE:-}" ]]; then
   388    # Should not have MAX_PODS_PER_NODE set for route-based clusters.
   389    echo -e "${color_red:-}Cannot set MAX_PODS_PER_NODE for route-based projects for ${PROJECT}." >&2
   390    exit 1
   391  fi
   392  if [[ "$(get-num-nodes)" -gt 100 ]]; then
   393    echo -e "${color_red:-}Cannot create cluster with more than 100 nodes for route-based projects for ${PROJECT}." >&2
   394    exit 1
   395  fi
   396fi
   397
   398# Enable GCE Alpha features.
   399if [[ -n "${GCE_ALPHA_FEATURES:-}" ]]; then
   400  PROVIDER_VARS="${PROVIDER_VARS:-} GCE_ALPHA_FEATURES"
   401fi
   402
   403# Disable Docker live-restore.
   404if [[ -n "${DISABLE_DOCKER_LIVE_RESTORE:-}" ]]; then
   405  PROVIDER_VARS="${PROVIDER_VARS:-} DISABLE_DOCKER_LIVE_RESTORE"
   406fi
   407
   408# Override default GLBC image
   409if [[ -n "${GCE_GLBC_IMAGE:-}" ]]; then
   410  PROVIDER_VARS="${PROVIDER_VARS:-} GCE_GLBC_IMAGE"
   411fi
   412CUSTOM_INGRESS_YAML=${CUSTOM_INGRESS_YAML:-}
   413
   414if [[ -z "${KUBE_ADMISSION_CONTROL:-}" ]]; then
   415  ADMISSION_CONTROL='NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,StorageObjectInUseProtection,PersistentVolumeClaimResize,RuntimeClass'
   416  # ResourceQuota must come last, or a creation is recorded, but the pod may be forbidden.
   417  ADMISSION_CONTROL="${ADMISSION_CONTROL},MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
   418else
   419  ADMISSION_CONTROL=${KUBE_ADMISSION_CONTROL}
   420fi
   421
   422ENABLE_APISERVER_DYNAMIC_AUDIT=${ENABLE_APISERVER_DYNAMIC_AUDIT:-false}
   423
   424# Optional: if set to true kube-up will automatically check for existing resources and clean them up.
   425KUBE_UP_AUTOMATIC_CLEANUP=${KUBE_UP_AUTOMATIC_CLEANUP:-false}
   426
   427# Optional: setting it to true denotes this is a testing cluster,
   428# so that we can use pulled kubernetes binaries, even if binaries
   429# are pre-installed in the image. Note that currently this logic
   430# is only supported in trusty or GCI.
   431TEST_CLUSTER=${TEST_CLUSTER:-true}
   432
   433# Storage backend. 'etcd2' and 'etcd3' are supported.
   434STORAGE_BACKEND=${STORAGE_BACKEND:-}
   435# Storage media type: application/json and application/vnd.kubernetes.protobuf are supported.
   436STORAGE_MEDIA_TYPE=${STORAGE_MEDIA_TYPE:-}
   437
   438NETWORK_PROVIDER=${NETWORK_PROVIDER:-kubenet} # none, kubenet
   439
   440# Network Policy plugin specific settings.
   441NETWORK_POLICY_PROVIDER=${NETWORK_POLICY_PROVIDER:-none} # calico
   442
   443export NON_MASQUERADE_CIDR='0.0.0.0/0'
   444
   445# How should the kubelet configure hairpin mode?
   446HAIRPIN_MODE=${HAIRPIN_MODE:-hairpin-veth} # promiscuous-bridge, hairpin-veth, none
   447
   448# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
   449export E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
   450
   451# Evict pods whenever compute resource availability on the nodes gets below a threshold.
   452EVICTION_HARD=${EVICTION_HARD:-memory.available<250Mi,nodefs.available<10%,nodefs.inodesFree<5%}
   453
   454# Optional: custom scheduling algorithm
   455SCHEDULING_ALGORITHM_PROVIDER=${SCHEDULING_ALGORITHM_PROVIDER:-}
   456
   457# Optional: install a default StorageClass
   458ENABLE_DEFAULT_STORAGE_CLASS=${ENABLE_DEFAULT_STORAGE_CLASS:-false}
   459
   460# Optional: install volume snapshot CRDs
   461ENABLE_VOLUME_SNAPSHOTS=${ENABLE_VOLUME_SNAPSHOTS:-true}
   462
   463# Optional: Enable legacy ABAC policy that makes all service accounts superusers.
   464# Disabling this by default in tests ensures default RBAC policies are sufficient from 1.6+
   465# Upgrade test jobs that go from a version < 1.6 to a version >= 1.6 should override this to be true.
   466ENABLE_LEGACY_ABAC=${ENABLE_LEGACY_ABAC:-false} # true, false
   467
   468# Enable a simple "AdvancedAuditing" setup for testing.
   469ENABLE_APISERVER_ADVANCED_AUDIT=${ENABLE_APISERVER_ADVANCED_AUDIT:-true} # true, false
   470ADVANCED_AUDIT_LOG_MODE=${ADVANCED_AUDIT_LOG_MODE:-batch} # batch, blocking
   471
   472ENABLE_BIG_CLUSTER_SUBNETS=${ENABLE_BIG_CLUSTER_SUBNETS:-false}
   473
   474# Optional: Enable log rotation for k8s services
   475ENABLE_LOGROTATE_FILES="${ENABLE_LOGROTATE_FILES:-true}"
   476PROVIDER_VARS="${PROVIDER_VARS:-} ENABLE_LOGROTATE_FILES"
   477if [[ -n "${LOGROTATE_FILES_MAX_COUNT:-}" ]]; then
   478  PROVIDER_VARS="${PROVIDER_VARS:-} LOGROTATE_FILES_MAX_COUNT"
   479fi
   480if [[ -n "${LOGROTATE_MAX_SIZE:-}" ]]; then
   481  PROVIDER_VARS="${PROVIDER_VARS:-} LOGROTATE_MAX_SIZE"
   482fi
   483
   484# Optional: Enable log rotation for pod logs
   485ENABLE_POD_LOG="${ENABLE_POD_LOG:-false}"
   486PROVIDER_VARS="${PROVIDER_VARS:-} ENABLE_POD_LOG"
   487
   488if [[ -n "${POD_LOG_MAX_FILE:-}" ]]; then
   489  PROVIDER_VARS="${PROVIDER_VARS:-} POD_LOG_MAX_FILE"
   490fi
   491
   492if [[ -n "${POD_LOG_MAX_SIZE:-}" ]]; then
   493  PROVIDER_VARS="${PROVIDER_VARS:-} POD_LOG_MAX_SIZE"
   494fi
   495
   496# Fluentd requirements
   497# YAML exists to trigger a configuration refresh when changes are made.
   498export FLUENTD_GCP_YAML_VERSION='v3.2.0'
   499FLUENTD_GCP_VERSION=${FLUENTD_GCP_VERSION:-1.6.17}
   500FLUENTD_GCP_MEMORY_LIMIT=${FLUENTD_GCP_MEMORY_LIMIT:-}
   501FLUENTD_GCP_CPU_REQUEST=${FLUENTD_GCP_CPU_REQUEST:-}
   502FLUENTD_GCP_MEMORY_REQUEST=${FLUENTD_GCP_MEMORY_REQUEST:-}
   503
   504# Heapster requirements
   505HEAPSTER_GCP_BASE_MEMORY=${HEAPSTER_GCP_BASE_MEMORY:-140Mi}
   506HEAPSTER_GCP_MEMORY_PER_NODE=${HEAPSTER_GCP_MEMORY_PER_NODE:-4}
   507HEAPSTER_GCP_BASE_CPU=${HEAPSTER_GCP_BASE_CPU:-80m}
   508HEAPSTER_GCP_CPU_PER_NODE=${HEAPSTER_GCP_CPU_PER_NODE:-0.5}
   509
   510# Default Stackdriver resources version exported by Fluentd-gcp addon
   511LOGGING_STACKDRIVER_RESOURCE_TYPES=${LOGGING_STACKDRIVER_RESOURCE_TYPES:-old}
   512
   513# Adding to PROVIDER_VARS, since this is GCP-specific.
   514PROVIDER_VARS="${PROVIDER_VARS:-} FLUENTD_GCP_YAML_VERSION FLUENTD_GCP_VERSION FLUENTD_GCP_MEMORY_LIMIT FLUENTD_GCP_CPU_REQUEST FLUENTD_GCP_MEMORY_REQUEST HEAPSTER_GCP_BASE_MEMORY HEAPSTER_GCP_MEMORY_PER_NODE HEAPSTER_GCP_BASE_CPU HEAPSTER_GCP_CPU_PER_NODE LOGGING_STACKDRIVER_RESOURCE_TYPES"
   515
   516# Fluentd configuration for node-journal
   517ENABLE_NODE_JOURNAL=${ENABLE_NODE_JOURNAL:-false}
   518
   519# prometheus-to-sd configuration
   520PROMETHEUS_TO_SD_ENDPOINT=${PROMETHEUS_TO_SD_ENDPOINT:-https://monitoring.googleapis.com/}
   521PROMETHEUS_TO_SD_PREFIX=${PROMETHEUS_TO_SD_PREFIX:-custom.googleapis.com}
   522ENABLE_PROMETHEUS_TO_SD=${ENABLE_PROMETHEUS_TO_SD:-true}
   523
   524# TODO(#51292): Make kube-proxy Daemonset default and remove the configuration here.
   525# Optional: [Experiment Only] Run kube-proxy as a DaemonSet if set to true, run as static pods otherwise.
   526KUBE_PROXY_DAEMONSET=${KUBE_PROXY_DAEMONSET:-false} # true, false
   527
   528# Control whether the startup scripts manage the lifecycle of kube-proxy
   529# When true, the startup scripts do not enable kube-proxy either as a daemonset addon or as a static pod
   530# regardless of the value of KUBE_PROXY_DAEMONSET.
   531# When false, the value of KUBE_PROXY_DAEMONSET controls whether kube-proxy comes up as a static pod or
   532# as an addon daemonset.
   533KUBE_PROXY_DISABLE="${KUBE_PROXY_DISABLE:-false}" # true, false
   534
   535# Optional: Change the kube-proxy implementation. Choices are [iptables, ipvs].
   536KUBE_PROXY_MODE=${KUBE_PROXY_MODE:-iptables}
   537
   538# Will be passed into the kube-proxy via `--detect-local-mode`
   539DETECT_LOCAL_MODE="${DETECT_LOCAL_MODE:-NodeCIDR}"
   540
   541# Optional: duration of cluster signed certificates.
   542CLUSTER_SIGNING_DURATION=${CLUSTER_SIGNING_DURATION:-}
   543
   544# Optional: enable certificate rotation of the kubelet certificates.
   545ROTATE_CERTIFICATES=${ROTATE_CERTIFICATES:-}
   546
   547# The number of services that are allowed to sync concurrently. Will be passed
   548# into kube-controller-manager via `--concurrent-service-syncs`
   549CONCURRENT_SERVICE_SYNCS=${CONCURRENT_SERVICE_SYNCS:-5}
   550
   551# The value kubernetes.default.svc.cluster.local is only usable for full
   552# OIDC discovery flows in Pods in the same cluster. For some providers
   553# with configurations that support non-traditional KSA authentication methods,
   554# this value may make sense, but if the expectation is traditional OIDC, don't
   555# use this value in production. If you do use it, the FQDN is preferred to
   556# kubernetes.default.svc, to avoid something outside the cluster attempting
   557# to resolve the partially qualified name.
   558export SERVICEACCOUNT_ISSUER='https://kubernetes.default.svc.cluster.local'
   559
   560# Taint Windows nodes by default to prevent Linux workloads from being
   561# scheduled onto them.
   562WINDOWS_NODE_TAINTS=${WINDOWS_NODE_TAINTS:-node.kubernetes.io/os=win1809:NoSchedule}
   563
   564# Whether to set up a private GCE cluster, i.e. a cluster where nodes have only private IPs.
   565export GCE_PRIVATE_CLUSTER=${KUBE_GCE_PRIVATE_CLUSTER:-false}
   566export GCE_PRIVATE_CLUSTER_PORTS_PER_VM=${KUBE_GCE_PRIVATE_CLUSTER_PORTS_PER_VM:-}
   567
   568export ETCD_LISTEN_CLIENT_IP=0.0.0.0
   569
   570export GCE_UPLOAD_KUBCONFIG_TO_MASTER_METADATA=true
   571
   572# Optoinal: Enable Windows CSI-Proxy
   573export ENABLE_CSI_PROXY="${ENABLE_CSI_PROXY:-true}"
   574
   575# KUBE_APISERVER_HEALTHCHECK_ON_HOST_IP decides whether
   576# kube-apiserver is healthchecked on host IP instead of 127.0.0.1.
   577export KUBE_APISERVER_HEALTHCHECK_ON_HOST_IP="${KUBE_APISERVER_HEALTHCHECK_ON_HOST_IP:-false}"
   578
   579# ETCD_PROGRESS_NOTIFY_INTERVAL defines the interval for etcd watch progress notify events.
   580export ETCD_PROGRESS_NOTIFY_INTERVAL="${ETCD_PROGRESS_NOTIFY_INTERVAL:-5s}"
   581
   582# Optional: Install Pigz on Windows.
   583# Pigz is a multi-core optimized version of unzip.exe.
   584# It improves container image pull performance since most time is spent
   585# unzipping the image layers to disk.
   586export WINDOWS_ENABLE_PIGZ="${WINDOWS_ENABLE_PIGZ:-true}"
   587
   588# Enable Windows DSR (Direct Server Return)
   589export WINDOWS_ENABLE_DSR="${WINDOWS_ENABLE_DSR:-false}"
   590
   591# Install Node Problem Detector (NPD) on Windows nodes.
   592# NPD analyzes the host for problems that can disrupt workloads.
   593export WINDOWS_ENABLE_NODE_PROBLEM_DETECTOR="${WINDOWS_ENABLE_NODE_PROBLEM_DETECTOR:-none}"
   594export WINDOWS_NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS="${WINDOWS_NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS:-}"
   595
   596# TLS_CIPHER_SUITES defines cipher suites allowed to be used by kube-apiserver.
   597# If this variable is unset or empty, kube-apiserver will allow its default set of cipher suites.
   598export TLS_CIPHER_SUITES=""
   599
   600# CLOUD_PROVIDER_FLAG defines the cloud-provider value presented to KCM, apiserver,
   601# and kubelet
   602export CLOUD_PROVIDER_FLAG="${CLOUD_PROVIDER_FLAG:-external}"
   603
   604# Don't run the node-ipam-controller on the KCM if cloud-provider external
   605if [[ "${CLOUD_PROVIDER_FLAG}" ==  "external" ]]; then
   606  RUN_CONTROLLERS="${RUN_CONTROLLERS:-*,-node-ipam-controller}"
   607fi
   608
   609# When ENABLE_AUTH_PROVIDER_GCP is set, following flags for out-of-tree credential provider for GCP
   610# are presented to kubelet:
   611# --image-credential-provider-config=${path-to-config}
   612# --image-credential-provider-bin-dir=${path-to-auth-provider-binary}
   613# Also, it is required that DisableKubeletCloudCredentialProviders and KubeletCredentialProviders
   614# feature gates are set to true for kubelet to use external credential provider.
   615export ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-true}"

View as plain text