1#!/usr/bin/env bash
2
3# Copyright 2017 The Kubernetes Authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../..
18
19source "${KUBE_ROOT}/test/kubemark/common/util.sh"
20
21# Wrapper for gcloud compute, running it $RETRIES times in case of failures.
22# Args:
23# $@: all stuff that goes after 'gcloud compute'
24function run-gcloud-compute-with-retries {
25 run-cmd-with-retries gcloud compute "$@"
26}
27
28function authenticate-docker {
29 echo "Configuring registry authentication"
30 mkdir -p "${HOME}/.docker"
31 gcloud beta auth configure-docker -q
32}
33
34function create-kubemark-master {
35 # We intentionally override env vars in subshell to preserve original values.
36 # shellcheck disable=SC2030,SC2031
37 (
38 # All calls to e2e-grow-cluster must share temp dir with initial e2e-up.sh.
39 kube::util::ensure-temp-dir
40 export KUBE_TEMP="${KUBE_TEMP}"
41
42 export KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark"
43 export KUBECONFIG_INTERNAL="${RESOURCE_DIRECTORY}/kubeconfig-internal.kubemark"
44 export CLUSTER_NAME="${CLUSTER_NAME}-kubemark"
45 export KUBE_CREATE_NODES=false
46 export KUBE_GCE_INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX}-kubemark"
47
48 # Quite tricky cidr setup: we set KUBE_GCE_ENABLE_IP_ALIASES=true to avoid creating
49 # cloud routes and RangeAllocator to assign cidrs by kube-controller-manager.
50 export KUBE_GCE_ENABLE_IP_ALIASES=true
51 export KUBE_GCE_NODE_IPAM_MODE=RangeAllocator
52
53 # Disable all addons. They are running outside of the kubemark cluster.
54 export KUBE_ENABLE_CLUSTER_AUTOSCALER=false
55 export KUBE_ENABLE_CLUSTER_DNS=false
56 export KUBE_ENABLE_NODE_LOGGING=false
57 export KUBE_ENABLE_METRICS_SERVER=false
58 export KUBE_ENABLE_L7_LOADBALANCING="none"
59
60 # Unset env variables set by kubetest for 'root cluster'. We need recompute them
61 # for kubemark master.
62 # TODO(mborsz): Figure out some better way to filter out such env variables than
63 # listing them here.
64 unset MASTER_SIZE MASTER_DISK_SIZE MASTER_ROOT_DISK_SIZE
65
66 # Set kubemark-specific overrides:
67 # for each defined env KUBEMARK_X=Y call export X=Y.
68 for var in ${!KUBEMARK_*}; do
69 dst_var=${var#KUBEMARK_}
70 val=${!var}
71 echo "Setting ${dst_var} to '${val}'"
72 export "${dst_var}"="${val}"
73 done
74
75 "${KUBE_ROOT}/hack/e2e-internal/e2e-up.sh"
76
77 if [[ "${KUBEMARK_HA_MASTER:-}" == "true" && -n "${KUBEMARK_MASTER_ADDITIONAL_ZONES:-}" ]]; then
78 for KUBE_GCE_ZONE in ${KUBEMARK_MASTER_ADDITIONAL_ZONES}; do
79 KUBE_GCE_ZONE="${KUBE_GCE_ZONE}" KUBE_REPLICATE_EXISTING_MASTER=true \
80 "${KUBE_ROOT}/hack/e2e-internal/e2e-grow-cluster.sh"
81 done
82 fi
83
84 # The e2e-up.sh script is not sourced, so we don't have access to variables that
85 # it sets. Instead, we read data which was written to the KUBE_TEMP directory.
86 # The cluster-location is either ZONE (say us-east1-a) or REGION (say us-east1).
87 # To get REGION from location, only first two parts are matched.
88 REGION=$(grep -o "^[a-z]*-[a-z0-9]*" "${KUBE_TEMP}"/cluster-location.txt)
89 MASTER_NAME="${KUBE_GCE_INSTANCE_PREFIX}"-master
90
91 if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
92 MASTER_INTERNAL_IP=$(gcloud compute addresses describe "${MASTER_NAME}-internal-ip" \
93 --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
94 fi
95 MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
96 --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
97
98 # If cluster uses private master IP, two kubeconfigs are created:
99 # - kubeconfig with public IP, which will be used to connect to the cluster
100 # from outside of the cluster network
101 # - kubeconfig with private IP (called internal kubeconfig), which will be
102 # used to create hollow nodes.
103 #
104 # Note that hollow nodes might use either of these kubeconfigs, but
105 # using internal one is better from performance and cost perspective, since
106 # traffic does not need to go through Cloud NAT.
107 if [[ -n "${MASTER_INTERNAL_IP:-}" ]]; then
108 echo "Writing internal kubeconfig to '${KUBECONFIG_INTERNAL}'"
109 ip_regexp=${MASTER_IP//./\\.} # escape ".", so that sed won't treat it as "any char"
110 sed "s/${ip_regexp}/${MASTER_INTERNAL_IP}/g" "${KUBECONFIG}" > "${KUBECONFIG_INTERNAL}"
111 fi
112 )
113}
114
115function delete-kubemark-master {
116 # We intentionally override env vars in subshell to preserve original values.
117 # shellcheck disable=SC2030,SC2031
118 (
119 export CLUSTER_NAME="${CLUSTER_NAME}-kubemark"
120 export KUBE_GCE_INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX}-kubemark"
121
122 export KUBE_DELETE_NETWORK=false
123
124 if [[ "${KUBEMARK_HA_MASTER:-}" == "true" && -n "${KUBEMARK_MASTER_ADDITIONAL_ZONES:-}" ]]; then
125 for KUBE_GCE_ZONE in ${KUBEMARK_MASTER_ADDITIONAL_ZONES}; do
126 KUBE_GCE_ZONE="${KUBE_GCE_ZONE}" KUBE_REPLICATE_EXISTING_MASTER=true \
127 "${KUBE_ROOT}/hack/e2e-internal/e2e-shrink-cluster.sh"
128 done
129 fi
130
131 "${KUBE_ROOT}/hack/e2e-internal/e2e-down.sh"
132 )
133}
134
135function calculate-node-labels {
136 echo "cloud.google.com/metadata-proxy-ready=true"
137}
View as plain text