1#!/usr/bin/env bash
2
3# Copyright 2015 The Kubernetes Authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17# Script that creates a Kubemark cluster for any given cloud provider.
18
19set -o errexit
20set -o nounset
21set -o pipefail
22
23TMP_ROOT="$(dirname "${BASH_SOURCE[@]}")/../.."
24KUBE_ROOT=$(readlink -e "${TMP_ROOT}" 2> /dev/null || perl -MCwd -e 'print Cwd::abs_path shift' "${TMP_ROOT}")
25
26source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh"
27source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh"
28source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh"
29source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh"
30
31if [[ -f "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/startup.sh" ]] ; then
32 source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/startup.sh"
33fi
34
35source "${KUBE_ROOT}/cluster/kubemark/util.sh"
36
37KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh"
38KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
39RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"
40LOCAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark"
41INTERNAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig-internal.kubemark"
42
43# Generate a random 6-digit alphanumeric tag for the kubemark image.
44# Used to uniquify image builds across different invocations of this script.
45KUBEMARK_IMAGE_TAG=$(head /dev/urandom | tr -dc 'a-z0-9' | fold -w 6 | head -n 1)
46
47# Create a docker image for hollow-node and upload it to the appropriate docker registry.
48function create-and-upload-hollow-node-image {
49 authenticate-docker
50 KUBEMARK_IMAGE_REGISTRY="${KUBEMARK_IMAGE_REGISTRY:-${CONTAINER_REGISTRY}/${PROJECT}}"
51 # Build+push the image through makefile.
52 build_cmd=("make" "${KUBEMARK_IMAGE_MAKE_TARGET}")
53 MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark"
54 KUBEMARK_BIN="$(kube::util::find-binary-for-platform kubemark linux/amd64)"
55 if [[ -z "${KUBEMARK_BIN}" ]]; then
56 echo 'Cannot find cmd/kubemark binary'
57 exit 1
58 fi
59 echo "Copying kubemark binary to ${MAKE_DIR}"
60 cp "${KUBEMARK_BIN}" "${MAKE_DIR}"
61 CURR_DIR=$(pwd)
62 cd "${MAKE_DIR}"
63 REGISTRY=${KUBEMARK_IMAGE_REGISTRY} IMAGE_TAG=${KUBEMARK_IMAGE_TAG} run-cmd-with-retries "${build_cmd[@]}"
64 rm kubemark
65 cd "$CURR_DIR"
66 echo "Created and uploaded the kubemark hollow-node image to docker registry."
67 # Cleanup the kubemark image after the script exits.
68 if [[ "${CLEANUP_KUBEMARK_IMAGE:-}" == "true" ]]; then
69 trap delete-kubemark-image EXIT
70 fi
71}
72
73function delete-kubemark-image {
74 delete-image "${KUBEMARK_IMAGE_REGISTRY}/kubemark:${KUBEMARK_IMAGE_TAG}"
75}
76
77# Generate secret and configMap for the hollow-node pods to work, prepare
78# manifests of the hollow-node and heapster replication controllers from
79# templates, and finally create these resources through kubectl.
80function create-kube-hollow-node-resources {
81 # Create kubemark namespace.
82 "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json"
83
84 # Create configmap for configuring hollow- kubelet, proxy and npd.
85 "${KUBECTL}" create configmap "node-configmap" --namespace="kubemark" \
86 --from-file=kernel.monitor="${RESOURCE_DIRECTORY}/kernel-monitor.json"
87
88 # Create secret for passing kubeconfigs to kubelet, kubeproxy and npd.
89 # It's bad that all component shares the same kubeconfig.
90 # TODO(https://github.com/kubernetes/kubernetes/issues/79883): Migrate all components to separate credentials.
91 "${KUBECTL}" create secret generic "kubeconfig" --type=Opaque --namespace="kubemark" \
92 --from-file=kubelet.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
93 --from-file=kubeproxy.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
94 --from-file=npd.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
95 --from-file=heapster.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
96 --from-file=cluster_autoscaler.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
97 --from-file=dns.kubeconfig="${HOLLOWNODE_KUBECONFIG}"
98
99 # Create addon pods.
100 # Heapster.
101 mkdir -p "${RESOURCE_DIRECTORY}/addons"
102 sed "s@{{MASTER_IP}}@${MASTER_IP}@g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json"
103 metrics_mem_per_node=4
104 metrics_mem=$((200 + metrics_mem_per_node*NUM_NODES))
105 sed -i'' -e "s@{{METRICS_MEM}}@${metrics_mem}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
106 metrics_cpu_per_node_numerator=${NUM_NODES}
107 metrics_cpu_per_node_denominator=2
108 metrics_cpu=$((80 + metrics_cpu_per_node_numerator / metrics_cpu_per_node_denominator))
109 sed -i'' -e "s@{{METRICS_CPU}}@${metrics_cpu}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
110 eventer_mem_per_node=500
111 eventer_mem=$((200 * 1024 + eventer_mem_per_node*NUM_NODES))
112 sed -i'' -e "s@{{EVENTER_MEM}}@${eventer_mem}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
113
114 # Cluster Autoscaler.
115 if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-}" == "true" ]]; then
116 echo "Setting up Cluster Autoscaler"
117 KUBEMARK_AUTOSCALER_MIG_NAME="${KUBEMARK_AUTOSCALER_MIG_NAME:-${NODE_INSTANCE_PREFIX}-group}"
118 KUBEMARK_AUTOSCALER_MIN_NODES="${KUBEMARK_AUTOSCALER_MIN_NODES:-0}"
119 KUBEMARK_AUTOSCALER_MAX_NODES="${KUBEMARK_AUTOSCALER_MAX_NODES:-10}"
120 NUM_NODES=${KUBEMARK_AUTOSCALER_MAX_NODES}
121 echo "Setting maximum cluster size to ${NUM_NODES}."
122 KUBEMARK_MIG_CONFIG="autoscaling.k8s.io/nodegroup: ${KUBEMARK_AUTOSCALER_MIG_NAME}"
123 sed "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/cluster-autoscaler_template.json" > "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
124 sed -i'' -e "s@{{kubemark_autoscaler_mig_name}}@${KUBEMARK_AUTOSCALER_MIG_NAME}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
125 sed -i'' -e "s@{{kubemark_autoscaler_min_nodes}}@${KUBEMARK_AUTOSCALER_MIN_NODES}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
126 sed -i'' -e "s@{{kubemark_autoscaler_max_nodes}}@${KUBEMARK_AUTOSCALER_MAX_NODES}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
127 fi
128
129 # Kube DNS.
130 if [[ "${ENABLE_KUBEMARK_KUBE_DNS:-}" == "true" ]]; then
131 echo "Setting up kube-dns"
132 sed "s@{{dns_domain}}@${KUBE_DNS_DOMAIN}@g" "${RESOURCE_DIRECTORY}/kube_dns_template.yaml" > "${RESOURCE_DIRECTORY}/addons/kube_dns.yaml"
133 fi
134
135 "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark"
136
137 # Create the replication controller for hollow-nodes.
138 # We allow to override the NUM_REPLICAS when running Cluster Autoscaler.
139 NUM_REPLICAS=${NUM_REPLICAS:-${NUM_NODES}}
140 sed "s@{{numreplicas}}@${NUM_REPLICAS}@g" "${RESOURCE_DIRECTORY}/hollow-node_template.yaml" > "${RESOURCE_DIRECTORY}/hollow-node.yaml"
141 proxy_cpu=20
142 if [ "${NUM_NODES}" -gt 1000 ]; then
143 proxy_cpu=50
144 fi
145 proxy_cpu=${KUBEMARK_HOLLOW_PROXY_MILLICPU:-$proxy_cpu}
146 proxy_mem_per_node=${KUBEMARK_HOLLOW_PROXY_MEM_PER_NODE_KB:-50}
147 proxy_mem=$((100 * 1024 + proxy_mem_per_node*NUM_NODES))
148 hollow_node_labels=${HOLLOW_NODE_LABELS:-$(calculate-node-labels)}
149 hollow_kubelet_params=$(eval "for param in ${HOLLOW_KUBELET_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done")
150 hollow_kubelet_params=${hollow_kubelet_params%?}
151 hollow_proxy_params=$(eval "for param in ${HOLLOW_PROXY_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done")
152 hollow_proxy_params=${hollow_proxy_params%?}
153
154 sed -i'' -e "s@{{hollow_kubelet_millicpu}}@${KUBEMARK_HOLLOW_KUBELET_MILLICPU:-40}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
155 sed -i'' -e "s@{{hollow_kubelet_mem_Ki}}@${KUBEMARK_HOLLOW_KUBELET_MEM_KB:-$((100*1024))}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
156 sed -i'' -e "s@{{hollow_proxy_millicpu}}@${proxy_cpu}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
157 sed -i'' -e "s@{{hollow_proxy_mem_Ki}}@${proxy_mem}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
158 sed -i'' -e "s@{{npd_millicpu}}@${KUBEMARK_NPD_MILLICPU:-20}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
159 sed -i'' -e "s@{{npd_mem_Ki}}@${KUBEMARK_NPD_MEM_KB:-$((20*1024))}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
160 sed -i'' -e "s@{{kubemark_image_registry}}@${KUBEMARK_IMAGE_REGISTRY}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
161 sed -i'' -e "s@{{kubemark_image_tag}}@${KUBEMARK_IMAGE_TAG}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
162 sed -i'' -e "s@{{master_ip}}@${MASTER_IP}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
163 sed -i'' -e "s@{{hollow_node_labels}}@${hollow_node_labels}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
164 sed -i'' -e "s@{{hollow_kubelet_params}}@${hollow_kubelet_params}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
165 sed -i'' -e "s@{{hollow_proxy_params}}@${hollow_proxy_params}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
166 sed -i'' -e "s@{{kubemark_mig_config}}@${KUBEMARK_MIG_CONFIG:-}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
167 "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark"
168
169 echo "Created secrets, configMaps, replication-controllers required for hollow-nodes."
170}
171
172# Wait until all hollow-nodes are running or there is a timeout.
173function wait-for-hollow-nodes-to-run-or-timeout {
174 echo -n "Waiting for all hollow-nodes to become Running"
175 start=$(date +%s)
176 nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true
177 ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1))
178
179 until [[ "${ready}" -ge "${NUM_REPLICAS}" ]]; do
180 echo -n "."
181 sleep 1
182 now=$(date +%s)
183 # Fail it if it already took more than 30 minutes.
184 if [ $((now - start)) -gt 1800 ]; then
185 echo ""
186 "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" describe node
187 # shellcheck disable=SC2154 # Color defined in sourced script
188 echo -e "${color_red} Timeout waiting for all hollow-nodes to become Running. ${color_norm}"
189 # Try listing nodes again - if it fails it means that API server is not responding
190 if "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node &> /dev/null; then
191 echo "Found only ${ready} ready hollow-nodes while waiting for ${NUM_NODES}."
192 else
193 echo "Got error while trying to list hollow-nodes. Probably API server is down."
194 fi
195 pods=$("${KUBECTL}" get pods -l name=hollow-node --namespace=kubemark) || true
196 running=$(($(echo "${pods}" | grep -c "Running")))
197 echo "${running} hollow-nodes are reported as 'Running'"
198 not_running=$(($(echo "${pods}" | grep -vc "Running") - 1))
199 echo "${not_running} hollow-nodes are reported as NOT 'Running'"
200 echo "${pods}" | grep -v Running
201 exit 1
202 fi
203 nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true
204 ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1))
205 done
206 # shellcheck disable=SC2154 # Color defined in sourced script
207 echo -e "${color_green} Done!${color_norm}"
208}
209
210############################### Main Function ########################################
211
212# Setup for hollow-nodes.
213function start-hollow-nodes {
214 # shellcheck disable=SC2154 # Color defined in sourced script
215 echo -e "${color_yellow}STARTING SETUP FOR HOLLOW-NODES${color_norm}"
216 create-and-upload-hollow-node-image
217 create-kube-hollow-node-resources
218 wait-for-hollow-nodes-to-run-or-timeout
219}
220
221# Annotates the node objects in the kubemark cluster to make their size
222# similar to regular nodes.
223# TODO(#90833): Replace this with image preloading from ClusterLoader to better
224# reflect the reality in kubemark tests.
225function resize-node-objects {
226 if [[ -z "$KUBEMARK_NODE_OBJECT_SIZE_BYTES" ]]; then
227 return 0
228 fi
229
230 annotation_size_bytes="${KUBEMARK_NODE_OBJECT_SIZE_BYTES}"
231 echo "Annotating node objects with ${annotation_size_bytes} byte label"
232 label=$( (< /dev/urandom tr -dc 'a-zA-Z0-9' | fold -w "$annotation_size_bytes"; true) | head -n 1)
233 "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get nodes -o name \
234 | xargs -P50 -r -I% "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" annotate --overwrite % label="$label"
235 echo "Annotating node objects completed"
236}
237
238
239detect-project &> /dev/null
240create-kubemark-master
241
242if [ -f "${INTERNAL_KUBECONFIG}" ]; then
243 HOLLOWNODE_KUBECONFIG="${INTERNAL_KUBECONFIG}"
244else
245 HOLLOWNODE_KUBECONFIG="${LOCAL_KUBECONFIG}"
246fi
247
248MASTER_IP=$(grep server "${HOLLOWNODE_KUBECONFIG}" | awk -F "/" '{print $3}')
249
250start-hollow-nodes
251resize-node-objects
252
253echo ""
254echo "Master IP: ${MASTER_IP}"
255echo "Kubeconfig for kubemark master is written in ${LOCAL_KUBECONFIG}"
View as plain text