1#!/usr/bin/env bash
2
3# Copyright 2014 The Kubernetes Authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17# This script runs e2e tests on Google Cloud Platform.
18# Usage: `hack/ginkgo-e2e.sh`.
19
20set -o errexit
21set -o nounset
22set -o pipefail
23
24KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
25source "${KUBE_ROOT}/cluster/common.sh"
26source "${KUBE_ROOT}/hack/lib/init.sh"
27
28# Find the ginkgo binary build as part of the release.
29ginkgo=$(kube::util::find-binary "ginkgo")
30e2e_test=$(kube::util::find-binary "e2e.test")
31
32# --- Setup some env vars.
33
34GINKGO_PARALLEL=${GINKGO_PARALLEL:-n} # set to 'y' to run tests in parallel
35CLOUD_CONFIG=${CLOUD_CONFIG:-""}
36
37# If 'y', Ginkgo's reporter will not use escape sequence to color output.
38#
39# Since Kubernetes 1.25, the default is to use colors only when connected to
40# a terminal. That is the right choice for all Prow jobs (Spyglass doesn't
41# render them properly).
42GINKGO_NO_COLOR=${GINKGO_NO_COLOR:-$(if [ -t 2 ]; then echo n; else echo y; fi)}
43
44# If 'y', will rerun failed tests once to give them a second chance.
45GINKGO_TOLERATE_FLAKES=${GINKGO_TOLERATE_FLAKES:-n}
46
47# If set, the command executed will be:
48# - `dlv exec` if set to "delve"
49# - `gdb` if set to "gdb"
50# NOTE: for this to work the e2e.test binary has to be compiled with
51# make DBG=1 WHAT=test/e2e/e2e.test
52E2E_TEST_DEBUG_TOOL=${E2E_TEST_DEBUG_TOOL:-}
53
54: "${KUBECTL:="${KUBE_ROOT}/cluster/kubectl.sh"}"
55: "${KUBE_CONFIG_FILE:="config-test.sh"}"
56
57export KUBECTL KUBE_CONFIG_FILE
58
59source "${KUBE_ROOT}/cluster/kube-util.sh"
60
61function detect-master-from-kubeconfig() {
62 export KUBECONFIG=${KUBECONFIG:-$DEFAULT_KUBECONFIG}
63
64 local cc
65 cc=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o jsonpath="{.current-context}")
66 if [[ -n "${KUBE_CONTEXT:-}" ]]; then
67 cc="${KUBE_CONTEXT}"
68 fi
69 local cluster
70 cluster=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o jsonpath="{.contexts[?(@.name == \"${cc}\")].context.cluster}")
71 KUBE_MASTER_URL=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o jsonpath="{.clusters[?(@.name == \"${cluster}\")].cluster.server}")
72}
73
74# ---- Do cloud-provider-specific setup
75if [[ -n "${KUBERNETES_CONFORMANCE_TEST:-}" ]]; then
76 echo "Conformance test: not doing test setup."
77 KUBERNETES_PROVIDER=${KUBERNETES_CONFORMANCE_PROVIDER:-"skeleton"}
78
79 detect-master-from-kubeconfig
80
81 auth_config=(
82 "--kubeconfig=${KUBECONFIG}"
83 )
84else
85 echo "Setting up for KUBERNETES_PROVIDER=\"${KUBERNETES_PROVIDER}\"."
86
87 prepare-e2e
88
89 detect-master >/dev/null
90
91 KUBE_MASTER_URL="${KUBE_MASTER_URL:-}"
92 if [[ -z "${KUBE_MASTER_URL:-}" && -n "${KUBE_MASTER_IP:-}" ]]; then
93 KUBE_MASTER_URL="https://${KUBE_MASTER_IP}"
94 fi
95
96 auth_config=(
97 "--kubeconfig=${KUBECONFIG:-$DEFAULT_KUBECONFIG}"
98 )
99fi
100
101if [[ -n "${NODE_INSTANCE_PREFIX:-}" ]]; then
102 NODE_INSTANCE_GROUP="${NODE_INSTANCE_PREFIX}-group"
103fi
104
105if [[ "${KUBERNETES_PROVIDER}" == "gce" ]]; then
106 set_num_migs
107 NODE_INSTANCE_GROUP=""
108 for ((i=1; i<=NUM_MIGS; i++)); do
109 if [[ ${i} == "${NUM_MIGS}" ]]; then
110 # We are assigning the same mig names as create-nodes function from cluster/gce/util.sh.
111 NODE_INSTANCE_GROUP="${NODE_INSTANCE_GROUP}${NODE_INSTANCE_PREFIX}-group"
112 else
113 NODE_INSTANCE_GROUP="${NODE_INSTANCE_GROUP}${NODE_INSTANCE_PREFIX}-group-${i},"
114 fi
115 done
116fi
117
118# TODO(kubernetes/test-infra#3330): Allow NODE_INSTANCE_GROUP to be
119# set before we get here, which eliminates any cluster/gke use if
120# KUBERNETES_CONFORMANCE_PROVIDER is set to "gke".
121if [[ -z "${NODE_INSTANCE_GROUP:-}" ]] && [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then
122 detect-node-instance-groups
123 NODE_INSTANCE_GROUP=$(kube::util::join , "${NODE_INSTANCE_GROUP[@]}")
124fi
125
126if [[ "${KUBERNETES_PROVIDER}" == "azure" ]]; then
127 if [[ ${CLOUD_CONFIG} == "" ]]; then
128 echo "Missing azure cloud config"
129 exit 1
130 fi
131fi
132
133# These arguments are understood by both Ginkgo test suite and CLI.
134# Some arguments (like --nodes) are only supported when using the CLI.
135# Those get set below when choosing the program.
136ginkgo_args=(
137 "--poll-progress-after=${GINKGO_POLL_PROGRESS_AFTER:-60m}"
138 "--poll-progress-interval=${GINKGO_POLL_PROGRESS_INTERVAL:-5m}"
139 "--source-root=${KUBE_ROOT}"
140)
141
142# NOTE: Ginkgo's default timeout has been reduced from 24h to 1h in V2, set it manually here as "24h"
143# for backward compatibility purpose.
144ginkgo_args+=("--timeout=${GINKGO_TIMEOUT:-24h}")
145
146if [[ -n "${CONFORMANCE_TEST_SKIP_REGEX:-}" ]]; then
147 ginkgo_args+=("--skip=${CONFORMANCE_TEST_SKIP_REGEX}")
148 ginkgo_args+=("--seed=1436380640")
149fi
150
151if [[ "${GINKGO_UNTIL_IT_FAILS:-}" == true ]]; then
152 ginkgo_args+=("--until-it-fails=true")
153fi
154
155FLAKE_ATTEMPTS=1
156if [[ "${GINKGO_TOLERATE_FLAKES}" == "y" ]]; then
157 FLAKE_ATTEMPTS=2
158fi
159ginkgo_args+=("--flake-attempts=${FLAKE_ATTEMPTS}")
160
161if [[ "${GINKGO_NO_COLOR}" == "y" ]]; then
162 ginkgo_args+=("--no-color")
163fi
164
165# The --host setting is used only when providing --auth_config
166# If --kubeconfig is used, the host to use is retrieved from the .kubeconfig
167# file and the one provided with --host is ignored.
168# Add path for things like running kubectl binary.
169PATH=$(dirname "${e2e_test}"):"${PATH}"
170export PATH
171
172# Choose the program to execute and additional arguments for it.
173#
174# Note that the e2e.test binary must have been built with "make DBG=1"
175# when using debuggers, otherwise debug information gets stripped.
176case "${E2E_TEST_DEBUG_TOOL:-ginkgo}" in
177 ginkgo)
178 program=("${ginkgo}")
179 if [[ -n "${GINKGO_PARALLEL_NODES:-}" ]]; then
180 program+=("--nodes=${GINKGO_PARALLEL_NODES}")
181 elif [[ ${GINKGO_PARALLEL} =~ ^[yY]$ ]]; then
182 program+=("--nodes=25")
183 fi
184 program+=("${ginkgo_args[@]:+${ginkgo_args[@]}}")
185 ;;
186 delve) program=("dlv" "exec") ;;
187 gdb) program=("gdb") ;;
188 *) kube::log::error_exit "Unsupported E2E_TEST_DEBUG_TOOL=${E2E_TEST_DEBUG_TOOL}" ;;
189esac
190
191# Move Ginkgo arguments that are understood by the suite when not using
192# the CLI.
193suite_args=()
194if [ "${E2E_TEST_DEBUG_TOOL:-ginkgo}" != "ginkgo" ]; then
195 for arg in "${ginkgo_args[@]}"; do
196 suite_args+=("--ginkgo.${arg#--}")
197 done
198fi
199
200# Generate full dumps of the test result and progress in <report-dir>/ginkgo/,
201# using the Ginkgo-specific JSON format and JUnit XML. Ignored if --report-dir
202# is not used.
203suite_args+=(--report-complete-ginkgo --report-complete-junit)
204
205# The following invocation is fairly complex. Let's dump it to simplify
206# determining what the final options are. Enabled by default in CI
207# environments like Prow.
208case "${GINKGO_SHOW_COMMAND:-${CI:-no}}" in y|yes|true) set -x ;; esac
209
210"${program[@]}" "${e2e_test}" -- \
211 "${auth_config[@]:+${auth_config[@]}}" \
212 --host="${KUBE_MASTER_URL}" \
213 --provider="${KUBERNETES_PROVIDER}" \
214 --gce-project="${PROJECT:-}" \
215 --gce-zone="${ZONE:-}" \
216 --gce-region="${REGION:-}" \
217 --gce-multizone="${MULTIZONE:-false}" \
218 --gke-cluster="${CLUSTER_NAME:-}" \
219 --kube-master="${KUBE_MASTER:-}" \
220 --cluster-tag="${CLUSTER_ID:-}" \
221 --cloud-config-file="${CLOUD_CONFIG:-}" \
222 --repo-root="${KUBE_ROOT}" \
223 --node-instance-group="${NODE_INSTANCE_GROUP:-}" \
224 --prefix="${KUBE_GCE_INSTANCE_PREFIX:-e2e}" \
225 --network="${KUBE_GCE_NETWORK:-${KUBE_GKE_NETWORK:-e2e}}" \
226 --node-tag="${NODE_TAG:-}" \
227 --master-tag="${MASTER_TAG:-}" \
228 --docker-config-file="${DOCKER_CONFIG_FILE:-}" \
229 --dns-domain="${KUBE_DNS_DOMAIN:-cluster.local}" \
230 --prepull-images="${PREPULL_IMAGES:-false}" \
231 ${MASTER_OS_DISTRIBUTION:+"--master-os-distro=${MASTER_OS_DISTRIBUTION}"} \
232 ${NODE_OS_DISTRIBUTION:+"--node-os-distro=${NODE_OS_DISTRIBUTION}"} \
233 ${NUM_NODES:+"--num-nodes=${NUM_NODES}"} \
234 ${E2E_REPORT_DIR:+"--report-dir=${E2E_REPORT_DIR}"} \
235 ${E2E_REPORT_PREFIX:+"--report-prefix=${E2E_REPORT_PREFIX}"} \
236 "${suite_args[@]:+${suite_args[@]}}" \
237 "${@}"
View as plain text