1#!/usr/bin/env bash
2
3# Copyright 2016 The Kubernetes Authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17# This script is only for demonstrating how to use the node test container. In
18# production environment, kubelet bootstrap will be more complicated, user
19# should configure the node test container accordingly.
20# In addition, this script will also be used in the node e2e test to let it use
21# the containerized test suite.
22
23# TODO(random-liu): Use standard installer to install kubelet.
24# TODO(random-liu): Use standard tool to start kubelet in production way (such
25# as systemd, supervisord etc.)
26
27# Refresh sudo credentials if needed
28if ping -c 1 -q metadata.google.internal &> /dev/null; then
29 echo 'Running on GCE, not asking for sudo credentials'
30elif sudo --non-interactive "$(which bash)" -c true 2> /dev/null; then
31 # if we can run bash without a password, it's a pretty safe bet that either
32 # we can run any command without a password, or that sudo credentials
33 # are already cached - and they've just been re-cached
34 echo 'No need to refresh sudo credentials'
35else
36 echo 'Updating sudo credentials'
37 sudo -v || exit 1
38fi
39
40# FOCUS is ginkgo focus to select which tests to run. By default, FOCUS is
41# initialized as "\[Conformance\]" in the test container to run all conformance
42# test.
43FOCUS=${FOCUS:-""}
44
45# SKIP is ginkgo skip to select which tests to skip. By default, SKIP is
46# initialized as "\[Flaky\]|\[Serial\]" in the test container skipping all
47# flaky and serial test.
48SKIP=${SKIP:-""}
49
50# TEST_ARGS is the test arguments. It could be used to override default test
51# arguments in the container.
52TEST_ARGS=${TEST_ARGS:-""}
53
54# REGISTRY is the image registry for node test image.
55REGISTRY=${REGISTRY:-"registry.k8s.io"}
56
57# ARCH is the architecture of current machine, the script will use this to
58# select corresponding test container image.
59ARCH=${ARCH:-"amd64"}
60
61# VERSION is the version of the test container image.
62VERSION=${VERSION:-"0.2"}
63
64# KUBELET_BIN is the kubelet binary name. If it is not specified, use the
65# default binary name "kubelet".
66KUBELET_BIN=${KUBELET_BIN:-"kubelet"}
67
68# KUBELET is the kubelet binary path. If it is not specified, assume kubelet is
69# in PATH.
70KUBELET=${KUBELET:-"$(which "$KUBELET_BIN")"}
71
72# LOG_DIR is the absolute path of the directory where the test will collect all
73# logs to. By default, use the current directory.
74LOG_DIR=${LOG_DIR:-$(pwd)}
75mkdir -p "$LOG_DIR"
76
77# NETWORK_PLUGIN is the network plugin used by kubelet. Do not use network
78# plugin by default.
79NETWORK_PLUGIN=${NETWORK_PLUGIN:-""}
80
81# CNI_CONF_DIR is the path to network plugin binaries.
82CNI_CONF_DIR=${CNI_CONF_DIR:-""}
83
84# CNI_BIN_DIR is the path to network plugin config files.
85CNI_BIN_DIR=${CNI_BIN_DIR:-""}
86
87# KUBELET_KUBECONFIG is the path to a kubeconfig file, specifying how to connect to the API server.
88KUBELET_KUBECONFIG=${KUBELET_KUBECONFIG:-"/var/lib/kubelet/kubeconfig"}
89
90# Creates a kubeconfig file for the kubelet.
91# Args: address (e.g. "http://localhost:8080"), destination file path
92function create-kubelet-kubeconfig() {
93 local api_addr="${1}"
94 local dest="${2}"
95 local dest_dir
96 dest_dir="$(dirname "${dest}")"
97 mkdir -p "${dest_dir}" &>/dev/null || sudo mkdir -p "${dest_dir}"
98 sudo=$(test -w "${dest_dir}" || echo "sudo -E")
99 cat <<EOF | ${sudo} tee "${dest}" > /dev/null
100apiVersion: v1
101kind: Config
102clusters:
103 - cluster:
104 server: ${api_addr}
105 name: local
106contexts:
107 - context:
108 cluster: local
109 name: local
110current-context: local
111EOF
112}
113
114# start_kubelet starts kubelet and redirect kubelet log to $LOG_DIR/kubelet.log.
115kubelet_log=kubelet.log
116start_kubelet() {
117 echo "Creating kubelet.kubeconfig"
118 create-kubelet-kubeconfig "http://localhost:8080" "${KUBELET_KUBECONFIG}"
119 echo "Starting kubelet..."
120 # we want to run this command as root but log the file to a normal user file
121 # (so disable SC2024)
122 # shellcheck disable=SC2024
123 if ! sudo -b "${KUBELET}" "$@" &>"${LOG_DIR}/${kubelet_log}"; then
124 echo "Failed to start kubelet"
125 exit 1
126 fi
127}
128
129# wait_kubelet retries for 10 times for kubelet to be ready by checking http://127.0.0.1:10248/healthz.
130wait_kubelet() {
131 echo "Health checking kubelet..."
132 healthCheckURL=http://127.0.0.1:10248/healthz
133 local maxRetry=10
134 local cur=1
135 while [ $cur -le $maxRetry ]; do
136 if curl -s $healthCheckURL > /dev/null; then
137 echo "Kubelet is ready"
138 break
139 fi
140 if [ $cur -eq $maxRetry ]; then
141 echo "Health check exceeds max retry"
142 exit 1
143 fi
144 echo "Kubelet is not ready"
145 sleep 1
146 ((cur++))
147 done
148}
149
150# kill_kubelet kills kubelet.
151kill_kubelet() {
152 echo "Stopping kubelet..."
153 if ! sudo pkill "${KUBELET_BIN}"; then
154 echo "Failed to stop kubelet."
155 exit 1
156 fi
157}
158
159# run_test runs the node test container.
160run_test() {
161 env=""
162 if [ -n "$FOCUS" ]; then
163 env="$env -e FOCUS=\"$FOCUS\""
164 fi
165 if [ -n "$SKIP" ]; then
166 env="$env -e SKIP=\"$SKIP\""
167 fi
168 if [ -n "$TEST_ARGS" ]; then
169 env="$env -e TEST_ARGS=\"$TEST_ARGS\""
170 fi
171 # The test assumes that inside the container:
172 # * kubelet manifest path is mounted to the same path;
173 # * log collect directory is mounted to /var/result;
174 # * root file system is mounted to /rootfs.
175 sudo sh -c "docker run -it --rm --privileged=true --net=host -v /:/rootfs \
176 -v $config_dir:$config_dir -v $LOG_DIR:/var/result ${env} $REGISTRY/node-test-$ARCH:$VERSION"
177}
178
179# Check whether kubelet is running. If kubelet is running, tell the user to stop
180# it before running the test.
181pid=$(pidof "${KUBELET_BIN}")
182if [ -n "$pid" ]; then
183 echo "Kubelet is running (pid=$pid), please stop it before running the test."
184 exit 1
185fi
186
187volume_stats_agg_period=10s
188serialize_image_pulls=false
189config_dir=$(mktemp -d)
190file_check_frequency=10s
191pod_cidr=10.100.0.0/24
192log_level=4
193start_kubelet --kubeconfig "${KUBELET_KUBECONFIG}" \
194 --volume-stats-agg-period $volume_stats_agg_period \
195 --serialize-image-pulls=$serialize_image_pulls \
196 --pod-manifest-path "${config_dir}" \
197 --file-check-frequency $file_check_frequency \
198 --pod-cidr=$pod_cidr \
199 --runtime-cgroups=/docker-daemon \
200 --kubelet-cgroups=/kubelet \
201 --system-cgroups=/system \
202 --cgroup-root=/ \
203 --v=$log_level \
204
205wait_kubelet
206
207run_test
208
209kill_kubelet
210
211# Clean up the kubelet config directory
212sudo rm -rf "${config_dir}"
View as plain text