1#!/usr/bin/env bash
2
3# Copyright 2014 The Kubernetes Authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17# LIMITATIONS
18# 1. Exit code is probably not always correct.
19# 2. There are no unittests.
20# 3. Will not work if the total length of paths to addons is greater than
21# bash can handle. Probably it is not a problem: ARG_MAX=2097152 on GCE.
22
23# cosmetic improvements to be done
24# 1. Improve the log function; add timestamp, file name, etc.
25# 2. Logging doesn't work from files that print things out.
26# 3. Kubectl prints the output to stderr (the output should be captured and then
27# logged)
28
29KUBECTL=${KUBECTL_BIN:-/usr/local/bin/kubectl}
30KUBECTL_OPTS=${KUBECTL_OPTS:-}
31# KUBECTL_PRUNE_WHITELIST is a list of resources whitelisted by default.
32# This is currently the same with the default in:
33# https://github.com/kubernetes/kubectl/blob/master/pkg/cmd/apply/prune.go.
34# To override the default list with other values, set
35# KUBECTL_PRUNE_WHITELIST_OVERRIDE environment variable to space-separated
36# names of resources to whitelist.
37if [ -z "${KUBECTL_PRUNE_WHITELIST_OVERRIDE:-}" ]; then
38 KUBECTL_PRUNE_WHITELIST=(
39 core/v1/ConfigMap
40 core/v1/Endpoints
41 core/v1/Namespace
42 core/v1/PersistentVolumeClaim
43 core/v1/PersistentVolume
44 core/v1/Pod
45 core/v1/ReplicationController
46 core/v1/Secret
47 core/v1/Service
48 batch/v1/Job
49 batch/v1/CronJob
50 apps/v1/DaemonSet
51 apps/v1/Deployment
52 apps/v1/ReplicaSet
53 apps/v1/StatefulSet
54 networking.k8s.io/v1/Ingress
55 )
56else
57 read -ra KUBECTL_PRUNE_WHITELIST <<< "${KUBECTL_PRUNE_WHITELIST_OVERRIDE}"
58fi
59
60# This variable is unused in this file, but not in those that source it.
61# shellcheck disable=SC2034
62ADDON_CHECK_INTERVAL_SEC=${TEST_ADDON_CHECK_INTERVAL_SEC:-60}
63ADDON_PATH=${ADDON_PATH:-/etc/kubernetes/addons}
64
65# This variable is unused in this file, but not in those that source it.
66# shellcheck disable=SC2034
67SYSTEM_NAMESPACE=kube-system
68
69# Addons could use this label with two modes:
70# - ADDON_MANAGER_LABEL=Reconcile
71# - ADDON_MANAGER_LABEL=EnsureExists
72ADDON_MANAGER_LABEL="addonmanager.kubernetes.io/mode"
73# This label is deprecated (only for Addon Manager). In future release
74# addon-manager may not respect it anymore. Addons with
75# CLUSTER_SERVICE_LABEL=true and without ADDON_MANAGER_LABEL=EnsureExists
76# will be reconciled for now.
77CLUSTER_SERVICE_LABEL="kubernetes.io/cluster-service"
78
79# Whether only one addon manager should be running in a multi-master setup.
80# Disabling this flag will force all addon managers to assume they are the
81# leaders.
82ADDON_MANAGER_LEADER_ELECTION=${ADDON_MANAGER_LEADER_ELECTION:-true}
83
84# Remember that you can't log from functions that print some output (because
85# logs are also printed on stdout).
86# $1 level
87# $2 message
88function log() {
89 # manage log levels manually here
90
91 # add the timestamp if you find it useful
92 case $1 in
93 DB3 )
94# echo "$1: $2"
95 ;;
96 DB2 )
97# echo "$1: $2"
98 ;;
99 DBG )
100# echo "$1: $2"
101 ;;
102 INFO )
103 echo "$1: $2"
104 ;;
105 WRN )
106 echo "$1: $2"
107 ;;
108 ERR )
109 echo "$1: $2"
110 ;;
111 * )
112 echo "INVALID_LOG_LEVEL $1: $2"
113 ;;
114 esac
115}
116
117# Generate kubectl prune-allowlist flags from provided resource list.
118function generate_prune_allowlist_flags() {
119 local -r resources=( "$@" )
120 for resource in "${resources[@]}"; do
121 # Check if $resource isn't composed just of whitespaces by replacing ' '
122 # with '' and checking whether the resulting string is not empty.
123 if [[ -n "${resource// /}" ]]; then
124 printf "%s" "--prune-allowlist ${resource} "
125 fi
126 done
127}
128
129# KUBECTL_EXTRA_PRUNE_WHITELIST is a list of extra allowed resources
130# besides the default ones.
131extra_prune_allowlist=
132if [ -n "${KUBECTL_EXTRA_PRUNE_WHITELIST:-}" ]; then
133 read -ra extra_prune_allowlist <<< "${KUBECTL_EXTRA_PRUNE_WHITELIST}"
134fi
135prune_allowlist=( "${KUBECTL_PRUNE_WHITELIST[@]}" "${extra_prune_allowlist[@]}" )
136prune_allowlist_flags=$(generate_prune_allowlist_flags "${prune_allowlist[@]}")
137
138log INFO "== Generated kubectl prune allowlist flags: $prune_allowlist_flags =="
139
140# $1 filename of addon to start.
141# $2 count of tries to start the addon.
142# $3 delay in seconds between two consecutive tries
143# $4 namespace
144function start_addon() {
145 local -r addon_filename=$1;
146 local -r tries=$2;
147 local -r delay=$3;
148 local -r namespace=$4
149
150 create_resource_from_string "$(cat "${addon_filename}")" "${tries}" "${delay}" "${addon_filename}" "${namespace}"
151}
152
153# $1 string with json or yaml.
154# $2 count of tries to start the addon.
155# $3 delay in seconds between two consecutive tries
156# $4 name of this object to use when logging about it.
157# $5 namespace for this object
158function create_resource_from_string() {
159 local -r config_string=$1;
160 local tries=$2;
161 local -r delay=$3;
162 local -r config_name=$4;
163 local -r namespace=$5;
164 while [ "${tries}" -gt 0 ]; do
165 reconcile_resource_from_string "${config_string}" "${config_name}" "${namespace}" && \
166 ensure_resource_from_string "${config_string}" "${config_name}" "${namespace}" && \
167 return 0;
168 (( tries-- ))
169 log WRN "== Failed to start ${config_name} in namespace ${namespace} at $(date -Is). ${tries} tries remaining. =="
170 sleep "${delay}";
171 done
172 return 1;
173}
174
175# Creates resources with addon mode Reconcile for create_resource_from_string.
176# Does not perform pruning.
177# $1 string with json or yaml.
178# $2 name of this object for logging
179# $3 namespace for the object
180function reconcile_resource_from_string() {
181 local -r config_string=$1;
182 local -r config_name=$2;
183 local -r namespace=$3;
184
185 # kubectl_output must be declared ahead of time to allow capturing kubectl's exit code and not local's exit code.
186 local kubectl_output;
187 # shellcheck disable=SC2086
188 # Disabling because "${KUBECTL_OPTS}" needs to allow for expansion here
189 kubectl_output=$(echo "${config_string}" | ${KUBECTL} ${KUBECTL_OPTS} apply -f - \
190 --namespace="${namespace}" -l ${ADDON_MANAGER_LABEL}=Reconcile 2>&1) && \
191 log INFO "== Successfully reconciled ${config_name} in namespace ${namespace} at $(date -Is)" && \
192 return 0;
193 if echo "${kubectl_output}" | grep --silent "no objects"; then
194 # Nothing to do.
195 return 0;
196 fi
197 echo "${kubectl_output}" # for visibility of errors
198 return 1;
199}
200
201# Creates resources with addon mode EnsureExists for create_resource_from_string.
202# Does not perform pruning.
203# $1 string with json or yaml.
204# $2 name of this object for logging
205# $3 namespace for the object
206function ensure_resource_from_string() {
207 local -r config_string=$1;
208 local -r config_name=$2;
209 local -r namespace=$3;
210
211 # Resources that are set to the addon mode EnsureExists should not be overwritten if they already exist.
212 local kubectl_output;
213 # shellcheck disable=SC2086
214 # Disabling because "${KUBECTL_OPTS}" needs to allow for expansion here
215 kubectl_output=$(echo "${config_string}" | ${KUBECTL} ${KUBECTL_OPTS} create -f - \
216 --namespace="${namespace}" -l ${ADDON_MANAGER_LABEL}=EnsureExists 2>&1) && \
217 log INFO "== Successfully started ${config_name} in namespace ${namespace} at $(date -Is)" && \
218 return 0;
219 # Detect an already exists failure for creating EnsureExists resources.
220 # All other errors should result in a retry.
221 if echo "${kubectl_output}" | grep --silent "AlreadyExists"; then
222 log INFO "== Skipping start ${config_name} in namespace ${namespace}, already exists at $(date -Is)"
223 return 0;
224 elif echo "${kubectl_output}" | grep --silent "no objects"; then
225 # Nothing to do.
226 return 0;
227 fi
228 echo "${kubectl_output}" # for visibility of errors
229 return 1;
230}
231
232function reconcile_addons() {
233 # TODO: Remove the first command in future release.
234 # Adding this for backward compatibility. Old addons have CLUSTER_SERVICE_LABEL=true and don't have
235 # ADDON_MANAGER_LABEL=EnsureExists will still be reconciled.
236 # Filter out `configured` message to not noisily log.
237 # `created`, `pruned` and errors will be logged.
238 log INFO "== Reconciling with deprecated label =="
239 # shellcheck disable=SC2086
240 # Disabling because "${KUBECTL_OPTS}" needs to allow for expansion here
241 ${KUBECTL} ${KUBECTL_OPTS} apply -f ${ADDON_PATH} \
242 -l ${CLUSTER_SERVICE_LABEL}=true,${ADDON_MANAGER_LABEL}!=EnsureExists \
243 --prune=true ${prune_allowlist_flags} --recursive | grep -v configured
244
245 log INFO "== Reconciling with addon-manager label =="
246 # shellcheck disable=SC2086
247 # Disabling because "${KUBECTL_OPTS}" needs to allow for expansion here
248 ${KUBECTL} ${KUBECTL_OPTS} apply -f ${ADDON_PATH} \
249 -l ${CLUSTER_SERVICE_LABEL}!=true,${ADDON_MANAGER_LABEL}=Reconcile \
250 --prune=true ${prune_allowlist_flags} --recursive | grep -v configured
251
252 log INFO "== Kubernetes addon reconcile completed at $(date -Is) =="
253}
254
255function ensure_addons() {
256 # Create objects already exist should fail.
257 # Filter out `AlreadyExists` message to not noisily log.
258 # shellcheck disable=SC2086
259 # Disabling because "${KUBECTL_OPTS}" needs to allow for expansion here
260 ${KUBECTL} ${KUBECTL_OPTS} create -f ${ADDON_PATH} \
261 -l ${ADDON_MANAGER_LABEL}=EnsureExists --recursive 2>&1 | grep -v AlreadyExists
262
263 log INFO "== Kubernetes addon ensure completed at $(date -Is) =="
264}
265
266function is_leader() {
267 # In multi-master setup, only one addon manager should be running. We use
268 # existing leader election in kube-controller-manager instead of implementing
269 # a separate mechanism here.
270 if ! $ADDON_MANAGER_LEADER_ELECTION; then
271 log INFO "Leader election disabled."
272 return 0;
273 fi
274 # shellcheck disable=SC2086
275 # Disabling because "${KUBECTL_OPTS}" needs to allow for expansion here
276 KUBE_CONTROLLER_MANAGER_LEADER=$(${KUBECTL} ${KUBECTL_OPTS} -n kube-system get leases.v1.coordination.k8s.io kube-controller-manager -o "jsonpath={.spec.holderIdentity}")
277
278 case "${KUBE_CONTROLLER_MANAGER_LEADER}" in
279 "")
280 log ERR "No leader election info found."
281 return 1
282 ;;
283
284 "${HOSTNAME}"_*)
285 log INFO "Leader is $KUBE_CONTROLLER_MANAGER_LEADER"
286 return 0
287 ;;
288
289 *)
290 log INFO "Leader is $KUBE_CONTROLLER_MANAGER_LEADER, not ${HOSTNAME}_*"
291 return 1
292 ;;
293 esac
294}
View as plain text