1#!/usr/bin/env bash
2
3# Copyright 2014 The Kubernetes Authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17# shellcheck disable=SC2034 # Variables sourced in other scripts.
18
19# Common utilities, variables and checks for all build scripts.
20set -o errexit
21set -o nounset
22set -o pipefail
23
24# Unset CDPATH, having it set messes up with script import paths
25unset CDPATH
26
27USER_ID=$(id -u)
28GROUP_ID=$(id -g)
29
30DOCKER_OPTS=${DOCKER_OPTS:-""}
31IFS=" " read -r -a DOCKER <<< "docker ${DOCKER_OPTS}"
32DOCKER_HOST=${DOCKER_HOST:-""}
33GOPROXY=${GOPROXY:-""}
34
35# This will canonicalize the path
36KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd -P)
37
38source "${KUBE_ROOT}/hack/lib/init.sh"
39
40# Constants
41readonly KUBE_BUILD_IMAGE_REPO=kube-build
42KUBE_BUILD_IMAGE_CROSS_TAG="$(cat "${KUBE_ROOT}/build/build-image/cross/VERSION")"
43readonly KUBE_BUILD_IMAGE_CROSS_TAG
44
45readonly KUBE_DOCKER_REGISTRY="${KUBE_DOCKER_REGISTRY:-registry.k8s.io}"
46KUBE_BASE_IMAGE_REGISTRY="${KUBE_BASE_IMAGE_REGISTRY:-registry.k8s.io/build-image}"
47readonly KUBE_BASE_IMAGE_REGISTRY
48
49# This version number is used to cause everyone to rebuild their data containers
50# and build image. This is especially useful for automated build systems like
51# Jenkins.
52#
53# Increment/change this number if you change the build image (anything under
54# build/build-image) or change the set of volumes in the data container.
55KUBE_BUILD_IMAGE_VERSION_BASE="$(cat "${KUBE_ROOT}/build/build-image/VERSION")"
56readonly KUBE_BUILD_IMAGE_VERSION_BASE
57readonly KUBE_BUILD_IMAGE_VERSION="${KUBE_BUILD_IMAGE_VERSION_BASE}-${KUBE_BUILD_IMAGE_CROSS_TAG}"
58
59# Make it possible to override the `kube-cross` image, and tag independent of `KUBE_BASE_IMAGE_REGISTRY`
60KUBE_CROSS_IMAGE="${KUBE_CROSS_IMAGE:-"${KUBE_BASE_IMAGE_REGISTRY}/kube-cross"}"
61readonly KUBE_CROSS_IMAGE
62KUBE_CROSS_VERSION="${KUBE_CROSS_VERSION:-"${KUBE_BUILD_IMAGE_CROSS_TAG}"}"
63readonly KUBE_CROSS_VERSION
64
65# Here we map the output directories across both the local and remote _output
66# directories:
67#
68# *_OUTPUT_ROOT - the base of all output in that environment.
69# *_OUTPUT_SUBPATH - location where golang stuff is built/cached. Also
70# persisted across docker runs with a volume mount.
71# *_OUTPUT_BINPATH - location where final binaries are placed. If the remote
72# is really remote, this is the stuff that has to be copied
73# back.
74# OUT_DIR can come in from the Makefile, so honor it.
75readonly LOCAL_OUTPUT_ROOT="${KUBE_ROOT}/${OUT_DIR:-_output}"
76readonly LOCAL_OUTPUT_SUBPATH="${LOCAL_OUTPUT_ROOT}/dockerized"
77readonly LOCAL_OUTPUT_BINPATH="${LOCAL_OUTPUT_SUBPATH}/bin"
78readonly LOCAL_OUTPUT_GOPATH="${LOCAL_OUTPUT_SUBPATH}/go"
79readonly LOCAL_OUTPUT_IMAGE_STAGING="${LOCAL_OUTPUT_ROOT}/images"
80
81# This is a symlink to binaries for "this platform" (e.g. build tools).
82readonly THIS_PLATFORM_BIN="${LOCAL_OUTPUT_ROOT}/bin"
83
84readonly KUBE_GO_PACKAGE=k8s.io/kubernetes
85readonly REMOTE_ROOT="/go/src/${KUBE_GO_PACKAGE}"
86readonly REMOTE_OUTPUT_ROOT="${REMOTE_ROOT}/_output"
87readonly REMOTE_OUTPUT_SUBPATH="${REMOTE_OUTPUT_ROOT}/dockerized"
88readonly REMOTE_OUTPUT_BINPATH="${REMOTE_OUTPUT_SUBPATH}/bin"
89readonly REMOTE_OUTPUT_GOPATH="${REMOTE_OUTPUT_SUBPATH}/go"
90
91# This is the port on the workstation host to expose RSYNC on. Set this if you
92# are doing something fancy with ssh tunneling.
93readonly KUBE_RSYNC_PORT="${KUBE_RSYNC_PORT:-}"
94
95# This is the port that rsync is running on *inside* the container. This may be
96# mapped to KUBE_RSYNC_PORT via docker networking.
97readonly KUBE_CONTAINER_RSYNC_PORT=8730
98
99# These are the default versions (image tags) for their respective base images.
100readonly __default_distroless_iptables_version=v0.5.3
101readonly __default_go_runner_version=v2.3.1-go1.22.2-bookworm.0
102readonly __default_setcap_version=bookworm-v1.0.2
103
104# These are the base images for the Docker-wrapped binaries.
105readonly KUBE_GORUNNER_IMAGE="${KUBE_GORUNNER_IMAGE:-$KUBE_BASE_IMAGE_REGISTRY/go-runner:$__default_go_runner_version}"
106readonly KUBE_APISERVER_BASE_IMAGE="${KUBE_APISERVER_BASE_IMAGE:-$KUBE_GORUNNER_IMAGE}"
107readonly KUBE_CONTROLLER_MANAGER_BASE_IMAGE="${KUBE_CONTROLLER_MANAGER_BASE_IMAGE:-$KUBE_GORUNNER_IMAGE}"
108readonly KUBE_SCHEDULER_BASE_IMAGE="${KUBE_SCHEDULER_BASE_IMAGE:-$KUBE_GORUNNER_IMAGE}"
109readonly KUBE_PROXY_BASE_IMAGE="${KUBE_PROXY_BASE_IMAGE:-$KUBE_BASE_IMAGE_REGISTRY/distroless-iptables:$__default_distroless_iptables_version}"
110readonly KUBECTL_BASE_IMAGE="${KUBECTL_BASE_IMAGE:-$KUBE_GORUNNER_IMAGE}"
111
112# This is the image used in a multi-stage build to apply capabilities to Docker-wrapped binaries.
113readonly KUBE_BUILD_SETCAP_IMAGE="${KUBE_BUILD_SETCAP_IMAGE:-$KUBE_BASE_IMAGE_REGISTRY/setcap:$__default_setcap_version}"
114
115# Get the set of master binaries that run in Docker (on Linux)
116# Entry format is "<binary-name>,<base-image>".
117# Binaries are placed in /usr/local/bin inside the image.
118# `make` users can override any or all of the base images using the associated
119# environment variables.
120#
121# $1 - server architecture
122kube::build::get_docker_wrapped_binaries() {
123 ### If you change any of these lists, please also update DOCKERIZED_BINARIES
124 ### in build/BUILD. And kube::golang::server_image_targets
125 local targets=(
126 "kube-apiserver,${KUBE_APISERVER_BASE_IMAGE}"
127 "kube-controller-manager,${KUBE_CONTROLLER_MANAGER_BASE_IMAGE}"
128 "kube-scheduler,${KUBE_SCHEDULER_BASE_IMAGE}"
129 "kube-proxy,${KUBE_PROXY_BASE_IMAGE}"
130 "kubectl,${KUBECTL_BASE_IMAGE}"
131 )
132
133 echo "${targets[@]}"
134}
135
136# ---------------------------------------------------------------------------
137# Basic setup functions
138
139# Verify that the right utilities and such are installed for building Kube. Set
140# up some dynamic constants.
141# Args:
142# $1 - boolean of whether to require functioning docker (default true)
143#
144# Vars set:
145# KUBE_ROOT_HASH
146# KUBE_BUILD_IMAGE_TAG_BASE
147# KUBE_BUILD_IMAGE_TAG
148# KUBE_BUILD_IMAGE
149# KUBE_BUILD_CONTAINER_NAME_BASE
150# KUBE_BUILD_CONTAINER_NAME
151# KUBE_DATA_CONTAINER_NAME_BASE
152# KUBE_DATA_CONTAINER_NAME
153# KUBE_RSYNC_CONTAINER_NAME_BASE
154# KUBE_RSYNC_CONTAINER_NAME
155# DOCKER_MOUNT_ARGS
156# LOCAL_OUTPUT_BUILD_CONTEXT
157# shellcheck disable=SC2120 # optional parameters
158function kube::build::verify_prereqs() {
159 local -r require_docker=${1:-true}
160 kube::log::status "Verifying Prerequisites...."
161 kube::build::ensure_tar || return 1
162 kube::build::ensure_rsync || return 1
163 if ${require_docker}; then
164 kube::build::ensure_docker_in_path || return 1
165 if kube::build::is_osx; then
166 kube::build::docker_available_on_osx || return 1
167 fi
168 kube::util::ensure_docker_daemon_connectivity || return 1
169
170 if (( KUBE_VERBOSE > 6 )); then
171 kube::log::status "Docker Version:"
172 "${DOCKER[@]}" version | kube::log::info_from_stdin
173 fi
174 fi
175
176 KUBE_GIT_BRANCH=$(git symbolic-ref --short -q HEAD 2>/dev/null || true)
177 KUBE_ROOT_HASH=$(kube::build::short_hash "${HOSTNAME:-}:${KUBE_ROOT}:${KUBE_GIT_BRANCH}")
178 KUBE_BUILD_IMAGE_TAG_BASE="build-${KUBE_ROOT_HASH}"
179 KUBE_BUILD_IMAGE_TAG="${KUBE_BUILD_IMAGE_TAG_BASE}-${KUBE_BUILD_IMAGE_VERSION}"
180 KUBE_BUILD_IMAGE="${KUBE_BUILD_IMAGE_REPO}:${KUBE_BUILD_IMAGE_TAG}"
181 KUBE_BUILD_CONTAINER_NAME_BASE="kube-build-${KUBE_ROOT_HASH}"
182 KUBE_BUILD_CONTAINER_NAME="${KUBE_BUILD_CONTAINER_NAME_BASE}-${KUBE_BUILD_IMAGE_VERSION}"
183 KUBE_RSYNC_CONTAINER_NAME_BASE="kube-rsync-${KUBE_ROOT_HASH}"
184 KUBE_RSYNC_CONTAINER_NAME="${KUBE_RSYNC_CONTAINER_NAME_BASE}-${KUBE_BUILD_IMAGE_VERSION}"
185 KUBE_DATA_CONTAINER_NAME_BASE="kube-build-data-${KUBE_ROOT_HASH}"
186 KUBE_DATA_CONTAINER_NAME="${KUBE_DATA_CONTAINER_NAME_BASE}-${KUBE_BUILD_IMAGE_VERSION}"
187 DOCKER_MOUNT_ARGS=(--volumes-from "${KUBE_DATA_CONTAINER_NAME}")
188 LOCAL_OUTPUT_BUILD_CONTEXT="${LOCAL_OUTPUT_IMAGE_STAGING}/${KUBE_BUILD_IMAGE}"
189
190 kube::version::get_version_vars
191 kube::version::save_version_vars "${KUBE_ROOT}/.dockerized-kube-version-defs"
192
193 # Without this, the user's umask can leak through.
194 umask 0022
195}
196
197# ---------------------------------------------------------------------------
198# Utility functions
199
200function kube::build::docker_available_on_osx() {
201 if [[ -z "${DOCKER_HOST}" ]]; then
202 if [[ -S "/var/run/docker.sock" ]] || [[ -S "$(docker context inspect --format '{{.Endpoints.docker.Host}}' | awk -F 'unix://' '{print $2}')" ]]; then
203 kube::log::status "Using docker on macOS"
204 return 0
205 fi
206
207 kube::log::status "No docker host is set."
208 kube::log::status "It looks like you're running Mac OS X, but Docker for Mac cannot be found."
209 kube::log::status "See: https://docs.docker.com/engine/installation/mac/ for installation instructions."
210 return 1
211 fi
212}
213
214function kube::build::is_osx() {
215 [[ "$(uname)" == "Darwin" ]]
216}
217
218function kube::build::is_gnu_sed() {
219 [[ $(sed --version 2>&1) == *GNU* ]]
220}
221
222function kube::build::ensure_rsync() {
223 if [[ -z "$(which rsync)" ]]; then
224 kube::log::error "Can't find 'rsync' in PATH, please fix and retry."
225 return 1
226 fi
227}
228
229function kube::build::ensure_docker_in_path() {
230 if [[ -z "$(which docker)" ]]; then
231 kube::log::error "Can't find 'docker' in PATH, please fix and retry."
232 kube::log::error "See https://docs.docker.com/installation/#installation for installation instructions."
233 return 1
234 fi
235}
236
237function kube::build::ensure_tar() {
238 if [[ -n "${TAR:-}" ]]; then
239 return
240 fi
241
242 # Find gnu tar if it is available, bomb out if not.
243 TAR=tar
244 if which gtar &>/dev/null; then
245 TAR=gtar
246 else
247 if which gnutar &>/dev/null; then
248 TAR=gnutar
249 fi
250 fi
251 if ! "${TAR}" --version | grep -q GNU; then
252 echo " !!! Cannot find GNU tar. Build on Linux or install GNU tar"
253 echo " on Mac OS X (brew install gnu-tar)."
254 return 1
255 fi
256}
257
258function kube::build::has_docker() {
259 which docker &> /dev/null
260}
261
262function kube::build::has_ip() {
263 which ip &> /dev/null && ip -Version | grep 'iproute2' &> /dev/null
264}
265
266# Detect if a specific image exists
267#
268# $1 - image repo name
269# $2 - image tag
270function kube::build::docker_image_exists() {
271 [[ -n $1 && -n $2 ]] || {
272 kube::log::error "Internal error. Image not specified in docker_image_exists."
273 exit 2
274 }
275
276 [[ $("${DOCKER[@]}" images -q "${1}:${2}") ]]
277}
278
279# Delete all images that match a tag prefix except for the "current" version
280#
281# $1: The image repo/name
282# $2: The tag base. We consider any image that matches $2*
283# $3: The current image not to delete if provided
284function kube::build::docker_delete_old_images() {
285 # In Docker 1.12, we can replace this with
286 # docker images "$1" --format "{{.Tag}}"
287 for tag in $("${DOCKER[@]}" images "${1}" | tail -n +2 | awk '{print $2}') ; do
288 if [[ "${tag}" != "${2}"* ]] ; then
289 V=3 kube::log::status "Keeping image ${1}:${tag}"
290 continue
291 fi
292
293 if [[ -z "${3:-}" || "${tag}" != "${3}" ]] ; then
294 V=2 kube::log::status "Deleting image ${1}:${tag}"
295 "${DOCKER[@]}" rmi "${1}:${tag}" >/dev/null
296 else
297 V=3 kube::log::status "Keeping image ${1}:${tag}"
298 fi
299 done
300}
301
302# Stop and delete all containers that match a pattern
303#
304# $1: The base container prefix
305# $2: The current container to keep, if provided
306function kube::build::docker_delete_old_containers() {
307 # In Docker 1.12 we can replace this line with
308 # docker ps -a --format="{{.Names}}"
309 for container in $("${DOCKER[@]}" ps -a | tail -n +2 | awk '{print $NF}') ; do
310 if [[ "${container}" != "${1}"* ]] ; then
311 V=3 kube::log::status "Keeping container ${container}"
312 continue
313 fi
314 if [[ -z "${2:-}" || "${container}" != "${2}" ]] ; then
315 V=2 kube::log::status "Deleting container ${container}"
316 kube::build::destroy_container "${container}"
317 else
318 V=3 kube::log::status "Keeping container ${container}"
319 fi
320 done
321}
322
323# Takes $1 and computes a short has for it. Useful for unique tag generation
324function kube::build::short_hash() {
325 [[ $# -eq 1 ]] || {
326 kube::log::error "Internal error. No data based to short_hash."
327 exit 2
328 }
329
330 local short_hash
331 if which md5 >/dev/null 2>&1; then
332 short_hash=$(md5 -q -s "$1")
333 else
334 short_hash=$(echo -n "$1" | md5sum)
335 fi
336 echo "${short_hash:0:10}"
337}
338
339# Pedantically kill, wait-on and remove a container. The -f -v options
340# to rm don't actually seem to get the job done, so force kill the
341# container, wait to ensure it's stopped, then try the remove. This is
342# a workaround for bug https://github.com/docker/docker/issues/3968.
343function kube::build::destroy_container() {
344 "${DOCKER[@]}" kill "$1" >/dev/null 2>&1 || true
345 if [[ $("${DOCKER[@]}" version --format '{{.Server.Version}}') = 17.06.0* ]]; then
346 # Workaround https://github.com/moby/moby/issues/33948.
347 # TODO: remove when 17.06.0 is not relevant anymore
348 DOCKER_API_VERSION=v1.29 "${DOCKER[@]}" wait "$1" >/dev/null 2>&1 || true
349 else
350 "${DOCKER[@]}" wait "$1" >/dev/null 2>&1 || true
351 fi
352 "${DOCKER[@]}" rm -f -v "$1" >/dev/null 2>&1 || true
353}
354
355# ---------------------------------------------------------------------------
356# Building
357
358
359function kube::build::clean() {
360 if kube::build::has_docker ; then
361 kube::build::docker_delete_old_containers "${KUBE_BUILD_CONTAINER_NAME_BASE}"
362 kube::build::docker_delete_old_containers "${KUBE_RSYNC_CONTAINER_NAME_BASE}"
363 kube::build::docker_delete_old_containers "${KUBE_DATA_CONTAINER_NAME_BASE}"
364 kube::build::docker_delete_old_images "${KUBE_BUILD_IMAGE_REPO}" "${KUBE_BUILD_IMAGE_TAG_BASE}"
365
366 V=2 kube::log::status "Cleaning all untagged docker images"
367 "${DOCKER[@]}" rmi "$("${DOCKER[@]}" images -q --filter 'dangling=true')" 2> /dev/null || true
368 fi
369
370 if [[ -d "${LOCAL_OUTPUT_ROOT}" ]]; then
371 kube::log::status "Removing _output directory"
372 # this ensures we can clean _output/local/go/cache which is not rw by default
373 chmod -R +w "${LOCAL_OUTPUT_ROOT}"
374 rm -rf "${LOCAL_OUTPUT_ROOT}"
375 fi
376}
377
378# Set up the context directory for the kube-build image and build it.
379function kube::build::build_image() {
380 mkdir -p "${LOCAL_OUTPUT_BUILD_CONTEXT}"
381 # Make sure the context directory owned by the right user for syncing sources to container.
382 chown -R "${USER_ID}":"${GROUP_ID}" "${LOCAL_OUTPUT_BUILD_CONTEXT}"
383
384 cp /etc/localtime "${LOCAL_OUTPUT_BUILD_CONTEXT}/"
385 chmod u+w "${LOCAL_OUTPUT_BUILD_CONTEXT}/localtime"
386
387 cp "${KUBE_ROOT}/build/build-image/Dockerfile" "${LOCAL_OUTPUT_BUILD_CONTEXT}/Dockerfile"
388 cp "${KUBE_ROOT}/build/build-image/rsyncd.sh" "${LOCAL_OUTPUT_BUILD_CONTEXT}/"
389 dd if=/dev/urandom bs=512 count=1 2>/dev/null | LC_ALL=C tr -dc 'A-Za-z0-9' | dd bs=32 count=1 2>/dev/null > "${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password"
390 chmod go= "${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password"
391
392 kube::build::docker_build "${KUBE_BUILD_IMAGE}" "${LOCAL_OUTPUT_BUILD_CONTEXT}" 'false' "--build-arg=KUBE_CROSS_IMAGE=${KUBE_CROSS_IMAGE} --build-arg=KUBE_CROSS_VERSION=${KUBE_CROSS_VERSION}"
393
394 # Clean up old versions of everything
395 kube::build::docker_delete_old_containers "${KUBE_BUILD_CONTAINER_NAME_BASE}" "${KUBE_BUILD_CONTAINER_NAME}"
396 kube::build::docker_delete_old_containers "${KUBE_RSYNC_CONTAINER_NAME_BASE}" "${KUBE_RSYNC_CONTAINER_NAME}"
397 kube::build::docker_delete_old_containers "${KUBE_DATA_CONTAINER_NAME_BASE}" "${KUBE_DATA_CONTAINER_NAME}"
398 kube::build::docker_delete_old_images "${KUBE_BUILD_IMAGE_REPO}" "${KUBE_BUILD_IMAGE_TAG_BASE}" "${KUBE_BUILD_IMAGE_TAG}"
399
400 kube::build::ensure_data_container
401 kube::build::sync_to_container
402}
403
404# Build a docker image from a Dockerfile.
405# $1 is the name of the image to build
406# $2 is the location of the "context" directory, with the Dockerfile at the root.
407# $3 is the value to set the --pull flag for docker build; true by default
408# $4 is the set of --build-args for docker.
409function kube::build::docker_build() {
410 kube::util::ensure-docker-buildx
411
412 local -r image=$1
413 local -r context_dir=$2
414 local -r pull="${3:-true}"
415 local build_args
416 IFS=" " read -r -a build_args <<< "$4"
417 readonly build_args
418 local -ra build_cmd=("${DOCKER[@]}" buildx build --load -t "${image}" "--pull=${pull}" "${build_args[@]}" "${context_dir}")
419
420 kube::log::status "Building Docker image ${image}"
421 local docker_output
422 docker_output=$(DOCKER_CLI_EXPERIMENTAL=enabled "${build_cmd[@]}" 2>&1) || {
423 cat <<EOF >&2
424+++ Docker build command failed for ${image}
425
426${docker_output}
427
428To retry manually, run:
429
430DOCKER_CLI_EXPERIMENTAL=enabled ${build_cmd[*]}
431
432EOF
433 return 1
434 }
435}
436
437function kube::build::ensure_data_container() {
438 # If the data container exists AND exited successfully, we can use it.
439 # Otherwise nuke it and start over.
440 local ret=0
441 local code=0
442
443 code=$(docker inspect \
444 -f '{{.State.ExitCode}}' \
445 "${KUBE_DATA_CONTAINER_NAME}" 2>/dev/null) || ret=$?
446 if [[ "${ret}" == 0 && "${code}" != 0 ]]; then
447 kube::build::destroy_container "${KUBE_DATA_CONTAINER_NAME}"
448 ret=1
449 fi
450 if [[ "${ret}" != 0 ]]; then
451 kube::log::status "Creating data container ${KUBE_DATA_CONTAINER_NAME}"
452 # We have to ensure the directory exists, or else the docker run will
453 # create it as root.
454 mkdir -p "${LOCAL_OUTPUT_GOPATH}"
455 # We want this to run as root to be able to chown, so non-root users can
456 # later use the result as a data container. This run both creates the data
457 # container and chowns the GOPATH.
458 #
459 # The data container creates volumes for all of the directories that store
460 # intermediates for the Go build. This enables incremental builds across
461 # Docker sessions. The *_cgo paths are re-compiled versions of the go std
462 # libraries for true static building.
463 local -ra docker_cmd=(
464 "${DOCKER[@]}" run
465 --volume "${REMOTE_ROOT}" # white-out the whole output dir
466 --volume /usr/local/go/pkg/linux_386_cgo
467 --volume /usr/local/go/pkg/linux_amd64_cgo
468 --volume /usr/local/go/pkg/linux_arm_cgo
469 --volume /usr/local/go/pkg/linux_arm64_cgo
470 --volume /usr/local/go/pkg/linux_ppc64le_cgo
471 --volume /usr/local/go/pkg/darwin_amd64_cgo
472 --volume /usr/local/go/pkg/darwin_386_cgo
473 --volume /usr/local/go/pkg/windows_amd64_cgo
474 --volume /usr/local/go/pkg/windows_386_cgo
475 --name "${KUBE_DATA_CONTAINER_NAME}"
476 --hostname "${HOSTNAME}"
477 "${KUBE_BUILD_IMAGE}"
478 chown -R "${USER_ID}":"${GROUP_ID}"
479 "${REMOTE_ROOT}"
480 /usr/local/go/pkg/
481 )
482 "${docker_cmd[@]}"
483 fi
484}
485
486# Run a command in the kube-build image. This assumes that the image has
487# already been built.
488function kube::build::run_build_command() {
489 kube::log::status "Running build command..."
490 kube::build::run_build_command_ex "${KUBE_BUILD_CONTAINER_NAME}" -- "$@"
491}
492
493# Run a command in the kube-build image. This assumes that the image has
494# already been built.
495#
496# Arguments are in the form of
497# <container name> <extra docker args> -- <command>
498function kube::build::run_build_command_ex() {
499 [[ $# != 0 ]] || { echo "Invalid input - please specify a container name." >&2; return 4; }
500 local container_name="${1}"
501 shift
502
503 local -a docker_run_opts=(
504 "--name=${container_name}"
505 "--user=$(id -u):$(id -g)"
506 "--hostname=${HOSTNAME}"
507 "-e=GOPROXY=${GOPROXY}"
508 "${DOCKER_MOUNT_ARGS[@]}"
509 )
510
511 local detach=false
512
513 [[ $# != 0 ]] || { echo "Invalid input - please specify docker arguments followed by --." >&2; return 4; }
514 # Everything before "--" is an arg to docker
515 until [ -z "${1-}" ] ; do
516 if [[ "$1" == "--" ]]; then
517 shift
518 break
519 fi
520 docker_run_opts+=("$1")
521 if [[ "$1" == "-d" || "$1" == "--detach" ]] ; then
522 detach=true
523 fi
524 shift
525 done
526
527 # Everything after "--" is the command to run
528 [[ $# != 0 ]] || { echo "Invalid input - please specify a command to run." >&2; return 4; }
529 local -a cmd=()
530 until [ -z "${1-}" ] ; do
531 cmd+=("$1")
532 shift
533 done
534
535 docker_run_opts+=(
536 --env "KUBE_FASTBUILD=${KUBE_FASTBUILD:-false}"
537 --env "KUBE_BUILDER_OS=${OSTYPE:-notdetected}"
538 --env "KUBE_VERBOSE=${KUBE_VERBOSE}"
539 --env "KUBE_BUILD_WITH_COVERAGE=${KUBE_BUILD_WITH_COVERAGE:-}"
540 --env "KUBE_BUILD_PLATFORMS=${KUBE_BUILD_PLATFORMS:-}"
541 --env "KUBE_CGO_OVERRIDES=' ${KUBE_CGO_OVERRIDES[*]:-} '"
542 --env "KUBE_STATIC_OVERRIDES=' ${KUBE_STATIC_OVERRIDES[*]:-} '"
543 --env "FORCE_HOST_GO=${FORCE_HOST_GO:-}"
544 --env "GO_VERSION=${GO_VERSION:-}"
545 --env "GOTOOLCHAIN=${GOTOOLCHAIN:-}"
546 --env "GOFLAGS=${GOFLAGS:-}"
547 --env "GOGCFLAGS=${GOGCFLAGS:-}"
548 --env "SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH:-}"
549 )
550
551 # use GOLDFLAGS only if it is set explicitly.
552 if [[ -v GOLDFLAGS ]]; then
553 docker_run_opts+=(
554 --env "GOLDFLAGS=${GOLDFLAGS:-}"
555 )
556 fi
557
558 if [[ -n "${DOCKER_CGROUP_PARENT:-}" ]]; then
559 kube::log::status "Using ${DOCKER_CGROUP_PARENT} as container cgroup parent"
560 docker_run_opts+=(--cgroup-parent "${DOCKER_CGROUP_PARENT}")
561 fi
562
563 # If we have stdin we can run interactive. This allows things like 'shell.sh'
564 # to work. However, if we run this way and don't have stdin, then it ends up
565 # running in a daemon-ish mode. So if we don't have a stdin, we explicitly
566 # attach stderr/stdout but don't bother asking for a tty.
567 if [[ -t 0 ]]; then
568 docker_run_opts+=(--interactive --tty)
569 elif [[ "${detach}" == false ]]; then
570 docker_run_opts+=("--attach=stdout" "--attach=stderr")
571 fi
572
573 local -ra docker_cmd=(
574 "${DOCKER[@]}" run "${docker_run_opts[@]}" "${KUBE_BUILD_IMAGE}")
575
576 # Clean up container from any previous run
577 kube::build::destroy_container "${container_name}"
578 "${docker_cmd[@]}" "${cmd[@]}"
579 if [[ "${detach}" == false ]]; then
580 kube::build::destroy_container "${container_name}"
581 fi
582}
583
584function kube::build::rsync_probe {
585 # Wait until rsync is up and running.
586 local tries=20
587 while (( tries > 0 )) ; do
588 if rsync "rsync://k8s@${1}:${2}/" \
589 --password-file="${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password" \
590 &> /dev/null ; then
591 return 0
592 fi
593 tries=$(( tries - 1))
594 sleep 0.1
595 done
596
597 return 1
598}
599
600# Start up the rsync container in the background. This should be explicitly
601# stopped with kube::build::stop_rsyncd_container.
602#
603# This will set the global var KUBE_RSYNC_ADDR to the effective port that the
604# rsync daemon can be reached out.
605function kube::build::start_rsyncd_container() {
606 IPTOOL=ifconfig
607 if kube::build::has_ip ; then
608 IPTOOL="ip address"
609 fi
610 kube::build::stop_rsyncd_container
611 V=3 kube::log::status "Starting rsyncd container"
612 kube::build::run_build_command_ex \
613 "${KUBE_RSYNC_CONTAINER_NAME}" -p 127.0.0.1:"${KUBE_RSYNC_PORT}":"${KUBE_CONTAINER_RSYNC_PORT}" -d \
614 -e ALLOW_HOST="$(${IPTOOL} | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1')" \
615 -- /rsyncd.sh >/dev/null
616
617 local mapped_port
618 if ! mapped_port=$("${DOCKER[@]}" port "${KUBE_RSYNC_CONTAINER_NAME}" "${KUBE_CONTAINER_RSYNC_PORT}" 2> /dev/null | cut -d: -f 2) ; then
619 kube::log::error "Could not get effective rsync port"
620 return 1
621 fi
622
623 local container_ip
624 container_ip=$("${DOCKER[@]}" inspect --format '{{ .NetworkSettings.IPAddress }}' "${KUBE_RSYNC_CONTAINER_NAME}")
625
626 # Sometimes we can reach rsync through localhost and a NAT'd port. Other
627 # times (when we are running in another docker container on the Jenkins
628 # machines) we have to talk directly to the container IP. There is no one
629 # strategy that works in all cases so we test to figure out which situation we
630 # are in.
631 if kube::build::rsync_probe 127.0.0.1 "${mapped_port}"; then
632 KUBE_RSYNC_ADDR="127.0.0.1:${mapped_port}"
633 return 0
634 elif kube::build::rsync_probe "${container_ip}" "${KUBE_CONTAINER_RSYNC_PORT}"; then
635 KUBE_RSYNC_ADDR="${container_ip}:${KUBE_CONTAINER_RSYNC_PORT}"
636 return 0
637 fi
638
639 kube::log::error "Could not connect to rsync container."
640 return 1
641}
642
643function kube::build::stop_rsyncd_container() {
644 V=3 kube::log::status "Stopping any currently running rsyncd container"
645 unset KUBE_RSYNC_ADDR
646 kube::build::destroy_container "${KUBE_RSYNC_CONTAINER_NAME}"
647}
648
649function kube::build::rsync {
650 local -a rsync_opts=(
651 --archive
652 "--password-file=${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password"
653 )
654 if (( KUBE_VERBOSE >= 6 )); then
655 rsync_opts+=("-iv")
656 fi
657 if (( KUBE_RSYNC_COMPRESS > 0 )); then
658 rsync_opts+=("--compress-level=${KUBE_RSYNC_COMPRESS}")
659 fi
660 V=3 kube::log::status "Running rsync"
661 rsync "${rsync_opts[@]}" "$@"
662}
663
664# This will launch rsyncd in a container and then sync the source tree to the
665# container over the local network.
666function kube::build::sync_to_container() {
667 kube::log::status "Syncing sources to container"
668
669 kube::build::start_rsyncd_container
670
671 # rsync filters are a bit confusing. Here we are syncing everything except
672 # output only directories and things that are not necessary like the git
673 # directory and generated files. The '- /' filter prevents rsync
674 # from trying to set the uid/gid/perms on the root of the sync tree.
675 # As an exception, we need to sync generated files in staging/, because
676 # they will not be re-generated by 'make'. Note that the 'H' filtered files
677 # are hidden from rsync so they will be deleted in the target container if
678 # they exist. This will allow them to be re-created in the container if
679 # necessary.
680 kube::build::rsync \
681 --delete \
682 --filter='- /_tmp/' \
683 --filter='- /_output/' \
684 --filter='- /' \
685 "${KUBE_ROOT}/" "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/"
686
687 kube::build::stop_rsyncd_container
688}
689
690# Copy all build results back out.
691function kube::build::copy_output() {
692 kube::log::status "Syncing out of container"
693
694 kube::build::start_rsyncd_container
695
696 # The filter syntax for rsync is a little obscure. It filters on files and
697 # directories. If you don't go in to a directory you won't find any files
698 # there. Rules are evaluated in order. The last two rules are a little
699 # magic. '+ */' says to go in to every directory and '- /**' says to ignore
700 # any file or directory that isn't already specifically allowed.
701 #
702 # We are looking to copy out all of the built binaries along with various
703 # generated files.
704 kube::build::rsync \
705 --prune-empty-dirs \
706 --filter='- /_temp/' \
707 --filter='+ /vendor/' \
708 --filter='+ /staging/***/Godeps/**' \
709 --filter='+ /_output/dockerized/bin/**' \
710 --filter='+ zz_generated.*' \
711 --filter='+ generated.proto' \
712 --filter='+ *.pb.go' \
713 --filter='+ types.go' \
714 --filter='+ */' \
715 --filter='- /**' \
716 "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/" "${KUBE_ROOT}"
717
718 kube::build::stop_rsyncd_container
719}
View as plain text