#!/usr/bin/env bash set -euE # Usage: ./docker/base-python.docker.gen > docker/.base-python.docker.stamp # # base-python.docker.gen is essentially just a 4 line script: # # iidfile=$(mktemp) # trap 'rm -f "$iidfile"' EXIT # docker build --iidfile="$iidfile" docker/base-python >&2 # cat "$iidfile" # # However, it has "optimizations" because that `docker build` is # really slow and painful: # # 0. (not an speed improvement itself, but nescessary for what # follows) generate a deterministic Docker tag based on the # inputs to the image; a sort of content-addressable scheme that # doesn't rely on having built the image first. # # 1. Rather than building the image locally, try to pull it # pre-build from any of the following Docker repos: # # - $BASE_PYTHON_REPO # - ${DEV_REGISTRY}/base-python # - docker.io/emissaryingress/base-python # # 2. If we do build it locally (because it couldn't be pulled), then # try pushing it to those Docker repos, so that # others/our-future-self can benefit from (1). OFF='' BLD='' RED='' GRN='' BLU='' if tput setaf 0 &>/dev/null; then OFF="$(tput sgr0)" BLD="$(tput bold)" RED="$(tput setaf 1)" GRN="$(tput setaf 2)" BLU="$(tput setaf 4)" fi msg() { # shellcheck disable=SC2059 printf "${BLU} => [${0##*/}]${OFF} $1${OFF}\n" "${@:2}" >&2 } stat_busy() { # shellcheck disable=SC2059 printf "${BLU} => [${0##*/}]${OFF} $1...${OFF}" "${@:2}" >&2 } stat_done() { # shellcheck disable=SC2059 printf " ${1:-done}${OFF}\n" >&2 } statnl_busy() { stat_busy "$@" printf '\n' >&2 } statnl_done() { # shellcheck disable=SC2059 printf "${BLU} => [${0##*/}]${OFF} ...${1:-done}${OFF}\n" >&2 } error() { # shellcheck disable=SC2059 printf "${RED} => [${0##*/}] ${BLD}error:${OFF} $1${OFF}\n" "${@:2}" >&2 } # Usage: tag=$(print-tag) # # print-tag generates and prints a Docker tag (without the leading # "REPO:" part) for the image, based on the inputs to the image. # # The inputs we care about (i.e. the things that should trigger a # rebuild) are: # # - The `docker/base-python/Dockerfile` file. # # - Whatever unpinned remote 3rd-party resources that Dockerfile # pulls in (mostly the Alpine package repos); but because we don't # have the whole repos as a file on disk, we fall back to a # truncated timestamp. This means that we rebuild periodically to # make sure we don't fall too far behind and then get surprised # when a rebuild is required for Dockerfile changes.) We have # defined "enough time" as a few days. See the variable # "build_every_n_days" below. print-tag() { python3 -c ' import datetime, hashlib # Arrange these 2 variables to reduce the likelihood that build_every_n_days # passes in the middle of a CI workflow; have it happen weekly during the # weekend. build_every_n_days = 7 # Periodic rebuild even if Dockerfile does not change epoch = datetime.datetime(2020, 11, 8, 5, 0) # 1AM EDT on a Sunday age = int((datetime.datetime.now() - epoch).days / build_every_n_days) age_start = epoch + datetime.timedelta(days=age*build_every_n_days) dockerfilehash = hashlib.sha256(open("docker/base-python/Dockerfile", "rb").read()).hexdigest() print("%sx%s-%s" % (age_start.strftime("%Y%m%d"), build_every_n_days, dockerfilehash[:16])) ' } main() { local tag tag=$(print-tag) # `repos` is a list of Docker repos where the base-python image # gets pulled-from/pushed-to. # # When pulling, we go down the list until we find a repo we # can successfully pull from; returning after the first # success; if we make through the list without a success, then # we build the image locally. # # When pushing, we attempt to push to *every* repo, but ignore # failures unless they *all* fail. local repos=() # add_repo REPO appends a repo to ${repos[@]}; if ${repos[@]} # doesn't already contain REPO. add_repo() { local needle straw needle="$1" # The `${repos[@]:+…}` non-emptiness check seems # pointless here, but it's important because macOS # still has Bash 3.2, and prior to Bash 4.4 (Sept # 2016), there was a bug where it would consider an # empty array to be unset, triggering `set -u`. The # other usages of "${repos[@]}" don't need this # because this is the only one where it might still be # empty. for straw in ${repos[@]:+"${repos[@]}"}; do if [[ "$straw" == "$needle" ]]; then return fi done repos+=("$needle") } if [[ -n "${BASE_PYTHON_REPO:-}" ]]; then add_repo "$BASE_PYTHON_REPO" fi if [[ -n "${DEV_REGISTRY:-}" ]]; then add_repo "${DEV_REGISTRY}/base-python" fi # We always include docker.io/emissaryingress/base-python as a # fallback, because rebuilding orjson takes so long that we # really want a cache-hit if at all possible. add_repo 'docker.io/emissaryingress/base-python' # Download local id='' for repo in "${repos[@]}"; do stat_busy 'Checking if %q exists locally' "$repo:$tag" if docker image inspect "$repo:$tag" &>/dev/null; then stat_done "${GRN}yes" id=$(docker image inspect "$repo:$tag" --format='{{.Id}}') break fi stat_done "${RED}no" stat_busy 'Checking if %q can be pulled' "$repo:$tag" if docker pull "$repo:$tag" &>/dev/null; then stat_done "${GRN}yes" id=$(docker image inspect "$repo:$tag" --format='{{.Id}}') break fi stat_done "${RED}no" done if [[ -z "$id" ]]; then # Build statnl_busy 'Building %q locally' "base-python:$tag" iidfile=$(mktemp) trap 'rm -f "$iidfile"' RETURN docker build --iidfile="$iidfile" docker/base-python >&2 id=$(cat "$iidfile") statnl_done 'done building' # Push pushed=0 for repo in "${repos[@]}"; do statnl_busy 'Attempting to push %q' "$repo:$tag" docker tag "$id" "$repo:$tag" >&2 if docker push "$repo:$tag" >&2; then statnl_done "${GRN}pushed" pushed=1 continue fi statnl_done "${RED}failed to push" done if ! (( pushed )); then error "Could not push locally-built image to any remote repositories" return 1 fi fi printf '%s\n' "$id" } main "$@"