1#!/usr/bin/env bash
2
3# Copyright 2019 The Kubernetes Authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17set -o errexit
18set -o nounset
19set -o pipefail
20
21# Go tools really don't like it if you have a symlink in `pwd`.
22cd "$(pwd -P)"
23
24KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
25source "${KUBE_ROOT}/hack/lib/init.sh"
26
27# Get all the default Go environment.
28kube::golang::setup_env
29
30# Turn off workspaces until we are ready for them later
31export GOWORK=off
32# Explicitly opt into go modules
33export GO111MODULE=on
34# Explicitly set GOFLAGS to ignore vendor, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor
35export GOFLAGS=-mod=mod
36# Ensure sort order doesn't depend on locale
37export LANG=C
38export LC_ALL=C
39# Detect problematic GOPROXY settings that prevent lookup of dependencies
40if [[ "${GOPROXY:-}" == "off" ]]; then
41 kube::log::error "Cannot run hack/update-vendor.sh with \$GOPROXY=off"
42 exit 1
43fi
44
45kube::util::require-jq
46
47TMP_DIR="${TMP_DIR:-$(mktemp -d /tmp/update-vendor.XXXX)}"
48LOG_FILE="${LOG_FILE:-${TMP_DIR}/update-vendor.log}"
49kube::log::status "logfile at ${LOG_FILE}"
50
51# Set up some FDs for this script to use, while capturing everything else to
52# the log. NOTHING ELSE should write to $LOG_FILE directly.
53exec 11>&1 # Real stdout, use this explicitly
54exec 22>&2 # Real stderr, use this explicitly
55exec 1>"${LOG_FILE}" # Automatic stdout
56exec 2>&1 # Automatic stderr
57set -x # Trace this script to stderr
58go env # For the log
59
60function finish {
61 ret=$?
62 if [[ ${ret} != 0 ]]; then
63 echo "An error has occurred. Please see more details in ${LOG_FILE}" >&22
64 fi
65 exit ${ret}
66}
67trap finish EXIT
68
69# ensure_require_replace_directives_for_all_dependencies:
70# - ensures all existing 'require' directives have an associated 'replace' directive pinning a version
71# - adds explicit 'require' directives for all transitive dependencies
72# - adds explicit 'replace' directives for all require directives (existing 'replace' directives take precedence)
73function ensure_require_replace_directives_for_all_dependencies() {
74 local local_tmp_dir
75 local_tmp_dir=$(mktemp -d "${TMP_DIR}/pin_replace.XXXX")
76
77 # collect 'require' directives that actually specify a version
78 local require_filter='(.Version != null) and (.Version != "v0.0.0") and (.Version != "v0.0.0-00010101000000-000000000000")'
79 # collect 'replace' directives that unconditionally pin versions (old=new@version)
80 local replace_filter='(.Old.Version == null) and (.New.Version != null)'
81
82 # Capture local require/replace directives before running any go commands that can modify the go.mod file
83 local require_json="${local_tmp_dir}/require.json"
84 local replace_json="${local_tmp_dir}/replace.json"
85 go mod edit -json \
86 | jq -r ".Require // [] | sort | .[] | select(${require_filter})" \
87 > "${require_json}"
88 go mod edit -json \
89 | jq -r ".Replace // [] | sort | .[] | select(${replace_filter})" \
90 > "${replace_json}"
91
92 # Propagate root replace/require directives into staging modules, in case we are downgrading, so they don't bump the root required version back up
93 for repo in $(kube::util::list_staging_repos); do
94 (
95 cd "staging/src/k8s.io/${repo}"
96 jq -r '"-require \(.Path)@\(.Version)"' < "${require_json}" \
97 | xargs -L 100 go mod edit -fmt
98 jq -r '"-replace \(.Old.Path)=\(.New.Path)@\(.New.Version)"' < "${replace_json}" \
99 | xargs -L 100 go mod edit -fmt
100 )
101 done
102
103 # tidy to ensure require directives are added for indirect dependencies
104 go mod tidy
105}
106
107function print_go_mod_section() {
108 local directive="$1"
109 local file="$2"
110
111 if [ -s "${file}" ]; then
112 echo "${directive} ("
113 cat "$file"
114 echo ")"
115 fi
116}
117
118function group_directives() {
119 local local_tmp_dir
120 local_tmp_dir=$(mktemp -d "${TMP_DIR}/group_replace.XXXX")
121 local go_mod_require_direct="${local_tmp_dir}/go.mod.require_direct.tmp"
122 local go_mod_require_indirect="${local_tmp_dir}/go.mod.require_indirect.tmp"
123 local go_mod_replace="${local_tmp_dir}/go.mod.replace.tmp"
124 local go_mod_other="${local_tmp_dir}/go.mod.other.tmp"
125 # separate replace and non-replace directives
126 awk "
127 # print lines between 'require (' ... ')' lines
128 /^require [(]/ { inrequire=1; next }
129 inrequire && /^[)]/ { inrequire=0; next }
130 inrequire && /\/\/ indirect/ { print > \"${go_mod_require_indirect}\"; next }
131 inrequire { print > \"${go_mod_require_direct}\"; next }
132
133 # print lines between 'replace (' ... ')' lines
134 /^replace [(]/ { inreplace=1; next }
135 inreplace && /^[)]/ { inreplace=0; next }
136 inreplace { print > \"${go_mod_replace}\"; next }
137
138 # print ungrouped replace directives with the replace directive trimmed
139 /^replace [^(]/ { sub(/^replace /,\"\"); print > \"${go_mod_replace}\"; next }
140
141 # print ungrouped require directives with the require directive trimmed
142 /^require [^(].*\/\/ indirect/ { sub(/^require /,\"\"); print > \"${go_mod_require_indirect}\"; next }
143 /^require [^(]/ { sub(/^require /,\"\"); print > \"${go_mod_require_direct}\"; next }
144
145 # otherwise print to the other file
146 { print > \"${go_mod_other}\" }
147 " < go.mod
148 {
149 cat "${go_mod_other}";
150 print_go_mod_section "require" "${go_mod_require_direct}"
151 print_go_mod_section "require" "${go_mod_require_indirect}"
152 print_go_mod_section "replace" "${go_mod_replace}"
153 } > go.mod
154
155 go mod edit -fmt
156}
157
158function add_generated_comments() {
159 local local_tmp_dir
160 local_tmp_dir=$(mktemp -d "${TMP_DIR}/add_generated_comments.XXXX")
161 local go_mod_nocomments="${local_tmp_dir}/go.mod.nocomments.tmp"
162
163 # drop comments before the module directive
164 awk "
165 BEGIN { dropcomments=1 }
166 /^module / { dropcomments=0 }
167 dropcomments && /^\/\// { next }
168 { print }
169 " < go.mod > "${go_mod_nocomments}"
170
171 # Add the specified comments
172 local comments="${1}"
173 {
174 echo "${comments}"
175 echo ""
176 cat "${go_mod_nocomments}"
177 } > go.mod
178
179 # Format
180 go mod edit -fmt
181}
182
183
184# Phase 1: ensure go.mod files for staging modules and main module
185
186for repo in $(kube::util::list_staging_repos); do
187 (
188 cd "staging/src/k8s.io/${repo}"
189
190 if [[ ! -f go.mod ]]; then
191 kube::log::status "go.mod: initialize ${repo}" >&11
192 rm -f Godeps/Godeps.json # remove before initializing, staging Godeps are not authoritative
193 go mod init "k8s.io/${repo}"
194 go mod edit -fmt
195 fi
196 )
197done
198
199if [[ ! -f go.mod ]]; then
200 kube::log::status "go.mod: initialize k8s.io/kubernetes" >&11
201 go mod init "k8s.io/kubernetes"
202 rm -f Godeps/Godeps.json # remove after initializing
203fi
204
205
206# Phase 2: ensure staging repo require/replace directives
207
208kube::log::status "go.mod: update staging references" >&11
209# Prune
210go mod edit -json \
211 | jq -r '.Require[]? | select(.Version == "v0.0.0") | "-droprequire \(.Path)"' \
212 | xargs -L 100 go mod edit -fmt
213go mod edit -json \
214 | jq -r '.Replace[]? | select(.New.Path | startswith("./staging/")) | "-dropreplace \(.Old.Path)"' \
215 | xargs -L 100 go mod edit -fmt
216# Re-add
217kube::util::list_staging_repos \
218 | while read -r X; do echo "-require k8s.io/${X}@v0.0.0"; done \
219 | xargs -L 100 go mod edit -fmt
220kube::util::list_staging_repos \
221 | while read -r X; do echo "-replace k8s.io/${X}=./staging/src/k8s.io/${X}"; done \
222 | xargs -L 100 go mod edit -fmt
223
224
225# Phase 3: capture required (minimum) versions from all modules, and replaced (pinned) versions from the root module
226
227# pin referenced versions
228ensure_require_replace_directives_for_all_dependencies
229# resolves/expands references in the root go.mod (if needed)
230go mod tidy
231# pin expanded versions
232ensure_require_replace_directives_for_all_dependencies
233# group require/replace directives
234group_directives
235
236# Phase 4: copy root go.mod to staging dirs and rewrite
237
238kube::log::status "go.mod: propagate to staging modules" >&11
239for repo in $(kube::util::list_staging_repos); do
240 (
241 cd "staging/src/k8s.io/${repo}"
242
243 echo "=== propagating to ${repo}"
244 # copy root go.mod, changing module name
245 sed "s#module k8s.io/kubernetes#module k8s.io/${repo}#" \
246 < "${KUBE_ROOT}/go.mod" \
247 > "${KUBE_ROOT}/staging/src/k8s.io/${repo}/go.mod"
248 # remove `require` directives for staging components (will get re-added as needed by `go list`)
249 kube::util::list_staging_repos \
250 | while read -r X; do echo "-droprequire k8s.io/${X}"; done \
251 | xargs -L 100 go mod edit
252 # rewrite `replace` directives for staging components to point to peer directories
253 kube::util::list_staging_repos \
254 | while read -r X; do echo "-replace k8s.io/${X}=../${X}"; done \
255 | xargs -L 100 go mod edit
256 )
257done
258
259
260# Phase 5: sort and tidy staging components
261
262kube::log::status "go.mod: sorting staging modules" >&11
263# tidy staging repos in reverse dependency order.
264# the content of dependencies' go.mod files affects what `go mod tidy` chooses to record in a go.mod file.
265tidy_unordered="${TMP_DIR}/tidy_unordered.txt"
266kube::util::list_staging_repos \
267 | xargs -I {} echo "k8s.io/{}" > "${tidy_unordered}"
268rm -f "${TMP_DIR}/tidy_deps.txt"
269# SC2094 checks that you do not read and write to the same file in a pipeline.
270# We do read from ${tidy_unordered} in the pipeline and mention it within the
271# pipeline (but only ready it again) so we disable the lint to assure shellcheck
272# that :this-is-fine:
273# shellcheck disable=SC2094
274while IFS= read -r repo; do
275 # record existence of the repo to ensure modules with no peer relationships still get included in the order
276 echo "${repo} ${repo}" >> "${TMP_DIR}/tidy_deps.txt"
277
278 (
279 cd "${KUBE_ROOT}/staging/src/${repo}"
280
281 # save the original go.mod, since go list doesn't just add missing entries, it also removes specific required versions from it
282 tmp_go_mod="${TMP_DIR}/tidy_${repo/\//_}_go.mod.original"
283 tmp_go_deps="${TMP_DIR}/tidy_${repo/\//_}_deps.txt"
284 cp go.mod "${tmp_go_mod}"
285
286 echo "=== sorting ${repo}"
287 # 'go list' calculates direct imports and updates go.mod so that go list -m lists our module dependencies
288 echo "=== computing imports for ${repo}"
289 go list all
290 # ignore errors related to importing `package main` packages, but catch
291 # other errors (https://github.com/golang/go/issues/59186)
292 errs=()
293 kube::util::read-array errs < <(
294 go list -e -tags=tools -json all | jq -r '.Error.Err | select( . != null )' \
295 | grep -v "is a program, not an importable package"
296 )
297 if (( "${#errs[@]}" != 0 )); then
298 for err in "${errs[@]}"; do
299 echo "${err}" >&2
300 done
301 exit 1
302 fi
303
304 # capture module dependencies
305 go list -m -f '{{if not .Main}}{{.Path}}{{end}}' all > "${tmp_go_deps}"
306
307 # restore the original go.mod file
308 cp "${tmp_go_mod}" go.mod
309
310 # list all module dependencies
311 for dep in $(join "${tidy_unordered}" "${tmp_go_deps}"); do
312 # record the relationship (put dep first, because we want to sort leaves first)
313 echo "${dep} ${repo}" >> "${TMP_DIR}/tidy_deps.txt"
314 # switch the required version to an explicit v0.0.0 (rather than an unknown v0.0.0-00010101000000-000000000000)
315 go mod edit -require "${dep}@v0.0.0"
316 done
317 )
318done < "${tidy_unordered}"
319
320kube::log::status "go.mod: tidying" >&11
321for repo in $(tsort "${TMP_DIR}/tidy_deps.txt"); do
322 (
323 cd "${KUBE_ROOT}/staging/src/${repo}"
324 echo "=== tidying ${repo}"
325
326 # prune replace directives that pin to the naturally selected version.
327 # do this before tidying, since tidy removes unused modules that
328 # don't provide any relevant packages, which forgets which version of the
329 # unused transitive dependency we had a require directive for,
330 # and prevents pruning the matching replace directive after tidying.
331 go list -m -json all |
332 jq -r 'select(.Replace != null) |
333 select(.Path == .Replace.Path) |
334 select(.Version == .Replace.Version) |
335 "-dropreplace \(.Replace.Path)"' |
336 xargs -L 100 go mod edit -fmt
337
338 go mod tidy -v
339
340 # disallow transitive dependencies on k8s.io/kubernetes
341 loopback_deps=()
342 kube::util::read-array loopback_deps < <(go list all 2>/dev/null | grep k8s.io/kubernetes/ || true)
343 if (( "${#loopback_deps[@]}" > 0 )); then
344 kube::log::error "${#loopback_deps[@]} disallowed ${repo} -> k8s.io/kubernetes dependencies exist via the following imports: $(go mod why "${loopback_deps[@]}")" >&22 2>&1
345 exit 1
346 fi
347
348 # prune unused pinned replace directives
349 comm -23 \
350 <(go mod edit -json | jq -r '.Replace[] | .Old.Path' | sort) \
351 <(go list -m -json all | jq -r .Path | sort) |
352 while read -r X; do echo "-dropreplace=${X}"; done |
353 xargs -L 100 go mod edit -fmt
354
355 # prune replace directives that pin to the naturally selected version
356 go list -m -json all |
357 jq -r 'select(.Replace != null) |
358 select(.Path == .Replace.Path) |
359 select(.Version == .Replace.Version) |
360 "-dropreplace \(.Replace.Path)"' |
361 xargs -L 100 go mod edit -fmt
362
363 # group require/replace directives
364 group_directives
365 )
366done
367echo "=== tidying root"
368go mod tidy
369
370# prune unused pinned non-local replace directives
371comm -23 \
372 <(go mod edit -json | jq -r '.Replace[] | select(.New.Path | startswith("./") | not) | .Old.Path' | sort) \
373 <(go list -m -json all | jq -r .Path | sort) |
374while read -r X; do echo "-dropreplace=${X}"; done |
375xargs -L 100 go mod edit -fmt
376
377# disallow transitive dependencies on k8s.io/kubernetes
378loopback_deps=()
379kube::util::read-array loopback_deps < <(go mod graph | grep ' k8s.io/kubernetes' || true)
380if (( "${#loopback_deps[@]}" > 0 )); then
381 kube::log::error "${#loopback_deps[@]} disallowed transitive k8s.io/kubernetes dependencies exist via the following imports:" >&22 2>&1
382 kube::log::error "${loopback_deps[@]}" >&22 2>&1
383 exit 1
384fi
385
386# Phase 6: add generated comments to go.mod files
387kube::log::status "go.mod: adding generated comments" >&11
388add_generated_comments "
389// This is a generated file. Do not edit directly.
390// Ensure you've carefully read
391// https://git.k8s.io/community/contributors/devel/sig-architecture/vendor.md
392// Run hack/pin-dependency.sh to change pinned dependency versions.
393// Run hack/update-vendor.sh to update go.mod files and the vendor directory.
394"
395for repo in $(kube::util::list_staging_repos); do
396 (
397 cd "staging/src/k8s.io/${repo}"
398 add_generated_comments "// This is a generated file. Do not edit directly."
399 )
400done
401
402
403# Phase 7: update internal modules
404kube::log::status "vendor: updating internal modules" >&11
405hack/update-internal-modules.sh
406
407
408# Phase 8: rebuild vendor directory
409(
410 kube::log::status "vendor: running 'go work vendor'" >&11
411 unset GOWORK
412 unset GOFLAGS
413 go work vendor
414)
415
416kube::log::status "vendor: updating vendor/LICENSES" >&11
417hack/update-vendor-licenses.sh
418
419kube::log::status "vendor: creating OWNERS file" >&11
420rm -f "vendor/OWNERS"
421cat <<__EOF__ > "vendor/OWNERS"
422# See the OWNERS docs at https://go.k8s.io/owners
423
424options:
425 # make root approval non-recursive
426 no_parent_owners: true
427approvers:
428- dep-approvers
429reviewers:
430- dep-reviewers
431__EOF__
432
433kube::log::status "NOTE: don't forget to handle vendor/* and LICENSE/* files that were added or removed" >&11
View as plain text