1#!/usr/bin/env bash
2
3# -e Stops execution in the instance of a command or pipeline error
4# -u Treat unset variables as an error and exit immediately
5set -eu
6
7if type realpath >/dev/null 2>&1 ; then
8 cd "$(realpath -- $(dirname -- "$0"))"
9fi
10
11#
12# Defaults
13#
14export RACE="false"
15STAGE="starting"
16STATUS="FAILURE"
17RUN=()
18UNIT_PACKAGES=()
19UNIT_FLAGS=()
20FILTER=()
21
22#
23# Print Functions
24#
25function print_outcome() {
26 if [ "$STATUS" == SUCCESS ]
27 then
28 echo -e "\e[32m"$STATUS"\e[0m"
29 else
30 echo -e "\e[31m"$STATUS"\e[0m while running \e[31m"$STAGE"\e[0m"
31 fi
32}
33
34function print_list_of_integration_tests() {
35 go test -tags integration -list=. ./test/integration/... | grep '^Test'
36 exit 0
37}
38
39function exit_msg() {
40 # complain to STDERR and exit with error
41 echo "$*" >&2
42 exit 2
43}
44
45function check_arg() {
46 if [ -z "$OPTARG" ]
47 then
48 exit_msg "No arg for --$OPT option, use: -h for help">&2
49 fi
50}
51
52function print_usage_exit() {
53 echo "$USAGE"
54 exit 0
55}
56
57function print_heading {
58 echo
59 echo -e "\e[34m\e[1m"$1"\e[0m"
60}
61
62function run_and_expect_silence() {
63 echo "$@"
64 result_file=$(mktemp -t bouldertestXXXX)
65 "$@" 2>&1 | tee "${result_file}"
66
67 # Fail if result_file is nonempty.
68 if [ -s "${result_file}" ]; then
69 rm "${result_file}"
70 exit 1
71 fi
72 rm "${result_file}"
73}
74
75#
76# Testing Helpers
77#
78function run_unit_tests() {
79 go test "${UNIT_FLAGS[@]}" "${UNIT_PACKAGES[@]}" "${FILTER[@]}"
80}
81
82#
83# Main CLI Parser
84#
85USAGE="$(cat -- <<-EOM
86
87Usage:
88Boulder test suite CLI, intended to be run inside of a Docker container:
89
90 docker compose run --use-aliases boulder ./$(basename "${0}") [OPTION]...
91
92With no options passed, runs standard battery of tests (lint, unit, and integration)
93
94 -l, --lints Adds lint to the list of tests to run
95 -u, --unit Adds unit to the list of tests to run
96 -v, --unit-verbose Enables verbose output for unit tests
97 -w, --unit-without-cache Disables go test caching for unit tests
98 -p <DIR>, --unit-test-package=<DIR> Run unit tests for specific go package(s)
99 -e, --enable-race-detection Enables race detection for unit and integration tests
100 -n, --config-next Changes BOULDER_CONFIG_DIR from test/config to test/config-next
101 -i, --integration Adds integration to the list of tests to run
102 -s, --start-py Adds start to the list of tests to run
103 -m, --gomod-vendor Adds gomod-vendor to the list of tests to run
104 -g, --generate Adds generate to the list of tests to run
105 -o, --list-integration-tests Outputs a list of the available integration tests
106 -f <REGEX>, --filter=<REGEX> Run only those tests matching the regular expression
107
108 Note:
109 This option disables the '"back in time"' integration test setup
110
111 For tests, the regular expression is split by unbracketed slash (/)
112 characters into a sequence of regular expressions
113
114 Example:
115 TestAkamaiPurgerDrainQueueFails/TestWFECORS
116 -h, --help Shows this help message
117
118EOM
119)"
120
121while getopts luvweciosmgnhp:f:-: OPT; do
122 if [ "$OPT" = - ]; then # long option: reformulate OPT and OPTARG
123 OPT="${OPTARG%%=*}" # extract long option name
124 OPTARG="${OPTARG#$OPT}" # extract long option argument (may be empty)
125 OPTARG="${OPTARG#=}" # if long option argument, remove assigning `=`
126 fi
127 case "$OPT" in
128 l | lints ) RUN+=("lints") ;;
129 u | unit ) RUN+=("unit") ;;
130 v | unit-verbose ) UNIT_FLAGS+=("-v") ;;
131 w | unit-without-cache ) UNIT_FLAGS+=("-count=1") ;;
132 p | unit-test-package ) check_arg; UNIT_PACKAGES+=("${OPTARG}") ;;
133 e | enable-race-detection ) RACE="true"; UNIT_FLAGS+=("-race") ;;
134 i | integration ) RUN+=("integration") ;;
135 o | list-integration-tests ) print_list_of_integration_tests ;;
136 f | filter ) check_arg; FILTER+=("${OPTARG}") ;;
137 s | start-py ) RUN+=("start") ;;
138 m | gomod-vendor ) RUN+=("gomod-vendor") ;;
139 g | generate ) RUN+=("generate") ;;
140 n | config-next ) BOULDER_CONFIG_DIR="test/config-next" ;;
141 h | help ) print_usage_exit ;;
142 ??* ) exit_msg "Illegal option --$OPT" ;; # bad long option
143 ? ) exit 2 ;; # bad short option (error reported via getopts)
144 esac
145done
146shift $((OPTIND-1)) # remove parsed options and args from $@ list
147
148# The list of segments to run. Order doesn't matter. Note: gomod-vendor
149# is specifically left out of the defaults, because we don't want to run
150# it locally (it could delete local state).
151if [ -z "${RUN[@]+x}" ]
152then
153 RUN+=("lints" "unit" "integration")
154fi
155
156# Filter is used by unit and integration but should not be used for both at the same time
157if [[ "${RUN[@]}" =~ unit ]] && [[ "${RUN[@]}" =~ integration ]] && [[ -n "${FILTER[@]+x}" ]]
158then
159 exit_msg "Illegal option: (-f, --filter) when specifying both (-u, --unit) and (-i, --integration)"
160fi
161
162# If unit + filter: set correct flags for go test
163if [[ "${RUN[@]}" =~ unit ]] && [[ -n "${FILTER[@]+x}" ]]
164then
165 FILTER=(--test.run "${FILTER[@]}")
166fi
167
168# If integration + filter: set correct flags for test/integration-test.py
169if [[ "${RUN[@]}" =~ integration ]] && [[ -n "${FILTER[@]+x}" ]]
170then
171 FILTER=(--filter "${FILTER[@]}")
172fi
173
174# If unit test packages are not specified: set flags to run unit tests
175# for all boulder packages
176if [ -z "${UNIT_PACKAGES[@]+x}" ]
177then
178 # '-p=1' configures unit tests to run serially, rather than in parallel. Our
179 # unit tests depend on mutating a database and then cleaning up after
180 # themselves. If these test were run in parallel, they could fail spuriously
181 # due to one test modifying a table (especially registrations) while another
182 # test is reading from it.
183 # https://github.com/letsencrypt/boulder/issues/1499
184 # https://pkg.go.dev/cmd/go#hdr-Testing_flags
185 UNIT_FLAGS+=("-p=1")
186 UNIT_PACKAGES+=("./...")
187fi
188
189print_heading "Boulder Test Suite CLI"
190print_heading "Settings:"
191
192# On EXIT, trap and print outcome
193trap "print_outcome" EXIT
194
195settings="$(cat -- <<-EOM
196 RUN: ${RUN[@]}
197 BOULDER_CONFIG_DIR: $BOULDER_CONFIG_DIR
198 UNIT_PACKAGES: ${UNIT_PACKAGES[@]}
199 UNIT_FLAGS: ${UNIT_FLAGS[@]}
200 FILTER: ${FILTER[@]}
201
202EOM
203)"
204
205echo "$settings"
206print_heading "Starting..."
207
208#
209# Run various linters.
210#
211STAGE="lints"
212if [[ "${RUN[@]}" =~ "$STAGE" ]] ; then
213 print_heading "Running Lints"
214 golangci-lint run --timeout 9m ./...
215 # Implicitly loads staticcheck.conf from the root of the boulder repository
216 staticcheck ./...
217 python3 test/grafana/lint.py
218 # Check for common spelling errors using codespell.
219 # Update .codespell.ignore.txt if you find false positives (NOTE: ignored
220 # words should be all lowercase).
221 run_and_expect_silence codespell \
222 --ignore-words=.codespell.ignore.txt \
223 --skip=.git,.gocache,go.sum,go.mod,vendor,bin,*.pyc,*.pem,*.der,*.resp,*.req,*.csr,.codespell.ignore.txt,.*.swp
224 # Check test JSON configs are formatted consistently
225 ./test/format-configs.py 'test/config*/*.json'
226 run_and_expect_silence git diff --exit-code .
227fi
228
229#
230# Unit Tests.
231#
232STAGE="unit"
233if [[ "${RUN[@]}" =~ "$STAGE" ]] ; then
234 print_heading "Running Unit Tests"
235 run_unit_tests
236fi
237
238#
239# Integration tests
240#
241STAGE="integration"
242if [[ "${RUN[@]}" =~ "$STAGE" ]] ; then
243 print_heading "Running Integration Tests"
244 python3 test/integration-test.py --chisel --gotest "${FILTER[@]}"
245fi
246
247# Test that just ./start.py works, which is a proxy for testing that
248# `docker compose up` works, since that just runs start.py (via entrypoint.sh).
249STAGE="start"
250if [[ "${RUN[@]}" =~ "$STAGE" ]] ; then
251 print_heading "Running Start Test"
252 python3 start.py &
253 for I in {1..115}; do
254 sleep 1
255 curl -s http://localhost:4001/directory && echo "Boulder took ${I} seconds to come up" && break
256 done
257 if [ "${I}" -eq 115 ]; then
258 echo "Boulder did not come up after ${I} seconds during ./start.py."
259 exit 1
260 fi
261fi
262
263# Run go mod vendor (happens only in CI) to check that the versions in
264# vendor/ really exist in the remote repo and match what we have.
265STAGE="gomod-vendor"
266if [[ "${RUN[@]}" =~ "$STAGE" ]] ; then
267 print_heading "Running Go Mod Tidy"
268 go mod tidy
269 print_heading "Running Go Mod Vendor"
270 go mod vendor
271 run_and_expect_silence git diff --exit-code .
272fi
273
274# Run generate to make sure all our generated code can be re-generated with
275# current tools.
276# Note: Some of the tools we use seemingly don't understand ./vendor yet, and
277# so will fail if imports are not available in $GOPATH.
278STAGE="generate"
279if [[ "${RUN[@]}" =~ "$STAGE" ]] ; then
280 print_heading "Running Generate"
281 # Additionally, we need to run go install before go generate because the stringer command
282 # (using in ./grpc/) checks imports, and depends on the presence of a built .a
283 # file to determine an import really exists. See
284 # https://golang.org/src/go/internal/gcimporter/gcimporter.go#L30
285 # Without this, we get error messages like:
286 # stringer: checking package: grpc/bcodes.go:6:2: could not import
287 # github.com/letsencrypt/boulder/probs (can't find import:
288 # github.com/letsencrypt/boulder/probs)
289 go install ./probs
290 go install ./vendor/google.golang.org/grpc/codes
291 run_and_expect_silence go generate ./...
292 run_and_expect_silence git diff --exit-code .
293fi
294
295# Because set -e stops execution in the instance of a command or pipeline
296# error; if we got here we assume success
297STATUS="SUCCESS"
View as plain text