1#!/usr/bin/env bash
2
3# Copyright 2014 The Kubernetes Authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17# This script verifies the following e2e test ownership policies
18# - tests MUST start with [sig-foo]
19# - tests SHOULD NOT have multiple [sig-foo] tags
20# TODO: these two can be dropped if KubeDescribe is gone from codebase
21# - tests MUST NOT have [k8s.io] in test names
22# - tests MUST NOT use KubeDescribe
23
24set -o errexit
25set -o nounset
26set -o pipefail
27
28# This will canonicalize the path
29KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd -P)
30source "${KUBE_ROOT}/hack/lib/init.sh"
31
32# Set REUSE_BUILD_OUTPUT=y to skip rebuilding dependencies if present
33REUSE_BUILD_OUTPUT=${REUSE_BUILD_OUTPUT:-n}
34# set VERBOSE_OUTPUT=y to output .jq files and shell commands
35VERBOSE_OUTPUT=${VERBOSE_OUTPUT:-n}
36
37if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
38 set -x
39fi
40
41pushd "${KUBE_ROOT}" > /dev/null
42
43# Setup a tmpdir to hold generated scripts and results
44tmpdir=$(mktemp -d -t verify-e2e-test-ownership.XXXX)
45readonly tmpdir
46trap 'rm -rf ${tmpdir}' EXIT
47
48# input
49spec_summaries="${KUBE_ROOT}/_output/specsummaries.json"
50# output
51results_json="${tmpdir}/results.json"
52summary_json="${tmpdir}/summary.json"
53failures_json="${tmpdir}/failures.json"
54
55# rebuild dependencies if necessary
56function ensure_dependencies() {
57 local -r ginkgo="${KUBE_ROOT}/_output/bin/ginkgo"
58 local -r e2e_test="${KUBE_ROOT}/_output/bin/e2e.test"
59 if ! { [ -f "${ginkgo}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then
60 make ginkgo
61 fi
62 if ! { [ -f "${e2e_test}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then
63 hack/make-rules/build.sh test/e2e/e2e.test
64 fi
65 if ! { [ -f "${spec_summaries}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then
66 "${ginkgo}" --dry-run=true "${e2e_test}" -- --spec-dump "${spec_summaries}" > /dev/null
67 fi
68}
69
70# evaluate ginkgo spec summaries against e2e test ownership polices
71# output to ${results_json}
72function generate_results_json() {
73 readonly results_jq=${tmpdir}/results.jq
74 cat >"${results_jq}" <<EOS
75 [.[] | select( .LeafNodeType == "It") | . as { ContainerHierarchyTexts: \$text, ContainerHierarchyLocations: \$code, LeafNodeText: \$leafText, LeafNodeLocation: \$leafCode} | {
76 calls: ([ \$text | range(0;length) as \$i | {
77 sig: ((\$text[\$i] | match("\\\[(sig-[^\\\]]+)\\\]") | .captures[0].string) // "unknown"),
78 text: \$text[\$i],
79 # unused, but if we ever wanted to have policies based on other tags...
80 # tags: \$text[\$i] | [match("(\\\[[^\\\]]+\\\])"; "g").string],
81 line: \$code[\$i] | "\(.FileName):\(.LineNumber)"
82 }] + [{
83 sig: ((\$leafText | match("\\\[(sig-[^\\\]]+)\\\]") | .captures[0].string) // "unknown"),
84 text: \$leafText,
85 # unused, but if we ever wanted to have policies based on other tags...
86 # tags: \$leafText | [match("(\\\[[^\\\]]+\\\])"; "g").string],
87 line: \$leafCode | "\(.FileName):\(.LineNumber)"
88 }]),
89 } | {
90 owner: .calls[0].sig,
91 calls: .calls,
92 testname: .calls | map(.text) | join(" "),
93 policies: [(
94 .calls[0] |
95 {
96 fail: (.sig == "unknown"),
97 level: "FAIL",
98 category: "unowned_test",
99 reason: "must start with [sig-foo]",
100 found: .,
101 }
102 ), (
103 .calls[1:] |
104 (map(select(.sig != "unknown")) // [] | {
105 fail: . | any,
106 level: "WARN",
107 category: "too_many_sigs",
108 reason: "should not have multiple [sig-foo] tags",
109 found: .,
110 })
111 )
112 ]
113 }]
114EOS
115 if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
116 echo "about to ${results_jq}..."
117 cat -n "${results_jq}"
118 echo
119 fi
120 <"${spec_summaries}" jq --slurp --from-file "${results_jq}" > "${results_json}"
121}
122
123# summarize e2e test policy results
124# output to ${summary_json}
125function generate_summary_json() {
126 summary_jq=${tmpdir}/summary.jq
127 cat >"${summary_jq}" <<EOS
128 . as \$results |
129 # for each policy category
130 reduce \$results[0].policies[] as \$p ({}; . + {
131 # add a convenience .policy field containing that policy's result
132 (\$p.category): \$results | map(. + {policy: .policies[] | select(.category == \$p.category)}) | {
133 level: \$p.level,
134 reason: \$p.reason,
135 passing: map(select(.policy.fail | not)) | length,
136 failing: map(select(.policy.fail)) | length,
137 testnames: map(select(.policy.fail) | .testname),
138 }
139 })
140 # add a meta policy based on whether any policy failed
141 + {
142 all_policies: \$results | {
143 level: "WARN",
144 reason: "should pass all policies",
145 passing: map(select(.policies | map(.fail) | any | not)) | length,
146 failing: map(select(.policies | map(.fail) | any)) | length,
147 testnames: map(select(.policies | map(.fail) | any) | .testname),
148 }
149 }
150 # if a policy has no failing tests, change its log output to PASS
151 | with_entries(.value += { log: (if (.value.failing == 0) then "PASS" else .value.level end) })
152 # sort by policies with the most failing tests first
153 | to_entries | sort_by(.value.failing) | reverse | from_entries
154EOS
155 if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
156 echo "about to run ${results_jq}..."
157 cat -n "${summary_jq}"
158 echo
159 fi
160 <"${results_json}" jq --from-file "${summary_jq}" > "${summary_json}"
161}
162
163# filter e2e policy tests results to tests that failed, with the policies they failed
164# output to ${failures_json}
165function generate_failures_json() {
166 local -r failures_jq="${tmpdir}/failures.jq"
167 cat >"${failures_jq}" <<EOS
168 .
169 # for each test
170 | map(
171 # filter down to failing policies; trim category, .reason is more verbose
172 .policies |= map(select(.fail) | del(.category))
173 # trim the full callstack, .found will contain the relevant call
174 | del(.calls)
175 )
176 # filter down to tests that have failed policies
177 | map(select(.policies | map (.fail) | any))
178EOS
179 if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
180 echo "about to run ${failures_jq}..."
181 cat -n "${failures_jq}"
182 echo
183 fi
184 <"${results_json}" jq --from-file "${failures_jq}" > "${failures_json}"
185}
186
187function output_results_and_exit_if_failed() {
188 local -r total_tests=$(<"${spec_summaries}" wc -l | awk '{print $1}')
189
190 # output results to console
191 (
192 echo "run at datetime: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
193 echo "based on commit: $(git log -n1 --date=iso-strict --pretty='%h - %cd - %s')"
194 echo
195 <"${failures_json}" cat
196 printf "%4s: e2e tests %-40s: %-4d\n" "INFO" "in total" "${total_tests}"
197 <"${summary_json}" jq -r 'to_entries[].value |
198 "printf \"%4s: ..failing %-40s: %-4d\\n\" \"\(.log)\" \"\(.reason)\" \"\(.failing)\""' | sh
199 ) | tee "${tmpdir}/output.txt"
200 # if we said "FAIL" in that output, we should fail
201 if <"${tmpdir}/output.txt" grep -q "^FAIL"; then
202 echo "FAIL"
203 exit 1
204 fi
205}
206
207ensure_dependencies
208generate_results_json
209generate_failures_json
210generate_summary_json
211output_results_and_exit_if_failed
212echo "PASS"
View as plain text