1#!/usr/bin/env bash
2
3# Copyright 2023 The Kubernetes Authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17# This script does following:
18# 1. Creates local registry if not already present. This registry is used to push the kms mock plugin image.
19# 2. Build and push the kms mock plugin image to the local registry.
20# 3. Connect local registry to kind network so that kind cluster created using kubetest2 in prow CI job can pull the kms mock plugin image.
21# 4. Create kind cluster using kubetest2 and run e2e tests.
22# 5. Collect logs and metrics from kind cluster.
23
24set -o errexit
25set -o nounset
26set -o pipefail
27
28readonly cluster_name="kms"
29readonly registry_name="kind-registry"
30readonly kind_network="kind"
31
32# build_and_push_mock_plugin builds and pushes the kms mock plugin image to the local registry.
33build_and_push_mock_plugin() {
34 docker buildx build \
35 --no-cache \
36 --platform linux/amd64 \
37 --output=type=docker \
38 -t localhost:5000/mock-kms-provider:e2e \
39 -f staging/src/k8s.io/kms/internal/plugins/_mock/Dockerfile staging/src/k8s.io/ \
40 --progress=plain;
41
42 docker push localhost:5000/mock-kms-provider:e2e
43}
44
45# create_registry creates local registry if not already present.
46create_registry() {
47 running="$(docker inspect -f '{{.State.Running}}' "${registry_name}" 2>/dev/null || true)"
48 if [ "${running}" != 'true' ]; then
49 echo "Creating local registry"
50 docker run \
51 -d --restart=always -p "5000:5000" --name "${registry_name}" \
52 registry:2
53 else
54 echo "Local registry is already running"
55 fi
56}
57
58# connect_registry connects local registry to kind network.
59connect_registry(){
60 # wait for the kind network to exist
61 # infinite loop here is fine because kubetest2 will timeout if kind cluster creation fails and that will terminate the CI job
62 for ((; ;)); do
63 if docker network ls | grep "${kind_network}"; then
64 break
65 else
66 echo "'docker network ls' does not have '${kind_network}' network yet. Retrying in 1 second..."
67 sleep 1
68 fi
69 done
70
71 containers=$(docker network inspect "${kind_network}" -f "{{range .Containers}}{{.Name}} {{end}}")
72 needs_connect="true"
73 for c in $containers; do
74 if [ "$c" = "${registry_name}" ]; then
75 needs_connect="false"
76 fi
77 done
78
79 if [ "${needs_connect}" = "true" ]; then
80 echo "connecting kind network to local registry"
81 docker network connect "${kind_network}" "${registry_name}"
82 else
83 echo "'${kind_network}' network is already connected to local registry"
84 fi
85}
86
87# create_cluster_and_run_test creates a kind cluster using kubetest2 and runs e2e tests.
88create_cluster_and_run_test() {
89 CLUSTER_CREATE_ATTEMPTED=true
90
91 TEST_ARGS=""
92 if [ "${SKIP_RUN_TESTS:-}" != "true" ]; then
93 # (--use-built-binaries) use the kubectl, e2e.test, and ginkgo binaries built during --build as opposed to from a GCS release tarball
94 TEST_ARGS="--test=ginkgo -- --focus-regex=\[Conformance\] --skip-regex=\[Serial\] --parallel 20 --use-built-binaries"
95 else
96 echo "Skipping running tests"
97 fi
98
99 # shellcheck disable=SC2086
100 kubetest2 kind -v 5 \
101 --build \
102 --up \
103 --rundir-in-artifacts \
104 --config test/e2e/testing-manifests/auth/encrypt/kind.yaml \
105 --cluster-name "${cluster_name}" ${TEST_ARGS}
106}
107
108cleanup() {
109 # CLUSTER_CREATE_ATTEMPTED is true once we run kubetest2 kind --up
110 if [ "${CLUSTER_CREATE_ATTEMPTED:-}" = true ]; then
111 if [ "${SKIP_COLLECT_LOGS:-}" != "true" ]; then
112 # collect logs and metrics
113 echo "Collecting logs"
114 mkdir -p "${ARTIFACTS}/logs"
115 kind "export" logs "${ARTIFACTS}/logs" --name "${cluster_name}"
116
117 echo "Collecting metrics"
118 mkdir -p "${ARTIFACTS}/metrics"
119 kubectl get --raw /metrics > "${ARTIFACTS}/metrics/kube-apiserver-metrics.txt"
120 else
121 echo "Skipping collecting logs and metrics"
122 fi
123
124 if [ "${SKIP_DELETE_CLUSTER:-}" != "true" ]; then
125 echo "Deleting kind cluster"
126 # delete cluster
127 kind delete cluster --name "${cluster_name}"
128 else
129 echo "Skipping deleting kind cluster"
130 fi
131 fi
132}
133
134main(){
135 # ensure artifacts (results) directory exists when not in CI
136 export ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
137 mkdir -p "${ARTIFACTS}"
138
139 export GO111MODULE=on;
140 go install sigs.k8s.io/kind@latest;
141 go install sigs.k8s.io/kubetest2@latest;
142 go install sigs.k8s.io/kubetest2/kubetest2-kind@latest;
143 go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest;
144
145 # The build e2e.test, ginkgo and kubectl binaries + copy to dockerized dir is
146 # because of https://github.com/kubernetes-sigs/kubetest2/issues/184
147 make all WHAT="test/e2e/e2e.test vendor/github.com/onsi/ginkgo/v2/ginkgo cmd/kubectl";
148 mkdir -p _output/dockerized/bin/linux/amd64;
149 for binary in kubectl e2e.test ginkgo; do
150 cp -f _output/local/go/bin/${binary} _output/dockerized/bin/linux/amd64/${binary};
151 done;
152
153 create_registry
154 build_and_push_mock_plugin
155 connect_registry &
156 create_cluster_and_run_test
157 cleanup
158}
159
160trap cleanup INT TERM
161main "$@"
View as plain text