1#############################################################################
2## The top section of this file is identical in the 3 cloudbuild.*yaml files.
3## Make sure any edits you make here are copied over to the other files too
4## if appropriate.
5##
6## TODO(al): consider if it's possible to merge these 3 files and control via
7## substitutions.
8#############################################################################
9
10timeout: 1200s
11options:
12 machineType: N1_HIGHCPU_32
13 volumes:
14 - name: go-modules
15 path: /go
16 env:
17 - GOPROXY=https://proxy.golang.org
18 - PROJECT_ROOT=github.com/google/certificate-transparency-go
19 - GOPATH=/go
20
21substitutions:
22 _CLUSTER_NAME: trillian-opensource-ci
23 _MASTER_ZONE: us-central1-a
24
25# Cloud Build logs sent to GCS bucket
26logsBucket: 'gs://trillian-cloudbuild-logs'
27
28steps:
29# First build a "ct_testbase" docker image which contains most of the tools we need for the later steps:
30- name: 'gcr.io/cloud-builders/docker'
31 entrypoint: 'bash'
32 args: ['-c', 'docker pull gcr.io/$PROJECT_ID/ct_testbase:latest || exit 0']
33- name: 'gcr.io/cloud-builders/docker'
34 args: [
35 'build',
36 '-t', 'gcr.io/$PROJECT_ID/ct_testbase:latest',
37 '--cache-from', 'gcr.io/$PROJECT_ID/ct_testbase:latest',
38 '-f', './integration/Dockerfile',
39 '.'
40 ]
41
42# prepare spins up an ephemeral trillian instance for testing use.
43- name: gcr.io/$PROJECT_ID/ct_testbase
44 entrypoint: 'bash'
45 id: 'prepare'
46 args:
47 - '-exc'
48 - |
49 # Use latest versions of Trillian docker images built by the Trillian CI cloudbuilders.
50 docker pull gcr.io/$PROJECT_ID/log_server:latest
51 docker tag gcr.io/$PROJECT_ID/log_server:latest deployment_trillian-log-server
52 docker pull gcr.io/$PROJECT_ID/log_signer:latest
53 docker tag gcr.io/$PROJECT_ID/log_signer:latest deployment_trillian-log-signer
54
55 # Bring up an ephemeral trillian instance using the docker-compose config in the Trillian repo:
56 export TRILLIAN_LOCATION="$$(go list -f '{{.Dir}}' github.com/google/trillian)"
57
58 # We need to fix up Trillian's docker-compose to connect to the CloudBuild network to that tests can use it:
59 echo -e "networks:\n default:\n external:\n name: cloudbuild" >> $${TRILLIAN_LOCATION}/examples/deployment/docker-compose.yml
60
61 docker-compose -f $${TRILLIAN_LOCATION}/examples/deployment/docker-compose.yml pull mysql trillian-log-server trillian-log-signer
62 docker-compose -f $${TRILLIAN_LOCATION}/examples/deployment/docker-compose.yml up -d mysql trillian-log-server trillian-log-signer
63
64# Install proto related bits and block on Trillian being ready
65- name: gcr.io/$PROJECT_ID/ct_testbase
66 id: 'ci-ready'
67 entrypoint: 'bash'
68 args:
69 - '-ec'
70 - |
71 go install \
72 github.com/golang/protobuf/proto \
73 github.com/golang/protobuf/protoc-gen-go \
74 github.com/golang/mock/mockgen \
75 go.etcd.io/etcd/v3 go.etcd.io/etcd/etcdctl/v3 \
76 github.com/fullstorydev/grpcurl/cmd/grpcurl
77
78 # Generate all protoc and mockgen files
79 go generate -run="protoc" ./...
80 go generate -run="mockgen" ./...
81
82 # Cache all the modules we'll need too
83 go mod download
84 go test ./...
85
86 # Wait for trillian logserver to be up
87 until nc -z deployment_trillian-log-server_1 8090; do echo .; sleep 5; done
88 waitFor: ['prepare']
89
90# Run the presubmit tests
91- name: gcr.io/$PROJECT_ID/ct_testbase
92 id: 'default_test'
93 env:
94 - 'GOFLAGS='
95 - 'PRESUBMIT_OPTS=--no-linters --no-generate'
96 - 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
97 - 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
98 waitFor: ['ci-ready']
99
100- name: gcr.io/$PROJECT_ID/ct_testbase
101 id: 'race_detection'
102 env:
103 - 'GOFLAGS=-race'
104 - 'PRESUBMIT_OPTS=--no-linters --no-generate'
105 - 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
106 - 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
107 waitFor: ['ci-ready']
108
109- name: gcr.io/$PROJECT_ID/ct_testbase
110 id: 'etcd_with_coverage'
111 env:
112 - 'GOFLAGS='
113 - 'PRESUBMIT_OPTS=--no-linters --no-generate --coverage'
114 - 'WITH_ETCD=true'
115 - 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
116 - 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
117 waitFor: ['ci-ready']
118
119- name: gcr.io/$PROJECT_ID/ct_testbase
120 id: 'etcd_with_race'
121 env:
122 - 'GOFLAGS=-race'
123 - 'PRESUBMIT_OPTS=--no-linters --no-generate'
124 - 'WITH_ETCD=true'
125 - 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
126 - 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
127 waitFor: ['ci-ready']
128
129- name: gcr.io/$PROJECT_ID/ct_testbase
130 id: 'with_pkcs11_and_race'
131 env:
132 - 'GOFLAGS=-race --tags=pkcs11'
133 - 'PRESUBMIT_OPTS=--no-linters --no-generate'
134 - 'WITH_PKCS11=true'
135 - 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
136 - 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
137 waitFor: ['ci-ready']
138
139# Collect and submit codecoverage reports
140- name: 'gcr.io/cloud-builders/curl'
141 id: 'codecov.io'
142 entrypoint: bash
143 args: ['-c', 'bash <(curl -s https://codecov.io/bash)']
144 env:
145 - 'VCS_COMMIT_ID=$COMMIT_SHA'
146 - 'VCS_BRANCH_NAME=$BRANCH_NAME'
147 - 'VCS_PULL_REQUEST=$_PR_NUMBER'
148 - 'CI_BUILD_ID=$BUILD_ID'
149 - 'CODECOV_TOKEN=$_CODECOV_TOKEN' # _CODECOV_TOKEN is specified in the cloud build trigger
150 waitFor: ['etcd_with_coverage']
151
152- name: gcr.io/$PROJECT_ID/ct_testbase
153 id: 'ci_complete'
154 entrypoint: /bin/true
155 waitFor: ['codecov.io', 'default_test', 'race_detection', 'etcd_with_coverage', 'etcd_with_race', 'with_pkcs11_and_race']
156
157############################################################################
158## End of replicated section.
159## Below are deployment specific steps for the CD env.
160############################################################################
161
162- id: build_ctfe
163 name: gcr.io/cloud-builders/docker
164 args:
165 - build
166 - --file=trillian/examples/deployment/docker/ctfe/Dockerfile
167 - --tag=gcr.io/${PROJECT_ID}/ctfe:${COMMIT_SHA}
168 - --cache-from=gcr.io/${PROJECT_ID}/ctfe
169 - .
170 waitFor: ["-"]
171- id: push_ctfe
172 name: gcr.io/cloud-builders/docker
173 args:
174 - push
175 - gcr.io/${PROJECT_ID}/ctfe:${COMMIT_SHA}
176 waitFor:
177 - build_ctfe
178- id: tag_latest_ctfe
179 name: gcr.io/cloud-builders/gcloud
180 args:
181 - container
182 - images
183 - add-tag
184 - gcr.io/${PROJECT_ID}/ctfe:${COMMIT_SHA}
185 - gcr.io/${PROJECT_ID}/ctfe:latest
186 waitFor:
187 - push_ctfe
188- id: build_envsubst
189 name: gcr.io/cloud-builders/docker
190 args:
191 - build
192 - trillian/examples/deployment/docker/envsubst
193 - -t
194 - envsubst
195 waitFor: ["-"]
196- id: envsubst_kubernetes_configs
197 name: envsubst
198 args:
199 - trillian/examples/deployment/kubernetes/ctfe-deployment.yaml
200 - trillian/examples/deployment/kubernetes/ctfe-service.yaml
201 - trillian/examples/deployment/kubernetes/ctfe-ingress.yaml
202 env:
203 - PROJECT_ID=${PROJECT_ID}
204 - IMAGE_TAG=${COMMIT_SHA}
205 waitFor:
206 - build_envsubst
207- id: update_kubernetes_configs
208 name: gcr.io/cloud-builders/kubectl
209 args:
210 - apply
211 - -f=trillian/examples/deployment/kubernetes/ctfe-deployment.yaml
212 - -f=trillian/examples/deployment/kubernetes/ctfe-service.yaml
213 - -f=trillian/examples/deployment/kubernetes/ctfe-ingress.yaml
214 env:
215 - CLOUDSDK_COMPUTE_ZONE=${_MASTER_ZONE}
216 - CLOUDSDK_CONTAINER_CLUSTER=${_CLUSTER_NAME}
217 waitFor:
218 - envsubst_kubernetes_configs
219 - push_ctfe
220
221images:
222- gcr.io/${PROJECT_ID}/ctfe:${COMMIT_SHA}
223- gcr.io/${PROJECT_ID}/ct_testbase:latest
View as plain text