1#!/bin/bash
2#
3# Copyright 2024 The Sigstore Authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17set -o errexit
18
19cleanup() {
20 code=$?
21 if [ $code -ne 0 ] ; then
22 echo "An error occurred. Waiting 30 seconds to start cleanup, press ^C to cancel cleanup."
23 sleep 30
24 fi
25 $@
26 exit $code
27}
28
29INSERT_RUNS=${INSERT_RUNS:-1000}
30SEARCH_ENTRIES=${SEARCH_ENTRIES:-100000}
31INDEX_BACKEND=${INDEX_BACKEND:-redis}
32REGION=${REGION:-us-west1}
33
34setup_bastion() {
35 echo "Configuring the bastion..."
36 sudo apt install kubernetes-client google-cloud-sdk-gke-gcloud-auth-plugin git redis-tools gnuplot prometheus minisign jq -y
37 if ! which hyperfine >/dev/null ; then
38 local tag=$(curl -H "Accept: application/json" -L https://github.com/sharkdp/hyperfine/releases/latest | jq -r .tag_name)
39 wget -O /tmp/hyperfine_${tag:1}_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/${tag}/hyperfine_${tag:1}_amd64.deb
40 sudo dpkg -i /tmp/hyperfine_${tag:1}_amd64.deb
41 fi
42 if ! which helm >/dev/null ; then
43 curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
44 fi
45 if ! which rekor-cli >/dev/null ; then
46 wget -O /tmp/rekor-cli-linux-amd64 https://github.com/sigstore/rekor/releases/latest/download/rekor-cli-linux-amd64
47 sudo install -m 0755 /tmp/rekor-cli-linux-amd64 /usr/local/bin/rekor-cli
48 fi
49 gcloud auth print-access-token >/dev/null 2>&1 || gcloud auth login
50 gcloud container clusters get-credentials rekor --region $REGION
51}
52
53setup_rekor() {
54 echo "Setting up rekor..."
55 helm repo add sigstore https://sigstore.github.io/helm-charts
56 helm repo update
57
58 sha=$(git ls-remote https://github.com/sigstore/rekor HEAD | awk '{print substr($1, 1, 7)}')
59 cat >values.yaml <<EOF
60server:
61 ingress:
62 enabled: false
63 image:
64 repository: projectsigstore/rekor/ci/rekor/rekor-server
65 version: '$sha'
66EOF
67
68 if [ "$INDEX_BACKEND" == "redis" ] ; then
69 export REDIS_IP=$(gcloud redis instances describe rekor-index --region $REGION --format='get(host)')
70 cat >index-values.yaml <<EOF
71redis:
72 enabled: false
73 hostname: $REDIS_IP
74server:
75 extraArgs:
76 - --search_index.storage_provider=redis
77EOF
78 helm upgrade -i rekor sigstore/rekor -n rekor-system --create-namespace --values values.yaml --values index-values.yaml
79 kubectl -n rekor-system rollout status deploy rekor-server
80 else
81 export MYSQL_IP=$(gcloud sql instances describe rekor-perf-tf --format='get(ipAddresses[0].ipAddress)')
82 cat >index-values.yaml <<EOF
83server:
84 extraArgs:
85 - --search_index.storage_provider=mysql
86 - --search_index.mysql.dsn=trillian:\$(MYSQL_PASSWORD)@tcp(${MYSQL_IP}:3306)/trillian
87EOF
88 helm upgrade -i rekor sigstore/rekor -n rekor-system --create-namespace --values values.yaml --values mysql-args-values.yaml
89 echo -n $MYSQL_PASS | kubectl -n rekor-system create secret generic mysql-credentials --save-config --dry-run=client --output=yaml --from-file=mysql-password=/dev/stdin | kubectl apply -f -
90 cat > patch.yaml <<EOF
91spec:
92 template:
93 spec:
94 containers:
95 - name: rekor-server
96 env:
97 - name: MYSQL_PASSWORD
98 valueFrom:
99 secretKeyRef:
100 name: mysql-credentials
101 key: mysql-password
102EOF
103 kubectl -n rekor-system patch deployment rekor-server --patch-file=patch.yaml
104 kubectl -n rekor-system rollout status deploy rekor-server
105 fi
106 kubectl apply -f - <<EOF
107apiVersion: v1
108kind: Service
109metadata:
110 name: rekor-server-nodeport
111 namespace: rekor-system
112spec:
113 selector:
114 app.kubernetes.io/component: server
115 app.kubernetes.io/instance: rekor
116 app.kubernetes.io/name: rekor
117 ports:
118 - name: http
119 port: 80
120 protocol: TCP
121 targetPort: 3000
122 nodePort: 30080
123 - name: metrics
124 port: 2112
125 protocol: TCP
126 targetPort: 2112
127 nodePort: 32112
128 type: NodePort
129EOF
130
131 node_address=$(kubectl get node -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')
132 export REKOR_URL=http://${node_address}:30080
133 export REKOR_METRICS=${node_address}:32112
134}
135
136setup_prometheus() {
137 echo "Setting up prometheus..."
138 sudo systemctl disable --now prometheus
139 mkdir -p prometheus >/dev/null
140 rm -rf prometheus/metrics2
141 cat >prometheus/prometheus.yml <<EOF
142scrape_configs:
143 - job_name: 'prometheus'
144 scrape_interval: 1s
145 static_configs:
146 - targets:
147 - '$REKOR_METRICS'
148EOF
149 setsid prometheus --storage.tsdb.path=./prometheus/metrics2 --config.file=prometheus/prometheus.yml >prometheus/prom.log 2>&1 &
150 export PROM_PID=$!
151}
152
153# Upload $INSERT_RUNS rekords of $INSERT_RUNS artifacts signed by 1 key
154insert() {
155 echo "Inserting entries..."
156 local N=$INSERT_RUNS
157 # Create N artifacts with different contents
158 export DIR=$(mktemp -d)
159 for i in $(seq 1 $N) ; do
160 echo hello${i} > ${DIR}/blob${i}
161 done
162 # Create a signing key
163 minisign -G -p $DIR/user1@example.com.pub -s $DIR/user1@example.com.key -W >/dev/null
164
165 echo "Signing $N artifacts with 1 key"
166 user=user1@example.com
167 local batch=0
168 while [ $batch -lt $N ] ; do
169 for i in $(seq 1 100) ; do
170 let id=$batch+$i
171 if [ $id -gt $N ] ; then
172 break
173 fi
174 sig=${DIR}/$(uuidgen).asc
175 (
176 minisign -S -x $sig -s $DIR/$user.key -m ${DIR}/blob${id}
177 rekor-cli upload --rekor_server $REKOR_URL --signature $sig --public-key ${DIR}/${user}.pub --artifact ${DIR}/blob${id} --pki-format=minisign
178 ) &
179 done
180 wait $(jobs -p | grep -v $PROM_PID)
181 let batch+=100
182 done
183
184 rm -rf $DIR
185}
186
187query_inserts() {
188 echo "Getting metrics for inserts..."
189 count=null
190
191 # may need to wait for the data to be scraped
192 tries=0
193 until [ "${count}" != "null" ] ; do
194 sleep 1
195 count=$(curl -s http://localhost:9090/api/v1/query --data-urlencode 'query=rekor_index_storage_latency_summary_count{success="true"}' | jq -r .data.result[0].value[1])
196 let 'tries+=1'
197 if [ $tries -eq 6 ] ; then
198 echo "count query failed, here is the raw result:"
199 set -x
200 curl -s http://localhost:9090/api/v1/query --data-urlencode 'query=rekor_index_storage_latency_summary_count{success="true"}'
201 set +x
202 echo
203 exit 1
204 fi
205 done
206
207 avg=$(curl -s http://localhost:9090/api/v1/query --data-urlencode 'query=rekor_index_storage_latency_summary_sum{success="true"}/rekor_index_storage_latency_summary_count{success="true"}' | jq -r .data.result[0].value[1])
208
209 if [ "${avg}" == "null" ] ; then
210 echo "avg query failed, here is the raw result:"
211 set -x
212 curl -s http://localhost:9090/api/v1/query --data-urlencode 'query=rekor_index_storage_latency_summary_sum{success="true"}/rekor_index_storage_latency_summary_count{success="true"}'
213 set +x
214 echo
215 exit 1
216 fi
217
218 echo "Insert latency: ${avg} (average over ${count} inserts)"
219 results=${INDEX_BACKEND}.dat
220 if [ "$INDEX_BACKEND" == "redis" ] ; then
221 x=1
222 else
223 x=0
224 fi
225 # output to gnuplot data set
226 echo "$x \"${INDEX_BACKEND} inserts\n(${count})\" $avg" > $results
227}
228
229upload() {
230 echo "Uploading entries..."
231 N=$SEARCH_ENTRIES
232
233 if [ ! -f indices.csv ] ; then
234 echo "Generating $N * 2 entries. This may take a while..."
235 # N artifacts, 1 user
236 for i in $(seq 1 $N) ; do
237 uuid=$(dbus-uuidgen)
238 echo user1@example.com,$uuid >> indices.csv
239 sha=$(echo $i | sha256sum | cut -d ' ' -f 1)
240 echo sha256:$sha,$uuid >> indices.csv
241 done
242
243 # 1 artifact, N users
244 sha=$(echo 1 | sha256sum | cut -d ' ' -f 1)
245 for i in $(seq 2 $N) ; do
246 uuid=$(dbus-uuidgen)
247 echo user${i}@example.com,$uuid >> indices.csv
248 echo sha256:$sha,$uuid >> indices.csv
249 done
250 fi
251
252 if [ "${INDEX_BACKEND}" == "redis" ] ; then
253 local dbsize=$(redis-cli -h $REDIS_IP dbsize | cut -d ' ' -f 2)
254 let wantsize=$SEARCH_ENTRIES*2
255 if [ ! $dbsize -ge $wantsize ] ; then
256 echo "Uploading entries into redis..."
257 while read LINE ; do
258 key=$(echo $LINE | cut -d',' -f1)
259 val=$(echo $LINE | cut -d',' -f2)
260 printf "*3\r\n\$5\r\nLPUSH\r\n\$${#key}\r\n${key}\r\n\$${#val}\r\n${val}\r\n"
261 done < indices.csv | redis-cli -h $REDIS_IP --pipe
262 fi
263 else
264 local dbsize=$(mysql -h $MYSQL_IP -P 3306 -utrillian -p${MYSQL_PASS} -D trillian -e "SELECT COUNT(*) FROM EntryIndex" --vertical | tail -1 | cut -d ' ' -f 2)
265 let wantsize=$SEARCH_ENTRIES*4
266 if [ ! $dbsize -ge $wantsize ] ; then
267 echo "Uploading entries into mysql..."
268 mysql -h $MYSQL_IP -P 3306 -utrillian -p${MYSQL_PASS} -D trillian -e "CREATE TABLE IF NOT EXISTS EntryIndex (
269 PK BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
270 EntryKey varchar(512) NOT NULL,
271 EntryUUID char(80) NOT NULL,
272 PRIMARY KEY(PK),
273 UNIQUE(EntryKey, EntryUUID)
274 );
275 LOAD DATA LOCAL INFILE './indices.csv'
276 INTO TABLE EntryIndex
277 FIELDS TERMINATED BY ','
278 LINES TERMINATED BY '\n' (EntryKey, EntryUUID);"
279 fi
280 fi
281}
282
283search() {
284 echo "Running search requests..."
285 sumblob1=$(echo 1 | sha256sum | cut -d ' ' -f1)
286 sumblob2=$(echo 2 | sha256sum | cut -d ' ' -f1)
287 sumblobnone=$(echo none | sha256sum | cut -d ' ' -f1)
288 # Search for entries using public key user1@example.com (should be many), user2@example.com (should be few), notreal@example.com (should be none)
289 hyperfine --style basic --warmup 10 --ignore-failure --parameter-list email user1@example.com,user2@example.com,notreal@example.com "rekor-cli search --rekor_server $REKOR_URL --email {email}"
290 # Search for entries using the sha256 sum of blob1 (should be many), blob2 (should be few), blobnone (should be none)
291 hyperfine --style basic --warmup 10 --ignore-failure --parameter-list sha ${sumblob1},${sumblob2},${sumblobnone} "rekor-cli search --rekor_server $REKOR_URL --sha sha256:{sha}"
292 # Search for entries using public key user1@example.com/user2@example.com/notreal@example.com OR/AND sha256 sum of blob1/blob2/blobnone
293 hyperfine --style basic --warmup 10 --ignore-failure --parameter-list email user1@example.com,user2@example.com,notreal@example.com \
294 --parameter-list sha ${sumblob1},${sumblob2},${sumblobnone} \
295 --parameter-list operator or,and \
296 "rekor-cli search --rekor_server $REKOR_URL --email {email} --sha sha256:{sha} --operator {operator}"
297}
298
299query_search() {
300 echo "Getting metrics for searches..."
301 count=null
302 # may need to wait for the data to be scraped
303 tries=0
304 until [ "${count}" != "null" ] ; do
305 sleep 1
306 count=$(curl -s http://localhost:9090/api/v1/query --data-urlencode 'query=rekor_api_latency_summary_count{path="/api/v1/index/retrieve"}' | jq -r .data.result[0].value[1])
307 let 'tries+=1'
308 if [ $tries -eq 6 ] ; then
309 echo "count query failed, here is the raw result:"
310 set -x
311 curl -s http://localhost:9090/api/v1/query --data-urlencode 'query=rekor_api_latency_summary_count{path="/api/v1/index/retrieve"}'
312 set +x
313 echo
314 fi
315 done
316
317 avg=$(curl -s http://localhost:9090/api/v1/query --data-urlencode 'query=rekor_api_latency_summary_sum{path="/api/v1/index/retrieve"}/rekor_api_latency_summary_count{path="/api/v1/index/retrieve"}' | jq -r .data.result[0].value[1])
318 if [ "${avg}" == "null" ] ; then
319 echo "avg query failed, here is the raw result:"
320 set -x
321 curl -s http://localhost:9090/api/v1/query --data-urlencode 'query=rekor_api_latency_summary_sum{path="/api/v1/index/retrieve"}/rekor_api_latency_summary_count{path="/api/v1/index/retrieve"}'
322 set +x
323 echo
324 fi
325
326 echo "Search latency: ${avg} (average over ${count} searches)"
327 results=${INDEX_BACKEND}.dat
328 if [ "$INDEX_BACKEND" == "redis" ] ; then
329 x=3
330 else
331 x=2
332 fi
333 # output to gnuplot data set
334 echo "$x \"${INDEX_BACKEND} searches\n(${count})\" $avg" >> $results
335}
336
337reset() {
338 echo "Resetting data..."
339 if [ "${INDEX_BACKEND}" == "redis" ] ; then
340 redis-cli -h $REDIS_IP flushall
341 else
342 mysql -h $MYSQL_IP -P 3306 -utrillian -p${MYSQL_PASS} -D trillian -e 'DELETE FROM EntryIndex;'
343 fi
344 kubectl -n rekor-system rollout restart deployment rekor-server
345}
346
347if [ "${BASH_SOURCE[0]}" == "${0}" ]; then
348 if [ "${INDEX_BACKEND}" != "redis" -a "${INDEX_BACKEND}" != "mysql" ] ; then
349 echo '$INDEX_BACKEND must be either redis or mysql.'
350 exit 1
351 fi
352
353 if [ "${INDEX_BACKEND}" == "mysql" -a "${MYSQL_PASS}" == "" ] ; then
354 echo '$MYSQL_PASS must be set when $INDEX_BACKEND is mysql.'
355 echo 'The trillian mysql user password can be found from your terraform host using `terraform output -json | jq -r .mysql_pass.value`.'
356 exit 1
357 fi
358
359 echo "Gathering insertion and retrieval metrics for index backend [${INDEX_BACKEND}]."
360
361 setup_bastion
362
363 setup_rekor
364
365 if [ -n "$RESET" ] ; then
366 reset
367 fi
368
369 setup_prometheus
370 cleanup_prom() {
371 echo "Cleaning up prometheus..."
372 pkill -x prometheus
373 }
374 trap 'cleanup cleanup_prom' EXIT
375
376 insert
377
378 query_inserts
379
380 upload
381
382 search
383
384 query_search
385fi
View as plain text