1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3"""
4This file contains basic infrastructure for running the integration test cases.
5Most test cases are in v2_integration.py. There are a few exceptions: Test cases
6that don't test either the v1 or v2 API are in this file, and test cases that
7have to run at a specific point in the cycle (e.g. after all other test cases)
8are also in this file.
9"""
10import argparse
11import datetime
12import inspect
13import json
14import os
15import random
16import re
17import requests
18import subprocess
19import shlex
20import signal
21import time
22
23import startservers
24
25import v2_integration
26from helpers import *
27
28from acme import challenges
29
30# Set the environment variable RACE to anything other than 'true' to disable
31# race detection. This significantly speeds up integration testing cycles
32# locally.
33race_detection = True
34if os.environ.get('RACE', 'true') != 'true':
35 race_detection = False
36
37def run_go_tests(filterPattern=None):
38 """
39 run_go_tests launches the Go integration tests. The go test command must
40 return zero or an exception will be raised. If the filterPattern is provided
41 it is used as the value of the `--test.run` argument to the go test command.
42 """
43 cmdLine = ["go", "test"]
44 if filterPattern is not None and filterPattern != "":
45 cmdLine = cmdLine + ["--test.run", filterPattern]
46 cmdLine = cmdLine + ["-tags", "integration", "-count=1", "-race", "./test/integration"]
47 subprocess.check_call(cmdLine, stderr=subprocess.STDOUT)
48
49exit_status = 1
50
51def main():
52 parser = argparse.ArgumentParser(description='Run integration tests')
53 parser.add_argument('--chisel', dest="run_chisel", action="store_true",
54 help="run integration tests using chisel")
55 parser.add_argument('--gotest', dest="run_go", action="store_true",
56 help="run Go integration tests")
57 parser.add_argument('--filter', dest="test_case_filter", action="store",
58 help="Regex filter for test cases")
59 # allow any ACME client to run custom command for integration
60 # testing (without having to implement its own busy-wait loop)
61 parser.add_argument('--custom', metavar="CMD", help="run custom command")
62 parser.set_defaults(run_chisel=False, test_case_filter="", skip_setup=False)
63 args = parser.parse_args()
64
65 if not (args.run_chisel or args.custom or args.run_go is not None):
66 raise(Exception("must run at least one of the letsencrypt or chisel tests with --chisel, --gotest, or --custom"))
67
68 if not startservers.install(race_detection=race_detection):
69 raise(Exception("failed to build"))
70
71 # Setup issuance hierarchy
72 startservers.setupHierarchy()
73
74 if not args.test_case_filter:
75 now = datetime.datetime.utcnow()
76
77 six_months_ago = now+datetime.timedelta(days=-30*6)
78 if not startservers.start(fakeclock=fakeclock(six_months_ago)):
79 raise(Exception("startservers failed (mocking six months ago)"))
80 setup_six_months_ago()
81 startservers.stop()
82
83 twenty_days_ago = now+datetime.timedelta(days=-20)
84 if not startservers.start(fakeclock=fakeclock(twenty_days_ago)):
85 raise(Exception("startservers failed (mocking twenty days ago)"))
86 setup_twenty_days_ago()
87 startservers.stop()
88
89 if not startservers.start(fakeclock=None):
90 raise(Exception("startservers failed"))
91
92 if args.run_chisel:
93 run_chisel(args.test_case_filter)
94
95 if args.run_go:
96 run_go_tests(args.test_case_filter)
97
98 if args.custom:
99 run(args.custom.split())
100
101 # Skip the last-phase checks when the test case filter is one, because that
102 # means we want to quickly iterate on a single test case.
103 if not args.test_case_filter:
104 run_cert_checker()
105 check_balance()
106
107 if not startservers.check():
108 raise(Exception("startservers.check failed"))
109
110 # This test is flaky, so it's temporarily disabled.
111 # TODO(#4583): Re-enable this test.
112 #check_slow_queries()
113
114 global exit_status
115 exit_status = 0
116
117def check_slow_queries():
118 """Checks that we haven't run any slow queries during the integration test.
119
120 This depends on flags set on mysqld in docker-compose.yml.
121
122 We skip the boulder_sa_test database because we manually run a bunch of
123 non-indexed queries in unittests. We skip actions by the setup and root
124 users because they're known to be non-indexed. Similarly we skip the
125 cert_checker, mailer, and janitor's work because they are known to be
126 slow (though we should eventually improve these).
127 The SELECT ... IN () on the authz2 table shows up in the slow query log
128 a lot. Presumably when there are a lot of entries in the IN() argument
129 and the table is small, it's not efficient to use the index. But we
130 should dig into this more.
131 """
132 query = """
133 SELECT * FROM mysql.slow_log
134 WHERE db != 'boulder_sa_test'
135 AND user_host NOT LIKE "test_setup%"
136 AND user_host NOT LIKE "root%"
137 AND user_host NOT LIKE "cert_checker%"
138 AND user_host NOT LIKE "mailer%"
139 AND user_host NOT LIKE "janitor%"
140 AND sql_text NOT LIKE 'SELECT status, expires FROM authz2 WHERE id IN %'
141 AND sql_text NOT LIKE '%LEFT JOIN orderToAuthz2 %'
142 \G
143 """
144 output = subprocess.check_output(
145 ["mysql", "-h", "boulder-proxysql", "-e", query],
146 stderr=subprocess.STDOUT).decode()
147 if len(output) > 0:
148 print(output)
149 raise Exception("Found slow queries in the slow query log")
150
151def run_chisel(test_case_filter):
152 for key, value in inspect.getmembers(v2_integration):
153 if callable(value) and key.startswith('test_') and re.search(test_case_filter, key):
154 value()
155 for key, value in globals().items():
156 if callable(value) and key.startswith('test_') and re.search(test_case_filter, key):
157 value()
158
159def check_balance():
160 """Verify that gRPC load balancing across backends is working correctly.
161
162 Fetch metrics from each backend and ensure the grpc_server_handled_total
163 metric is present, which means that backend handled at least one request.
164 """
165 addresses = [
166 "sa1.service.consul:8003",
167 "sa2.service.consul:8103",
168 "publisher1.service.consul:8009",
169 "publisher2.service.consul:8109",
170 "va1.service.consul:8004",
171 "va2.service.consul:8104",
172 "ca1.service.consul:8001",
173 "ca2.service.consul:8104",
174 "ra1.service.consul:8002",
175 "ra2.service.consul:8102",
176 ]
177 for address in addresses:
178 metrics = requests.get("http://%s/metrics" % address)
179 if not "grpc_server_handled_total" in metrics.text:
180 raise(Exception("no gRPC traffic processed by %s; load balancing problem?")
181 % address)
182
183def run_cert_checker():
184 run(["./bin/boulder", "cert-checker", "-config", "%s/cert-checker.json" % config_dir])
185
186if __name__ == "__main__":
187 main()
View as plain text