1import os
2import sys
3import time
4from typing import Any, Optional
5
6import pexpect
7import pytest
8import requests
9
10DOCKER_IMAGE = os.environ.get("AMBASSADOR_DOCKER_IMAGE", None)
11
12child: Optional[pexpect.spawnbase.SpawnBase] = None # see docker_start()
13diagd_host: Optional[str] = None # see docker_start()
14child_name = "diagd-unset" # see docker_start() and docker_kill()
15
16SEQUENCES = [
17 (["env_ok", "chime"], ["boot1", "now-healthy"]),
18 (["env_ok", "chime", "scout_cache_reset", "chime"], ["boot1", "now-healthy", "healthy"]),
19 (["env_ok", "chime", "env_bad", "chime"], ["boot1", "now-healthy", "now-unhealthy"]),
20 (["env_bad", "chime"], ["boot1", "unhealthy"]),
21 (
22 ["env_bad", "chime", "chime", "scout_cache_reset", "chime"],
23 ["boot1", "unhealthy", "unhealthy"],
24 ),
25 (
26 ["chime", "chime", "chime", "env_ok", "chime", "chime"],
27 ["boot1", "unhealthy", "now-healthy"],
28 ),
29]
30
31
32def docker_start(logfile) -> bool:
33 # Use a global here so that the child process doesn't get killed
34 global child
35
36 global child_name
37 child_name = f"diagd-{int(time.time() * 1000)}"
38
39 global diagd_host
40
41 cmd = f"docker run --name {child_name} --rm -p 9999:9999 {DOCKER_IMAGE} --dev-magic"
42 diagd_host = "localhost:9999"
43
44 child = pexpect.spawn(cmd, encoding="utf-8")
45 child.logfile = logfile
46
47 i = child.expect([pexpect.EOF, pexpect.TIMEOUT, "LocalScout: mode boot, action boot1"])
48
49 if i == 0:
50 print("diagd died?")
51 return False
52 elif i == 1:
53 print("diagd timed out?")
54 return False
55
56 # Set up port forwarding in the Ambassador container from all:9999, where
57 # this test will connect, to localhost:9998, where diagd is listening. This
58 # is necessary because diagd rejects (403) requests originating outside the
59 # container for security reasons.
60
61 # Copy the simple port forwarding script into the container
62 child2 = pexpect.spawn(
63 f"docker cp python/tests/_forward.py {child_name}:/tmp/", encoding="utf-8"
64 )
65 child2.logfile = logfile
66
67 if child2.expect([pexpect.EOF, pexpect.TIMEOUT]) == 1:
68 print("docker cp timed out?")
69 return False
70
71 child2.close()
72 if child2.exitstatus != 0:
73 print("docker cp failed?")
74 return False
75
76 # Run the port forwarding script
77 child2 = pexpect.spawn(
78 f'docker exec -d {child_name} python /tmp/_forward.py localhost 9998 "" 9999',
79 encoding="utf-8",
80 )
81 child2.logfile = logfile
82
83 if child2.expect([pexpect.EOF, pexpect.TIMEOUT]) == 1:
84 print("docker exec timed out?")
85 return False
86
87 child2.close()
88 if child2.exitstatus != 0:
89 print("docker exec failed?")
90 return False
91
92 return True
93
94
95def docker_kill(logfile):
96 cmd = f"docker kill {child_name}"
97
98 child = pexpect.spawn(cmd, encoding="utf-8")
99 child.logfile = logfile
100
101 child.expect([pexpect.EOF, pexpect.TIMEOUT])
102
103
104def wait_for_diagd(logfile) -> bool:
105 status = False
106 tries_left = 5
107
108 while tries_left >= 0:
109 logfile.write(f"...checking diagd ({tries_left})\n")
110
111 try:
112 global diagd_host
113 response = requests.get(
114 f"http://{diagd_host}/_internal/v0/ping",
115 headers={"X-Ambassador-Diag-IP": "127.0.0.1"},
116 )
117
118 if response.status_code == 200:
119 logfile.write(" got it\n")
120 status = True
121 break
122 else:
123 logfile.write(f" failed {response.status_code}\n")
124 except requests.exceptions.RequestException as e:
125 logfile.write(f" failed {e}\n")
126
127 tries_left -= 1
128 time.sleep(2)
129
130 return status
131
132
133def check_http(logfile, cmd: str) -> bool:
134 try:
135 global diagd_host
136 response = requests.post(
137 f"http://{diagd_host}/_internal/v0/fs",
138 headers={"X-Ambassador-Diag-IP": "127.0.0.1"},
139 params={"path": f"cmd:{cmd}"},
140 )
141 text = response.text
142
143 if response.status_code != 200:
144 logfile.write(f"{cmd}: wanted 200 but got {response.status_code} {text}\n")
145 return False
146
147 return True
148 except Exception as e:
149 logfile.write(f"Could not do HTTP: {e}\n")
150
151 return False
152
153
154def fetch_events(logfile) -> Any:
155 try:
156 global diagd_host
157 response = requests.get(
158 f"http://{diagd_host}/_internal/v0/events",
159 headers={"X-Ambassador-Diag-IP": "127.0.0.1"},
160 )
161
162 if response.status_code != 200:
163 logfile.write(f"events: wanted 200 but got {response.status_code} {response.text}\n")
164 return None
165
166 data = response.json()
167
168 return data
169 except Exception as e:
170 logfile.write(f"events: could not do HTTP: {e}\n")
171
172 return None
173
174
175def check_chimes(logfile) -> bool:
176 result = True
177
178 i = 0
179
180 covered = {
181 "F-F-F": False,
182 "F-F-T": False,
183 # 'F-T-F': False, # This particular key can never be generated
184 # 'F-T-T': False, # This particular key can never be generated
185 "T-F-F": False,
186 "T-F-T": False,
187 "T-T-F": False,
188 "T-T-T": False,
189 }
190
191 for cmds, wanted_verdict in SEQUENCES:
192 logfile.write(f"RESETTING for sequence {i}\n")
193
194 if not check_http(logfile, "chime_reset"):
195 logfile.write(f"could not reset for sequence {i}\n")
196 result = False
197 continue
198
199 j = 0
200 for cmd in cmds:
201 logfile.write(f" sending {cmd} for sequence {i}.{j}\n")
202
203 if not check_http(logfile, cmd):
204 logfile.write(f"could not do {cmd} for sequence {i}.{j}\n")
205 result = False
206 break
207
208 j += 1
209
210 if not result:
211 continue
212
213 events = fetch_events(logfile)
214
215 if not events:
216 result = False
217 continue
218
219 # logfile.write(json.dumps(events, sort_keys=True, indent=4))
220
221 logfile.write(" ----\n")
222 verdict = []
223
224 for timestamp, mode, action, data in events:
225 verdict.append(action)
226
227 action_key = data.get("action_key", None)
228
229 if action_key:
230 covered[action_key] = True
231
232 logfile.write(f" {action} - {action_key}\n")
233
234 # logfile.write(json.dumps(verdict, sort_keys=True, indent=4\n))
235
236 if verdict != wanted_verdict:
237 logfile.write(f"verdict mismatch for sequence {i}:\n")
238 logfile.write(f' wanted {" ".join(wanted_verdict)}\n')
239 logfile.write(f' got {" ".join(verdict)}\n')
240
241 i += 1
242
243 for key in sorted(covered.keys()):
244 if not covered[key]:
245 logfile.write(f"missing coverage for {key}\n")
246 result = False
247
248 return result
249
250
251@pytest.mark.flaky(reruns=1, reruns_delay=10)
252def test_scout():
253 test_status = False
254
255 with open("/tmp/test_scout_output", "w") as logfile:
256 if not DOCKER_IMAGE:
257 logfile.write("No $AMBASSADOR_DOCKER_IMAGE??\n")
258 else:
259 if docker_start(logfile):
260 if wait_for_diagd(logfile) and check_chimes(logfile):
261 test_status = True
262
263 docker_kill(logfile)
264
265 if not test_status:
266 with open("/tmp/test_scout_output", "r") as logfile:
267 for line in logfile:
268 print(line.rstrip())
269
270 assert test_status, "test failed"
271
272
273if __name__ == "__main__":
274 pytest.main(sys.argv)
View as plain text