...

Text file src/github.com/emissary-ingress/emissary/v3/python/tests/integration/test_scout.py

Documentation: github.com/emissary-ingress/emissary/v3/python/tests/integration

     1import os
     2import sys
     3import time
     4from typing import Any, Optional
     5
     6import pexpect
     7import pytest
     8import requests
     9
    10DOCKER_IMAGE = os.environ.get("AMBASSADOR_DOCKER_IMAGE", None)
    11
    12child: Optional[pexpect.spawnbase.SpawnBase] = None  # see docker_start()
    13diagd_host: Optional[str] = None  # see docker_start()
    14child_name = "diagd-unset"  # see docker_start() and docker_kill()
    15
    16SEQUENCES = [
    17    (["env_ok", "chime"], ["boot1", "now-healthy"]),
    18    (["env_ok", "chime", "scout_cache_reset", "chime"], ["boot1", "now-healthy", "healthy"]),
    19    (["env_ok", "chime", "env_bad", "chime"], ["boot1", "now-healthy", "now-unhealthy"]),
    20    (["env_bad", "chime"], ["boot1", "unhealthy"]),
    21    (
    22        ["env_bad", "chime", "chime", "scout_cache_reset", "chime"],
    23        ["boot1", "unhealthy", "unhealthy"],
    24    ),
    25    (
    26        ["chime", "chime", "chime", "env_ok", "chime", "chime"],
    27        ["boot1", "unhealthy", "now-healthy"],
    28    ),
    29]
    30
    31
    32def docker_start(logfile) -> bool:
    33    # Use a global here so that the child process doesn't get killed
    34    global child
    35
    36    global child_name
    37    child_name = f"diagd-{int(time.time() * 1000)}"
    38
    39    global diagd_host
    40
    41    cmd = f"docker run --name {child_name} --rm -p 9999:9999 {DOCKER_IMAGE} --dev-magic"
    42    diagd_host = "localhost:9999"
    43
    44    child = pexpect.spawn(cmd, encoding="utf-8")
    45    child.logfile = logfile
    46
    47    i = child.expect([pexpect.EOF, pexpect.TIMEOUT, "LocalScout: mode boot, action boot1"])
    48
    49    if i == 0:
    50        print("diagd died?")
    51        return False
    52    elif i == 1:
    53        print("diagd timed out?")
    54        return False
    55
    56    # Set up port forwarding in the Ambassador container from all:9999, where
    57    # this test will connect, to localhost:9998, where diagd is listening. This
    58    # is necessary because diagd rejects (403) requests originating outside the
    59    # container for security reasons.
    60
    61    # Copy the simple port forwarding script into the container
    62    child2 = pexpect.spawn(
    63        f"docker cp python/tests/_forward.py {child_name}:/tmp/", encoding="utf-8"
    64    )
    65    child2.logfile = logfile
    66
    67    if child2.expect([pexpect.EOF, pexpect.TIMEOUT]) == 1:
    68        print("docker cp timed out?")
    69        return False
    70
    71    child2.close()
    72    if child2.exitstatus != 0:
    73        print("docker cp failed?")
    74        return False
    75
    76    # Run the port forwarding script
    77    child2 = pexpect.spawn(
    78        f'docker exec -d {child_name} python /tmp/_forward.py localhost 9998 "" 9999',
    79        encoding="utf-8",
    80    )
    81    child2.logfile = logfile
    82
    83    if child2.expect([pexpect.EOF, pexpect.TIMEOUT]) == 1:
    84        print("docker exec timed out?")
    85        return False
    86
    87    child2.close()
    88    if child2.exitstatus != 0:
    89        print("docker exec failed?")
    90        return False
    91
    92    return True
    93
    94
    95def docker_kill(logfile):
    96    cmd = f"docker kill {child_name}"
    97
    98    child = pexpect.spawn(cmd, encoding="utf-8")
    99    child.logfile = logfile
   100
   101    child.expect([pexpect.EOF, pexpect.TIMEOUT])
   102
   103
   104def wait_for_diagd(logfile) -> bool:
   105    status = False
   106    tries_left = 5
   107
   108    while tries_left >= 0:
   109        logfile.write(f"...checking diagd ({tries_left})\n")
   110
   111        try:
   112            global diagd_host
   113            response = requests.get(
   114                f"http://{diagd_host}/_internal/v0/ping",
   115                headers={"X-Ambassador-Diag-IP": "127.0.0.1"},
   116            )
   117
   118            if response.status_code == 200:
   119                logfile.write("   got it\n")
   120                status = True
   121                break
   122            else:
   123                logfile.write(f"   failed {response.status_code}\n")
   124        except requests.exceptions.RequestException as e:
   125            logfile.write(f"   failed {e}\n")
   126
   127        tries_left -= 1
   128        time.sleep(2)
   129
   130    return status
   131
   132
   133def check_http(logfile, cmd: str) -> bool:
   134    try:
   135        global diagd_host
   136        response = requests.post(
   137            f"http://{diagd_host}/_internal/v0/fs",
   138            headers={"X-Ambassador-Diag-IP": "127.0.0.1"},
   139            params={"path": f"cmd:{cmd}"},
   140        )
   141        text = response.text
   142
   143        if response.status_code != 200:
   144            logfile.write(f"{cmd}: wanted 200 but got {response.status_code} {text}\n")
   145            return False
   146
   147        return True
   148    except Exception as e:
   149        logfile.write(f"Could not do HTTP: {e}\n")
   150
   151        return False
   152
   153
   154def fetch_events(logfile) -> Any:
   155    try:
   156        global diagd_host
   157        response = requests.get(
   158            f"http://{diagd_host}/_internal/v0/events",
   159            headers={"X-Ambassador-Diag-IP": "127.0.0.1"},
   160        )
   161
   162        if response.status_code != 200:
   163            logfile.write(f"events: wanted 200 but got {response.status_code} {response.text}\n")
   164            return None
   165
   166        data = response.json()
   167
   168        return data
   169    except Exception as e:
   170        logfile.write(f"events: could not do HTTP: {e}\n")
   171
   172        return None
   173
   174
   175def check_chimes(logfile) -> bool:
   176    result = True
   177
   178    i = 0
   179
   180    covered = {
   181        "F-F-F": False,
   182        "F-F-T": False,
   183        # 'F-T-F': False,   # This particular key can never be generated
   184        # 'F-T-T': False,   # This particular key can never be generated
   185        "T-F-F": False,
   186        "T-F-T": False,
   187        "T-T-F": False,
   188        "T-T-T": False,
   189    }
   190
   191    for cmds, wanted_verdict in SEQUENCES:
   192        logfile.write(f"RESETTING for sequence {i}\n")
   193
   194        if not check_http(logfile, "chime_reset"):
   195            logfile.write(f"could not reset for sequence {i}\n")
   196            result = False
   197            continue
   198
   199        j = 0
   200        for cmd in cmds:
   201            logfile.write(f"   sending {cmd} for sequence {i}.{j}\n")
   202
   203            if not check_http(logfile, cmd):
   204                logfile.write(f"could not do {cmd} for sequence {i}.{j}\n")
   205                result = False
   206                break
   207
   208            j += 1
   209
   210        if not result:
   211            continue
   212
   213        events = fetch_events(logfile)
   214
   215        if not events:
   216            result = False
   217            continue
   218
   219        # logfile.write(json.dumps(events, sort_keys=True, indent=4))
   220
   221        logfile.write("   ----\n")
   222        verdict = []
   223
   224        for timestamp, mode, action, data in events:
   225            verdict.append(action)
   226
   227            action_key = data.get("action_key", None)
   228
   229            if action_key:
   230                covered[action_key] = True
   231
   232            logfile.write(f"     {action} - {action_key}\n")
   233
   234        # logfile.write(json.dumps(verdict, sort_keys=True, indent=4\n))
   235
   236        if verdict != wanted_verdict:
   237            logfile.write(f"verdict mismatch for sequence {i}:\n")
   238            logfile.write(f'  wanted {" ".join(wanted_verdict)}\n')
   239            logfile.write(f'  got    {" ".join(verdict)}\n')
   240
   241        i += 1
   242
   243    for key in sorted(covered.keys()):
   244        if not covered[key]:
   245            logfile.write(f"missing coverage for {key}\n")
   246            result = False
   247
   248    return result
   249
   250
   251@pytest.mark.flaky(reruns=1, reruns_delay=10)
   252def test_scout():
   253    test_status = False
   254
   255    with open("/tmp/test_scout_output", "w") as logfile:
   256        if not DOCKER_IMAGE:
   257            logfile.write("No $AMBASSADOR_DOCKER_IMAGE??\n")
   258        else:
   259            if docker_start(logfile):
   260                if wait_for_diagd(logfile) and check_chimes(logfile):
   261                    test_status = True
   262
   263                docker_kill(logfile)
   264
   265    if not test_status:
   266        with open("/tmp/test_scout_output", "r") as logfile:
   267            for line in logfile:
   268                print(line.rstrip())
   269
   270    assert test_status, "test failed"
   271
   272
   273if __name__ == "__main__":
   274    pytest.main(sys.argv)

View as plain text