...

Text file src/github.com/datawire/ambassador/v2/python/ambassador_cli/ambassador.py

Documentation: github.com/datawire/ambassador/v2/python/ambassador_cli

     1# Copyright 2018-2020 Datawire. All rights reserved.
     2#
     3# Licensed under the Apache License, Version 2.0 (the "License");
     4# you may not use this file except in compliance with the License.
     5# You may obtain a copy of the License at
     6#
     7#     http://www.apache.org/licenses/LICENSE-2.0
     8#
     9# Unless required by applicable law or agreed to in writing, software
    10# distributed under the License is distributed on an "AS IS" BASIS,
    11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12# See the License for the specific language governing permissions and
    13# limitations under the License
    14
    15########
    16# This is the ambassador CLI. Despite the impression given by its name, it is actually
    17# primarily a debugging tool at this point: the most useful thing to do with it is to
    18# run "ambassador dump --watt path-to-watt-snapshot-file" and have it spit out the IR,
    19# etc.
    20########
    21
    22import cProfile
    23import json
    24import logging
    25import os
    26import sys
    27import traceback
    28from typing import TYPE_CHECKING, ClassVar, Optional, Set
    29from typing import cast as typecast
    30
    31import click
    32
    33from ambassador import IR, Config, Diagnostics, Scout, Version
    34from ambassador.envoy import EnvoyConfig, V2Config, V3Config
    35from ambassador.fetch import ResourceFetcher
    36from ambassador.utils import (
    37    NullSecretHandler,
    38    RichStatus,
    39    SecretHandler,
    40    SecretInfo,
    41    Timer,
    42    dump_json,
    43    parse_json,
    44)
    45
    46if TYPE_CHECKING:
    47    from ambassador.ir import IRResource  # pragma: no cover
    48
    49__version__ = Version
    50
    51logging.basicConfig(
    52    level=logging.INFO,
    53    format="%%(asctime)s ambassador-cli %s %%(levelname)s: %%(message)s" % __version__,
    54    datefmt="%Y-%m-%d %H:%M:%S",
    55)
    56
    57logger = logging.getLogger("ambassador")
    58
    59
    60def handle_exception(what, e, **kwargs):
    61    tb = "\n".join(traceback.format_exception(*sys.exc_info()))
    62
    63    scout = Scout()
    64    result = scout.report(action=what, mode="cli", exception=str(e), traceback=tb, **kwargs)
    65
    66    logger.debug("Scout %s, result: %s" % ("enabled" if scout._scout else "disabled", result))
    67
    68    logger.error("%s: %s\n%s" % (what, e, tb))
    69
    70    show_notices(result)
    71
    72
    73def show_notices(result: dict, printer=logger.log):
    74    notices = result.get("notices", [])
    75
    76    for notice in notices:
    77        lvl = logging.getLevelName(notice.get("level", "ERROR"))
    78
    79        printer(lvl, notice.get("message", "?????"))
    80
    81
    82def stdout_printer(lvl, msg):
    83    print("%s: %s" % (logging.getLevelName(lvl), msg))
    84
    85
    86def version():
    87    """
    88    Show Ambassador's version
    89    """
    90
    91    print("Ambassador %s" % __version__)
    92
    93    scout = Scout()
    94
    95    print("Ambassador Scout version %s" % scout.version)
    96    print("Ambassador Scout semver  %s" % scout.get_semver(scout.version))
    97
    98    result = scout.report(action="version", mode="cli")
    99    show_notices(result, printer=stdout_printer)
   100
   101
   102def showid():
   103    """
   104    Show Ambassador's installation ID
   105    """
   106
   107    scout = Scout()
   108
   109    print("Ambassador Scout installation ID %s" % scout.install_id)
   110
   111    result = scout.report(action="showid", mode="cli")
   112    show_notices(result, printer=stdout_printer)
   113
   114
   115def file_checker(path: str) -> bool:
   116    logger.debug("CLI file checker: pretending %s exists" % path)
   117    return True
   118
   119
   120class CLISecretHandler(SecretHandler):
   121    # HOOK: if you're using dump and you need it to pretend that certain missing secrets
   122    # are present, add them to LoadableSecrets. At Some Point(tm) there will be a switch
   123    # to add these from the command line, but Flynn didn't actually need that for the
   124    # debugging he was doing...
   125
   126    LoadableSecrets: ClassVar[Set[str]] = set(
   127        # "ssl-certificate.mynamespace"
   128    )
   129
   130    def load_secret(
   131        self, resource: "IRResource", secret_name: str, namespace: str
   132    ) -> Optional[SecretInfo]:
   133        # Only allow a secret to be _loaded_ if it's marked Loadable.
   134
   135        key = f"{secret_name}.{namespace}"
   136
   137        if key in CLISecretHandler.LoadableSecrets:
   138            self.logger.info(f"CLISecretHandler: loading {key}")
   139            return SecretInfo(
   140                secret_name,
   141                namespace,
   142                "mocked-loadable-secret",
   143                "-mocked-cert-",
   144                "-mocked-key-",
   145                decode_b64=False,
   146            )
   147
   148        self.logger.debug(f"CLISecretHandler: cannot load {key}")
   149        return None
   150
   151
   152@click.command()
   153@click.argument("config_dir_path", type=click.Path())
   154@click.option("--secret-dir-path", type=click.Path(), help="Directory into which to save secrets")
   155@click.option("--watt", is_flag=True, help="If set, input must be a WATT snapshot")
   156@click.option("--debug", is_flag=True, help="If set, generate debugging output")
   157@click.option("--debug_scout", is_flag=True, help="If set, generate debugging output")
   158@click.option(
   159    "--k8s", is_flag=True, help="If set, assume configuration files are annotated K8s manifests"
   160)
   161@click.option(
   162    "--recurse", is_flag=True, help="If set, recurse into directories below config_dir_path"
   163)
   164@click.option("--stats", is_flag=True, help="If set, dump statistics to stderr")
   165@click.option("--nopretty", is_flag=True, help="If set, do not pretty print the dumped JSON")
   166@click.option("--aconf", is_flag=True, help="If set, dump the Ambassador config")
   167@click.option("--ir", is_flag=True, help="If set, dump the IR")
   168@click.option("--v2", is_flag=True, help="If set, dump the Envoy V2 config")
   169@click.option("--v3", is_flag=True, help="If set, dump the Envoy V3 config")
   170@click.option("--diag", is_flag=True, help="If set, dump the Diagnostics overview")
   171@click.option("--everything", is_flag=True, help="If set, dump everything")
   172@click.option("--features", is_flag=True, help="If set, dump the feature set")
   173@click.option("--profile", is_flag=True, help="If set, profile with the cProfile module")
   174def dump(
   175    config_dir_path: str,
   176    *,
   177    secret_dir_path=None,
   178    watt=False,
   179    debug=False,
   180    debug_scout=False,
   181    k8s=False,
   182    recurse=False,
   183    stats=False,
   184    nopretty=False,
   185    everything=False,
   186    aconf=False,
   187    ir=False,
   188    v2=False,
   189    v3=False,
   190    diag=False,
   191    features=False,
   192    profile=False,
   193):
   194    """
   195    Dump various forms of an Ambassador configuration for debugging
   196
   197    Use --aconf, --ir, and --envoy to control what gets dumped. If none are requested, the IR
   198    will be dumped.
   199
   200    :param config_dir_path: Configuration directory to scan for Ambassador YAML files
   201    """
   202
   203    if not secret_dir_path:
   204        secret_dir_path = "/tmp/cli-secrets"
   205
   206        if not os.path.isdir(secret_dir_path):
   207            secret_dir_path = os.path.dirname(secret_dir_path)
   208
   209    if debug:
   210        logger.setLevel(logging.DEBUG)
   211
   212    if debug_scout:
   213        logging.getLogger("ambassador.scout").setLevel(logging.DEBUG)
   214
   215    if everything:
   216        aconf = True
   217        ir = True
   218        v2 = True
   219        v3 = True
   220        diag = True
   221        features = True
   222    elif not (aconf or ir or v2 or v3 or diag or features):
   223        aconf = True
   224        ir = True
   225        v2 = True
   226        v3 = False
   227        diag = False
   228        features = False
   229
   230    dump_aconf = aconf
   231    dump_ir = ir
   232    dump_v2 = v2
   233    dump_v3 = v3
   234    dump_diag = diag
   235    dump_features = features
   236
   237    od = {}
   238    diagconfig: Optional[EnvoyConfig] = None
   239
   240    _profile: Optional[cProfile.Profile] = None
   241    _rc = 0
   242
   243    if profile:
   244        _profile = cProfile.Profile()
   245        _profile.enable()
   246
   247    try:
   248        total_timer = Timer("total")
   249        total_timer.start()
   250
   251        fetch_timer = Timer("fetch resources")
   252        with fetch_timer:
   253            aconf = Config()
   254
   255            fetcher = ResourceFetcher(logger, aconf)
   256
   257            if watt:
   258                fetcher.parse_watt(open(config_dir_path, "r").read())
   259            else:
   260                fetcher.load_from_filesystem(config_dir_path, k8s=k8s, recurse=True)
   261
   262        load_timer = Timer("load fetched resources")
   263        with load_timer:
   264            aconf.load_all(fetcher.sorted())
   265
   266        # aconf.post_error("Error from string, boo yah")
   267        # aconf.post_error(RichStatus.fromError("Error from RichStatus"))
   268
   269        irgen_timer = Timer("ir generation")
   270        with irgen_timer:
   271            secret_handler = NullSecretHandler(logger, config_dir_path, secret_dir_path, "0")
   272
   273            ir = IR(aconf, file_checker=file_checker, secret_handler=secret_handler)
   274
   275        aconf_timer = Timer("aconf")
   276        with aconf_timer:
   277            if dump_aconf:
   278                od["aconf"] = aconf.as_dict()
   279
   280        ir_timer = Timer("ir")
   281        with ir_timer:
   282            if dump_ir:
   283                od["ir"] = ir.as_dict()
   284
   285        v2_timer = Timer("v2")
   286        with v2_timer:
   287            if dump_v2:
   288                v2config = V2Config(ir)
   289                diagconfig = v2config
   290                od["v2"] = v2config.as_dict()
   291        v3_timer = Timer("v3")
   292        with v3_timer:
   293            if dump_v3:
   294                v3config = V3Config(ir)
   295                diagconfig = v3config
   296                od["v3"] = v3config.as_dict()
   297        diag_timer = Timer("diag")
   298        with diag_timer:
   299            if dump_diag:
   300                if not diagconfig:
   301                    diagconfig = V2Config(ir)
   302                    diagconfigv3 = V3Config(ir)
   303                econf = typecast(EnvoyConfig, diagconfig)
   304                econfv3 = typecast(EnvoyConfig, diagconfigv3)
   305                diag = Diagnostics(ir, econf)
   306                diagv3 = Diagnostics(ir, econfv3)
   307                od["diag"] = diag.as_dict()
   308                od["elements"] = econf.elements
   309                od["diagv3"] = diagv3.as_dict()
   310                od["elementsv3"] = econfv3.elements
   311
   312        features_timer = Timer("features")
   313        with features_timer:
   314            if dump_features:
   315                od["features"] = ir.features()
   316
   317        # scout = Scout()
   318        # scout_args = {}
   319        #
   320        # if ir and not os.environ.get("AMBASSADOR_DISABLE_FEATURES", None):
   321        #     scout_args["features"] = ir.features()
   322        #
   323        # result = scout.report(action="dump", mode="cli", **scout_args)
   324        # show_notices(result)
   325
   326        dump_timer = Timer("dump JSON")
   327
   328        with dump_timer:
   329            js = dump_json(od, pretty=not nopretty)
   330            jslen = len(js)
   331
   332        write_timer = Timer("write JSON")
   333        with write_timer:
   334            sys.stdout.write(js)
   335            sys.stdout.write("\n")
   336
   337        total_timer.stop()
   338
   339        route_count = 0
   340        vhost_count = 0
   341        filter_chain_count = 0
   342        filter_count = 0
   343        apiversion = "v2" if v2 else "v3"
   344        if apiversion in od:
   345            for listener in od[apiversion]["static_resources"]["listeners"]:
   346                for fc in listener["filter_chains"]:
   347                    filter_chain_count += 1
   348                    for f in fc["filters"]:
   349                        filter_count += 1
   350                        for vh in f["typed_config"]["route_config"]["virtual_hosts"]:
   351                            vhost_count += 1
   352                            route_count += len(vh["routes"])
   353
   354        if stats:
   355            sys.stderr.write("STATS:\n")
   356            sys.stderr.write("  config bytes:  %d\n" % jslen)
   357            sys.stderr.write("  vhosts:        %d\n" % vhost_count)
   358            sys.stderr.write("  filter chains: %d\n" % filter_chain_count)
   359            sys.stderr.write("  filters:       %d\n" % filter_count)
   360            sys.stderr.write("  routes:        %d\n" % route_count)
   361            sys.stderr.write(
   362                "  routes/vhosts: %.3f\n" % float(float(route_count) / float(vhost_count))
   363            )
   364            sys.stderr.write("TIMERS:\n")
   365            sys.stderr.write("  fetch resources:  %.3fs\n" % fetch_timer.average)
   366            sys.stderr.write("  load resources:   %.3fs\n" % load_timer.average)
   367            sys.stderr.write("  ir generation:    %.3fs\n" % irgen_timer.average)
   368            sys.stderr.write("  aconf:            %.3fs\n" % aconf_timer.average)
   369            sys.stderr.write("  envoy v2:         %.3fs\n" % v2_timer.average)
   370            sys.stderr.write("  diag:             %.3fs\n" % diag_timer.average)
   371            sys.stderr.write("  features:         %.3fs\n" % features_timer.average)
   372            sys.stderr.write("  dump json:        %.3fs\n" % dump_timer.average)
   373            sys.stderr.write("  write json:       %.3fs\n" % write_timer.average)
   374            sys.stderr.write("  ----------------------\n")
   375            sys.stderr.write("  total: %.3fs\n" % total_timer.average)
   376    except Exception as e:
   377        handle_exception("EXCEPTION from dump", e, config_dir_path=config_dir_path)
   378        _rc = 1
   379
   380    if _profile:
   381        _profile.disable()
   382        _profile.dump_stats("ambassador.profile")
   383
   384    sys.exit(_rc)
   385
   386
   387@click.command()
   388@click.argument("config_dir_path", type=click.Path())
   389def validate(config_dir_path: str):
   390    """
   391    Validate an Ambassador configuration. This is an extension of "config" that
   392    redirects output to devnull and always exits on error.
   393
   394    :param config_dir_path: Configuration directory to scan for Ambassador YAML files
   395    """
   396    config(config_dir_path, os.devnull, exit_on_error=True)
   397
   398
   399@click.command()
   400@click.argument("config_dir_path", type=click.Path())
   401@click.argument("output_json_path", type=click.Path())
   402@click.option("--debug", is_flag=True, help="If set, generate debugging output")
   403@click.option(
   404    "--debug-scout", is_flag=True, help="If set, generate debugging output when talking to Scout"
   405)
   406@click.option(
   407    "--check", is_flag=True, help="If set, generate configuration only if it doesn't already exist"
   408)
   409@click.option(
   410    "--k8s", is_flag=True, help="If set, assume configuration files are annotated K8s manifests"
   411)
   412@click.option(
   413    "--exit-on-error",
   414    is_flag=True,
   415    help="If set, will exit with status 1 on any configuration error",
   416)
   417@click.option(
   418    "--ir", type=click.Path(), help="Pathname to which to dump the IR (not dumped if not present)"
   419)
   420@click.option(
   421    "--aconf",
   422    type=click.Path(),
   423    help="Pathname to which to dump the aconf (not dumped if not present)",
   424)
   425def config(
   426    config_dir_path: str,
   427    output_json_path: str,
   428    *,
   429    debug=False,
   430    debug_scout=False,
   431    check=False,
   432    k8s=False,
   433    ir=None,
   434    aconf=None,
   435    exit_on_error=False,
   436):
   437    """
   438    Generate an Envoy configuration
   439
   440    :param config_dir_path: Configuration directory to scan for Ambassador YAML files
   441
   442    :param output_json_path: Path to output envoy.json
   443    """
   444
   445    if debug:
   446        logger.setLevel(logging.DEBUG)
   447
   448    if debug_scout:
   449        logging.getLogger("ambassador.scout").setLevel(logging.DEBUG)
   450
   451    try:
   452        logger.debug("CHECK MODE  %s" % check)
   453        logger.debug("CONFIG DIR  %s" % config_dir_path)
   454        logger.debug("OUTPUT PATH %s" % output_json_path)
   455
   456        dump_aconf: Optional[str] = aconf
   457        dump_ir: Optional[str] = ir
   458
   459        # Bypass the existence check...
   460        output_exists = False
   461
   462        if check:
   463            # ...oh no wait, they explicitly asked for the existence check!
   464            # Assume that the file exists (ie, we'll do nothing) unless we
   465            # determine otherwise.
   466            output_exists = True
   467
   468            try:
   469                parse_json(open(output_json_path, "r").read())
   470            except FileNotFoundError:
   471                logger.debug("output file does not exist")
   472                output_exists = False
   473            except OSError:
   474                logger.warning("output file is not sane?")
   475                output_exists = False
   476            except json.decoder.JSONDecodeError:
   477                logger.warning("output file is not valid JSON")
   478                output_exists = False
   479
   480            logger.info("Output file %s" % ("exists" if output_exists else "does not exist"))
   481
   482        rc = RichStatus.fromError("impossible error")
   483
   484        if not output_exists:
   485            # Either we didn't need to check, or the check didn't turn up
   486            # a valid config. Regenerate.
   487            logger.info("Generating new Envoy configuration...")
   488
   489            aconf = Config()
   490            fetcher = ResourceFetcher(logger, aconf)
   491            fetcher.load_from_filesystem(config_dir_path, k8s=k8s)
   492            aconf.load_all(fetcher.sorted())
   493
   494            if dump_aconf:
   495                with open(dump_aconf, "w") as output:
   496                    output.write(aconf.as_json())
   497                    output.write("\n")
   498
   499            # If exit_on_error is set, log _errors and exit with status 1
   500            if exit_on_error and aconf.errors:
   501                raise Exception("errors in: {0}".format(", ".join(aconf.errors.keys())))
   502
   503            secret_handler = NullSecretHandler(logger, config_dir_path, config_dir_path, "0")
   504
   505            ir = IR(aconf, file_checker=file_checker, secret_handler=secret_handler)
   506
   507            if dump_ir:
   508                with open(dump_ir, "w") as output:
   509                    output.write(ir.as_json())
   510                    output.write("\n")
   511
   512            logger.info("Writing envoy V2 configuration")
   513            v2config = V2Config(ir)
   514            rc = RichStatus.OK(msg="huh_v2")
   515
   516            if rc:
   517                with open(output_json_path, "w") as output:
   518                    output.write(v2config.as_json())
   519                    output.write("\n")
   520            else:
   521                logger.error("Could not generate new Envoy configuration: %s" % rc.error)
   522
   523        scout = Scout()
   524        result = scout.report(action="config", mode="cli")
   525        show_notices(result)
   526    except Exception as e:
   527        handle_exception(
   528            "EXCEPTION from config",
   529            e,
   530            config_dir_path=config_dir_path,
   531            output_json_path=output_json_path,
   532        )
   533
   534        # This is fatal.
   535        sys.exit(1)
   536
   537
   538def version_callback(ctx: click.core.Context, param: click.Parameter, value: bool) -> None:
   539    if not value:
   540        return
   541    version()
   542    ctx.exit()
   543
   544
   545def showid_callback(ctx: click.core.Context, param: click.Parameter, value: bool) -> None:
   546    if not value:
   547        return
   548    showid()
   549    ctx.exit()
   550
   551
   552@click.group(
   553    no_args_is_help=False,
   554    commands=[config, dump, validate],
   555)
   556@click.option(
   557    "--version",
   558    is_flag=True,
   559    expose_value=False,
   560    callback=version_callback,
   561    help="Show the Emissary version number and exit.",
   562)
   563@click.option(
   564    "--showid",
   565    is_flag=True,
   566    expose_value=False,
   567    callback=showid_callback,
   568    help="Show the cluster ID and exit.",
   569)
   570def main():
   571    """Generate an Envoy config, or manage an Ambassador deployment. Use
   572
   573        ambassador.py command --help
   574
   575    for more help, or
   576
   577        ambassador.py --version
   578
   579    to see Ambassador's version.
   580    """
   581
   582
   583if __name__ == "__main__":
   584    main()

View as plain text