diff --git a/README.md b/README.md index 7d34b7e..5ae94a5 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,21 @@ g3dt synth generate AusDiab_Simulated --llm -n 5 -e test `g3dt config show --env ` prints the resolved provider, model, and key path. +### Kubernetes restart targets + +`k8s restart-schema`, `k8s restart-ms`, `dict deploy`, and `synth deploy` +restart the commons' schema microservices **serially, in a configured order**, +waiting for each rollout to go Healthy before the next; `k8s restart-etl` and +the deploy flows run a named ETL cronjob. Both targets resolve with precedence +**CLI flags > SSM > default**: the CDK config's optional `k8s` block publishes +`app/restart_services` (comma-separated deployment names) and +`app/etl_cronjob`, `--restart-services` / `--etl-cronjob` override them for +one run, and environments deployed without the block keep the classic Gen3 set +(`sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment` +/ `etl-cronjob`). A commons that manages some service outside this flow (e.g. +a manually redeployed frontend) simply omits it from the list in its wrapper +config. `g3dt config show --env ` prints the resolved values. + ## Verifying download access (check-download) Registration alone does not prove a file can be downloaded. Two failure modes diff --git a/pyproject.toml b/pyproject.toml index a3af157..049b523 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gen3-dataops-toolkit" -version = "3.4.0" +version = "3.5.0" description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments" authors = ["JoshuaHarris391 "] readme = "README.md" diff --git a/src/g3dt/cli/config_cmds.py b/src/g3dt/cli/config_cmds.py index 35f5444..dc9ce42 100644 --- a/src/g3dt/cli/config_cmds.py +++ b/src/g3dt/cli/config_cmds.py @@ -123,6 +123,10 @@ def show( typer.echo(f" llm_provider : {e.llm_provider}") typer.echo(f" llm_model : {e.llm_model or '(not set — pass --llm-model or add the llm block to the CDK config)'}") typer.echo(f" llm_api_key_file : {config.llm_api_key_file() or '(not set — g3dt config set llm_api_key_file )'}") + # k8s restart targets: from SSM (the CDK's optional k8s block), restarted + # in the listed order by restart-schema/restart-ms/dict deploy/synth deploy. + typer.echo(f" restart_services : {e.restart_services}") + typer.echo(f" etl_cronjob : {e.etl_cronjob}") if study: s = study_of(study, env) typer.secho(f"Study: {study} -> {s.key}", bold=True) @@ -217,6 +221,16 @@ def check(label: str, file_value, ssm_value) -> None: for camel, leaf in {"provider": "llm_provider", "model": "llm_model"}.items(): if camel in llm: check(f"llm.{camel}", llm.get(camel), rc.get(f"app/{leaf}")) + k8s = inputs.get("k8s") or {} + if "schemaRestartServices" in k8s: + # The CDK publishes the list comma-joined; compare the same shape. + check( + "k8s.schemaRestartServices", + ",".join(k8s.get("schemaRestartServices") or []), + rc.get("app/restart_services"), + ) + if "etlCronjob" in k8s: + check("k8s.etlCronjob", k8s.get("etlCronjob"), rc.get("app/etl_cronjob")) if not drift: typer.secho( diff --git a/src/g3dt/cli/dict_cmds.py b/src/g3dt/cli/dict_cmds.py index d03b898..0953371 100644 --- a/src/g3dt/cli/dict_cmds.py +++ b/src/g3dt/cli/dict_cmds.py @@ -103,6 +103,11 @@ def deploy( version: str = typer.Option( None, "--version", help="Dictionary git tag (default: the env's version)." ), + restart_services: Optional[str] = typer.Option( + None, "--restart-services", + help="Comma-separated deployment names restarted after the upload, in " + "order; default: the env's SSM app/restart_services.", + ), ) -> None: """Pull + upload the dictionary and restart Gen3 schema microservices. @@ -123,7 +128,10 @@ def deploy( """ e = env_of(env) warn_if_overridden(e, version) + env_vars = script_env(e, _version(e, version)) + if restart_services: + env_vars["G3DT_RESTART_SERVICES"] = restart_services runner.run( runner.bash_script("services/dictionary/deploy_dd.sh", env), - env=script_env(e, _version(e, version)), + env=env_vars, ) diff --git a/src/g3dt/cli/k8s.py b/src/g3dt/cli/k8s.py index 65d9755..11073af 100644 --- a/src/g3dt/cli/k8s.py +++ b/src/g3dt/cli/k8s.py @@ -3,9 +3,18 @@ These use ``argocd login --sso`` (a browser flow), so they cannot run headless on EC2. The wrapped scripts receive their settings as ``G3DT_*`` environment variables resolved from SSM — they read no config files. + +The restart targets resolve with precedence CLI flags > SSM > default: the +CDK config's optional ``k8s`` block publishes ``app/restart_services`` (a +comma-separated list, restarted in order) and ``app/etl_cronjob``; +``--restart-services`` / ``--etl-cronjob`` override them for one run, and +environments deployed without the block keep the classic Gen3 set +(sheepdog, peregrine, guppy, portal / etl-cronjob). """ from __future__ import annotations +from typing import Optional + import typer from g3dt.config import script_env @@ -18,37 +27,78 @@ _ETL = "services/k8s_ops/argocd_restart_etl.sh" _ETL_AND_MS = "services/k8s_ops/restart_etl_and_ms.sh" +_RESTART_SERVICES_HELP = ( + "Comma-separated deployment names to restart, in order; default: the " + "env's SSM app/restart_services (the CDK config's k8s.schemaRestartServices)." +) +_ETL_CRONJOB_HELP = ( + "ETL cronjob name; default: the env's SSM app/etl_cronjob " + "(the CDK config's k8s.etlCronjob)." +) + + +def restart_env(e, restart_services: Optional[str] = None, + etl_cronjob: Optional[str] = None) -> dict: + """script_env plus per-run restart-target overrides (flags beat SSM).""" + env_vars = script_env(e) + if restart_services: + env_vars["G3DT_RESTART_SERVICES"] = restart_services + if etl_cronjob: + env_vars["G3DT_ETL_CRONJOB"] = etl_cronjob + return env_vars + @app.command(name="restart-schema") def restart_schema( env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."), sync: bool = typer.Option(False, "--sync", "-s", help="argocd app sync first."), + restart_services: Optional[str] = typer.Option( + None, "--restart-services", help=_RESTART_SERVICES_HELP + ), ) -> None: - """Restart sheepdog/peregrine/guppy/portal (schema microservices).""" + """Restart the schema microservices, in the env's configured order.""" e = env_of(env) args = ["-d", e.domain, "-a", e.app_name, "-n", e.namespace] if sync: args.append("-s") - runner.run(runner.bash_script(_SCHEMA, *args), env=script_env(e)) + runner.run( + runner.bash_script(_SCHEMA, *args), + env=restart_env(e, restart_services=restart_services), + ) @app.command(name="restart-etl") def restart_etl( env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."), sync: bool = typer.Option(False, "--sync", "-s", help="argocd app sync first."), + etl_cronjob: Optional[str] = typer.Option( + None, "--etl-cronjob", help=_ETL_CRONJOB_HELP + ), ) -> None: """Create + run the ETL cronjob and wait for completion.""" e = env_of(env) args = ["-e", env] if sync: args.append("-s") - runner.run(runner.bash_script(_ETL, *args), env=script_env(e)) + runner.run( + runner.bash_script(_ETL, *args), + env=restart_env(e, etl_cronjob=etl_cronjob), + ) @app.command(name="restart-ms") def restart_ms( env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."), + restart_services: Optional[str] = typer.Option( + None, "--restart-services", help=_RESTART_SERVICES_HELP + ), + etl_cronjob: Optional[str] = typer.Option( + None, "--etl-cronjob", help=_ETL_CRONJOB_HELP + ), ) -> None: """Restart both ETL and schema microservices (wraps restart_etl_and_ms.sh).""" e = env_of(env) - runner.run(runner.bash_script(_ETL_AND_MS, env), env=script_env(e)) + runner.run( + runner.bash_script(_ETL_AND_MS, env), + env=restart_env(e, restart_services=restart_services, etl_cronjob=etl_cronjob), + ) diff --git a/src/g3dt/cli/synth.py b/src/g3dt/cli/synth.py index 7ebad35..cc1ed81 100644 --- a/src/g3dt/cli/synth.py +++ b/src/g3dt/cli/synth.py @@ -168,17 +168,31 @@ def deploy( help="Path to the file holding the LLM API key; default: the marker's " "llm_api_key_file (set once: g3dt config set llm_api_key_file ).", ), + restart_services: Optional[str] = typer.Option( + None, "--restart-services", + help="Comma-separated deployment names restarted during the deploy, in " + "order; default: the env's SSM app/restart_services.", + ), + etl_cronjob: Optional[str] = typer.Option( + None, "--etl-cronjob", + help="ETL cronjob name; default: the env's SSM app/etl_cronjob.", + ), ) -> None: """Full end-to-end synthetic deploy (dict + LLM-generate + upload + restarts). Wraps services/synthetic_data/full_deploy_dd_and_synth.sh (LLM-backed - generation). Provider/model come from the env's SSM tree unless - overridden; the API key path comes from --llm-api-key-file or the marker. + generation). Provider/model and the restart targets come from the env's + SSM tree unless overridden; the API key path comes from + --llm-api-key-file or the marker. """ e = env_of(env) safety.confirm_prod_strict("synthetic full deploy", env) env_vars = script_env(e) env_vars.update(_llm_env_overrides(e, llm_provider, llm_model, llm_api_key_file)) + if restart_services: + env_vars["G3DT_RESTART_SERVICES"] = restart_services + if etl_cronjob: + env_vars["G3DT_ETL_CRONJOB"] = etl_cronjob runner.run( runner.bash_script( "services/synthetic_data/full_deploy_dd_and_synth.sh", env diff --git a/src/g3dt/config.py b/src/g3dt/config.py index 82e47d5..c54cef7 100644 --- a/src/g3dt/config.py +++ b/src/g3dt/config.py @@ -58,6 +58,17 @@ #: when the ``--llm`` path is used and no model is configured anywhere. DEFAULT_LLM_PROVIDER = "anthropic" +#: Kubernetes restart targets, published by the CDK's OPTIONAL ``k8s`` config +#: block as ``app/restart_services`` (comma-separated, restarted in order) and +#: ``app/etl_cronjob``. Optional app inputs: environments deployed without the +#: block keep the classic Gen3 set. Consumed by every restart path — `g3dt +#: k8s restart-schema/restart-etl/restart-ms`, `dict deploy`, `synth deploy` — +#: via $G3DT_RESTART_SERVICES / $G3DT_ETL_CRONJOB in the service scripts. +DEFAULT_RESTART_SERVICES = ( + "sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment" +) +DEFAULT_ETL_CRONJOB = "etl-cronjob" + #: Marker locations, most specific first. MARKER_PATHS = ("g3dt.yaml", "~/.g3dt/g3dt.yaml", "/etc/g3dt/g3dt.yaml") @@ -257,6 +268,9 @@ class EnvConfig: # with guidance rather than silently picking a model. llm_provider: str = DEFAULT_LLM_PROVIDER llm_model: Optional[str] = None + # Optional k8s restart targets (SSM app/restart_services, app/etl_cronjob). + restart_services: str = DEFAULT_RESTART_SERVICES + etl_cronjob: str = DEFAULT_ETL_CRONJOB def _app_or_default(rc, leaf: str, default: str) -> str: @@ -320,6 +334,10 @@ def resolve_env(env: str, project: Optional[str] = None) -> EnvConfig: # the config has an llm block, so absence means "use the defaults". llm_provider=_app_or_default(rc, "llm_provider", DEFAULT_LLM_PROVIDER), llm_model=(_app_or_default(rc, "llm_model", "") or None), + restart_services=_app_or_default( + rc, "restart_services", DEFAULT_RESTART_SERVICES + ), + etl_cronjob=_app_or_default(rc, "etl_cronjob", DEFAULT_ETL_CRONJOB), ) @@ -428,6 +446,8 @@ def script_env(e: EnvConfig, version: Optional[str] = None) -> Dict[str, str]: "G3DT_SCHEMA_REPO": e.schema_repo, "G3DT_LLM_PROVIDER": e.llm_provider, "G3DT_LLM_MODEL": e.llm_model, + "G3DT_RESTART_SERVICES": e.restart_services, + "G3DT_ETL_CRONJOB": e.etl_cronjob, } env.update({k: v for k, v in values.items() if v is not None}) return env diff --git a/src/g3dt/services/k8s_ops/argocd_restart_etl.sh b/src/g3dt/services/k8s_ops/argocd_restart_etl.sh index bb0abf3..487f78c 100755 --- a/src/g3dt/services/k8s_ops/argocd_restart_etl.sh +++ b/src/g3dt/services/k8s_ops/argocd_restart_etl.sh @@ -6,7 +6,7 @@ usage() { echo " -d DOMAIN The domain for argocd login (example: cd.cad.test.biocommons.org.au)" echo " -a APPNAME The application name (example: uatgen3)" echo " -n NAMESPACE The namespace for the resources (example: cad)" - echo " -c ETL_CRONJOB The name of the ETL cronjob to run (default: etl-cronjob)" + echo " -c ETL_CRONJOB The name of the ETL cronjob to run (default: \$G3DT_ETL_CRONJOB — the env's SSM app/etl_cronjob, set by g3dt — else etl-cronjob)" echo " -t CONTAINER The name of the container to check logs from (default: tube)" echo " -l Bypass login" echo " -s Sync the argocd app before restarting resources" @@ -15,8 +15,9 @@ usage() { set -eo pipefail -# default values -ETL_CRONJOB="etl-cronjob" +# default values. The cronjob name comes from the environment's SSM tree +# (app/etl_cronjob, exported by g3dt as G3DT_ETL_CRONJOB). +ETL_CRONJOB="${G3DT_ETL_CRONJOB:-etl-cronjob}" CONTAINER_TO_CHECK="tube" LOGIN_REQUIRED=true SYNC_APP=false diff --git a/src/g3dt/services/k8s_ops/argocd_restart_ms.sh b/src/g3dt/services/k8s_ops/argocd_restart_ms.sh index 8fc58b4..80912a8 100755 --- a/src/g3dt/services/k8s_ops/argocd_restart_ms.sh +++ b/src/g3dt/services/k8s_ops/argocd_restart_ms.sh @@ -6,8 +6,11 @@ usage() { echo "Usage: $0 [-d DOMAIN] [-a APPNAME] [-r RESOURCES] [-n NAMESPACE] [-k KIND] [-l] [-s]" echo " -d DOMAIN The domain for argocd login (example: cd.cad.test.biocommons.org.au)" echo " -a APPNAME The application name (example: uatgen3)" - echo " -r RESOURCES Comma-separated string of microservice names to restart (example: sheepdog-deployment,peregrine-deployment )" - echo " -n NAMESPACE The namespace for the resources (example: cad)" + echo " -r RESOURCES Comma-separated microservice names, restarted in order" + echo " (default: \$G3DT_RESTART_SERVICES — the env's SSM" + echo " app/restart_services, set by g3dt — else the classic" + echo " sheepdog,peregrine,guppy,portal set)" + echo " -n NAMESPACE The namespace for the resources (default: \$G3DT_NAMESPACE)" echo " -k KIND The kind of resource to restart (default: Deployment)" echo " -l Bypass login" echo " -s Sync the argocd app before restarting resources" @@ -16,7 +19,10 @@ usage() { set -eo pipefail -# Parse command line arguments +# Parse command line arguments. The restart set defaults to the environment's +# SSM app/restart_services (exported by g3dt as G3DT_RESTART_SERVICES). +IFS=',' read -r -a RESOURCES <<< "${G3DT_RESTART_SERVICES:-sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment}" +NAMESPACE="${G3DT_NAMESPACE:-}" LOGIN_REQUIRED=true KIND="Deployment" SYNC_APP=false @@ -52,6 +58,11 @@ while getopts "d:a:r:n:k:hls" opt; do esac done +if [ -z "$NAMESPACE" ]; then + echo "No namespace given: pass -n, or run via the g3dt CLI (which sets G3DT_NAMESPACE)." >&2 + exit 1 +fi + # Check if argocd is installed if ! command -v argocd &> /dev/null then diff --git a/src/g3dt/services/k8s_ops/argocd_restart_schema.sh b/src/g3dt/services/k8s_ops/argocd_restart_schema.sh index 2035046..e7a4971 100755 --- a/src/g3dt/services/k8s_ops/argocd_restart_schema.sh +++ b/src/g3dt/services/k8s_ops/argocd_restart_schema.sh @@ -6,8 +6,11 @@ usage() { echo "Usage: $0 [-d DOMAIN] [-a APPNAME] [-r RESOURCES] [-n NAMESPACE] [-k KIND] [-l] [-s]" echo " -d DOMAIN The domain for argocd login (example: cd.cad.test.biocommons.org.au)" echo " -a APPNAME The application name (example: uatgen3)" - echo " -r RESOURCES Comma-separated string of microservice names to restart (default: \"sheepdog-deployment\" \"peregrine-deployment\" \"guppy-deployment\" \"portal-deployment\")" - echo " -n NAMESPACE The namespace for the resources (default: cad)" + echo " -r RESOURCES Comma-separated microservice names, restarted in order" + echo " (default: \$G3DT_RESTART_SERVICES — the env's SSM" + echo " app/restart_services, set by g3dt — else the classic" + echo " sheepdog,peregrine,guppy,portal set)" + echo " -n NAMESPACE The namespace for the resources (default: \$G3DT_NAMESPACE)" echo " -k KIND The kind of resource to restart (default: Deployment)" echo " -l Bypass login" echo " -s Run 'argocd app sync' before restarts" @@ -16,9 +19,11 @@ usage() { set -eo pipefail -# Set default values -RESOURCES=("sheepdog-deployment" "peregrine-deployment" "guppy-deployment" "portal-deployment") -NAMESPACE="cad" +# Set default values. The restart set comes from the environment's SSM tree +# (app/restart_services, exported by g3dt as G3DT_RESTART_SERVICES); the +# classic Gen3 set is the fallback for direct invocations outside g3dt. +IFS=',' read -r -a RESOURCES <<< "${G3DT_RESTART_SERVICES:-sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment}" +NAMESPACE="${G3DT_NAMESPACE:-}" KIND="Deployment" LOGIN_REQUIRED=true SYNC_APP=false @@ -56,6 +61,11 @@ while getopts "d:a:r:n:k:hls" opt; do esac done +if [ -z "$NAMESPACE" ]; then + echo "No namespace given: pass -n, or run via the g3dt CLI (which sets G3DT_NAMESPACE)." >&2 + exit 1 +fi + # Check if argocd is installed if ! command -v argocd &> /dev/null then diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 7fc7440..c8b116f 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -13,6 +13,7 @@ from typer.testing import CliRunner from g3dt.cli.main import app +from g3dt import config as config_mod from g3dt.config import ConfigError, EnvConfig, StudyConfig, dictionary_url runner = CliRunner() @@ -636,3 +637,52 @@ def test_indexd_check_download_samples_registry_with_limit(mock_run, _env): ) assert result.exit_code == 0, result.output assert _argv(mock_run)[2:] == ["--env", "staging", "--limit", "5"] + + +@patch("g3dt.cli.k8s.env_of", side_effect=_env_cfg) +@patch("g3dt.cli._internal.runner.run") +def test_k8s_restart_schema_flag_overrides_restart_services(mock_run, _env): + """ + Inputs: k8s restart-schema --restart-services sheepdog-deployment + Expected Output: the override reaches the script as G3DT_RESTART_SERVICES + in the subprocess env (the script's default; flags > SSM > default), so a + one-off restart of a subset needs no redeploy. + """ + result = runner.invoke( + app, + ["k8s", "restart-schema", "--env", "test", + "--restart-services", "sheepdog-deployment"], + ) + assert result.exit_code == 0, result.output + env = mock_run.call_args.kwargs["env"] + assert env["G3DT_RESTART_SERVICES"] == "sheepdog-deployment" + + +@patch("g3dt.cli.k8s.env_of", side_effect=_env_cfg) +@patch("g3dt.cli._internal.runner.run") +def test_k8s_restart_schema_default_exports_env_restart_services(mock_run, _env): + """ + Inputs: k8s restart-schema with no --restart-services flag + Expected Output: the subprocess env carries the EnvConfig's (SSM-resolved) + restart set via script_env, so the script restarts exactly what the + deployment configured. + """ + result = runner.invoke(app, ["k8s", "restart-schema", "--env", "test"]) + assert result.exit_code == 0, result.output + env = mock_run.call_args.kwargs["env"] + assert env["G3DT_RESTART_SERVICES"] == config_mod.DEFAULT_RESTART_SERVICES + + +@patch("g3dt.cli.k8s.env_of", side_effect=_env_cfg) +@patch("g3dt.cli._internal.runner.run") +def test_k8s_restart_etl_flag_overrides_cronjob(mock_run, _env): + """ + Inputs: k8s restart-etl --etl-cronjob custom-etl + Expected Output: G3DT_ETL_CRONJOB carries the override to the script. + """ + result = runner.invoke( + app, ["k8s", "restart-etl", "--env", "test", "--etl-cronjob", "custom-etl"] + ) + assert result.exit_code == 0, result.output + env = mock_run.call_args.kwargs["env"] + assert env["G3DT_ETL_CRONJOB"] == "custom-etl" diff --git a/tests/test_cli_config.py b/tests/test_cli_config.py index 1b1a38c..986abcd 100644 --- a/tests/test_cli_config.py +++ b/tests/test_cli_config.py @@ -356,3 +356,45 @@ def test_script_env_exports_llm_facts_only_when_model_set(): without_model = config.script_env(config.EnvConfig(**base)) assert "G3DT_LLM_MODEL" not in without_model + + +@mock_aws +def test_restart_targets_resolved_from_optional_app_inputs(): + """ + Inputs: an env tree with app/restart_services and app/etl_cronjob set + (what the CDK's optional k8s config block publishes) + Expected: EnvConfig carries both, so every restart path (k8s restart-*, + dict deploy, synth deploy) uses the deployment's own targets — + e.g. a commons that excludes portal because its frontend is + redeployed manually. + """ + _seed_env("etl", "test") + ssm = boto3.client("ssm", region_name=REGION) + ssm.put_parameter( + Name="/etl/test/app/restart_services", + Value="sheepdog-deployment,guppy-deployment", Type="String", + ) + ssm.put_parameter( + Name="/etl/test/app/etl_cronjob", Value="my-etl", Type="String" + ) + + e = config.resolve_env("test") + assert e.restart_services == "sheepdog-deployment,guppy-deployment" + assert e.etl_cronjob == "my-etl" + + +@mock_aws +def test_restart_targets_default_when_absent(): + """ + Inputs: an env tree WITHOUT app/restart_services or app/etl_cronjob + (every deployment whose config has no k8s block) + Expected: the classic Gen3 defaults — NOT a resolution error. These keys + are deliberately not in REQUIRED_APP_KEYS so existing + environments keep restarting exactly what they always did. + """ + _seed_env("etl", "test") + + e = config.resolve_env("test") + assert e.restart_services == config.DEFAULT_RESTART_SERVICES + assert e.etl_cronjob == config.DEFAULT_ETL_CRONJOB + assert "portal-deployment" in e.restart_services diff --git a/tests/test_restart_scripts_sh.py b/tests/test_restart_scripts_sh.py new file mode 100644 index 0000000..8639666 --- /dev/null +++ b/tests/test_restart_scripts_sh.py @@ -0,0 +1,184 @@ +"""End-to-end sequencing tests for the ArgoCD restart scripts. + +``argocd_restart_schema.sh`` / ``argocd_restart_ms.sh`` restart the commons' +schema microservices **serially, in list order**, waiting for each rollout to +report Healthy before starting the next; ``argocd_restart_etl.sh`` creates and +watches a run of a named ETL cronjob. Since v3.5.0 the targets are no longer +hardcoded: the list and cronjob name default to ``$G3DT_RESTART_SERVICES`` / +``$G3DT_ETL_CRONJOB`` — the env's SSM ``app/restart_services`` / +``app/etl_cronjob`` facts, exported by g3dt — with the classic Gen3 set as the +fallback for direct invocations. + +That resolution and the restart ORDER are only observable by running the real +scripts, so these tests stub ``argocd``, ``jq``, ``kubectl``, and ``sleep`` on +PATH (recording every call, answering Healthy/succeeded immediately) and +assert exactly which resources are restarted, in which sequence. This is the +closest an offline test can get to a live `g3dt dict deploy` restart cycle. +""" +import os +import subprocess +from pathlib import Path + +import pytest + +K8S_OPS = Path(__file__).resolve().parent.parent / "src" / "g3dt" / "services" / "k8s_ops" + +CLASSIC = [ + "sheepdog-deployment", + "peregrine-deployment", + "guppy-deployment", + "portal-deployment", +] + + +@pytest.fixture +def stub_bin(tmp_path): + """Stub argocd/jq/kubectl/sleep on PATH, recording every invocation. + + - ``argocd`` logs its args and exits 0 (its ``app get`` output is unused + because ``jq`` is also stubbed). + - ``jq`` answers '"Healthy"' so the per-resource wait loop exits on the + first check. + - ``sleep`` is a no-op so the serial waits don't slow the suite. + - ``kubectl`` answers the etl script's job lifecycle: created job name, + succeeded status, a pod name, and logs containing "Exit code: 0". + """ + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + record = tmp_path / "record.txt" + + (bin_dir / "argocd").write_text( + '#!/usr/bin/env bash\necho "argocd $*" >> "$STUB_RECORD"\nexit 0\n' + ) + (bin_dir / "jq").write_text( + '#!/usr/bin/env bash\necho \'"Healthy"\'\n' + ) + (bin_dir / "sleep").write_text("#!/usr/bin/env bash\nexit 0\n") + (bin_dir / "kubectl").write_text( + "#!/usr/bin/env bash\n" + 'echo "kubectl $*" >> "$STUB_RECORD"\n' + 'case "$*" in\n' + " *current-context*) echo ctx ;;\n" + " *create\\ job*) echo job.batch/test-job ;;\n" + " *succeeded*) echo 1 ;;\n" + " *failed*) echo '' ;;\n" + " *get\\ pods*) echo test-pod ;;\n" + ' *logs*) echo "Exit code: 0" ;;\n' + "esac\nexit 0\n" + ) + for f in bin_dir.iterdir(): + f.chmod(0o755) + return bin_dir, record + + +def _run(script, stub_bin, args=(), extra_env=None, expect_rc=0): + bin_dir, record = stub_bin + env = dict( + os.environ, + PATH=f"{bin_dir}:{os.environ['PATH']}", + STUB_RECORD=str(record), + ) + env.pop("G3DT_RESTART_SERVICES", None) + env.pop("G3DT_ETL_CRONJOB", None) + env.pop("G3DT_NAMESPACE", None) + if extra_env: + env.update(extra_env) + result = subprocess.run( + ["bash", str(K8S_OPS / script), "-l", *args], + env=env, capture_output=True, text=True, + ) + assert result.returncode == expect_rc, result.stdout + result.stderr + return record.read_text() if record.exists() else "" + + +def _restart_order(recorded): + """Deployment names from the 'actions run ... restart' lines, in order.""" + names = [] + for line in recorded.splitlines(): + if "actions run" in line and "restart" in line: + parts = line.split() + names.append(parts[parts.index("--resource-name") + 1]) + return names + + +@pytest.mark.parametrize("script", ["argocd_restart_schema.sh", "argocd_restart_ms.sh"]) +def test_env_restart_services_define_the_set_and_order(script, stub_bin): + """ + Inputs: G3DT_RESTART_SERVICES with a custom, reordered subset (what an + env like omix3 publishes — no portal, since its frontend is + redeployed manually outside this flow) + Expected: exactly those deployments are restarted, serially, in the given + order — for both the schema and ms variants of the script. + """ + recorded = _run( + script, stub_bin, + args=("-d", "cd.example.org", "-a", "testgen3", "-n", "omix3"), + extra_env={ + "G3DT_RESTART_SERVICES": "guppy-deployment,sheepdog-deployment", + }, + ) + assert _restart_order(recorded) == ["guppy-deployment", "sheepdog-deployment"] + assert "portal-deployment" not in recorded + + +@pytest.mark.parametrize("script", ["argocd_restart_schema.sh", "argocd_restart_ms.sh"]) +def test_classic_set_when_nothing_configured(script, stub_bin): + """ + Inputs: no G3DT_RESTART_SERVICES and no -r (a pre-k8s-block deployment, + or a direct invocation outside g3dt) + Expected: the classic Gen3 four, in the historical order — existing + environments keep restarting exactly what they always did. + """ + recorded = _run( + script, stub_bin, + args=("-d", "cd.example.org", "-a", "testgen3", "-n", "cad"), + ) + assert _restart_order(recorded) == CLASSIC + + +def test_r_flag_beats_env(stub_bin): + """ + Inputs: both G3DT_RESTART_SERVICES and an explicit -r + Expected: -r wins — the flag is the per-run escape hatch above SSM. + """ + recorded = _run( + "argocd_restart_schema.sh", stub_bin, + args=("-d", "d", "-a", "a", "-n", "ns", "-r", "portal-deployment"), + extra_env={"G3DT_RESTART_SERVICES": "sheepdog-deployment"}, + ) + assert _restart_order(recorded) == ["portal-deployment"] + + +def test_missing_namespace_fails_fast(stub_bin): + """ + Inputs: no -n and no G3DT_NAMESPACE + Expected: exit 1 before any argocd call. The old script silently defaulted + to the legacy 'cad' namespace, which would restart another + project's services when run outside g3dt. + """ + recorded = _run( + "argocd_restart_schema.sh", stub_bin, + args=("-d", "d", "-a", "a"), + expect_rc=1, + ) + assert "actions run" not in recorded + + +def test_etl_cronjob_name_from_env(stub_bin): + """ + Inputs: G3DT_ETL_CRONJOB=custom-etl (the env's SSM app/etl_cronjob) + Expected: the job is created from cronjob/custom-etl; with nothing set the + classic etl-cronjob name is used. + """ + recorded = _run( + "argocd_restart_etl.sh", stub_bin, + args=("-d", "d", "-a", "a", "-n", "ns"), + extra_env={"G3DT_ETL_CRONJOB": "custom-etl"}, + ) + assert "--from=cronjob/custom-etl" in recorded + + recorded = _run( + "argocd_restart_etl.sh", stub_bin, + args=("-d", "d", "-a", "a", "-n", "ns"), + ) + assert "--from=cronjob/etl-cronjob" in recorded