feat: configurable k8s restart targets (services list + etl cronjob)

README.md

@@ -174,6 +174,21 @@ g3dt synth generate AusDiab_Simulated --llm -n 5 -e test
 `g3dt config show --env <env>` prints the resolved provider, model, and key
 path.
 
+### Kubernetes restart targets
+
+`k8s restart-schema`, `k8s restart-ms`, `dict deploy`, and `synth deploy`
+restart the commons' schema microservices **serially, in a configured order**,
+waiting for each rollout to go Healthy before the next; `k8s restart-etl` and
+the deploy flows run a named ETL cronjob. Both targets resolve with precedence
+**CLI flags > SSM > default**: the CDK config's optional `k8s` block publishes
+`app/restart_services` (comma-separated deployment names) and
+`app/etl_cronjob`, `--restart-services` / `--etl-cronjob` override them for
+one run, and environments deployed without the block keep the classic Gen3 set
+(`sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment`
+/ `etl-cronjob`). A commons that manages some service outside this flow (e.g.
+a manually redeployed frontend) simply omits it from the list in its wrapper
+config. `g3dt config show --env <env>` prints the resolved values.
+
 ## Verifying download access (check-download)
 
 Registration alone does not prove a file can be downloaded. Two failure modes

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gen3-dataops-toolkit"
-version = "3.4.0"
+version = "3.5.0"
 description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"

src/g3dt/cli/config_cmds.py

@@ -123,6 +123,10 @@ def show(
     typer.echo(f"  llm_provider       : {e.llm_provider}")
     typer.echo(f"  llm_model          : {e.llm_model or '(not set — pass --llm-model or add the llm block to the CDK config)'}")
     typer.echo(f"  llm_api_key_file   : {config.llm_api_key_file() or '(not set — g3dt config set llm_api_key_file <path>)'}")
+    # k8s restart targets: from SSM (the CDK's optional k8s block), restarted
+    # in the listed order by restart-schema/restart-ms/dict deploy/synth deploy.
+    typer.echo(f"  restart_services   : {e.restart_services}")
+    typer.echo(f"  etl_cronjob        : {e.etl_cronjob}")
     if study:
         s = study_of(study, env)
         typer.secho(f"Study: {study} -> {s.key}", bold=True)
@@ -217,6 +221,16 @@ def check(label: str, file_value, ssm_value) -> None:
     for camel, leaf in {"provider": "llm_provider", "model": "llm_model"}.items():
         if camel in llm:
             check(f"llm.{camel}", llm.get(camel), rc.get(f"app/{leaf}"))
+    k8s = inputs.get("k8s") or {}
+    if "schemaRestartServices" in k8s:
+        # The CDK publishes the list comma-joined; compare the same shape.
+        check(
+            "k8s.schemaRestartServices",
+            ",".join(k8s.get("schemaRestartServices") or []),
+            rc.get("app/restart_services"),
+        )
+    if "etlCronjob" in k8s:
+        check("k8s.etlCronjob", k8s.get("etlCronjob"), rc.get("app/etl_cronjob"))
 
     if not drift:
         typer.secho(

src/g3dt/cli/dict_cmds.py

@@ -103,6 +103,11 @@ def deploy(
     version: str = typer.Option(
         None, "--version", help="Dictionary git tag (default: the env's version)."
     ),
+    restart_services: Optional[str] = typer.Option(
+        None, "--restart-services",
+        help="Comma-separated deployment names restarted after the upload, in "
+        "order; default: the env's SSM app/restart_services.",
+    ),
 ) -> None:
     """Pull + upload the dictionary and restart Gen3 schema microservices.
 
@@ -123,7 +128,10 @@ def deploy(
     """
     e = env_of(env)
     warn_if_overridden(e, version)
+    env_vars = script_env(e, _version(e, version))
+    if restart_services:
+        env_vars["G3DT_RESTART_SERVICES"] = restart_services
     runner.run(
         runner.bash_script("services/dictionary/deploy_dd.sh", env),
-        env=script_env(e, _version(e, version)),
+        env=env_vars,
     )

src/g3dt/cli/k8s.py

@@ -3,9 +3,18 @@
 These use ``argocd login --sso`` (a browser flow), so they cannot run headless
 on EC2. The wrapped scripts receive their settings as ``G3DT_*`` environment
 variables resolved from SSM — they read no config files.
+
+The restart targets resolve with precedence CLI flags > SSM > default: the
+CDK config's optional ``k8s`` block publishes ``app/restart_services`` (a
+comma-separated list, restarted in order) and ``app/etl_cronjob``;
+``--restart-services`` / ``--etl-cronjob`` override them for one run, and
+environments deployed without the block keep the classic Gen3 set
+(sheepdog, peregrine, guppy, portal / etl-cronjob).
 """
 from __future__ import annotations
 
+from typing import Optional
+
 import typer
 
 from g3dt.config import script_env
@@ -18,37 +27,78 @@
 _ETL = "services/k8s_ops/argocd_restart_etl.sh"
 _ETL_AND_MS = "services/k8s_ops/restart_etl_and_ms.sh"
 
+_RESTART_SERVICES_HELP = (
+    "Comma-separated deployment names to restart, in order; default: the "
+    "env's SSM app/restart_services (the CDK config's k8s.schemaRestartServices)."
+)
+_ETL_CRONJOB_HELP = (
+    "ETL cronjob name; default: the env's SSM app/etl_cronjob "
+    "(the CDK config's k8s.etlCronjob)."
+)
+
+
+def restart_env(e, restart_services: Optional[str] = None,
+                etl_cronjob: Optional[str] = None) -> dict:
+    """script_env plus per-run restart-target overrides (flags beat SSM)."""
+    env_vars = script_env(e)
+    if restart_services:
+        env_vars["G3DT_RESTART_SERVICES"] = restart_services
+    if etl_cronjob:
+        env_vars["G3DT_ETL_CRONJOB"] = etl_cronjob
+    return env_vars
+
 
 @app.command(name="restart-schema")
 def restart_schema(
     env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
     sync: bool = typer.Option(False, "--sync", "-s", help="argocd app sync first."),
+    restart_services: Optional[str] = typer.Option(
+        None, "--restart-services", help=_RESTART_SERVICES_HELP
+    ),
 ) -> None:
-    """Restart sheepdog/peregrine/guppy/portal (schema microservices)."""
+    """Restart the schema microservices, in the env's configured order."""
     e = env_of(env)
     args = ["-d", e.domain, "-a", e.app_name, "-n", e.namespace]
     if sync:
         args.append("-s")
-    runner.run(runner.bash_script(_SCHEMA, *args), env=script_env(e))
+    runner.run(
+        runner.bash_script(_SCHEMA, *args),
+        env=restart_env(e, restart_services=restart_services),
+    )
 
 
 @app.command(name="restart-etl")
 def restart_etl(
     env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
     sync: bool = typer.Option(False, "--sync", "-s", help="argocd app sync first."),
+    etl_cronjob: Optional[str] = typer.Option(
+        None, "--etl-cronjob", help=_ETL_CRONJOB_HELP
+    ),
 ) -> None:
     """Create + run the ETL cronjob and wait for completion."""
     e = env_of(env)
     args = ["-e", env]
     if sync:
         args.append("-s")
-    runner.run(runner.bash_script(_ETL, *args), env=script_env(e))
+    runner.run(
+        runner.bash_script(_ETL, *args),
+        env=restart_env(e, etl_cronjob=etl_cronjob),
+    )
 
 
 @app.command(name="restart-ms")
 def restart_ms(
     env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
+    restart_services: Optional[str] = typer.Option(
+        None, "--restart-services", help=_RESTART_SERVICES_HELP
+    ),
+    etl_cronjob: Optional[str] = typer.Option(
+        None, "--etl-cronjob", help=_ETL_CRONJOB_HELP
+    ),
 ) -> None:
     """Restart both ETL and schema microservices (wraps restart_etl_and_ms.sh)."""
     e = env_of(env)
-    runner.run(runner.bash_script(_ETL_AND_MS, env), env=script_env(e))
+    runner.run(
+        runner.bash_script(_ETL_AND_MS, env),
+        env=restart_env(e, restart_services=restart_services, etl_cronjob=etl_cronjob),
+    )

src/g3dt/cli/synth.py

@@ -168,17 +168,31 @@ def deploy(
         help="Path to the file holding the LLM API key; default: the marker's "
         "llm_api_key_file (set once: g3dt config set llm_api_key_file <path>).",
     ),
+    restart_services: Optional[str] = typer.Option(
+        None, "--restart-services",
+        help="Comma-separated deployment names restarted during the deploy, in "
+        "order; default: the env's SSM app/restart_services.",
+    ),
+    etl_cronjob: Optional[str] = typer.Option(
+        None, "--etl-cronjob",
+        help="ETL cronjob name; default: the env's SSM app/etl_cronjob.",
+    ),
 ) -> None:
     """Full end-to-end synthetic deploy (dict + LLM-generate + upload + restarts).
 
     Wraps services/synthetic_data/full_deploy_dd_and_synth.sh (LLM-backed
-    generation). Provider/model come from the env's SSM tree unless
-    overridden; the API key path comes from --llm-api-key-file or the marker.
+    generation). Provider/model and the restart targets come from the env's
+    SSM tree unless overridden; the API key path comes from
+    --llm-api-key-file or the marker.
     """
     e = env_of(env)
     safety.confirm_prod_strict("synthetic full deploy", env)
     env_vars = script_env(e)
     env_vars.update(_llm_env_overrides(e, llm_provider, llm_model, llm_api_key_file))
+    if restart_services:
+        env_vars["G3DT_RESTART_SERVICES"] = restart_services
+    if etl_cronjob:
+        env_vars["G3DT_ETL_CRONJOB"] = etl_cronjob
     runner.run(
         runner.bash_script(
             "services/synthetic_data/full_deploy_dd_and_synth.sh", env

src/g3dt/config.py

@@ -58,6 +58,17 @@
 #: when the ``--llm`` path is used and no model is configured anywhere.
 DEFAULT_LLM_PROVIDER = "anthropic"
 
+#: Kubernetes restart targets, published by the CDK's OPTIONAL ``k8s`` config
+#: block as ``app/restart_services`` (comma-separated, restarted in order) and
+#: ``app/etl_cronjob``. Optional app inputs: environments deployed without the
+#: block keep the classic Gen3 set. Consumed by every restart path — `g3dt
+#: k8s restart-schema/restart-etl/restart-ms`, `dict deploy`, `synth deploy` —
+#: via $G3DT_RESTART_SERVICES / $G3DT_ETL_CRONJOB in the service scripts.
+DEFAULT_RESTART_SERVICES = (
+    "sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment"
+)
+DEFAULT_ETL_CRONJOB = "etl-cronjob"
+
 #: Marker locations, most specific first.
 MARKER_PATHS = ("g3dt.yaml", "~/.g3dt/g3dt.yaml", "/etc/g3dt/g3dt.yaml")
 
@@ -257,6 +268,9 @@ class EnvConfig:
     # with guidance rather than silently picking a model.
     llm_provider: str = DEFAULT_LLM_PROVIDER
     llm_model: Optional[str] = None
+    # Optional k8s restart targets (SSM app/restart_services, app/etl_cronjob).
+    restart_services: str = DEFAULT_RESTART_SERVICES
+    etl_cronjob: str = DEFAULT_ETL_CRONJOB
 
 
 def _app_or_default(rc, leaf: str, default: str) -> str:
@@ -320,6 +334,10 @@ def resolve_env(env: str, project: Optional[str] = None) -> EnvConfig:
         # the config has an llm block, so absence means "use the defaults".
         llm_provider=_app_or_default(rc, "llm_provider", DEFAULT_LLM_PROVIDER),
         llm_model=(_app_or_default(rc, "llm_model", "") or None),
+        restart_services=_app_or_default(
+            rc, "restart_services", DEFAULT_RESTART_SERVICES
+        ),
+        etl_cronjob=_app_or_default(rc, "etl_cronjob", DEFAULT_ETL_CRONJOB),
     )
 
 
@@ -428,6 +446,8 @@ def script_env(e: EnvConfig, version: Optional[str] = None) -> Dict[str, str]:
         "G3DT_SCHEMA_REPO": e.schema_repo,
         "G3DT_LLM_PROVIDER": e.llm_provider,
         "G3DT_LLM_MODEL": e.llm_model,
+        "G3DT_RESTART_SERVICES": e.restart_services,
+        "G3DT_ETL_CRONJOB": e.etl_cronjob,
     }
     env.update({k: v for k, v in values.items() if v is not None})
     return env

src/g3dt/services/k8s_ops/argocd_restart_etl.sh

@@ -6,7 +6,7 @@ usage() {
     echo "  -d DOMAIN      The domain for argocd login (example: cd.cad.test.biocommons.org.au)"
     echo "  -a APPNAME     The application name (example: uatgen3)"
     echo "  -n NAMESPACE   The namespace for the resources (example: cad)"
-    echo "  -c ETL_CRONJOB The name of the ETL cronjob to run (default: etl-cronjob)"
+    echo "  -c ETL_CRONJOB The name of the ETL cronjob to run (default: \$G3DT_ETL_CRONJOB — the env's SSM app/etl_cronjob, set by g3dt — else etl-cronjob)"
     echo "  -t CONTAINER   The name of the container to check logs from (default: tube)"
     echo "  -l             Bypass login"
     echo "  -s             Sync the argocd app before restarting resources"
@@ -15,8 +15,9 @@ usage() {
 
 set -eo pipefail
 
-# default values
-ETL_CRONJOB="etl-cronjob"
+# default values. The cronjob name comes from the environment's SSM tree
+# (app/etl_cronjob, exported by g3dt as G3DT_ETL_CRONJOB).
+ETL_CRONJOB="${G3DT_ETL_CRONJOB:-etl-cronjob}"
 CONTAINER_TO_CHECK="tube"
 LOGIN_REQUIRED=true
 SYNC_APP=false

src/g3dt/services/k8s_ops/argocd_restart_ms.sh

@@ -6,8 +6,11 @@ usage() {
     echo "Usage: $0 [-d DOMAIN] [-a APPNAME] [-r RESOURCES] [-n NAMESPACE] [-k KIND] [-l] [-s]"
     echo "  -d DOMAIN      The domain for argocd login (example: cd.cad.test.biocommons.org.au)"
     echo "  -a APPNAME     The application name (example: uatgen3)"
-    echo "  -r RESOURCES   Comma-separated string of microservice names to restart (example: sheepdog-deployment,peregrine-deployment )"
-    echo "  -n NAMESPACE   The namespace for the resources (example: cad)"
+    echo "  -r RESOURCES   Comma-separated microservice names, restarted in order"
+    echo "                 (default: \$G3DT_RESTART_SERVICES — the env's SSM"
+    echo "                 app/restart_services, set by g3dt — else the classic"
+    echo "                 sheepdog,peregrine,guppy,portal set)"
+    echo "  -n NAMESPACE   The namespace for the resources (default: \$G3DT_NAMESPACE)"
     echo "  -k KIND        The kind of resource to restart (default: Deployment)"
     echo "  -l             Bypass login"
     echo "  -s             Sync the argocd app before restarting resources"
@@ -16,7 +19,10 @@ usage() {
 
 set -eo pipefail
 
-# Parse command line arguments
+# Parse command line arguments. The restart set defaults to the environment's
+# SSM app/restart_services (exported by g3dt as G3DT_RESTART_SERVICES).
+IFS=',' read -r -a RESOURCES <<< "${G3DT_RESTART_SERVICES:-sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment}"
+NAMESPACE="${G3DT_NAMESPACE:-}"
 LOGIN_REQUIRED=true
 KIND="Deployment"
 SYNC_APP=false
@@ -52,6 +58,11 @@ while getopts "d:a:r:n:k:hls" opt; do
     esac
 done
 
+if [ -z "$NAMESPACE" ]; then
+    echo "No namespace given: pass -n, or run via the g3dt CLI (which sets G3DT_NAMESPACE)." >&2
+    exit 1
+fi
+
 # Check if argocd is installed
 if ! command -v argocd &> /dev/null
 then

src/g3dt/services/k8s_ops/argocd_restart_schema.sh

@@ -6,8 +6,11 @@ usage() {
     echo "Usage: $0 [-d DOMAIN] [-a APPNAME] [-r RESOURCES] [-n NAMESPACE] [-k KIND] [-l] [-s]"
     echo "  -d DOMAIN      The domain for argocd login (example: cd.cad.test.biocommons.org.au)"
     echo "  -a APPNAME     The application name (example: uatgen3)"
-    echo "  -r RESOURCES   Comma-separated string of microservice names to restart (default: \"sheepdog-deployment\" \"peregrine-deployment\" \"guppy-deployment\" \"portal-deployment\")"
-    echo "  -n NAMESPACE   The namespace for the resources (default: cad)"
+    echo "  -r RESOURCES   Comma-separated microservice names, restarted in order"
+    echo "                 (default: \$G3DT_RESTART_SERVICES — the env's SSM"
+    echo "                 app/restart_services, set by g3dt — else the classic"
+    echo "                 sheepdog,peregrine,guppy,portal set)"
+    echo "  -n NAMESPACE   The namespace for the resources (default: \$G3DT_NAMESPACE)"
     echo "  -k KIND        The kind of resource to restart (default: Deployment)"
     echo "  -l             Bypass login"
     echo "  -s             Run 'argocd app sync' before restarts"
@@ -16,9 +19,11 @@ usage() {
 
 set -eo pipefail
 
-# Set default values
-RESOURCES=("sheepdog-deployment" "peregrine-deployment" "guppy-deployment" "portal-deployment")
-NAMESPACE="cad"
+# Set default values. The restart set comes from the environment's SSM tree
+# (app/restart_services, exported by g3dt as G3DT_RESTART_SERVICES); the
+# classic Gen3 set is the fallback for direct invocations outside g3dt.
+IFS=',' read -r -a RESOURCES <<< "${G3DT_RESTART_SERVICES:-sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment}"
+NAMESPACE="${G3DT_NAMESPACE:-}"
 KIND="Deployment"
 LOGIN_REQUIRED=true
 SYNC_APP=false
@@ -56,6 +61,11 @@ while getopts "d:a:r:n:k:hls" opt; do
     esac
 done
 
+if [ -z "$NAMESPACE" ]; then
+    echo "No namespace given: pass -n, or run via the g3dt CLI (which sets G3DT_NAMESPACE)." >&2
+    exit 1
+fi
+
 # Check if argocd is installed
 if ! command -v argocd &> /dev/null
 then