diff --git a/config/forge-saw/README.md b/config/forge-saw/README.md index 6f1ea23..7133e2c 100644 --- a/config/forge-saw/README.md +++ b/config/forge-saw/README.md @@ -148,8 +148,51 @@ oc get secret openshell-credentials -n guy-ziv-evalflow -o json \ `aeh-openshell-eval` TCP-checks `:17670` and optionally runs `openshell sandbox list`. If SAW is down the **run fails**. That step does not install SAW. +## NetworkPolicy (Forge shared namespace) + +When SAW VMs and Tekton share a namespace with default-deny policies, evaluate +pods must be allowed to reach the agent gateway on `:17670`. Use the canonical +Pipeline name `abevalflow-pipeline-openshell` and its referenced Task name +`evaluate` so the injected `tekton.dev/pipeline` and `tekton.dev/task` pod +labels match. + +The egress policy permits SAW gateways in the evaluate namespace, LiteLLM, +MLflow and MinIO either there or in `gz-forge-eval`, DNS, and public TCP/443 +for the Task's OpenShell download, Python package installs and public OIDC +issuer. Kubernetes NetworkPolicy cannot limit the HTTPS rule to GitHub or +PyPI hostnames. If the OIDC issuer is private, add an allow rule for its actual +namespace and pod labels or route before enabling that mode. The shared Task +also installs packages at runtime; an image with those dependencies baked in +would allow a narrower deployment policy. + +Same-namespace template: + +```bash +sed "s/NAMESPACE/${EVAL_NS}/g" config/forge-saw/networkpolicy-ci-openshell.yaml \ + | oc apply -f - +``` + +If the earlier two-selector template was applied, remove its old egress policy +after applying this one; `oc apply` does not delete objects omitted from a file: + +```bash +oc delete networkpolicy abevalflow-allow-ci-egress-by-part-of \ + -n "$EVAL_NS" --ignore-not-found +``` + +Copies of the Pipeline or Evaluate Task under another name need matching +NetworkPolicy selectors; otherwise gateway preflight times out. + ## Image Stock `agent-eval-harness:v1.0.x` cannot import `agent_eval.openshell`. Point `aeh-openshell-image` at an orchestrator image built from GuyZivRH -`agent-eval-harness` **main** that also includes the `openshell` CLI. +`agent-eval-harness` at the pinned OpenShell Pipeline revision that also +includes the `openshell` CLI. Use `main` only as an explicit development +override. + +The OpenShell Pipeline profile explicitly sets `openshell-user-fixture` to +`fixtures/USER.md` for the Forge submission. The shared Evaluate Task defaults +this parameter to empty. Set it to an empty value for a submission without an +installation fixture, or provide another path relative to its submission +directory. A configured path must exist when Evaluate runs. diff --git a/config/forge-saw/networkpolicy-ci-openshell.yaml b/config/forge-saw/networkpolicy-ci-openshell.yaml new file mode 100644 index 0000000..7e82997 --- /dev/null +++ b/config/forge-saw/networkpolicy-ci-openshell.yaml @@ -0,0 +1,170 @@ +# NetworkPolicies for OpenShell OpenClaw evals when SAW VMs and the Tekton +# pipeline share one namespace (Forge workspace style). +# +# Select the evaluate Task pods in the canonical OpenShell Pipeline by the +# injected tekton.dev/pipeline and tekton.dev/task labels. +# +# Renamed copies of the Pipeline need a matching policy selector or gateway +# preflight will be blocked by default-deny. +# +# Apply after substituting NAMESPACE for the SAW/evaluate namespace. LiteLLM, +# MLflow and MinIO may also run in gz-forge-eval (the example PipelineRun). +# Public HTTPS is needed for CLI downloads, Python packages and public OIDC +# issuers. Private OIDC issuers need an additional deployment-specific rule. +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: abevalflow-allow-ci-agent-gateway + namespace: NAMESPACE + labels: + app.kubernetes.io/part-of: abevalflow + app.kubernetes.io/component: forge-saw +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: openshell-saw-agent + vm.kubevirt.io/name: openshell-saw-agent + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + tekton.dev/pipeline: abevalflow-pipeline-openshell + tekton.dev/task: evaluate + ports: + - protocol: TCP + port: 17670 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: abevalflow-allow-ci-integ-gateway + namespace: NAMESPACE + labels: + app.kubernetes.io/part-of: abevalflow + app.kubernetes.io/component: forge-saw +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: openshell-saw-integ + vm.kubevirt.io/name: openshell-saw-integ + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + tekton.dev/pipeline: abevalflow-pipeline-openshell + tekton.dev/task: evaluate + ports: + - protocol: TCP + port: 17670 + - protocol: TCP + port: 18082 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: abevalflow-allow-ci-egress-by-pipeline + namespace: NAMESPACE + labels: + app.kubernetes.io/part-of: abevalflow + app.kubernetes.io/component: forge-saw +spec: + podSelector: + matchLabels: + tekton.dev/pipeline: abevalflow-pipeline-openshell + tekton.dev/task: evaluate + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: openshell-saw-agent + vm.kubevirt.io/name: openshell-saw-agent + ports: + - protocol: TCP + port: 17670 + - protocol: TCP + port: 8443 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: openshell-saw-integ + vm.kubevirt.io/name: openshell-saw-integ + ports: + - protocol: TCP + port: 17670 + - protocol: TCP + port: 18082 + - protocol: TCP + port: 8443 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: litellm + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: gz-forge-eval + podSelector: + matchLabels: + app.kubernetes.io/name: litellm + ports: + - protocol: TCP + port: 4000 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: mlflow + app.kubernetes.io/part-of: abevalflow + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: gz-forge-eval + podSelector: + matchLabels: + app.kubernetes.io/name: mlflow + app.kubernetes.io/part-of: abevalflow + ports: + - protocol: TCP + port: 5000 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: abevalflow + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: gz-forge-eval + podSelector: + matchLabels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: abevalflow + ports: + - protocol: TCP + port: 9000 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - protocol: UDP + port: 5353 + - protocol: TCP + port: 5353 + # Setup and runtime HTTPS: OpenShell release download, PyPI packages, + # and a public OIDC issuer when client credentials are configured. + # NetworkPolicy cannot restrict this rule to DNS names. + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + ports: + - protocol: TCP + port: 443 diff --git a/config/forge-saw/networkpolicy-gateway-from-abeval.yaml b/config/forge-saw/networkpolicy-gateway-from-abeval.yaml index af1aaef..54a410f 100644 --- a/config/forge-saw/networkpolicy-gateway-from-abeval.yaml +++ b/config/forge-saw/networkpolicy-gateway-from-abeval.yaml @@ -1,6 +1,10 @@ # Allow evaluate Task pods to reach the OpenShell gateway on :17670. # bootstrap.sh rewrites namespace / from-namespace (SAW_NS, EVAL_NS). # Apply in the SAW namespace, not from the evaluate PipelineRun. +# +# Match evaluate Task pods in the canonical Pipeline. A bare +# podSelector: {} from the whole EVAL_NS is broader than needed when SAW +# and eval share a Forge workspace namespace — see networkpolicy-ci-openshell.yaml. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -20,7 +24,10 @@ spec: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: guy-ziv-evalflow - - podSelector: {} + podSelector: + matchLabels: + tekton.dev/pipeline: abevalflow-pipeline-openshell + tekton.dev/task: evaluate ports: - protocol: TCP port: 17670 diff --git a/pipeline/pipelines/ci-pipeline-openshell.yaml b/pipeline/pipelines/ci-pipeline-openshell.yaml index 07a41c5..2cbeb3e 100644 --- a/pipeline/pipelines/ci-pipeline-openshell.yaml +++ b/pipeline/pipelines/ci-pipeline-openshell.yaml @@ -5,6 +5,7 @@ metadata: namespace: ab-eval-flow labels: app.kubernetes.io/name: abevalflow + app.kubernetes.io/part-of: abevalflow app.kubernetes.io/version: "2.0" app.kubernetes.io/profile: aeh-openshell-openclaw spec: @@ -102,8 +103,10 @@ spec: default: "https://github.com/GuyZivRH/agent-eval-harness.git" - name: agent-eval-harness-repo-revision type: string - default: "feat/aeh-openshell-openclaw" - description: Harness revision with agent_eval.openshell (feature-branch pin) + default: "8d58e500d0eb55a3106819ccacf26da6bf7ea166" + description: >- + Verified GuyZivRH/agent-eval-harness commit containing the OpenClaw + fixes merged in #9. Override with main only for development runs. - name: openshell-cli-version type: string default: "0.0.116" @@ -112,10 +115,16 @@ spec: default: "https://openshell.openshell.svc.cluster.local:17670" - name: openshell-sandbox-image type: string - default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a" + default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" - name: openshell-provider type: string default: "forge-ai-gateway,m365-read-intervm,slack-read-proxy,drafts-service-agent" + - name: openshell-user-fixture + type: string + default: "fixtures/USER.md" + description: >- + OpenClaw Forge installation fixture relative to the submission directory. + Override with an empty value for submissions without this fixture. - name: openshell-mtls-secret type: string default: "openshell-mtls" @@ -279,6 +288,8 @@ spec: value: $(params.openshell-sandbox-image) - name: openshell-provider value: $(params.openshell-provider) + - name: openshell-user-fixture + value: $(params.openshell-user-fixture) - name: openshell-mtls-secret value: $(params.openshell-mtls-secret) - name: openshell-ai-gateway-ca-secret diff --git a/pipeline/runs/openshell-openclaw-pipelinerun.yaml b/pipeline/runs/openshell-openclaw-pipelinerun.yaml index f46241e..885d566 100644 --- a/pipeline/runs/openshell-openclaw-pipelinerun.yaml +++ b/pipeline/runs/openshell-openclaw-pipelinerun.yaml @@ -16,12 +16,12 @@ spec: value: "openclaw-forge" - name: eval-engine value: "aeh_openshell_openclaw" - # Until this branch is merged, pass the feature revision so store/parse scripts match. - # After merge to main, omit these (Pipeline defaults are main). + # This example targets the merged upstream main for both submission and + # Pipeline helper source. Replace either value with a commit SHA to pin a run. - name: revision - value: "feat/aeh-openshell-openclaw" + value: "main" - name: pipeline-repo-revision - value: "feat/aeh-openshell-openclaw" + value: "main" # [Optional] Add a harness repository or revision for this run; these replace Pipeline defaults. # - name: agent-eval-harness-repo-url # value: "" @@ -36,9 +36,11 @@ spec: value: "forge-agent-upstream-tls" # To use another sandbox image, replace the value below with . - name: openshell-sandbox-image - value: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a" + value: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" - name: openshell-provider value: "forge-ai-gateway,m365-read-intervm,slack-read-proxy,drafts-service-agent" + - name: openshell-user-fixture + value: "fixtures/USER.md" - name: aeh-openshell-image value: "registry.access.redhat.com/ubi9/python-311:9.6" - name: llm-model diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index 827346c..8ce8940 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -184,10 +184,10 @@ spec: default: "https://github.com/GuyZivRH/agent-eval-harness.git" - name: agent-eval-harness-repo-revision type: string - default: "feat/aeh-openshell-openclaw" + default: "8d58e500d0eb55a3106819ccacf26da6bf7ea166" description: >- - Branch or tag of GuyZivRH/agent-eval-harness to clone. Must include the - OpenShell create keep-alive fix (`--detach` + `sleep infinity`). + Commit SHA of GuyZivRH/agent-eval-harness with the OpenShell fixes. + Branches and tags are accepted as explicit overrides. - name: openshell-gateway-endpoint type: string default: "http://openshell.openshell.svc.cluster.local:8080" @@ -196,7 +196,7 @@ spec: Override only if using a different gateway (e.g. forge-saw :17670). - name: openshell-sandbox-image type: string - default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a" + default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" description: >- Immutable SAW Chief-of-Staff image with the daily-briefing skill and governed M365/Slack tools. @@ -209,6 +209,10 @@ spec: type: string default: "forge-ai-gateway,m365-read-intervm,slack-read-proxy,drafts-service-agent" description: SAW provider explicitly attached to each evaluation sandbox + - name: openshell-user-fixture + type: string + default: "" + description: Optional installation fixture path relative to the submission directory - name: openshell-mtls-secret type: string default: "openshell-mtls" @@ -276,14 +280,15 @@ spec: # Clone/update agent-eval-harness repo (required for latest OpenShell logic) HARNESS_DIR="$(workspaces.source.path)/_harness" echo "=== Cloning harness repo: $(params.agent-eval-harness-repo-url) @ $(params.agent-eval-harness-repo-revision) ===" - if [ -d "$HARNESS_DIR/.git" ]; then - cd "$HARNESS_DIR" - git fetch origin "$(params.agent-eval-harness-repo-revision)" --depth 1 - git reset --hard FETCH_HEAD + if [ ! -d "$HARNESS_DIR/.git" ]; then + mkdir -p "$HARNESS_DIR" + git -C "$HARNESS_DIR" init -q + git -C "$HARNESS_DIR" remote add origin "$(params.agent-eval-harness-repo-url)" else - git clone --depth 1 --branch "$(params.agent-eval-harness-repo-revision)" \ - "$(params.agent-eval-harness-repo-url)" "$HARNESS_DIR" + git -C "$HARNESS_DIR" remote set-url origin "$(params.agent-eval-harness-repo-url)" fi + git -C "$HARNESS_DIR" fetch --depth 1 origin "$(params.agent-eval-harness-repo-revision)" + git -C "$HARNESS_DIR" checkout --detach -f FETCH_HEAD echo "Harness repo ready at $(git -C "$HARNESS_DIR" rev-parse --short HEAD)" # Create results directories @@ -1680,6 +1685,8 @@ spec: value: "$(params.openshell-sandbox-image)" - name: AGENT_EVAL_OPENSHELL_PROVIDER value: "$(params.openshell-provider)" + - name: OPENSHELL_USER_FIXTURE + value: "$(params.openshell-user-fixture)" - name: AGENT_EVAL_FORGE_AI_GATEWAY_CA_FILE value: "/tmp/forge-ai-gateway-ca/ca.crt" - name: FORGE_SAW_PROFILE @@ -1799,24 +1806,17 @@ spec: grep -vE 'client.secret|secret|token' /tmp/openshell-gateway-add.log || true exit 1 } - OIDC_TOKEN_JSON=/tmp/openshell-oidc-token.json - curl -ksSf --max-time 20 \ - -u "${OPENSHELL_OIDC_CLIENT_ID}:${OPENSHELL_OIDC_CLIENT_SECRET}" \ - -d grant_type=client_credentials \ - "${OPENSHELL_OIDC_ISSUER%/}/protocol/openid-connect/token" \ - >"$OIDC_TOKEN_JSON" - python3 - "$OIDC_TOKEN_JSON" "$HOME/.config/openshell/gateways/$GW_NAME/oidc_token.json" \ - "$OPENSHELL_OIDC_ISSUER" "$OPENSHELL_OIDC_CLIENT_ID" <<'PY' - import json, sys, time - src, dst, issuer, client_id = sys.argv[1:] - payload = json.load(open(src, encoding="utf-8")) - token = payload["access_token"] - expires_at = int(time.time()) + int(payload.get("expires_in", 300)) - json.dump({"access_token": token, "expires_at": expires_at, - "issuer": issuer, "client_id": client_id}, open(dst, "w", encoding="utf-8")) - PY - chmod 600 "$HOME/.config/openshell/gateways/$GW_NAME/oidc_token.json" + OIDC_TOKEN_CACHE="$HOME/.config/openshell/gateways/$GW_NAME/oidc_token.json" + OIDC_REFRESH_HELPER="$(workspaces.source.path)/_pipeline/scripts/refresh_openshell_oidc.py" + python3 "$OIDC_REFRESH_HELPER" --cache "$OIDC_TOKEN_CACHE" echo "OpenShell OIDC token cache populated via client credentials" + # A case can run for 15 minutes, while the client-credentials token + # expires after roughly five. Refresh the CLI cache throughout this + # step so later artifact downloads and sandbox creation stay authorized. + python3 "$OIDC_REFRESH_HELPER" --cache "$OIDC_TOKEN_CACHE" --interval 120 & + OIDC_REFRESH_PID=$! + trap 'kill "$OIDC_REFRESH_PID" 2>/dev/null || true' EXIT + echo "OpenShell OIDC token refresher started" export OPENSHELL_GATEWAY="$GW_NAME" echo "OpenShell OIDC gateway configured: ${GW_NAME}" else @@ -2061,6 +2061,19 @@ spec: echo "Model: $MODEL" export AGENT_EVAL_HARNESS_ROOT="$HARNESS_DIR" + # Only submissions that explicitly configure a fixture stage one. + if [ -n "$OPENSHELL_USER_FIXTURE" ] && [ -z "${AGENT_EVAL_FORGE_USER_FILE:-}" ]; then + case "$OPENSHELL_USER_FIXTURE" in + /*|..|../*|*/../*|*/..) echo "ERROR: installation fixture must be submission-relative" >&2; exit 1 ;; + esac + FIXTURE_FILE="$SUBMISSION_DIR/$OPENSHELL_USER_FIXTURE" + if [ ! -f "$FIXTURE_FILE" ]; then + echo "ERROR: configured installation fixture is missing: $FIXTURE_FILE" >&2 + exit 1 + fi + export AGENT_EVAL_FORGE_USER_FILE="$FIXTURE_FILE" + echo "Using configured installation profile fixture" + fi if [ "$(params.enable-mlflow)" = "true" ] && [ -n "$(params.mlflow-tracking-uri)" ]; then export MLFLOW_TRACKING_URI="$(params.mlflow-tracking-uri)" echo "MLFLOW_TRACKING_URI set for AEH harness + CI logger" diff --git a/pipeline/tasks/phases/prepare.yaml b/pipeline/tasks/phases/prepare.yaml index 33f0f21..cdc7d79 100644 --- a/pipeline/tasks/phases/prepare.yaml +++ b/pipeline/tasks/phases/prepare.yaml @@ -122,16 +122,17 @@ spec: echo "=== PREPARE PHASE: Clone Pipeline Repo ===" PIPELINE_DIR="$(workspaces.source.path)/_pipeline" - if [ -d "$PIPELINE_DIR/.git" ]; then - echo "Pipeline repo already cloned, updating..." - cd "$PIPELINE_DIR" - git fetch origin "$(params.pipeline-repo-revision)" - git checkout "$(params.pipeline-repo-revision)" 2>/dev/null || git checkout FETCH_HEAD + PIPELINE_REV="$(params.pipeline-repo-revision)" + if [ ! -d "$PIPELINE_DIR/.git" ]; then + mkdir -p "$PIPELINE_DIR" + git -C "$PIPELINE_DIR" init -q + git -C "$PIPELINE_DIR" remote add origin "$(params.pipeline-repo-url)" else - git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ - "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + git -C "$PIPELINE_DIR" remote set-url origin "$(params.pipeline-repo-url)" fi - echo "Pipeline repo ready at $(params.pipeline-repo-revision)" + git -C "$PIPELINE_DIR" fetch --depth 1 origin "$PIPELINE_REV" + git -C "$PIPELINE_DIR" checkout --detach -f FETCH_HEAD + echo "Pipeline repo ready at $(git -C "$PIPELINE_DIR" rev-parse --short HEAD)" # Step 3: Generate tests (conditional) - name: generate-tests diff --git a/scripts/refresh_openshell_oidc.py b/scripts/refresh_openshell_oidc.py new file mode 100644 index 0000000..6003dda --- /dev/null +++ b/scripts/refresh_openshell_oidc.py @@ -0,0 +1,81 @@ +"""Populate and periodically refresh the OpenShell CLI OIDC token cache.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import tempfile +import time +from pathlib import Path + + +def fetch_token(issuer: str, client_id: str, client_secret: str) -> dict: + # Match the existing Task's curl transport, including its CA handling. + result = subprocess.run( + [ + "curl", + "-ksSf", + "--max-time", + "20", + "-u", + f"{client_id}:{client_secret}", + "-d", + "grant_type=client_credentials", + f"{issuer.rstrip('/')}/protocol/openid-connect/token", + ], + capture_output=True, + check=True, + ) + return json.loads(result.stdout) + + +def write_cache(path: Path, payload: dict, issuer: str, client_id: str) -> None: + cache = { + "access_token": payload["access_token"], + "expires_at": int(time.time()) + int(payload.get("expires_in", 300)), + "issuer": issuer, + "client_id": client_id, + } + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp(prefix=".oidc-token-", dir=path.parent) + try: + os.fchmod(fd, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump(cache, stream) + os.replace(tmp_name, path) + finally: + if os.path.exists(tmp_name): + os.unlink(tmp_name) + + +def refresh(path: Path, issuer: str, client_id: str, client_secret: str) -> None: + write_cache(path, fetch_token(issuer, client_id, client_secret), issuer, client_id) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cache", type=Path, required=True) + parser.add_argument("--interval", type=int, default=0, help="Seconds between refreshes; zero writes once") + args = parser.parse_args() + if args.interval < 0: + parser.error("--interval must be nonnegative") + + issuer = os.environ["OPENSHELL_OIDC_ISSUER"] + client_id = os.environ["OPENSHELL_OIDC_CLIENT_ID"] + client_secret = os.environ["OPENSHELL_OIDC_CLIENT_SECRET"] + if not args.interval: + refresh(args.cache, issuer, client_id, client_secret) + return + + while True: + time.sleep(args.interval) + try: + refresh(args.cache, issuer, client_id, client_secret) + except (OSError, ValueError, KeyError, subprocess.CalledProcessError) as exc: + print(f"WARN: OpenShell OIDC token refresh failed ({type(exc).__name__}); retrying", flush=True) + + +if __name__ == "__main__": + main() diff --git a/submissions/openclaw-forge/fixtures/USER.md b/submissions/openclaw-forge/fixtures/USER.md new file mode 100644 index 0000000..e88fe80 --- /dev/null +++ b/submissions/openclaw-forge/fixtures/USER.md @@ -0,0 +1,7 @@ +# Synthetic evaluation identity + +- Display name: Alex Rivera +- Role: Engineering executive +- Initials: AR +- Primary email: tbx-demo2@dev.mscloud.ibm.com +- Time zone: America/New_York diff --git a/tests/test_openshell_networkpolicy.py b/tests/test_openshell_networkpolicy.py new file mode 100644 index 0000000..0fa9ec2 --- /dev/null +++ b/tests/test_openshell_networkpolicy.py @@ -0,0 +1,41 @@ +"""Check the OpenShell NetworkPolicy's intended Tekton pod identity.""" + +from __future__ import annotations + +from pathlib import Path + +import yaml + +REPO = Path(__file__).resolve().parents[1] +EVALUATE_SELECTOR = {"tekton.dev/pipeline": "abevalflow-pipeline-openshell", "tekton.dev/task": "evaluate"} + + +def _matches(selector: dict[str, str], labels: dict[str, str]) -> bool: + return all(labels.get(key) == value for key, value in selector.items()) + + +def test_gateway_and_egress_select_only_evaluate_task_pods(): + pipeline = yaml.safe_load((REPO / "pipeline/pipelines/ci-pipeline-openshell.yaml").read_text()) + evaluate = next(task for task in pipeline["spec"]["tasks"] if task["name"] == "evaluate") + assert pipeline["metadata"]["name"] == EVALUATE_SELECTOR["tekton.dev/pipeline"] + assert evaluate["taskRef"]["name"] == EVALUATE_SELECTOR["tekton.dev/task"] + + policies = list(yaml.safe_load_all((REPO / "config/forge-saw/networkpolicy-ci-openshell.yaml").read_text())) + assert len(policies) == 3 + ingress_selectors = [ + rule["podSelector"]["matchLabels"] for policy in policies[:2] for rule in policy["spec"]["ingress"][0]["from"] + ] + egress_selector = policies[2]["spec"]["podSelector"]["matchLabels"] + remote_gateway = yaml.safe_load((REPO / "config/forge-saw/networkpolicy-gateway-from-abeval.yaml").read_text()) + remote_selector = remote_gateway["spec"]["ingress"][0]["from"][0]["podSelector"]["matchLabels"] + assert ingress_selectors + [egress_selector, remote_selector] == [EVALUATE_SELECTOR] * 4 + + evaluate_pod = {**EVALUATE_SELECTOR, "app.kubernetes.io/part-of": "abevalflow"} + prepare_pod = {**evaluate_pod, "tekton.dev/task": "prepare"} + unrelated_pod = {"app.kubernetes.io/part-of": "abevalflow"} + other_pipeline = {**evaluate_pod, "tekton.dev/pipeline": "other-pipeline"} + for selector in ingress_selectors + [egress_selector, remote_selector]: + assert _matches(selector, evaluate_pod) + assert not _matches(selector, prepare_pod) + assert not _matches(selector, unrelated_pod) + assert not _matches(selector, other_pipeline) diff --git a/tests/test_openshell_pipeline_profile.py b/tests/test_openshell_pipeline_profile.py index b44f222..1875801 100644 --- a/tests/test_openshell_pipeline_profile.py +++ b/tests/test_openshell_pipeline_profile.py @@ -8,6 +8,7 @@ REPO = Path(__file__).resolve().parents[1] PIPELINE = REPO / "pipeline" / "pipelines" / "ci-pipeline-openshell.yaml" +HARNESS_PIN = "8d58e500d0eb55a3106819ccacf26da6bf7ea166" CI = REPO / "pipeline" / "pipelines" / "ci-pipeline.yaml" CI_DEV = REPO / "pipeline" / "pipelines" / "ci-pipeline-dev.yaml" @@ -41,8 +42,10 @@ def test_openshell_defaults(self): assert defaults["enable-ai-generation"] == "false" assert defaults["aeh-runner"] == "openshell" assert defaults["aeh-openshell-image"] == "registry.access.redhat.com/ubi9/python-311:9.6" + assert defaults["agent-eval-harness-repo-revision"] == HARNESS_PIN + assert defaults["openshell-user-fixture"] == "fixtures/USER.md" assert defaults["openshell-sandbox-image"] == ( - "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a" + "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" ) assert defaults["enable-mlflow"] == "true" assert defaults["mlflow-tracking-uri"] == ("http://abevalflow-mlflow.gz-forge-eval.svc.cluster.local:5000") @@ -63,6 +66,8 @@ def test_evaluate_openshell_step_logs_mlflow(self): def test_openshell_eval_uses_llm_param_instead_of_inference_secret(self): task = _load(REPO / "pipeline" / "tasks" / "phases" / "evaluate.yaml") + assert _param_defaults(task["spec"])["openshell-user-fixture"] == "" + assert _param_defaults(task["spec"])["agent-eval-harness-repo-revision"] == HARNESS_PIN openshell = next(step for step in task["spec"]["steps"] if step["name"] == "aeh-openshell-eval") env_from_secret_names = { @@ -80,6 +85,36 @@ def test_openshell_eval_uses_llm_param_instead_of_inference_secret(self): assert env["value"] == "$(params.llm-api-key)" assert "valueFrom" not in env + def test_forge_briefing_has_installation_user_fixture(self): + fixture = REPO / "submissions" / "openclaw-forge" / "fixtures" / "USER.md" + fields = {} + for line in fixture.read_text().splitlines(): + if line.startswith("- ") and ": " in line: + key, value = line[2:].split(": ", 1) + fields[key.lower()] = value.strip() + assert all(fields.get(key) and not fields[key].startswith("<") for key in ("display name", "role", "initials")) + scene = _load(REPO / "submissions" / "openclaw-forge" / "scenes" / "monday-acquisition.yaml") + assert fields["primary email"] == scene["m365"]["user"] + + pipeline = _load(PIPELINE) + evaluate = next(item for item in pipeline["spec"]["tasks"] if item["name"] == "evaluate") + forwarded = {item["name"]: item["value"] for item in evaluate["params"]} + assert forwarded["openshell-user-fixture"] == "$(params.openshell-user-fixture)" + example = _load(REPO / "pipeline" / "runs" / "openshell-openclaw-pipelinerun.yaml") + example_params = {item["name"]: item["value"] for item in example["spec"]["params"]} + assert example_params["openshell-user-fixture"] == "fixtures/USER.md" + + task = _load(REPO / "pipeline" / "tasks" / "phases" / "evaluate.yaml") + step = next(s for s in task["spec"]["steps"] if s["name"] == "aeh-openshell-eval") + assert next(env for env in step["env"] if env["name"] == "OPENSHELL_USER_FIXTURE")["value"] == ( + "$(params.openshell-user-fixture)" + ) + script = step["script"] + assert 'FIXTURE_FILE="$SUBMISSION_DIR/$OPENSHELL_USER_FIXTURE"' in script + assert "Using configured installation profile fixture" in script + assert "fixtures/USER.md" not in script + assert script.index('export AGENT_EVAL_FORGE_USER_FILE="$FIXTURE_FILE"') < script.index("scripts/run_aeh.py") + def test_harbor_profiles_still_include_test(self): for path in (CI, CI_DEV): names = _task_names(_load(path)["spec"]) diff --git a/tests/test_prepare_pipeline_checkout.py b/tests/test_prepare_pipeline_checkout.py new file mode 100644 index 0000000..0c2e7bf --- /dev/null +++ b/tests/test_prepare_pipeline_checkout.py @@ -0,0 +1,51 @@ +"""Exercise the Prepare Task pipeline checkout with a local Git remote.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import yaml + + +def _git(*args: str, cwd: Path | None = None) -> str: + return subprocess.check_output(["git", *args], cwd=cwd, text=True).strip() + + +def test_pipeline_checkout_accepts_branch_tag_and_full_sha(tmp_path: Path): + source = tmp_path / "source" + remote = tmp_path / "remote.git" + _git("init", "-q", "-b", "main", str(source)) + (source / "example.txt").write_text("main") + _git("add", "example.txt", cwd=source) + _git("-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-qm", "initial", cwd=source) + main_sha = _git("rev-parse", "HEAD", cwd=source) + _git("tag", "v1", cwd=source) + _git("checkout", "-qb", "feature", cwd=source) + (source / "example.txt").write_text("feature") + _git("add", "example.txt", cwd=source) + _git("-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-qm", "feature", cwd=source) + feature_sha = _git("rev-parse", "HEAD", cwd=source) + _git("init", "--bare", "-q", str(remote)) + _git("remote", "add", "origin", str(remote), cwd=source) + _git("push", "-q", "origin", "main", "feature", "v1", cwd=source) + + repo = Path(__file__).resolve().parents[1] + task = yaml.safe_load((repo / "pipeline/tasks/phases/prepare.yaml").read_text()) + script = next(step["script"] for step in task["spec"]["steps"] if step["name"] == "clone-pipeline-repo") + + def checkout(revision: str, workspace: Path) -> str: + workspace.mkdir(exist_ok=True) + rendered = ( + script.replace("$(workspaces.source.path)", str(workspace)) + .replace("$(params.pipeline-repo-revision)", revision) + .replace("$(params.pipeline-repo-url)", str(remote)) + ) + subprocess.run(["bash", "-c", rendered], check=True, capture_output=True, text=True) + return _git("rev-parse", "HEAD", cwd=workspace / "_pipeline") + + workspace = tmp_path / "workspace" + assert checkout("main", workspace) == main_sha + assert checkout("v1", workspace) == main_sha + assert checkout("feature", workspace) == feature_sha + assert checkout(main_sha, tmp_path / "fresh-workspace") == main_sha diff --git a/tests/test_refresh_openshell_oidc.py b/tests/test_refresh_openshell_oidc.py new file mode 100644 index 0000000..3c81737 --- /dev/null +++ b/tests/test_refresh_openshell_oidc.py @@ -0,0 +1,67 @@ +"""Behavior tests for the OpenShell OIDC cache helper.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from unittest.mock import patch + +import yaml + +from scripts import refresh_openshell_oidc as oidc + + +def test_initial_and_refresh_writes_are_atomic_and_private(tmp_path: Path): + cache = tmp_path / "gateways" / "ci" / "oidc_token.json" + issuer = "https://issuer.example" + with patch.object(oidc.time, "time", return_value=1000): + oidc.write_cache(cache, {"access_token": "first", "expires_in": 300}, issuer, "client") + assert cache.stat().st_mode & 0o777 == 0o600 + assert json.loads(cache.read_text()) == { + "access_token": "first", + "expires_at": 1300, + "issuer": issuer, + "client_id": "client", + } + + real_replace = os.replace + + def observe_replace(source: str, destination: Path) -> None: + # A reader still sees the complete old cache until replacement. + assert json.loads(cache.read_text())["access_token"] == "first" + assert Path(source).stat().st_mode & 0o777 == 0o600 + real_replace(source, destination) + + with patch.object(oidc.os, "replace", side_effect=observe_replace): + with patch.object(oidc.time, "time", return_value=1100): + oidc.write_cache(cache, {"access_token": "second", "expires_in": 240}, issuer, "client") + assert json.loads(cache.read_text())["access_token"] == "second" + assert json.loads(cache.read_text())["expires_at"] == 1340 + assert cache.stat().st_mode & 0o777 == 0o600 + assert list(cache.parent.glob(".oidc-token-*")) == [] + + +def test_failed_fetch_preserves_existing_cache(tmp_path: Path): + cache = tmp_path / "oidc_token.json" + oidc.write_cache(cache, {"access_token": "valid"}, "issuer", "client") + original = cache.read_bytes() + with patch.object(oidc, "fetch_token", side_effect=ValueError("bad response")): + try: + oidc.refresh(cache, "issuer", "client", "secret") + except ValueError: + pass + else: + raise AssertionError("refresh should fail") + assert cache.read_bytes() == original + + +def test_evaluate_uses_one_helper_for_initial_and_periodic_refresh(): + task = yaml.safe_load((Path(__file__).parents[1] / "pipeline/tasks/phases/evaluate.yaml").read_text()) + step = next(item for item in task["spec"]["steps"] if item["name"] == "aeh-openshell-eval") + script = step["script"] + assert script.count('python3 "$OIDC_REFRESH_HELPER" --cache "$OIDC_TOKEN_CACHE"') == 2 + assert "--interval 120 &" in script + assert script.index('python3 "$OIDC_REFRESH_HELPER" --cache "$OIDC_TOKEN_CACHE"') < script.index( + 'python "$PIPELINE_DIR/scripts/run_aeh.py"' + )