From c35031bf35574ec353871f3da4c11649f93db0ce Mon Sep 17 00:00:00 2001 From: Tarun Kumar Reddy Etikala Date: Fri, 2 Oct 2026 10:13:55 -0400 Subject: [PATCH 1/6] Make Forge OpenShell evals publish full briefs reliably Port the verified OIDC refresh, USER.md fixture, pinned SAW image, and published_brief gate onto main. Add same-namespace NetworkPolicy templates that match the canonical Pipeline name or app.kubernetes.io/part-of=abevalflow so ad-hoc Pipeline copies no longer time out on gateway preflight. Co-authored-by: Cursor --- config/forge-saw/README.md | 18 ++ .../forge-saw/networkpolicy-ci-openshell.yaml | 213 ++++++++++++++++++ .../networkpolicy-gateway-from-abeval.yaml | 15 +- pipeline/pipelines/ci-pipeline-openshell.yaml | 10 +- .../runs/openshell-openclaw-pipelinerun.yaml | 7 +- pipeline/tasks/phases/evaluate.yaml | 51 ++++- submissions/openclaw-forge/fixtures/USER.md | 7 + tests/test_openshell_pipeline_profile.py | 20 +- 8 files changed, 333 insertions(+), 8 deletions(-) create mode 100644 config/forge-saw/networkpolicy-ci-openshell.yaml create mode 100644 submissions/openclaw-forge/fixtures/USER.md diff --git a/config/forge-saw/README.md b/config/forge-saw/README.md index 6f1ea23..b2b566c 100644 --- a/config/forge-saw/README.md +++ b/config/forge-saw/README.md @@ -148,6 +148,24 @@ oc get secret openshell-credentials -n guy-ziv-evalflow -o json \ `aeh-openshell-eval` TCP-checks `:17670` and optionally runs `openshell sandbox list`. If SAW is down the **run fails**. That step does not install SAW. +## NetworkPolicy (Forge shared namespace) + +When SAW VMs and Tekton share a namespace with default-deny policies, evaluate +pods must be allowed to reach the agent gateway on `:17670`. Use the canonical +Pipeline name `abevalflow-pipeline-openshell` (so `tekton.dev/pipeline` matches) +and label PipelineRuns with `app.kubernetes.io/part-of: abevalflow`. + +Same-namespace template: + +```bash +sed "s/NAMESPACE/${EVAL_NS}/g" config/forge-saw/networkpolicy-ci-openshell.yaml \ + | oc apply -f - +``` + +Do not create ad-hoc copies of the Pipeline under a different name unless those +PipelineRuns also carry the `part-of=abevalflow` label and the NetworkPolicies +above are applied — otherwise gateway preflight times out. + ## Image Stock `agent-eval-harness:v1.0.x` cannot import `agent_eval.openshell`. Point diff --git a/config/forge-saw/networkpolicy-ci-openshell.yaml b/config/forge-saw/networkpolicy-ci-openshell.yaml new file mode 100644 index 0000000..5bd6f18 --- /dev/null +++ b/config/forge-saw/networkpolicy-ci-openshell.yaml @@ -0,0 +1,213 @@ +# NetworkPolicies for OpenShell OpenClaw evals when SAW VMs and the Tekton +# pipeline share one namespace (Forge workspace style). +# +# Select evaluate pods by either: +# - tekton.dev/pipeline=abevalflow-pipeline-openshell (canonical Pipeline name) +# - app.kubernetes.io/part-of=abevalflow (set on PipelineRun labels) +# +# Do NOT rename/copy the Pipeline to an ad-hoc name without also labeling the +# PipelineRun with app.kubernetes.io/part-of=abevalflow — otherwise default-deny +# blocks TCP to the gateway on :17670 and evaluate fails preflight. +# +# Apply after substituting NAMESPACE (and AGENT_VM / INTEG_VM if different): +# sed "s/NAMESPACE/forge-nommen/g" networkpolicy-ci-openshell.yaml | oc apply -f - +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: abevalflow-allow-ci-agent-gateway + namespace: NAMESPACE + labels: + app.kubernetes.io/part-of: abevalflow + app.kubernetes.io/component: forge-saw +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: openshell-saw-agent + vm.kubevirt.io/name: openshell-saw-agent + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + tekton.dev/pipeline: abevalflow-pipeline-openshell + - podSelector: + matchLabels: + app.kubernetes.io/part-of: abevalflow + ports: + - protocol: TCP + port: 17670 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: abevalflow-allow-ci-integ-gateway + namespace: NAMESPACE + labels: + app.kubernetes.io/part-of: abevalflow + app.kubernetes.io/component: forge-saw +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: openshell-saw-integ + vm.kubevirt.io/name: openshell-saw-integ + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + tekton.dev/pipeline: abevalflow-pipeline-openshell + - podSelector: + matchLabels: + app.kubernetes.io/part-of: abevalflow + ports: + - protocol: TCP + port: 17670 + - protocol: TCP + port: 18082 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: abevalflow-allow-ci-egress-by-pipeline + namespace: NAMESPACE + labels: + app.kubernetes.io/part-of: abevalflow + app.kubernetes.io/component: forge-saw +spec: + podSelector: + matchLabels: + tekton.dev/pipeline: abevalflow-pipeline-openshell + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: openshell-saw-agent + vm.kubevirt.io/name: openshell-saw-agent + ports: + - protocol: TCP + port: 17670 + - protocol: TCP + port: 8443 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: openshell-saw-integ + vm.kubevirt.io/name: openshell-saw-integ + ports: + - protocol: TCP + port: 17670 + - protocol: TCP + port: 18082 + - protocol: TCP + port: 8443 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: litellm + ports: + - protocol: TCP + port: 4000 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: mlflow + app.kubernetes.io/part-of: abevalflow + ports: + - protocol: TCP + port: 5000 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: abevalflow + ports: + - protocol: TCP + port: 9000 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - protocol: UDP + port: 5353 + - protocol: TCP + port: 5353 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: abevalflow-allow-ci-egress-by-part-of + namespace: NAMESPACE + labels: + app.kubernetes.io/part-of: abevalflow + app.kubernetes.io/component: forge-saw +spec: + podSelector: + matchLabels: + app.kubernetes.io/part-of: abevalflow + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: openshell-saw-agent + vm.kubevirt.io/name: openshell-saw-agent + ports: + - protocol: TCP + port: 17670 + - protocol: TCP + port: 8443 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: openshell-saw-integ + vm.kubevirt.io/name: openshell-saw-integ + ports: + - protocol: TCP + port: 17670 + - protocol: TCP + port: 18082 + - protocol: TCP + port: 8443 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: litellm + ports: + - protocol: TCP + port: 4000 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: mlflow + app.kubernetes.io/part-of: abevalflow + ports: + - protocol: TCP + port: 5000 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: abevalflow + ports: + - protocol: TCP + port: 9000 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - protocol: UDP + port: 5353 + - protocol: TCP + port: 5353 diff --git a/config/forge-saw/networkpolicy-gateway-from-abeval.yaml b/config/forge-saw/networkpolicy-gateway-from-abeval.yaml index af1aaef..9071391 100644 --- a/config/forge-saw/networkpolicy-gateway-from-abeval.yaml +++ b/config/forge-saw/networkpolicy-gateway-from-abeval.yaml @@ -1,6 +1,11 @@ # Allow evaluate Task pods to reach the OpenShell gateway on :17670. # bootstrap.sh rewrites namespace / from-namespace (SAW_NS, EVAL_NS). # Apply in the SAW namespace, not from the evaluate PipelineRun. +# +# Prefer matching Tekton pods by the canonical Pipeline name or the +# app.kubernetes.io/part-of=abevalflow label on PipelineRuns. A bare +# podSelector: {} from the whole EVAL_NS is broader than needed when SAW +# and eval share a Forge workspace namespace — see networkpolicy-ci-openshell.yaml. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -20,7 +25,15 @@ spec: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: guy-ziv-evalflow - - podSelector: {} + podSelector: + matchLabels: + tekton.dev/pipeline: abevalflow-pipeline-openshell + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: guy-ziv-evalflow + podSelector: + matchLabels: + app.kubernetes.io/part-of: abevalflow ports: - protocol: TCP port: 17670 diff --git a/pipeline/pipelines/ci-pipeline-openshell.yaml b/pipeline/pipelines/ci-pipeline-openshell.yaml index 07a41c5..63032d7 100644 --- a/pipeline/pipelines/ci-pipeline-openshell.yaml +++ b/pipeline/pipelines/ci-pipeline-openshell.yaml @@ -5,6 +5,7 @@ metadata: namespace: ab-eval-flow labels: app.kubernetes.io/name: abevalflow + app.kubernetes.io/part-of: abevalflow app.kubernetes.io/version: "2.0" app.kubernetes.io/profile: aeh-openshell-openclaw spec: @@ -102,8 +103,11 @@ spec: default: "https://github.com/GuyZivRH/agent-eval-harness.git" - name: agent-eval-harness-repo-revision type: string - default: "feat/aeh-openshell-openclaw" - description: Harness revision with agent_eval.openshell (feature-branch pin) + default: "main" + description: >- + Harness revision with agent_eval.openshell. Use a tip that includes the + OpenClaw brief-reader/gateway fixes (GuyZivRH/agent-eval-harness#9) until + that lands on upstream main. - name: openshell-cli-version type: string default: "0.0.116" @@ -112,7 +116,7 @@ spec: default: "https://openshell.openshell.svc.cluster.local:17670" - name: openshell-sandbox-image type: string - default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a" + default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" - name: openshell-provider type: string default: "forge-ai-gateway,m365-read-intervm,slack-read-proxy,drafts-service-agent" diff --git a/pipeline/runs/openshell-openclaw-pipelinerun.yaml b/pipeline/runs/openshell-openclaw-pipelinerun.yaml index f46241e..9b9b1de 100644 --- a/pipeline/runs/openshell-openclaw-pipelinerun.yaml +++ b/pipeline/runs/openshell-openclaw-pipelinerun.yaml @@ -5,6 +5,11 @@ apiVersion: tekton.dev/v1 kind: PipelineRun metadata: generateName: aeh-openshell-openclaw- + labels: + # Required when NetworkPolicies select on part-of (and recommended always). + # Keep pipelineRef.name=abevalflow-pipeline-openshell so tekton.dev/pipeline + # also matches the canonical CI NetworkPolicies. + app.kubernetes.io/part-of: abevalflow spec: pipelineRef: name: abevalflow-pipeline-openshell @@ -36,7 +41,7 @@ spec: value: "forge-agent-upstream-tls" # To use another sandbox image, replace the value below with . - name: openshell-sandbox-image - value: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a" + value: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" - name: openshell-provider value: "forge-ai-gateway,m365-read-intervm,slack-read-proxy,drafts-service-agent" - name: aeh-openshell-image diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index 827346c..b67c8cb 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -184,7 +184,7 @@ spec: default: "https://github.com/GuyZivRH/agent-eval-harness.git" - name: agent-eval-harness-repo-revision type: string - default: "feat/aeh-openshell-openclaw" + default: "main" description: >- Branch or tag of GuyZivRH/agent-eval-harness to clone. Must include the OpenShell create keep-alive fix (`--detach` + `sleep infinity`). @@ -196,7 +196,7 @@ spec: Override only if using a different gateway (e.g. forge-saw :17670). - name: openshell-sandbox-image type: string - default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a" + default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" description: >- Immutable SAW Chief-of-Staff image with the daily-briefing skill and governed M365/Slack tools. @@ -1817,6 +1817,46 @@ spec: PY chmod 600 "$HOME/.config/openshell/gateways/$GW_NAME/oidc_token.json" echo "OpenShell OIDC token cache populated via client credentials" + # A case can run for 15 minutes, while the client-credentials token + # expires after roughly five. Refresh the CLI cache throughout this + # step so later artifact downloads and sandbox creation stay authorized. + OIDC_TOKEN_CACHE="$HOME/.config/openshell/gateways/$GW_NAME/oidc_token.json" + ( + umask 077 + trap 'rm -f "${OIDC_TOKEN_JSON}.refresh"' EXIT + while sleep 120; do + if curl -ksSf --max-time 20 \ + -u "${OPENSHELL_OIDC_CLIENT_ID}:${OPENSHELL_OIDC_CLIENT_SECRET}" \ + -d grant_type=client_credentials \ + "${OPENSHELL_OIDC_ISSUER%/}/protocol/openid-connect/token" \ + >"${OIDC_TOKEN_JSON}.refresh" 2>/dev/null; then + python3 - "${OIDC_TOKEN_JSON}.refresh" "$OIDC_TOKEN_CACHE" \ + "$OPENSHELL_OIDC_ISSUER" "$OPENSHELL_OIDC_CLIENT_ID" <<'REFRESH' || true + import json, os, sys, tempfile, time + src, dst, issuer, client_id = sys.argv[1:] + with open(src, encoding="utf-8") as stream: + payload = json.load(stream) + cache = {"access_token": payload["access_token"], + "expires_at": int(time.time()) + int(payload.get("expires_in", 300)), + "issuer": issuer, "client_id": client_id} + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(dst), prefix=".oidc-token-") + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump(cache, stream) + os.chmod(tmp, 0o600) + os.replace(tmp, dst) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + REFRESH + else + echo "WARN: OpenShell OIDC token refresh failed; retrying" >&2 + fi + done + ) & + OIDC_REFRESH_PID=$! + trap 'kill "$OIDC_REFRESH_PID" 2>/dev/null || true' EXIT + echo "OpenShell OIDC token refresher started" export OPENSHELL_GATEWAY="$GW_NAME" echo "OpenShell OIDC gateway configured: ${GW_NAME}" else @@ -2061,6 +2101,13 @@ spec: echo "Model: $MODEL" export AGENT_EVAL_HARNESS_ROOT="$HARNESS_DIR" + # A published Forge briefing needs installation identity. Keep this + # synthetic fixture scoped to submissions that supply one; an explicit + # harness setting takes precedence for other deployments. + if [ -z "${AGENT_EVAL_FORGE_USER_FILE:-}" ] && [ -f "$SUBMISSION_DIR/fixtures/USER.md" ]; then + export AGENT_EVAL_FORGE_USER_FILE="$SUBMISSION_DIR/fixtures/USER.md" + echo "Using submission installation profile fixture" + fi if [ "$(params.enable-mlflow)" = "true" ] && [ -n "$(params.mlflow-tracking-uri)" ]; then export MLFLOW_TRACKING_URI="$(params.mlflow-tracking-uri)" echo "MLFLOW_TRACKING_URI set for AEH harness + CI logger" diff --git a/submissions/openclaw-forge/fixtures/USER.md b/submissions/openclaw-forge/fixtures/USER.md new file mode 100644 index 0000000..e88fe80 --- /dev/null +++ b/submissions/openclaw-forge/fixtures/USER.md @@ -0,0 +1,7 @@ +# Synthetic evaluation identity + +- Display name: Alex Rivera +- Role: Engineering executive +- Initials: AR +- Primary email: tbx-demo2@dev.mscloud.ibm.com +- Time zone: America/New_York diff --git a/tests/test_openshell_pipeline_profile.py b/tests/test_openshell_pipeline_profile.py index b44f222..27ae5f0 100644 --- a/tests/test_openshell_pipeline_profile.py +++ b/tests/test_openshell_pipeline_profile.py @@ -42,7 +42,7 @@ def test_openshell_defaults(self): assert defaults["aeh-runner"] == "openshell" assert defaults["aeh-openshell-image"] == "registry.access.redhat.com/ubi9/python-311:9.6" assert defaults["openshell-sandbox-image"] == ( - "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a" + "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" ) assert defaults["enable-mlflow"] == "true" assert defaults["mlflow-tracking-uri"] == ("http://abevalflow-mlflow.gz-forge-eval.svc.cluster.local:5000") @@ -80,6 +80,24 @@ def test_openshell_eval_uses_llm_param_instead_of_inference_secret(self): assert env["value"] == "$(params.llm-api-key)" assert "valueFrom" not in env + def test_forge_briefing_has_installation_user_fixture(self): + fixture = REPO / "submissions" / "openclaw-forge" / "fixtures" / "USER.md" + fields = {} + for line in fixture.read_text().splitlines(): + if line.startswith("- ") and ": " in line: + key, value = line[2:].split(": ", 1) + fields[key.lower()] = value.strip() + assert all(fields.get(key) and not fields[key].startswith("<") + for key in ("display name", "role", "initials")) + scene = _load(REPO / "submissions" / "openclaw-forge" / "scenes" / "monday-acquisition.yaml") + assert fields["primary email"] == scene["m365"]["user"] + + task = _load(REPO / "pipeline" / "tasks" / "phases" / "evaluate.yaml") + step = next(s for s in task["spec"]["steps"] if s["name"] == "aeh-openshell-eval") + script = step["script"] + assert 'AGENT_EVAL_FORGE_USER_FILE="$SUBMISSION_DIR/fixtures/USER.md"' in script + assert script.index('[ -f "$SUBMISSION_DIR/fixtures/USER.md" ]') < script.index("scripts/run_aeh.py") + def test_harbor_profiles_still_include_test(self): for path in (CI, CI_DEV): names = _task_names(_load(path)["spec"]) From eea6414c832a5502778c77793cea9b8d6b309a33 Mon Sep 17 00:00:00 2001 From: Tarun Kumar Reddy Etikala Date: Fri, 2 Oct 2026 10:27:36 -0400 Subject: [PATCH 2/6] Allow prepare to clone pipeline repo by commit SHA Depth-1 --branch fails for bare SHAs; fall back to full clone + checkout so clean-source pins work the same way as the submission revision. Co-authored-by: Cursor --- pipeline/tasks/phases/prepare.yaml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pipeline/tasks/phases/prepare.yaml b/pipeline/tasks/phases/prepare.yaml index 33f0f21..9da41f6 100644 --- a/pipeline/tasks/phases/prepare.yaml +++ b/pipeline/tasks/phases/prepare.yaml @@ -122,16 +122,21 @@ spec: echo "=== PREPARE PHASE: Clone Pipeline Repo ===" PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + PIPELINE_REV="$(params.pipeline-repo-revision)" if [ -d "$PIPELINE_DIR/.git" ]; then echo "Pipeline repo already cloned, updating..." cd "$PIPELINE_DIR" - git fetch origin "$(params.pipeline-repo-revision)" - git checkout "$(params.pipeline-repo-revision)" 2>/dev/null || git checkout FETCH_HEAD + git fetch origin "$PIPELINE_REV" --depth 1 2>/dev/null \ + || git fetch origin "$PIPELINE_REV" + git checkout "$PIPELINE_REV" 2>/dev/null || git checkout FETCH_HEAD else - git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ - "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + if ! git clone --depth 1 --branch "$PIPELINE_REV" \ + "$(params.pipeline-repo-url)" "$PIPELINE_DIR" 2>/dev/null; then + git clone "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + git -C "$PIPELINE_DIR" checkout "$PIPELINE_REV" + fi fi - echo "Pipeline repo ready at $(params.pipeline-repo-revision)" + echo "Pipeline repo ready at $(git -C "$PIPELINE_DIR" rev-parse --short HEAD)" # Step 3: Generate tests (conditional) - name: generate-tests From c5b4da24f654f60a8c664f36763c27ee616606fd Mon Sep 17 00:00:00 2001 From: "Sana Fayyazgit config --global user.email sanafayyaz315@gmail.comgit config --global user.name Sana" Date: Thu, 8 Oct 2026 13:29:37 +0100 Subject: [PATCH 3/6] Format Forge fixture test for CI --- tests/test_openshell_pipeline_profile.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_openshell_pipeline_profile.py b/tests/test_openshell_pipeline_profile.py index 27ae5f0..e3207ab 100644 --- a/tests/test_openshell_pipeline_profile.py +++ b/tests/test_openshell_pipeline_profile.py @@ -87,8 +87,7 @@ def test_forge_briefing_has_installation_user_fixture(self): if line.startswith("- ") and ": " in line: key, value = line[2:].split(": ", 1) fields[key.lower()] = value.strip() - assert all(fields.get(key) and not fields[key].startswith("<") - for key in ("display name", "role", "initials")) + assert all(fields.get(key) and not fields[key].startswith("<") for key in ("display name", "role", "initials")) scene = _load(REPO / "submissions" / "openclaw-forge" / "scenes" / "monday-acquisition.yaml") assert fields["primary email"] == scene["m365"]["user"] From 3232787d4b868f7c4d15d3d4f33edf96852f1e3f Mon Sep 17 00:00:00 2001 From: "Sana Fayyazgit config --global user.email sanafayyaz315@gmail.comgit config --global user.name Sana" Date: Thu, 8 Oct 2026 15:13:58 +0100 Subject: [PATCH 4/6] Refresh Forge OpenShell example and harness main guidance --- pipeline/pipelines/ci-pipeline-openshell.yaml | 5 ++--- pipeline/runs/openshell-openclaw-pipelinerun.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pipeline/pipelines/ci-pipeline-openshell.yaml b/pipeline/pipelines/ci-pipeline-openshell.yaml index 63032d7..b4ff9bf 100644 --- a/pipeline/pipelines/ci-pipeline-openshell.yaml +++ b/pipeline/pipelines/ci-pipeline-openshell.yaml @@ -105,9 +105,8 @@ spec: type: string default: "main" description: >- - Harness revision with agent_eval.openshell. Use a tip that includes the - OpenClaw brief-reader/gateway fixes (GuyZivRH/agent-eval-harness#9) until - that lands on upstream main. + GuyZivRH/agent-eval-harness main includes the OpenClaw brief-reader and + gateway fixes merged in #9. Pin a commit SHA for reproducible runs. - name: openshell-cli-version type: string default: "0.0.116" diff --git a/pipeline/runs/openshell-openclaw-pipelinerun.yaml b/pipeline/runs/openshell-openclaw-pipelinerun.yaml index 9b9b1de..1254a2d 100644 --- a/pipeline/runs/openshell-openclaw-pipelinerun.yaml +++ b/pipeline/runs/openshell-openclaw-pipelinerun.yaml @@ -21,12 +21,12 @@ spec: value: "openclaw-forge" - name: eval-engine value: "aeh_openshell_openclaw" - # Until this branch is merged, pass the feature revision so store/parse scripts match. - # After merge to main, omit these (Pipeline defaults are main). + # This example targets the merged upstream main for both submission and + # Pipeline helper source. Replace either value with a commit SHA to pin a run. - name: revision - value: "feat/aeh-openshell-openclaw" + value: "main" - name: pipeline-repo-revision - value: "feat/aeh-openshell-openclaw" + value: "main" # [Optional] Add a harness repository or revision for this run; these replace Pipeline defaults. # - name: agent-eval-harness-repo-url # value: "" From 4dc11e62c3a1a0b69317853c5f9d5642cba29113 Mon Sep 17 00:00:00 2001 From: "Sana Fayyazgit config --global user.email sanafayyaz315@gmail.comgit config --global user.name Sana" Date: Fri, 9 Oct 2026 11:47:12 +0100 Subject: [PATCH 5/6] Address PR 109 OpenShell reliability review --- config/forge-saw/README.md | 37 +++++- .../forge-saw/networkpolicy-ci-openshell.yaml | 121 ++++++------------ .../networkpolicy-gateway-from-abeval.yaml | 10 +- pipeline/pipelines/ci-pipeline-openshell.yaml | 14 +- .../runs/openshell-openclaw-pipelinerun.yaml | 7 +- pipeline/tasks/phases/evaluate.yaml | 98 +++++--------- pipeline/tasks/phases/prepare.yaml | 18 +-- scripts/refresh_openshell_oidc.py | 81 ++++++++++++ tests/test_openshell_networkpolicy.py | 43 +++++++ tests/test_openshell_pipeline_profile.py | 22 +++- tests/test_prepare_pipeline_checkout.py | 51 ++++++++ tests/test_refresh_openshell_oidc.py | 67 ++++++++++ 12 files changed, 386 insertions(+), 183 deletions(-) create mode 100644 scripts/refresh_openshell_oidc.py create mode 100644 tests/test_openshell_networkpolicy.py create mode 100644 tests/test_prepare_pipeline_checkout.py create mode 100644 tests/test_refresh_openshell_oidc.py diff --git a/config/forge-saw/README.md b/config/forge-saw/README.md index b2b566c..7133e2c 100644 --- a/config/forge-saw/README.md +++ b/config/forge-saw/README.md @@ -152,8 +152,18 @@ If SAW is down the **run fails**. That step does not install SAW. When SAW VMs and Tekton share a namespace with default-deny policies, evaluate pods must be allowed to reach the agent gateway on `:17670`. Use the canonical -Pipeline name `abevalflow-pipeline-openshell` (so `tekton.dev/pipeline` matches) -and label PipelineRuns with `app.kubernetes.io/part-of: abevalflow`. +Pipeline name `abevalflow-pipeline-openshell` and its referenced Task name +`evaluate` so the injected `tekton.dev/pipeline` and `tekton.dev/task` pod +labels match. + +The egress policy permits SAW gateways in the evaluate namespace, LiteLLM, +MLflow and MinIO either there or in `gz-forge-eval`, DNS, and public TCP/443 +for the Task's OpenShell download, Python package installs and public OIDC +issuer. Kubernetes NetworkPolicy cannot limit the HTTPS rule to GitHub or +PyPI hostnames. If the OIDC issuer is private, add an allow rule for its actual +namespace and pod labels or route before enabling that mode. The shared Task +also installs packages at runtime; an image with those dependencies baked in +would allow a narrower deployment policy. Same-namespace template: @@ -162,12 +172,27 @@ sed "s/NAMESPACE/${EVAL_NS}/g" config/forge-saw/networkpolicy-ci-openshell.yaml | oc apply -f - ``` -Do not create ad-hoc copies of the Pipeline under a different name unless those -PipelineRuns also carry the `part-of=abevalflow` label and the NetworkPolicies -above are applied — otherwise gateway preflight times out. +If the earlier two-selector template was applied, remove its old egress policy +after applying this one; `oc apply` does not delete objects omitted from a file: + +```bash +oc delete networkpolicy abevalflow-allow-ci-egress-by-part-of \ + -n "$EVAL_NS" --ignore-not-found +``` + +Copies of the Pipeline or Evaluate Task under another name need matching +NetworkPolicy selectors; otherwise gateway preflight times out. ## Image Stock `agent-eval-harness:v1.0.x` cannot import `agent_eval.openshell`. Point `aeh-openshell-image` at an orchestrator image built from GuyZivRH -`agent-eval-harness` **main** that also includes the `openshell` CLI. +`agent-eval-harness` at the pinned OpenShell Pipeline revision that also +includes the `openshell` CLI. Use `main` only as an explicit development +override. + +The OpenShell Pipeline profile explicitly sets `openshell-user-fixture` to +`fixtures/USER.md` for the Forge submission. The shared Evaluate Task defaults +this parameter to empty. Set it to an empty value for a submission without an +installation fixture, or provide another path relative to its submission +directory. A configured path must exist when Evaluate runs. diff --git a/config/forge-saw/networkpolicy-ci-openshell.yaml b/config/forge-saw/networkpolicy-ci-openshell.yaml index 5bd6f18..7e82997 100644 --- a/config/forge-saw/networkpolicy-ci-openshell.yaml +++ b/config/forge-saw/networkpolicy-ci-openshell.yaml @@ -1,16 +1,16 @@ # NetworkPolicies for OpenShell OpenClaw evals when SAW VMs and the Tekton # pipeline share one namespace (Forge workspace style). # -# Select evaluate pods by either: -# - tekton.dev/pipeline=abevalflow-pipeline-openshell (canonical Pipeline name) -# - app.kubernetes.io/part-of=abevalflow (set on PipelineRun labels) +# Select the evaluate Task pods in the canonical OpenShell Pipeline by the +# injected tekton.dev/pipeline and tekton.dev/task labels. # -# Do NOT rename/copy the Pipeline to an ad-hoc name without also labeling the -# PipelineRun with app.kubernetes.io/part-of=abevalflow — otherwise default-deny -# blocks TCP to the gateway on :17670 and evaluate fails preflight. +# Renamed copies of the Pipeline need a matching policy selector or gateway +# preflight will be blocked by default-deny. # -# Apply after substituting NAMESPACE (and AGENT_VM / INTEG_VM if different): -# sed "s/NAMESPACE/forge-nommen/g" networkpolicy-ci-openshell.yaml | oc apply -f - +# Apply after substituting NAMESPACE for the SAW/evaluate namespace. LiteLLM, +# MLflow and MinIO may also run in gz-forge-eval (the example PipelineRun). +# Public HTTPS is needed for CLI downloads, Python packages and public OIDC +# issuers. Private OIDC issuers need an additional deployment-specific rule. --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -31,9 +31,7 @@ spec: - podSelector: matchLabels: tekton.dev/pipeline: abevalflow-pipeline-openshell - - podSelector: - matchLabels: - app.kubernetes.io/part-of: abevalflow + tekton.dev/task: evaluate ports: - protocol: TCP port: 17670 @@ -57,9 +55,7 @@ spec: - podSelector: matchLabels: tekton.dev/pipeline: abevalflow-pipeline-openshell - - podSelector: - matchLabels: - app.kubernetes.io/part-of: abevalflow + tekton.dev/task: evaluate ports: - protocol: TCP port: 17670 @@ -78,6 +74,7 @@ spec: podSelector: matchLabels: tekton.dev/pipeline: abevalflow-pipeline-openshell + tekton.dev/task: evaluate policyTypes: [Egress] egress: - to: @@ -106,6 +103,12 @@ spec: - podSelector: matchLabels: app.kubernetes.io/name: litellm + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: gz-forge-eval + podSelector: + matchLabels: + app.kubernetes.io/name: litellm ports: - protocol: TCP port: 4000 @@ -114,76 +117,10 @@ spec: matchLabels: app.kubernetes.io/name: mlflow app.kubernetes.io/part-of: abevalflow - ports: - - protocol: TCP - port: 5000 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - app.kubernetes.io/part-of: abevalflow - ports: - - protocol: TCP - port: 9000 - - to: - namespaceSelector: matchLabels: - kubernetes.io/metadata.name: openshift-dns - ports: - - protocol: UDP - port: 53 - - protocol: TCP - port: 53 - - protocol: UDP - port: 5353 - - protocol: TCP - port: 5353 ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: abevalflow-allow-ci-egress-by-part-of - namespace: NAMESPACE - labels: - app.kubernetes.io/part-of: abevalflow - app.kubernetes.io/component: forge-saw -spec: - podSelector: - matchLabels: - app.kubernetes.io/part-of: abevalflow - policyTypes: [Egress] - egress: - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: openshell-saw-agent - vm.kubevirt.io/name: openshell-saw-agent - ports: - - protocol: TCP - port: 17670 - - protocol: TCP - port: 8443 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: openshell-saw-integ - vm.kubevirt.io/name: openshell-saw-integ - ports: - - protocol: TCP - port: 17670 - - protocol: TCP - port: 18082 - - protocol: TCP - port: 8443 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: litellm - ports: - - protocol: TCP - port: 4000 - - to: - - podSelector: + kubernetes.io/metadata.name: gz-forge-eval + podSelector: matchLabels: app.kubernetes.io/name: mlflow app.kubernetes.io/part-of: abevalflow @@ -195,6 +132,13 @@ spec: matchLabels: app.kubernetes.io/name: minio app.kubernetes.io/part-of: abevalflow + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: gz-forge-eval + podSelector: + matchLabels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: abevalflow ports: - protocol: TCP port: 9000 @@ -211,3 +155,16 @@ spec: port: 5353 - protocol: TCP port: 5353 + # Setup and runtime HTTPS: OpenShell release download, PyPI packages, + # and a public OIDC issuer when client credentials are configured. + # NetworkPolicy cannot restrict this rule to DNS names. + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + ports: + - protocol: TCP + port: 443 diff --git a/config/forge-saw/networkpolicy-gateway-from-abeval.yaml b/config/forge-saw/networkpolicy-gateway-from-abeval.yaml index 9071391..54a410f 100644 --- a/config/forge-saw/networkpolicy-gateway-from-abeval.yaml +++ b/config/forge-saw/networkpolicy-gateway-from-abeval.yaml @@ -2,8 +2,7 @@ # bootstrap.sh rewrites namespace / from-namespace (SAW_NS, EVAL_NS). # Apply in the SAW namespace, not from the evaluate PipelineRun. # -# Prefer matching Tekton pods by the canonical Pipeline name or the -# app.kubernetes.io/part-of=abevalflow label on PipelineRuns. A bare +# Match evaluate Task pods in the canonical Pipeline. A bare # podSelector: {} from the whole EVAL_NS is broader than needed when SAW # and eval share a Forge workspace namespace — see networkpolicy-ci-openshell.yaml. apiVersion: networking.k8s.io/v1 @@ -28,12 +27,7 @@ spec: podSelector: matchLabels: tekton.dev/pipeline: abevalflow-pipeline-openshell - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: guy-ziv-evalflow - podSelector: - matchLabels: - app.kubernetes.io/part-of: abevalflow + tekton.dev/task: evaluate ports: - protocol: TCP port: 17670 diff --git a/pipeline/pipelines/ci-pipeline-openshell.yaml b/pipeline/pipelines/ci-pipeline-openshell.yaml index b4ff9bf..2cbeb3e 100644 --- a/pipeline/pipelines/ci-pipeline-openshell.yaml +++ b/pipeline/pipelines/ci-pipeline-openshell.yaml @@ -103,10 +103,10 @@ spec: default: "https://github.com/GuyZivRH/agent-eval-harness.git" - name: agent-eval-harness-repo-revision type: string - default: "main" + default: "8d58e500d0eb55a3106819ccacf26da6bf7ea166" description: >- - GuyZivRH/agent-eval-harness main includes the OpenClaw brief-reader and - gateway fixes merged in #9. Pin a commit SHA for reproducible runs. + Verified GuyZivRH/agent-eval-harness commit containing the OpenClaw + fixes merged in #9. Override with main only for development runs. - name: openshell-cli-version type: string default: "0.0.116" @@ -119,6 +119,12 @@ spec: - name: openshell-provider type: string default: "forge-ai-gateway,m365-read-intervm,slack-read-proxy,drafts-service-agent" + - name: openshell-user-fixture + type: string + default: "fixtures/USER.md" + description: >- + OpenClaw Forge installation fixture relative to the submission directory. + Override with an empty value for submissions without this fixture. - name: openshell-mtls-secret type: string default: "openshell-mtls" @@ -282,6 +288,8 @@ spec: value: $(params.openshell-sandbox-image) - name: openshell-provider value: $(params.openshell-provider) + - name: openshell-user-fixture + value: $(params.openshell-user-fixture) - name: openshell-mtls-secret value: $(params.openshell-mtls-secret) - name: openshell-ai-gateway-ca-secret diff --git a/pipeline/runs/openshell-openclaw-pipelinerun.yaml b/pipeline/runs/openshell-openclaw-pipelinerun.yaml index 1254a2d..885d566 100644 --- a/pipeline/runs/openshell-openclaw-pipelinerun.yaml +++ b/pipeline/runs/openshell-openclaw-pipelinerun.yaml @@ -5,11 +5,6 @@ apiVersion: tekton.dev/v1 kind: PipelineRun metadata: generateName: aeh-openshell-openclaw- - labels: - # Required when NetworkPolicies select on part-of (and recommended always). - # Keep pipelineRef.name=abevalflow-pipeline-openshell so tekton.dev/pipeline - # also matches the canonical CI NetworkPolicies. - app.kubernetes.io/part-of: abevalflow spec: pipelineRef: name: abevalflow-pipeline-openshell @@ -44,6 +39,8 @@ spec: value: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" - name: openshell-provider value: "forge-ai-gateway,m365-read-intervm,slack-read-proxy,drafts-service-agent" + - name: openshell-user-fixture + value: "fixtures/USER.md" - name: aeh-openshell-image value: "registry.access.redhat.com/ubi9/python-311:9.6" - name: llm-model diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index b67c8cb..8ce8940 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -184,10 +184,10 @@ spec: default: "https://github.com/GuyZivRH/agent-eval-harness.git" - name: agent-eval-harness-repo-revision type: string - default: "main" + default: "8d58e500d0eb55a3106819ccacf26da6bf7ea166" description: >- - Branch or tag of GuyZivRH/agent-eval-harness to clone. Must include the - OpenShell create keep-alive fix (`--detach` + `sleep infinity`). + Commit SHA of GuyZivRH/agent-eval-harness with the OpenShell fixes. + Branches and tags are accepted as explicit overrides. - name: openshell-gateway-endpoint type: string default: "http://openshell.openshell.svc.cluster.local:8080" @@ -209,6 +209,10 @@ spec: type: string default: "forge-ai-gateway,m365-read-intervm,slack-read-proxy,drafts-service-agent" description: SAW provider explicitly attached to each evaluation sandbox + - name: openshell-user-fixture + type: string + default: "" + description: Optional installation fixture path relative to the submission directory - name: openshell-mtls-secret type: string default: "openshell-mtls" @@ -276,14 +280,15 @@ spec: # Clone/update agent-eval-harness repo (required for latest OpenShell logic) HARNESS_DIR="$(workspaces.source.path)/_harness" echo "=== Cloning harness repo: $(params.agent-eval-harness-repo-url) @ $(params.agent-eval-harness-repo-revision) ===" - if [ -d "$HARNESS_DIR/.git" ]; then - cd "$HARNESS_DIR" - git fetch origin "$(params.agent-eval-harness-repo-revision)" --depth 1 - git reset --hard FETCH_HEAD + if [ ! -d "$HARNESS_DIR/.git" ]; then + mkdir -p "$HARNESS_DIR" + git -C "$HARNESS_DIR" init -q + git -C "$HARNESS_DIR" remote add origin "$(params.agent-eval-harness-repo-url)" else - git clone --depth 1 --branch "$(params.agent-eval-harness-repo-revision)" \ - "$(params.agent-eval-harness-repo-url)" "$HARNESS_DIR" + git -C "$HARNESS_DIR" remote set-url origin "$(params.agent-eval-harness-repo-url)" fi + git -C "$HARNESS_DIR" fetch --depth 1 origin "$(params.agent-eval-harness-repo-revision)" + git -C "$HARNESS_DIR" checkout --detach -f FETCH_HEAD echo "Harness repo ready at $(git -C "$HARNESS_DIR" rev-parse --short HEAD)" # Create results directories @@ -1680,6 +1685,8 @@ spec: value: "$(params.openshell-sandbox-image)" - name: AGENT_EVAL_OPENSHELL_PROVIDER value: "$(params.openshell-provider)" + - name: OPENSHELL_USER_FIXTURE + value: "$(params.openshell-user-fixture)" - name: AGENT_EVAL_FORGE_AI_GATEWAY_CA_FILE value: "/tmp/forge-ai-gateway-ca/ca.crt" - name: FORGE_SAW_PROFILE @@ -1799,61 +1806,14 @@ spec: grep -vE 'client.secret|secret|token' /tmp/openshell-gateway-add.log || true exit 1 } - OIDC_TOKEN_JSON=/tmp/openshell-oidc-token.json - curl -ksSf --max-time 20 \ - -u "${OPENSHELL_OIDC_CLIENT_ID}:${OPENSHELL_OIDC_CLIENT_SECRET}" \ - -d grant_type=client_credentials \ - "${OPENSHELL_OIDC_ISSUER%/}/protocol/openid-connect/token" \ - >"$OIDC_TOKEN_JSON" - python3 - "$OIDC_TOKEN_JSON" "$HOME/.config/openshell/gateways/$GW_NAME/oidc_token.json" \ - "$OPENSHELL_OIDC_ISSUER" "$OPENSHELL_OIDC_CLIENT_ID" <<'PY' - import json, sys, time - src, dst, issuer, client_id = sys.argv[1:] - payload = json.load(open(src, encoding="utf-8")) - token = payload["access_token"] - expires_at = int(time.time()) + int(payload.get("expires_in", 300)) - json.dump({"access_token": token, "expires_at": expires_at, - "issuer": issuer, "client_id": client_id}, open(dst, "w", encoding="utf-8")) - PY - chmod 600 "$HOME/.config/openshell/gateways/$GW_NAME/oidc_token.json" + OIDC_TOKEN_CACHE="$HOME/.config/openshell/gateways/$GW_NAME/oidc_token.json" + OIDC_REFRESH_HELPER="$(workspaces.source.path)/_pipeline/scripts/refresh_openshell_oidc.py" + python3 "$OIDC_REFRESH_HELPER" --cache "$OIDC_TOKEN_CACHE" echo "OpenShell OIDC token cache populated via client credentials" # A case can run for 15 minutes, while the client-credentials token # expires after roughly five. Refresh the CLI cache throughout this # step so later artifact downloads and sandbox creation stay authorized. - OIDC_TOKEN_CACHE="$HOME/.config/openshell/gateways/$GW_NAME/oidc_token.json" - ( - umask 077 - trap 'rm -f "${OIDC_TOKEN_JSON}.refresh"' EXIT - while sleep 120; do - if curl -ksSf --max-time 20 \ - -u "${OPENSHELL_OIDC_CLIENT_ID}:${OPENSHELL_OIDC_CLIENT_SECRET}" \ - -d grant_type=client_credentials \ - "${OPENSHELL_OIDC_ISSUER%/}/protocol/openid-connect/token" \ - >"${OIDC_TOKEN_JSON}.refresh" 2>/dev/null; then - python3 - "${OIDC_TOKEN_JSON}.refresh" "$OIDC_TOKEN_CACHE" \ - "$OPENSHELL_OIDC_ISSUER" "$OPENSHELL_OIDC_CLIENT_ID" <<'REFRESH' || true - import json, os, sys, tempfile, time - src, dst, issuer, client_id = sys.argv[1:] - with open(src, encoding="utf-8") as stream: - payload = json.load(stream) - cache = {"access_token": payload["access_token"], - "expires_at": int(time.time()) + int(payload.get("expires_in", 300)), - "issuer": issuer, "client_id": client_id} - fd, tmp = tempfile.mkstemp(dir=os.path.dirname(dst), prefix=".oidc-token-") - try: - with os.fdopen(fd, "w", encoding="utf-8") as stream: - json.dump(cache, stream) - os.chmod(tmp, 0o600) - os.replace(tmp, dst) - finally: - if os.path.exists(tmp): - os.unlink(tmp) - REFRESH - else - echo "WARN: OpenShell OIDC token refresh failed; retrying" >&2 - fi - done - ) & + python3 "$OIDC_REFRESH_HELPER" --cache "$OIDC_TOKEN_CACHE" --interval 120 & OIDC_REFRESH_PID=$! trap 'kill "$OIDC_REFRESH_PID" 2>/dev/null || true' EXIT echo "OpenShell OIDC token refresher started" @@ -2101,12 +2061,18 @@ spec: echo "Model: $MODEL" export AGENT_EVAL_HARNESS_ROOT="$HARNESS_DIR" - # A published Forge briefing needs installation identity. Keep this - # synthetic fixture scoped to submissions that supply one; an explicit - # harness setting takes precedence for other deployments. - if [ -z "${AGENT_EVAL_FORGE_USER_FILE:-}" ] && [ -f "$SUBMISSION_DIR/fixtures/USER.md" ]; then - export AGENT_EVAL_FORGE_USER_FILE="$SUBMISSION_DIR/fixtures/USER.md" - echo "Using submission installation profile fixture" + # Only submissions that explicitly configure a fixture stage one. + if [ -n "$OPENSHELL_USER_FIXTURE" ] && [ -z "${AGENT_EVAL_FORGE_USER_FILE:-}" ]; then + case "$OPENSHELL_USER_FIXTURE" in + /*|..|../*|*/../*|*/..) echo "ERROR: installation fixture must be submission-relative" >&2; exit 1 ;; + esac + FIXTURE_FILE="$SUBMISSION_DIR/$OPENSHELL_USER_FIXTURE" + if [ ! -f "$FIXTURE_FILE" ]; then + echo "ERROR: configured installation fixture is missing: $FIXTURE_FILE" >&2 + exit 1 + fi + export AGENT_EVAL_FORGE_USER_FILE="$FIXTURE_FILE" + echo "Using configured installation profile fixture" fi if [ "$(params.enable-mlflow)" = "true" ] && [ -n "$(params.mlflow-tracking-uri)" ]; then export MLFLOW_TRACKING_URI="$(params.mlflow-tracking-uri)" diff --git a/pipeline/tasks/phases/prepare.yaml b/pipeline/tasks/phases/prepare.yaml index 9da41f6..cdc7d79 100644 --- a/pipeline/tasks/phases/prepare.yaml +++ b/pipeline/tasks/phases/prepare.yaml @@ -123,19 +123,15 @@ spec: PIPELINE_DIR="$(workspaces.source.path)/_pipeline" PIPELINE_REV="$(params.pipeline-repo-revision)" - if [ -d "$PIPELINE_DIR/.git" ]; then - echo "Pipeline repo already cloned, updating..." - cd "$PIPELINE_DIR" - git fetch origin "$PIPELINE_REV" --depth 1 2>/dev/null \ - || git fetch origin "$PIPELINE_REV" - git checkout "$PIPELINE_REV" 2>/dev/null || git checkout FETCH_HEAD + if [ ! -d "$PIPELINE_DIR/.git" ]; then + mkdir -p "$PIPELINE_DIR" + git -C "$PIPELINE_DIR" init -q + git -C "$PIPELINE_DIR" remote add origin "$(params.pipeline-repo-url)" else - if ! git clone --depth 1 --branch "$PIPELINE_REV" \ - "$(params.pipeline-repo-url)" "$PIPELINE_DIR" 2>/dev/null; then - git clone "$(params.pipeline-repo-url)" "$PIPELINE_DIR" - git -C "$PIPELINE_DIR" checkout "$PIPELINE_REV" - fi + git -C "$PIPELINE_DIR" remote set-url origin "$(params.pipeline-repo-url)" fi + git -C "$PIPELINE_DIR" fetch --depth 1 origin "$PIPELINE_REV" + git -C "$PIPELINE_DIR" checkout --detach -f FETCH_HEAD echo "Pipeline repo ready at $(git -C "$PIPELINE_DIR" rev-parse --short HEAD)" # Step 3: Generate tests (conditional) diff --git a/scripts/refresh_openshell_oidc.py b/scripts/refresh_openshell_oidc.py new file mode 100644 index 0000000..6003dda --- /dev/null +++ b/scripts/refresh_openshell_oidc.py @@ -0,0 +1,81 @@ +"""Populate and periodically refresh the OpenShell CLI OIDC token cache.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import tempfile +import time +from pathlib import Path + + +def fetch_token(issuer: str, client_id: str, client_secret: str) -> dict: + # Match the existing Task's curl transport, including its CA handling. + result = subprocess.run( + [ + "curl", + "-ksSf", + "--max-time", + "20", + "-u", + f"{client_id}:{client_secret}", + "-d", + "grant_type=client_credentials", + f"{issuer.rstrip('/')}/protocol/openid-connect/token", + ], + capture_output=True, + check=True, + ) + return json.loads(result.stdout) + + +def write_cache(path: Path, payload: dict, issuer: str, client_id: str) -> None: + cache = { + "access_token": payload["access_token"], + "expires_at": int(time.time()) + int(payload.get("expires_in", 300)), + "issuer": issuer, + "client_id": client_id, + } + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp(prefix=".oidc-token-", dir=path.parent) + try: + os.fchmod(fd, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump(cache, stream) + os.replace(tmp_name, path) + finally: + if os.path.exists(tmp_name): + os.unlink(tmp_name) + + +def refresh(path: Path, issuer: str, client_id: str, client_secret: str) -> None: + write_cache(path, fetch_token(issuer, client_id, client_secret), issuer, client_id) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cache", type=Path, required=True) + parser.add_argument("--interval", type=int, default=0, help="Seconds between refreshes; zero writes once") + args = parser.parse_args() + if args.interval < 0: + parser.error("--interval must be nonnegative") + + issuer = os.environ["OPENSHELL_OIDC_ISSUER"] + client_id = os.environ["OPENSHELL_OIDC_CLIENT_ID"] + client_secret = os.environ["OPENSHELL_OIDC_CLIENT_SECRET"] + if not args.interval: + refresh(args.cache, issuer, client_id, client_secret) + return + + while True: + time.sleep(args.interval) + try: + refresh(args.cache, issuer, client_id, client_secret) + except (OSError, ValueError, KeyError, subprocess.CalledProcessError) as exc: + print(f"WARN: OpenShell OIDC token refresh failed ({type(exc).__name__}); retrying", flush=True) + + +if __name__ == "__main__": + main() diff --git a/tests/test_openshell_networkpolicy.py b/tests/test_openshell_networkpolicy.py new file mode 100644 index 0000000..90bc42e --- /dev/null +++ b/tests/test_openshell_networkpolicy.py @@ -0,0 +1,43 @@ +"""Check the OpenShell NetworkPolicy's intended Tekton pod identity.""" + +from __future__ import annotations + +from pathlib import Path + +import yaml + +REPO = Path(__file__).resolve().parents[1] +EVALUATE_SELECTOR = {"tekton.dev/pipeline": "abevalflow-pipeline-openshell", "tekton.dev/task": "evaluate"} + + +def _matches(selector: dict[str, str], labels: dict[str, str]) -> bool: + return all(labels.get(key) == value for key, value in selector.items()) + + +def test_gateway_and_egress_select_only_evaluate_task_pods(): + pipeline = yaml.safe_load((REPO / "pipeline/pipelines/ci-pipeline-openshell.yaml").read_text()) + evaluate = next(task for task in pipeline["spec"]["tasks"] if task["name"] == "evaluate") + assert pipeline["metadata"]["name"] == EVALUATE_SELECTOR["tekton.dev/pipeline"] + assert evaluate["taskRef"]["name"] == EVALUATE_SELECTOR["tekton.dev/task"] + + policies = list(yaml.safe_load_all((REPO / "config/forge-saw/networkpolicy-ci-openshell.yaml").read_text())) + assert len(policies) == 3 + ingress_selectors = [ + rule["podSelector"]["matchLabels"] + for policy in policies[:2] + for rule in policy["spec"]["ingress"][0]["from"] + ] + egress_selector = policies[2]["spec"]["podSelector"]["matchLabels"] + remote_gateway = yaml.safe_load((REPO / "config/forge-saw/networkpolicy-gateway-from-abeval.yaml").read_text()) + remote_selector = remote_gateway["spec"]["ingress"][0]["from"][0]["podSelector"]["matchLabels"] + assert ingress_selectors + [egress_selector, remote_selector] == [EVALUATE_SELECTOR] * 4 + + evaluate_pod = {**EVALUATE_SELECTOR, "app.kubernetes.io/part-of": "abevalflow"} + prepare_pod = {**evaluate_pod, "tekton.dev/task": "prepare"} + unrelated_pod = {"app.kubernetes.io/part-of": "abevalflow"} + other_pipeline = {**evaluate_pod, "tekton.dev/pipeline": "other-pipeline"} + for selector in ingress_selectors + [egress_selector, remote_selector]: + assert _matches(selector, evaluate_pod) + assert not _matches(selector, prepare_pod) + assert not _matches(selector, unrelated_pod) + assert not _matches(selector, other_pipeline) diff --git a/tests/test_openshell_pipeline_profile.py b/tests/test_openshell_pipeline_profile.py index e3207ab..1875801 100644 --- a/tests/test_openshell_pipeline_profile.py +++ b/tests/test_openshell_pipeline_profile.py @@ -8,6 +8,7 @@ REPO = Path(__file__).resolve().parents[1] PIPELINE = REPO / "pipeline" / "pipelines" / "ci-pipeline-openshell.yaml" +HARNESS_PIN = "8d58e500d0eb55a3106819ccacf26da6bf7ea166" CI = REPO / "pipeline" / "pipelines" / "ci-pipeline.yaml" CI_DEV = REPO / "pipeline" / "pipelines" / "ci-pipeline-dev.yaml" @@ -41,6 +42,8 @@ def test_openshell_defaults(self): assert defaults["enable-ai-generation"] == "false" assert defaults["aeh-runner"] == "openshell" assert defaults["aeh-openshell-image"] == "registry.access.redhat.com/ubi9/python-311:9.6" + assert defaults["agent-eval-harness-repo-revision"] == HARNESS_PIN + assert defaults["openshell-user-fixture"] == "fixtures/USER.md" assert defaults["openshell-sandbox-image"] == ( "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" ) @@ -63,6 +66,8 @@ def test_evaluate_openshell_step_logs_mlflow(self): def test_openshell_eval_uses_llm_param_instead_of_inference_secret(self): task = _load(REPO / "pipeline" / "tasks" / "phases" / "evaluate.yaml") + assert _param_defaults(task["spec"])["openshell-user-fixture"] == "" + assert _param_defaults(task["spec"])["agent-eval-harness-repo-revision"] == HARNESS_PIN openshell = next(step for step in task["spec"]["steps"] if step["name"] == "aeh-openshell-eval") env_from_secret_names = { @@ -91,11 +96,24 @@ def test_forge_briefing_has_installation_user_fixture(self): scene = _load(REPO / "submissions" / "openclaw-forge" / "scenes" / "monday-acquisition.yaml") assert fields["primary email"] == scene["m365"]["user"] + pipeline = _load(PIPELINE) + evaluate = next(item for item in pipeline["spec"]["tasks"] if item["name"] == "evaluate") + forwarded = {item["name"]: item["value"] for item in evaluate["params"]} + assert forwarded["openshell-user-fixture"] == "$(params.openshell-user-fixture)" + example = _load(REPO / "pipeline" / "runs" / "openshell-openclaw-pipelinerun.yaml") + example_params = {item["name"]: item["value"] for item in example["spec"]["params"]} + assert example_params["openshell-user-fixture"] == "fixtures/USER.md" + task = _load(REPO / "pipeline" / "tasks" / "phases" / "evaluate.yaml") step = next(s for s in task["spec"]["steps"] if s["name"] == "aeh-openshell-eval") + assert next(env for env in step["env"] if env["name"] == "OPENSHELL_USER_FIXTURE")["value"] == ( + "$(params.openshell-user-fixture)" + ) script = step["script"] - assert 'AGENT_EVAL_FORGE_USER_FILE="$SUBMISSION_DIR/fixtures/USER.md"' in script - assert script.index('[ -f "$SUBMISSION_DIR/fixtures/USER.md" ]') < script.index("scripts/run_aeh.py") + assert 'FIXTURE_FILE="$SUBMISSION_DIR/$OPENSHELL_USER_FIXTURE"' in script + assert "Using configured installation profile fixture" in script + assert "fixtures/USER.md" not in script + assert script.index('export AGENT_EVAL_FORGE_USER_FILE="$FIXTURE_FILE"') < script.index("scripts/run_aeh.py") def test_harbor_profiles_still_include_test(self): for path in (CI, CI_DEV): diff --git a/tests/test_prepare_pipeline_checkout.py b/tests/test_prepare_pipeline_checkout.py new file mode 100644 index 0000000..0c2e7bf --- /dev/null +++ b/tests/test_prepare_pipeline_checkout.py @@ -0,0 +1,51 @@ +"""Exercise the Prepare Task pipeline checkout with a local Git remote.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import yaml + + +def _git(*args: str, cwd: Path | None = None) -> str: + return subprocess.check_output(["git", *args], cwd=cwd, text=True).strip() + + +def test_pipeline_checkout_accepts_branch_tag_and_full_sha(tmp_path: Path): + source = tmp_path / "source" + remote = tmp_path / "remote.git" + _git("init", "-q", "-b", "main", str(source)) + (source / "example.txt").write_text("main") + _git("add", "example.txt", cwd=source) + _git("-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-qm", "initial", cwd=source) + main_sha = _git("rev-parse", "HEAD", cwd=source) + _git("tag", "v1", cwd=source) + _git("checkout", "-qb", "feature", cwd=source) + (source / "example.txt").write_text("feature") + _git("add", "example.txt", cwd=source) + _git("-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-qm", "feature", cwd=source) + feature_sha = _git("rev-parse", "HEAD", cwd=source) + _git("init", "--bare", "-q", str(remote)) + _git("remote", "add", "origin", str(remote), cwd=source) + _git("push", "-q", "origin", "main", "feature", "v1", cwd=source) + + repo = Path(__file__).resolve().parents[1] + task = yaml.safe_load((repo / "pipeline/tasks/phases/prepare.yaml").read_text()) + script = next(step["script"] for step in task["spec"]["steps"] if step["name"] == "clone-pipeline-repo") + + def checkout(revision: str, workspace: Path) -> str: + workspace.mkdir(exist_ok=True) + rendered = ( + script.replace("$(workspaces.source.path)", str(workspace)) + .replace("$(params.pipeline-repo-revision)", revision) + .replace("$(params.pipeline-repo-url)", str(remote)) + ) + subprocess.run(["bash", "-c", rendered], check=True, capture_output=True, text=True) + return _git("rev-parse", "HEAD", cwd=workspace / "_pipeline") + + workspace = tmp_path / "workspace" + assert checkout("main", workspace) == main_sha + assert checkout("v1", workspace) == main_sha + assert checkout("feature", workspace) == feature_sha + assert checkout(main_sha, tmp_path / "fresh-workspace") == main_sha diff --git a/tests/test_refresh_openshell_oidc.py b/tests/test_refresh_openshell_oidc.py new file mode 100644 index 0000000..f2e4f23 --- /dev/null +++ b/tests/test_refresh_openshell_oidc.py @@ -0,0 +1,67 @@ +"""Behavior tests for the OpenShell OIDC cache helper.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from unittest.mock import patch + +import yaml + +from scripts import refresh_openshell_oidc as oidc + + +def test_initial_and_refresh_writes_are_atomic_and_private(tmp_path: Path): + cache = tmp_path / "gateways" / "ci" / "oidc_token.json" + issuer = "https://issuer.example" + with patch.object(oidc.time, "time", return_value=1000): + oidc.write_cache(cache, {"access_token": "first", "expires_in": 300}, issuer, "client") + assert cache.stat().st_mode & 0o777 == 0o600 + assert json.loads(cache.read_text()) == { + "access_token": "first", + "expires_at": 1300, + "issuer": issuer, + "client_id": "client", + } + + real_replace = os.replace + + def observe_replace(source: str, destination: Path) -> None: + # A reader still sees the complete old cache until replacement. + assert json.loads(cache.read_text())["access_token"] == "first" + assert Path(source).stat().st_mode & 0o777 == 0o600 + real_replace(source, destination) + + with patch.object(oidc.os, "replace", side_effect=observe_replace): + with patch.object(oidc.time, "time", return_value=1100): + oidc.write_cache(cache, {"access_token": "second", "expires_in": 240}, issuer, "client") + assert json.loads(cache.read_text())["access_token"] == "second" + assert json.loads(cache.read_text())["expires_at"] == 1340 + assert cache.stat().st_mode & 0o777 == 0o600 + assert list(cache.parent.glob(".oidc-token-*")) == [] + + +def test_failed_fetch_preserves_existing_cache(tmp_path: Path): + cache = tmp_path / "oidc_token.json" + oidc.write_cache(cache, {"access_token": "valid"}, "issuer", "client") + original = cache.read_bytes() + with patch.object(oidc, "fetch_token", side_effect=ValueError("bad response")): + try: + oidc.refresh(cache, "issuer", "client", "secret") + except ValueError: + pass + else: + raise AssertionError("refresh should fail") + assert cache.read_bytes() == original + + +def test_evaluate_uses_one_helper_for_initial_and_periodic_refresh(): + task = yaml.safe_load((Path(__file__).parents[1] / "pipeline/tasks/phases/evaluate.yaml").read_text()) + step = next(item for item in task["spec"]["steps"] if item["name"] == "aeh-openshell-eval") + script = step["script"] + assert script.count('python3 "$OIDC_REFRESH_HELPER" --cache "$OIDC_TOKEN_CACHE"') == 2 + assert '--interval 120 &' in script + assert script.index('python3 "$OIDC_REFRESH_HELPER" --cache "$OIDC_TOKEN_CACHE"') < script.index( + 'python "$PIPELINE_DIR/scripts/run_aeh.py"' + ) From a23898cd3d8fc70e0a2702902936b17f60a3d4e4 Mon Sep 17 00:00:00 2001 From: "Sana Fayyazgit config --global user.email sanafayyaz315@gmail.comgit config --global user.name Sana" Date: Fri, 9 Oct 2026 11:57:38 +0100 Subject: [PATCH 6/6] Format OpenShell review tests for CI --- tests/test_openshell_networkpolicy.py | 4 +--- tests/test_refresh_openshell_oidc.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_openshell_networkpolicy.py b/tests/test_openshell_networkpolicy.py index 90bc42e..0fa9ec2 100644 --- a/tests/test_openshell_networkpolicy.py +++ b/tests/test_openshell_networkpolicy.py @@ -23,9 +23,7 @@ def test_gateway_and_egress_select_only_evaluate_task_pods(): policies = list(yaml.safe_load_all((REPO / "config/forge-saw/networkpolicy-ci-openshell.yaml").read_text())) assert len(policies) == 3 ingress_selectors = [ - rule["podSelector"]["matchLabels"] - for policy in policies[:2] - for rule in policy["spec"]["ingress"][0]["from"] + rule["podSelector"]["matchLabels"] for policy in policies[:2] for rule in policy["spec"]["ingress"][0]["from"] ] egress_selector = policies[2]["spec"]["podSelector"]["matchLabels"] remote_gateway = yaml.safe_load((REPO / "config/forge-saw/networkpolicy-gateway-from-abeval.yaml").read_text()) diff --git a/tests/test_refresh_openshell_oidc.py b/tests/test_refresh_openshell_oidc.py index f2e4f23..3c81737 100644 --- a/tests/test_refresh_openshell_oidc.py +++ b/tests/test_refresh_openshell_oidc.py @@ -61,7 +61,7 @@ def test_evaluate_uses_one_helper_for_initial_and_periodic_refresh(): step = next(item for item in task["spec"]["steps"] if item["name"] == "aeh-openshell-eval") script = step["script"] assert script.count('python3 "$OIDC_REFRESH_HELPER" --cache "$OIDC_TOKEN_CACHE"') == 2 - assert '--interval 120 &' in script + assert "--interval 120 &" in script assert script.index('python3 "$OIDC_REFRESH_HELPER" --cache "$OIDC_TOKEN_CACHE"') < script.index( 'python "$PIPELINE_DIR/scripts/run_aeh.py"' )