Stabilize Forge OpenShell evals for full brief publication

config/forge-saw/README.md

@@ -148,6 +148,24 @@ oc get secret openshell-credentials -n guy-ziv-evalflow -o json \
 `aeh-openshell-eval` TCP-checks `:17670` and optionally runs `openshell sandbox list`.
 If SAW is down the **run fails**. That step does not install SAW.
 
+## NetworkPolicy (Forge shared namespace)
+
+When SAW VMs and Tekton share a namespace with default-deny policies, evaluate
+pods must be allowed to reach the agent gateway on `:17670`. Use the canonical
+Pipeline name `abevalflow-pipeline-openshell` (so `tekton.dev/pipeline` matches)
+and label PipelineRuns with `app.kubernetes.io/part-of: abevalflow`.
+
+Same-namespace template:
+
+```bash
+sed "s/NAMESPACE/${EVAL_NS}/g" config/forge-saw/networkpolicy-ci-openshell.yaml \
+  | oc apply -f -
+```
+
+Do not create ad-hoc copies of the Pipeline under a different name unless those
+PipelineRuns also carry the `part-of=abevalflow` label and the NetworkPolicies
+above are applied — otherwise gateway preflight times out.
+
 ## Image
 
 Stock `agent-eval-harness:v1.0.x` cannot import `agent_eval.openshell`. Point

config/forge-saw/networkpolicy-ci-openshell.yaml

@@ -0,0 +1,213 @@
+# NetworkPolicies for OpenShell OpenClaw evals when SAW VMs and the Tekton
+# pipeline share one namespace (Forge workspace style).
+#
+# Select evaluate pods by either:
+#   - tekton.dev/pipeline=abevalflow-pipeline-openshell  (canonical Pipeline name)
+#   - app.kubernetes.io/part-of=abevalflow               (set on PipelineRun labels)
+#
+# Do NOT rename/copy the Pipeline to an ad-hoc name without also labeling the
+# PipelineRun with app.kubernetes.io/part-of=abevalflow — otherwise default-deny
+# blocks TCP to the gateway on :17670 and evaluate fails preflight.
+#
+# Apply after substituting NAMESPACE (and AGENT_VM / INTEG_VM if different):
+#   sed "s/NAMESPACE/forge-nommen/g" networkpolicy-ci-openshell.yaml | oc apply -f -
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: abevalflow-allow-ci-agent-gateway
+  namespace: NAMESPACE
+  labels:
+    app.kubernetes.io/part-of: abevalflow
+    app.kubernetes.io/component: forge-saw
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: openshell-saw-agent
+      vm.kubevirt.io/name: openshell-saw-agent
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              tekton.dev/pipeline: abevalflow-pipeline-openshell
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/part-of: abevalflow
+      ports:
+        - protocol: TCP
+          port: 17670
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: abevalflow-allow-ci-integ-gateway
+  namespace: NAMESPACE
+  labels:
+    app.kubernetes.io/part-of: abevalflow
+    app.kubernetes.io/component: forge-saw
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: openshell-saw-integ
+      vm.kubevirt.io/name: openshell-saw-integ
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              tekton.dev/pipeline: abevalflow-pipeline-openshell
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/part-of: abevalflow
+      ports:
+        - protocol: TCP
+          port: 17670
+        - protocol: TCP
+          port: 18082
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: abevalflow-allow-ci-egress-by-pipeline
+  namespace: NAMESPACE
+  labels:
+    app.kubernetes.io/part-of: abevalflow
+    app.kubernetes.io/component: forge-saw
+spec:
+  podSelector:
+    matchLabels:
+      tekton.dev/pipeline: abevalflow-pipeline-openshell
+  policyTypes: [Egress]
+  egress:
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: openshell-saw-agent
+              vm.kubevirt.io/name: openshell-saw-agent
+      ports:
+        - protocol: TCP
+          port: 17670
+        - protocol: TCP
+          port: 8443
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: openshell-saw-integ
+              vm.kubevirt.io/name: openshell-saw-integ
+      ports:
+        - protocol: TCP
+          port: 17670
+        - protocol: TCP
+          port: 18082
+        - protocol: TCP
+          port: 8443
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: litellm
+      ports:
+        - protocol: TCP
+          port: 4000
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: mlflow
+              app.kubernetes.io/part-of: abevalflow
+      ports:
+        - protocol: TCP
+          port: 5000
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: minio
+              app.kubernetes.io/part-of: abevalflow
+      ports:
+        - protocol: TCP
+          port: 9000
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: openshift-dns
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+        - protocol: UDP
+          port: 5353
+        - protocol: TCP
+          port: 5353
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: abevalflow-allow-ci-egress-by-part-of
+  namespace: NAMESPACE
+  labels:
+    app.kubernetes.io/part-of: abevalflow
+    app.kubernetes.io/component: forge-saw
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/part-of: abevalflow
+  policyTypes: [Egress]
+  egress:
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: openshell-saw-agent
+              vm.kubevirt.io/name: openshell-saw-agent
+      ports:
+        - protocol: TCP
+          port: 17670
+        - protocol: TCP
+          port: 8443
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: openshell-saw-integ
+              vm.kubevirt.io/name: openshell-saw-integ
+      ports:
+        - protocol: TCP
+          port: 17670
+        - protocol: TCP
+          port: 18082
+        - protocol: TCP
+          port: 8443
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: litellm
+      ports:
+        - protocol: TCP
+          port: 4000
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: mlflow
+              app.kubernetes.io/part-of: abevalflow
+      ports:
+        - protocol: TCP
+          port: 5000
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: minio
+              app.kubernetes.io/part-of: abevalflow
+      ports:
+        - protocol: TCP
+          port: 9000
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: openshift-dns
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+        - protocol: UDP
+          port: 5353
+        - protocol: TCP
+          port: 5353

config/forge-saw/networkpolicy-gateway-from-abeval.yaml

@@ -1,6 +1,11 @@
 # Allow evaluate Task pods to reach the OpenShell gateway on :17670.
 # bootstrap.sh rewrites namespace / from-namespace (SAW_NS, EVAL_NS).
 # Apply in the SAW namespace, not from the evaluate PipelineRun.
+#
+# Prefer matching Tekton pods by the canonical Pipeline name or the
+# app.kubernetes.io/part-of=abevalflow label on PipelineRuns. A bare
+# podSelector: {} from the whole EVAL_NS is broader than needed when SAW
+# and eval share a Forge workspace namespace — see networkpolicy-ci-openshell.yaml.
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
@@ -20,7 +25,15 @@ spec:
         - namespaceSelector:
             matchLabels:
               kubernetes.io/metadata.name: guy-ziv-evalflow
-        - podSelector: {}
+          podSelector:
+            matchLabels:
+              tekton.dev/pipeline: abevalflow-pipeline-openshell
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: guy-ziv-evalflow
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/part-of: abevalflow
       ports:
         - protocol: TCP
           port: 17670

pipeline/pipelines/ci-pipeline-openshell.yaml

@@ -5,6 +5,7 @@ metadata:
   namespace: ab-eval-flow
   labels:
     app.kubernetes.io/name: abevalflow
+    app.kubernetes.io/part-of: abevalflow
     app.kubernetes.io/version: "2.0"
     app.kubernetes.io/profile: aeh-openshell-openclaw
 spec:
@@ -102,8 +103,11 @@ spec:
       default: "https://github.com/GuyZivRH/agent-eval-harness.git"
     - name: agent-eval-harness-repo-revision
       type: string
-      default: "feat/aeh-openshell-openclaw"
-      description: Harness revision with agent_eval.openshell (feature-branch pin)
+      default: "main"
+      description: >-
+        Harness revision with agent_eval.openshell. Use a tip that includes the
+        OpenClaw brief-reader/gateway fixes (GuyZivRH/agent-eval-harness#9) until
+        that lands on upstream main.
     - name: openshell-cli-version
       type: string
       default: "0.0.116"
@@ -112,7 +116,7 @@ spec:
       default: "https://openshell.openshell.svc.cluster.local:17670"
     - name: openshell-sandbox-image
       type: string
-      default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a"
+      default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9"
     - name: openshell-provider
       type: string
       default: "forge-ai-gateway,m365-read-intervm,slack-read-proxy,drafts-service-agent"

pipeline/runs/openshell-openclaw-pipelinerun.yaml

@@ -5,10 +5,18 @@ apiVersion: tekton.dev/v1
 kind: PipelineRun
 metadata:
   generateName: aeh-openshell-openclaw-
+  labels:
+    # Required when NetworkPolicies select on part-of (and recommended always).
+    # Keep pipelineRef.name=abevalflow-pipeline-openshell so tekton.dev/pipeline
+    # also matches the canonical CI NetworkPolicies.
+    app.kubernetes.io/part-of: abevalflow
 spec:
   pipelineRef:
     name: abevalflow-pipeline-openshell
   params:
+    # [Optional] Add a submission repository for this run; this replaces the deployed Pipeline default.
+    # - name: repo-url
+    #   value: "<submission-repo-url>"
     - name: submission-dir
       value: "openclaw-forge"
     - name: eval-engine
@@ -19,15 +27,21 @@ spec:
       value: "feat/aeh-openshell-openclaw"
     - name: pipeline-repo-revision
       value: "feat/aeh-openshell-openclaw"
+    # [Optional] Add a harness repository or revision for this run; these replace Pipeline defaults.
+    # - name: agent-eval-harness-repo-url
+    #   value: "<harness-repo-url>"
+    # - name: agent-eval-harness-repo-revision
+    #   value: "<harness-commit-or-branch>"
     - name: openshell-gateway-endpoint
       # Certificate-valid hostname, resolved to this namespace's Service below.
       value: "https://host.containers.internal:17670"
     - name: openshell-mtls-secret
       value: "openshell-gateway-mtls"
     - name: openshell-ai-gateway-ca-secret
       value: "forge-agent-upstream-tls"
+    # To use another sandbox image, replace the value below with <image@sha256:digest>.
     - name: openshell-sandbox-image
-      value: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a"
+      value: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9"
     - name: openshell-provider
       value: "forge-ai-gateway,m365-read-intervm,slack-read-proxy,drafts-service-agent"
     - name: aeh-openshell-image