Port Forge OpenShell reliability fixes onto current main

config/forge-saw/README.md

@@ -148,8 +148,51 @@ oc get secret openshell-credentials -n guy-ziv-evalflow -o json \
 `aeh-openshell-eval` TCP-checks `:17670` and optionally runs `openshell sandbox list`.
 If SAW is down the **run fails**. That step does not install SAW.
 
+## NetworkPolicy (Forge shared namespace)
+
+When SAW VMs and Tekton share a namespace with default-deny policies, evaluate
+pods must be allowed to reach the agent gateway on `:17670`. Use the canonical
+Pipeline name `abevalflow-pipeline-openshell` and its referenced Task name
+`evaluate` so the injected `tekton.dev/pipeline` and `tekton.dev/task` pod
+labels match.
+
+The egress policy permits SAW gateways in the evaluate namespace, LiteLLM,
+MLflow and MinIO either there or in `gz-forge-eval`, DNS, and public TCP/443
+for the Task's OpenShell download, Python package installs and public OIDC
+issuer. Kubernetes NetworkPolicy cannot limit the HTTPS rule to GitHub or
+PyPI hostnames. If the OIDC issuer is private, add an allow rule for its actual
+namespace and pod labels or route before enabling that mode. The shared Task
+also installs packages at runtime; an image with those dependencies baked in
+would allow a narrower deployment policy.
+
+Same-namespace template:
+
+```bash
+sed "s/NAMESPACE/${EVAL_NS}/g" config/forge-saw/networkpolicy-ci-openshell.yaml \
+  | oc apply -f -
+```
+
+If the earlier two-selector template was applied, remove its old egress policy
+after applying this one; `oc apply` does not delete objects omitted from a file:
+
+```bash
+oc delete networkpolicy abevalflow-allow-ci-egress-by-part-of \
+  -n "$EVAL_NS" --ignore-not-found
+```
+
+Copies of the Pipeline or Evaluate Task under another name need matching
+NetworkPolicy selectors; otherwise gateway preflight times out.
+
 ## Image
 
 Stock `agent-eval-harness:v1.0.x` cannot import `agent_eval.openshell`. Point
 `aeh-openshell-image` at an orchestrator image built from GuyZivRH
-`agent-eval-harness` **main** that also includes the `openshell` CLI.
+`agent-eval-harness` at the pinned OpenShell Pipeline revision that also
+includes the `openshell` CLI. Use `main` only as an explicit development
+override.
+
+The OpenShell Pipeline profile explicitly sets `openshell-user-fixture` to
+`fixtures/USER.md` for the Forge submission. The shared Evaluate Task defaults
+this parameter to empty. Set it to an empty value for a submission without an
+installation fixture, or provide another path relative to its submission
+directory. A configured path must exist when Evaluate runs.

config/forge-saw/networkpolicy-ci-openshell.yaml

@@ -0,0 +1,170 @@
+# NetworkPolicies for OpenShell OpenClaw evals when SAW VMs and the Tekton
+# pipeline share one namespace (Forge workspace style).
+#
+# Select the evaluate Task pods in the canonical OpenShell Pipeline by the
+# injected tekton.dev/pipeline and tekton.dev/task labels.
+#
+# Renamed copies of the Pipeline need a matching policy selector or gateway
+# preflight will be blocked by default-deny.
+#
+# Apply after substituting NAMESPACE for the SAW/evaluate namespace. LiteLLM,
+# MLflow and MinIO may also run in gz-forge-eval (the example PipelineRun).
+# Public HTTPS is needed for CLI downloads, Python packages and public OIDC
+# issuers. Private OIDC issuers need an additional deployment-specific rule.
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: abevalflow-allow-ci-agent-gateway
+  namespace: NAMESPACE
+  labels:
+    app.kubernetes.io/part-of: abevalflow
+    app.kubernetes.io/component: forge-saw
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: openshell-saw-agent
+      vm.kubevirt.io/name: openshell-saw-agent
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              tekton.dev/pipeline: abevalflow-pipeline-openshell
+              tekton.dev/task: evaluate
+      ports:
+        - protocol: TCP
+          port: 17670
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: abevalflow-allow-ci-integ-gateway
+  namespace: NAMESPACE
+  labels:
+    app.kubernetes.io/part-of: abevalflow
+    app.kubernetes.io/component: forge-saw
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: openshell-saw-integ
+      vm.kubevirt.io/name: openshell-saw-integ
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              tekton.dev/pipeline: abevalflow-pipeline-openshell
+              tekton.dev/task: evaluate
+      ports:
+        - protocol: TCP
+          port: 17670
+        - protocol: TCP
+          port: 18082
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: abevalflow-allow-ci-egress-by-pipeline
+  namespace: NAMESPACE
+  labels:
+    app.kubernetes.io/part-of: abevalflow
+    app.kubernetes.io/component: forge-saw
+spec:
+  podSelector:
+    matchLabels:
+      tekton.dev/pipeline: abevalflow-pipeline-openshell
+      tekton.dev/task: evaluate
+  policyTypes: [Egress]
+  egress:
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: openshell-saw-agent
+              vm.kubevirt.io/name: openshell-saw-agent
+      ports:
+        - protocol: TCP
+          port: 17670
+        - protocol: TCP
+          port: 8443
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: openshell-saw-integ
+              vm.kubevirt.io/name: openshell-saw-integ
+      ports:
+        - protocol: TCP
+          port: 17670
+        - protocol: TCP
+          port: 18082
+        - protocol: TCP
+          port: 8443
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: litellm
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: gz-forge-eval
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: litellm
+      ports:
+        - protocol: TCP
+          port: 4000
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: mlflow
+              app.kubernetes.io/part-of: abevalflow
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: gz-forge-eval
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: mlflow
+              app.kubernetes.io/part-of: abevalflow
+      ports:
+        - protocol: TCP
+          port: 5000
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: minio
+              app.kubernetes.io/part-of: abevalflow
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: gz-forge-eval
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: minio
+              app.kubernetes.io/part-of: abevalflow
+      ports:
+        - protocol: TCP
+          port: 9000
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: openshift-dns
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+        - protocol: UDP
+          port: 5353
+        - protocol: TCP
+          port: 5353
+    # Setup and runtime HTTPS: OpenShell release download, PyPI packages,
+    # and a public OIDC issuer when client credentials are configured.
+    # NetworkPolicy cannot restrict this rule to DNS names.
+    - to:
+        - ipBlock:
+            cidr: 0.0.0.0/0
+            except:
+              - 10.0.0.0/8
+              - 172.16.0.0/12
+              - 192.168.0.0/16
+      ports:
+        - protocol: TCP
+          port: 443

config/forge-saw/networkpolicy-gateway-from-abeval.yaml

@@ -1,6 +1,10 @@
 # Allow evaluate Task pods to reach the OpenShell gateway on :17670.
 # bootstrap.sh rewrites namespace / from-namespace (SAW_NS, EVAL_NS).
 # Apply in the SAW namespace, not from the evaluate PipelineRun.
+#
+# Match evaluate Task pods in the canonical Pipeline. A bare
+# podSelector: {} from the whole EVAL_NS is broader than needed when SAW
+# and eval share a Forge workspace namespace — see networkpolicy-ci-openshell.yaml.
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
@@ -20,7 +24,10 @@ spec:
         - namespaceSelector:
             matchLabels:
               kubernetes.io/metadata.name: guy-ziv-evalflow
-        - podSelector: {}
+          podSelector:
+            matchLabels:
+              tekton.dev/pipeline: abevalflow-pipeline-openshell
+              tekton.dev/task: evaluate
       ports:
         - protocol: TCP
           port: 17670

pipeline/pipelines/ci-pipeline-openshell.yaml

@@ -5,6 +5,7 @@ metadata:
   namespace: ab-eval-flow
   labels:
     app.kubernetes.io/name: abevalflow
+    app.kubernetes.io/part-of: abevalflow
     app.kubernetes.io/version: "2.0"
     app.kubernetes.io/profile: aeh-openshell-openclaw
 spec:
@@ -102,8 +103,10 @@ spec:
       default: "https://github.com/GuyZivRH/agent-eval-harness.git"
     - name: agent-eval-harness-repo-revision
       type: string
-      default: "feat/aeh-openshell-openclaw"
-      description: Harness revision with agent_eval.openshell (feature-branch pin)
+      default: "8d58e500d0eb55a3106819ccacf26da6bf7ea166"
+      description: >-
+        Verified GuyZivRH/agent-eval-harness commit containing the OpenClaw
+        fixes merged in #9. Override with main only for development runs.
     - name: openshell-cli-version
       type: string
       default: "0.0.116"
@@ -112,10 +115,16 @@ spec:
       default: "https://openshell.openshell.svc.cluster.local:17670"
     - name: openshell-sandbox-image
       type: string
-      default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a"
+      default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9"
     - name: openshell-provider
       type: string
       default: "forge-ai-gateway,m365-read-intervm,slack-read-proxy,drafts-service-agent"
+    - name: openshell-user-fixture
+      type: string
+      default: "fixtures/USER.md"
+      description: >-
+        OpenClaw Forge installation fixture relative to the submission directory.
+        Override with an empty value for submissions without this fixture.
     - name: openshell-mtls-secret
       type: string
       default: "openshell-mtls"
@@ -279,6 +288,8 @@ spec:
           value: $(params.openshell-sandbox-image)
         - name: openshell-provider
           value: $(params.openshell-provider)
+        - name: openshell-user-fixture
+          value: $(params.openshell-user-fixture)
         - name: openshell-mtls-secret
           value: $(params.openshell-mtls-secret)
         - name: openshell-ai-gateway-ca-secret

pipeline/runs/openshell-openclaw-pipelinerun.yaml

@@ -16,12 +16,12 @@ spec:
       value: "openclaw-forge"
     - name: eval-engine
       value: "aeh_openshell_openclaw"
-    # Until this branch is merged, pass the feature revision so store/parse scripts match.
-    # After merge to main, omit these (Pipeline defaults are main).
+    # This example targets the merged upstream main for both submission and
+    # Pipeline helper source. Replace either value with a commit SHA to pin a run.
     - name: revision
-      value: "feat/aeh-openshell-openclaw"
+      value: "main"
     - name: pipeline-repo-revision
-      value: "feat/aeh-openshell-openclaw"
+      value: "main"
     # [Optional] Add a harness repository or revision for this run; these replace Pipeline defaults.
     # - name: agent-eval-harness-repo-url
     #   value: "<harness-repo-url>"
@@ -36,9 +36,11 @@ spec:
       value: "forge-agent-upstream-tls"
     # To use another sandbox image, replace the value below with <image@sha256:digest>.
     - name: openshell-sandbox-image
-      value: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a"
+      value: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9"
     - name: openshell-provider
       value: "forge-ai-gateway,m365-read-intervm,slack-read-proxy,drafts-service-agent"
+    - name: openshell-user-fixture
+      value: "fixtures/USER.md"
     - name: aeh-openshell-image
       value: "registry.access.redhat.com/ubi9/python-311:9.6"
     - name: llm-model