From 3427d609b143295db807f876bff6d2cd243d5fb1 Mon Sep 17 00:00:00 2001 From: vaibhavdabas16 Date: Wed, 9 Sep 2026 20:40:46 +0530 Subject: [PATCH 1/3] fix(harbor): gate step setup on the real readiness contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setup accepted any 200 from /api/status as "the runtime is ready". The runtime server starts answering before its CDP handler attaches and sets eval_interceptor_ready, so a task could begin in that window with interception inactive and silently fail to score Stage 1 — the failure looks like an agent miss, not a harness fault. The step healthcheck in task.toml already checked the right condition, so the two now share one `runtime_ready_command()` and cannot drift: runtime server up, request interceptor armed, CDP endpoint live. The TOML form is escaped once via json.dumps and the shell form is used verbatim, which also fixes the escaping that would have made a literal copy of the healthcheck string never match inside a shell script. The wait is also raised from 60s to a configurable 180s, and a timeout now tails the runtime-server log instead of exiting with one opaque line — remote sandboxes (#331 §3) provision slower than a local daemon and were the case 60s was never sized for. --- src/clawbench/eval/harbor_adapter.py | 47 ++++++++++++++++++---------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/src/clawbench/eval/harbor_adapter.py b/src/clawbench/eval/harbor_adapter.py index 4a0c0996..b13c87b2 100644 --- a/src/clawbench/eval/harbor_adapter.py +++ b/src/clawbench/eval/harbor_adapter.py @@ -131,6 +131,25 @@ def copy_environment(env_dir: Path) -> None: chmod_executable(script) +def runtime_ready_command(cdp_url: str) -> str: + """Shell test for "the ClawBench runtime is ready to score this task". + + Ready means three things at once: the runtime server answers, its request + interceptor is armed, and the browser's CDP endpoint is live. A 200 from + ``/api/status`` alone is not enough — the server starts answering before + the CDP handler has attached, so a task that begins in that window runs + with interception inactive and silently cannot score Stage 1. + + Harbor's per-step healthcheck and the step's own setup script both use + this, so the two can never drift apart. + """ + return ( + "curl -sf http://127.0.0.1:7878/api/status " + + "| grep -q '\"eval_interceptor_ready\":true' " + + f"&& curl -sf {cdp_url}/json/version >/dev/null" + ) + + def playwright_mcp_server(cdp_url: str) -> dict[str, Any]: return { "name": "playwright", @@ -207,11 +226,8 @@ def task_toml( + '\nCLAWBENCH_RECORDING_MODE = "provider-download"' ) mcp_servers = mcp_servers_toml([playwright_mcp_server(REMOTE_BRIDGE_CDP_URL)]) - healthcheck_command = ( - "curl -sf http://127.0.0.1:7878/api/status | grep -q '" - + '\\"eval_interceptor_ready\\":true' - + f"' && curl -sf {cdp_url}/json/version >/dev/null" - ) + # The shell form is escaped once for TOML; setup.sh takes it verbatim. + escaped_healthcheck = json.dumps(runtime_ready_command(cdp_url)) return ( f"""schema_version = "1.3" source = "clawbench-v2" @@ -260,7 +276,7 @@ def task_toml( timeout_sec = 300.0 [steps.healthcheck] -command = "{healthcheck_command}" +command = {escaped_healthcheck} interval_sec = 2.0 timeout_sec = 5.0 start_period_sec = 2.0 @@ -273,15 +289,9 @@ def task_toml( def setup_script(browser_runtime: str = "local") -> str: kernel_setup = "" - readiness = ( - " if curl -sf http://127.0.0.1:7878/api/status >/dev/null \\\n" - " && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then\n" - ) + cdp_url = REMOTE_BRIDGE_CDP_URL if browser_runtime == "kernel" else LOCAL_CDP_URL + readiness = f" if {runtime_ready_command(cdp_url)}; then\n" if browser_runtime == "kernel": - readiness = ( - " if curl -sf http://127.0.0.1:7878/api/status >/dev/null \\\n" - " && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then\n" - ) kernel_setup = ( "# Create the Kernel browser and replay before the runtime server" " starts so it can bridge the provider CDP endpoint.\n" @@ -310,7 +320,10 @@ def setup_script(browser_runtime: str = "local") -> str: {kernel_setup}/app/src/harbor/start-runtime.sh -for _ in $(seq 1 60); do +# Remote sandboxes (e2b, daytona, modal) cold-start slower than a local +# container daemon, so this waits well past the local worst case rather than +# failing a trial on provisioning latency. +for _ in $(seq 1 "${{CLAWBENCH_RUNTIME_READY_TIMEOUT_S:-180}}"); do {readiness} rm -f /app/setup.sh trap - EXIT exit 0 @@ -318,7 +331,9 @@ def setup_script(browser_runtime: str = "local") -> str: sleep 1 done -echo "ClawBench Harbor runtime did not become ready" >&2 +echo "ClawBench Harbor runtime did not become ready: \ +runtime server, request interceptor, or CDP endpoint never came up" >&2 +tail -n 40 /tmp/clawbench-run/runtime-server.log >&2 || true exit 1 """ From 96b2b3c38821f4e88dfddaefa89f8434c13e0d2c Mon Sep 17 00:00:00 2001 From: vaibhavdabas16 Date: Wed, 9 Sep 2026 20:40:46 +0530 Subject: [PATCH 2/3] fix(harbor): poll for runtime readiness instead of fixed sleeps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit start-runtime.sh slept a fixed 1-2s after starting Xvfb, uvicorn, and Chromium before moving on. Those numbers encode local container-daemon timing; on a remote sandbox the next step can run against a service that is not listening yet, and in remote-browser mode the script declared the CDP bridge ready after a bare `sleep 1`. Each step now polls for the thing it needs — the X socket, the runtime server, Chromium's CDP endpoint — with a bounded timeout that reports which component never came up. CLAWBENCH_RUNTIME_WAIT_TIMEOUT_S overrides the 60s default for slow providers. --- src/clawbench/runtime/harbor/start-runtime.sh | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/clawbench/runtime/harbor/start-runtime.sh b/src/clawbench/runtime/harbor/start-runtime.sh index 43d11a0b..17772433 100644 --- a/src/clawbench/runtime/harbor/start-runtime.sh +++ b/src/clawbench/runtime/harbor/start-runtime.sh @@ -3,6 +3,24 @@ set -euo pipefail mkdir -p /data /tmp/clawbench-run +# Remote sandboxes (e2b, daytona, modal) start slower and less predictably +# than a local container daemon, so every step below polls for the thing it +# needs instead of sleeping a fixed number of seconds and hoping. +WAIT_TIMEOUT_S="${CLAWBENCH_RUNTIME_WAIT_TIMEOUT_S:-60}" + +wait_for_url() { + local url=$1 label=$2 elapsed=0 + while [ "$elapsed" -lt "$WAIT_TIMEOUT_S" ]; do + if curl -sf "$url" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + echo "timed out after ${WAIT_TIMEOUT_S}s waiting for $label ($url)" >&2 + return 1 +} + if [ -f /tmp/clawbench-run/runtime.pid ] && kill -0 "$(cat /tmp/clawbench-run/runtime.pid)" 2>/dev/null; then echo "ClawBench Harbor runtime is already running." exit 0 @@ -20,13 +38,19 @@ if [ "$REMOTE_MODE" = false ]; then export DISPLAY="${DISPLAY:-:99}" Xvfb "$DISPLAY" -screen 0 1920x1080x24 >/tmp/clawbench-run/xvfb.log 2>&1 & echo "$!" > /tmp/clawbench-run/xvfb.pid - sleep 1 + # Xvfb serves no HTTP endpoint to poll; wait for its socket to appear. + for _ in $(seq 1 "$WAIT_TIMEOUT_S"); do + if [ -e "/tmp/.X11-unix/X${DISPLAY#:}" ]; then + break + fi + sleep 1 + done fi cd /app/src/runtime-server uv run --no-sync uvicorn server:app --host 0.0.0.0 --port 7878 >/tmp/clawbench-run/runtime-server.log 2>&1 & echo "$!" > /tmp/clawbench-run/runtime-server.pid -sleep 1 +wait_for_url http://127.0.0.1:7878/api/status "runtime server" if [ "$REMOTE_MODE" = true ]; then # Keep the agent-facing CDP endpoint identical across browser runtimes. @@ -129,13 +153,12 @@ LOAD_EXTS="/app/src/chrome-extension" about:blank >/tmp/clawbench-run/chrome.log 2>&1 & echo "$!" > /tmp/clawbench-run/chrome.pid -sleep 2 +wait_for_url http://127.0.0.1:9222/json/version "Chromium CDP" socat TCP-LISTEN:9223,fork,reuseaddr,bind=0.0.0.0 TCP:127.0.0.1:9222 >/tmp/clawbench-run/socat.log 2>&1 & echo "$!" > /tmp/clawbench-run/socat.pid x11vnc -display "$DISPLAY" -nopw -shared -forever -rfbport 5900 -xkb >/tmp/clawbench-run/x11vnc.log 2>&1 & echo "$!" > /tmp/clawbench-run/x11vnc.pid -sleep 1 /opt/novnc/utils/novnc_proxy --vnc localhost:5900 --listen 6080 >/tmp/clawbench-run/novnc.log 2>&1 & echo "$!" > /tmp/clawbench-run/novnc.pid From adc683a7fbb7b7103870d7387bf8808eb78205b5 Mon Sep 17 00:00:00 2001 From: vaibhavdabas16 Date: Wed, 9 Sep 2026 20:40:47 +0530 Subject: [PATCH 3/3] docs(harbor): document running on remote sandboxes Adds a `-e e2b` section covering what the generated tasks assume about a sandbox (no bind mounts outside the task dir, no local X11 or GPU, polled readiness), the rule that every credential stays an env reference resolved by --env-file/--ve, and the per-trial resource floor. States plainly that the end-to-end e2b smoke run in #331 is still open. Tests assert the healthcheck and setup script share one readiness contract in both local and kernel modes, that startup polls rather than sleeps, and that a generated task carries no host paths or baked secrets. --- CHANGELOG.md | 3 + docs/harbor.md | 29 ++++++++++ tests/test_harbor_adapter.py | 106 +++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb130c2c..6daa7556 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] ### Added +- Documented running the generated Harbor dataset on remote sandboxes (`harbor run -e e2b`), including what the tasks assume about the sandbox, how credentials are passed, and the per-trial resource floor. - Added `scripts/export_openeval.py`, an additive script exporting a batch's `rescore-summary.json` as an [EvalPort](https://github.com/adhabnr-ux/evalport) `ResultSet` Thanks to [@adhabnr-ux](https://github.com/adhabnr-ux). - Added a `--browser-runtime kernel` mode to the Harbor adapter that runs each task against one Kernel cloud browser, exposing only a credential-free CDP bridge to the agent, and finalizes the replay and deletes the browser during verification. @@ -18,6 +19,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - Changed the default Harbor version to `0.22.0`. ### Fixed +- Harbor task setup now waits for the same condition as the step healthcheck — runtime server up, request interceptor armed, and CDP live — instead of accepting any 200 from `/api/status`. The server answers before its CDP handler attaches, so a task could previously start with interception inactive and silently fail to score Stage 1. +- `start-runtime.sh` polls for the runtime server and Chromium's CDP endpoint instead of sleeping a fixed number of seconds, so a slower remote sandbox no longer fails a trial on provisioning latency. - Host-timeout container termination now uses the lazy container-engine resolver. - Added host-side container and batch-job timeouts so a wedged run cannot stall a batch indefinitely. - Fixed a judge-provider outage (or an unparseable judge reply) being recorded as an agent failure. `run.py` now exits 3 instead of 1 when the judge never renders a verdict, `batch.py` gives it its own `judge_inconclusive` bucket in `batch-summary.json` instead of folding it into `failed`, and `clawbench-rescore` now retries a cached `match: null` verdict even without `--force`. diff --git a/docs/harbor.md b/docs/harbor.md index 6373e6aa..95b44d35 100644 --- a/docs/harbor.md +++ b/docs/harbor.md @@ -136,6 +136,35 @@ Generated tasks register a pinned Playwright MCP package (`@playwright/mcp@0.0.7 Export `KERNEL_API_KEY` (and optionally `KERNEL_BASE_URL` for non-production gateways) before `harbor run`; no extra flags are needed. +## Remote sandboxes (`-e e2b`) + +Harbor can run each trial on a remote sandbox provider instead of the local container daemon: + +```bash +uvx --from harbor==0.22.0 harbor run -p ./harbor-datasets/clawbench-v2 \ + -e e2b \ + -a hermes -m deepseek/deepseek-v4-flash \ + --env-file .env \ + --ve CLAWBENCH_JUDGE_BASE_URL="$CLAWBENCH_JUDGE_BASE_URL" \ + --ve CLAWBENCH_JUDGE_API_KEY="$CLAWBENCH_JUDGE_API_KEY" \ + --ve CLAWBENCH_JUDGE_MODEL="${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" \ + --ve CLAWBENCH_JUDGE_API_TYPE="${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" +``` + +The generated tasks are written to be provider-agnostic, so `-e daytona` and `-e modal` take the same shape. + +**What the generated task assumes about its sandbox.** Nothing host-specific: + +- **No bind mounts outside the task directory.** Everything the runtime needs is baked into the environment image or written under `/data`, `/tmp/clawbench-run`, and `/my-info` at setup time. +- **No local X11 or GPU.** Chromium runs headful under Xvfb with SwiftShader (`--use-gl=angle --use-angle=swiftshader`), and `--disable-dev-shm-usage` keeps it off a small `/dev/shm`. +- **Readiness is polled, never slept for.** `start-runtime.sh` waits for the runtime server and Chromium's CDP endpoint to actually answer before continuing, and the step's setup script waits for the same condition Harbor's healthcheck checks — runtime server up, **request interceptor armed**, CDP live. A 200 from `/api/status` alone is not readiness: the server answers before the CDP handler attaches, and a task that starts in that window runs with interception inactive. Override the budgets with `CLAWBENCH_RUNTIME_WAIT_TIMEOUT_S` (default 60s, per component) and `CLAWBENCH_RUNTIME_READY_TIMEOUT_S` (default 180s, overall) if your provider provisions slowly. + +**Credentials never live in the dataset.** Every secret in `task.toml` is an environment reference (`${PURELY_MAIL_API_KEY}`, `${CLAWBENCH_JUDGE_API_KEY}`, `${KERNEL_API_KEY}`), resolved at run time from `--env-file` and `--ve`. Pass them that way and nothing sensitive is committed or shipped to the provider's image registry. + +**Resource floor.** Budget the same **1 CPU core and ~2 GB RAM per concurrent trial** as a local run — a remote sandbox still runs a full Chromium. Providers bill per sandbox-second, so `--timeout-multiplier` and per-task `time_limit` translate directly into cost. + +> **Not yet verified end-to-end.** The e2b smoke run in [#331](https://github.com/TIGER-AI-Lab/ClawBench/issues/331) is still open; this documents what the generated tasks require, not a passing run. + ## Making it fast A full V2 sweep is 129 containerized browser sessions, each capped by the task's `time_limit`. Serial, that is a very long night. What actually moves the needle, in order: diff --git a/tests/test_harbor_adapter.py b/tests/test_harbor_adapter.py index c7afe7b8..83565ecf 100644 --- a/tests/test_harbor_adapter.py +++ b/tests/test_harbor_adapter.py @@ -2,6 +2,7 @@ import json import os +import re import subprocess import sys import textwrap @@ -9,13 +10,17 @@ from pathlib import Path from clawbench.eval.harbor_adapter import ( + LOCAL_CDP_URL, + REMOTE_BRIDGE_CDP_URL, discover_cases, + runtime_ready_command, sanitize_task_name, unique_output_name, write_harbor_task, ) from clawbench.runtime.harbor import verify as harbor_verify from clawbench.runtime.harbor.verify import write_reward +from clawbench.utils.paths import RUNTIME_ROOT SRC_ROOT = Path(__file__).resolve().parents[1] / "src" @@ -285,3 +290,104 @@ def test_harbor_verifier_omits_unknown_judge_match_metric(tmp_path: Path) -> Non reward = json.loads((tmp_path / "reward.json").read_text()) assert reward == {"reward": 0.0, "intercepted": 0.0} + + +def test_healthcheck_and_setup_share_one_readiness_contract(tmp_path: Path) -> None: + """A 200 from /api/status is not readiness: the interceptor must be armed. + + The server answers before the CDP handler attaches, so a task that starts + in that window runs with interception inactive and cannot score Stage 1. + """ + out = write_harbor_task( + task_dir=_write_case(tmp_path / "v2", "v2-047-x", _task()), + task=_task(), + output_root=tmp_path / "out", + output_name="v2-047-x", + org="clawbench", + dataset_name="v2", + ) + + config = tomllib.loads((out / "task.toml").read_text()) + healthcheck = config["steps"][0]["healthcheck"]["command"] + setup = (out / "steps" / "run" / "workdir" / "setup.sh").read_text() + + assert healthcheck == runtime_ready_command(LOCAL_CDP_URL) + assert healthcheck in setup + # Unescaped in the shell script, escaped exactly once in the TOML source. + assert "'\"eval_interceptor_ready\":true'" in setup + assert '\\"eval_interceptor_ready\\":true' in (out / "task.toml").read_text() + + +def test_setup_waits_long_enough_for_a_remote_sandbox(tmp_path: Path) -> None: + out = write_harbor_task( + task_dir=_write_case(tmp_path / "v2", "v2-047-x", _task()), + task=_task(), + output_root=tmp_path / "out", + output_name="v2-047-x", + org="clawbench", + dataset_name="v2", + ) + + setup = (out / "steps" / "run" / "workdir" / "setup.sh").read_text() + + assert "CLAWBENCH_RUNTIME_READY_TIMEOUT_S:-180" in setup + # A timeout must say what failed rather than exiting silently. + assert "runtime-server.log" in setup + + +def test_kernel_setup_uses_the_bridge_readiness_contract(tmp_path: Path) -> None: + out = write_harbor_task( + task_dir=_write_case(tmp_path / "v2", "v2-047-x", _task()), + task=_task(), + output_root=tmp_path / "out", + output_name="v2-047-x", + org="clawbench", + dataset_name="v2", + browser_runtime="kernel", + ) + + config = tomllib.loads((out / "task.toml").read_text()) + setup = (out / "steps" / "run" / "workdir" / "setup.sh").read_text() + + assert config["steps"][0]["healthcheck"]["command"] in setup + assert runtime_ready_command(REMOTE_BRIDGE_CDP_URL) in setup + assert "kernel-browser.py start" in setup + + +def test_runtime_startup_polls_instead_of_sleeping() -> None: + """Fixed sleeps encode local daemon timing; remote sandboxes are slower.""" + script = (RUNTIME_ROOT / "harbor" / "start-runtime.sh").read_text() + + assert "wait_for_url http://127.0.0.1:7878/api/status" in script + assert "wait_for_url http://127.0.0.1:9222/json/version" in script + assert "CLAWBENCH_RUNTIME_WAIT_TIMEOUT_S" in script + # No bare `sleep ` left standing in for a readiness check; the only + # sleeps remaining are the one-second steps inside the polling loops. + assert re.findall(r"^sleep \d+$", script, re.MULTILINE) == [] + + +def test_generated_task_carries_no_host_paths_or_baked_secrets(tmp_path: Path) -> None: + """A remote sandbox has none of the host's filesystem, and no .env.""" + out = write_harbor_task( + task_dir=_write_case(tmp_path / "v2", "v2-047-x", _task()), + task=_task(), + output_root=tmp_path / "out", + output_name="v2-047-x", + org="clawbench", + dataset_name="v2", + ) + + config = tomllib.loads((out / "task.toml").read_text()) + env = config["environment"]["env"] + + # Every credential is a variable reference resolved by --ve/--env-file at + # run time, never a literal baked into the committed dataset. + for key, value in env.items(): + if "KEY" in key or "MAIL" in key: + assert value.startswith("${"), f"{key} is not an env reference: {value}" + + setup = (out / "steps" / "run" / "workdir" / "setup.sh").read_text() + test = (out / "steps" / "run" / "tests" / "test.sh").read_text() + for script in (setup, test): + assert str(tmp_path) not in script + assert "/host" not in script