From b3daf62836c63cfc30351e153a49555eb91e72c4 Mon Sep 17 00:00:00 2001 From: vaibhavdabas16 Date: Wed, 9 Sep 2026 20:45:29 +0530 Subject: [PATCH 1/6] feat(runner): record code, corpus, and agent revisions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run-meta.json recorded what was run — task, model, harness name, image ids — but not the revisions those names resolved to. Two runs of "openclaw on v2" a month apart are indistinguishable in the artifact even when the agent, the corpus, and ClawBench itself have all moved, which makes a published leaderboard row labelled rather than reproducible. This collects the ClawBench version, commit, branch and dirty state; the corpus suite and the revision of the commit that last touched it; and the agent and plugin versions pinned by the harness Dockerfile — reading the pins the image was built from rather than asking a running container. Every lookup is best-effort and returns None rather than raising: a PyPI install has no git repository and a container host may have no git at all, and a missing provenance field must never fail a run that otherwise succeeded. `dirty: None` deliberately means "the lookup failed", which is not the same claim as False. Lookups are cached because a batch run builds one block per task and the answers cannot change within a process. --- .../runner/run_support/provenance.py | 198 ++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 src/clawbench/runner/run_support/provenance.py diff --git a/src/clawbench/runner/run_support/provenance.py b/src/clawbench/runner/run_support/provenance.py new file mode 100644 index 00000000..d6066c72 --- /dev/null +++ b/src/clawbench/runner/run_support/provenance.py @@ -0,0 +1,198 @@ +"""Which exact code, corpus, and agent produced a run. + +`run-meta.json` already records *what* was run — task, model, harness name, +image ids. It does not record the revisions those names resolved to, so two +runs of "openclaw on v2" a month apart are indistinguishable in the artifact +even when the agent, the corpus, and ClawBench itself all moved. + +This module fills that gap: the ClawBench version and commit, the corpus +revision, and the agent versions pinned by the harness image. Every lookup is +best-effort and returns ``None`` rather than raising — a PyPI install has no +git repository, a container host may have no `git` at all, and a missing +provenance field must never fail a run that otherwise succeeded. + +Results are cached because a batch run builds one of these per task and the +answers cannot change inside a single process. +""" + +from __future__ import annotations + +import re +import subprocess +from functools import lru_cache +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Any + +from clawbench.utils.paths import ASSET_ROOT, HARNESS_ROOT, SOURCE_ROOT + +_GIT_TIMEOUT_S = 10 + +# Version pins declared in a harness Dockerfile: `pkg@1.2.3` for npm and +# `pkg==1.2.3` for pip. This reads the pins the image was built from rather +# than asking the agent for its version, which would need a running container. +_NPM_PIN_RE = re.compile(r"(? str | None: + try: + result = subprocess.run( + ["git", "-C", str(repo), *args], + capture_output=True, + text=True, + timeout=_GIT_TIMEOUT_S, + ) + except (OSError, subprocess.SubprocessError): + return None + if result.returncode != 0: + return None + return result.stdout.strip() or None + + +@lru_cache(maxsize=1) +def clawbench_version() -> str | None: + try: + return version("clawbench-eval") + except PackageNotFoundError: + return None + + +@lru_cache(maxsize=1) +def _repo_root() -> Path | None: + """The ClawBench git checkout, when running from source rather than a wheel.""" + if SOURCE_ROOT is None or not (SOURCE_ROOT / ".git").exists(): + return None + return SOURCE_ROOT + + +@lru_cache(maxsize=1) +def clawbench_commit() -> dict[str, Any]: + """Commit, branch, and dirty state of the ClawBench checkout.""" + repo = _repo_root() + if repo is None: + return {"commit": None, "branch": None, "dirty": None} + status = _git(repo, "status", "--porcelain") + return { + "commit": _git(repo, "rev-parse", "HEAD"), + "branch": _git(repo, "rev-parse", "--abbrev-ref", "HEAD"), + # `git status` succeeding with no output means a clean tree; a failed + # lookup returns None, which is not the same claim as "clean". + "dirty": (status != "") if status is not None else None, + } + + +def _relative_to_assets(path: Path) -> str | None: + try: + return path.resolve().relative_to(ASSET_ROOT.resolve()).as_posix() + except (OSError, ValueError): + return None + + +@lru_cache(maxsize=8) +def _corpus_commit(suite_path: str) -> str | None: + """Last commit that touched this corpus directory.""" + repo = _repo_root() + if repo is None: + return None + return _git(repo, "log", "-1", "--format=%H", "--", suite_path) + + +def corpus_meta(task_dir: Path | None) -> dict[str, Any]: + """Which corpus a task came from, and at which revision. + + A task outside the bundled corpora (an explicit ``--cases-dir``) reports + its suite name but no revision — its history is not ClawBench's to claim. + """ + if task_dir is None: + return {"suite": None, "path": None, "revision": None} + relative = _relative_to_assets(task_dir) + if relative is None: + return {"suite": task_dir.parent.name, "path": None, "revision": None} + # e.g. "test-cases/v2/v2-047-daily-life-personal-care-taskrabbit" + parts = relative.split("/") + suite_path = "/".join(parts[:2]) + return { + "suite": parts[1] if len(parts) > 1 else parts[0], + "path": suite_path, + "revision": _corpus_commit(suite_path), + } + + +@lru_cache(maxsize=32) +def harness_pins(harness: str) -> dict[str, str]: + """Agent and plugin versions pinned by a harness Dockerfile. + + Reads the pins the image was built from — `opencode-ai@1.4.4`, + `@playwright/mcp@0.0.70`, `pip install foo==1.2` — so a trace records the + exact agent build it used. Returns an empty mapping for a harness with no + version pins, or one whose Dockerfile cannot be read. + """ + if harness in ("human", ""): + return {} + try: + from clawbench.runner.run_support.harness_registry import HARNESS_REGISTRY + + dockerfile = HARNESS_REGISTRY.harness_dockerfiles.get(harness) + except (ImportError, ValueError): + dockerfile = None + if dockerfile is None: + candidates = sorted(HARNESS_ROOT.glob(f"{harness}/Dockerfile.*")) + dockerfile = candidates[0] if candidates else None + if dockerfile is None or not dockerfile.is_file(): + return {} + try: + text = dockerfile.read_text(encoding="utf-8", errors="replace") + except OSError: + return {} + + pins: dict[str, str] = {} + for line in text.splitlines(): + if _PIN_SKIP_RE.match(line): + continue + for pattern in (_NPM_PIN_RE, _PIP_PIN_RE): + for name, pinned in pattern.findall(line): + pins.setdefault(name, pinned) + return pins + + +def _agent_version(harness: str, pins: dict[str, str]) -> str | None: + """The pin that is the agent itself, not one of its plugins. + + Package names rarely equal the harness name exactly (`opencode` ships as + `opencode-ai`), so fall back to the pin whose package name contains it. + """ + if harness in pins: + return pins[harness] + for name, pinned in pins.items(): + if harness in name: + return pinned + return None + + +def harness_meta(harness: str, image_id: str | None) -> dict[str, Any]: + pins = harness_pins(harness) + return { + "name": harness, + "image_id": image_id, + "pinned_versions": pins or None, + # Named separately because it is the one a leaderboard row cites. + "agent_version": _agent_version(harness, pins), + } + + +def make_provenance( + *, + harness: str, + harness_image_id: str | None, + task_dir: Path | None, +) -> dict[str, Any]: + """The provenance block written into ``run-meta.json``.""" + return { + "clawbench_version": clawbench_version(), + **clawbench_commit(), + "corpus": corpus_meta(task_dir), + "harness": harness_meta(harness, harness_image_id), + } From 993e2e8f2bb1b5e932533be41f10fb8353f58659 Mon Sep 17 00:00:00 2001 From: vaibhavdabas16 Date: Wed, 9 Sep 2026 20:45:29 +0530 Subject: [PATCH 2/6] feat(runner): write the provenance block into run-meta.json Adds `provenance` alongside `runtime`, reusing the harness image id that _runtime_meta already resolves rather than inspecting the image twice. --- src/clawbench/runner/run_support/metadata.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/clawbench/runner/run_support/metadata.py b/src/clawbench/runner/run_support/metadata.py index c7f44522..c816631d 100644 --- a/src/clawbench/runner/run_support/metadata.py +++ b/src/clawbench/runner/run_support/metadata.py @@ -17,6 +17,7 @@ harness_image, ) from clawbench.runner.run_support.docker import container_engine_version, image_id +from clawbench.runner.run_support.provenance import make_provenance from clawbench.runner.run_support.task import normalize_extra_info SECRET_CONFIG_RE = re.compile( @@ -230,6 +231,7 @@ def make_run_meta( temperature = model_cfg.get("temperature") if model_cfg else None max_tokens = model_cfg.get("max_tokens") if model_cfg else None + runtime = _runtime_meta(harness) meta = { "test_case": case_name, **metadata, @@ -252,7 +254,14 @@ def make_run_meta( "infra_flags": classification["infra_flags"], "run_metrics": classification["metrics"], "usage": classification["metrics"].get("usage"), - "runtime": _runtime_meta(harness), + "runtime": runtime, + # Which code, corpus, and agent build produced this trace — the part + # that makes a published row reproducible rather than merely labelled. + "provenance": make_provenance( + harness=harness, + harness_image_id=runtime.get("harness_image_id"), + task_dir=task_dir, + ), "browser_runtime": browser_runtime, "task": _task_meta( task=task, From 1136fee68d329e24d26732a1b5fb788e664c1622 Mon Sep 17 00:00:00 2001 From: vaibhavdabas16 Date: Wed, 9 Sep 2026 20:45:30 +0530 Subject: [PATCH 3/6] docs: document run provenance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a Provenance section to the trace cookbook covering the block's shape, what corpus.revision and harness.pinned_versions actually mean, and why every field can be null — with the filter to apply before comparing runs across code versions. Tests cover pin extraction for each bundled harness, that base-image and COPY lines are not mistaken for agent pins, pip-style pins, corpus resolution for bundled and external case dirs, and that a missing git checkout or git binary yields nulls rather than an exception. --- CHANGELOG.md | 1 + docs/trace-cookbook.md | 43 +++++- tests/test_provenance.py | 208 +++++++++++++++++++++++++++++ tests/test_results_and_metadata.py | 6 + 4 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 tests/test_provenance.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cb130c2c..f82a5d54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] ### Added +- `run-meta.json` now carries a `provenance` block: the ClawBench version, commit, branch, and dirty state; the corpus suite and the revision of the commit that last touched it; and the agent and plugin versions pinned by the harness Dockerfile. Every field is best-effort and null outside a git checkout, so a run never fails on a missing one. See [`docs/trace-cookbook.md`](docs/trace-cookbook.md#provenance). - Added `scripts/export_openeval.py`, an additive script exporting a batch's `rescore-summary.json` as an [EvalPort](https://github.com/adhabnr-ux/evalport) `ResultSet` Thanks to [@adhabnr-ux](https://github.com/adhabnr-ux). - Added a `--browser-runtime kernel` mode to the Harbor adapter that runs each task against one Kernel cloud browser, exposing only a credential-free CDP bridge to the agent, and finalizes the replay and deletes the browser during verification. diff --git a/docs/trace-cookbook.md b/docs/trace-cookbook.md index 8fc0f38e..7aa6e75c 100644 --- a/docs/trace-cookbook.md +++ b/docs/trace-cookbook.md @@ -27,7 +27,7 @@ Each run directory contains: | `screenshots/*.png` | Timestamped PNG per action | Vision grounding, GUI datasets | | `recording.mp4` | Full session video (H.264, 15 fps) | Qualitative analysis, demos | | `interception.json` | The final blocked request | Outcome labels (Stage-1) | -| `run-meta.json` | Model, harness, task, timing | Joins and filtering | +| `run-meta.json` | Model, harness, task, timing, provenance | Joins and filtering | Pull a single model or task without downloading everything: @@ -56,6 +56,47 @@ outcome = json.loads((run / "interception.json").read_text()) print(meta["model"], len(msgs), "messages,", len(acts), "actions") ``` +### Provenance + +`run-meta.json` carries a `provenance` block naming the exact revisions behind +the names in the rest of the file, so a row on a leaderboard can be traced back +to the code, corpus, and agent build that produced it: + +```json +"provenance": { + "clawbench_version": "0.10.0", + "commit": "3f3599d...", + "branch": "main", + "dirty": false, + "corpus": {"suite": "v2", "path": "test-cases/v2", "revision": "62ee923..."}, + "harness": { + "name": "openclaw", + "image_id": "sha256:...", + "agent_version": "2026.3.13", + "pinned_versions": {"openclaw": "2026.3.13"} + } +} +``` + +`corpus.revision` is the last commit that touched that suite, so two runs with +the same revision saw the same task text. `harness.pinned_versions` comes from +the version pins in the harness Dockerfile — the agent and any plugins the +image was built with. + +Every field is best-effort. A run from a PyPI install has no git checkout and +reports `commit: null`; a task from an explicit `--cases-dir` reports its suite +name but `revision: null`, because its history is not ClawBench's to claim. +`dirty: null` means the lookup failed, which is not the same claim as `false`. +Filter on these before comparing runs: + +```python +same_code = { + run for run in runs + if run["provenance"]["commit"] == reference["provenance"]["commit"] + and run["provenance"]["dirty"] is False +} +``` + ## Recipes **1. Agent SFT / distillation data.** `agent-messages.jsonl` from passing runs diff --git a/tests/test_provenance.py b/tests/test_provenance.py new file mode 100644 index 00000000..c85b97ce --- /dev/null +++ b/tests/test_provenance.py @@ -0,0 +1,208 @@ +"""Run provenance: which code, corpus, and agent build produced a trace.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from clawbench.runner.run_support import provenance +from clawbench.utils.paths import ASSET_ROOT + + +@pytest.fixture(autouse=True) +def _clear_provenance_caches() -> None: + for fn in ( + provenance.clawbench_version, + provenance._repo_root, + provenance.clawbench_commit, + provenance._corpus_commit, + provenance.harness_pins, + ): + fn.cache_clear() + + +# --------------------------------------------------------------------------- +# Harness pins +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("harness", "package"), + [ + ("openclaw", "openclaw"), + ("opencode", "opencode-ai"), + ("claude-code", "@anthropic-ai/claude-code"), + ("browser-use", "browser-use"), + ], +) +def test_bundled_harnesses_report_their_pinned_agent( + harness: str, package: str +) -> None: + pins = provenance.harness_pins(harness) + + assert package in pins, f"{harness} Dockerfile pin not detected: {pins}" + assert pins[package][0].isdigit() + assert provenance.harness_meta(harness, None)["agent_version"] == pins[package] + + +def test_base_image_lines_are_not_mistaken_for_agent_pins() -> None: + """`FROM node:24-slim` and `COPY --from=...uv:0.11.6` are not agent pins.""" + pins = provenance.harness_pins("opencode") + + assert "node" not in pins + assert "uv" not in pins + assert pins["@playwright/mcp"] == "0.0.70" + + +def test_harness_without_pins_reports_nothing_rather_than_guessing() -> None: + meta = provenance.harness_meta("null", "sha256:abc") + + assert meta["pinned_versions"] is None + assert meta["agent_version"] is None + assert meta["image_id"] == "sha256:abc" + + +def test_human_and_unknown_harnesses_are_safe() -> None: + assert provenance.harness_pins("human") == {} + assert provenance.harness_pins("not-a-harness") == {} + + +def test_pip_pins_are_detected(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + harness_dir = tmp_path / "demo" + harness_dir.mkdir() + (harness_dir / "Dockerfile.demo").write_text( + "FROM python:3.11-slim\nRUN pip install demo-agent==2.4.1 helper-plugin==0.9\n", + encoding="utf-8", + ) + monkeypatch.setattr(provenance, "HARNESS_ROOT", tmp_path) + + pins = provenance.harness_pins("demo") + + assert pins == {"demo-agent": "2.4.1", "helper-plugin": "0.9"} + assert provenance.harness_meta("demo", None)["agent_version"] == "2.4.1" + + +# --------------------------------------------------------------------------- +# Corpus revision +# --------------------------------------------------------------------------- + + +def test_bundled_task_reports_its_suite_and_path() -> None: + task_dir = ASSET_ROOT / "test-cases" / "v2" / "example-task" + + corpus = provenance.corpus_meta(task_dir) + + assert corpus["suite"] == "v2" + assert corpus["path"] == "test-cases/v2" + + +def test_external_cases_dir_claims_no_revision(tmp_path: Path) -> None: + """An explicit --cases-dir has a history that is not ClawBench's to claim.""" + corpus = provenance.corpus_meta(tmp_path / "my-suite" / "task-1") + + assert corpus["suite"] == "my-suite" + assert corpus["path"] is None + assert corpus["revision"] is None + + +def test_missing_task_dir_is_all_null() -> None: + assert provenance.corpus_meta(None) == { + "suite": None, + "path": None, + "revision": None, + } + + +# --------------------------------------------------------------------------- +# ClawBench revision +# --------------------------------------------------------------------------- + + +def test_commit_lookup_outside_a_checkout_is_null_not_a_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A PyPI install has no git repository; that must not fail a run.""" + monkeypatch.setattr(provenance, "SOURCE_ROOT", None) + provenance._repo_root.cache_clear() + provenance.clawbench_commit.cache_clear() + + assert provenance.clawbench_commit() == { + "commit": None, + "branch": None, + "dirty": None, + } + + +def test_missing_git_binary_is_null_not_a_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def no_git(*args: object, **kwargs: object) -> None: + raise FileNotFoundError("git") + + monkeypatch.setattr(subprocess, "run", no_git) + + assert provenance._git(Path("."), "rev-parse", "HEAD") is None + + +def test_dirty_is_null_when_the_lookup_itself_failed( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Unknown is not the same claim as clean.""" + monkeypatch.setattr(provenance, "_repo_root", lambda: tmp_path) + monkeypatch.setattr(provenance, "_git", lambda *args: None) + provenance.clawbench_commit.cache_clear() + + assert provenance.clawbench_commit()["dirty"] is None + + +def test_clean_and_dirty_trees_are_distinguished( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(provenance, "_repo_root", lambda: tmp_path) + monkeypatch.setattr( + provenance, + "_git", + lambda repo, *args: "" if args[0] == "status" else "abc123", + ) + provenance.clawbench_commit.cache_clear() + assert provenance.clawbench_commit()["dirty"] is False + + monkeypatch.setattr( + provenance, + "_git", + lambda repo, *args: " M file" if args[0] == "status" else "abc123", + ) + provenance.clawbench_commit.cache_clear() + assert provenance.clawbench_commit()["dirty"] is True + + +# --------------------------------------------------------------------------- +# The assembled block +# --------------------------------------------------------------------------- + + +def test_provenance_block_has_a_stable_shape() -> None: + block = provenance.make_provenance( + harness="openclaw", + harness_image_id="sha256:abc", + task_dir=ASSET_ROOT / "test-cases" / "v2" / "example-task", + ) + + assert set(block) == { + "clawbench_version", + "commit", + "branch", + "dirty", + "corpus", + "harness", + } + assert set(block["corpus"]) == {"suite", "path", "revision"} + assert set(block["harness"]) == { + "name", + "image_id", + "pinned_versions", + "agent_version", + } + assert block["harness"]["name"] == "openclaw" diff --git a/tests/test_results_and_metadata.py b/tests/test_results_and_metadata.py index f407ed76..6a4ee138 100644 --- a/tests/test_results_and_metadata.py +++ b/tests/test_results_and_metadata.py @@ -258,3 +258,9 @@ def test_run_metadata_redacts_model_and_judge_secrets( assert meta["browser_runtime"]["cleanup_status"] == "released" assert meta["usage"]["estimated_cost_usd"] == 0.0042 assert meta["run_metrics"]["usage"]["total_tokens"] == 123 + + provenance = meta["provenance"] + assert provenance["harness"]["name"] == "openclaw" + assert provenance["harness"]["agent_version"] is not None + assert provenance["corpus"]["suite"] == "v1" + assert provenance["harness"]["image_id"] == meta["runtime"]["harness_image_id"] From b7bc2e053fc1fa6626b70dc85e8061adfb84c162 Mon Sep 17 00:00:00 2001 From: vaibhavdabas16 Date: Thu, 10 Sep 2026 11:00:13 +0530 Subject: [PATCH 4/6] fix(provenance): address review on #347 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four defects Perry2004 caught, all real: `dirty` could never be False. `_git()` collapsed empty output into None, so a clean `git status --porcelain` — which succeeds with no output — was indistinguishable from a failed lookup. `_git()` now returns None only when the command could not run, and "" when it ran and said nothing; callers that want a non-empty value ask for it explicitly. Dockerfile pins were reported as fact even under --no-build, where the image can be arbitrarily older than the Dockerfile on disk. Pins are now claimed only when this run built the image, with a `pins_source` field of "dockerfile" or "unverified" saying which. The signal is set by docker_build() rather than read off the --no-build flag, because clawbench-batch builds once and then runs every child with --no-build — keying off the flag would have marked the entire batch path unverified. Pip extras dropped the pin entirely: `litellm[proxy]==1.77.3` matched nothing at all, so the pin vanished silently rather than being recorded. Only `name@version` was recognised, so revision pins — `pkg@github:o/r#sha`, `pkg@git+https://…#ref`, and pip's PEP 508 `pkg @ git+…@ref` — were missed. That is exactly how a preview-stage agent like DeepSeek Harness is pinned, which is the case #309 needs. A floating dist-tag (`@next`) is still not collected: it names a moving target, so recording it as a pin would be a false claim. --- src/clawbench/runner/run_support/docker.py | 6 ++ .../runner/run_support/provenance.py | 77 +++++++++++++++---- 2 files changed, 70 insertions(+), 13 deletions(-) diff --git a/src/clawbench/runner/run_support/docker.py b/src/clawbench/runner/run_support/docker.py index 82a3ff32..b08e75de 100644 --- a/src/clawbench/runner/run_support/docker.py +++ b/src/clawbench/runner/run_support/docker.py @@ -24,6 +24,7 @@ engine, harness_image, ) +from clawbench.runner.run_support.provenance import IMAGE_BUILT_ENV from clawbench.runner.run_support.usage import ( fetch_openrouter_pricing, format_usage_status, @@ -248,6 +249,11 @@ def docker_build(harness: str = DEFAULT_HARNESS) -> None: _build_one(BASE_DOCKERFILE, BASE_IMAGE) _build_one(_HARNESS_DOCKERFILES[harness], target_image) + # Record that this image really was built from the Dockerfile in this + # checkout, so run provenance can tell its version pins apart from a + # possibly-stale image reused via --no-build. clawbench-batch builds once + # here and its child runs inherit the environment. + os.environ[IMAGE_BUILT_ENV] = harness console.print(f"[green]✓[/] Container image ready ({target_image})") diff --git a/src/clawbench/runner/run_support/provenance.py b/src/clawbench/runner/run_support/provenance.py index d6066c72..7bf7f0fd 100644 --- a/src/clawbench/runner/run_support/provenance.py +++ b/src/clawbench/runner/run_support/provenance.py @@ -17,6 +17,7 @@ from __future__ import annotations +import os import re import subprocess from functools import lru_cache @@ -28,16 +29,42 @@ _GIT_TIMEOUT_S = 10 -# Version pins declared in a harness Dockerfile: `pkg@1.2.3` for npm and -# `pkg==1.2.3` for pip. This reads the pins the image was built from rather -# than asking the agent for its version, which would need a running container. -_NPM_PIN_RE = re.compile(r"(? · pkg@git+https://…# +# pip pkg==1.2.3 · pkg[extra]==1.2.3 · pkg @ git+https://…@ +# +# A floating dist-tag (`pkg@latest`, `pkg@next`) is deliberately NOT collected: +# it names a moving target, so recording it as a pin would be a false claim. +_VERSION_OR_REVISION = r"\d[\w.+-]*|(?:github:|git\+)[^\s\"']+" +_NPM_PIN_RE = re.compile( + r"(? str | None: + """Run a git command, or return ``None`` if it could not run. + + ``None`` means *the lookup failed*. A command that succeeded with no + output returns ``""`` — for ``git status --porcelain`` that empty string + is the meaningful answer "clean", so it must not be folded into ``None``. + """ try: result = subprocess.run( ["git", "-C", str(repo), *args], @@ -49,7 +76,7 @@ def _git(repo: Path, *args: str) -> str | None: return None if result.returncode != 0: return None - return result.stdout.strip() or None + return result.stdout.strip() @lru_cache(maxsize=1) @@ -76,10 +103,12 @@ def clawbench_commit() -> dict[str, Any]: return {"commit": None, "branch": None, "dirty": None} status = _git(repo, "status", "--porcelain") return { - "commit": _git(repo, "rev-parse", "HEAD"), - "branch": _git(repo, "rev-parse", "--abbrev-ref", "HEAD"), - # `git status` succeeding with no output means a clean tree; a failed - # lookup returns None, which is not the same claim as "clean". + # An empty commit or branch would mean a successful lookup that said + # nothing, which is no more useful than a failed one. + "commit": _git(repo, "rev-parse", "HEAD") or None, + "branch": _git(repo, "rev-parse", "--abbrev-ref", "HEAD") or None, + # "" is a clean tree; None is a lookup that failed, which is not the + # same claim as clean. "dirty": (status != "") if status is not None else None, } @@ -97,7 +126,7 @@ def _corpus_commit(suite_path: str) -> str | None: repo = _repo_root() if repo is None: return None - return _git(repo, "log", "-1", "--format=%H", "--", suite_path) + return _git(repo, "log", "-1", "--format=%H", "--", suite_path) or None def corpus_meta(task_dir: Path | None) -> dict[str, Any]: @@ -152,9 +181,9 @@ def harness_pins(harness: str) -> dict[str, str]: for line in text.splitlines(): if _PIN_SKIP_RE.match(line): continue - for pattern in (_NPM_PIN_RE, _PIP_PIN_RE): + for pattern in _PIN_PATTERNS: for name, pinned in pattern.findall(line): - pins.setdefault(name, pinned) + pins.setdefault(name.strip(), pinned) return pins @@ -172,7 +201,28 @@ def _agent_version(harness: str, pins: dict[str, str]) -> str | None: return None +def image_built_from_dockerfile(harness: str) -> bool: + """Whether the image this run uses was built from the Dockerfile we read. + + With ``--no-build`` the container image can be arbitrarily older than the + Dockerfile on disk, so its pins are not evidence of what actually ran. + ``docker_build()`` records the harness it built; ``clawbench-batch`` builds + once and its children inherit that environment, so a batch run still + reports real pins while a bare ``--no-build`` run does not. + """ + return os.environ.get(IMAGE_BUILT_ENV) == harness + + def harness_meta(harness: str, image_id: str | None) -> dict[str, Any]: + if not image_built_from_dockerfile(harness): + # Claim nothing rather than report a pin the running image may not have. + return { + "name": harness, + "image_id": image_id, + "pinned_versions": None, + "agent_version": None, + "pins_source": "unverified", + } pins = harness_pins(harness) return { "name": harness, @@ -180,6 +230,7 @@ def harness_meta(harness: str, image_id: str | None) -> dict[str, Any]: "pinned_versions": pins or None, # Named separately because it is the one a leaderboard row cites. "agent_version": _agent_version(harness, pins), + "pins_source": "dockerfile", } From 373f10bcf9097f57249d8ce40a22e0576cc34c93 Mon Sep 17 00:00:00 2001 From: vaibhavdabas16 Date: Thu, 10 Sep 2026 11:00:13 +0530 Subject: [PATCH 5/6] test(provenance): exercise real git instead of mocking it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The clean/dirty test monkeypatched `_git` to return "", bypassing the very `or None` conversion that made `dirty` unable to ever be False — the mock asserted the intended behaviour while the real code did the opposite. It now runs against a real temporary repository, and fails against the old implementation. Adds coverage for pip extras, all three revision-pin spellings, dist-tags being excluded, URL userinfo not being mistaken for a pin, and pins being claimed only for a harness whose image this run built. --- tests/test_provenance.py | 188 ++++++++++++++++++++++++++--- tests/test_results_and_metadata.py | 4 +- 2 files changed, 171 insertions(+), 21 deletions(-) diff --git a/tests/test_provenance.py b/tests/test_provenance.py index c85b97ce..0d5cabcb 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -23,6 +23,41 @@ def _clear_provenance_caches() -> None: fn.cache_clear() +@pytest.fixture +def built_image(monkeypatch: pytest.MonkeyPatch): + """Mark a harness image as built from the Dockerfile in this checkout.""" + + def mark(harness: str) -> None: + monkeypatch.setenv(provenance.IMAGE_BUILT_ENV, harness) + + return mark + + +def _git_repo(path: Path) -> Path: + """A real git repository — mocking `_git` is what hid the dirty-flag bug.""" + path.mkdir(parents=True, exist_ok=True) + for args in ( + ["init", "-q", "."], + [ + "-c", + "user.email=t@example.test", + "-c", + "user.name=t", + "commit", + "-q", + "--allow-empty", + "-m", + "init", + ], + ): + result = subprocess.run( + ["git", "-C", str(path), *args], capture_output=True, text=True + ) + if result.returncode != 0: + pytest.skip(f"git unavailable: {result.stderr.strip()}") + return path + + # --------------------------------------------------------------------------- # Harness pins # --------------------------------------------------------------------------- @@ -38,8 +73,9 @@ def _clear_provenance_caches() -> None: ], ) def test_bundled_harnesses_report_their_pinned_agent( - harness: str, package: str + harness: str, package: str, built_image ) -> None: + built_image(harness) pins = provenance.harness_pins(harness) assert package in pins, f"{harness} Dockerfile pin not detected: {pins}" @@ -56,12 +92,42 @@ def test_base_image_lines_are_not_mistaken_for_agent_pins() -> None: assert pins["@playwright/mcp"] == "0.0.70" -def test_harness_without_pins_reports_nothing_rather_than_guessing() -> None: +def test_harness_without_pins_reports_nothing_rather_than_guessing( + built_image, +) -> None: + built_image("null") meta = provenance.harness_meta("null", "sha256:abc") assert meta["pinned_versions"] is None assert meta["agent_version"] is None assert meta["image_id"] == "sha256:abc" + # We did look; the Dockerfile simply pins nothing. + assert meta["pins_source"] == "dockerfile" + + +def test_reused_image_claims_no_pins(monkeypatch: pytest.MonkeyPatch) -> None: + """--no-build can run an image far older than the Dockerfile on disk. + + Its pins are then not evidence of what actually ran, so nothing is claimed. + """ + monkeypatch.delenv(provenance.IMAGE_BUILT_ENV, raising=False) + + meta = provenance.harness_meta("openclaw", "sha256:abc") + + assert meta["pins_source"] == "unverified" + assert meta["pinned_versions"] is None + assert meta["agent_version"] is None + # The image id is still a fact about the run, and is still reported. + assert meta["image_id"] == "sha256:abc" + + +def test_pins_are_claimed_for_the_built_harness_only( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(provenance.IMAGE_BUILT_ENV, "openclaw") + + assert provenance.harness_meta("openclaw", None)["pins_source"] == "dockerfile" + assert provenance.harness_meta("codex", None)["pins_source"] == "unverified" def test_human_and_unknown_harnesses_are_safe() -> None: @@ -69,14 +135,21 @@ def test_human_and_unknown_harnesses_are_safe() -> None: assert provenance.harness_pins("not-a-harness") == {} -def test_pip_pins_are_detected(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def _demo_harness(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, body: str) -> None: harness_dir = tmp_path / "demo" - harness_dir.mkdir() - (harness_dir / "Dockerfile.demo").write_text( + harness_dir.mkdir(exist_ok=True) + (harness_dir / "Dockerfile.demo").write_text(body, encoding="utf-8") + monkeypatch.setattr(provenance, "HARNESS_ROOT", tmp_path) + monkeypatch.setenv(provenance.IMAGE_BUILT_ENV, "demo") + provenance.harness_pins.cache_clear() + + +def test_pip_pins_are_detected(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _demo_harness( + tmp_path, + monkeypatch, "FROM python:3.11-slim\nRUN pip install demo-agent==2.4.1 helper-plugin==0.9\n", - encoding="utf-8", ) - monkeypatch.setattr(provenance, "HARNESS_ROOT", tmp_path) pins = provenance.harness_pins("demo") @@ -84,6 +157,72 @@ def test_pip_pins_are_detected(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) assert provenance.harness_meta("demo", None)["agent_version"] == "2.4.1" +def test_pip_extras_do_not_drop_the_pin( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """`litellm[proxy]==1.77.3` used to match nothing, losing the pin silently.""" + _demo_harness( + tmp_path, + monkeypatch, + 'FROM python:3.11-slim\nRUN pip install "litellm[proxy,extra]==1.77.3"\n', + ) + + assert provenance.harness_pins("demo") == {"litellm[proxy,extra]": "1.77.3"} + + +@pytest.mark.parametrize( + ("line", "expected"), + [ + ( + "RUN npm install -g demo@github:acme/demo#a1b2c3d4e5f6a7b8", + {"demo": "github:acme/demo#a1b2c3d4e5f6a7b8"}, + ), + ( + "RUN npm install -g demo@git+https://github.com/acme/demo.git#v1.2.3", + {"demo": "git+https://github.com/acme/demo.git#v1.2.3"}, + ), + ( + 'RUN pip install "demo @ git+https://github.com/acme/demo@abc1234"', + {"demo": "git+https://github.com/acme/demo@abc1234"}, + ), + ], +) +def test_revision_pins_are_detected( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + line: str, + expected: dict[str, str], +) -> None: + """A preview-stage agent is pinned to a commit, not a released version.""" + _demo_harness(tmp_path, monkeypatch, f"FROM python:3.11-slim\n{line}\n") + + assert provenance.harness_pins("demo") == expected + + +def test_floating_dist_tags_are_not_recorded_as_pins( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """`@next` names a moving target; calling it a pin would be a false claim.""" + _demo_harness( + tmp_path, monkeypatch, "FROM node:24-slim\nRUN npm install -g demo@next\n" + ) + + assert provenance.harness_pins("demo") == {} + assert provenance.harness_meta("demo", None)["agent_version"] is None + + +def test_url_userinfo_is_not_mistaken_for_a_pin( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _demo_harness( + tmp_path, + monkeypatch, + "FROM python:3.11-slim\nRUN curl -sf https://user@host.example/x\n", + ) + + assert provenance.harness_pins("demo") == {} + + # --------------------------------------------------------------------------- # Corpus revision # --------------------------------------------------------------------------- @@ -157,23 +296,30 @@ def test_dirty_is_null_when_the_lookup_itself_failed( assert provenance.clawbench_commit()["dirty"] is None +def test_git_distinguishes_empty_output_from_failure(tmp_path: Path) -> None: + """A clean `git status` succeeds with no output; that is an answer. + + Folding "" into None here is what made `dirty` unable to ever be False. + """ + repo = _git_repo(tmp_path / "repo") + + assert provenance._git(repo, "status", "--porcelain") == "" + assert provenance._git(repo, "rev-parse", "HEAD") + assert provenance._git(repo, "not-a-git-command") is None + + def test_clean_and_dirty_trees_are_distinguished( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - monkeypatch.setattr(provenance, "_repo_root", lambda: tmp_path) - monkeypatch.setattr( - provenance, - "_git", - lambda repo, *args: "" if args[0] == "status" else "abc123", - ) + repo = _git_repo(tmp_path / "repo") + monkeypatch.setattr(provenance, "_repo_root", lambda: repo) + provenance.clawbench_commit.cache_clear() - assert provenance.clawbench_commit()["dirty"] is False + clean = provenance.clawbench_commit() + assert clean["dirty"] is False + assert clean["commit"] - monkeypatch.setattr( - provenance, - "_git", - lambda repo, *args: " M file" if args[0] == "status" else "abc123", - ) + (repo / "changed.txt").write_text("edited", encoding="utf-8") provenance.clawbench_commit.cache_clear() assert provenance.clawbench_commit()["dirty"] is True @@ -183,7 +329,8 @@ def test_clean_and_dirty_trees_are_distinguished( # --------------------------------------------------------------------------- -def test_provenance_block_has_a_stable_shape() -> None: +def test_provenance_block_has_a_stable_shape(built_image) -> None: + built_image("openclaw") block = provenance.make_provenance( harness="openclaw", harness_image_id="sha256:abc", @@ -204,5 +351,6 @@ def test_provenance_block_has_a_stable_shape() -> None: "image_id", "pinned_versions", "agent_version", + "pins_source", } assert block["harness"]["name"] == "openclaw" diff --git a/tests/test_results_and_metadata.py b/tests/test_results_and_metadata.py index 6a4ee138..a4b65d69 100644 --- a/tests/test_results_and_metadata.py +++ b/tests/test_results_and_metadata.py @@ -261,6 +261,8 @@ def test_run_metadata_redacts_model_and_judge_secrets( provenance = meta["provenance"] assert provenance["harness"]["name"] == "openclaw" - assert provenance["harness"]["agent_version"] is not None assert provenance["corpus"]["suite"] == "v1" assert provenance["harness"]["image_id"] == meta["runtime"]["harness_image_id"] + # This run reused an existing image, so its pins are not claimed. + assert provenance["harness"]["pins_source"] == "unverified" + assert provenance["harness"]["agent_version"] is None From 78754abe84bc49573d34096d81cfc54fe438ca0b Mon Sep 17 00:00:00 2001 From: vaibhavdabas16 Date: Thu, 10 Sep 2026 11:00:14 +0530 Subject: [PATCH 6/6] docs: document pins_source and what counts as a pin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records which pin spellings are collected and why a floating dist-tag is not, and explains when pinned_versions describes the image that actually ran — including that a batch run reports "dockerfile" while a bare clawbench-run --no-build reports "unverified". --- CHANGELOG.md | 2 +- docs/trace-cookbook.md | 26 ++++++++++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f82a5d54..c2b86aca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] ### Added -- `run-meta.json` now carries a `provenance` block: the ClawBench version, commit, branch, and dirty state; the corpus suite and the revision of the commit that last touched it; and the agent and plugin versions pinned by the harness Dockerfile. Every field is best-effort and null outside a git checkout, so a run never fails on a missing one. See [`docs/trace-cookbook.md`](docs/trace-cookbook.md#provenance). +- `run-meta.json` now carries a `provenance` block: the ClawBench version, commit, branch, and dirty state; the corpus suite and the revision of the commit that last touched it; and the agent and plugin versions pinned by the harness Dockerfile, with a `pins_source` saying whether those pins describe the image that actually ran. Every field is best-effort and null outside a git checkout, so a run never fails on a missing one. See [`docs/trace-cookbook.md`](docs/trace-cookbook.md#provenance). - Added `scripts/export_openeval.py`, an additive script exporting a batch's `rescore-summary.json` as an [EvalPort](https://github.com/adhabnr-ux/evalport) `ResultSet` Thanks to [@adhabnr-ux](https://github.com/adhabnr-ux). - Added a `--browser-runtime kernel` mode to the Harbor adapter that runs each task against one Kernel cloud browser, exposing only a credential-free CDP bridge to the agent, and finalizes the replay and deletes the browser during verification. diff --git a/docs/trace-cookbook.md b/docs/trace-cookbook.md index 7aa6e75c..8629f381 100644 --- a/docs/trace-cookbook.md +++ b/docs/trace-cookbook.md @@ -73,15 +73,33 @@ to the code, corpus, and agent build that produced it: "name": "openclaw", "image_id": "sha256:...", "agent_version": "2026.3.13", - "pinned_versions": {"openclaw": "2026.3.13"} + "pinned_versions": {"openclaw": "2026.3.13"}, + "pins_source": "dockerfile" } } ``` `corpus.revision` is the last commit that touched that suite, so two runs with -the same revision saw the same task text. `harness.pinned_versions` comes from -the version pins in the harness Dockerfile — the agent and any plugins the -image was built with. +the same revision saw the same task text. + +`harness.pinned_versions` comes from the version pins in the harness Dockerfile +— the agent and any plugins the image was built with. Both released versions +(`opencode-ai@1.4.4`, `litellm[proxy]==1.77.3`) and pinned revisions +(`pkg@github:owner/repo#`, `pkg @ git+https://…@`) count. A floating +dist-tag like `@next` is deliberately **not** recorded: it names a moving +target, so calling it a pin would be a false claim, and `agent_version` is +`null` for a harness pinned that way. + +`pins_source` says whether those pins describe the image that actually ran: + +| Value | Meaning | +|---|---| +| `"dockerfile"` | The image was built from this checkout's Dockerfile during this run, so its pins are the versions that ran. | +| `"unverified"` | The run reused an existing image (`--no-build`) that may predate the Dockerfile on disk. `pinned_versions` and `agent_version` are `null` — nothing is claimed. | + +`clawbench-batch` builds the image once and then runs every task with +`--no-build`, so a batch run still reports `"dockerfile"`; a bare +`clawbench-run --no-build` reports `"unverified"`. Every field is best-effort. A run from a PyPI install has no git checkout and reports `commit: null`; a task from an explicit `--cases-dir` reports its suite