fix(metabench): isolate native probes and harden benchmark reporting

.cargo/mutants.linux.toml

@@ -45,6 +45,7 @@ profile = "mutants"
 # Only run unit tests and integration tests, not doctests, benchmarks or examples.
 # Retain fatal-path tests needed to detect mutations in termination guards;
 # skip the remaining intentional-abort tests only during mutation runs.
+# Exact metabench deadlines use FakeClock/FakeProcess; keep real-time stress outside mutation runs.
 additional_cargo_test_args = [
     "--locked", "--tests", "--",
     "--skip", "abort::tests::subprocess_oracle_rejects_normal_return",
@@ -55,9 +56,17 @@ additional_cargo_test_args = [
     "--skip", "observation::tests::poisoned_publisher_aborts_instead_of_overwriting_failed_state",
     "--skip", "observation::tests::generation_overflow_aborts_instead_of_reusing_a_lease_identity",
     "--skip", "thread::tests::poisoned_pool_aborts_before_lending_an_owner",
+    "--skip", "runner::tests::captured_logs_bound_sustained_output_and_timeout_cleanup",
 ]
 
 exclude_re = [
+    # Windows and unsupported-platform pipe adapters are compiled out on Linux.
+    'crates/metabench/src/worker_log\.rs:\d+:\d+: (?:.* in (?:read_available_windows|prepare_reader_unsupported|read_available_unsupported)$|replace (?:read_available_windows|prepare_reader_unsupported|read_available_unsupported) ->)',
+    # Windows metadata admission is compiled out on Linux.
+    '\bin reject_windows_reparse_point$',
+    '\breplace reject_windows_reparse_point ->',
+    # O_NOFOLLOW and O_NONBLOCK have disjoint bits, so XOR is equivalent to OR.
+    'crates/metabench/src/artifact.rs:\d+:\d+: replace \| with \^ in open_regular_file$',
     # Shared exclusions copied from mutants.toml.
     # There are no API contracts for what these must return because they
     # are for human-readable stringification. No purpose mutating them.

.cargo/mutants.toml

@@ -34,6 +34,7 @@ profile = "mutants"
 # Only run unit tests and integration tests, not doctests, benchmarks or examples.
 # Retain fatal-path tests needed to detect mutations in termination guards;
 # skip the remaining intentional-abort tests only during mutation runs.
+# Exact metabench deadlines use FakeClock/FakeProcess; keep real-time stress outside mutation runs.
 additional_cargo_test_args = [
     "--locked", "--tests", "--",
     "--skip", "abort::tests::subprocess_oracle_rejects_normal_return",
@@ -44,9 +45,12 @@ additional_cargo_test_args = [
     "--skip", "observation::tests::poisoned_publisher_aborts_instead_of_overwriting_failed_state",
     "--skip", "observation::tests::generation_overflow_aborts_instead_of_reusing_a_lease_identity",
     "--skip", "thread::tests::poisoned_pool_aborts_before_lending_an_owner",
+    "--skip", "runner::tests::captured_logs_bound_sustained_output_and_timeout_cleanup",
 ]
 
 exclude_re = [
+    # O_NOFOLLOW and O_NONBLOCK have disjoint bits, so XOR is equivalent to OR.
+    'crates/metabench/src/artifact.rs:\d+:\d+: replace \| with \^ in open_regular_file$',
     # There are no API contracts for what these must return because they
     # are for human-readable stringification. No purpose mutating them.
     "impl Debug",

.cargo/mutants.windows.toml

@@ -40,6 +40,7 @@ profile = "mutants"
 # Only run unit tests and integration tests, not doctests, benchmarks or examples.
 # Retain fatal-path tests needed to detect mutations in termination guards;
 # skip the remaining intentional-abort tests only during mutation runs.
+# Exact metabench deadlines use FakeClock/FakeProcess; keep real-time stress outside mutation runs.
 additional_cargo_test_args = [
     "--locked", "--tests", "--",
     "--skip", "abort::tests::subprocess_oracle_rejects_normal_return",
@@ -50,9 +51,14 @@ additional_cargo_test_args = [
     "--skip", "observation::tests::poisoned_publisher_aborts_instead_of_overwriting_failed_state",
     "--skip", "observation::tests::generation_overflow_aborts_instead_of_reusing_a_lease_identity",
     "--skip", "thread::tests::poisoned_pool_aborts_before_lending_an_owner",
+    "--skip", "runner::tests::captured_logs_bound_sustained_output_and_timeout_cleanup",
 ]
 
 exclude_re = [
+    # Unix and unsupported-platform pipe adapters are compiled out on Windows.
+    'crates/metabench/src/worker_log\.rs:\d+:\d+: (?:.* in (?:prepare_reader_unix|prepare_reader_with|read_available_unix|prepare_reader_unsupported|read_available_unsupported)$|replace (?:prepare_reader_unix|prepare_reader_with|read_available_unix|prepare_reader_unsupported|read_available_unsupported) ->)',
+    # Unix artifact-open flags are compiled out on Windows.
+    'crates/metabench/src/artifact.rs:\d+:\d+: replace \| with (\^|&) in open_regular_file$',
     # Shared exclusions copied from mutants.toml.
     # There are no API contracts for what these must return because they
     # are for human-readable stringification. No purpose mutating them.

.github/workflows/repository-checks.yml

@@ -103,10 +103,41 @@ jobs:
       - name: Test no_std with serde
         run: cargo +1.95 test --package internity --no-default-features --features serde --test no_std_api --locked
 
+  metabench-native-contracts:
+    name: Metabench native contracts (Linux)
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      CARGO_BUILD_JOBS: "2"
+      METABENCH_REQUIRE_NATIVE_CONTRACTS: "1"
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: ./.github/actions/anvil-setup
+        with:
+          group: none
+          free-disk-space: "true"
+      - name: Install Rust
+        run: just anvil-toolchain-stable-install
+      - name: Install native prerequisites
+        run: |
+          mkdir -p "$GITHUB_WORKSPACE/target/metabench-contract-scratch"
+          export TMPDIR="$GITHUB_WORKSPACE/target/metabench-contract-scratch"
+          sudo apt-get update --quiet
+          sudo apt-get install --yes --quiet valgrind
+          cargo +1.95 install --locked --version 0.19.4 gungraun-runner
+          gungraun-runner --version
+          valgrind --version
+      - name: Run native listing and smoke contracts
+        run: |
+          mkdir -p "$GITHUB_WORKSPACE/target/metabench-contract-scratch"
+          export TMPDIR="$GITHUB_WORKSPACE/target/metabench-contract-scratch"
+          cargo +1.95 test --package metabench --test spawned_benchmark --locked gungraun_listing_is_forwarded -- --exact --nocapture
+          cargo +1.95 test --package metabench --test spawned_benchmark --locked default_engine_smoke_checks_do_not_publish_measurements -- --exact --nocapture
+
   required-repository-checks:
     name: Required repository checks
     if: always()
-    needs: [release-script-tests, simd-no-std-arm, simd-no-std-x86, internity-32-bit-handles, internity-no-std-serde]
+    needs: [release-script-tests, simd-no-std-arm, simd-no-std-x86, internity-32-bit-handles, internity-no-std-serde, metabench-native-contracts]
     runs-on: ubuntu-latest
     steps:
       - name: Verify repository checks
@@ -116,6 +147,7 @@ jobs:
           SIMD_NO_STD_X86: ${{ needs.simd-no-std-x86.result }}
           INTERNITY_32_BIT_HANDLES: ${{ needs.internity-32-bit-handles.result }}
           INTERNITY_NO_STD_SERDE: ${{ needs.internity-no-std-serde.result }}
+          METABENCH_NATIVE_CONTRACTS: ${{ needs.metabench-native-contracts.result }}
         run: |
           if [[ "$RELEASE_SCRIPT_TESTS" != "success" ]]; then
             echo "::error::release-script-tests concluded $RELEASE_SCRIPT_TESTS"
@@ -137,3 +169,7 @@ jobs:
             echo "::error::internity-no-std-serde concluded $INTERNITY_NO_STD_SERDE"
             exit 1
           fi
+          if [[ "$METABENCH_NATIVE_CONTRACTS" != "success" ]]; then
+            echo "::error::metabench-native-contracts concluded $METABENCH_NATIVE_CONTRACTS"
+            exit 1
+          fi

Cargo.toml

@@ -124,6 +124,7 @@ deadpool = { version = "0.13.0", default-features = false }
 derive_more = { version = "2.0.1", default-features = false }
 duct = { version = "1.1.1", default-features = false }
 dynosaur = { version = "0.3.0", default-features = false }
+either-or-both = { version = "0.3", default-features = false }
 events_once = { version = "0.5.32", default-features = false }
 fakeable = { path = "crates/fakeable", default-features = false, version = "0.2.0" }
 fakeable_impl = { path = "crates/fakeable_impl", default-features = false, version = "=0.2.0" }
@@ -148,7 +149,7 @@ futures-util = { version = "0.3.31", default-features = false }
 getrandom = { version = "0.4.3", default-features = false }
 # Keep the library and summary parser aligned with gungraun-runner's schema.
 gungraun = { version = "=0.19.4", default-features = false }
-gungraun-summary = { version = "=6.0.0", default-features = false }
+gungraun-runner = { version = "=0.19.4", default-features = false }
 h3 = { version = "0.0.8", default-features = false }
 h3-quinn = { version = "0.0.10", default-features = false }
 hashbrown = { version = "0.17.0", default-features = false }

crates/arty/benches/arty_contention.rs

@@ -228,16 +228,16 @@ fn criterion_benchmarks(criterion: &mut Criterion) {
 }
 
 #[benchmark(ARTY, "arty_contention/relocated_cache", "Arty")]
-#[bench::concurrency_w1(&mut ArtyCase::new(1), 10)]
-#[bench::concurrency_w8(&mut ArtyCase::new(8), 10)]
-fn arty_workload(case: &mut ArtyCase, concurrency: u64) -> Duration {
+#[bench::concurrency_w1(&ArtyCase::new(1), 10)]
+#[bench::concurrency_w8(&ArtyCase::new(8), 10)]
+fn arty_workload(case: &ArtyCase, concurrency: u64) -> Duration {
     case.run(usize::try_from(concurrency).expect("benchmark concurrency fits in usize"))
 }
 
 #[benchmark(TOKIO, "arty_contention/relocated_cache", "Tokio")]
-#[bench::concurrency_w1(&mut TokioCase::new(1), 10)]
-#[bench::concurrency_w8(&mut TokioCase::new(8), 10)]
-fn tokio_workload(case: &mut TokioCase, concurrency: u64) -> Duration {
+#[bench::concurrency_w1(&TokioCase::new(1), 10)]
+#[bench::concurrency_w8(&TokioCase::new(8), 10)]
+fn tokio_workload(case: &TokioCase, concurrency: u64) -> Duration {
     case.run(usize::try_from(concurrency).expect("benchmark concurrency fits in usize"))
 }
 

crates/http_headers/benches/http_headers_micro.rs

@@ -2141,6 +2141,7 @@ fn downstream_agent() -> (HeaderMap, DownstreamAgent) {
     (empty_map(), DownstreamAgent(field_value(USER_AGENT_VALUE)))
 }
 
+#[derive(Clone, Copy)]
 struct RawUserAgentSource;
 
 impl FieldSource for RawUserAgentSource {

crates/http_headers/docs/PERF.md

@@ -5,7 +5,7 @@ is not a promise of current timing on different hardware or dependency
 versions; use the repository's metabench targets to measure the current
 checkout.
 
-Each cell reports metabench's median wall-clock time, Callgrind instruction
+Each cell reports the historical Criterion median wall-clock time, Callgrind instruction
 count, allocation count, and total allocated bytes for one typed decode and
 read. Benchmarks consume the decoded value and force comparable semantic work
 when `headers 0.4.1` defers parsing. Setup uses a prebuilt `HeaderMap` outside
@@ -14,6 +14,11 @@ and 60 samples per arm. Each cell is formatted as time, instructions, then
 allocation count / allocated bytes. `n/a` means `headers 0.4.1` does not
 provide that typed header.
 
+These measurements have not been rerun for the metabench schema 9 migration.
+Current `scripts/perf_report.rs` output reads Criterion's primary estimate from
+`criterion/time`, not the historical median metric. A median across independent
+runs, where used, is a separate aggregation of per-run estimates.
+
 | Header | `headers 0.4.1` | `http_headers` (owned) | `http_headers` (borrowed) |
 |---|---:|---:|---:|
 | Accept | n/a | 399.0 ns<br>1,387 instr<br>1 allocs / 24 B | 343.5 ns<br>1,055 instr<br>0 allocs / 0 B |

crates/http_headers/scripts/perf_report.rs

@@ -21,7 +21,7 @@ use std::{env, fs};
 
 use serde::Deserialize;
 
-const SCHEMA_VERSION: u32 = 8;
+const SCHEMA_VERSION: u32 = 9;
 const DEFAULT_ARTIFACT: &str = "target/metabench/http_headers_per_header/report.json";
 const DEFAULT_REPORT: &str = "crates/http_headers/docs/PERF.md";
 
@@ -369,7 +369,7 @@ fn render(report: &Report) -> Result<String, String> {
     .map_err(|error| format!("cannot render report disclaimer: {error}"))?;
     writeln!(
         output,
-        "Each cell reports metabench's median wall-clock time, Callgrind instruction\n\
+        "Each cell reports Criterion's primary wall-clock estimate, Callgrind instruction\n\
          count, allocation count, and total allocated bytes for one typed decode and\n\
          read. Benchmarks consume the decoded value and force comparable semantic work\n\
          when `headers 0.4.1` defers parsing. Setup uses a prebuilt `HeaderMap` outside\n\
@@ -421,16 +421,16 @@ fn required_entry<'a>(
 }
 
 fn measurements(entry: &Entry) -> Result<Measurements, String> {
-    let time = metric(entry, "criterion", "median")?;
+    let time = metric(entry, "criterion", "time")?;
     if time.unit.as_deref() != Some("ns") {
         return Err(format!(
-            "`{}` criterion median has unit {}, expected ns",
+            "`{}` criterion primary estimate (`time`) has unit {}, expected ns",
             entry.identity,
             time.unit.as_deref().unwrap_or("<missing>")
         ));
     }
     Ok(Measurements {
-        time_ns: finite_nonnegative(entry, "criterion/median", float_value(time.value))?,
+        time_ns: finite_nonnegative(entry, "criterion/time", float_value(time.value))?,
         instructions: integer_metric(entry, "gungraun.callgrind", "Ir")?,
         allocations: integer_metric(entry, "alloc_tracker", "Allocations")?,
         allocated_bytes: integer_metric(entry, "alloc_tracker", "Allocated bytes")?,
@@ -503,3 +503,46 @@ fn grouped(value: u64) -> String {
 fn bytes(value: u64) -> String {
     format!("{} B", grouped(value))
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{Report, SCHEMA_VERSION, measurements};
+
+    #[test]
+    fn schema_nine_primary_estimate_preserves_allocation_metrics() {
+        let mut report: Report = serde_json::from_str(
+            r#"{
+                "schema_version": 9,
+                "entries": [{
+                    "identity": "example",
+                    "results": {
+                        "criterion": {"metrics": {"time": {"value": 12.5, "unit": "ns"}}},
+                        "gungraun.callgrind": {"metrics": {"Ir": {"value": 123}}},
+                        "alloc_tracker": {"metrics": {
+                            "Allocations": {"value": 2},
+                            "Allocated bytes": {"value": 64}
+                        }}
+                    }
+                }]
+            }"#,
+        )
+        .unwrap();
+        assert_eq!(report.schema_version, SCHEMA_VERSION);
+        let entry = &mut report.entries[0];
+        let result = measurements(entry).unwrap();
+        assert_eq!(result.time_ns, 12.5);
+        assert_eq!(result.instructions, 123);
+        assert_eq!(result.allocations, 2);
+        assert_eq!(result.allocated_bytes, 64);
+
+        let metrics = &mut entry.results.get_mut("criterion").unwrap().metrics;
+        let time = metrics.remove("time").unwrap();
+        metrics.insert("median".to_string(), time);
+        assert!(
+            measurements(entry)
+                .err()
+                .unwrap()
+                .contains("criterion/time")
+        );
+    }
+}

crates/internity/docs/COMPARISON.md

@@ -17,6 +17,12 @@ treated as a distinct row.
 > predate cached hashes and reusable shared snapshots; do not treat them as
 > measurements of the current implementation.
 
+The historical timing data has not been rerun for the metabench schema 9
+migration. Current report scripts read Criterion's primary estimates from
+`criterion/time`; those estimates are distinct from these historical medians.
+A median across independent runs, where used, is a separate aggregation of
+per-run estimates.
+
 ---
 
 ## 1. Master comparison table

crates/internity/docs/PERF.md

@@ -157,6 +157,12 @@ The established comparison timing figures below are wall-clock medians measured
 machine-dependent: the ratios between rows are the durable signal, not the
 absolute values.
 
+These historical timing figures have not been rerun for the metabench schema 9
+migration. Current `scripts/perf_report.rs` output reads Criterion's primary
+estimate from `criterion/time`, not the historical median metric. A median
+across independent runs, where used, is a separate aggregation of per-run
+estimates.
+
 This report is a curated set of customer-facing scenarios. The crate also carries
 internal Callgrind instruction-count benches in the consolidated metabench target
 (`benches/internity.rs`); the matched cases used above are part of that target.
@@ -207,7 +213,7 @@ One table per operation, comparing internity against every other interner measur
 
 ## Concurrent scaling
 
-One table per operation, with a column per thread count. Each cell is the wall-clock median of the whole parallel phase at that thread count, so the total work grows with the thread count and the numbers show how each interner holds up under contention rather than a per-thread speedup. The trailing column is the delta against internity at the highest thread count, which is where the designs separate most; per-thread-count deltas are omitted to keep the tables readable.
+One table per operation, with a column per thread count. Each historical cell is the wall-clock median of the whole parallel phase at that thread count, so the total work grows with the thread count and the numbers show how each interner holds up under contention rather than a per-thread speedup. The trailing column is the delta against internity at the highest thread count, which is where the designs separate most; per-thread-count deltas are omitted to keep the tables readable.
 
 ### `insert` — concurrent