diff --git a/.cargo/mutants.linux.toml b/.cargo/mutants.linux.toml index 444b1b2fb..d069f44f7 100644 --- a/.cargo/mutants.linux.toml +++ b/.cargo/mutants.linux.toml @@ -45,6 +45,7 @@ profile = "mutants" # Only run unit tests and integration tests, not doctests, benchmarks or examples. # Retain fatal-path tests needed to detect mutations in termination guards; # skip the remaining intentional-abort tests only during mutation runs. +# Exact metabench deadlines use FakeClock/FakeProcess; keep real-time stress outside mutation runs. additional_cargo_test_args = [ "--locked", "--tests", "--", "--skip", "abort::tests::subprocess_oracle_rejects_normal_return", @@ -55,9 +56,17 @@ additional_cargo_test_args = [ "--skip", "observation::tests::poisoned_publisher_aborts_instead_of_overwriting_failed_state", "--skip", "observation::tests::generation_overflow_aborts_instead_of_reusing_a_lease_identity", "--skip", "thread::tests::poisoned_pool_aborts_before_lending_an_owner", + "--skip", "runner::tests::captured_logs_bound_sustained_output_and_timeout_cleanup", ] exclude_re = [ + # Windows and unsupported-platform pipe adapters are compiled out on Linux. + 'crates/metabench/src/worker_log\.rs:\d+:\d+: (?:.* in (?:read_available_windows|prepare_reader_unsupported|read_available_unsupported)$|replace (?:read_available_windows|prepare_reader_unsupported|read_available_unsupported) ->)', + # Windows metadata admission is compiled out on Linux. + '\bin reject_windows_reparse_point$', + '\breplace reject_windows_reparse_point ->', + # O_NOFOLLOW and O_NONBLOCK have disjoint bits, so XOR is equivalent to OR. + 'crates/metabench/src/artifact.rs:\d+:\d+: replace \| with \^ in open_regular_file$', # Shared exclusions copied from mutants.toml. # There are no API contracts for what these must return because they # are for human-readable stringification. No purpose mutating them. diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 2c098d48a..cd99edc30 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -34,6 +34,7 @@ profile = "mutants" # Only run unit tests and integration tests, not doctests, benchmarks or examples. # Retain fatal-path tests needed to detect mutations in termination guards; # skip the remaining intentional-abort tests only during mutation runs. +# Exact metabench deadlines use FakeClock/FakeProcess; keep real-time stress outside mutation runs. additional_cargo_test_args = [ "--locked", "--tests", "--", "--skip", "abort::tests::subprocess_oracle_rejects_normal_return", @@ -44,9 +45,12 @@ additional_cargo_test_args = [ "--skip", "observation::tests::poisoned_publisher_aborts_instead_of_overwriting_failed_state", "--skip", "observation::tests::generation_overflow_aborts_instead_of_reusing_a_lease_identity", "--skip", "thread::tests::poisoned_pool_aborts_before_lending_an_owner", + "--skip", "runner::tests::captured_logs_bound_sustained_output_and_timeout_cleanup", ] exclude_re = [ + # O_NOFOLLOW and O_NONBLOCK have disjoint bits, so XOR is equivalent to OR. + 'crates/metabench/src/artifact.rs:\d+:\d+: replace \| with \^ in open_regular_file$', # There are no API contracts for what these must return because they # are for human-readable stringification. No purpose mutating them. "impl Debug", diff --git a/.cargo/mutants.windows.toml b/.cargo/mutants.windows.toml index 316fa451f..0dde463b6 100644 --- a/.cargo/mutants.windows.toml +++ b/.cargo/mutants.windows.toml @@ -40,6 +40,7 @@ profile = "mutants" # Only run unit tests and integration tests, not doctests, benchmarks or examples. # Retain fatal-path tests needed to detect mutations in termination guards; # skip the remaining intentional-abort tests only during mutation runs. +# Exact metabench deadlines use FakeClock/FakeProcess; keep real-time stress outside mutation runs. additional_cargo_test_args = [ "--locked", "--tests", "--", "--skip", "abort::tests::subprocess_oracle_rejects_normal_return", @@ -50,9 +51,14 @@ additional_cargo_test_args = [ "--skip", "observation::tests::poisoned_publisher_aborts_instead_of_overwriting_failed_state", "--skip", "observation::tests::generation_overflow_aborts_instead_of_reusing_a_lease_identity", "--skip", "thread::tests::poisoned_pool_aborts_before_lending_an_owner", + "--skip", "runner::tests::captured_logs_bound_sustained_output_and_timeout_cleanup", ] exclude_re = [ + # Unix and unsupported-platform pipe adapters are compiled out on Windows. + 'crates/metabench/src/worker_log\.rs:\d+:\d+: (?:.* in (?:prepare_reader_unix|prepare_reader_with|read_available_unix|prepare_reader_unsupported|read_available_unsupported)$|replace (?:prepare_reader_unix|prepare_reader_with|read_available_unix|prepare_reader_unsupported|read_available_unsupported) ->)', + # Unix artifact-open flags are compiled out on Windows. + 'crates/metabench/src/artifact.rs:\d+:\d+: replace \| with (\^|&) in open_regular_file$', # Shared exclusions copied from mutants.toml. # There are no API contracts for what these must return because they # are for human-readable stringification. No purpose mutating them. diff --git a/.github/workflows/repository-checks.yml b/.github/workflows/repository-checks.yml index b8f8ade5f..ce47f688a 100644 --- a/.github/workflows/repository-checks.yml +++ b/.github/workflows/repository-checks.yml @@ -103,10 +103,41 @@ jobs: - name: Test no_std with serde run: cargo +1.95 test --package internity --no-default-features --features serde --test no_std_api --locked + metabench-native-contracts: + name: Metabench native contracts (Linux) + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + CARGO_BUILD_JOBS: "2" + METABENCH_REQUIRE_NATIVE_CONTRACTS: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ./.github/actions/anvil-setup + with: + group: none + free-disk-space: "true" + - name: Install Rust + run: just anvil-toolchain-stable-install + - name: Install native prerequisites + run: | + mkdir -p "$GITHUB_WORKSPACE/target/metabench-contract-scratch" + export TMPDIR="$GITHUB_WORKSPACE/target/metabench-contract-scratch" + sudo apt-get update --quiet + sudo apt-get install --yes --quiet valgrind + cargo +1.95 install --locked --version 0.19.4 gungraun-runner + gungraun-runner --version + valgrind --version + - name: Run native listing and smoke contracts + run: | + mkdir -p "$GITHUB_WORKSPACE/target/metabench-contract-scratch" + export TMPDIR="$GITHUB_WORKSPACE/target/metabench-contract-scratch" + cargo +1.95 test --package metabench --test spawned_benchmark --locked gungraun_listing_is_forwarded -- --exact --nocapture + cargo +1.95 test --package metabench --test spawned_benchmark --locked default_engine_smoke_checks_do_not_publish_measurements -- --exact --nocapture + required-repository-checks: name: Required repository checks if: always() - needs: [release-script-tests, simd-no-std-arm, simd-no-std-x86, internity-32-bit-handles, internity-no-std-serde] + needs: [release-script-tests, simd-no-std-arm, simd-no-std-x86, internity-32-bit-handles, internity-no-std-serde, metabench-native-contracts] runs-on: ubuntu-latest steps: - name: Verify repository checks @@ -116,6 +147,7 @@ jobs: SIMD_NO_STD_X86: ${{ needs.simd-no-std-x86.result }} INTERNITY_32_BIT_HANDLES: ${{ needs.internity-32-bit-handles.result }} INTERNITY_NO_STD_SERDE: ${{ needs.internity-no-std-serde.result }} + METABENCH_NATIVE_CONTRACTS: ${{ needs.metabench-native-contracts.result }} run: | if [[ "$RELEASE_SCRIPT_TESTS" != "success" ]]; then echo "::error::release-script-tests concluded $RELEASE_SCRIPT_TESTS" @@ -137,3 +169,7 @@ jobs: echo "::error::internity-no-std-serde concluded $INTERNITY_NO_STD_SERDE" exit 1 fi + if [[ "$METABENCH_NATIVE_CONTRACTS" != "success" ]]; then + echo "::error::metabench-native-contracts concluded $METABENCH_NATIVE_CONTRACTS" + exit 1 + fi diff --git a/Cargo.lock b/Cargo.lock index 50e41255a..4eeaf3cea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2459,20 +2459,6 @@ dependencies = [ "serde", ] -[[package]] -name = "gungraun-summary" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f967da1beab0153f3f1e52408a71d3bd0fbea3c9e0bf0431f9a3a3c9e96130" -dependencies = [ - "either-or-both", - "gungraun-runner", - "indexmap", - "serde", - "serde_json", - "thiserror", -] - [[package]] name = "h2" version = "0.4.19" @@ -3517,17 +3503,22 @@ dependencies = [ "bolero", "command-group", "criterion", + "either-or-both", "gungraun", - "gungraun-summary", + "gungraun-runner", "jiff", + "libc", "metabench_macros", "mutants", "nix 0.31.3", + "ohno", + "regex", "serde", "serde_json", "tempfile", "terminal_size", "unicode-width 0.2.2", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e228c685a..5a9525290 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,6 +124,7 @@ deadpool = { version = "0.13.0", default-features = false } derive_more = { version = "2.0.1", default-features = false } duct = { version = "1.1.1", default-features = false } dynosaur = { version = "0.3.0", default-features = false } +either-or-both = { version = "0.3", default-features = false } events_once = { version = "0.5.32", default-features = false } fakeable = { path = "crates/fakeable", default-features = false, version = "0.2.0" } fakeable_impl = { path = "crates/fakeable_impl", default-features = false, version = "=0.2.0" } @@ -148,7 +149,7 @@ futures-util = { version = "0.3.31", default-features = false } getrandom = { version = "0.4.3", default-features = false } # Keep the library and summary parser aligned with gungraun-runner's schema. gungraun = { version = "=0.19.4", default-features = false } -gungraun-summary = { version = "=6.0.0", default-features = false } +gungraun-runner = { version = "=0.19.4", default-features = false } h3 = { version = "0.0.8", default-features = false } h3-quinn = { version = "0.0.10", default-features = false } hashbrown = { version = "0.17.0", default-features = false } diff --git a/crates/arty/benches/arty_contention.rs b/crates/arty/benches/arty_contention.rs index aebb64db0..9ca8c303b 100644 --- a/crates/arty/benches/arty_contention.rs +++ b/crates/arty/benches/arty_contention.rs @@ -228,16 +228,16 @@ fn criterion_benchmarks(criterion: &mut Criterion) { } #[benchmark(ARTY, "arty_contention/relocated_cache", "Arty")] -#[bench::concurrency_w1(&mut ArtyCase::new(1), 10)] -#[bench::concurrency_w8(&mut ArtyCase::new(8), 10)] -fn arty_workload(case: &mut ArtyCase, concurrency: u64) -> Duration { +#[bench::concurrency_w1(&ArtyCase::new(1), 10)] +#[bench::concurrency_w8(&ArtyCase::new(8), 10)] +fn arty_workload(case: &ArtyCase, concurrency: u64) -> Duration { case.run(usize::try_from(concurrency).expect("benchmark concurrency fits in usize")) } #[benchmark(TOKIO, "arty_contention/relocated_cache", "Tokio")] -#[bench::concurrency_w1(&mut TokioCase::new(1), 10)] -#[bench::concurrency_w8(&mut TokioCase::new(8), 10)] -fn tokio_workload(case: &mut TokioCase, concurrency: u64) -> Duration { +#[bench::concurrency_w1(&TokioCase::new(1), 10)] +#[bench::concurrency_w8(&TokioCase::new(8), 10)] +fn tokio_workload(case: &TokioCase, concurrency: u64) -> Duration { case.run(usize::try_from(concurrency).expect("benchmark concurrency fits in usize")) } diff --git a/crates/http_headers/benches/http_headers_micro.rs b/crates/http_headers/benches/http_headers_micro.rs index c9af3f563..bd443c3f9 100644 --- a/crates/http_headers/benches/http_headers_micro.rs +++ b/crates/http_headers/benches/http_headers_micro.rs @@ -2141,6 +2141,7 @@ fn downstream_agent() -> (HeaderMap, DownstreamAgent) { (empty_map(), DownstreamAgent(field_value(USER_AGENT_VALUE))) } +#[derive(Clone, Copy)] struct RawUserAgentSource; impl FieldSource for RawUserAgentSource { diff --git a/crates/http_headers/docs/PERF.md b/crates/http_headers/docs/PERF.md index 36f356ac4..250d40ce6 100644 --- a/crates/http_headers/docs/PERF.md +++ b/crates/http_headers/docs/PERF.md @@ -5,7 +5,7 @@ is not a promise of current timing on different hardware or dependency versions; use the repository's metabench targets to measure the current checkout. -Each cell reports metabench's median wall-clock time, Callgrind instruction +Each cell reports the historical Criterion median wall-clock time, Callgrind instruction count, allocation count, and total allocated bytes for one typed decode and read. Benchmarks consume the decoded value and force comparable semantic work when `headers 0.4.1` defers parsing. Setup uses a prebuilt `HeaderMap` outside @@ -14,6 +14,11 @@ and 60 samples per arm. Each cell is formatted as time, instructions, then allocation count / allocated bytes. `n/a` means `headers 0.4.1` does not provide that typed header. +These measurements have not been rerun for the metabench schema 9 migration. +Current `scripts/perf_report.rs` output reads Criterion's primary estimate from +`criterion/time`, not the historical median metric. A median across independent +runs, where used, is a separate aggregation of per-run estimates. + | Header | `headers 0.4.1` | `http_headers` (owned) | `http_headers` (borrowed) | |---|---:|---:|---:| | Accept | n/a | 399.0 ns
1,387 instr
1 allocs / 24 B | 343.5 ns
1,055 instr
0 allocs / 0 B | diff --git a/crates/http_headers/scripts/perf_report.rs b/crates/http_headers/scripts/perf_report.rs index ca77f2f74..6decf2ba0 100755 --- a/crates/http_headers/scripts/perf_report.rs +++ b/crates/http_headers/scripts/perf_report.rs @@ -21,7 +21,7 @@ use std::{env, fs}; use serde::Deserialize; -const SCHEMA_VERSION: u32 = 8; +const SCHEMA_VERSION: u32 = 9; const DEFAULT_ARTIFACT: &str = "target/metabench/http_headers_per_header/report.json"; const DEFAULT_REPORT: &str = "crates/http_headers/docs/PERF.md"; @@ -369,7 +369,7 @@ fn render(report: &Report) -> Result { .map_err(|error| format!("cannot render report disclaimer: {error}"))?; writeln!( output, - "Each cell reports metabench's median wall-clock time, Callgrind instruction\n\ + "Each cell reports Criterion's primary wall-clock estimate, Callgrind instruction\n\ count, allocation count, and total allocated bytes for one typed decode and\n\ read. Benchmarks consume the decoded value and force comparable semantic work\n\ when `headers 0.4.1` defers parsing. Setup uses a prebuilt `HeaderMap` outside\n\ @@ -421,16 +421,16 @@ fn required_entry<'a>( } fn measurements(entry: &Entry) -> Result { - let time = metric(entry, "criterion", "median")?; + let time = metric(entry, "criterion", "time")?; if time.unit.as_deref() != Some("ns") { return Err(format!( - "`{}` criterion median has unit {}, expected ns", + "`{}` criterion primary estimate (`time`) has unit {}, expected ns", entry.identity, time.unit.as_deref().unwrap_or("") )); } Ok(Measurements { - time_ns: finite_nonnegative(entry, "criterion/median", float_value(time.value))?, + time_ns: finite_nonnegative(entry, "criterion/time", float_value(time.value))?, instructions: integer_metric(entry, "gungraun.callgrind", "Ir")?, allocations: integer_metric(entry, "alloc_tracker", "Allocations")?, allocated_bytes: integer_metric(entry, "alloc_tracker", "Allocated bytes")?, @@ -503,3 +503,46 @@ fn grouped(value: u64) -> String { fn bytes(value: u64) -> String { format!("{} B", grouped(value)) } + +#[cfg(test)] +mod tests { + use super::{Report, SCHEMA_VERSION, measurements}; + + #[test] + fn schema_nine_primary_estimate_preserves_allocation_metrics() { + let mut report: Report = serde_json::from_str( + r#"{ + "schema_version": 9, + "entries": [{ + "identity": "example", + "results": { + "criterion": {"metrics": {"time": {"value": 12.5, "unit": "ns"}}}, + "gungraun.callgrind": {"metrics": {"Ir": {"value": 123}}}, + "alloc_tracker": {"metrics": { + "Allocations": {"value": 2}, + "Allocated bytes": {"value": 64} + }} + } + }] + }"#, + ) + .unwrap(); + assert_eq!(report.schema_version, SCHEMA_VERSION); + let entry = &mut report.entries[0]; + let result = measurements(entry).unwrap(); + assert_eq!(result.time_ns, 12.5); + assert_eq!(result.instructions, 123); + assert_eq!(result.allocations, 2); + assert_eq!(result.allocated_bytes, 64); + + let metrics = &mut entry.results.get_mut("criterion").unwrap().metrics; + let time = metrics.remove("time").unwrap(); + metrics.insert("median".to_string(), time); + assert!( + measurements(entry) + .err() + .unwrap() + .contains("criterion/time") + ); + } +} diff --git a/crates/internity/docs/COMPARISON.md b/crates/internity/docs/COMPARISON.md index 33c332501..de3204fde 100644 --- a/crates/internity/docs/COMPARISON.md +++ b/crates/internity/docs/COMPARISON.md @@ -17,6 +17,12 @@ treated as a distinct row. > predate cached hashes and reusable shared snapshots; do not treat them as > measurements of the current implementation. +The historical timing data has not been rerun for the metabench schema 9 +migration. Current report scripts read Criterion's primary estimates from +`criterion/time`; those estimates are distinct from these historical medians. +A median across independent runs, where used, is a separate aggregation of +per-run estimates. + --- ## 1. Master comparison table diff --git a/crates/internity/docs/PERF.md b/crates/internity/docs/PERF.md index aa50e64bc..7814dc59d 100644 --- a/crates/internity/docs/PERF.md +++ b/crates/internity/docs/PERF.md @@ -157,6 +157,12 @@ The established comparison timing figures below are wall-clock medians measured machine-dependent: the ratios between rows are the durable signal, not the absolute values. +These historical timing figures have not been rerun for the metabench schema 9 +migration. Current `scripts/perf_report.rs` output reads Criterion's primary +estimate from `criterion/time`, not the historical median metric. A median +across independent runs, where used, is a separate aggregation of per-run +estimates. + This report is a curated set of customer-facing scenarios. The crate also carries internal Callgrind instruction-count benches in the consolidated metabench target (`benches/internity.rs`); the matched cases used above are part of that target. @@ -207,7 +213,7 @@ One table per operation, comparing internity against every other interner measur ## Concurrent scaling -One table per operation, with a column per thread count. Each cell is the wall-clock median of the whole parallel phase at that thread count, so the total work grows with the thread count and the numbers show how each interner holds up under contention rather than a per-thread speedup. The trailing column is the delta against internity at the highest thread count, which is where the designs separate most; per-thread-count deltas are omitted to keep the tables readable. +One table per operation, with a column per thread count. Each historical cell is the wall-clock median of the whole parallel phase at that thread count, so the total work grows with the thread count and the numbers show how each interner holds up under contention rather than a per-thread speedup. The trailing column is the delta against internity at the highest thread count, which is where the designs separate most; per-thread-count deltas are omitted to keep the tables readable. ### `insert` — concurrent diff --git a/crates/internity/scripts/perf_report.rs b/crates/internity/scripts/perf_report.rs index 5cacd2199..17168e8ee 100755 --- a/crates/internity/scripts/perf_report.rs +++ b/crates/internity/scripts/perf_report.rs @@ -24,14 +24,14 @@ serde_json = "1" //! live-heap measurement remains the dedicated `internity_mem` target because it //! reports steady-state retained bytes, not a metabench engine metric. -use std::collections::{BTreeMap, btree_map::Entry}; -use std::env; +use std::collections::BTreeMap; +use std::collections::btree_map::Entry; use std::error::Error; use std::fmt::Write as _; -use std::fs; use std::io::Write as _; use std::path::{Path, PathBuf}; use std::process::{Command, ExitCode}; +use std::{env, fs}; use clap::Parser; use serde::Deserialize; @@ -129,10 +129,7 @@ const THREAD_COUNTS: &[&str] = &["1", "2", "4", "8"]; /// Rows that have no repeatable single-threaded criterion timing in this suite /// because they are backed by a process-global, persistent table. -const CRITERION_OMITTED: &[&str] = &[ - "internity_compare/insert/ustr", - "internity_compare/insert/string_cache", -]; +const CRITERION_OMITTED: &[&str] = &["internity_compare/insert/ustr", "internity_compare/insert/string_cache"]; #[derive(Deserialize)] struct JsonReport { @@ -190,7 +187,7 @@ fn index_report(report: JsonReport) -> Result { } fn criterion_time_ns(report: &ReportIndex, identity: &str) -> Option { - let metric = report.get(identity)?.results.get("criterion")?.metrics.get("median")?; + let metric = report.get(identity)?.results.get("criterion")?.metrics.get("time")?; let value = metric.value.as_f64(); match metric.unit.as_deref()? { "ps" => Some(value * 1e-3), @@ -206,6 +203,40 @@ fn lookup_time(report: &ReportIndex, key: &str) -> Option { criterion_time_ns(report, key) } +#[cfg(test)] +mod tests { + use super::{JsonReport, criterion_time_ns, index_report}; + + #[test] + fn schema_nine_primary_estimate_and_units() { + for (value, unit, expected) in [ + (serde_json::json!(1250), "ps", 1.25), + (serde_json::json!(1.25), "ns", 1.25), + (serde_json::json!(1.25), "µs", 1250.0), + (serde_json::json!(1.25), "us", 1250.0), + (serde_json::json!(1.25), "ms", 1_250_000.0), + (serde_json::json!(1.25), "s", 1_250_000_000.0), + ] { + let fixture = serde_json::json!({ + "schema_version": 9, + "entries": [{ + "identity": "example", + "results": {"criterion": {"metrics": { + "time": {"value": value, "unit": unit} + }}} + }] + }); + let report: JsonReport = serde_json::from_value(fixture).unwrap(); + let mut report = index_report(report).unwrap(); + assert_eq!(criterion_time_ns(&report, "example"), Some(expected)); + let metrics = &mut report.get_mut("example").unwrap().results.get_mut("criterion").unwrap().metrics; + let time = metrics.remove("time").unwrap(); + metrics.insert("median".to_string(), time); + assert_eq!(criterion_time_ns(&report, "example"), None); + } + } +} + fn validate_criterion(report: &ReportIndex) -> Result<(), BoxErr> { let mut missing = Vec::new(); for (_, group, rows) in SINGLE_GROUPS { @@ -262,7 +293,7 @@ fn header() -> String { refresh these numbers.\n\n", ); out.push_str( - "All timing figures are wall-clock medians measured by Criterion. They are\n\ + "All timing figures are primary wall-clock estimates measured by Criterion. They are\n\ machine-dependent: the ratios between rows are the durable signal, not the\n\ absolute values.\n\n", ); @@ -331,14 +362,12 @@ fn render_single(out: &mut String, report: &ReportIndex) { /// Render each concurrent operation as a single table: one row per interner, one /// column per thread count, and a trailing delta at the highest thread count. fn render_concurrent(out: &mut String, report: &ReportIndex) { - let top = THREAD_COUNTS - .last() - .expect("THREAD_COUNTS is a non-empty compile-time constant"); + let top = THREAD_COUNTS.last().expect("THREAD_COUNTS is a non-empty compile-time constant"); out.push_str("## Concurrent scaling\n\n"); out.push_str( "One table per operation, with a column per thread count. Each cell is the \ - wall-clock median of the whole parallel phase at that thread count, so the total \ + primary wall-clock estimate of the whole parallel phase at that thread count, so the total \ work grows with the thread count and the numbers show how each interner holds up \ under contention rather than a per-thread speedup. The trailing column is the \ delta against internity at the highest thread count, which is where the designs \ @@ -366,11 +395,7 @@ fn render_concurrent(out: &mut String, report: &ReportIndex) { let _ = write!(out, " {} |", fmt_ns(time)); } let top_time = lookup_time(report, &format!("{group}/{row}/{top}")); - let _ = writeln!( - out, - " {} |", - fmt_delta(top_time, reference, *row == "internity") - ); + let _ = writeln!(out, " {} |", fmt_delta(top_time, reference, *row == "internity")); } out.push('\n'); } @@ -450,14 +475,7 @@ fn run_bench(cwd: &Path, bench: &str, extra: &[&str], label: &str) -> Result Result<(), BoxErr> { +fn run_metabench(cwd: &Path, json_path: &Path, markdown_path: &Path, samples: &str, measurement: &str, warmup: &str) -> Result<(), BoxErr> { println!("==> Running criterion metabench suite"); let mut cmd = Command::new(env::var("CARGO").unwrap_or_else(|_| "cargo".into())); cmd.current_dir(cwd).arg("bench").arg("--bench").arg("internity").arg("--"); @@ -506,8 +524,7 @@ fn run(args: &Args) -> Result<(), BoxErr> { let warmup = args.warm_up_time.unwrap_or(1).to_string(); let metabench_dir = crate_dir.join("target").join("metabench").join("internity"); - fs::create_dir_all(&metabench_dir) - .map_err(|e| format!("creating {}: {e}", metabench_dir.display()))?; + fs::create_dir_all(&metabench_dir).map_err(|e| format!("creating {}: {e}", metabench_dir.display()))?; let json_path = metabench_dir.join("report.json"); let markdown_path = metabench_dir.join("report.md"); @@ -530,11 +547,7 @@ fn run(args: &Args) -> Result<(), BoxErr> { let out_path = docs_dir.join("PERF.md"); fs::write(&out_path, &report).map_err(|e| format!("writing {}: {e}", out_path.display()))?; - println!( - "Wrote {} ({} metabench identities)", - out_path.display(), - entry_count, - ); + println!("Wrote {} ({} metabench identities)", out_path.display(), entry_count,); println!("==> Done. Report written to docs/PERF.md"); Ok(()) } diff --git a/crates/metabench/Cargo.toml b/crates/metabench/Cargo.toml index 2279618f3..534b5544a 100644 --- a/crates/metabench/Cargo.toml +++ b/crates/metabench/Cargo.toml @@ -24,12 +24,7 @@ exclude = true min-lines-percent = 1 [package.metadata.cargo_check_external_types] -allowed_external_types = [ - "alloc_tracker::allocator::Allocator", - "criterion", - "gungraun", - "metabench_macros::*", -] +allowed_external_types = ["criterion", "gungraun", "metabench_macros::*"] [package.metadata.docs.rs] all-features = true @@ -41,10 +36,14 @@ bench = false alloc_tracker = { workspace = true } command-group = { workspace = true } criterion = { workspace = true } +either-or-both = { workspace = true } gungraun = { workspace = true, features = ["default"] } -gungraun-summary = { workspace = true } +# Parse summaries with the matching runner, avoiding a separately resolved wrapper. +gungraun-runner = { workspace = true, features = ["summary"] } jiff = { workspace = true, features = ["std"] } metabench_macros = { workspace = true } +ohno = { workspace = true } +regex = { workspace = true, features = ["std", "perf", "unicode"] } serde = { workspace = true, features = ["derive", "std"] } serde_json = { workspace = true, features = ["std"] } tempfile = { workspace = true, features = ["getrandom"] } @@ -54,8 +53,16 @@ unicode-width = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] nix = { workspace = true, features = ["fs", "poll"] } +[target.'cfg(unix)'.dependencies] +libc = { workspace = true } + +[target.'cfg(windows)'.dependencies] +windows-sys = { workspace = true, features = ["Win32_Foundation", "Win32_System_Pipes"] } + [dev-dependencies] bolero = { workspace = true, features = ["std"] } + +[target.'cfg(not(target_os = "linux"))'.dev-dependencies] mutants = { workspace = true } # >>> anvil-managed: anvil-lints diff --git a/crates/metabench/README.md b/crates/metabench/README.md index 9e272b5e2..ffe484316 100644 --- a/crates/metabench/README.md +++ b/crates/metabench/README.md @@ -83,6 +83,11 @@ markdown reports in your `target` directory. ## Measurement engines +A **native case** is a case supplied by a benchmark’s Gungraun declarations, +including its implicit single case when no case attribute is present. +**Probes** are the separate allocation, perf, and `VTune` workers that execute +these cases; they do not use Criterion registrations. + As mentioned, metabench supports five distinct benchmark tools: Criterion, Gungraun, allocation tracking, Linux `perf`, and Intel `VTune`: @@ -94,15 +99,19 @@ tracking, Linux `perf`, and Intel `VTune`: enabled, system calls, and heap activity. * **Allocation tracking** reports allocation count and allocated bytes for - one invocation of every discovered Criterion case. + one invocation of every declared native case. Counts + are process-wide, so concurrent threads can contribute allocations. * **Linux `perf`** reports supported hardware and software counters for one - invocation of every discovered Criterion case. Counters cover only the - annotated workload. Use repeated `--perf-arg` options for additional + invocation of every declared native case. Collection starts and stops in + a private caller around the workload, excluding input preparation and + returned-value teardown. The collection window can include caller and + control overhead; it is not a claim of exact function-only counters. + Use repeated `--perf-arg` options for additional `perf stat` arguments. * **Intel `VTune`** reports hardware event counts for one invocation of every - discovered Criterion case, using the same annotated-workload + declared native case, using the same private caller measurement as Linux `perf`. Because `VTune`’s hardware-event collection is a statistical sampler rather than an exact counter, reported counts are sampling estimates that can read zero for a short workload, and the @@ -123,9 +132,6 @@ cargo bench --bench parser -- --criterion Some limitations around engines: -* Benchmark functions declared with `const fn` can be measured by Criterion and Gungraun, but not - by allocation tracking, `perf`, or `VTune`. - * Gungraun and Linux `perf` are only available on Linux platforms. `VTune` is available on Linux and Windows, provided a `vtune` installation is reachable on `PATH`. `VTune`’s integration test exercises the control @@ -133,6 +139,34 @@ Some limitations around engines: `vtune` fixture (see `tests/spawned_benchmark.rs`), rather than a real `VTune` install. +## Workload boundaries + +The attribute preserves a non-inlined workload symbol without injecting +runtime collector checks into the function. Criterion callbacks call it +directly, and Gungraun uses its normal adapter. Neither executes the private +allocation, perf, or `VTune` collection controls. + +Allocation and hardware workers reuse the existing `#[bench::...]` and +`#[benches::...]` declarations through separate generated adapters. Native +suite and group hooks run outside collection. Case argument evaluation and +setup precede collection; teardown and returned-value destruction follow it. +Calls to annotated functions during setup cannot consume the measurement. +Collection guards stop on unwinding. + +No new registration language or public measurement helper is required. Probes +do not run Criterion registration or infer its arbitrary callbacks. A +Criterion-only case without a corresponding native declaration has no probe +result. Matching case names do not prove matching Criterion inputs or cleanup; +those remain author responsibilities. + +Probe execution reuses native case arguments and lifecycle hooks, not +Valgrind configuration such as sandboxes or environment overrides. Configure +hardware tools with their own forwarded arguments. + +Input destruction inside the benchmark function is included; return owned +inputs with the output if their destruction should happen outside collection. +Functions declared with `const fn` can also have runtime probes. + ## Configure Criterion Provide a Criterion factory to customize measurements or other Criterion @@ -152,6 +186,9 @@ metabench::main!( ## Configure Gungraun +The installed `gungraun-runner` must match the Gungraun library version +(currently 0.19.4). + Benchmark-level `gungraun_config`, `gungraun_setup`, and `gungraun_teardown` options work with native `#[bench::...]` and `#[benches::...]` cases: @@ -214,12 +251,51 @@ under `target/metabench//`. Reports include benchmark metadata, environment information, measured engines, and their metrics. JSON retains every available metric and includes human-readable display names. +Criterion’s `Time` is its slope estimate when available, otherwise its mean, +with the statistic and matching confidence level and bounds retained in JSON +and Markdown. Custom measurement units remain unchanged. `Input bytes` denotes +the declared work size, not allocated memory or a throughput rate. +Confidence levels must be strictly between zero and one, with finite, ordered +bounds. Bootstrap intervals need not contain the point estimate. Markdown +encodes report text as literal content, and terminal cells visibly escape +controls and untrusted line breaks; JSON preserves the original strings. When the JSON output path already exists, it is used as the baseline before being replaced. `--baseline` selects another report, while `--no-baseline` -disables comparison. Metrics with compatible names, units, and directions +disables comparison. Report schema 9 distinguishes these estimates from old +median-only reports. Old automatic baselines are warned about and skipped; +an explicitly selected unsupported schema is an error. Different target, +compiler, CPU, profile, or compiler flags also warn and skip comparison. +Compiler identity is captured when the executable is built, not from the +runtime `RUSTC` override or the compiler currently on `PATH`. +These checks do not prove identical hardware, features, inputs, or boundaries. +Metrics with compatible names, units, directions, and timing statistics are classified as improved, stable, or regressed. The default regression threshold is five percent; change it with `--regression-threshold`. +These percentage labels are not statistical significance or equivalence tests. + +Smoke runs (`--test`, including a forwarded native `--test`) run Criterion or +Gungraun cases briefly without collecting probe metrics or publishing reports. +Allocation, perf, and `VTune` smoke selections fall back to Criterion registration, +which must be provided even when the target declares native probe cases. +An invocation +without arguments is also smoke mode, matching Cargo’s harness-free test +invocation; pass an engine selector to request measurements directly. +A measurement selection matching no cases fails and preserves existing reports. +Failed multi-engine runs can write separate `.partial` reports but never +replace the complete rolling baseline. + +Default native artifacts and combined reports follow the Cargo output root +of the running executable, including `--target-dir`, target triples, and recognized +custom profiles. `CARGO_TARGET_DIR` identifies a target root whose name matches +the target triple without overriding a different executable root selected by Cargo. +A relocated executable in an unknown layout needs an explicit +`CARGO_TARGET_DIR`; metabench does not guess a workspace root. +Hidden worker stdout and stderr share a pipe. Each run log retains at most +1 MiB of raw output, keeping the tail with an explicit truncation marker. +Failures and timeouts show the final 8 KiB and the log path. Capture is bounded +while workers run, and inherited pipe writers cannot delay cleanup indefinitely. +`--show-engine-output` instead inherits the terminal without capturing a log. ## Command-line options @@ -258,12 +334,33 @@ selected. The `BENCH_ENGINE` environment variable may select `criterion`, `gungraun`, `perf`, `vtune`, or `allocations` when no engine selector is present. `CRITERION_HOME` and `GUNGRAUN_HOME` select the corresponding engine data directories. +Filters remain engine-specific: use Criterion arguments for Criterion and +probes, and Gungraun arguments for Gungraun. Probes apply the Criterion name +filter (and `--exact`) to declared native case identities, then run a private +indexed worker. Criterion timing, profiling, and baseline options do not +affect probe execution. A common cross-engine filter is not supported. +Native Gungraun symbols include readable registration-identity suffixes and +collision-safe hashes; run manifests retain native-to-canonical mappings. +Renamed native symbols do not automatically reconnect older native histories. ## Custom memory allocator The system allocator is used by default; `main!(..., allocator = MyAllocator)` selects another underlying allocator. +Put `allocator` after `benchmarks` or `groups` in [`main!`][__link3]. The executable +may define only one global allocator. Any existing `alloc_tracker` +instrumentation must resolve to the same Cargo package source as the metabench +tracker; a registry dependency and a separate path dependency have independent +tracking state even if their names and versions match. +The generated allocator checks one process-wide atomic flag on allocation, +reallocation, and deallocation. Tracking is enabled only in allocation workers; Criterion, +Gungraun, perf, and `VTune` bypass the tracking counters and thread-local state. +The flag check still adds overhead to allocating workloads; it is not a +zero-overhead claim. +Both paths use the same underlying allocator instance, including custom +allocators. Allocation workers track process-wide totals, with snapshots +excluding setup and teardown from reported counts.
@@ -274,3 +371,4 @@ This crate was developed as part of u64 { + black_box(value).wrapping_add(1) +} + +#[inline(never)] +fn plain(value: u64) -> u64 { + black_box(value).wrapping_add(1) +} + +#[library_benchmark] +fn annotated_control() -> u64 { + black_box(annotated(black_box(42))) +} + +#[library_benchmark] +fn plain_control() -> u64 { + black_box(plain(black_box(42))) +} + +library_benchmark_group!( + name = controls; + benchmarks = annotated_control, plain_control +); +main!(library_benchmark_groups = controls); diff --git a/crates/metabench/examples/measurement_boundaries.rs b/crates/metabench/examples/measurement_boundaries.rs new file mode 100644 index 000000000..ada03fdc4 --- /dev/null +++ b/crates/metabench/examples/measurement_boundaries.rs @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Prepared cases with unmeasured calls to annotated functions during setup. + +use std::hint::black_box; + +use criterion::{BatchSize, BenchmarkId, Criterion}; + +// Allocating destruction must follow collection; return this value to defer it. +struct Output(Vec); + +impl Drop for Output { + fn drop(&mut self) { + black_box(self.0.len()); + drop(black_box(vec![42; 64])); + } +} + +// Preparation and allocating destruction stay outside collection by returning ownership. +struct Input { + size: usize, + prepared: Vec, +} + +impl Drop for Input { + fn drop(&mut self) { + black_box(self.prepared.len()); + drop(black_box(vec![42; 64])); + } +} + +fn input(size: usize) -> Input { + Input { + size, + prepared: vec![42; 11], + } +} + +fn prepare(size: usize) -> Input { + if std::env::var_os("METABENCH_TEST_TIMEOUT").is_some() { + eprintln!("selected workload is waiting"); + std::thread::sleep(std::time::Duration::from_mins(1)); + } + assert!(std::env::var_os("METABENCH_TEST_FAIL_SETUP").is_none(), "native setup failed"); + drop(other()); + drop(allocate(input(5))); + drop(allocate(input(size))); + input(size) +} + +#[metabench::benchmark(ALLOCATE, "boundaries", "allocate", gungraun_setup = prepare)] +#[bench::small(32)] +#[bench::large(128)] +fn allocate(input: Input) -> (Input, Output) { + let output = Output(black_box(vec![42; black_box(input.size)])); + (input, output) +} + +#[metabench::benchmark(OTHER, "boundaries", "other")] +fn other() -> Vec { + black_box(vec![42; 7]) +} + +#[metabench::benchmark(CONST, "boundaries", "const")] +const fn constant() -> u64 { + 42 +} + +fn criterion_benchmarks(criterion: &mut Criterion) { + assert!( + std::env::var_os("METABENCH_TEST_NO_CRITERION").is_none(), + "Criterion must not run for native probes" + ); + if std::env::var_os("METABENCH_TEST_REQUIRE_UNTRACKED").is_some() { + let session = alloc_tracker::Session::new().no_stdout().no_file(); + let span = session.operation("untracked").measure_process().iterations(1); + drop(black_box(vec![42; 128])); + drop(span); + let report = session.to_report(); + for (_, operation) in report.operations() { + assert_eq!(operation.total_allocations_count(), 0, "Criterion must bypass allocation tracking"); + assert_eq!(operation.total_bytes_allocated(), 0, "Criterion must bypass allocation tracking"); + } + } + drop(other()); + drop(allocate(input(3))); + let mut group = criterion.benchmark_group(ALLOCATE.group_name()); + for (case, size) in [("small", 32), ("large", 128)] { + group.bench_function(BenchmarkId::new(ALLOCATE.benchmark_name(), case), |bencher| { + bencher.iter_batched(|| prepare(size), allocate, BatchSize::SmallInput); + }); + } + group.bench_function(CONST.benchmark_name(), |bencher| bencher.iter(constant)); + group.finish(); +} + +metabench::main!(criterion = criterion_benchmarks, benchmarks = [ALLOCATE, OTHER, CONST],); diff --git a/crates/metabench/examples/native_probe_contracts.rs b/crates/metabench/examples/native_probe_contracts.rs new file mode 100644 index 000000000..560d15594 --- /dev/null +++ b/crates/metabench/examples/native_probe_contracts.rs @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Native iterator cases and lifecycle hooks without Criterion registration. + +use std::hint::black_box; +use std::io::Write as _; +use std::sync::OnceLock; + +static TEST_OPTIONS: OnceLock = OnceLock::new(); + +struct TestOptions { + require_untracked: bool, + ordering: bool, + panic_workload: bool, +} + +fn event(value: &str) { + if let Some(path) = std::env::var_os("METABENCH_TEST_LIFECYCLE") { + let mut file = std::fs::OpenOptions::new() + .append(true) + .create(true) + .open(path) + .expect("open lifecycle log"); + writeln!(file, "{value}").expect("write lifecycle log"); + } +} + +fn suite_setup() { + TEST_OPTIONS.get_or_init(|| TestOptions { + require_untracked: std::env::var_os("METABENCH_TEST_REQUIRE_UNTRACKED").is_some(), + ordering: std::env::var_os("METABENCH_TEST_ORDERING").is_some(), + panic_workload: std::env::var_os("METABENCH_TEST_PANIC_WORKLOAD").is_some(), + }); + println!("native suite setup"); + event("suite setup"); +} +fn suite_teardown() { + event("suite teardown"); +} +fn group_setup() { + event("group setup"); +} +fn group_teardown() { + event("group teardown"); +} + +fn prepare(size: usize) -> Vec { + event("case setup"); + black_box(vec![1; size]) +} + +fn cleanup(output: (Vec, Vec)) { + event("case teardown"); + assert_eq!(output.0.len(), output.1.len()); + drop(output); + drop(black_box(vec![1; 256])); +} + +#[metabench::benchmark(ITER, "contracts", "iterator", gungraun_setup = prepare, gungraun_teardown = cleanup)] +#[benches::sizes(iter = [8, 16])] +#[cfg_attr(all(), bench::conditional(args = (24), setup = prepare, teardown = cleanup))] +#[cfg_attr(any(), bench::disabled(32))] +fn allocate(input: Vec) -> (Vec, Vec) { + // Windows environment lookups allocate even when the variable is absent. + let options = TEST_OPTIONS + .get() + .expect("suite setup initializes test options before workloads run"); + if options.require_untracked { + let session = alloc_tracker::Session::new().no_stdout().no_file(); + let span = session.operation("untracked").measure_process().iterations(1); + drop(black_box(vec![42; 128])); + drop(span); + for (_, operation) in session.to_report().operations() { + assert_eq!( + operation.total_allocations_count(), + 0, + "hardware workers must bypass allocation tracking" + ); + assert_eq!( + operation.total_bytes_allocated(), + 0, + "hardware workers must bypass allocation tracking" + ); + } + } + if options.ordering { + event("workload entry"); + assert!(!options.panic_workload, "native workload panic"); + } + let output = black_box(vec![42; black_box(input.len())]); + if options.ordering { + event("workload exit"); + } + (input, output) +} + +#[expect(clippy::panic, reason = "process fixture fails if a probe executes Criterion registration")] +fn criterion_benchmarks(_: &mut criterion::Criterion) { + panic!("native probes must not execute Criterion registration"); +} + +metabench::main!( + criterion = criterion_benchmarks, + gungraun = { setup = suite_setup(); teardown = suite_teardown(); }, + groups = { + CONTRACTS { + benchmarks = [ITER], + gungraun_setup = group_setup(), + gungraun_teardown = group_teardown(), + }, + }, +); diff --git a/crates/metabench/examples/threaded_allocations.rs b/crates/metabench/examples/threaded_allocations.rs new file mode 100644 index 000000000..5c731ad76 --- /dev/null +++ b/crates/metabench/examples/threaded_allocations.rs @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Deterministic process-wide allocation accounting with prestarted threads. + +#![expect(clippy::unwrap_used, reason = "process fixture failures provide sufficient context")] + +use std::hint::black_box; +use std::sync::{Arc, Barrier}; +use std::thread::{self, JoinHandle}; + +// Return ownership so final thread allocations and joins happen after collection. +struct Workers { + start: Arc, + complete: Arc, + finish: Arc, + threads: [Option>; 2], +} + +impl Drop for Workers { + fn drop(&mut self) { + self.finish.wait(); + for thread in &mut self.threads { + thread.take().unwrap().join().unwrap(); + } + } +} + +fn prepare() -> Workers { + let ready = Arc::new(Barrier::new(3)); + let start = Arc::new(Barrier::new(3)); + let complete = Arc::new(Barrier::new(3)); + let finish = Arc::new(Barrier::new(3)); + let threads = [37, 83].map(|size| { + let ready = Arc::clone(&ready); + let start = Arc::clone(&start); + let complete = Arc::clone(&complete); + let finish = Arc::clone(&finish); + Some(thread::spawn(move || { + // Initialize each thread's tracker and finish startup before collection. + drop(black_box(vec![1_u8; 2048])); + ready.wait(); + start.wait(); + drop(black_box(vec![2_u8; black_box(size)])); + drop(black_box(vec![3_u8; black_box(19)])); + complete.wait(); + finish.wait(); + drop(black_box(vec![4_u8; 4096])); + })) + }); + ready.wait(); + Workers { + start, + complete, + finish, + threads, + } +} + +#[metabench::benchmark(THREADED, "threads", "allocations", gungraun_setup = prepare)] +fn allocate_on_threads(workers: Workers) -> Workers { + workers.start.wait(); + workers.complete.wait(); + workers +} + +#[expect(clippy::panic, reason = "process fixture fails if a native probe executes Criterion registration")] +fn criterion_benchmarks(_: &mut criterion::Criterion) { + panic!("the process fixture only runs native allocation probes"); +} + +metabench::main!(criterion = criterion_benchmarks, benchmarks = [THREADED],); diff --git a/crates/metabench/src/__private.rs b/crates/metabench/src/__private.rs index 4f31803b2..c2927d342 100644 --- a/crates/metabench/src/__private.rs +++ b/crates/metabench/src/__private.rs @@ -3,12 +3,13 @@ //! Dependencies and implementation details used by macro expansions. -pub use alloc_tracker::Allocator; pub use criterion; pub use gungraun; pub use crate::allocation::begin as begin_allocation_measurement; +pub use crate::allocator::Allocator; pub use crate::perf::begin as begin_perf_measurement; +pub use crate::probe::{ProbeBenchmark, ProbeGroup, run_probe_groups}; pub use crate::runner::{EngineSet, run}; pub use crate::vtune::begin as begin_vtune_measurement; diff --git a/crates/metabench/src/allocation.rs b/crates/metabench/src/allocation.rs index 82042f83d..5c9e3daef 100644 --- a/crates/metabench/src/allocation.rs +++ b/crates/metabench/src/allocation.rs @@ -15,7 +15,7 @@ use crate::report::{Metric, MetricDirection, NativeResult}; pub(crate) const ARTIFACT_ENV: &str = "METABENCH_INTERNAL_ALLOCATION_ARTIFACT"; pub(crate) const IDENTITY_ENV: &str = "METABENCH_INTERNAL_ALLOCATION_IDENTITY"; -static ACTIVE: LazyLock = LazyLock::new(|| env::var_os(ARTIFACT_ENV).is_some()); +static ACTIVE: AtomicBool = AtomicBool::new(false); static MEASURED: AtomicBool = AtomicBool::new(false); static SESSION: LazyLock = LazyLock::new(|| Session::new().no_stdout().no_file()); @@ -27,25 +27,25 @@ struct AllocationArtifact { } #[inline] -/// Begins measuring the first workload invocation in an allocation worker. +/// Begins measuring the first explicit workload boundary in an allocation worker. #[doc(hidden)] pub fn begin() -> Option { - (*ACTIVE && !MEASURED.swap(true, Ordering::Relaxed)).then(|| { + (ACTIVE.load(Ordering::Relaxed) && !MEASURED.swap(true, Ordering::Relaxed)).then(|| { let operation = SESSION.operation("workload"); operation.measure_process().iterations(1) }) } -/// Resolves the lazy state behind [`begin`] ahead of any measurement. +/// Enables collection after validating an allocation worker's token and mode. /// -/// [`begin`] runs inside the region a profiler collects, so leaving these -/// `LazyLock`s cold would charge the first workload for a `getenv` call, the -/// surrounding one-time synchronization, and the tracking session setup. -#[cfg_attr(coverage_nightly, coverage(off))] // not exercised by the instrumented test binary; see prime()'s doc comment -#[cfg_attr(test, mutants::skip)] // warm-up only affects timing, not any observable behavior +/// Enable tracking and initialize the session before the workload boundary so +/// session setup cannot contribute allocations to the selected workload. +#[cfg_attr(coverage_nightly, coverage(off))] // exercised in generated allocation-worker executables pub(crate) fn prime() { - if *ACTIVE { + if env::var_os(ARTIFACT_ENV).is_some() { + crate::allocator::enable_tracking(); LazyLock::force(&SESSION); + ACTIVE.store(true, Ordering::Relaxed); } } @@ -63,7 +63,7 @@ pub(crate) fn write_worker_artifact() -> Result<(), Error> { .find_map(|(name, operation)| (name == "workload").then_some(operation)) .ok_or_else(|| Error::ArtifactFormat { path: path.clone(), - message: format!("benchmark '{identity}' did not execute its instrumented workload"), + message: format!("benchmark '{identity}' did not execute its generated allocation probe"), })?; if operation.total_iterations() != 1 { return Err(Error::ArtifactFormat { @@ -85,7 +85,7 @@ pub(crate) fn write_worker_artifact() -> Result<(), Error> { } pub(crate) fn read_artifact(path: &Path, native_identity: String, relative_path: String) -> Result { - let artifact: AllocationArtifact = read_json(path)?; + let artifact: AllocationArtifact = crate::artifact::read_json(path)?; Ok(NativeResult { identity: artifact.identity, native_identity, @@ -132,16 +132,15 @@ fn write_json(path: &Path, value: &impl Serialize) -> Result<(), Error> { }) } -fn read_json(path: &Path) -> Result -where - T: for<'de> Deserialize<'de>, -{ - let file = fs::File::open(path).map_err(|source| Error::ArtifactIo { - path: path.to_owned(), - source, - })?; - serde_json::from_reader(file).map_err(|source| Error::ArtifactJson { - path: path.to_owned(), - source, - }) +#[cfg_attr(coverage_nightly, coverage(off))] +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn inactive_collection_does_not_consume_the_first_workload() { + assert!(begin().is_none()); + assert!(begin().is_none()); + assert!(!MEASURED.load(Ordering::Relaxed)); + } } diff --git a/crates/metabench/src/allocator.rs b/crates/metabench/src/allocator.rs new file mode 100644 index 000000000..ff13251df --- /dev/null +++ b/crates/metabench/src/allocator.rs @@ -0,0 +1,212 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicBool, Ordering}; + +static TRACKING_ENABLED: AtomicBool = AtomicBool::new(false); + +pub(crate) fn enable_tracking() { + TRACKING_ENABLED.store(true, Ordering::Relaxed); +} + +/// Allocator used by generated entry points; tracking is enabled only in allocation workers. +#[doc(hidden)] +#[derive(Debug)] +pub struct Allocator { + inner: A, +} + +impl Allocator { + /// Uses the system allocator. + #[must_use] + pub const fn system() -> Self { + Self::new(System) + } +} + +impl Allocator { + /// Wraps the allocator shared by tracked and untracked operations. + #[must_use] + pub const fn new(inner: A) -> Self { + Self { inner } + } + + fn tracker(&self) -> alloc_tracker::Allocator> { + // Borrow the same allocator in both paths; custom allocator state must not be duplicated. + alloc_tracker::Allocator::new(AllocatorRef(&self.inner)) + } +} + +// SAFETY: Both paths forward the caller's allocation contract to the same underlying allocator. +unsafe impl GlobalAlloc for Allocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + if TRACKING_ENABLED.load(Ordering::Relaxed) { + // SAFETY: The caller supplies a valid allocation layout. + unsafe { self.tracker().alloc(layout) } + } else { + // SAFETY: The caller supplies a valid allocation layout. + unsafe { self.inner.alloc(layout) } + } + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + if TRACKING_ENABLED.load(Ordering::Relaxed) { + // SAFETY: The caller supplies a valid allocation layout. + unsafe { self.tracker().alloc_zeroed(layout) } + } else { + // SAFETY: The caller supplies a valid allocation layout. + unsafe { self.inner.alloc_zeroed(layout) } + } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + if TRACKING_ENABLED.load(Ordering::Relaxed) { + // SAFETY: The caller supplies the live allocation, original layout, and valid new size. + unsafe { self.tracker().realloc(ptr, layout, new_size) } + } else { + // SAFETY: The caller supplies the live allocation, original layout, and valid new size. + unsafe { self.inner.realloc(ptr, layout, new_size) } + } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + if TRACKING_ENABLED.load(Ordering::Relaxed) { + // SAFETY: Both paths allocate through self.inner; the caller supplies its live allocation and layout. + unsafe { self.tracker().dealloc(ptr, layout) } + } else { + // SAFETY: Both paths allocate through self.inner; the caller supplies its live allocation and layout. + unsafe { self.inner.dealloc(ptr, layout) } + } + } +} + +struct AllocatorRef<'a, A>(&'a A); + +// SAFETY: This adapter forwards all operations unchanged to the borrowed GlobalAlloc implementation. +unsafe impl GlobalAlloc for AllocatorRef<'_, A> { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + // SAFETY: The caller supplies a valid allocation layout. + unsafe { self.0.alloc(layout) } + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + // SAFETY: The caller supplies a valid allocation layout. + unsafe { self.0.alloc_zeroed(layout) } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + // SAFETY: The caller supplies the live allocation, original layout, and valid new size. + unsafe { self.0.realloc(ptr, layout, new_size) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + // SAFETY: The caller supplies a live allocation from this allocator and its original layout. + unsafe { self.0.dealloc(ptr, layout) } + } +} + +#[cfg_attr(coverage_nightly, coverage(off))] +#[cfg(test)] +mod tests { + use std::ptr::NonNull; + use std::sync::atomic::AtomicUsize; + + use alloc_tracker::Session; + + use super::*; + + #[derive(Default)] + struct CountingAllocator { + allocations: AtomicUsize, + zeroed: AtomicUsize, + reallocations: AtomicUsize, + frees: AtomicUsize, + } + + // SAFETY: All operations preserve the GlobalAlloc contract by forwarding unchanged to System. + unsafe impl GlobalAlloc for CountingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + self.allocations.fetch_add(1, Ordering::Relaxed); + // SAFETY: The caller supplies a valid allocation layout. + unsafe { System.alloc(layout) } + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + self.zeroed.fetch_add(1, Ordering::Relaxed); + // SAFETY: The caller supplies a valid allocation layout. + unsafe { System.alloc_zeroed(layout) } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + self.reallocations.fetch_add(1, Ordering::Relaxed); + // SAFETY: The caller supplies a live System allocation and valid layouts. + unsafe { System.realloc(ptr, layout, new_size) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + self.frees.fetch_add(1, Ordering::Relaxed); + // SAFETY: The caller supplies a live System allocation and its original layout. + unsafe { System.dealloc(ptr, layout) } + } + } + + struct DisableOnDrop; + + impl Drop for DisableOnDrop { + fn drop(&mut self) { + TRACKING_ENABLED.store(false, Ordering::Relaxed); + } + } + + #[test] + fn only_enabled_operations_are_tracked_and_both_paths_share_allocator_state() { + let _reset = DisableOnDrop; + let allocator = Allocator::new(CountingAllocator::default()); + let session = Session::new().no_stdout().no_file(); + let layout = Layout::from_size_align(16, 8).unwrap(); + let resized = Layout::from_size_align(32, 8).unwrap(); + + for (name, enabled) in [("disabled", false), ("enabled", true)] { + TRACKING_ENABLED.store(enabled, Ordering::Relaxed); + let span = session.operation(name).measure_process().iterations(1); + // SAFETY: The layout has non-zero size and valid alignment. + let ptr = NonNull::new(unsafe { allocator.alloc(layout) }).unwrap(); + // SAFETY: The allocation contains at least one writable byte. + unsafe { ptr.as_ptr().write(42) }; + // SAFETY: The pointer is live, its original layout matches, and the new size is valid. + let ptr = NonNull::new(unsafe { allocator.realloc(ptr.as_ptr(), layout, resized.size()) }).unwrap(); + // SAFETY: Successful reallocation preserves the initialized first byte. + assert_eq!(unsafe { ptr.as_ptr().read() }, 42); + // SAFETY: The resized layout matches this live allocation. + unsafe { allocator.dealloc(ptr.as_ptr(), resized) }; + // SAFETY: The layout has non-zero size and valid alignment. + let zeroed = NonNull::new(unsafe { allocator.alloc_zeroed(layout) }).unwrap(); + // SAFETY: The allocation contains layout.size() initialized bytes. + let bytes = unsafe { std::slice::from_raw_parts(zeroed.as_ptr(), layout.size()) }; + assert!(bytes.iter().all(|byte| *byte == 0)); + // SAFETY: The layout matches this live allocation. + unsafe { allocator.dealloc(zeroed.as_ptr(), layout) }; + drop(span); + } + + let report = session.to_report(); + for (name, operation) in report.operations() { + let (count, bytes) = if name == "enabled" { (3, 64) } else { (0, 0) }; + assert_eq!(operation.total_allocations_count(), count, "{name}"); + assert_eq!(operation.total_bytes_allocated(), bytes, "{name}"); + } + assert_eq!(allocator.inner.allocations.load(Ordering::Relaxed), 2); + assert_eq!(allocator.inner.zeroed.load(Ordering::Relaxed), 2); + assert_eq!(allocator.inner.reallocations.load(Ordering::Relaxed), 2); + assert_eq!(allocator.inner.frees.load(Ordering::Relaxed), 4); + + TRACKING_ENABLED.store(false, Ordering::Relaxed); + // SAFETY: A valid layout is supplied; the allocation is freed with the same allocator and layout. + let ptr = NonNull::new(unsafe { allocator.alloc(layout) }).unwrap(); + enable_tracking(); + // SAFETY: Enabling tracking does not change ownership of this live allocation. + unsafe { allocator.dealloc(ptr.as_ptr(), layout) }; + assert_eq!(allocator.inner.frees.load(Ordering::Relaxed), 5); + } +} diff --git a/crates/metabench/src/arguments.rs b/crates/metabench/src/arguments.rs index ba3cd3af6..9d2fac2dc 100644 --- a/crates/metabench/src/arguments.rs +++ b/crates/metabench/src/arguments.rs @@ -69,6 +69,12 @@ pub(crate) enum FailureMode { } impl Arguments { + pub(crate) fn smoke(&self) -> bool { + self.cargo_test + || self.native_args.criterion.iter().any(|argument| argument == "--test") + || self.native_args.gungraun.iter().any(|argument| argument == "--test") + } + pub(crate) fn parse() -> Result { Self::parse_from(env::args_os().skip(1), env::var_os("BENCH_ENGINE")) } @@ -110,10 +116,10 @@ impl Arguments { || modes.contains(&Mode::Perf) || modes.contains(&Mode::Vtune) { - if self.cargo_bench || !self.cargo_test { + if !self.smoke() { push_argument_unique(&mut self.native_args.criterion, "--bench"); } - if self.cargo_test { + if self.smoke() { push_argument_unique(&mut self.native_args.criterion, "--test"); } if self.list { @@ -121,10 +127,10 @@ impl Arguments { } } if modes.contains(&Mode::Gungraun) { - if self.cargo_bench { + if self.cargo_bench && !self.smoke() { push_argument_unique(&mut self.native_args.gungraun, "--bench"); } - if self.cargo_test { + if self.smoke() { push_argument_unique(&mut self.native_args.gungraun, "--test"); } if self.list { @@ -154,7 +160,11 @@ impl Arguments { let mut trailing = Vec::new(); let mut cargo_bench = false; let mut cargo_test = false; - let mut arguments = arguments.into_iter(); + let mut arguments = arguments.into_iter().peekable(); + if arguments.peek().is_none() { + // Cargo invokes harness-free `cargo test --bench` binaries without --bench. + cargo_test = true; + } while let Some(argument) = arguments.next() { if argument == OsStr::new("--") { @@ -460,6 +470,20 @@ mod tests { assert_eq!(arguments.args_for(Mode::Gungraun), ["--baseline"]); } + #[test] + fn gungraun_bench_flag_requires_cargo_bench_and_non_smoke_mode() { + for (options, expected) in [ + (&["--gungraun"][..], &[][..]), + (&["--gungraun", "--bench"][..], &["--bench"][..]), + (&["--gungraun", "--test"][..], &["--test"][..]), + (&["--gungraun", "--bench", "--test"][..], &["--test"][..]), + ] { + let mut arguments = parse(options, None).unwrap(); + arguments.finalize_native_args(&[Mode::Gungraun]).unwrap(); + assert_eq!(arguments.args_for(Mode::Gungraun), expected); + } + } + #[test] fn parses_perf_selection_and_arguments() { let mut arguments = Arguments::parse_from(["--perf".into(), "--perf-arg".into(), "--event=cpu-clock".into()], None).unwrap(); @@ -505,6 +529,28 @@ mod tests { )); } + #[test] + fn smoke_intent_covers_cargo_and_forwarded_native_arguments() { + for options in [ + &[][..], + &["--test"][..], + &["--criterion", "--criterion-arg=--test"][..], + &["--allocations", "--", "--test"][..], + ] { + let mut arguments = Arguments::parse_from(options.iter().map(OsString::from), None).unwrap(); + arguments.finalize_native_args(&arguments.modes.clone()).unwrap(); + assert!(arguments.smoke()); + assert!(arguments.criterion_args().contains(&"--test".into())); + assert!(!arguments.criterion_args().contains(&"--bench".into())); + } + for options in [&["--bench"][..], &["--criterion"][..], &["--allocations"][..]] { + let mut arguments = Arguments::parse_from(options.iter().map(OsString::from), None).unwrap(); + arguments.finalize_native_args(&arguments.modes.clone()).unwrap(); + assert!(!arguments.smoke()); + assert!(arguments.criterion_args().contains(&"--bench".into())); + } + } + #[test] fn defaults_exclude_opt_in_perf_and_keep_going() { let arguments = parse(&[], None).unwrap(); diff --git a/crates/metabench/src/artifact.rs b/crates/metabench/src/artifact.rs index 55020c4d3..202cc1f3a 100644 --- a/crates/metabench/src/artifact.rs +++ b/crates/metabench/src/artifact.rs @@ -6,15 +6,21 @@ use std::ffi::OsString; use std::fmt::Debug; use std::fs::{File, OpenOptions}; use std::hash::Hash; -use std::io::{self, BufReader, BufWriter, Write as _}; +use std::io::{self, BufWriter, Read as _, Write as _}; +#[cfg(unix)] +use std::os::unix::fs::{DirBuilderExt as _, OpenOptionsExt as _}; +#[cfg(windows)] +use std::os::windows::fs::{MetadataExt as _, OpenOptionsExt as _}; use std::path::{Path, PathBuf}; use std::process::{self, ExitStatus}; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; use std::{env, fs}; -use gungraun_summary::either_or_both::EitherOrBoth; -use gungraun_summary::v6::{BenchmarkSummary, Metric as GungraunMetric, MetricsSummary, ToolMetricSummary, ValgrindTool}; +use either_or_both::EitherOrBoth; +use gungraun_runner::api::ValgrindTool; +use gungraun_runner::metrics::model::{Metric as GungraunMetric, MetricsSummary}; +use gungraun_runner::summary::model::{BenchmarkSummary, ToolMetricSummary}; use serde::{Deserialize, Serialize}; use crate::error::Error; @@ -48,6 +54,8 @@ struct EngineRunManifest { success: bool, exit_code: Option, discovered_native_identities: Vec, + #[serde(default)] + native_to_canonical: BTreeMap, } impl ArtifactDirectory { @@ -72,7 +80,7 @@ impl ArtifactDirectory { let gungraun = gungraun_runs .as_ref() .map_or_else(|| run.join("gungraun"), |runs| runs.join(&run_name)); - fs::create_dir_all(&run).map_err(|source| Error::ArtifactIo { path: run.clone(), source })?; + Self::create_private_run(&run)?; let mut marked_runs = BTreeSet::from([run.clone()]); let mut active_locks = vec![mark_run_active(&run)?]; Self::prune_owned_runs(&runs, &run)?; @@ -84,6 +92,7 @@ impl ArtifactDirectory { .flatten() { if marked_runs.insert(current.to_owned()) { + Self::create_private_run(current)?; active_locks.push(mark_run_active(current)?); } Self::prune_owned_runs(owned_runs, current)?; @@ -97,6 +106,26 @@ impl ArtifactDirectory { }) } + fn create_private_run(path: &Path) -> Result<(), Error> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|source| Error::ArtifactIo { + path: parent.to_owned(), + source, + })?; + } + let directory = fs::DirBuilder::new(); + #[cfg(unix)] + let directory = { + let mut directory = directory; + directory.mode(0o700); + directory + }; + directory.create(path).map_err(|source| Error::ArtifactIo { + path: path.to_owned(), + source, + }) + } + fn prune_owned_runs(runs: &Path, current: &Path) -> Result<(), Error> { let mut directories = fs::read_dir(runs) .map_err(|source| Error::ArtifactIo { @@ -175,6 +204,7 @@ pub(crate) fn write_run_manifest( success: status.is_some_and(|status| status.success()), exit_code: status.and_then(|status| status.code()), discovered_native_identities: Vec::new(), + native_to_canonical: BTreeMap::new(), }; write_json(&manifest_root.join("manifests").join(format!("{engine}.json")), &manifest) } @@ -188,6 +218,10 @@ pub(crate) fn record_discovered_identities(manifest_root: &Path, engine: &str, r .collect::>() .into_iter() .collect(); + manifest.native_to_canonical = results + .iter() + .map(|result| (result.native_identity.clone(), result.identity.clone())) + .collect(); write_json(&path, &manifest) } @@ -201,7 +235,8 @@ struct CriterionMetadata { #[derive(Debug, Deserialize)] struct CriterionEstimates { - median: CriterionEstimate, + slope: Option, + mean: CriterionEstimate, } #[derive(Debug, Deserialize)] @@ -212,6 +247,7 @@ struct CriterionEstimate { #[derive(Debug, Deserialize)] struct CriterionConfidenceInterval { + confidence_level: f64, lower_bound: f64, upper_bound: f64, } @@ -234,13 +270,18 @@ pub(crate) fn parse_criterion( let identity = identities.resolve_criterion(&native_identity); let estimates_path = path.with_file_name("estimates.json"); let estimates = read_json::(&estimates_path)?; - let median = estimates.median; - validate_criterion_estimate(&estimates_path, &median)?; - let mut metric = Metric::float("criterion", "median", median.point_estimate, MetricDirection::LowerIsBetter); + let (statistic, estimate) = match estimates.slope { + Some(slope) => ("slope", slope), + None => ("mean", estimates.mean), + }; + validate_criterion_estimate(&estimates_path, &estimate)?; + let mut metric = Metric::float("criterion", "time", estimate.point_estimate, MetricDirection::LowerIsBetter); metric.unit = unit.map(str::to_owned); - metric.lower_bound = Some(median.confidence_interval.lower_bound); - metric.upper_bound = Some(median.confidence_interval.upper_bound); - let mut metrics = BTreeMap::from([("median".to_owned(), metric)]); + metric.statistic = Some(statistic.to_owned()); + metric.confidence_level = Some(estimate.confidence_interval.confidence_level); + metric.lower_bound = Some(estimate.confidence_interval.lower_bound); + metric.upper_bound = Some(estimate.confidence_interval.upper_bound); + let mut metrics = BTreeMap::from([("time".to_owned(), metric)]); if let Some((name, value)) = metadata.throughput.as_ref().and_then(parse_criterion_throughput) { metrics.insert(name.clone(), Metric::integer("criterion", &name, value, MetricDirection::Unknown)); } @@ -263,10 +304,12 @@ pub(crate) fn parse_gungraun(root: &Path, artifact_root: &Path, identities: &[Be let identities = CanonicalIdentityIndex::new(identities); let mut results = Vec::new(); for path in paths { - validate_json_size(&path, MAX_ARTIFACT_JSON_BYTES)?; - let summary = gungraun_summary::v6::parse(&path).map_err(|error| Error::ArtifactFormat { - path: path.clone(), - message: error.to_string(), + let summary = read_json::(&path).map_err(|error| match error { + Error::ArtifactJson { path, source } => Error::ArtifactFormat { + path, + message: source.to_string(), + }, + error => error, })?; if summary.version != "6" { return Err(Error::ArtifactFormat { @@ -311,11 +354,7 @@ struct PerfStatRecord { } pub(crate) fn parse_perf(path: &Path, artifact_root: &Path, identity: String, native_identity: String) -> Result { - validate_json_size(path, MAX_ARTIFACT_JSON_BYTES)?; - let contents = fs::read_to_string(path).map_err(|source| Error::ArtifactIo { - path: path.to_owned(), - source, - })?; + let contents = read_artifact_text(path, "JSON")?; let mut metrics = BTreeMap::new(); for (index, line) in contents.lines().enumerate() { if line.trim().is_empty() { @@ -431,11 +470,7 @@ const VTUNE_EVENT_COUNT_COLUMN: &str = "Hardware Event Count:Self"; /// against a live install) parses correctly as long as it is quoted per CSV /// convention, e.g. `INST_RETIRED.ANY,"1,234,567"`. pub(crate) fn parse_vtune(path: &Path, artifact_root: &Path, identity: String, native_identity: String) -> Result { - validate_artifact_size(path, MAX_ARTIFACT_JSON_BYTES, "CSV")?; - let contents = fs::read_to_string(path).map_err(|source| Error::ArtifactIo { - path: path.to_owned(), - source, - })?; + let contents = read_artifact_text(path, "CSV")?; let mut lines = contents.lines().enumerate(); let (name_column, count_column, expected_field_count) = match lines.next() { Some((_, header)) => { @@ -547,10 +582,6 @@ fn criterion_identity(metadata: &CriterionMetadata) -> String { .join("/") } -pub(crate) fn allocation_criterion_identity(native_identity: &str, identities: &[BenchmarkIdentity]) -> Option { - CanonicalIdentityIndex::new(identities).resolve_allocation_criterion(native_identity) -} - fn parse_criterion_throughput(value: &serde_json::Value) -> Option<(String, u64)> { let object = value.as_object()?; let (kind, value) = object.iter().next()?; @@ -573,14 +604,13 @@ fn gungraun_canonical_identity(summary: &BenchmarkSummary) -> String { } let group = parts.last().copied().unwrap_or("gungraun"); match &summary.id { - Some(id) if id != "default" => format!("{group}/{}/{id}", summary.function_name), + Some(id) => format!("{group}/{}/{id}", summary.function_name), _ => format!("{group}/{}", summary.function_name), } } struct CanonicalIdentityIndex { criterion: BTreeMap, - allocation_criterion: BTreeMap, gungraun: BTreeMap, } @@ -590,20 +620,11 @@ impl CanonicalIdentityIndex { .iter() .map(|identity| (identity.criterion_identity(), identity.canonical_identity())) .collect(); - let allocation_criterion = identities - .iter() - .filter(|identity| identity.allocation_tracking()) - .map(|identity| (identity.criterion_identity(), identity.canonical_identity())) - .collect(); let gungraun = identities .iter() .map(|identity| (identity.gungraun_identity(), identity.canonical_identity())) .collect(); - Self { - criterion, - allocation_criterion, - gungraun, - } + Self { criterion, gungraun } } fn resolve_criterion(&self, native: &str) -> String { @@ -614,10 +635,6 @@ impl CanonicalIdentityIndex { Self::resolve(&self.criterion, native) } - fn resolve_allocation_criterion(&self, native: &str) -> Option { - Self::resolve(&self.allocation_criterion, native) - } - fn resolve_gungraun(&self, native: &str, default_identity: String) -> String { Self::resolve(&self.gungraun, native).unwrap_or(default_identity) } @@ -700,7 +717,13 @@ fn validate_criterion_estimate(path: &Path, estimate: &CriterionEstimate) -> Res let point = estimate.point_estimate; let lower = estimate.confidence_interval.lower_bound; let upper = estimate.confidence_interval.upper_bound; - if !point.is_finite() || !lower.is_finite() || !upper.is_finite() { + let confidence = estimate.confidence_interval.confidence_level; + if !confidence.is_finite() || confidence <= 0.0 || confidence >= 1.0 { + Err(Error::ArtifactFormat { + path: path.to_owned(), + message: format!("Criterion confidence level must be between zero and one; found {confidence}"), + }) + } else if !point.is_finite() || !lower.is_finite() || !upper.is_finite() { Err(Error::ArtifactFormat { path: path.to_owned(), message: "contains a non-finite Criterion estimate".to_owned(), @@ -710,10 +733,10 @@ fn validate_criterion_estimate(path: &Path, estimate: &CriterionEstimate) -> Res path: path.to_owned(), message: format!("Criterion estimates must be non-negative; found {lower}..{point}..{upper}"), }) - } else if lower > point || point > upper { + } else if lower > upper { Err(Error::ArtifactFormat { path: path.to_owned(), - message: format!("Criterion confidence interval must contain its point estimate; found {lower}..{point}..{upper}"), + message: format!("Criterion confidence interval bounds must be ordered; found {lower}..{upper}"), }) } else { Ok(()) @@ -847,43 +870,112 @@ fn mark_run_active(path: &Path) -> Result { Ok(file) } -fn read_json(path: &Path) -> Result { - validate_json_size(path, MAX_ARTIFACT_JSON_BYTES)?; - let file = fs::File::open(path).map_err(|source| Error::ArtifactIo { +pub(crate) fn read_json(path: &Path) -> Result { + let bytes = read_regular_file(path, MAX_ARTIFACT_JSON_BYTES, "JSON")?; + serde_json::from_slice(&bytes).map_err(|source| Error::ArtifactJson { + path: path.to_owned(), + source, + }) +} + +fn read_artifact_text(path: &Path, format: &str) -> Result { + let bytes = read_regular_file(path, MAX_ARTIFACT_JSON_BYTES, format)?; + String::from_utf8(bytes).map_err(|source| Error::ArtifactIo { + path: path.to_owned(), + source: io::Error::new(io::ErrorKind::InvalidData, source), + }) +} + +pub(crate) fn read_regular_file(path: &Path, limit: u64, format: &str) -> Result, Error> { + let file = open_regular_file(path)?; + read_bounded_file(file, path, limit, format) +} + +/// Admits the same regular-file handle later consumed by the bounded reader. +fn open_regular_file(path: &Path) -> Result { + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + // Reject final links and avoid waiting for a FIFO writer before checking the file type. + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + #[cfg(windows)] + { + // Inspect rather than follow reparse points, denying concurrent writes and path replacement. + const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000; + const FILE_SHARE_READ: u32 = 1; + options.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT).share_mode(FILE_SHARE_READ); + } + let file = options.open(path).map_err(|source| Error::ArtifactIo { path: path.to_owned(), source, })?; - serde_json::from_reader(BufReader::new(file)).map_err(|source| Error::ArtifactJson { + let metadata = file.metadata().map_err(|source| Error::ArtifactIo { path: path.to_owned(), source, - }) + })?; + #[cfg(windows)] + { + reject_windows_reparse_point(path, metadata.file_attributes())?; + } + if !metadata.is_file() { + return Err(Error::ArtifactFormat { + path: path.to_owned(), + message: "artifact must be a regular file".to_owned(), + }); + } + Ok(file) } -fn validate_json_size(path: &Path, limit: u64) -> Result<(), Error> { - validate_artifact_size(path, limit, "JSON") +#[cfg(windows)] +fn reject_windows_reparse_point(path: &Path, attributes: u32) -> Result<(), Error> { + const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x400; + if attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return Err(Error::ArtifactFormat { + path: path.to_owned(), + message: "artifact must not be a reparse point".to_owned(), + }); + } + Ok(()) } -/// Rejects an artifact larger than `limit`, naming its `format` (e.g. -/// `"JSON"` or `"CSV"`) in the error so the diagnostic matches the actual -/// file being validated rather than always describing it as JSON. -fn validate_artifact_size(path: &Path, limit: u64, format: &str) -> Result<(), Error> { - let length = fs::metadata(path) +fn read_bounded_file(file: File, path: &Path, limit: u64, format: &str) -> Result, Error> { + let length = file + .metadata() .map_err(|source| Error::ArtifactIo { path: path.to_owned(), source, })? .len(); if length > limit { - Err(Error::ArtifactFormat { + return Err(Error::ArtifactFormat { path: path.to_owned(), message: format!("{format} file is {length} bytes; limit is {limit} bytes"), - }) - } else { - Ok(()) + }); + } + read_bounded_stream(file, path, limit, format) +} + +fn read_bounded_stream(reader: impl io::Read, path: &Path, limit: u64, format: &str) -> Result, Error> { + let mut bytes = Vec::new(); + reader + .take(limit.saturating_add(1)) + .read_to_end(&mut bytes) + .map_err(|source| Error::ArtifactIo { + path: path.to_owned(), + source, + })?; + if u64::try_from(bytes.len()).unwrap_or(u64::MAX) > limit { + return Err(Error::ArtifactFormat { + path: path.to_owned(), + message: format!("{format} stream exceeds the {limit}-byte limit"), + }); } + Ok(bytes) } -fn write_json(path: &Path, value: &impl Serialize) -> Result<(), Error> { +pub(crate) fn write_json(path: &Path, value: &impl Serialize) -> Result<(), Error> { let parent = path .parent() .filter(|parent| !parent.as_os_str().is_empty()) @@ -924,9 +1016,238 @@ fn write_json(path: &Path, value: &impl Serialize) -> Result<(), Error> { #[cfg(test)] mod tests { use std::fmt::Write as _; + #[cfg(unix)] + use std::os::unix::fs::{PermissionsExt as _, symlink}; use super::*; + #[test] + fn private_run_creation_creates_parents_and_rejects_existing_paths() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("runs").join("new-run"); + ArtifactDirectory::create_private_run(&path).unwrap(); + assert!(path.is_dir()); + assert!(matches!( + ArtifactDirectory::create_private_run(&path), + Err(Error::ArtifactIo { path: failed, source }) + if failed == path && source.kind() == io::ErrorKind::AlreadyExists + )); + let file = directory.path().join("file"); + fs::write(&file, b"preserve me").unwrap(); + assert!(matches!( + ArtifactDirectory::create_private_run(&file.join("run")), + Err(Error::ArtifactIo { path: failed, .. }) if failed == file + )); + assert_eq!(fs::read(file).unwrap(), b"preserve me"); + } + + #[cfg(windows)] + #[test] + fn json_admission_rejects_reparse_attributes_without_symlink_privileges() { + let path = Path::new("artifact.json"); + for attributes in [0, 0x80, 0x1000] { + reject_windows_reparse_point(path, attributes).unwrap(); + } + for attributes in [0x400, 0x480, 0x1400] { + assert!(matches!( + reject_windows_reparse_point(path, attributes), + Err(Error::ArtifactFormat { path: failed, message }) + if failed == path && message == "artifact must not be a reparse point" + )); + } + } + + #[test] + fn json_admission_limits_the_opened_stream_and_ignores_path_replacement() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("artifact.json"); + fs::write(&path, b"null").unwrap(); + let file = open_regular_file(&path).unwrap(); + assert_eq!(file.metadata().unwrap().len(), 4); + #[cfg(unix)] + { + fs::write(&path, b"null ").unwrap(); + assert!( + read_bounded_stream(file, &path, 4, "JSON") + .unwrap_err() + .to_string() + .contains("stream exceeds") + ); + } + #[cfg(windows)] + { + // Admission denies write sharing while the artifact handle is open. + fs::write(&path, b"null ").unwrap_err(); + assert_eq!(read_bounded_file(file, &path, 4, "JSON").unwrap(), b"null"); + fs::write(&path, b"null ").unwrap(); + } + assert!( + read_bounded_stream(io::Cursor::new(b"null "), &path, 4, "JSON") + .unwrap_err() + .to_string() + .contains("stream exceeds") + ); + assert!(matches!( + read_bounded_file(open_regular_file(&path).unwrap(), &path, 4, "JSON"), + Err(Error::ArtifactFormat { .. }) + )); + fs::write(&path, b"null").unwrap(); + assert_eq!( + read_bounded_file(open_regular_file(&path).unwrap(), &path, 4, "JSON").unwrap(), + b"null" + ); + #[cfg(unix)] + { + let file = open_regular_file(&path).unwrap(); + fs::rename(&path, directory.path().join("original.json")).unwrap(); + fs::write(&path, b"not the opened artifact").unwrap(); + assert_eq!(read_bounded_file(file, &path, 4, "JSON").unwrap(), b"null"); + } + } + + #[test] + fn json_admission_rejects_oversized_allocation_identities() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("allocation.json"); + let mut file = File::create(&path).unwrap(); + file.write_all(b"{\"identity\":\"").unwrap(); + for _ in 0..(MAX_ARTIFACT_JSON_BYTES / 8_192) { + file.write_all(&[b'x'; 8_192]).unwrap(); + } + file.write_all(b"\",\"allocated_bytes\":1,\"allocation_count\":1}").unwrap(); + drop(file); + assert!(matches!( + crate::allocation::read_artifact(&path, "native".to_owned(), "allocation.json".to_owned()), + Err(Error::ArtifactFormat { .. }) + )); + read_json::(directory.path()).unwrap_err(); + } + + #[test] + fn hardware_artifact_admission_rejects_invalid_files_before_parsing() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("artifact"); + for csv in [false, true] { + let parse = if csv { parse_vtune } else { parse_perf }; + let parse = |path: &Path| parse(path, directory.path(), "group/case".into(), "native/case".into()); + parse(directory.path()).unwrap_err(); + fs::write(&path, [0xff]).unwrap(); + assert!(matches!( + parse(&path), + Err(Error::ArtifactIo { source, .. }) if source.kind() == io::ErrorKind::InvalidData + )); + File::create(&path).unwrap().set_len(MAX_ARTIFACT_JSON_BYTES + 1).unwrap(); + assert!(matches!( + parse(&path), + Err(Error::ArtifactFormat { message, .. }) + if message.starts_with(if csv { "CSV file" } else { "JSON file" }) + )); + #[cfg(unix)] + { + let link = directory.path().join(if csv { "csv-link" } else { "json-link" }); + symlink(&path, &link).unwrap(); + assert!(matches!(parse(&link), Err(Error::ArtifactIo { .. }))); + } + #[cfg(target_os = "linux")] + { + let fifo = directory.path().join(if csv { "csv-fifo" } else { "json-fifo" }); + nix::unistd::mkfifo(&fifo, nix::sys::stat::Mode::S_IRUSR | nix::sys::stat::Mode::S_IWUSR).unwrap(); + assert!(matches!( + parse(&fifo), + Err(Error::ArtifactFormat { message, .. }) if message.contains("regular file") + )); + } + } + } + + #[test] + fn bounded_hardware_streams_check_growth_and_retain_the_output_format() { + let path = Path::new("hardware-output"); + for format in ["JSON", "CSV"] { + assert_eq!(read_bounded_stream(io::Cursor::new(b"data"), path, 4, format).unwrap(), b"data"); + assert!(matches!( + read_bounded_stream(io::Cursor::new(b"data grows"), path, 4, format), + Err(Error::ArtifactFormat { message, .. }) + if message == format!("{format} stream exceeds the 4-byte limit") + )); + } + } + + #[cfg(unix)] + #[test] + fn json_admission_rejects_links_and_fifos_without_waiting_for_a_writer() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("artifact.json"); + let link = directory.path().join("link.json"); + fs::write(&path, b"null").unwrap(); + symlink(&path, &link).unwrap(); + read_json::(&link).unwrap_err(); + #[cfg(target_os = "linux")] + { + let fifo = directory.path().join("fifo.json"); + nix::unistd::mkfifo(&fifo, nix::sys::stat::Mode::S_IRUSR | nix::sys::stat::Mode::S_IWUSR).unwrap(); + assert!( + read_json::(&fifo) + .unwrap_err() + .to_string() + .contains("regular file") + ); + } + } + + #[cfg(unix)] + #[test] + fn artifact_run_directories_are_private() { + let directory = tempfile::tempdir().unwrap(); + let artifacts = ArtifactDirectory::create_with_homes( + &directory.path().join("target"), + Some(directory.path().join("criterion").into_os_string()), + Some(directory.path().join("gungraun").into_os_string()), + ) + .unwrap(); + for path in [artifacts.path().to_owned(), artifacts.criterion_path(), artifacts.gungraun_path()] { + assert_eq!(fs::metadata(path).unwrap().permissions().mode() & 0o777, 0o700); + } + } + + #[test] + fn explicit_default_case_is_distinct_and_joins_matching_engine_results() { + let source: serde_json::Value = read_json(&fixture("gungraun/minimal/summary.json")).unwrap(); + let directory = tempfile::tempdir().unwrap(); + let mut all = Vec::new(); + for id in [None, Some("default")] { + let path = directory.path().join(if id.is_some() { "named" } else { "unnamed" }); + let mut summary = source.clone(); + summary["id"] = id.into(); + write_json(&path.join("summary.json"), &summary).unwrap(); + let results = parse_gungraun(&path, directory.path(), &[]).unwrap(); + let expected = if id.is_some() { + "adapter/metabench_adapter/default" + } else { + "adapter/metabench_adapter" + }; + assert_eq!(results[0].native_identity, expected); + assert_eq!(results[0].identity, expected); + all.extend(results); + } + let mut time = Metric::float("criterion", "ns", 1.0, MetricDirection::LowerIsBetter); + time.statistic = Some("mean".to_owned()); + time.lower_bound = Some(1.0); + time.upper_bound = Some(1.0); + time.confidence_level = Some(0.95); + all.push(NativeResult { + identity: "adapter/metabench_adapter/default".to_owned(), + native_identity: "adapter/metabench_adapter/default".to_owned(), + source: "criterion".to_owned(), + metrics: BTreeMap::from([("time".to_owned(), time)]), + raw_artifacts: Vec::new(), + }); + let report = crate::report::BenchmarkReport::from_results(all, vec![], String::new()).unwrap(); + assert_eq!(report.entries.len(), 2); + let named = report.entries.iter().find(|entry| entry.identity.ends_with("/default")).unwrap(); + assert_eq!(named.results.len(), 2); + } + #[test] fn accepts_only_performance_measurement_tools() { for tool in [ValgrindTool::Callgrind, ValgrindTool::Cachegrind, ValgrindTool::DHAT] { @@ -953,7 +1274,7 @@ mod tests { fs::write(new.join("benchmark.json"), r#"{"group_id":"group","function_id":"bench"}"#).unwrap(); fs::write( new.join("estimates.json"), - format!(r#"{{"median":{{"confidence_interval":{{"lower_bound":{lower},"upper_bound":{upper}}},"point_estimate":{point}}}}}"#), + format!(r#"{{"mean":{{"confidence_interval":{{"confidence_level":0.95,"lower_bound":{lower},"upper_bound":{upper}}},"point_estimate":{point}}}}}"#), ) .unwrap(); } @@ -970,12 +1291,14 @@ mod tests { assert_eq!(result.native_identity, "group/bench"); assert_eq!(result.source, "criterion"); assert_eq!(result.raw_artifacts, ["new/benchmark.json", "new/estimates.json"]); - let median = &result.metrics["median"]; + let median = &result.metrics["time"]; assert_eq!(median.value, crate::report::MetricValue::Float(20.0)); assert_eq!(median.direction, MetricDirection::LowerIsBetter); assert_eq!(median.unit.as_deref(), Some("ns")); assert_eq!(median.lower_bound, Some(10.0)); assert_eq!(median.upper_bound, Some(30.0)); + assert_eq!(median.statistic.as_deref(), Some("mean")); + assert_eq!(median.confidence_level, Some(0.95)); assert_eq!( result.metrics["declared_throughput.bytes"].value, crate::report::MetricValue::Integer(10) @@ -985,6 +1308,63 @@ mod tests { assert_eq!(index.resolve_criterion("other/bench/case_0"), "other/bench/case_0"); } + #[test] + fn criterion_preserves_primary_estimate_and_its_confidence_interval() { + let directory = tempfile::tempdir().unwrap(); + write_criterion(directory.path(), 20.0, 10.0, 30.0); + let path = directory.path().join("new/estimates.json"); + let mut estimates: serde_json::Value = read_json(&path).unwrap(); + estimates["slope"] = serde_json::json!({ + "point_estimate": 100.0, + "confidence_interval": {"lower_bound": 90.0, "upper_bound": 110.0, "confidence_level": 0.99} + }); + estimates["median"] = serde_json::json!({ + "point_estimate": 2.0, + "confidence_interval": {"lower_bound": 1.0, "upper_bound": 3.0, "confidence_level": 0.90} + }); + write_json(&path, &estimates).unwrap(); + let results = parse_criterion(directory.path(), directory.path(), &[], Some("cycles")).unwrap(); + let time = &results[0].metrics["time"]; + assert_eq!(time.value, crate::report::MetricValue::Float(100.0)); + assert_eq!(time.lower_bound, Some(90.0)); + assert_eq!(time.upper_bound, Some(110.0)); + assert_eq!(time.confidence_level, Some(0.99)); + assert_eq!(time.statistic.as_deref(), Some("slope")); + assert_eq!(time.unit.as_deref(), Some("cycles")); + estimates["slope"] = serde_json::Value::Null; + write_json(&path, &estimates).unwrap(); + let results = parse_criterion(directory.path(), directory.path(), &[], None).unwrap(); + assert_eq!(results[0].metrics["time"].value, crate::report::MetricValue::Float(20.0)); + assert_eq!(results[0].metrics["time"].statistic.as_deref(), Some("mean")); + assert_eq!(results[0].metrics["time"].lower_bound, Some(10.0)); + assert_eq!(results[0].metrics["time"].upper_bound, Some(30.0)); + assert_eq!(results[0].metrics["time"].confidence_level, Some(0.95)); + assert_eq!(results[0].metrics["time"].unit, None); + for confidence in [0.0, 1.0, -0.1] { + estimates["mean"]["confidence_interval"]["confidence_level"] = serde_json::json!(confidence); + write_json(&path, &estimates).unwrap(); + assert!(matches!( + parse_criterion(directory.path(), directory.path(), &[], None), + Err(Error::ArtifactFormat { .. }) + )); + } + } + + #[test] + fn criterion_accepts_zero_and_equal_interval_bounds() { + for (point, lower, upper) in [(0.0, 0.0, 0.0), (1.0, 0.0, 2.0), (1.0, 1.0, 1.0)] { + let estimate = CriterionEstimate { + point_estimate: point, + confidence_interval: CriterionConfidenceInterval { + lower_bound: lower, + upper_bound: upper, + confidence_level: 0.95, + }, + }; + validate_criterion_estimate(Path::new("estimates.json"), &estimate).unwrap(); + } + } + #[test] fn generated_native_identities_resolve_only_on_segment_boundaries() { let identities = [BenchmarkIdentity::__new("group", "bench", "native_group", "native_bench", true)]; @@ -1038,7 +1418,7 @@ mod tests { Err(Error::ArtifactFormat { .. }) )); - for (point, lower, upper) in [(-1.0, 0.0, 1.0), (1.0, -1.0, 2.0), (1.0, 2.0, 3.0), (3.0, 1.0, 2.0)] { + for (point, lower, upper) in [(-1.0, 0.0, 1.0), (1.0, -1.0, 2.0), (1.0, 3.0, 2.0), (1.0, 0.0, -1.0)] { let directory = tempfile::tempdir().unwrap(); write_criterion(directory.path(), point, lower, upper); assert!(matches!( @@ -1048,6 +1428,55 @@ mod tests { } } + #[test] + fn criterion_accepts_bootstrap_intervals_excluding_the_point() { + let directory = tempfile::tempdir_in(env!("CARGO_MANIFEST_DIR")).unwrap(); + for point in [1.0, 4.0] { + write_criterion(directory.path(), point, 2.0, 3.0); + let path = directory.path().join("new/estimates.json"); + let mut estimates: serde_json::Value = read_json(&path).unwrap(); + for statistic in ["mean", "slope"] { + if statistic == "slope" { + estimates["slope"] = estimates["mean"].clone(); + write_json(&path, &estimates).unwrap(); + } + let results = parse_criterion(directory.path(), directory.path(), &[], None).unwrap(); + let time = &results[0].metrics["time"]; + assert_eq!(time.value, crate::report::MetricValue::Float(point)); + assert_eq!(time.lower_bound, Some(2.0)); + assert_eq!(time.upper_bound, Some(3.0)); + assert_eq!(time.statistic.as_deref(), Some(statistic)); + assert_eq!(time.confidence_level, Some(0.95)); + } + } + } + + #[test] + fn criterion_rejects_non_finite_estimate_components() { + for invalid in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + for component in 0..4 { + let mut estimate = CriterionEstimate { + point_estimate: 2.0, + confidence_interval: CriterionConfidenceInterval { + lower_bound: 1.0, + upper_bound: 3.0, + confidence_level: 0.95, + }, + }; + match component { + 0 => estimate.point_estimate = invalid, + 1 => estimate.confidence_interval.lower_bound = invalid, + 2 => estimate.confidence_interval.upper_bound = invalid, + _ => estimate.confidence_interval.confidence_level = invalid, + } + assert!(matches!( + validate_criterion_estimate(Path::new("estimates.json"), &estimate), + Err(Error::ArtifactFormat { .. }) + )); + } + } + } + #[test] fn parses_gungraun_schema_and_strips_native_diffs() { let source = fs::read_to_string(fixture("gungraun/minimal/summary.json")).unwrap(); @@ -1390,7 +1819,18 @@ mod tests { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("large.json"); fs::File::create(&path).unwrap().set_len(11).unwrap(); - assert!(matches!(validate_json_size(&path, 10), Err(Error::ArtifactFormat { .. }))); + assert!(matches!(read_regular_file(&path, 10, "JSON"), Err(Error::ArtifactFormat { .. }))); + } + + #[test] + fn run_manifest_retains_native_to_canonical_identity_mapping() { + let directory = tempfile::tempdir().unwrap(); + write_run_manifest(directory.path(), directory.path(), "target", "criterion", &[], None).unwrap(); + let results = parse_criterion(&fixture("criterion/minimal"), &fixture("criterion"), &[], None).unwrap(); + record_discovered_identities(directory.path(), "criterion", &results).unwrap(); + let manifest: EngineRunManifest = read_json(&directory.path().join("manifests/criterion.json")).unwrap(); + assert_eq!(manifest.discovered_native_identities, [results[0].native_identity.clone()]); + assert_eq!(manifest.native_to_canonical[&results[0].native_identity], results[0].identity); } #[test] diff --git a/crates/metabench/src/entrypoint.rs b/crates/metabench/src/entrypoint.rs index ab66e30c5..ccb7d929b 100644 --- a/crates/metabench/src/entrypoint.rs +++ b/crates/metabench/src/entrypoint.rs @@ -9,6 +9,7 @@ macro_rules! __metabench_main { criterion = $criterion_benchmarks:path, gungraun = { $($gungraun:tt)* }, identities = $identities:expr + , probes = $probes:expr $(, allocator = $allocator:path)? $(,)? ) => { @@ -21,6 +22,7 @@ macro_rules! __metabench_main { criterion_unit = ::core::option::Option::Some("ns"); gungraun = { $($gungraun)* }; identities = $identities; + probes = $probes; allocator = [$($allocator)?]; ); }; @@ -32,6 +34,7 @@ macro_rules! __metabench_main { }, gungraun = { $($gungraun:tt)* }, identities = $identities:expr + , probes = $probes:expr $(, allocator = $allocator:path)? $(,)? ) => { @@ -44,6 +47,7 @@ macro_rules! __metabench_main { criterion_unit = $crate::__metabench_main!(@unit $( $unit)?); gungraun = { $($gungraun)* }; identities = $identities; + probes = $probes; allocator = [$($allocator)?]; ); }; @@ -59,6 +63,7 @@ macro_rules! __metabench_main { criterion_unit = $criterion_unit:expr; gungraun = { $($gungraun:tt)* }; identities = $identities:expr; + probes = $probes:expr; allocator = [$($allocator:path)?]; ) => { $crate::__metabench_main!(@allocator $($allocator)?); @@ -82,6 +87,10 @@ macro_rules! __metabench_main { pub(super) fn run() { main(); } + + pub(super) fn run_probe() -> Result<(), String> { + $crate::__private::run_probe_groups($probes, __run_setup, __run_teardown) + } } fn main() { @@ -90,6 +99,7 @@ macro_rules! __metabench_main { __metabench_criterion::run, __metabench_gungraun::run, $criterion_unit, + __metabench_gungraun::run_probe, ), $identities, env!("CARGO_CRATE_NAME"), diff --git a/crates/metabench/src/error.rs b/crates/metabench/src/error.rs index f53d4f9a6..bddf4bee8 100644 --- a/crates/metabench/src/error.rs +++ b/crates/metabench/src/error.rs @@ -6,9 +6,12 @@ use std::path::PathBuf; use std::time::Duration; use std::{error, fmt, io}; +use crate::runner::terminal_text; + #[derive(Debug)] pub(crate) enum Error { InvalidMode(String), + Probe(String), UnsupportedMode { mode: &'static str, reason: &'static str, @@ -25,7 +28,6 @@ pub(crate) enum Error { ConflictingOutputOptions, ConflictingBaselineOptions, CurrentExecutable(io::Error), - CurrentDirectory(io::Error), HelpOutput(io::Error), ConsoleOutput(io::Error), Spawn { @@ -91,6 +93,7 @@ impl fmt::Display for Error { #[expect(clippy::renamed_function_params, reason = "the descriptive name improves readability")] fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::Probe(message) => write!(formatter, "native benchmark probe failed: {}", terminal_text(message)), Self::InvalidMode(mode) => { write!( formatter, @@ -130,7 +133,6 @@ impl fmt::Display for Error { Self::ConflictingOutputOptions => formatter.write_str("--no-output, --output, --export-json, and --export-md cannot conflict"), Self::ConflictingBaselineOptions => formatter.write_str("--no-baseline cannot be combined with --baseline"), Self::CurrentExecutable(error) => write!(formatter, "failed to locate the benchmark executable: {error}"), - Self::CurrentDirectory(error) => write!(formatter, "failed to locate the benchmark working directory: {error}"), Self::HelpOutput(error) => write!(formatter, "failed to write help text: {error}"), Self::ConsoleOutput(error) => write!(formatter, "failed to write console output: {error}"), Self::Spawn { mode, source } => write!(formatter, "failed to launch the {mode} worker: {source}"), @@ -144,9 +146,9 @@ impl fmt::Display for Error { Self::CreateWorkerToken(error) => write!(formatter, "failed to create an internal worker token: {error}"), Self::ConsumeWorkerToken(error) => write!(formatter, "failed to consume an internal worker token: {error}"), Self::PerfControl(error) => write!(formatter, "failed to control Linux perf counters: {error}"), - Self::PerfWorkloadNotMeasured => formatter.write_str("the perf worker did not execute an instrumented workload"), + Self::PerfWorkloadNotMeasured => formatter.write_str("the perf worker did not execute its generated probe"), Self::VtuneControl(error) => write!(formatter, "failed to control the VTune collection: {error}"), - Self::VtuneWorkloadNotMeasured => formatter.write_str("the vtune worker did not execute an instrumented workload"), + Self::VtuneWorkloadNotMeasured => formatter.write_str("the vtune worker did not execute its generated probe"), Self::MultipleModeFailures(failures) => { formatter.write_str("multiple benchmark failures occurred")?; for failure in failures { @@ -162,7 +164,9 @@ impl fmt::Display for Error { "unsupported report schema {found} in {}; expected schema {expected}", path.display() ), - Self::DuplicateReportBenchmark(identity) => write!(formatter, "report contains duplicate benchmark '{identity}'"), + Self::DuplicateReportBenchmark(identity) => { + write!(formatter, "report contains duplicate benchmark '{}'", terminal_text(identity)) + } Self::Metadata(message) => write!(formatter, "failed to collect environment metadata: {message}"), Self::InvalidThreshold(value) => write!( formatter, @@ -183,7 +187,6 @@ impl error::Error for Error { fn source(&self) -> Option<&(dyn error::Error + 'static)> { match self { Self::CurrentExecutable(source) - | Self::CurrentDirectory(source) | Self::HelpOutput(source) | Self::ConsoleOutput(source) | Self::CreateWorkerToken(source) @@ -205,6 +208,23 @@ impl error::Error for Error { mod tests { use super::*; + #[test] + fn probe_diagnostics_escape_untrusted_messages() { + let message = "unsupported option --bad\n\u{1b}[31m\r\t\u{061c}\u{200e}\u{200f}\u{202e}\u{2066}"; + let error = Error::Probe(message.to_owned()); + assert_eq!( + error.to_string(), + r"native benchmark probe failed: unsupported option --bad\n\u{1b}[31m\r\t\u{61c}\u{200e}\u{200f}\u{202e}\u{2066}" + ); + assert!(matches!(error, Error::Probe(raw) if raw == message)); + } + + #[test] + fn duplicate_report_diagnostics_escape_disk_supplied_identities() { + let error = Error::DuplicateReportBenchmark("case\n\u{1b}[31m\u{202e}".into()); + assert_eq!(error.to_string(), r"report contains duplicate benchmark 'case\n\u{1b}[31m\u{202e}'"); + } + #[test] fn source_returns_the_wrapped_io_error_for_control_protocol_variants() { let error = Error::VtuneControl(io::Error::other("control pipe closed")); diff --git a/crates/metabench/src/identity.rs b/crates/metabench/src/identity.rs index 1779e0b9e..27d0ab643 100644 --- a/crates/metabench/src/identity.rs +++ b/crates/metabench/src/identity.rs @@ -72,10 +72,6 @@ impl BenchmarkIdentity { pub(crate) fn gungraun_identity(self) -> String { join_identity(self.gungraun_group, self.gungraun_benchmark) } - - pub(crate) const fn allocation_tracking(self) -> bool { - self.allocation_tracking - } } fn join_identity(group: &str, benchmark: &str) -> String { diff --git a/crates/metabench/src/lib.rs b/crates/metabench/src/lib.rs index d9a0a0446..0c4683ae3 100644 --- a/crates/metabench/src/lib.rs +++ b/crates/metabench/src/lib.rs @@ -78,6 +78,11 @@ //! //! # Measurement engines //! +//! A **native case** is a case supplied by a benchmark's Gungraun declarations, +//! including its implicit single case when no case attribute is present. +//! **Probes** are the separate allocation, perf, and `VTune` workers that execute +//! these cases; they do not use Criterion registrations. +//! //! As mentioned, metabench supports five distinct benchmark tools: Criterion, Gungraun, allocation //! tracking, Linux `perf`, and Intel `VTune`: //! @@ -89,15 +94,19 @@ //! enabled, system calls, and heap activity. //! //! - **Allocation tracking** reports allocation count and allocated bytes for -//! one invocation of every discovered Criterion case. +//! one invocation of every declared native case. Counts +//! are process-wide, so concurrent threads can contribute allocations. //! //! - **Linux `perf`** reports supported hardware and software counters for one -//! invocation of every discovered Criterion case. Counters cover only the -//! annotated workload. Use repeated `--perf-arg` options for additional +//! invocation of every declared native case. Collection starts and stops in +//! a private caller around the workload, excluding input preparation and +//! returned-value teardown. The collection window can include caller and +//! control overhead; it is not a claim of exact function-only counters. +//! Use repeated `--perf-arg` options for additional //! `perf stat` arguments. //! //! - **Intel `VTune`** reports hardware event counts for one invocation of every -//! discovered Criterion case, using the same annotated-workload +//! declared native case, using the same private caller //! measurement as Linux `perf`. Because `VTune`'s hardware-event collection //! is a statistical sampler rather than an exact counter, reported counts //! are sampling estimates that can read zero for a short workload, and the @@ -118,9 +127,6 @@ //! //! Some limitations around engines: //! -//! - Benchmark functions declared with `const fn` can be measured by Criterion and Gungraun, but not -//! by allocation tracking, `perf`, or `VTune`. -//! //! - Gungraun and Linux `perf` are only available on Linux platforms. `VTune` is //! available on Linux and Windows, provided a `vtune` installation is //! reachable on `PATH`. `VTune`'s integration test exercises the control @@ -128,6 +134,34 @@ //! `vtune` fixture (see `tests/spawned_benchmark.rs`), rather than a real //! `VTune` install. //! +//! # Workload boundaries +//! +//! The attribute preserves a non-inlined workload symbol without injecting +//! runtime collector checks into the function. Criterion callbacks call it +//! directly, and Gungraun uses its normal adapter. Neither executes the private +//! allocation, perf, or `VTune` collection controls. +//! +//! Allocation and hardware workers reuse the existing `#[bench::...]` and +//! `#[benches::...]` declarations through separate generated adapters. Native +//! suite and group hooks run outside collection. Case argument evaluation and +//! setup precede collection; teardown and returned-value destruction follow it. +//! Calls to annotated functions during setup cannot consume the measurement. +//! Collection guards stop on unwinding. +//! +//! No new registration language or public measurement helper is required. Probes +//! do not run Criterion registration or infer its arbitrary callbacks. A +//! Criterion-only case without a corresponding native declaration has no probe +//! result. Matching case names do not prove matching Criterion inputs or cleanup; +//! those remain author responsibilities. +//! +//! Probe execution reuses native case arguments and lifecycle hooks, not +//! Valgrind configuration such as sandboxes or environment overrides. Configure +//! hardware tools with their own forwarded arguments. +//! +//! Input destruction inside the benchmark function is included; return owned +//! inputs with the output if their destruction should happen outside collection. +//! Functions declared with `const fn` can also have runtime probes. +//! //! # Configure Criterion //! //! Provide a Criterion factory to customize measurements or other Criterion @@ -147,6 +181,9 @@ //! //! # Configure Gungraun //! +//! The installed `gungraun-runner` must match the Gungraun library version +//! (currently 0.19.4). +//! //! Benchmark-level `gungraun_config`, `gungraun_setup`, and //! `gungraun_teardown` options work with native `#[bench::...]` and //! `#[benches::...]` cases: @@ -209,12 +246,51 @@ //! metadata, environment information, measured engines, and their metrics. //! JSON retains every available metric and includes human-readable display //! names. +//! Criterion's `Time` is its slope estimate when available, otherwise its mean, +//! with the statistic and matching confidence level and bounds retained in JSON +//! and Markdown. Custom measurement units remain unchanged. `Input bytes` denotes +//! the declared work size, not allocated memory or a throughput rate. +//! Confidence levels must be strictly between zero and one, with finite, ordered +//! bounds. Bootstrap intervals need not contain the point estimate. Markdown +//! encodes report text as literal content, and terminal cells visibly escape +//! controls and untrusted line breaks; JSON preserves the original strings. //! //! When the JSON output path already exists, it is used as the baseline before //! being replaced. `--baseline` selects another report, while `--no-baseline` -//! disables comparison. Metrics with compatible names, units, and directions +//! disables comparison. Report schema 9 distinguishes these estimates from old +//! median-only reports. Old automatic baselines are warned about and skipped; +//! an explicitly selected unsupported schema is an error. Different target, +//! compiler, CPU, profile, or compiler flags also warn and skip comparison. +//! Compiler identity is captured when the executable is built, not from the +//! runtime `RUSTC` override or the compiler currently on `PATH`. +//! These checks do not prove identical hardware, features, inputs, or boundaries. +//! Metrics with compatible names, units, directions, and timing statistics //! are classified as improved, stable, or regressed. The default regression //! threshold is five percent; change it with `--regression-threshold`. +//! These percentage labels are not statistical significance or equivalence tests. +//! +//! Smoke runs (`--test`, including a forwarded native `--test`) run Criterion or +//! Gungraun cases briefly without collecting probe metrics or publishing reports. +//! Allocation, perf, and `VTune` smoke selections fall back to Criterion registration, +//! which must be provided even when the target declares native probe cases. +//! An invocation +//! without arguments is also smoke mode, matching Cargo's harness-free test +//! invocation; pass an engine selector to request measurements directly. +//! A measurement selection matching no cases fails and preserves existing reports. +//! Failed multi-engine runs can write separate `.partial` reports but never +//! replace the complete rolling baseline. +//! +//! Default native artifacts and combined reports follow the Cargo output root +//! of the running executable, including `--target-dir`, target triples, and recognized +//! custom profiles. `CARGO_TARGET_DIR` identifies a target root whose name matches +//! the target triple without overriding a different executable root selected by Cargo. +//! A relocated executable in an unknown layout needs an explicit +//! `CARGO_TARGET_DIR`; metabench does not guess a workspace root. +//! Hidden worker stdout and stderr share a pipe. Each run log retains at most +//! 1 MiB of raw output, keeping the tail with an explicit truncation marker. +//! Failures and timeouts show the final 8 KiB and the log path. Capture is bounded +//! while workers run, and inherited pipe writers cannot delay cleanup indefinitely. +//! `--show-engine-output` instead inherits the terminal without capturing a log. //! //! # Command-line options //! @@ -253,17 +329,39 @@ //! `gungraun`, `perf`, `vtune`, or `allocations` when no engine selector is present. //! `CRITERION_HOME` and `GUNGRAUN_HOME` select the corresponding engine data //! directories. +//! Filters remain engine-specific: use Criterion arguments for Criterion and +//! probes, and Gungraun arguments for Gungraun. Probes apply the Criterion name +//! filter (and `--exact`) to declared native case identities, then run a private +//! indexed worker. Criterion timing, profiling, and baseline options do not +//! affect probe execution. A common cross-engine filter is not supported. +//! Native Gungraun symbols include readable registration-identity suffixes and +//! collision-safe hashes; run manifests retain native-to-canonical mappings. +//! Renamed native symbols do not automatically reconnect older native histories. //! //! # Custom memory allocator //! //! The system allocator //! is used by default; `main!(..., allocator = MyAllocator)` selects another //! underlying allocator. +//! Put `allocator` after `benchmarks` or `groups` in [`main!`]. The executable +//! may define only one global allocator. Any existing `alloc_tracker` +//! instrumentation must resolve to the same Cargo package source as the metabench +//! tracker; a registry dependency and a separate path dependency have independent +//! tracking state even if their names and versions match. +//! The generated allocator checks one process-wide atomic flag on allocation, +//! reallocation, and deallocation. Tracking is enabled only in allocation workers; Criterion, +//! Gungraun, perf, and `VTune` bypass the tracking counters and thread-local state. +//! The flag check still adds overhead to allocating workloads; it is not a +//! zero-overhead claim. +//! Both paths use the same underlying allocator instance, including custom +//! allocators. Allocation workers track process-wide totals, with snapshots +//! excluding setup and teardown from reported counts. #[doc(hidden)] pub mod __private; mod allocation; +mod allocator; mod arguments; mod artifact; mod entrypoint; @@ -272,9 +370,12 @@ mod identity; mod metric_display; mod mode; mod perf; +mod probe; +mod probe_error; mod report; mod runner; mod vtune; +mod worker_log; #[doc(inline)] pub use identity::BenchmarkIdentity; diff --git a/crates/metabench/src/metric_display.rs b/crates/metabench/src/metric_display.rs index f161c486e..d4a75b535 100644 --- a/crates/metabench/src/metric_display.rs +++ b/crates/metabench/src/metric_display.rs @@ -31,17 +31,18 @@ impl TableMetric { fn descriptor(key: &str) -> (Cow<'static, str>, u8, u8) { let known = match key { - "criterion/median" => ("Time", 1, 1), + "criterion/time" => ("Time", 1, 1), + "criterion/median" => ("Median time", 1, 1), "alloc_tracker/Allocations" => ("Allocs", 1, 12), - "alloc_tracker/Allocated bytes" => ("Bytes", 1, 13), + "alloc_tracker/Allocated bytes" => ("Allocated bytes", 1, 13), "gungraun.callgrind/Ir" => ("Instr", 1, 2), "gungraun.cachegrind/Ir" => ("Instr", 1, 3), - "gungraun.callgrind/EstimatedCycles" => ("Cycles", 1, 4), + "gungraun.callgrind/EstimatedCycles" => ("Est cycles", 1, 4), "perf/instructions" => ("HW Instr", 1, 5), "perf/cycles" => ("HW Cycles", 1, 6), "criterion/declared_throughput.bits" => ("Bits", 1, 7), - "criterion/declared_throughput.bytes" => ("Bytes", 1, 8), - "criterion/declared_throughput.bytes_decimal" => ("Decimal Bytes", 1, 9), + "criterion/declared_throughput.bytes" => ("Input bytes", 1, 8), + "criterion/declared_throughput.bytes_decimal" => ("Input bytes (decimal)", 1, 9), "criterion/declared_throughput.elements" => ("Elements", 1, 10), "criterion/declared_throughput.elements_and_bytes" => ("Elements And Bytes", 1, 11), @@ -201,4 +202,23 @@ mod tests { assert_eq!(metric.label, "INST RETIRED ANY"); assert_eq!(metric.priority, 10); } + + #[test] + fn primary_metric_descriptors_preserve_labels_and_order() { + for (engine, key, label, rank) in [ + ("criterion", "time", "Time", 1), + ("criterion", "median", "Median time", 1), + ("gungraun.callgrind", "Ir", "Instr", 2), + ("gungraun.cachegrind", "Ir", "Instr", 3), + ("gungraun.callgrind", "EstimatedCycles", "Est cycles", 4), + ("criterion", "declared_throughput.bits", "Bits", 7), + ("criterion", "declared_throughput.bytes", "Input bytes", 8), + ("criterion", "declared_throughput.bytes_decimal", "Input bytes (decimal)", 9), + ("alloc_tracker", "Allocated bytes", "Allocated bytes", 13), + ] { + let metric = TableMetric::new(engine.to_owned(), key.to_owned()); + assert_eq!(metric.label, label); + assert_eq!(metric.sort_key(), (1, rank, engine, key)); + } + } } diff --git a/crates/metabench/src/perf.rs b/crates/metabench/src/perf.rs index f94aba354..9013e70d4 100644 --- a/crates/metabench/src/perf.rs +++ b/crates/metabench/src/perf.rs @@ -7,8 +7,8 @@ use std::io::{self, Read as _, Write as _}; #[cfg(target_os = "linux")] use std::os::fd::AsFd as _; use std::path::PathBuf; +use std::sync::Mutex; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{LazyLock, Mutex}; #[cfg(target_os = "linux")] use nix::poll::{PollFd, PollFlags, poll}; @@ -18,15 +18,7 @@ use crate::error::Error; pub(crate) const ACK_ENV: &str = "METABENCH_INTERNAL_PERF_ACK"; pub(crate) const CONTROL_ENV: &str = "METABENCH_INTERNAL_PERF_CONTROL"; -// `perf` control is only ever driven by our own Linux-only worker process -// (see `runner::run_perf`), which is the only thing that sets `CONTROL_ENV`. -// Still, force this to `false` off Linux regardless of the environment: -// `Control::command` only guards its FIFO read with a poll timeout under -// `#[cfg(target_os = "linux")]`, so contaminated environment variables on -// another platform (for example inherited across a `cargo bench` re-exec) -// would otherwise attempt an unbounded blocking read on a FIFO that is -// never written to. -static ACTIVE: LazyLock = LazyLock::new(|| cfg!(target_os = "linux") && env::var_os(CONTROL_ENV).is_some()); +static ACTIVE: AtomicBool = AtomicBool::new(false); static ERROR: Mutex> = Mutex::new(None); static MEASURED: AtomicBool = AtomicBool::new(false); @@ -48,7 +40,7 @@ impl Drop for Guard { #[inline] #[doc(hidden)] pub fn begin() -> Option { - if !*ACTIVE || MEASURED.swap(true, Ordering::Relaxed) { + if !ACTIVE.load(Ordering::Relaxed) || MEASURED.swap(true, Ordering::Relaxed) { return None; } match Control::connect().and_then(|mut control| { @@ -63,16 +55,13 @@ pub fn begin() -> Option { } } -/// Resolves the environment probe behind [`begin`] ahead of any measurement. +/// Enables collection after validating a perf worker's token and mode. /// -/// [`begin`] runs inside the region a profiler collects, so leaving its -/// `LazyLock` cold would charge the first workload for a `getenv` call and the -/// surrounding one-time synchronization. Callgrind attributes that to the code -/// under test, inflating every reported instruction count by a fixed amount. +/// Initialize the collector before a private probe prepares and invokes its +/// workload. Ordinary Criterion callbacks do not call [`begin`]. #[cfg_attr(coverage_nightly, coverage(off))] // not exercised by the instrumented test binary; see prime()'s doc comment -#[cfg_attr(test, mutants::skip)] // warm-up only affects timing, not any observable behavior pub(crate) fn prime() { - let _active = *ACTIVE; + ACTIVE.store(cfg!(target_os = "linux") && env::var_os(CONTROL_ENV).is_some(), Ordering::Relaxed); } pub(crate) fn finish_worker() -> Result<(), Error> { @@ -139,3 +128,55 @@ fn required_path(name: &str) -> Result { .map(PathBuf::from) .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("missing {name}"))) } + +#[cfg_attr(coverage_nightly, coverage(off))] +#[cfg(test)] +mod tests { + use std::process::Command; + + use super::*; + + #[test] + fn collector_activation_and_failed_begin_are_isolated() { + const FIXTURE: &str = "METABENCH_TEST_PERF_STATE"; + if let Some(scenario) = env::var_os(FIXTURE) { + if scenario == "forced" { + ACTIVE.store(true, Ordering::Relaxed); + assert!(begin().is_none()); + assert!(MEASURED.load(Ordering::Relaxed)); + assert!(matches!( + finish_worker(), + Err(Error::PerfControl(error)) if error.kind() == io::ErrorKind::NotFound + )); + assert!(begin().is_none()); + return; + } + ACTIVE.store(true, Ordering::Relaxed); + prime(); + assert_eq!( + ACTIVE.load(Ordering::Relaxed), + scenario == "configured" && cfg!(target_os = "linux") + ); + assert!(begin().is_none()); + assert_eq!( + MEASURED.load(Ordering::Relaxed), + scenario == "configured" && cfg!(target_os = "linux") + ); + assert!(finish_worker().is_err()); + return; + } + let directory = tempfile::tempdir().unwrap(); + for scenario in ["disabled", "configured", "forced"] { + let mut command = Command::new(env::current_exe().unwrap()); + command + .args(["--exact", "perf::tests::collector_activation_and_failed_begin_are_isolated"]) + .env(FIXTURE, scenario) + .env_remove(CONTROL_ENV) + .env_remove(ACK_ENV); + if scenario == "configured" { + command.env(CONTROL_ENV, directory.path().join("missing-control")); + } + assert!(command.status().unwrap().success()); + } + } +} diff --git a/crates/metabench/src/probe.rs b/crates/metabench/src/probe.rs new file mode 100644 index 000000000..39b834297 --- /dev/null +++ b/crates/metabench/src/probe.rs @@ -0,0 +1,525 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::collections::BTreeSet; +use std::ffi::OsString; +use std::path::Path; +use std::{env, panic}; + +use gungraun::__internal::{InternalLibFunctionKind, InternalMacroLibBench}; +use serde::{Deserialize, Serialize}; + +use crate::BenchmarkIdentity; +use crate::probe_error::{ + DuplicateProbeBackendIdentity, DuplicateProbeIdentity, InvalidProbeArguments, MissingProbeDiscoveryPath, ProbeCaseLimit, ProbeError, + ProbeIndexOutOfBounds, ProbeSelectionMismatch, +}; + +pub(crate) const DISCOVERY_ENV: &str = "METABENCH_INTERNAL_PROBE_DISCOVERY"; +// Conservative headroom for large suites while bounding discovery records and worker fan-out. +pub(crate) const MAX_PROBE_CASES: usize = 10_000; + +#[doc(hidden)] +#[derive(Debug)] +pub struct ProbeBenchmark { + pub identity: BenchmarkIdentity, + pub cases: &'static [InternalMacroLibBench], +} + +#[doc(hidden)] +#[derive(Debug)] +pub struct ProbeGroup { + pub benchmarks: &'static [ProbeBenchmark], + pub setup: fn(bool) -> bool, + pub teardown: fn(bool) -> bool, +} + +#[derive(Debug, Deserialize, Serialize)] +pub(crate) struct ProbeTarget { + pub(crate) identity: String, + pub(crate) native_identity: String, + group: usize, + benchmark: usize, + case: usize, + iteration: Option, +} + +impl ProbeTarget { + pub(crate) fn arguments(&self) -> Vec { + vec![ + "--metabench-probe".into(), + self.group.to_string().into(), + self.benchmark.to_string().into(), + self.case.to_string().into(), + self.iteration.map_or_else(|| "-".to_owned(), |index| index.to_string()).into(), + ] + } +} + +fn case_identity(mut identity: String, case: &InternalMacroLibBench, iteration: Option) -> String { + if let Some(id) = case.id_display { + identity.push('/'); + identity.push_str(id); + } + if let Some(index) = iteration { + identity.push('_'); + identity.push_str(&index.to_string()); + } + identity +} + +fn with_lifecycle( + setup: fn(bool) -> bool, + teardown: fn(bool) -> bool, + body: impl FnOnce() -> Result<(), ProbeError>, +) -> Result<(), ProbeError> { + setup(true); + let result = panic::catch_unwind(panic::AssertUnwindSafe(body)); + teardown(true); + match result { + Ok(result) => result, + Err(payload) => panic::resume_unwind(payload), + } +} + +fn execute_case(case: &InternalMacroLibBench, iteration: Option) -> Result<(), ProbeError> { + match (case.func, iteration) { + (InternalLibFunctionKind::Default(run), None) => { + run(); + Ok(()) + } + (InternalLibFunctionKind::Iter(run), Some(index)) => { + let count = run(None); + if index >= count { + return Err(ProbeIndexOutOfBounds::new("iterator", index, count).into()); + } + run(Some(index)); + Ok(()) + } + _ => Err(ProbeSelectionMismatch::new(matches!(case.func, InternalLibFunctionKind::Iter(_)), iteration).into()), + } +} + +fn discover_targets(groups: &[ProbeGroup]) -> Result, ProbeError> { + let mut targets = Vec::new(); + let mut identities = BTreeSet::new(); + let mut native_identities = BTreeSet::new(); + for (group_index, group) in groups.iter().enumerate() { + with_lifecycle(group.setup, group.teardown, || { + for (benchmark_index, benchmark) in group.benchmarks.iter().enumerate() { + for (case_index, case) in benchmark.cases.iter().enumerate() { + let (count, iterator) = match case.func { + InternalLibFunctionKind::Default(_) => (1, false), + InternalLibFunctionKind::Iter(count) => { + let count = count(None); + if count > MAX_PROBE_CASES.saturating_sub(targets.len()) { + return Err(ProbeCaseLimit::new(MAX_PROBE_CASES).into()); + } + (count, true) + } + }; + for index in 0..count { + let iteration = iterator.then_some(index); + if targets.len() == MAX_PROBE_CASES { + return Err(ProbeCaseLimit::new(MAX_PROBE_CASES).into()); + } + let identity = case_identity(benchmark.identity.to_string(), case, iteration); + let native_identity = case_identity(benchmark.identity.gungraun_identity(), case, iteration); + if !identities.insert(identity.clone()) { + return Err(DuplicateProbeIdentity::new(identity).into()); + } + if !native_identities.insert(native_identity.clone()) { + return Err(DuplicateProbeBackendIdentity::new(native_identity).into()); + } + targets.push(ProbeTarget { + native_identity, + identity, + group: group_index, + benchmark: benchmark_index, + case: case_index, + iteration, + }); + } + } + } + Ok(()) + })?; + } + Ok(targets) +} + +fn publish_targets(output: Option, targets: &[ProbeTarget]) -> Result<(), ProbeError> { + let output = output.ok_or_else(MissingProbeDiscoveryPath::new)?; + crate::artifact::write_json(Path::new(&output), &targets)?; + Ok(()) +} + +/// Executes only generated probe adapters, reusing the pinned native case table. +/// +/// # Errors +/// +/// Returns an error for invalid worker arguments, indices, or discovery output. +#[doc(hidden)] +pub fn run_probe_groups(groups: &[ProbeGroup], suite_setup: fn(bool) -> bool, suite_teardown: fn(bool) -> bool) -> Result<(), String> { + let arguments = env::args().skip(1).collect::>(); + run_probe_groups_inner(groups, suite_setup, suite_teardown, &arguments).map_err(|error| error.to_string()) +} + +fn run_probe_groups_inner( + groups: &[ProbeGroup], + suite_setup: fn(bool) -> bool, + suite_teardown: fn(bool) -> bool, + arguments: &[String], +) -> Result<(), ProbeError> { + let listing = arguments == ["--metabench-probe-list"]; + let selected = if listing { + None + } else { + if arguments.len() != 5 || arguments[0] != "--metabench-probe" { + return Err(InvalidProbeArguments::new().into()); + } + let parse = |index: usize| arguments[index].parse::().map_err(ProbeError::from); + Some(( + parse(1)?, + parse(2)?, + parse(3)?, + if arguments[4] == "-" { None } else { Some(parse(4)?) }, + )) + }; + with_lifecycle(suite_setup, suite_teardown, || { + if let Some((group, benchmark, case, iteration)) = selected { + let group = groups + .get(group) + .ok_or_else(|| ProbeIndexOutOfBounds::new("group", group, groups.len()))?; + let benchmark = group + .benchmarks + .get(benchmark) + .ok_or_else(|| ProbeIndexOutOfBounds::new("benchmark", benchmark, group.benchmarks.len()))?; + let case = benchmark + .cases + .get(case) + .ok_or_else(|| ProbeIndexOutOfBounds::new("case", case, benchmark.cases.len()))?; + with_lifecycle(group.setup, group.teardown, || execute_case(case, iteration)) + } else { + let targets = discover_targets(groups)?; + publish_targets(env::var_os(DISCOVERY_ENV), &targets) + } + }) +} + +#[cfg_attr(coverage_nightly, coverage(off))] +#[cfg(test)] +mod tests { + use std::cell::{Cell, RefCell}; + use std::error::Error as _; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + + static SETUP: AtomicUsize = AtomicUsize::new(0); + static TEARDOWN: AtomicUsize = AtomicUsize::new(0); + + thread_local! { + static ITERATION_COUNT: Cell = const { Cell::new(0) }; + static COUNT_CALLS: Cell = const { Cell::new(0) }; + static EXECUTIONS: RefCell>> = const { RefCell::new(Vec::new()) }; + } + + fn default_case() { + EXECUTIONS.with_borrow_mut(|calls| calls.push(None)); + } + + fn iterator_case(index: Option) -> usize { + match index { + None => COUNT_CALLS.set(COUNT_CALLS.get() + 1), + Some(index) => EXECUTIONS.with_borrow_mut(|calls| calls.push(Some(index))), + } + ITERATION_COUNT.get() + } + + const fn native_case(func: InternalLibFunctionKind) -> InternalMacroLibBench { + InternalMacroLibBench { + args_display: None, + config: None, + consts_display: None, + func, + id_display: Some("case"), + } + } + + fn reset_callbacks(count: usize) { + ITERATION_COUNT.set(count); + COUNT_CALLS.set(0); + EXECUTIONS.with_borrow_mut(Vec::clear); + } + + #[test] + fn case_identity_distinguishes_explicit_default_from_absent_id() { + let mut case = native_case(InternalLibFunctionKind::Default(default_case)); + case.id_display = None; + assert_eq!(case_identity("group/workload".into(), &case, None), "group/workload"); + case.id_display = Some("default"); + assert_eq!(case_identity("group/workload".into(), &case, None), "group/workload/default"); + assert_eq!(case_identity("group/workload".into(), &case, Some(0)), "group/workload/default_0"); + } + + #[test] + fn selection_checks_kind_and_iterator_bounds_before_execution() { + for count in [0_usize, 1, 3] { + for iteration in [None, Some(0), Some(count.saturating_sub(1)), Some(count)] { + for iterator in [false, true] { + reset_callbacks(count); + let case = native_case(if iterator { + InternalLibFunctionKind::Iter(iterator_case) + } else { + InternalLibFunctionKind::Default(default_case) + }); + let result = execute_case(&case, iteration); + let valid = if iterator { + iteration.is_some_and(|index| index < count) + } else { + iteration.is_none() + }; + if valid { + result.unwrap(); + EXECUTIONS.with_borrow(|calls| assert_eq!(calls, &[iteration])); + } else { + let error = result.unwrap_err(); + if iterator && let Some(index) = iteration { + let source = error.source().unwrap().downcast_ref::().unwrap(); + assert_eq!((source.level, source.index, source.count), ("iterator", index, count)); + } else { + let source = error.source().unwrap().downcast_ref::().unwrap(); + assert_eq!((source.iterator, source.iteration), (iterator, iteration)); + } + EXECUTIONS.with_borrow(|calls| assert_eq!(calls, &[] as &[Option])); + } + assert_eq!(COUNT_CALLS.get(), usize::from(iterator && iteration.is_some())); + } + } + } + } + + #[test] + fn discovery_enforces_exact_and_remaining_case_budgets_without_execution() { + static DEFAULT: [InternalMacroLibBench; 1] = [native_case(InternalLibFunctionKind::Default(default_case))]; + static ITERATOR: [InternalMacroLibBench; 1] = [native_case(InternalLibFunctionKind::Iter(iterator_case))]; + static DEFAULT_BENCHMARKS: [ProbeBenchmark; 1] = [ProbeBenchmark { + identity: BenchmarkIdentity::__new("group", "default", "native_group", "native_default", true), + cases: &DEFAULT, + }]; + static ITERATOR_BENCHMARKS: [ProbeBenchmark; 1] = [ProbeBenchmark { + identity: BenchmarkIdentity::__new("group", "iterator", "native_group", "native_iterator", true), + cases: &ITERATOR, + }]; + let group = |benchmarks| ProbeGroup { + benchmarks, + setup: |enabled| enabled, + teardown: |enabled| enabled, + }; + for prefix in [0, 1] { + for count in [0, 1, 9_999, 10_000, 10_001] { + reset_callbacks(count); + let mut groups = Vec::new(); + if prefix == 1 { + groups.push(group(&DEFAULT_BENCHMARKS)); + } + groups.push(group(&ITERATOR_BENCHMARKS)); + let result = discover_targets(&groups); + if prefix + count <= 10_000 { + let targets = result.unwrap(); + assert_eq!(targets.len(), prefix + count); + for (index, target) in targets.iter().skip(prefix).enumerate() { + assert_eq!( + (target.group, target.benchmark, target.case, target.iteration), + (prefix, 0, 0, Some(index)) + ); + assert_eq!(target.identity, format!("group/iterator/case_{index}")); + assert_eq!(target.native_identity, format!("native_group/native_iterator/case_{index}")); + } + } else { + let error = result.unwrap_err(); + assert_eq!(error.source().unwrap().downcast_ref::().unwrap().limit, 10_000); + } + assert_eq!(COUNT_CALLS.get(), 1); + EXECUTIONS.with_borrow(|calls| assert_eq!(calls, &[] as &[Option])); + } + } + for count in [10_000, 10_001] { + reset_callbacks(9_999); + let mut groups = vec![group(&ITERATOR_BENCHMARKS)]; + groups.extend((9_999..count).map(|_| group(&DEFAULT_BENCHMARKS))); + let result = discover_targets(&groups); + if count == 10_000 { + assert_eq!(result.unwrap().len(), count); + } else { + let error = result.unwrap_err(); + assert_eq!(error.source().unwrap().downcast_ref::().unwrap().limit, 10_000); + } + assert_eq!(COUNT_CALLS.get(), 1); + EXECUTIONS.with_borrow(|calls| assert_eq!(calls, &[] as &[Option])); + } + } + + #[test] + fn discovery_rejects_colliding_iterator_and_declared_case_identities() { + const ITERATOR: InternalMacroLibBench = InternalMacroLibBench { + id_display: Some("sizes"), + ..native_case(InternalLibFunctionKind::Iter(iterator_case)) + }; + const DECLARED: InternalMacroLibBench = InternalMacroLibBench { + id_display: Some("sizes_0"), + ..native_case(InternalLibFunctionKind::Default(default_case)) + }; + static ITERATOR_FIRST: [ProbeBenchmark; 1] = [ProbeBenchmark { + identity: BenchmarkIdentity::__new("group", "iterator", "native_group", "native_iterator", true), + cases: &[ITERATOR, DECLARED], + }]; + static DECLARED_FIRST: [ProbeBenchmark; 1] = [ProbeBenchmark { + identity: BenchmarkIdentity::__new("group", "iterator", "native_group", "native_iterator", true), + cases: &[DECLARED, ITERATOR], + }]; + for benchmarks in [&ITERATOR_FIRST, &DECLARED_FIRST] { + reset_callbacks(1); + let groups = [ProbeGroup { + benchmarks, + setup: |enabled| enabled, + teardown: |enabled| enabled, + }]; + let error = discover_targets(&groups).unwrap_err(); + assert_eq!( + error.source().unwrap().downcast_ref::().unwrap().identity, + "group/iterator/sizes_0" + ); + EXECUTIONS.with_borrow(|calls| assert!(calls.is_empty())); + } + } + + #[test] + fn discovery_rejects_distinct_canonical_cases_with_a_shared_backend_identity() { + static CASES: [InternalMacroLibBench; 1] = [native_case(InternalLibFunctionKind::Default(default_case))]; + static BENCHMARKS: [ProbeBenchmark; 2] = [ + ProbeBenchmark { + identity: BenchmarkIdentity::__new("group", "first", "native_group", "same", true), + cases: &CASES, + }, + ProbeBenchmark { + identity: BenchmarkIdentity::__new("group", "second", "native_group", "same", true), + cases: &CASES, + }, + ]; + let groups = [ProbeGroup { + benchmarks: &BENCHMARKS, + setup: |enabled| enabled, + teardown: |enabled| enabled, + }]; + let error = discover_targets(&groups).unwrap_err(); + assert_eq!( + error + .source() + .unwrap() + .downcast_ref::() + .unwrap() + .identity, + "native_group/same/case" + ); + } + + fn setup(enabled: bool) -> bool { + assert!(enabled); + SETUP.fetch_add(1, Ordering::Relaxed); + true + } + + fn teardown(enabled: bool) -> bool { + assert!(enabled); + TEARDOWN.fetch_add(1, Ordering::Relaxed); + true + } + + #[test] + fn lifecycle_tears_down_after_success_error_and_panic() { + with_lifecycle(setup, teardown, || Ok(())).unwrap(); + let error = with_lifecycle(setup, teardown, || Err(InvalidProbeArguments::new().into())).unwrap_err(); + assert!(error.source().unwrap().is::()); + let panic = panic::catch_unwind(|| with_lifecycle(setup, teardown, || panic!("workload failed"))); + panic.unwrap_err(); + assert_eq!(SETUP.load(Ordering::Relaxed), 3); + assert_eq!(TEARDOWN.load(Ordering::Relaxed), 3); + } + + #[test] + fn indexed_arguments_round_trip_without_native_filters() { + let target = ProbeTarget { + identity: "group/benchmark/case_2".into(), + native_identity: "native/case_2".into(), + group: 1, + benchmark: 3, + case: 4, + iteration: Some(2), + }; + assert_eq!(target.arguments(), ["--metabench-probe", "1", "3", "4", "2"].map(OsString::from)); + let encoded = serde_json::to_vec(&target).unwrap(); + let decoded: ProbeTarget = serde_json::from_slice(&encoded).unwrap(); + assert_eq!(decoded.arguments(), target.arguments()); + assert_eq!(decoded.native_identity, target.native_identity); + assert_eq!(ProbeTarget { iteration: None, ..target }.arguments()[4], "-"); + } + + #[test] + fn worker_selection_preserves_typed_argument_and_index_errors() { + static CASES: [InternalMacroLibBench; 1] = [native_case(InternalLibFunctionKind::Default(default_case))]; + static BENCHMARKS: [ProbeBenchmark; 1] = [ProbeBenchmark { + identity: BenchmarkIdentity::__new("group", "default", "native_group", "native_default", true), + cases: &CASES, + }]; + let groups = [ProbeGroup { + benchmarks: &BENCHMARKS, + setup: |enabled| enabled, + teardown: |enabled| enabled, + }]; + let run = |arguments: &[&str]| { + run_probe_groups_inner( + &groups, + |enabled| enabled, + |enabled| enabled, + &arguments.iter().map(|argument| (*argument).to_owned()).collect::>(), + ) + }; + for arguments in [&[][..], &["--other"][..], &["--metabench-probe", "0"][..]] { + let error = run(arguments).unwrap_err(); + assert!(error.source().unwrap().is::()); + assert_eq!(error.to_string(), "expected private native probe worker arguments"); + } + let error = run(&["--metabench-probe", "invalid", "0", "0", "-"]).unwrap_err(); + assert!(error.source().unwrap().is::()); + for (arguments, level) in [ + (["--metabench-probe", "1", "0", "0", "-"], "group"), + (["--metabench-probe", "0", "1", "0", "-"], "benchmark"), + (["--metabench-probe", "0", "0", "1", "-"], "case"), + ] { + let error = run(&arguments).unwrap_err(); + let source = error.source().unwrap().downcast_ref::().unwrap(); + assert_eq!((source.level, source.index, source.count), (level, 1, 1)); + assert_eq!(error.to_string(), format!("probe {level} index is out of bounds")); + } + } + + #[test] + fn probe_artifact_errors_retain_the_original_source_until_worker_rendering() { + let missing = publish_targets(None, &[]).unwrap_err(); + assert!(missing.source().unwrap().is::()); + assert_eq!(missing.to_string(), "missing native probe discovery artifact path"); + let error = ProbeError::from(crate::error::Error::ArtifactIo { + path: "discovery.json".into(), + source: std::io::Error::new(std::io::ErrorKind::PermissionDenied, "write denied"), + }); + let artifact = error.source().unwrap().downcast_ref::().unwrap(); + assert!(matches!( + artifact, + crate::error::Error::ArtifactIo { path, source } + if path == Path::new("discovery.json") && source.kind() == std::io::ErrorKind::PermissionDenied + )); + assert_eq!(error.to_string(), "failed to access artifact discovery.json: write denied"); + } +} diff --git a/crates/metabench/src/probe_error.rs b/crates/metabench/src/probe_error.rs new file mode 100644 index 000000000..6a4cb89e2 --- /dev/null +++ b/crates/metabench/src/probe_error.rs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::num::ParseIntError; + +use crate::error::Error; + +#[ohno::error] +#[display("expected private native probe worker arguments")] +pub(crate) struct InvalidProbeArguments; + +#[ohno::error] +#[display("probe {level} index is out of bounds")] +#[cfg_attr( + not(test), + expect(dead_code, reason = "retain structured indices without changing existing worker diagnostics") +)] +pub(crate) struct ProbeIndexOutOfBounds { + pub(crate) level: &'static str, + pub(crate) index: usize, + pub(crate) count: usize, +} + +#[ohno::error] +#[display("probe iterator selection does not match the native case")] +#[cfg_attr( + not(test), + expect( + dead_code, + reason = "retain structured selection details without changing existing worker diagnostics" + ) +)] +pub(crate) struct ProbeSelectionMismatch { + pub(crate) iterator: bool, + pub(crate) iteration: Option, +} + +#[ohno::error] +#[display("native probe case count exceeds {limit}")] +pub(crate) struct ProbeCaseLimit { + pub(crate) limit: usize, +} + +#[ohno::error] +#[display("duplicate native probe identity {identity:?}; rename the conflicting case or benchmark")] +pub(crate) struct DuplicateProbeIdentity { + pub(crate) identity: String, +} + +#[ohno::error] +#[display("duplicate native probe backend identity {identity:?}; rename the conflicting case or benchmark")] +pub(crate) struct DuplicateProbeBackendIdentity { + pub(crate) identity: String, +} + +#[ohno::error] +#[display("missing native probe discovery artifact path")] +pub(crate) struct MissingProbeDiscoveryPath; + +#[ohno::error] +#[from(InvalidProbeArguments)] +#[from(ProbeIndexOutOfBounds)] +#[from(ProbeSelectionMismatch)] +#[from(ProbeCaseLimit)] +#[from(DuplicateProbeIdentity)] +#[from(DuplicateProbeBackendIdentity)] +#[from(MissingProbeDiscoveryPath)] +#[from(ParseIntError)] +#[from(Error)] +pub(crate) struct ProbeError; diff --git a/crates/metabench/src/report.rs b/crates/metabench/src/report.rs index 34f1eab9f..9ed875db4 100644 --- a/crates/metabench/src/report.rs +++ b/crates/metabench/src/report.rs @@ -3,8 +3,9 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fmt::{self, Write as _}; -use std::io::{BufReader, BufWriter, Write as _}; +use std::io::{BufWriter, Write as _}; use std::path::{Path, PathBuf}; +#[cfg(any(target_os = "macos", target_os = "windows", test))] use std::process::Command; use std::{env, fs, io}; @@ -12,11 +13,13 @@ use jiff::Timestamp; use serde::{Deserialize, Serialize}; use unicode_width::UnicodeWidthStr; +use crate::artifact::read_regular_file; use crate::error::Error; use crate::metric_display::TableMetric; +use crate::runner::terminal_text; pub(crate) const DEFAULT_REGRESSION_THRESHOLD: f64 = 5.0; -const REPORT_SCHEMA_VERSION: u32 = 8; +const REPORT_SCHEMA_VERSION: u32 = 9; const MAX_REPORT_JSON_BYTES: u64 = 64 * 1024 * 1024; const DEFAULT_CONSOLE_WIDTH: usize = 120; const MARKDOWN_TABLE_WIDTH: usize = 180; @@ -69,6 +72,10 @@ pub(crate) struct Metric { #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) upper_bound: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) confidence_level: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) statistic: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) change_percentage: Option, } @@ -81,6 +88,8 @@ impl Metric { direction, lower_bound: None, upper_bound: None, + confidence_level: None, + statistic: None, change_percentage: None, } } @@ -93,6 +102,8 @@ impl Metric { direction, lower_bound: None, upper_bound: None, + confidence_level: None, + statistic: None, change_percentage: None, } } @@ -185,7 +196,7 @@ impl BenchmarkReport { } let report = Self { schema_version: REPORT_SCHEMA_VERSION, - metadata: EnvironmentMetadata::collect(backends, artifact_root)?, + metadata: EnvironmentMetadata::collect(backends, artifact_root), entries: entries.into_values().collect(), }; validate_report(&report)?; @@ -201,23 +212,8 @@ impl BenchmarkReport { message: format!("publication is incomplete; marker {} is present", incomplete.display()), }); } - let length = fs::metadata(path) - .map_err(|source| Error::ReportIo { - path: path.to_owned(), - source, - })? - .len(); - if length > MAX_REPORT_JSON_BYTES { - return Err(Error::ReportFormat { - path: path.to_owned(), - message: format!("JSON file is {length} bytes; limit is {MAX_REPORT_JSON_BYTES} bytes"), - }); - } - let file = fs::File::open(path).map_err(|source| Error::ReportIo { - path: path.to_owned(), - source, - })?; - let mut report: Self = serde_json::from_reader(BufReader::new(file)).map_err(|source| Error::ReportJson { + let bytes = read_regular_file(path, MAX_REPORT_JSON_BYTES, "JSON").map_err(Self::report_read_error)?; + let mut report: Self = serde_json::from_slice(&bytes).map_err(|source| Error::ReportJson { path: path.to_owned(), source, })?; @@ -233,6 +229,14 @@ impl BenchmarkReport { Ok(report) } + fn report_read_error(error: Error) -> Error { + match error { + Error::ArtifactIo { path, source } => Error::ReportIo { path, source }, + Error::ArtifactFormat { path, message } => Error::ReportFormat { path, message }, + error => error, + } + } + pub(crate) fn apply_baseline(&mut self, baseline: &Self, threshold: f64) -> Result<(), Error> { if !threshold.is_finite() || threshold < 0.0 { return Err(Error::InvalidThreshold(threshold.to_string())); @@ -269,7 +273,11 @@ impl BenchmarkReport { let Some(old) = previous_result.metrics.get(name) else { continue; }; - if current.unit != old.unit || current.direction != old.direction || current.direction == MetricDirection::Unknown { + if current.unit != old.unit + || current.statistic != old.statistic + || current.direction != old.direction + || current.direction == MetricDirection::Unknown + { continue; } let change = percentage_shift(old.value.as_f64(), current.value.as_f64()); @@ -411,23 +419,20 @@ pub(crate) struct EnvironmentMetadata { } impl EnvironmentMetadata { - fn collect(measured_backends: Vec, artifact_root: String) -> Result { + pub(crate) fn comparable_to(&self, other: &Self) -> bool { + self.target_triple == other.target_triple + && self.rustc_version == other.rustc_version + && self.host_cpu == other.host_cpu + && self.profile == other.profile + && self.compiler_flags == other.compiler_flags + } + + fn collect(measured_backends: Vec, artifact_root: String) -> Self { let timestamp_utc = Timestamp::now().to_string(); - let rustc = env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); - let output = Command::new(rustc) - .arg("-Vv") - .output() - .map_err(|error| Error::Metadata(error.to_string()))?; - if !output.status.success() { - return Err(Error::Metadata(format!("rustc -Vv exited with {}", output.status))); - } - Ok(Self { + Self { timestamp_utc, target_triple: env!("METABENCH_TARGET").to_owned(), - rustc_version: String::from_utf8(output.stdout) - .map_err(|error| Error::Metadata(error.to_string()))? - .trim() - .to_owned(), + rustc_version: env!("METABENCH_RUSTC_VERSION").to_owned(), host_cpu: host_cpu(), profile: env!("METABENCH_PROFILE").to_owned(), compiler_flags: option_env!("METABENCH_RUSTFLAGS") @@ -435,7 +440,7 @@ impl EnvironmentMetadata { .filter(|flags| !flags.is_empty()), measured_backends, artifact_root, - }) + } } } @@ -511,18 +516,24 @@ fn render_metric_markdown(name: &str, metric: &Metric) -> String { let _ = write!(value, " {unit}"); } if let (Some(lower), Some(upper)) = (metric.lower_bound, metric.upper_bound) { - if name == "median" { + if matches!(name, "time" | "median") { let _ = write!(value, " ({lower:.2}..{upper:.2})"); } else { let _ = write!(value, " ({lower:.4}..{upper:.4})"); } + if let Some(level) = metric.confidence_level { + let _ = write!(value, " {:.1}% CI", level * 100.0); + } + if let Some(statistic) = &metric.statistic { + let _ = write!(value, " [{statistic}]"); + } } value } fn render_metric_value(name: &str, metric: &Metric) -> String { match (name, metric.value) { - ("median", MetricValue::Float(value)) => format!("{value:.2}"), + ("time" | "median", MetricValue::Float(value)) => format!("{value:.2}"), _ => metric.value.to_string(), } } @@ -562,7 +573,44 @@ fn classify(changes: &[f64], threshold: f64) -> BenchmarkStatus { } fn escape_table(value: &str) -> String { - value.replace('|', "\\|").replace('\n', "
") + let mut escaped = String::with_capacity(value.len()); + for character in value.chars() { + if matches!(character, '\u{061c}' | '\u{200e}' | '\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}') { + let _ = write!(escaped, "\u{{:x}}", u32::from(character)); + continue; + } + if character.is_control() + || matches!( + character, + '&' | '<' + | '>' + | '"' + | '\'' + | '\\' + | '`' + | '*' + | '_' + | '{' + | '}' + | '[' + | ']' + | '(' + | ')' + | '#' + | '+' + | '-' + | '.' + | '!' + | '|' + ) + || matches!(character, '\u{2028}' | '\u{2029}') + { + let _ = write!(escaped, "&#{};", u32::from(character)); + } else { + escaped.push(character); + } + } + escaped } fn table_headers(metrics: &[TableMetric]) -> Vec { @@ -648,11 +696,11 @@ fn render_console_grid(headers: &[String], rows: &[Vec]) -> String { fn console_column_widths(headers: &[String], rows: &[Vec]) -> Vec { let mut widths = headers .iter() - .map(|header| UnicodeWidthStr::width(header.as_str())) + .map(|header| UnicodeWidthStr::width(terminal_text(header).as_str())) .collect::>(); for row in rows { for (index, value) in row.iter().enumerate() { - widths[index] = widths[index].max(UnicodeWidthStr::width(value.as_str())); + widths[index] = widths[index].max(UnicodeWidthStr::width(terminal_text(value).as_str())); } } widths @@ -660,10 +708,11 @@ fn console_column_widths(headers: &[String], rows: &[Vec]) -> Vec fn render_console_row(output: &mut String, cells: &[String], widths: &[usize]) { for (index, cell) in cells.iter().enumerate() { + let cell = terminal_text(cell); if index != 0 { output.push_str(" "); } - output.push_str(cell); + output.push_str(&cell); output.push_str(&" ".repeat(widths[index] - UnicodeWidthStr::width(cell.as_str()))); } output.push('\n'); @@ -675,6 +724,41 @@ fn validate_report(report: &BenchmarkReport) -> Result<(), Error> { if !identities.insert(entry.identity.as_str()) { return Err(Error::DuplicateReportBenchmark(entry.identity.clone())); } + for (engine, result) in entry.results.iter().chain(entry.prior.iter().flat_map(|prior| prior.iter())) { + for (name, metric) in &result.metrics { + let invalid = match (metric.lower_bound, metric.upper_bound) { + (Some(lower), Some(upper)) => { + metric.confidence_level.is_none() + || !lower.is_finite() + || !upper.is_finite() + || lower > upper + || (engine == "criterion" && matches!(name.as_str(), "time" | "median") && (lower < 0.0 || upper < 0.0)) + } + (None, None) => metric.confidence_level.is_some() || (engine == "criterion" && name == "time"), + _ => true, + }; + let invalid_confidence = metric + .confidence_level + .is_some_and(|level| !level.is_finite() || level <= 0.0 || level >= 1.0); + let invalid_timing = engine == "criterion" + && matches!(name.as_str(), "time" | "median") + && (!metric.value.as_f64().is_finite() || metric.value.as_f64() < 0.0); + let invalid_statistic = engine == "criterion" + && name == "time" + && metric.statistic.as_deref().is_none_or(|statistic| statistic.trim().is_empty()); + if invalid || invalid_confidence || invalid_timing || invalid_statistic { + return Err(Error::ReportFormat { + path: "".into(), + message: format!( + "invalid estimate or confidence metadata for {} ({}/{})", + terminal_text(&entry.identity), + terminal_text(engine), + terminal_text(name) + ), + }); + } + } + } } Ok(()) } @@ -852,8 +936,52 @@ fn cpu_identity_fallback(architecture: &str) -> String { #[cfg_attr(coverage_nightly, coverage(off))] #[cfg(test)] mod tests { + #[cfg(unix)] + use std::os::unix::fs::{PermissionsExt as _, symlink}; + use super::*; + #[test] + fn report_admission_errors_retain_report_context_and_existing_causes() { + let path = PathBuf::from("baseline.json"); + assert!(matches!( + BenchmarkReport::report_read_error(Error::ArtifactIo { + path: path.clone(), + source: io::Error::new(io::ErrorKind::PermissionDenied, "read denied"), + }), + Error::ReportIo { path: failed, source } + if failed == path && source.kind() == io::ErrorKind::PermissionDenied + )); + assert!(matches!( + BenchmarkReport::report_read_error(Error::ArtifactFormat { path: path.clone(), message: "not regular".into() }), + Error::ReportFormat { path: failed, message } if failed == path && message == "not regular" + )); + assert!(matches!( + BenchmarkReport::report_read_error(Error::Metadata("original error".into())), + Error::Metadata(message) if message == "original error" + )); + } + + #[cfg(unix)] + #[test] + fn baseline_admission_rejects_links_and_fifos_without_waiting_for_a_writer() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("baseline.json"); + let link = directory.path().join("link.json"); + fs::write(&path, b"{}").unwrap(); + symlink(&path, &link).unwrap(); + assert!(matches!(BenchmarkReport::read_json(&link), Err(Error::ReportIo { .. }))); + #[cfg(target_os = "linux")] + { + let fifo = directory.path().join("fifo.json"); + nix::unistd::mkfifo(&fifo, nix::sys::stat::Mode::S_IRUSR | nix::sys::stat::Mode::S_IWUSR).unwrap(); + assert!(matches!( + BenchmarkReport::read_json(&fifo), + Err(Error::ReportFormat { message, .. }) if message.contains("regular file") + )); + } + } + fn metadata() -> EnvironmentMetadata { EnvironmentMetadata { timestamp_utc: "2026-01-02T03:04:05Z".to_owned(), @@ -907,8 +1035,13 @@ mod tests { } fn engine(value: f64, direction: MetricDirection) -> EngineResult { + let mut time = Metric::float("criterion", "time", value, direction); + time.statistic = Some("mean".to_owned()); + time.lower_bound = Some(value); + time.upper_bound = Some(value); + time.confidence_level = Some(0.95); EngineResult { - metrics: BTreeMap::from([("median".to_owned(), Metric::float("criterion", "median", value, direction))]), + metrics: BTreeMap::from([("time".to_owned(), time)]), raw_artifacts: vec!["raw.json".to_owned()], } } @@ -930,16 +1063,22 @@ mod tests { fn merges_sources_by_canonical_identity() { let results = ["criterion", "gungraun.callgrind"] .into_iter() - .zip(["median", "Ir"]) - .map(|(source, metric)| NativeResult { - identity: "group/bench".to_owned(), - native_identity: source.to_owned(), - source: source.to_owned(), - metrics: BTreeMap::from([( - metric.to_owned(), - Metric::integer(source, metric, 1, MetricDirection::LowerIsBetter), - )]), - raw_artifacts: Vec::new(), + .zip(["time", "Ir"]) + .map(|(source, name)| { + let mut metric = Metric::integer(source, name, 1, MetricDirection::LowerIsBetter); + if source == "criterion" { + metric.statistic = Some("mean".to_owned()); + metric.lower_bound = Some(1.0); + metric.upper_bound = Some(1.0); + metric.confidence_level = Some(0.95); + } + NativeResult { + identity: "group/bench".to_owned(), + native_identity: source.to_owned(), + source: source.to_owned(), + metrics: BTreeMap::from([(name.to_owned(), metric)]), + raw_artifacts: Vec::new(), + } }); let report = BenchmarkReport::from_results(results, Vec::new(), String::new()).unwrap(); assert_eq!(report.entries.len(), 1); @@ -1032,6 +1171,7 @@ mod tests { for (identity, direction, unit, expected) in [ ("other", MetricDirection::LowerIsBetter, None, BenchmarkStatus::Uncompared), ("bench", MetricDirection::Unknown, None, BenchmarkStatus::Uncompared), + ("bench", MetricDirection::HigherIsBetter, None, BenchmarkStatus::Uncompared), ("bench", MetricDirection::LowerIsBetter, Some("ns"), BenchmarkStatus::Uncompared), ("bench", MetricDirection::LowerIsBetter, None, BenchmarkStatus::Stable), ] { @@ -1041,7 +1181,7 @@ mod tests { .get_mut("criterion") .unwrap() .metrics - .get_mut("median") + .get_mut("time") .unwrap() .unit = unit.map(str::to_owned); current.apply_baseline(&baseline, 5.0).unwrap(); @@ -1074,18 +1214,18 @@ mod tests { assert!(!console.contains("criterion")); assert!(!console.contains("alloc_tracker")); assert!(console.contains("測試")); - assert!(console.contains("b|ench\n測試 (PRIOR)")); + assert!(console.contains("b|ench\\n測試 (PRIOR)")); let mut markdown = Vec::new(); current.write_markdown(&mut markdown).unwrap(); let markdown = String::from_utf8(markdown).unwrap(); - assert!(markdown.contains("b\\|ench
測試")); + assert!(markdown.contains("b|ench 測試")); assert!(markdown.contains("Benchmark")); assert!(markdown.contains("Time")); assert!(markdown.contains("Allocs")); - assert!(markdown.contains("10.00")); - assert!(markdown.contains("b\\|ench
測試 (PRIOR)")); - assert!(markdown.contains("8.00")); + assert!(markdown.contains("10.00")); + assert!(markdown.contains("b|ench 測試 (PRIOR)")); + assert!(markdown.contains("8.00")); assert!(markdown.ends_with("\n\n")); } @@ -1144,7 +1284,7 @@ mod tests { assert!(lines[1].split('|').nth(2).unwrap().trim_end().ends_with(':')); let time_cell = lines[2].split('|').nth(2).unwrap(); assert!(time_cell.len() - time_cell.trim_start().len() > 1); - assert!(time_cell.trim().starts_with("10.00")); + assert!(time_cell.trim().starts_with("10.00")); } #[test] @@ -1171,13 +1311,13 @@ mod tests { #[test] fn time_values_use_two_decimal_places_only_for_display() { - let mut time = Metric::float("criterion", "median", 12.3456, MetricDirection::LowerIsBetter); + let mut time = Metric::float("criterion", "time", 12.3456, MetricDirection::LowerIsBetter); time.unit = Some("ns".to_owned()); time.lower_bound = Some(11.2345); time.upper_bound = Some(13.4567); - assert_eq!(render_metric_console("median", &time), "12.35 ns"); - assert_eq!(render_metric_markdown("median", &time), "12.35 ns (11.23..13.46)"); + assert_eq!(render_metric_console("time", &time), "12.35 ns"); + assert_eq!(render_metric_markdown("time", &time), "12.35 ns (11.23..13.46)"); assert_eq!(time.value, MetricValue::Float(12.3456)); let counter = Metric::float("perf", "cycles", 12.3456, MetricDirection::LowerIsBetter); @@ -1282,6 +1422,375 @@ mod tests { BenchmarkReport::read_json(&path).unwrap(); } + #[test] + fn different_timing_statistics_are_not_compared() { + let mut baseline = report("bench", 10.0, MetricDirection::LowerIsBetter); + let mut current = report("bench", 100.0, MetricDirection::LowerIsBetter); + baseline.entries[0] + .results + .get_mut("criterion") + .unwrap() + .metrics + .get_mut("time") + .unwrap() + .statistic = Some("mean".to_owned()); + current.entries[0] + .results + .get_mut("criterion") + .unwrap() + .metrics + .get_mut("time") + .unwrap() + .statistic = Some("slope".to_owned()); + current.apply_baseline(&baseline, 5.0).unwrap(); + assert_eq!(current.entries[0].status, BenchmarkStatus::Uncompared); + assert_eq!(current.entries[0].results["criterion"].metrics["time"].change_percentage, None); + } + + #[test] + fn environment_compatibility_ignores_run_location_but_not_measurement_conditions() { + let base = report("bench", 10.0, MetricDirection::LowerIsBetter).metadata; + let mut other = base.clone(); + other.timestamp_utc = "different time".to_owned(); + other.artifact_root = "different output directory".to_owned(); + other.measured_backends = vec!["different subset".to_owned()]; + assert!(base.comparable_to(&other)); + for field in 0..5 { + let mut other = base.clone(); + match field { + 0 => other.target_triple.push_str("-different"), + 1 => other.rustc_version.push_str("-different"), + 2 => other.host_cpu.push_str("-different"), + 3 => other.profile.push_str("-different"), + _ => other.compiler_flags = Some("-different".to_owned()), + } + assert!(!base.comparable_to(&other)); + } + } + + #[test] + fn compiler_metadata_uses_the_build_identity() { + if env::var_os("METABENCH_TEST_AMBIENT_COMPILER").is_some() { + let current = EnvironmentMetadata::collect(vec![], "artifacts".to_owned()); + assert_eq!(current.rustc_version, env!("METABENCH_RUSTC_VERSION")); + assert!(current.rustc_version.starts_with("rustc ")); + let mut other = current.clone(); + other.rustc_version = "rustc other build".to_owned(); + assert!(!current.comparable_to(&other)); + return; + } + let directory = tempfile::tempdir_in(env!("CARGO_MANIFEST_DIR")).unwrap(); + for version in ["one", "two"] { + let ambient = directory.path().join(format!("ambient-rustc-{version}")); + #[cfg(unix)] + { + fs::write(&ambient, format!("#!/bin/sh\nprintf 'rustc ambient {version}\\n'\n")).unwrap(); + fs::set_permissions(&ambient, fs::Permissions::from_mode(0o700)).unwrap(); + } + let status = Command::new(env::current_exe().unwrap()) + .args(["--exact", "report::tests::compiler_metadata_uses_the_build_identity"]) + .env("METABENCH_TEST_AMBIENT_COMPILER", "1") + .env("RUSTC", &ambient) + .status() + .unwrap(); + assert!(status.success()); + } + } + + fn primary_estimate_report() -> BenchmarkReport { + serde_json::from_str( + r#"{ + "schema_version": 9, + "metadata": { + "timestamp_utc": "2026-01-02T03:04:05Z", "target_triple": "test-target", + "rustc_version": "rustc test", "host_cpu": "test cpu", "profile": "release", + "compiler_flags": null, "measured_backends": ["criterion"], "artifact_root": "artifacts" + }, + "entries": [{ + "identity": "bench", "status": "uncompared", + "results": {"criterion": {"metrics": {"time": { + "value": 12.35, "unit": "ns", "direction": "lower_is_better", + "lower_bound": 11.23, "upper_bound": 13.46, + "confidence_level": 0.95, "statistic": "slope" + }}}}, + "prior": {"criterion": {"metrics": {"time": { + "value": 20.0, "unit": "cycles", "direction": "lower_is_better", + "lower_bound": 10.0, "upper_bound": 30.0, + "confidence_level": 0.99, "statistic": "mean" + }}}} + }] + }"#, + ) + .unwrap() + } + + #[test] + fn primary_estimate_wire_fields_and_markdown_cells_are_preserved() { + let directory = tempfile::tempdir_in(env!("CARGO_MANIFEST_DIR")).unwrap(); + let path = directory.path().join("report.json"); + let source = primary_estimate_report(); + fs::write(&path, serde_json::to_vec(&source).unwrap()).unwrap(); + let actual = BenchmarkReport::read_json(&path).unwrap(); + let wire = serde_json::to_value(&actual).unwrap(); + for (row, statistic, value, unit, lower, upper, confidence) in [ + ("results", "slope", 12.35, "ns", 11.23, 13.46, 0.95), + ("prior", "mean", 20.0, "cycles", 10.0, 30.0, 0.99), + ] { + let metric = &wire["entries"][0][row]["criterion"]["metrics"]["time"]; + assert_eq!(metric["statistic"], statistic); + assert_eq!(metric["confidence_level"], confidence); + assert_eq!(metric["value"], value); + assert_eq!(metric["unit"], unit); + assert_eq!(metric["lower_bound"], lower); + assert_eq!(metric["upper_bound"], upper); + } + let mut output = Vec::new(); + actual.write_markdown(&mut output).unwrap(); + let markdown = String::from_utf8(output).unwrap(); + let rows = markdown.split_once("## Results\n\n").unwrap().1.lines().collect::>(); + assert_eq!( + rows[2].split('|').nth(2).unwrap().trim(), + "12.35 ns (11.23..13.46) 95.0% CI [slope]" + ); + assert_eq!( + rows[3].split('|').nth(2).unwrap().trim(), + "20.00 cycles (10.00..30.00) 99.0% CI [mean]" + ); + } + + #[test] + fn schema_nine_rejects_incoherent_confidence_in_current_and_prior_metrics() { + let directory = tempfile::tempdir_in(env!("CARGO_MANIFEST_DIR")).unwrap(); + let path = directory.path().join("report.json"); + for row in ["results", "prior"] { + for invalid in [ + serde_json::json!({"statistic": null}), + serde_json::json!({"statistic": ""}), + serde_json::json!({"statistic": " "}), + serde_json::json!({"confidence_level": null}), + serde_json::json!({"confidence_level": 0.0}), + serde_json::json!({"confidence_level": 1.0}), + serde_json::json!({"confidence_level": 2.0}), + serde_json::json!({"confidence_level": -0.1}), + serde_json::json!({"confidence_level": 0.95, "lower_bound": null, "upper_bound": null}), + serde_json::json!({"lower_bound": null}), + serde_json::json!({"upper_bound": null}), + serde_json::json!({"lower_bound": 40.0}), + serde_json::json!({"lower_bound": -1.0}), + serde_json::json!({"value": -1.0}), + serde_json::json!({"lower_bound": null, "upper_bound": null, "confidence_level": null}), + ] { + let mut wire = serde_json::to_value(primary_estimate_report()).unwrap(); + let metric = wire["entries"][0][row]["criterion"]["metrics"]["time"].as_object_mut().unwrap(); + metric.extend(invalid.as_object().unwrap().clone()); + fs::write(&path, serde_json::to_vec(&wire).unwrap()).unwrap(); + assert!(matches!(BenchmarkReport::read_json(&path), Err(Error::ReportFormat { .. }))); + let mut wire = serde_json::to_value(primary_estimate_report()).unwrap(); + let metric = wire["entries"][0][row]["criterion"]["metrics"]["time"].as_object_mut().unwrap(); + for field in ["lower_bound", "upper_bound", "confidence_level"] { + metric.remove(field); + } + fs::write(&path, serde_json::to_vec(&wire).unwrap()).unwrap(); + assert!(matches!(BenchmarkReport::read_json(&path), Err(Error::ReportFormat { .. }))); + } + let mut wire = serde_json::to_value(primary_estimate_report()).unwrap(); + wire["entries"][0][row]["criterion"]["metrics"]["time"] + .as_object_mut() + .unwrap() + .remove("statistic"); + fs::write(&path, serde_json::to_vec(&wire).unwrap()).unwrap(); + assert!(matches!(BenchmarkReport::read_json(&path), Err(Error::ReportFormat { .. }))); + } + let mut valid = primary_estimate_report(); + valid.entries[0] + .results + .get_mut("criterion") + .unwrap() + .metrics + .get_mut("time") + .unwrap() + .value = MetricValue::Float(100.0); + valid.entries[0] + .prior + .as_mut() + .unwrap() + .get_mut("criterion") + .unwrap() + .metrics + .get_mut("time") + .unwrap() + .value = MetricValue::Float(1.0); + fs::write(&path, serde_json::to_vec(&valid).unwrap()).unwrap(); + let actual = BenchmarkReport::read_json(&path).unwrap(); + assert_eq!( + actual.entries[0].results["criterion"].metrics["time"].value, + MetricValue::Float(100.0) + ); + assert_eq!( + actual.entries[0].prior.as_ref().unwrap()["criterion"].metrics["time"].value, + MetricValue::Float(1.0) + ); + for invalid in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + for component in 0..4 { + let mut baseline = valid.clone(); + let metric = baseline.entries[0] + .results + .get_mut("criterion") + .unwrap() + .metrics + .get_mut("time") + .unwrap(); + match component { + 0 => metric.value = MetricValue::Float(invalid), + 1 => metric.lower_bound = Some(invalid), + 2 => metric.upper_bound = Some(invalid), + _ => metric.confidence_level = Some(invalid), + } + let mut current = report("bench", 5.0, MetricDirection::LowerIsBetter); + assert!(matches!(current.apply_baseline(&baseline, 5.0), Err(Error::ReportFormat { .. }))); + } + } + } + + #[test] + fn timing_estimates_accept_zero_and_equal_interval_bounds() { + for (value, lower, upper) in [(0.0, 0.0, 0.0), (1.0, 0.0, 2.0), (1.0, 1.0, 1.0)] { + let mut current = report("bench", value, MetricDirection::LowerIsBetter); + let time = current.entries[0] + .results + .get_mut("criterion") + .unwrap() + .metrics + .get_mut("time") + .unwrap(); + time.lower_bound = Some(lower); + time.upper_bound = Some(upper); + super::validate_report(¤t).unwrap(); + current.entries[0].prior = Some(current.entries[0].results.clone()); + super::validate_report(¤t).unwrap(); + } + } + + #[test] + fn invalid_metric_diagnostics_escape_untrusted_identity_engine_and_name() { + let mut report = report("identity\n\u{1b}", 1.0, MetricDirection::LowerIsBetter); + let mut result = report.entries[0].results.remove("criterion").unwrap(); + let mut metric = result.metrics.remove("time").unwrap(); + metric.lower_bound = Some(2.0); + metric.upper_bound = Some(1.0); + result.metrics.insert("metric\r\u{202e}".into(), metric); + report.entries[0].results.insert("engine\t\u{200f}".into(), result); + let message = validate_report(&report).unwrap_err().to_string(); + assert!(message.contains(r"identity\n\u{1b}")); + assert!(message.contains(r"engine\t\u{200f}")); + assert!(message.contains(r"metric\r\u{202e}")); + for character in ['\n', '\r', '\t', '\u{1b}', '\u{200f}', '\u{202e}'] { + assert!(!message.contains(character)); + } + } + + #[test] + fn markdown_encodes_untrusted_text_without_changing_json() { + let text = " [link](javascript:bad) `code` *bold* _em_ \\ | <\r\n\u{1b}\u{85}\u{202e}"; + let mut report = primary_estimate_report(); + report.entries[0].identity = text.to_owned(); + report.metadata.artifact_root = text.to_owned(); + report.metadata.compiler_flags = Some(text.to_owned()); + for result in report.entries[0].results.values_mut() { + for metric in result.metrics.values_mut() { + metric.unit = Some(text.to_owned()); + metric.statistic = Some(text.to_owned()); + } + } + report.entries[0] + .prior + .as_mut() + .unwrap() + .get_mut("criterion") + .unwrap() + .metrics + .get_mut("time") + .unwrap() + .statistic = Some(text.to_owned()); + let mut output = Vec::new(); + report.write_markdown(&mut output).unwrap(); + let markdown = String::from_utf8(output).unwrap(); + assert!(markdown.contains("<img src=x onerror='alert(1)'>")); + assert!(markdown.contains("[link](javascript:bad)")); + assert!( + markdown + .contains("`code` *bold* _em_ \ | &lt; …\u{202e}") + ); + assert!(!markdown.contains(" allocation::prime(), + Mode::Perf => perf::prime(), + Mode::Vtune => vtune::prime(), + Mode::Criterion | Mode::Gungraun => {} + } + if matches!(mode, Mode::Allocations | Mode::Perf | Mode::Vtune) { + (engines.probe)().map_err(Error::Probe)?; + if env::args_os().nth(1).as_deref() == Some(OsStr::new("--metabench-probe-list")) { + return Ok(()); + } + } else { + engines.run(mode); + } if mode == Mode::Allocations { allocation::write_worker_artifact()?; } else if mode == Mode::Perf { @@ -132,14 +156,6 @@ fn run_inner(engines: EngineSet, identities: &'static [BenchmarkIdentity], bench run_parent(&mut arguments, engines, identities, benchmark_target) } -/// Determines whether a `--list` invocation must add [`Mode::Criterion`] to the native modes so -/// that a non-native engine (allocations, perf, or vtune) has a Criterion instance to list -/// benchmark identities from. -fn needs_criterion_for_listing(list: bool, modes: &[Mode], native_modes: &[Mode]) -> bool { - list && (modes.contains(&Mode::Allocations) || modes.contains(&Mode::Perf) || modes.contains(&Mode::Vtune)) - && !native_modes.contains(&Mode::Criterion) -} - fn run_parent( arguments: &mut Arguments, engines: EngineSet, @@ -149,18 +165,33 @@ fn run_parent( let modes = arguments.modes.clone(); arguments.finalize_native_args(&modes)?; let executable = current_executable(env::current_exe)?; - let report_root = default_target_directory_with(|name| env::var_os(name), env::current_dir)? + if arguments.smoke() && !arguments.list { + let mut smoke_modes = modes + .iter() + .copied() + .filter(|mode| matches!(mode, Mode::Criterion | Mode::Gungraun)) + .collect::>(); + if smoke_modes.is_empty() { + smoke_modes.push(Mode::Criterion); + } + let summary = orchestrate_modes(&smoke_modes, arguments.failure_mode, |mode| { + let status = launch_worker(mode, &executable, arguments.args_for(mode), None, true, arguments.timeout)?; + ensure_success(mode, status) + })?; + return finish_failures(summary.failures); + } + let report_root = effective_target_directory(&executable, |name| env::var_os(name))? .join("metabench") .join(benchmark_target); - let artifacts = (!arguments.list).then(|| ArtifactDirectory::create(&report_root)).transpose()?; - let mut native_modes = modes + let has_probes = modes + .iter() + .any(|mode| matches!(mode, Mode::Allocations | Mode::Perf | Mode::Vtune)); + let artifacts = create_run_artifacts(&report_root, arguments.list, has_probes)?; + let native_modes = modes .iter() .copied() .filter(|mode| !matches!(mode, Mode::Allocations | Mode::Perf | Mode::Vtune)) .collect::>(); - if needs_criterion_for_listing(arguments.list, &modes, &native_modes) { - native_modes.push(Mode::Criterion); - } let mut summary = orchestrate_modes(&native_modes, arguments.failure_mode, |mode| { if !arguments.list { print_progress(mode)?; @@ -190,6 +221,13 @@ fn run_parent( let mut failures = std::mem::take(&mut summary.failures); if arguments.list { + if has_probes { + let artifacts = artifacts.as_ref().expect("probe listing creates an artifact directory above"); + match discover_native_probe_targets(&executable, artifacts, arguments) { + Ok(targets) => print_probe_targets(&targets)?, + Err(error) => append_failure(&mut failures, error), + } + } return finish_failures(failures); } @@ -198,7 +236,6 @@ fn run_parent( &executable, arguments, artifacts.as_ref().expect("non-list runs always create an artifact directory"), - identities, &mut summary, &mut failures, )?; @@ -225,12 +262,27 @@ fn run_parent( finish_failures(failures) } +fn create_run_artifacts(report_root: &Path, list: bool, has_probes: bool) -> Result, Error> { + (!list || has_probes).then(|| ArtifactDirectory::create(report_root)).transpose() +} + +fn print_probe_targets(targets: &[ProbeTarget]) -> Result<(), Error> { + let mut stdout = io::stdout().lock(); + write_probe_targets(&mut stdout, targets) +} + +fn write_probe_targets(stdout: &mut impl io::Write, targets: &[ProbeTarget]) -> Result<(), Error> { + for target in targets { + writeln!(stdout, "{}: benchmark", terminal_text(&target.identity)).map_err(Error::ConsoleOutput)?; + } + Ok(()) +} + fn run_probe_modes( modes: &[Mode], executable: &Path, arguments: &Arguments, artifacts: &ArtifactDirectory, - identities: &[BenchmarkIdentity], summary: &mut EngineRunSummary, failures: &mut Vec, ) -> Result, Error> { @@ -239,29 +291,12 @@ fn run_probe_modes( if !matches!(mode, Mode::Allocations | Mode::Perf | Mode::Vtune) { continue; } + print_progress(mode)?; let measured = match mode { - Mode::Allocations => run_allocations( - executable, - arguments, - artifacts, - identities, - summary.completed.contains(&Mode::Criterion), - ), - Mode::Perf => run_perf( - executable, - arguments, - artifacts, - identities, - summary.completed.contains(&Mode::Criterion), - ), - Mode::Vtune => run_vtune( - executable, - arguments, - artifacts, - identities, - summary.completed.contains(&Mode::Criterion), - ), + Mode::Allocations => run_allocations(executable, arguments, artifacts), + Mode::Perf => run_perf(executable, arguments, artifacts), + Mode::Vtune => run_vtune(executable, arguments, artifacts), Mode::Criterion | Mode::Gungraun => unreachable!("the probe mode list is fixed above"), }; match measured { @@ -350,11 +385,13 @@ fn build_report( let mut results = probe_results; if completed.contains(&Mode::Criterion) { let criterion = artifact::parse_criterion(&artifacts.criterion_path(), artifacts.root(), identities, criterion_unit)?; + require_backend_results(&criterion, Mode::Criterion, &artifacts.criterion_path())?; artifact::record_discovered_identities(artifacts.path(), Mode::Criterion.as_str(), &criterion)?; results.extend(criterion); } if completed.contains(&Mode::Gungraun) { let gungraun = artifact::parse_gungraun(&artifacts.gungraun_path(), artifacts.root(), identities)?; + require_backend_results(&gungraun, Mode::Gungraun, &artifacts.gungraun_path())?; artifact::record_discovered_identities(artifacts.path(), Mode::Gungraun.as_str(), &gungraun)?; results.extend(gungraun); } @@ -363,6 +400,12 @@ fn build_report( completed.iter().map(|mode| mode.as_str().to_owned()).collect(), artifacts.root().to_string_lossy().into_owned(), )?; + if report.entries.is_empty() { + return Err(Error::ArtifactFormat { + path: artifacts.path().to_owned(), + message: "no benchmark cases matched; existing reports were preserved".to_owned(), + }); + } let (json_path, markdown_path) = report_paths(arguments, benchmark_target)?; let baseline = if let Some(path) = arguments.baseline.as_deref() { Some((path, false)) @@ -377,7 +420,17 @@ fn build_report( if let Some((path, automatic)) = baseline && let Some(baseline) = read_baseline(path, automatic)? { - report.apply_baseline(&baseline, arguments.regression_threshold)?; + if report.metadata.comparable_to(&baseline.metadata) { + report.apply_baseline(&baseline, arguments.regression_threshold)?; + } else { + write_baseline_warning( + &mut io::stderr(), + &format!( + "skipping incompatible baseline {} (target, compiler, CPU, profile, or flags differ)", + path.display() + ), + )?; + } } // A `--keep-going` run where some selected engine failed must not overwrite the // rolling automatic-baseline file with results from only the engines that @@ -398,6 +451,19 @@ fn build_report( Ok((report, json_path, markdown_path)) } +fn require_backend_results(results: &[crate::report::NativeResult], mode: Mode, path: &Path) -> Result<(), Error> { + if results.is_empty() { + return Err(Error::ArtifactFormat { + path: path.to_owned(), + message: format!( + "{} produced no benchmark measurements; existing reports were preserved", + mode.as_str() + ), + }); + } + Ok(()) +} + /// Derives a sibling path for partial-run diagnostics, e.g. `report.json` becomes /// `report.partial.json`, keeping such output clearly separate from the rolling /// baseline file at the original path. @@ -409,66 +475,98 @@ fn partial_report_path(path: &Path) -> PathBuf { } fn read_baseline(path: &Path, automatic: bool) -> Result, Error> { + read_baseline_with_output(path, automatic, &mut io::stderr()) +} + +fn write_baseline_warning(output: &mut impl io::Write, detail: &str) -> Result<(), Error> { + writeln!(output, "metabench: {}", terminal_text(detail)).map_err(Error::ConsoleOutput) +} + +fn read_baseline_with_output(path: &Path, automatic: bool, output: &mut impl io::Write) -> Result, Error> { match BenchmarkReport::read_json(path) { Ok(report) => Ok(Some(report)), - Err(Error::UnsupportedReportSchema { .. }) if automatic => Ok(None), + Err(error @ Error::UnsupportedReportSchema { .. }) if automatic => { + write_baseline_warning(output, &format!("skipping incompatible automatic baseline: {error}"))?; + Ok(None) + } Err(error) => Err(error), } } -#[derive(Debug)] -struct CriterionProbeTarget { - identity: String, - native_identity: String, +fn probe_selection_arguments(criterion_arguments: &[OsString]) -> Result, Error> { + let mut probe_arguments = Vec::new(); + let mut value_expected = false; + let mut discard_value = false; + for argument in criterion_arguments { + if discard_value { + discard_value = false; + continue; + } + if value_expected { + probe_arguments.push(argument.clone()); + value_expected = false; + continue; + } + let text = argument.to_string_lossy(); + let option = text.split('=').next().unwrap_or(&text); + if criterion_measurement_only_option(option) { + discard_value = !text.contains('='); + continue; + } + if matches!(text.as_ref(), "--noplot" | "-n") { + continue; + } + if matches!(text.as_ref(), "--bench" | "--test" | "--list" | "--quick") { + continue; + } + if text.starts_with('-') { + value_expected = criterion_option_takes_value(&text); + } + probe_arguments.push(argument.clone()); + } + if value_expected || discard_value { + return Err(Error::Probe("missing a value for a forwarded Criterion option".to_owned())); + } + Ok(probe_arguments) } -fn criterion_probe_arguments(criterion_arguments: &[OsString], native_identity: &str) -> Vec { - let mut probe_arguments = criterion_arguments.to_vec(); - probe_arguments.push(native_identity.into()); - if !probe_arguments.iter().any(|argument| argument == "--exact") { - probe_arguments.push("--exact".into()); - } - probe_arguments +fn criterion_measurement_only_option(option: &str) -> bool { + matches!( + option, + "--save-baseline" | "-s" | "--baseline" | "-b" | "--load-baseline" | "--baseline-lenient" | "--profile-time" + ) +} + +fn criterion_option_takes_value(option: &str) -> bool { + matches!( + option, + "--sample-size" + | "--warm-up-time" + | "--measurement-time" + | "--nresamples" + | "--noise-threshold" + | "--confidence-level" + | "--significance-level" + | "--plotting-backend" + | "--output-format" + | "--format" + | "--color" + | "-c" + ) } fn run_allocations( executable: &Path, arguments: &Arguments, artifacts: &ArtifactDirectory, - identities: &[BenchmarkIdentity], - criterion_completed: bool, ) -> Result, Error> { - let targets = if criterion_completed { - artifact::parse_criterion(&artifacts.criterion_path(), artifacts.root(), identities, None)? - .into_iter() - .filter_map(|result| { - artifact::allocation_criterion_identity(&result.native_identity, identities).map(|identity| CriterionProbeTarget { - identity, - native_identity: result.native_identity, - }) - }) - .collect() - } else { - discover_criterion_probe_targets( - executable, - artifacts, - identities, - arguments.args_for(Mode::Criterion), - arguments.timeout, - )? - }; - if targets.is_empty() { - return Err(Error::ArtifactFormat { - path: artifacts.path().to_owned(), - message: "contains no registered Criterion benchmarks for allocation tracking".to_owned(), - }); - } + let targets = discover_native_probe_targets(executable, artifacts, arguments)?; let mut results = Vec::with_capacity(targets.len()); for (index, target) in targets.into_iter().enumerate() { let artifact_path = artifacts.path().join("allocations").join(format!("{index}.json")); let criterion_home = artifacts.path().join("allocation-probes").join(index.to_string()); - let native_arguments = criterion_probe_arguments(arguments.criterion_args(), &target.native_identity); + let native_arguments = target.arguments(); let status = launch_worker_with( Mode::Allocations, executable, @@ -491,102 +589,114 @@ fn run_allocations( Ok(results) } -fn discover_criterion_probe_targets( +fn discover_native_probe_targets( executable: &Path, artifacts: &ArtifactDirectory, - identities: &[BenchmarkIdentity], - criterion_arguments: &[OsString], - timeout: Option, -) -> Result, Error> { - let output_path = artifacts.path().join("criterion-probe-discovery.txt"); - let criterion_home = artifacts.path().join("criterion-probe-discovery"); - let mut discovery_arguments = criterion_arguments - .iter() - .filter(|argument| { - ![ - OsStr::new("--bench"), - OsStr::new("--test"), - OsStr::new("--list"), - OsStr::new("--exact"), - ] - .contains(&argument.as_os_str()) - }) - .cloned() - .collect::>(); - discovery_arguments.push("--list".into()); + arguments: &Arguments, +) -> Result, Error> { + let home = tempfile::Builder::new() + .prefix("native-probe-discovery-") + .tempdir_in(artifacts.path()) + .map_err(|source| Error::ArtifactIo { + path: artifacts.path().to_owned(), + source, + })? + .keep(); + let output_path = home.join("cases.json"); let status = launch_worker_with( - Mode::Criterion, + Mode::Allocations, executable, - &discovery_arguments, + &["--metabench-probe-list".into()], Some(artifacts), false, - timeout, - Some(&criterion_home), + arguments.timeout, + Some(&home), Some(&output_path), None, )?; - ensure_success(Mode::Criterion, status)?; - let output = fs::read_to_string(&output_path).map_err(|source| Error::ArtifactIo { - path: output_path.clone(), - source, - })?; - let targets = output - .lines() - .filter_map(|line| line.strip_suffix(": benchmark")) - .filter_map(|native_identity| { - artifact::allocation_criterion_identity(native_identity, identities).map(|identity| CriterionProbeTarget { - identity, - native_identity: native_identity.to_owned(), - }) - }) - .collect::>(); + ensure_success(Mode::Allocations, status)?; + let targets = read_probe_targets(&output_path)?; + let targets = select_native_probe_targets(targets, arguments.criterion_args())?; if targets.is_empty() { return Err(Error::ArtifactFormat { path: output_path, - message: "Criterion listing contained no benchmarks".to_owned(), + message: "no native benchmark cases matched; existing reports were preserved".to_owned(), }); } Ok(targets) } -fn run_perf( - executable: &Path, - arguments: &Arguments, - artifacts: &ArtifactDirectory, - identities: &[BenchmarkIdentity], - criterion_completed: bool, -) -> Result, Error> { +fn read_probe_targets(path: &Path) -> Result, Error> { + let targets: Vec = artifact::read_json(path)?; + if targets.len() > crate::probe::MAX_PROBE_CASES { + return Err(Error::ArtifactFormat { + path: path.to_owned(), + message: format!("native discovery exceeds the {}-case limit", crate::probe::MAX_PROBE_CASES), + }); + } + Ok(targets) +} + +fn select_native_probe_targets(targets: Vec, arguments: &[OsString]) -> Result, Error> { + let selection = probe_selection_arguments(arguments)?; + let mut filter = None; + let mut exact = false; + let mut value_expected = false; + for argument in &selection { + if value_expected { + value_expected = false; + continue; + } + let text = argument.to_string_lossy(); + if text == "--exact" { + exact = true; + } else if text.starts_with('-') { + let option = text.split('=').next().unwrap_or(&text); + if criterion_option_takes_value(option) { + value_expected = !text.contains('='); + } else if !matches!(option, "--verbose" | "--quiet" | "--debug" | "--discard-baseline" | "-v") { + return Err(Error::Probe(format!( + "unsupported Criterion option for native probe selection: {text}" + ))); + } + } else if filter.replace(text.into_owned()).is_some() { + return Err(Error::Probe("only one case selection filter is supported".to_owned())); + } + } + let pattern = filter + .as_deref() + .filter(|_| !exact) + .map(regex::Regex::new) + .transpose() + .map_err(|error| Error::Probe(format!("invalid case filter: {error}")))?; + Ok(targets + .into_iter() + .filter(|target| { + filter.as_deref().is_none_or(|filter| { + if exact { + target.identity == filter + } else { + pattern.as_ref().is_some_and(|pattern| pattern.is_match(&target.identity)) + } + }) + }) + .collect()) +} + +fn run_perf(executable: &Path, arguments: &Arguments, artifacts: &ArtifactDirectory) -> Result, Error> { if !cfg!(target_os = "linux") { return Err(Error::UnsupportedMode { mode: Mode::Perf.as_str(), reason: "Linux perf is supported only on Linux", }); } - let targets = if criterion_completed { - artifact::parse_criterion(&artifacts.criterion_path(), artifacts.root(), identities, None)? - .into_iter() - .filter_map(|result| { - artifact::allocation_criterion_identity(&result.native_identity, identities).map(|identity| CriterionProbeTarget { - identity, - native_identity: result.native_identity, - }) - }) - .collect() - } else { - discover_criterion_probe_targets(executable, artifacts, identities, arguments.criterion_args(), arguments.timeout)? - }; - if targets.is_empty() { - return Err(Error::ArtifactFormat { - path: artifacts.path().to_owned(), - message: "contains no registered Criterion benchmarks for perf measurement".to_owned(), - }); - } + let targets = discover_native_probe_targets(executable, artifacts, arguments)?; let mut results = Vec::with_capacity(targets.len()); for (index, target) in targets.into_iter().enumerate() { let artifact_path = artifacts.perf_path(index); let criterion_home = artifacts.path().join("perf-probes").join(index.to_string()); - let native_arguments = criterion_probe_arguments(arguments.criterion_args(), &target.native_identity); + let native_arguments = target.arguments(); let status = launch_perf_worker( executable, &native_arguments, @@ -617,9 +727,6 @@ fn launch_perf_worker( show_output: bool, timeout: Option, ) -> Result { - use nix::sys::stat::Mode as FileMode; - use nix::unistd::mkfifo; - validate_perf_arguments(perf_arguments)?; let fifo_directory = tempfile::Builder::new() .prefix("metabench-perf-") @@ -663,6 +770,7 @@ fn launch_perf_worker( let control_argument = format!("fifo:{control_text},{acknowledgement_text}"); let mut command = Command::new("perf"); + clear_collector_environment(&mut command); command .arg("stat") .args(perf_arguments) @@ -688,13 +796,12 @@ fn launch_perf_worker( .env_remove("CARGO_CRITERION_PORT") .env(perf::CONTROL_ENV, &control) .env(perf::ACK_ENV, &acknowledgement); - if !show_output { - command.stdout(Stdio::null()).stderr(Stdio::null()); - } + let log_path = artifact_path.with_extension("log"); + let log = capture_worker_output(&mut command, &log_path, show_output)?; let token = create_worker_token()?; command.env(WORKER_TOKEN_ENV, token.path()); - let status = wait_for_worker(Mode::Perf, &mut command, timeout); + let status = wait_for_logged_worker(Mode::Perf, &mut command, timeout, &log_path, log); drop(control_guard); drop(acknowledgement_guard); let cleanup = match token.path().try_exists() { @@ -750,39 +857,15 @@ fn validate_perf_arguments(arguments: &[OsString]) -> Result<(), Error> { Ok(()) } -fn run_vtune( - executable: &Path, - arguments: &Arguments, - artifacts: &ArtifactDirectory, - identities: &[BenchmarkIdentity], - criterion_completed: bool, -) -> Result, Error> { - let targets = if criterion_completed { - artifact::parse_criterion(&artifacts.criterion_path(), artifacts.root(), identities, None)? - .into_iter() - .filter_map(|result| { - artifact::allocation_criterion_identity(&result.native_identity, identities).map(|identity| CriterionProbeTarget { - identity, - native_identity: result.native_identity, - }) - }) - .collect() - } else { - discover_criterion_probe_targets(executable, artifacts, identities, arguments.criterion_args(), arguments.timeout)? - }; - if targets.is_empty() { - return Err(Error::ArtifactFormat { - path: artifacts.path().to_owned(), - message: "contains no registered Criterion benchmarks for vtune measurement".to_owned(), - }); - } +fn run_vtune(executable: &Path, arguments: &Arguments, artifacts: &ArtifactDirectory) -> Result, Error> { + let targets = discover_native_probe_targets(executable, artifacts, arguments)?; let mut results = Vec::with_capacity(targets.len()); for (index, target) in targets.into_iter().enumerate() { let artifact_path = artifacts.vtune_path(index); let result_dir = artifacts.vtune_result_path(index); let criterion_home = artifacts.path().join("vtune-probes").join(index.to_string()); - let native_arguments = criterion_probe_arguments(arguments.criterion_args(), &target.native_identity); + let native_arguments = target.arguments(); let status = launch_vtune_worker( executable, &native_arguments, @@ -847,6 +930,7 @@ fn launch_vtune_worker( } let mut command = Command::new("vtune"); + clear_collector_environment(&mut command); command .arg("-collect-with") .arg("runsa") @@ -861,13 +945,12 @@ fn launch_vtune_worker( .arg("--") .arg(executable) .args(native_arguments); - if !show_output { - command.stdout(Stdio::null()).stderr(Stdio::null()); - } + let log_path = artifact_path.with_extension("log"); + let log = capture_worker_output(&mut command, &log_path, show_output)?; let token = create_worker_token()?; command.env(WORKER_TOKEN_ENV, token.path()); - let status = wait_for_worker(Mode::Vtune, &mut command, timeout); + let status = wait_for_logged_worker(Mode::Vtune, &mut command, timeout, &log_path, log); let cleanup = match token.path().try_exists() { Ok(true) => fs::remove_dir(token.path()).map_err(Error::ConsumeWorkerToken), Ok(false) => Ok(()), @@ -879,15 +962,15 @@ fn launch_vtune_worker( } let mut report_command = Command::new("vtune"); + clear_collector_environment(&mut report_command); report_command .args(["-report", "hw-events", "-r"]) .arg(result_dir) .args(["-format", "csv", "-report-output"]) .arg(artifact_path); - if !show_output { - report_command.stdout(Stdio::null()).stderr(Stdio::null()); - } - wait_for_worker(Mode::Vtune, &mut report_command, timeout) + let report_log = artifact_path.with_extension("report.log"); + let log = capture_worker_output(&mut report_command, &report_log, show_output)?; + wait_for_logged_worker(Mode::Vtune, &mut report_command, timeout, &report_log, log) } fn validate_vtune_arguments(arguments: &[OsString]) -> Result<(), Error> { @@ -960,10 +1043,11 @@ fn launch_worker_with( show_output: bool, timeout: Option, criterion_home: Option<&Path>, - captured_stdout: Option<&Path>, + discovery_output: Option<&Path>, allocation_target: Option<(&str, &Path)>, ) -> Result { let mut command = Command::new(executable); + clear_collector_environment(&mut command); command.args(native_arguments).env(WORKER_MODE_ENV, mode.as_str()); if let Some(artifacts) = artifacts { match mode { @@ -987,27 +1071,24 @@ fn launch_worker_with( if let Some((identity, path)) = allocation_target { command.env(allocation::IDENTITY_ENV, identity).env(allocation::ARTIFACT_ENV, path); } - if let Some(path) = captured_stdout { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent).map_err(|source| Error::ArtifactIo { - path: parent.to_owned(), - source, - })?; - } - let output = fs::File::create(path).map_err(|source| Error::ArtifactIo { - path: path.to_owned(), - source, - })?; - command.stdout(output); - if !show_output { - command.stderr(Stdio::null()); - } - } else if !show_output { - command.stdout(Stdio::null()).stderr(Stdio::null()); - } + if let Some(path) = discovery_output { + command.env(crate::probe::DISCOVERY_ENV, path); + } + let log_root = criterion_home + .map(Path::to_owned) + .or_else(|| artifacts.map(|artifacts| artifacts.path().join("logs"))); + let log_path = log_root.map(|root| root.join(format!("{}.log", mode.as_str()))); + let log = log_path + .as_deref() + .map(|path| capture_worker_output(&mut command, path, show_output)) + .transpose()? + .flatten(); let token = create_worker_token()?; command.env(WORKER_TOKEN_ENV, token.path()); - let status = wait_for_worker(mode, &mut command, timeout); + let status = match &log_path { + Some(path) => wait_for_logged_worker(mode, &mut command, timeout, path, log), + None => wait_for_worker(mode, &mut command, timeout), + }; let cleanup = match token.path().try_exists() { Ok(true) => fs::remove_dir(token.path()).map_err(Error::ConsumeWorkerToken), Ok(false) => Ok(()), @@ -1020,6 +1101,226 @@ fn launch_worker_with( } } +fn clear_collector_environment(command: &mut Command) { + for name in [ + allocation::ARTIFACT_ENV, + allocation::IDENTITY_ENV, + perf::CONTROL_ENV, + perf::ACK_ENV, + vtune::RESULT_DIR_ENV, + crate::probe::DISCOVERY_ENV, + ] { + command.env_remove(name); + } +} + +pub(crate) fn terminal_text(text: &str) -> String { + let mut output = String::with_capacity(text.len()); + for character in text.chars() { + if character.is_control() + || matches!(character, '\u{061c}' | '\u{200e}' | '\u{200f}' | '\u{2028}' | '\u{2029}' | '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}') + { + output.extend(character.escape_default()); + } else { + output.push(character); + } + } + output +} + +fn capture_worker_output(command: &mut Command, path: &Path, show_output: bool) -> Result, Error> { + if show_output { + return Ok(None); + } + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|source| Error::ArtifactIo { + path: parent.to_owned(), + source, + })?; + } + let log = fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(path) + .map_err(|source| Error::ArtifactIo { + path: path.to_owned(), + source, + })?; + WorkerLog::new(log, command).map(Some).map_err(|source| Error::ArtifactIo { + path: path.to_owned(), + source, + }) +} + +fn wait_for_logged_worker( + mode: Mode, + command: &mut Command, + timeout: Option, + path: &Path, + log: Option, +) -> Result { + wait_for_logged_worker_with_output(mode, command, timeout, path, log, &mut io::stderr()) +} + +fn wait_for_logged_worker_with_output( + mode: Mode, + command: &mut Command, + timeout: Option, + path: &Path, + log: Option, + output: &mut impl io::Write, +) -> Result { + let Some(mut log) = log else { + return wait_for_worker(mode, command, timeout); + }; + let result = wait_for_captured_worker(mode, command, timeout, path, &mut log); + let persisted = log.persist().map_err(|source| Error::ArtifactIo { + path: path.to_owned(), + source, + }); + let result = combine_worker_results(mode, result, persisted); + if !result.as_ref().is_ok_and(ExitStatus::success) { + let diagnostics = write_worker_excerpt(output, mode, path, &log.excerpt(), log.truncated()).map_err(|source| Error::ArtifactIo { + path: path.to_owned(), + source, + }); + return combine_worker_results(mode, result, diagnostics); + } + result +} + +fn combine_worker_results(mode: Mode, result: Result, supplemental: Result<(), Error>) -> Result { + let result = if supplemental.is_err() { + result.and_then(|status| ensure_success(mode, status).map(|()| status)) + } else { + result + }; + combine_results(result, supplemental) +} + +#[cfg(test)] +fn write_worker_log(output: &mut impl io::Write, mode: Mode, path: &Path, mut log: &fs::File, truncated: bool) -> io::Result<()> { + let length = log.metadata()?.len(); + log.seek(io::SeekFrom::Start(length.saturating_sub(8_192)))?; + let mut bytes = Vec::new(); + log.take(8_192).read_to_end(&mut bytes)?; + write_worker_excerpt(output, mode, path, &bytes, truncated) +} + +fn write_worker_excerpt(output: &mut impl io::Write, mode: Mode, path: &Path, bytes: &[u8], truncated: bool) -> io::Result<()> { + writeln!( + output, + "metabench: {} worker log: {}{}\n{}", + mode.as_str(), + terminal_text(&path.to_string_lossy()), + if truncated { " (output truncated; retained tail)" } else { "" }, + terminal_text(&String::from_utf8_lossy(bytes)) + ) +} + +fn wait_for_captured_worker( + mode: Mode, + command: &mut Command, + timeout: Option, + path: &Path, + log: &mut WorkerLog, +) -> Result { + let spawned = command.group_spawn().map_err(|source| spawn_error(mode, source)); + WorkerLog::release_command_writers(command); + let mut child = spawned?; + wait_for_captured_process(mode, &mut child, timeout, path, || log.drain(), &SystemClock) +} + +/// Waits for worker completion and output EOF, bounding inherited writers' final drain. +/// +/// Read errors terminate and reap immediately instead of entering the final-drain state. +fn wait_for_captured_process( + mode: Mode, + child: &mut impl WorkerProcess, + timeout: Option, + path: &Path, + mut drain: impl FnMut() -> io::Result<(bool, bool)>, + clock: &impl Clock, +) -> Result { + let started = clock.mark(); + let mut completed = None; + let mut drain_started = None; + loop { + let (eof, had_output) = match drain() { + Ok(state) => state, + Err(source) => { + let error = Error::ArtifactIo { + path: path.to_owned(), + source, + }; + let cleanup = kill_and_reap_worker(mode, child, clock); + return combine_results(Err(error), cleanup); + } + }; + if completed.is_none() { + completed = match child.try_wait() { + Ok(Some(status)) => Some(Ok(status)), + Ok(None) => match timeout { + Some(timeout) if clock.elapsed(&started) >= timeout => { + // Termination and reaping consume the same grace as the remaining output drain. + drain_started = Some(clock.mark()); + Some(terminate_timed_out_worker(mode, child, timeout, clock)) + } + _ => None, + }, + Err(source) => { + let error = spawn_error(mode, source); + drain_started = Some(clock.mark()); + let cleanup = kill_and_reap_worker(mode, child, clock); + Some(combine_results(Err(error), cleanup)) + } + }; + if completed.is_some() && drain_started.is_none() { + drain_started = Some(clock.mark()); + } + } + if (eof + || drain_started + .as_ref() + .is_some_and(|mark| clock.elapsed(mark) >= TERMINATION_REAP_GRACE)) + && let Some(result) = completed.take() + { + if eof { + return result; + } + let error = Error::ArtifactIo { + path: path.to_owned(), + source: io::Error::new(io::ErrorKind::TimedOut, "worker output pipe remained open after the drain deadline"), + }; + let cleanup = kill_and_reap_worker(mode, child, clock); + return combine_results(combine_results(result, Err(error)), cleanup); + } + clock.sleep(if had_output { + Duration::from_millis(1) + } else { + WORKER_POLL_INTERVAL + }); + } +} + +fn kill_and_reap_worker(mode: Mode, child: &mut impl WorkerProcess, clock: &impl Clock) -> Result<(), Error> { + child.kill().map_err(|source| termination_error(mode, source))?; + let started = clock.mark(); + loop { + match child.try_wait().map_err(|source| termination_error(mode, source))? { + Some(_) => return Ok(()), + None if clock.elapsed(&started) < TERMINATION_REAP_GRACE => clock.sleep(WORKER_POLL_INTERVAL), + None => { + return Err(termination_error( + mode, + io::Error::new(io::ErrorKind::TimedOut, "worker did not exit before the termination reap deadline"), + )); + } + } + } +} + fn wait_for_worker(mode: Mode, command: &mut Command, timeout: Option) -> Result { let mut child = command.group_spawn().map_err(|source| Error::Spawn { mode: mode.as_str(), @@ -1177,7 +1478,7 @@ fn write_console_output(mut destination: impl io::Write, output: &str) -> Result } fn report_paths(arguments: &Arguments, benchmark_target: &str) -> Result<(Option, Option), Error> { - let root = default_target_directory_with(|name| env::var_os(name), env::current_dir)? + let root = effective_target_directory(¤t_executable(env::current_exe)?, |name| env::var_os(name))? .join("metabench") .join(benchmark_target); report_paths_with_root(arguments, &root) @@ -1199,29 +1500,43 @@ fn report_paths_with_root(arguments: &Arguments, root: &Path) -> Result<(Option< Ok(paths) } -fn default_target_directory_with( - var_os: impl Fn(&str) -> Option, - current_dir: impl FnOnce() -> io::Result, -) -> Result { - if let Some(directory) = var_os("CARGO_TARGET_DIR") { - return Ok(directory.into()); +fn effective_target_directory(executable: &Path, var_os: impl Fn(&str) -> Option) -> Result { + let configured = var_os("CARGO_TARGET_DIR").map(PathBuf::from); + let parent = executable.parent(); + let profile = parent.and_then(|parent| { + if matches!(parent.file_name().and_then(OsStr::to_str), Some("deps" | "examples")) { + parent.parent() + } else { + Some(parent) + } + }); + if let Some(profile) = profile + && (matches!(profile.file_name().and_then(OsStr::to_str), Some("debug" | "release")) || profile.join(".fingerprint").is_dir()) + && let Some(root) = profile.parent() + { + return Ok( + if root.file_name() == Some(OsStr::new(env!("METABENCH_TARGET"))) + && !configured.as_deref().is_some_and(|configured| paths_alias(configured, root)) + { + root.parent().unwrap_or(root).to_owned() + } else { + root.to_owned() + }, + ); + } + if let Some(directory) = configured { + return Ok(directory); } - let current = current_dir().map_err(Error::CurrentDirectory)?; - let workspace = current - .ancestors() - .find(|directory| is_workspace_manifest(&directory.join("Cargo.toml"))) - .unwrap_or(¤t); - Ok(workspace.join("target")) + Err(Error::ArtifactFormat { + path: executable.to_owned(), + message: "cannot infer the Cargo target directory from this executable; set CARGO_TARGET_DIR for a relocated binary".to_owned(), + }) } fn current_executable(current_exe: impl FnOnce() -> io::Result) -> Result { current_exe().map_err(Error::CurrentExecutable) } -fn is_workspace_manifest(path: &Path) -> bool { - fs::read_to_string(path).is_ok_and(|manifest| manifest.lines().any(|line| line.trim() == "[workspace]")) -} - struct WorkerToken { directory: tempfile::TempDir, } @@ -1267,44 +1582,404 @@ fn consume_worker_token_from(token: Option<&OsStr>, temporary_directory: &Path) mod tests { use std::cell::Cell; use std::collections::VecDeque; + #[cfg(unix)] + use std::os::unix::process::ExitStatusExt as _; + #[cfg(windows)] + use std::os::windows::process::ExitStatusExt as _; use super::*; #[test] - fn criterion_probe_arguments_preserve_configuration_and_select_one_target() { - let configured = [OsString::from("--sample-size"), OsString::from("20"), OsString::from("--bench")]; + fn artifacts_are_created_for_measurements_and_probe_listing_only() { + let directory = tempfile::tempdir_in(".").unwrap(); + for (list, probes, expected) in [(false, false, true), (false, true, true), (true, false, false), (true, true, true)] { + let root = directory.path().join(format!("{list}-{probes}")); + let artifacts = create_run_artifacts(&root, list, probes).unwrap(); + assert_eq!(artifacts.is_some(), expected); + assert_eq!(root.exists(), expected); + if let Some(artifacts) = artifacts { + assert!(artifacts.path().is_dir()); + } + } + } + + #[test] + fn collector_environment_is_removed_without_removing_worker_or_native_settings() { + let collectors = [ + allocation::ARTIFACT_ENV, + allocation::IDENTITY_ENV, + perf::CONTROL_ENV, + perf::ACK_ENV, + vtune::RESULT_DIR_ENV, + crate::probe::DISCOVERY_ENV, + ]; + let mut command = Command::new("unused"); + for name in collectors { + command.env(name, "stale collector configuration"); + } + command.env(WORKER_MODE_ENV, "criterion").env("CRITERION_HOME", "native"); + clear_collector_environment(&mut command); + let settings = command.get_envs().collect::>(); + for name in collectors { + assert!(settings.contains(&(OsStr::new(name), None))); + } + assert!(settings.contains(&(OsStr::new(WORKER_MODE_ENV), Some(OsStr::new("criterion"))))); + assert!(settings.contains(&(OsStr::new("CRITERION_HOME"), Some(OsStr::new("native"))))); + } + + #[test] + fn native_discovery_receiver_enforces_case_budget_and_listing_encoding() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("discovery.json"); + let case = serde_json::json!({ + "identity": "group/escape\u{1b}[31m\r\nforged", + "native_identity": "native", + "group": 0, "benchmark": 0, "case": 0, "iteration": null + }); + for (count, accepted) in [(10_000, true), (10_001, false)] { + artifact::write_json(&path, &vec![case.clone(); count]).unwrap(); + let result = read_probe_targets(&path); + assert_eq!(result.is_ok(), accepted); + if let Ok(targets) = result { + let mut output = Vec::new(); + write_probe_targets(&mut output, &targets[..1]).unwrap(); + assert_eq!( + String::from_utf8(output).unwrap(), + "group/escape\\u{1b}[31m\\r\\nforged: benchmark\n" + ); + assert_eq!(targets[0].identity, "group/escape\u{1b}[31m\r\nforged"); + assert_eq!( + serde_json::from_slice::(&fs::read(&path).unwrap()).unwrap()[0], + case + ); + } else { + assert!(result.unwrap_err().to_string().contains("10000-case limit")); + } + } + } + + #[test] + fn captured_logs_are_exclusive_and_excerpt_encoding_preserves_raw_bytes() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("worker.log"); + let mut command = Command::new("unused"); + let mut log = capture_worker_output(&mut command, &path, false).unwrap().unwrap(); + let raw = b"failure\x1b[31m\r\nforged"; + log.file.write_all(raw).unwrap(); + assert!(matches!( + capture_worker_output(&mut Command::new("unused"), &path, false), + Err(Error::ArtifactIo { source, .. }) if source.kind() == io::ErrorKind::AlreadyExists + )); + let mut output = Vec::new(); + write_worker_log(&mut output, Mode::Allocations, &path, &log.file, false).unwrap(); + assert!(String::from_utf8(output).unwrap().ends_with("failure\\u{1b}[31m\\r\\nforged\n")); + assert_eq!(fs::read(&path).unwrap(), raw); + #[cfg(unix)] + { + fs::rename(&path, directory.path().join("retained.log")).unwrap(); + fs::write(&path, b"substituted log").unwrap(); + let mut output = Vec::new(); + write_worker_log(&mut output, Mode::Allocations, &path, &log.file, false).unwrap(); + assert!(!String::from_utf8(output).unwrap().contains("substituted")); + } + } + + #[test] + fn captured_logs_never_truncate_preexisting_link_targets() { + let directory = tempfile::tempdir().unwrap(); + let sentinel = directory.path().join("sentinel"); + let path = directory.path().join("worker.log"); + fs::write(&sentinel, b"preserve me").unwrap(); + fs::hard_link(&sentinel, &path).unwrap(); + assert!(matches!( + capture_worker_output(&mut Command::new("unused"), &path, false), + Err(Error::ArtifactIo { source, .. }) if source.kind() == io::ErrorKind::AlreadyExists + )); + assert_eq!(fs::read(&sentinel).unwrap(), b"preserve me"); + #[cfg(unix)] + { + fs::remove_file(&path).unwrap(); + std::os::unix::fs::symlink(&sentinel, &path).unwrap(); + assert!(matches!( + capture_worker_output(&mut Command::new("unused"), &path, false), + Err(Error::ArtifactIo { source, .. }) if source.kind() == io::ErrorKind::AlreadyExists + )); + assert_eq!(fs::read(&sentinel).unwrap(), b"preserve me"); + } + } + + #[test] + fn captured_output_fixture() { + let Some(scenario) = env::var_os("METABENCH_LOG_TEST_SCENARIO") else { + return; + }; + let mut stdout = io::stdout().lock(); + let mut stderr = io::stderr().lock(); + let noisy = scenario != "success" && scenario != "failure"; + let chunks = if noisy { 512 } else { 1 }; + let bytes = [b'x'; 8_192]; + let started = Instant::now(); + loop { + for _ in 0..chunks { + stdout.write_all(&bytes).unwrap(); + stderr.write_all(&bytes).unwrap(); + } + stdout.write_all(b"\ncritical stdout diagnostic\n").unwrap(); + stdout.flush().unwrap(); + stderr.write_all(b"critical stderr diagnostic\xff\x1b[31m\n").unwrap(); + stderr.flush().unwrap(); + // A disabled parent deadline must fail its assertion without leaving this fixture running forever. + if scenario != "timeout" || started.elapsed() >= Duration::from_secs(2) { + std::process::exit(i32::from(scenario != "success")); + } + } + } + + fn captured_fixture_command(scenario: &str) -> Command { + let mut command = Command::new(env::current_exe().unwrap()); + command + .args(["--exact", "runner::tests::captured_output_fixture", "--nocapture"]) + .env("METABENCH_LOG_TEST_SCENARIO", scenario); + command + } + + #[test] + fn captured_logs_keep_bounded_raw_tail_for_noisy_failures_and_normal_workers() { + for scenario in ["success", "failure", "noisy-failure"] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("worker.log"); + let mut command = captured_fixture_command(scenario); + let mut log = capture_worker_output(&mut command, &path, false).unwrap().unwrap(); + let result = wait_for_captured_worker(Mode::Criterion, &mut command, Some(Duration::from_secs(10)), &path, &mut log); + assert_eq!(result.unwrap().success(), scenario == "success"); + assert!(log.file.metadata().unwrap().len() <= crate::worker_log::LOG_LIMIT as u64); + log.persist().unwrap(); + let bytes = fs::read(&path).unwrap(); + assert!(bytes.len() <= crate::worker_log::LOG_LIMIT); + assert_eq!(log.truncated(), scenario == "noisy-failure"); + assert!( + bytes + .windows(b"critical stdout diagnostic".len()) + .any(|bytes| bytes == b"critical stdout diagnostic") + ); + assert!(bytes.ends_with(b"critical stderr diagnostic\xff\x1b[31m\n")); + if log.truncated() { + assert_eq!(bytes.len(), crate::worker_log::LOG_LIMIT); + assert!(bytes.starts_with(b"[metabench: output truncated;")); + } + let mut output = Vec::new(); + write_worker_excerpt(&mut output, Mode::Criterion, &path, &log.excerpt(), log.truncated()).unwrap(); + let excerpt = String::from_utf8(output).unwrap(); + assert!(excerpt.contains("critical stdout diagnostic")); + assert!(excerpt.contains("critical stderr diagnostic")); + assert!(excerpt.contains("\\u{1b}[31m\\n")); + assert_eq!(excerpt.contains("output truncated"), scenario == "noisy-failure"); + } + } + + #[test] + fn captured_logs_bound_sustained_output_and_timeout_cleanup() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("worker.log"); + let mut command = captured_fixture_command("timeout"); + let log = capture_worker_output(&mut command, &path, false).unwrap(); + let started = Instant::now(); + let error = wait_for_logged_worker_with_output( + Mode::Criterion, + &mut command, + Some(Duration::from_millis(150)), + &path, + log, + &mut Vec::new(), + ) + .unwrap_err(); + assert!(error.to_string().contains("timeout"), "{error}"); + assert!(started.elapsed() < Duration::from_secs(5)); + assert!(fs::metadata(&path).unwrap().len() <= crate::worker_log::LOG_LIMIT as u64); + } + + #[cfg(unix)] + #[test] + fn captured_logs_bound_inherited_writer_lifetime_after_worker_exit() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("worker.log"); + let mut command = Command::new("sh"); + command.args(["-c", "sleep 30 & printf 'parent diagnostic\\n'; exit 0"]); + let log = capture_worker_output(&mut command, &path, false).unwrap(); + let started = Instant::now(); + let error = wait_for_logged_worker_with_output(Mode::Criterion, &mut command, None, &path, log, &mut Vec::new()).unwrap_err(); + assert!(error.to_string().contains("pipe remained open"), "{error}"); + assert!(started.elapsed() < Duration::from_secs(5)); + assert_eq!(fs::read(&path).unwrap(), b"parent diagnostic\n"); + } + + #[test] + fn captured_logs_publish_failures_through_the_original_open_handle() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("worker.log"); + let mut command = captured_fixture_command("failure"); + let log = capture_worker_output(&mut command, &path, false).unwrap(); + #[cfg(unix)] + { + fs::rename(&path, directory.path().join("retained.log")).unwrap(); + fs::write(&path, b"substituted log").unwrap(); + } + let mut output = Vec::new(); + let status = wait_for_logged_worker_with_output( + Mode::Criterion, + &mut command, + Some(Duration::from_secs(10)), + &path, + log, + &mut output, + ) + .unwrap(); + assert!(!status.success()); + assert!(String::from_utf8(output).unwrap().contains("critical stderr diagnostic")); + let retained = if cfg!(unix) { + directory.path().join("retained.log") + } else { + path.clone() + }; + assert!(fs::read(retained).unwrap().ends_with(b"critical stderr diagnostic\xff\x1b[31m\n")); + #[cfg(unix)] + assert_eq!(fs::read(&path).unwrap(), b"substituted log"); + } + + #[test] + fn captured_logs_keep_diagnostics_when_file_publication_fails() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("readonly.log"); + fs::write(&path, b"preserve me").unwrap(); + let mut command = captured_fixture_command("failure"); + let log = WorkerLog::new(fs::File::open(&path).unwrap(), &mut command).unwrap(); + let mut output = Vec::new(); + let error = wait_for_logged_worker_with_output( + Mode::Criterion, + &mut command, + Some(Duration::from_secs(10)), + &path, + Some(log), + &mut output, + ) + .unwrap_err(); + assert!(error.to_string().contains("failed to access artifact")); + assert!(error.to_string().contains("exited with status 1")); + assert_eq!(fs::read(&path).unwrap(), b"preserve me"); + let diagnostics = String::from_utf8(output).unwrap(); + assert!(diagnostics.contains("critical stdout diagnostic")); + assert!(diagnostics.contains("critical stderr diagnostic")); + } + + #[test] + fn captured_logs_show_output_does_not_create_a_file() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("worker.log"); + let mut command = Command::new("unused"); + assert!(capture_worker_output(&mut command, &path, true).unwrap().is_none()); + assert!(!path.exists()); + } + #[test] + fn empty_native_backend_cannot_promote_nonempty_probe_results() { + let directory = tempfile::tempdir().unwrap(); + let base = directory.path().join("report"); + let json = base.with_extension("json"); + let markdown = base.with_extension("md"); + fs::write(&json, b"json sentinel").unwrap(); + fs::write(&markdown, b"markdown sentinel").unwrap(); + let arguments = Arguments::parse_from( + [ + "--criterion".into(), + "--allocations".into(), + "--output".into(), + base.into_os_string(), + ], + None, + ) + .unwrap(); + for mode in [Mode::Criterion, Mode::Gungraun] { + let artifacts = ArtifactDirectory::create(&directory.path().join("artifacts")).unwrap(); + fs::create_dir_all(artifacts.criterion_path()).unwrap(); + fs::create_dir_all(artifacts.gungraun_path()).unwrap(); + let probe = crate::report::NativeResult { + identity: "native/work".to_owned(), + native_identity: "native/work".to_owned(), + source: "alloc_tracker".to_owned(), + metrics: [( + "Allocations".to_owned(), + crate::report::Metric::integer("alloc_tracker", "Allocations", 1, crate::report::MetricDirection::LowerIsBetter), + )] + .into(), + raw_artifacts: Vec::new(), + }; + let error = build_report( + &artifacts, + &[mode, Mode::Allocations], + &[], + &arguments, + "target", + None, + vec![probe], + true, + ) + .unwrap_err() + .to_string(); + assert!(error.contains(mode.as_str())); + assert!(error.contains("no benchmark measurements")); + assert_eq!(fs::read(&json).unwrap(), b"json sentinel"); + assert_eq!(fs::read(&markdown).unwrap(), b"markdown sentinel"); + } + } + #[test] + fn native_probe_selection_validates_and_applies_filters() { + let targets = || { + serde_json::from_value(serde_json::json!([ + {"identity":"group/work/small","native_identity":"native/small","group":0,"benchmark":0,"case":0,"iteration":null}, + {"identity":"group/work/large","native_identity":"native/large","group":0,"benchmark":0,"case":1,"iteration":null} + ])) + .unwrap() + }; + let args = |values: &[&str]| values.iter().map(OsString::from).collect::>(); + let selected = select_native_probe_targets(targets(), &args(&["small$", "--sample-size", "10"])).unwrap(); + assert_eq!(selected.len(), 1); + assert_eq!(selected[0].identity, "group/work/small"); + assert!( + select_native_probe_targets(targets(), &args(&["small", "--exact"])) + .unwrap() + .is_empty() + ); assert_eq!( - criterion_probe_arguments(&configured, "group/benchmark"), - ["--sample-size", "20", "--bench", "group/benchmark", "--exact"] + select_native_probe_targets(targets(), &args(&["group/work/large", "--exact"])) + .unwrap() + .len(), + 1 ); + for invalid in [ + &["["][..], + &["small", "large"], + &["--unknown"], + &["--sample-size"], + &["--profile-time"], + ] { + assert!(select_native_probe_targets(targets(), &args(invalid)).is_err(), "{invalid:?}"); + } } #[test] - fn needs_criterion_for_listing_requires_list_a_native_only_engine_and_absence_of_criterion() { - // Not listing: never inject Criterion, regardless of the other modes. - assert!(!needs_criterion_for_listing(false, &[Mode::Vtune], &[])); - // Listing, but only native (non-Perf/Allocations/Vtune) modes selected: nothing to inject for. - assert!(!needs_criterion_for_listing(true, &[Mode::Criterion], &[Mode::Criterion])); - // Listing with Criterion already present among the native modes: no duplicate injection. - assert!(!needs_criterion_for_listing(true, &[Mode::Vtune], &[Mode::Criterion])); - // Listing with each non-native engine selected and no Criterion present yet: inject it. - assert!(needs_criterion_for_listing(true, &[Mode::Allocations], &[])); - assert!(needs_criterion_for_listing(true, &[Mode::Perf], &[])); - assert!(needs_criterion_for_listing(true, &[Mode::Vtune], &[])); + fn probe_selection_preserves_option_values_but_discards_benchmark_mode() { + let configured = [OsString::from("--sample-size"), OsString::from("20"), OsString::from("--bench")]; + + assert_eq!(probe_selection_arguments(&configured).unwrap(), ["--sample-size", "20"]); } #[cfg(unix)] fn exit_status(code: i32) -> ExitStatus { - use std::os::unix::process::ExitStatusExt as _; - ExitStatus::from_raw(code << 8) } #[cfg(windows)] fn exit_status(code: i32) -> ExitStatus { - use std::os::windows::process::ExitStatusExt as _; - ExitStatus::from_raw(u32::try_from(code).unwrap()) } @@ -1372,6 +2047,199 @@ mod tests { } } + #[test] + fn captured_process_waits_for_both_exit_and_output_eof() { + for (statuses, drains) in [ + (vec![Ok(None), Ok(Some(exit_status(3)))], vec![(true, false), (true, false)]), + (vec![Ok(Some(exit_status(3)))], vec![(false, true), (true, false)]), + ] { + let mut process = FakeProcess::new(statuses); + let clock = FakeClock::new(); + let mut drains = VecDeque::from(drains); + let status = wait_for_captured_process( + Mode::Criterion, + &mut process, + None, + Path::new("worker.log"), + || Ok(drains.pop_front().unwrap()), + &clock, + ) + .unwrap(); + assert_eq!(status.code(), Some(3)); + assert!(drains.is_empty()); + assert_eq!(process.kill_calls, 0); + } + } + + #[test] + fn captured_process_drains_until_the_exact_post_exit_deadline() { + let mut process = FakeProcess::new([Ok(Some(exit_status(0))), Ok(Some(exit_status(0)))]); + let clock = FakeClock::new(); + let mut polls = 0; + let error = wait_for_captured_process( + Mode::Criterion, + &mut process, + None, + Path::new("worker.log"), + || { + polls += 1; + clock.elapsed.set(match polls { + 1 => Duration::ZERO, + 2 => TERMINATION_REAP_GRACE.checked_sub(Duration::from_nanos(1)).unwrap(), + 3 => TERMINATION_REAP_GRACE, + _ => panic!("draining must stop at the deadline"), + }); + Ok((false, false)) + }, + &clock, + ) + .unwrap_err(); + assert!(error.to_string().contains("pipe remained open")); + assert_eq!(polls, 3); + assert_eq!(process.kill_calls, 1); + assert_eq!(clock.elapsed.get(), TERMINATION_REAP_GRACE); + } + + #[test] + fn captured_process_read_errors_kill_and_reap_and_preserve_cleanup_errors() { + let mut process = FakeProcess::new([Err(io::Error::other("reap failed"))]); + let error = wait_for_captured_process( + Mode::Criterion, + &mut process, + None, + Path::new("worker.log"), + || Err(io::Error::other("read failed")), + &FakeClock::new(), + ) + .unwrap_err() + .to_string(); + assert!(error.contains("read failed")); + assert!(error.contains("reap failed")); + assert_eq!(process.kill_calls, 1); + assert_eq!(process.wait_calls, 0); + } + + #[test] + fn captured_process_timeout_drain_deadline_includes_the_reap_grace() { + let polls = TERMINATION_REAP_GRACE.as_nanos() / WORKER_POLL_INTERVAL.as_nanos(); + let mut process = FakeProcess::new((0..=polls + 1).map(|_| Ok(None))); + let clock = FakeClock::new(); + let mut drains = 0; + let error = wait_for_captured_process( + Mode::Criterion, + &mut process, + Some(Duration::ZERO), + Path::new("worker.log"), + || { + drains += 1; + assert_eq!(drains, 1, "reaping has already exhausted the output drain deadline"); + Ok((false, false)) + }, + &clock, + ) + .unwrap_err() + .to_string(); + assert!(error.contains("worker exceeded")); + assert!(error.contains("pipe remained open")); + assert_eq!(clock.elapsed.get(), TERMINATION_REAP_GRACE); + assert_eq!(process.kill_calls, 2); + assert_eq!(process.wait_calls, 0); + } + + #[test] + fn captured_process_deadline_preempts_a_finite_successful_exit() { + let deadline = WORKER_POLL_INTERVAL * 2; + for timeout in [Some(deadline), None] { + let mut process = FakeProcess::new([Ok(None), Ok(None), Ok(None), Ok(Some(exit_status(0)))]); + let clock = FakeClock::new(); + let mut polls = 0; + let result = wait_for_captured_process( + Mode::Criterion, + &mut process, + timeout, + Path::new("worker.log"), + || { + polls += 1; + assert!(polls <= 4, "the scripted worker exits successfully within four polls"); + Ok((polls >= 3, false)) + }, + &clock, + ); + if timeout.is_some() { + assert!(matches!( + result, + Err(Error::WorkerTimedOut { mode: "criterion", timeout }) if timeout == deadline + )); + assert_eq!(polls, 3); + assert_eq!(clock.elapsed.get(), deadline); + assert_eq!(process.kill_calls, 1); + } else { + assert!(result.unwrap().success()); + assert_eq!(polls, 4); + assert_eq!(clock.elapsed.get(), WORKER_POLL_INTERVAL * 3); + assert_eq!(process.kill_calls, 0); + } + assert!(process.try_wait.is_empty()); + assert_eq!(process.wait_calls, 0); + } + } + + #[test] + fn captured_process_timeout_is_enforced_while_the_pipe_is_pending() { + let mut process = FakeProcess::new([Ok(None), Ok(None), Ok(Some(exit_status(9)))]); + let clock = FakeClock::new(); + let mut drains = VecDeque::from([(false, false), (false, false), (true, false)]); + let error = wait_for_captured_process( + Mode::Criterion, + &mut process, + Some(WORKER_POLL_INTERVAL), + Path::new("worker.log"), + || Ok(drains.pop_front().unwrap()), + &clock, + ) + .unwrap_err(); + assert!(matches!(error, Error::WorkerTimedOut { .. })); + assert!(drains.is_empty()); + assert_eq!(process.kill_calls, 1); + assert_eq!(clock.elapsed.get(), WORKER_POLL_INTERVAL * 2); + } + + #[test] + fn kill_and_reap_polls_pending_exit_but_stops_at_the_exact_grace_deadline() { + let mut process = FakeProcess::new([Ok(None), Ok(Some(exit_status(9)))]); + let clock = FakeClock::new(); + kill_and_reap_worker(Mode::Criterion, &mut process, &clock).unwrap(); + assert_eq!(process.kill_calls, 1); + assert_eq!(process.wait_calls, 0); + assert_eq!(clock.elapsed.get(), WORKER_POLL_INTERVAL); + assert!(process.try_wait.is_empty()); + + let polls = TERMINATION_REAP_GRACE.as_nanos() / WORKER_POLL_INTERVAL.as_nanos(); + let mut statuses = (0..=polls).map(|_| Ok(None)).collect::>(); + // A late exit also makes an accidentally unbounded grace finish deterministically. + statuses.push(Ok(Some(exit_status(9)))); + let mut process = FakeProcess::new(statuses); + let clock = FakeClock::new(); + let error = kill_and_reap_worker(Mode::Criterion, &mut process, &clock).unwrap_err(); + assert!(matches!(error, Error::WorkerTermination { source, .. } if source.kind() == io::ErrorKind::TimedOut)); + assert_eq!(clock.elapsed.get(), TERMINATION_REAP_GRACE); + assert_eq!(process.kill_calls, 1); + assert_eq!(process.wait_calls, 0); + assert_eq!(process.try_wait.len(), 1); + } + + #[test] + fn kill_and_reap_propagates_failed_kill_without_polling() { + let mut process = FakeProcess::new([Ok(Some(exit_status(0)))]); + process.kill = Some(Err(io::Error::other("kill denied"))); + let clock = FakeClock::new(); + let error = kill_and_reap_worker(Mode::Criterion, &mut process, &clock).unwrap_err(); + assert!(error.to_string().contains("kill denied")); + assert_eq!(process.kill_calls, 1); + assert_eq!(process.try_wait.len(), 1); + assert_eq!(clock.elapsed.get(), Duration::ZERO); + } + struct FailingOutput; impl io::Write for FailingOutput { @@ -1410,10 +2278,46 @@ mod tests { assert!(read_baseline(&path, true).unwrap().is_none()); assert!(matches!( read_baseline(&path, false), - Err(Error::UnsupportedReportSchema { found: 7, expected: 8, .. }) + Err(Error::UnsupportedReportSchema { found: 7, expected: 9, .. }) )); } + #[test] + fn baseline_warnings_encode_hostile_paths_and_schema_errors() { + let mut output = Vec::new(); + let hostile = Path::new("baseline\x1b[31m\r\nforged\u{202e}.json"); + write_baseline_warning( + &mut output, + &format!( + "skipping incompatible baseline {} (target, compiler, CPU, profile, or flags differ)", + hostile.display() + ), + ) + .unwrap(); + let warning = String::from_utf8(output).unwrap(); + assert!(warning.contains("baseline\\u{1b}[31m\\r\\nforged\\u{202e}.json")); + assert_eq!(warning.lines().count(), 1); + + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("baseline\u{202e}.json"); + let report = serde_json::json!({ + "schema_version": 7, + "metadata": { + "timestamp_utc": "", "target_triple": "", "rustc_version": "", + "host_cpu": "", "profile": "", "compiler_flags": null, + "measured_backends": [], "artifact_root": "" + }, + "entries": [] + }); + fs::write(&path, serde_json::to_vec(&report).unwrap()).unwrap(); + let mut output = Vec::new(); + assert!(read_baseline_with_output(&path, true, &mut output).unwrap().is_none()); + let warning = String::from_utf8(output).unwrap(); + assert!(warning.contains("baseline\\u{202e}.json")); + assert!(warning.contains("unsupported report schema")); + assert!(!warning.contains('\u{202e}')); + } + #[test] fn perf_arguments_cannot_override_measurement_protocol() { for argument in [ @@ -1691,12 +2595,99 @@ mod tests { } #[test] - fn environment_and_output_boundaries_map_exact_errors() { + fn probe_filters_are_replaced_without_losing_option_values() { + let arguments = ["--sample-size", "20", "--color", "never", "group", "--exact", "--bench"].map(OsString::from); assert_eq!( - default_target_directory_with( - |name| (name == "CARGO_TARGET_DIR").then(|| OsString::from("custom-target")), - || Err(io::Error::other("must not be called")), + probe_selection_arguments(&arguments).unwrap(), + ["--sample-size", "20", "--color", "never", "group", "--exact"] + ); + assert_eq!( + probe_selection_arguments(&[ + "--profile-time".into(), + "60".into(), + "--load-baseline=old".into(), + "--baseline".into(), + "old".into(), + "--noplot".into(), + "group".into(), + ],) + .unwrap(), + ["group"] + ); + } + + #[test] + fn native_probe_selection_accepts_criterion_aliases_and_output_options() { + for arguments in [ + vec![ + "-s", "old", "-b", "old", "-n", "-c", "never", "--format", "terse", "group", "--exact", + ], + vec!["-s=old", "-b=old", "-n", "-c=never", "--format=terse", "group", "--exact"], + ] { + let arguments = arguments.into_iter().map(OsString::from).collect::>(); + let targets = serde_json::from_str( + r#"[{"identity":"group","native_identity":"native/group","group":0,"benchmark":0,"case":0,"iteration":null}]"#, ) + .unwrap(); + let selected = select_native_probe_targets(targets, &arguments).unwrap(); + assert_eq!(selected.len(), 1); + assert_eq!(selected[0].identity, "group"); + } + for arguments in [vec!["-s"], vec!["-b"], vec!["-c"], vec!["--format"]] { + assert!(matches!( + probe_selection_arguments(&arguments.into_iter().map(OsString::from).collect::>()), + Err(Error::Probe(_)) + )); + } + } + + #[test] + fn configured_target_root_disambiguates_a_triple_named_directory() { + let configured = PathBuf::from("custom").join(env!("METABENCH_TARGET")); + for executable in [ + configured.join("release/deps/benchmark"), + configured.join(env!("METABENCH_TARGET")).join("release/deps/benchmark"), + ] { + assert_eq!( + effective_target_directory(&executable, |name| (name == "CARGO_TARGET_DIR").then(|| configured.clone().into())).unwrap(), + configured + ); + } + assert_eq!( + effective_target_directory(&configured.join("release/deps/benchmark"), |_| Some(configured.join(".").into())).unwrap(), + configured + ); + } + + #[test] + fn target_directory_follows_executable_layout_over_environment() { + for (executable, expected) in [ + ("custom/release/deps/benchmark", "custom"), + ("custom/debug/examples/benchmark", "custom"), + ("custom/release/benchmark", "custom"), + ] { + assert_eq!( + effective_target_directory(Path::new(executable), |_| Some("ignored".into())).unwrap(), + PathBuf::from(expected) + ); + } + let executable = PathBuf::from("custom") + .join(env!("METABENCH_TARGET")) + .join("release/deps/benchmark"); + assert_eq!(effective_target_directory(&executable, |_| None).unwrap(), PathBuf::from("custom")); + let custom = tempfile::tempdir().unwrap(); + fs::create_dir_all(custom.path().join("special/.fingerprint")).unwrap(); + assert_eq!( + effective_target_directory(&custom.path().join("special/deps/benchmark"), |_| None).unwrap(), + custom.path() + ); + } + + #[test] + fn environment_and_output_boundaries_map_exact_errors() { + assert_eq!( + effective_target_directory(Path::new("relocated/benchmark"), |name| (name == "CARGO_TARGET_DIR") + .then(|| OsString::from("custom-target")),) .unwrap(), PathBuf::from("custom-target") ); @@ -1705,8 +2696,8 @@ mod tests { Err(Error::CurrentExecutable(_)) )); assert!(matches!( - default_target_directory_with(|_| None, || Err(io::Error::other("no directory"))), - Err(Error::CurrentDirectory(_)) + effective_target_directory(Path::new("relocated/benchmark"), |_| None), + Err(Error::ArtifactFormat { .. }) )); let mut progress = Vec::new(); diff --git a/crates/metabench/src/vtune.rs b/crates/metabench/src/vtune.rs index 98828db7e..89d1f177c 100644 --- a/crates/metabench/src/vtune.rs +++ b/crates/metabench/src/vtune.rs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! Control protocol for measuring exactly the annotated workload under a live +//! Control protocol for measuring an explicit workload boundary under a live //! `vtune` collection, mirroring [`crate::perf`]'s Linux `perf` integration. //! //! Unlike `perf`, which is driven through a pair of FIFOs the worker polls @@ -15,13 +15,13 @@ //! ). //! `runner::launch_vtune_worker` starts the collection with `--start-paused` //! so nothing is measured until this module's [`begin`] resumes it, and this -//! module's [`Guard::drop`] pauses it again the instant the annotated +//! module's [`Guard::drop`] pauses it again the instant the selected //! workload returns. //! //! # Sampling limitations //! //! `VTune`'s hardware-event collection is a statistical sampler, not an exact -//! event counter: only the single Criterion invocation that first calls +//! event counter: only the single native probe invocation that first calls //! [`begin`] is measured (the `MEASURED` flag admits exactly one resume per //! process), and unlike the exact per-invocation counts `perf` produces, //! `VTune`'s reported counts are sampling estimates whose accuracy improves @@ -35,14 +35,14 @@ use std::env; use std::process::Command; +use std::sync::Mutex; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{LazyLock, Mutex}; use crate::error::Error; pub(crate) const RESULT_DIR_ENV: &str = "METABENCH_INTERNAL_VTUNE_RESULT_DIR"; -static ACTIVE: LazyLock = LazyLock::new(|| env::var_os(RESULT_DIR_ENV).is_some()); +static ACTIVE: AtomicBool = AtomicBool::new(false); static ERROR: Mutex> = Mutex::new(None); static MEASURED: AtomicBool = AtomicBool::new(false); @@ -64,14 +64,10 @@ impl Drop for Guard { #[inline] #[doc(hidden)] pub fn begin() -> Option { - if !*ACTIVE { + if !ACTIVE.load(Ordering::Relaxed) { return None; } - // Fetch and validate the result directory before claiming the single - // `MEASURED` slot: if `ACTIVE` was cached `true` while the env var was - // set but it is now gone (only possible within the same process), this - // records a control error instead of silently marking the workload - // "measured" with no collection actually resumed. + // Validate before claiming the one measurement slot. let Some(result_dir) = env::var_os(RESULT_DIR_ENV) else { record_error(Error::VtuneControl(std::io::Error::other( "METABENCH_INTERNAL_VTUNE_RESULT_DIR was unset after vtune::begin's environment was primed", @@ -79,10 +75,6 @@ pub fn begin() -> Option { return None; }; if MEASURED.swap(true, Ordering::Relaxed) { - // Only the first Criterion invocation in this process is ever - // measured; see this module's "Sampling limitations" doc for why - // that single-invocation, sampling-based measurement is `VTune`'s - // documented trade-off rather than a bug to fix here. return None; } let result_dir = std::path::PathBuf::from(result_dir); @@ -97,15 +89,13 @@ pub fn begin() -> Option { } } -/// Resolves the environment probe behind [`begin`] ahead of any measurement. +/// Enables collection after validating a `VTune` worker's token and mode. /// -/// See [`crate::perf::prime`] for why this must run before the measured -/// workload: leaving the `LazyLock` cold would charge the first workload for -/// the `getenv` call and its surrounding one-time synchronization. +/// Initialize the collector before a private probe invokes its workload, +/// as in [`crate::perf::prime`]. #[cfg_attr(coverage_nightly, coverage(off))] // not exercised by the instrumented test binary; see prime()'s doc comment -#[cfg_attr(test, mutants::skip)] // warm-up only affects timing, not any observable behavior pub(crate) fn prime() { - let _active = *ACTIVE; + ACTIVE.store(env::var_os(RESULT_DIR_ENV).is_some(), Ordering::Relaxed); } pub(crate) fn finish_worker() -> Result<(), Error> { diff --git a/crates/metabench/src/worker_log.rs b/crates/metabench/src/worker_log.rs new file mode 100644 index 000000000..1ced36e21 --- /dev/null +++ b/crates/metabench/src/worker_log.rs @@ -0,0 +1,390 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::collections::VecDeque; +use std::fs::File; +use std::io::{self, Read as _, Seek as _, Write as _}; +#[cfg(unix)] +use std::os::fd::AsRawFd as _; +#[cfg(windows)] +use std::os::windows::io::AsRawHandle as _; +use std::process::{Command, Stdio}; +#[cfg(windows)] +use std::ptr; + +#[cfg(windows)] +use windows_sys::Win32::Foundation::ERROR_BROKEN_PIPE; +#[cfg(windows)] +use windows_sys::Win32::System::Pipes::PeekNamedPipe; + +#[cfg(not(any(unix, windows)))] +use self::prepare_reader_unsupported as prepare_reader; +#[cfg(not(any(unix, windows)))] +use self::read_available_unsupported as read_available; +#[cfg(windows)] +use self::read_available_windows as read_available; +#[cfg(unix)] +use self::{prepare_reader_unix as prepare_reader, read_available_unix as read_available}; + +// Conservative diagnostic headroom without unbounded per-worker memory or storage. +pub(crate) const LOG_LIMIT: usize = 1_048_576; +const TRUNCATION: &[u8] = b"[metabench: output truncated; only the final bytes are retained]\n"; +const TAIL_LIMIT: usize = LOG_LIMIT - TRUNCATION.len(); +// Consume at most a quarter of the retained log before checking the worker deadline again. +const DRAIN_BUDGET: usize = 262_144; + +/// Owns a merged output pipe and a bounded raw tail, without a reader thread. +pub(crate) struct WorkerLog { + pub(crate) file: File, + reader: io::PipeReader, + output: OutputTail, +} + +struct OutputTail { + tail: VecDeque, + truncated: bool, + eof: bool, +} + +impl OutputTail { + fn new() -> Self { + Self { + tail: VecDeque::with_capacity(TAIL_LIMIT), + truncated: false, + eof: false, + } + } + + fn drain_with(&mut self, mut read: impl FnMut(&mut [u8]) -> io::Result) -> io::Result<(bool, bool)> { + if self.eof { + return Ok((true, false)); + } + let mut bytes = [0; 8_192]; + let mut had_output = false; + for _ in 0..DRAIN_BUDGET / bytes.len() { + match read(&mut bytes) { + Ok(0) => { + self.eof = true; + return Ok((true, had_output)); + } + Ok(count) => { + had_output = true; + self.retain(&bytes[..count]); + } + Err(error) if error.kind() == io::ErrorKind::WouldBlock => return Ok((false, had_output)), + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) => return Err(error), + } + } + Ok((false, had_output)) + } + + fn retain(&mut self, bytes: &[u8]) { + let excess = (self.tail.len() + bytes.len()).saturating_sub(TAIL_LIMIT); + if excess != 0 { + self.tail.drain(..excess); + self.truncated = true; + } + self.tail.extend(bytes); + } + + fn excerpt(&self) -> Vec { + self.tail.iter().skip(self.tail.len().saturating_sub(8_192)).copied().collect() + } +} + +impl WorkerLog { + pub(crate) fn new(file: File, command: &mut Command) -> io::Result { + let (reader, writer) = io::pipe()?; + #[cfg(not(windows))] + prepare_reader(&reader)?; + command.stdout(writer.try_clone()?); + command.stderr(writer); + Ok(Self { + file, + reader, + output: OutputTail::new(), + }) + } + + pub(crate) fn release_command_writers(command: &mut Command) { + // Command retains its Stdio handles after spawning; they would prevent EOF. + command.stdout(Stdio::inherit()).stderr(Stdio::inherit()); + } + + /// Bounds work per poll so continuous output cannot starve the worker deadline. + /// + /// Returns whether EOF was reached and whether any bytes were consumed. + pub(crate) fn drain(&mut self) -> io::Result<(bool, bool)> { + self.output.drain_with(|bytes| read_available(&mut self.reader, bytes)) + } + + pub(crate) fn persist(&mut self) -> io::Result<()> { + self.file.rewind()?; + if self.output.truncated { + self.file.write_all(TRUNCATION)?; + } + let (first, second) = self.output.tail.as_slices(); + self.file.write_all(first)?; + self.file.write_all(second)?; + Ok(()) + } + + pub(crate) const fn truncated(&self) -> bool { + self.output.truncated + } + + pub(crate) fn excerpt(&self) -> Vec { + self.output.excerpt() + } +} + +#[cfg(unix)] +fn prepare_reader_unix(reader: &io::PipeReader) -> io::Result<()> { + prepare_reader_with(|operation, flags| { + // SAFETY: reader owns a live descriptor; F_GETFL/F_SETFL take no pointer arguments. + unsafe { libc::fcntl(reader.as_raw_fd(), operation, flags) } + }) +} + +#[cfg(unix)] +fn prepare_reader_with(mut fcntl: impl FnMut(libc::c_int, libc::c_int) -> libc::c_int) -> io::Result<()> { + let flags = fcntl(libc::F_GETFL, 0); + if flags == -1 { + return Err(io::Error::last_os_error()); + } + if fcntl(libc::F_SETFL, flags | libc::O_NONBLOCK) == -1 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +#[cfg(unix)] +fn read_available_unix(reader: &mut io::PipeReader, bytes: &mut [u8]) -> io::Result { + reader.read(bytes) +} + +#[cfg(windows)] +fn read_available_windows(reader: &mut io::PipeReader, bytes: &mut [u8]) -> io::Result { + let mut available = 0; + // SAFETY: reader owns the live anonymous pipe handle. The only non-null + // output pointer addresses a writable u32; the zero-length buffer is null. + let result = unsafe { + PeekNamedPipe( + reader.as_raw_handle(), + ptr::null_mut(), + 0, + ptr::null_mut(), + &raw mut available, + ptr::null_mut(), + ) + }; + if result == 0 { + let error = io::Error::last_os_error(); + return if error.raw_os_error() == Some(ERROR_BROKEN_PIPE.cast_signed()) { + // ERROR_BROKEN_PIPE: all writers have closed. + Ok(0) + } else { + Err(error) + }; + } + if available == 0 { + return Err(io::ErrorKind::WouldBlock.into()); + } + let count = bytes.len().min(available as usize); + // This is the sole reader, so the bytes observed above cannot be consumed + // by another reader between the peek and this bounded read. + reader.read(&mut bytes[..count]) +} + +#[cfg(not(any(unix, windows)))] +fn prepare_reader_unsupported(_reader: &io::PipeReader) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "worker output capture requires Unix or Windows", + )) +} + +#[cfg(not(any(unix, windows)))] +fn read_available_unsupported(_reader: &mut io::PipeReader, _bytes: &mut [u8]) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "worker output capture requires Unix or Windows", + )) +} + +#[cfg(test)] +#[cfg_attr(coverage_nightly, coverage(off))] +mod tests { + use super::*; + + fn log() -> OutputTail { + OutputTail::new() + } + + #[test] + fn drain_limits_continuous_output_to_one_poll_budget() { + let mut log = log(); + let mut consumed = 0; + let state = log + .drain_with(|bytes| { + consumed += bytes.len(); + assert!(consumed <= DRAIN_BUDGET, "a poll must not starve the worker deadline"); + bytes.fill(b'x'); + Ok(bytes.len()) + }) + .unwrap(); + assert_eq!(state, (false, true)); + assert_eq!(consumed, DRAIN_BUDGET); + assert_eq!(log.tail.len(), DRAIN_BUDGET); + } + + #[test] + fn drain_retries_interrupts_preserves_output_and_stops_on_would_block() { + let mut log = log(); + let mut calls = 0; + let state = log + .drain_with(|bytes| { + calls += 1; + match calls { + 1 => Err(io::ErrorKind::Interrupted.into()), + 2 => { + bytes[..3].copy_from_slice(b"raw"); + Ok(3) + } + 3 => Err(io::ErrorKind::WouldBlock.into()), + _ => panic!("WouldBlock must end the poll"), + } + }) + .unwrap(); + assert_eq!(calls, 3); + assert_eq!(state, (false, true)); + assert_eq!(log.excerpt(), b"raw"); + assert!(!log.eof); + } + + #[test] + fn drain_propagates_fatal_errors_instead_of_retrying_or_suppressing_them() { + let mut log = log(); + let mut calls = 0; + let error = log + .drain_with(|_| { + calls += 1; + assert_eq!(calls, 1, "fatal errors must not be retried"); + Err(io::Error::new(io::ErrorKind::PermissionDenied, "read denied")) + }) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::PermissionDenied); + assert_eq!(error.to_string(), "read denied"); + assert!(!log.eof); + } + + #[test] + fn drain_records_eof_once_and_keeps_the_bytes_before_it() { + let mut log = log(); + let mut calls = 0; + assert_eq!( + log.drain_with(|bytes| { + calls += 1; + if calls == 1 { + bytes[0] = 0xff; + Ok(1) + } else { + Ok(0) + } + }) + .unwrap(), + (true, true) + ); + assert_eq!(calls, 2); + assert_eq!(log.excerpt(), [0xff]); + assert_eq!(log.drain_with(|_| panic!("EOF is permanent")).unwrap(), (true, false)); + } + + #[cfg(any(unix, windows))] + #[test] + fn available_pipe_reads_distinguish_pending_data_and_eof() { + #[cfg(windows)] + { + const FIXTURE: &str = "METABENCH_TEST_WINDOWS_PIPE"; + if std::env::var_os(FIXTURE).is_none() { + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "worker_log::tests::available_pipe_reads_distinguish_pending_data_and_eof", + ]) + .env(FIXTURE, "1") + .spawn() + .unwrap(); + let started = std::time::Instant::now(); + loop { + if let Some(status) = child.try_wait().unwrap() { + assert!(status.success()); + return; + } + if started.elapsed() >= std::time::Duration::from_secs(10) { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("reading an empty pipe must not block"); + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + } + } + let (mut reader, mut writer) = io::pipe().unwrap(); + #[cfg(unix)] + { + prepare_reader(&reader).unwrap(); + prepare_reader(&reader).unwrap(); + // SAFETY: reader owns a live descriptor and F_GETFL takes no pointers. + let flags = unsafe { libc::fcntl(reader.as_raw_fd(), libc::F_GETFL) }; + assert_ne!(flags & libc::O_NONBLOCK, 0); + } + let mut bytes = [0; 4]; + assert_eq!( + read_available(&mut reader, &mut bytes).unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + writer.write_all(b"raw").unwrap(); + assert_eq!(read_available(&mut reader, &mut bytes).unwrap(), 3); + assert_eq!(&bytes[..3], b"raw"); + drop(writer); + assert_eq!(read_available(&mut reader, &mut bytes).unwrap(), 0); + } + + #[cfg(unix)] + #[test] + fn nonblocking_setup_preserves_flags_and_propagates_each_syscall_failure() { + for flags in [0, libc::O_NONBLOCK, libc::O_APPEND | libc::O_NONBLOCK] { + let mut calls = 0; + prepare_reader_with(|operation, value| { + calls += 1; + match calls { + 1 => { + assert_eq!((operation, value), (libc::F_GETFL, 0)); + flags + } + 2 => { + assert_eq!((operation, value), (libc::F_SETFL, flags | libc::O_NONBLOCK)); + 0 + } + _ => panic!("only flag lookup and update are needed"), + } + }) + .unwrap(); + assert_eq!(calls, 2); + } + for failing_call in [1, 2] { + let mut calls = 0; + assert!( + prepare_reader_with(|_, _| { + calls += 1; + assert!(calls <= failing_call, "setup must stop at the failed syscall"); + if calls == failing_call { -1 } else { 0 } + }) + .is_err() + ); + assert_eq!(calls, failing_call); + } + } +} diff --git a/crates/metabench/tests/fixtures/artifacts/criterion/duplicate/a/new/estimates.json b/crates/metabench/tests/fixtures/artifacts/criterion/duplicate/a/new/estimates.json index d18920544..abff6889d 100644 --- a/crates/metabench/tests/fixtures/artifacts/criterion/duplicate/a/new/estimates.json +++ b/crates/metabench/tests/fixtures/artifacts/criterion/duplicate/a/new/estimates.json @@ -1 +1 @@ -{"median":{"confidence_interval":{"lower_bound":10.0,"upper_bound":30.0},"point_estimate":20.0}} +{"mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":10.0,"upper_bound":30.0},"point_estimate":20.0}} diff --git a/crates/metabench/tests/fixtures/artifacts/criterion/duplicate/b/new/estimates.json b/crates/metabench/tests/fixtures/artifacts/criterion/duplicate/b/new/estimates.json index 280fd2b0c..63f25a08e 100644 --- a/crates/metabench/tests/fixtures/artifacts/criterion/duplicate/b/new/estimates.json +++ b/crates/metabench/tests/fixtures/artifacts/criterion/duplicate/b/new/estimates.json @@ -1 +1 @@ -{"median":{"confidence_interval":{"lower_bound":11.0,"upper_bound":31.0},"point_estimate":21.0}} +{"mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":11.0,"upper_bound":31.0},"point_estimate":21.0}} diff --git a/crates/metabench/tests/fixtures/artifacts/criterion/invalid_numeric/new/estimates.json b/crates/metabench/tests/fixtures/artifacts/criterion/invalid_numeric/new/estimates.json index f731aa324..7ada0d583 100644 --- a/crates/metabench/tests/fixtures/artifacts/criterion/invalid_numeric/new/estimates.json +++ b/crates/metabench/tests/fixtures/artifacts/criterion/invalid_numeric/new/estimates.json @@ -1 +1 @@ -{"median":{"confidence_interval":{"lower_bound":1.0,"upper_bound":3.0},"point_estimate":-1.0}} +{"mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":1.0,"upper_bound":3.0},"point_estimate":-1.0}} diff --git a/crates/metabench/tests/fixtures/artifacts/criterion/minimal/new/estimates.json b/crates/metabench/tests/fixtures/artifacts/criterion/minimal/new/estimates.json index d18920544..abff6889d 100644 --- a/crates/metabench/tests/fixtures/artifacts/criterion/minimal/new/estimates.json +++ b/crates/metabench/tests/fixtures/artifacts/criterion/minimal/new/estimates.json @@ -1 +1 @@ -{"median":{"confidence_interval":{"lower_bound":10.0,"upper_bound":30.0},"point_estimate":20.0}} +{"mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":10.0,"upper_bound":30.0},"point_estimate":20.0}} diff --git a/crates/metabench/tests/fixtures/artifacts/criterion/missing_function/new/estimates.json b/crates/metabench/tests/fixtures/artifacts/criterion/missing_function/new/estimates.json index 5c4142408..2a9c7672b 100644 --- a/crates/metabench/tests/fixtures/artifacts/criterion/missing_function/new/estimates.json +++ b/crates/metabench/tests/fixtures/artifacts/criterion/missing_function/new/estimates.json @@ -1 +1 @@ -{"median":{"confidence_interval":{"lower_bound":1.0,"upper_bound":3.0},"point_estimate":2.0}} +{"mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":1.0,"upper_bound":3.0},"point_estimate":2.0}} diff --git a/crates/metabench/tests/fixtures/artifacts/criterion/unicode/new/estimates.json b/crates/metabench/tests/fixtures/artifacts/criterion/unicode/new/estimates.json index 5c4142408..2a9c7672b 100644 --- a/crates/metabench/tests/fixtures/artifacts/criterion/unicode/new/estimates.json +++ b/crates/metabench/tests/fixtures/artifacts/criterion/unicode/new/estimates.json @@ -1 +1 @@ -{"median":{"confidence_interval":{"lower_bound":1.0,"upper_bound":3.0},"point_estimate":2.0}} +{"mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":1.0,"upper_bound":3.0},"point_estimate":2.0}} diff --git a/crates/metabench/tests/fixtures/artifacts/traversal/cycle/new/estimates.json b/crates/metabench/tests/fixtures/artifacts/traversal/cycle/new/estimates.json index 5c4142408..2a9c7672b 100644 --- a/crates/metabench/tests/fixtures/artifacts/traversal/cycle/new/estimates.json +++ b/crates/metabench/tests/fixtures/artifacts/traversal/cycle/new/estimates.json @@ -1 +1 @@ -{"median":{"confidence_interval":{"lower_bound":1.0,"upper_bound":3.0},"point_estimate":2.0}} +{"mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":1.0,"upper_bound":3.0},"point_estimate":2.0}} diff --git a/crates/metabench/tests/probe_protocol.rs b/crates/metabench/tests/probe_protocol.rs new file mode 100644 index 000000000..7975d9fc5 --- /dev/null +++ b/crates/metabench/tests/probe_protocol.rs @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Authenticated generated-worker rejection tests for the indexed probe protocol. + +#![expect(clippy::unwrap_used, reason = "test failures provide sufficient context")] + +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn fixture() -> PathBuf { + let output = Command::new(std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into())) + .args([ + "build", + "--quiet", + "--jobs", + "2", + "--package", + "metabench", + "--example", + "native_probe_contracts", + "--message-format=json", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + String::from_utf8(output.stdout) + .unwrap() + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .find_map(|message| { + (message["target"]["name"] == "native_probe_contracts") + .then(|| message["executable"].as_str().map(PathBuf::from)) + .flatten() + }) + .unwrap() +} + +fn assert_rejected(executable: &Path, directory: &Path, arguments: &[String], expected: &str) { + let token = tempfile::Builder::new().prefix("metabench-worker-").tempdir_in(directory).unwrap(); + let events = directory.join("events"); + let _ = std::fs::remove_file(&events); + let mut command = Command::new(executable); + command + .args(arguments) + .env("TMPDIR", directory) + .env("TMP", directory) + .env("TEMP", directory) + .env("METABENCH_INTERNAL_WORKER_MODE", "allocations") + .env("METABENCH_INTERNAL_WORKER_TOKEN", token.path()) + .env("METABENCH_TEST_LIFECYCLE", &events); + for name in [ + "BENCH_ENGINE", + "METABENCH_INTERNAL_PROBE_DISCOVERY", + "METABENCH_INTERNAL_PERF_CONTROL", + "METABENCH_INTERNAL_PERF_ACK", + "METABENCH_INTERNAL_VTUNE_RESULT_DIR", + "METABENCH_INTERNAL_ALLOCATION_IDENTITY", + "METABENCH_INTERNAL_ALLOCATION_ARTIFACT", + "CARGO_CRITERION_PORT", + ] { + command.env_remove(name); + } + let output = command.output().unwrap(); + assert_eq!(output.status.code(), Some(2), "{arguments:?}: {output:?}"); + assert!(!token.path().exists(), "worker must consume its authentication token"); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert_eq!( + stderr, + format!("metabench: native benchmark probe failed: {expected}\n"), + "{arguments:?}" + ); + assert!(!stderr.contains("panicked"), "{arguments:?}: {stderr}"); + let events = std::fs::read_to_string(events).unwrap_or_default(); + assert!( + !events.contains("case setup") && !events.contains("case teardown"), + "{arguments:?}: {events}" + ); +} + +#[test] +fn authenticated_workers_reject_malformed_arguments_and_table_indices() { + let executable = fixture(); + let directory = tempfile::Builder::new() + .prefix("metabench-protocol-") + .tempdir_in(Path::new(env!("CARGO_MANIFEST_DIR"))) + .unwrap(); + let base = ["--metabench-probe", "0", "0", "0", "-"].map(String::from); + let protocol_error = "expected private native probe worker arguments"; + for count in 0..5 { + assert_rejected(&executable, directory.path(), &base[..count], protocol_error); + } + let mut extra = base.to_vec(); + extra.push("extra".into()); + assert_rejected(&executable, directory.path(), &extra, protocol_error); + for verb in ["--unknown", "--metabench-probe-list", "metabench-probe"] { + let mut arguments = base.clone(); + arguments[0] = verb.into(); + assert_rejected(&executable, directory.path(), &arguments, protocol_error); + } + for field in 1..5 { + for invalid in ["not-a-number", "-1", "99999999999999999999999999999999999999999"] { + let mut arguments = base.clone(); + arguments[field] = invalid.into(); + let expected = invalid.parse::().unwrap_err().to_string(); + assert_rejected(&executable, directory.path(), &arguments, &expected); + } + } + for (field, expected) in [ + (1, "probe group index is out of bounds"), + (2, "probe benchmark index is out of bounds"), + (3, "probe case index is out of bounds"), + ] { + for index in ["1".to_owned(), usize::MAX.to_string()] { + let mut arguments = base.clone(); + // The fixture has two native cases, unlike its single group and benchmark. + arguments[field] = if field == 3 && index == "1" { "2".into() } else { index }; + assert_rejected(&executable, directory.path(), &arguments, expected); + } + } +} diff --git a/crates/metabench/tests/spawned_benchmark.rs b/crates/metabench/tests/spawned_benchmark.rs index 42c7f3153..da12ce668 100644 --- a/crates/metabench/tests/spawned_benchmark.rs +++ b/crates/metabench/tests/spawned_benchmark.rs @@ -5,17 +5,46 @@ #![expect(clippy::unwrap_used, reason = "test failures provide sufficient context")] +#[cfg(target_os = "linux")] +use std::io; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt as _; +use std::panic; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; use std::sync::{Mutex, OnceLock}; static CARGO_SUBPROCESS: Mutex<()> = Mutex::new(()); +// The lock serializes Cargo, not mutable state; release it before resuming a test failure. +fn with_cargo_lock(body: impl FnOnce() -> T) -> T { + let guard = CARGO_SUBPROCESS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let result = panic::catch_unwind(panic::AssertUnwindSafe(body)); + drop(guard); + match result { + Ok(value) => value, + Err(payload) => panic::resume_unwind(payload), + } +} + +#[test] +fn serialized_failure_preserves_the_payload_without_new_lock_poison() { + let failure = panic::catch_unwind(|| { + with_cargo_lock(|| panic::panic_any(("serialized failure", CARGO_SUBPROCESS.is_poisoned()))); + }) + .unwrap_err(); + let (message, previously_poisoned) = *failure.downcast::<(&'static str, bool)>().unwrap(); + assert_eq!(message, "serialized failure"); + with_cargo_lock(|| assert_eq!(CARGO_SUBPROCESS.is_poisoned(), previously_poisoned)); +} + fn cargo_target() -> &'static Path { static TARGET: OnceLock = OnceLock::new(); TARGET - .get_or_init(|| Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/metabench-spawned-benchmark")) + .get_or_init(|| { + std::env::var_os("CARGO_TARGET_DIR").map_or_else(|| Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target"), PathBuf::from) + }) .as_path() } @@ -25,9 +54,12 @@ fn cargo() -> Command { "BENCH_ENGINE", "METABENCH_INTERNAL_WORKER_MODE", "METABENCH_INTERNAL_WORKER_TOKEN", + "METABENCH_INTERNAL_PROBE_DISCOVERY", "METABENCH_INTERNAL_PERF_CONTROL", "METABENCH_INTERNAL_PERF_ACK", "METABENCH_INTERNAL_VTUNE_RESULT_DIR", + "METABENCH_INTERNAL_ALLOCATION_ARTIFACT", + "METABENCH_INTERNAL_ALLOCATION_IDENTITY", "CRITERION_HOME", "CARGO_CRITERION_PORT", "GUNGRAUN_HOME", @@ -50,32 +82,37 @@ fn cargo() -> Command { } command .env("CARGO_TARGET_DIR", cargo_target()) + .env("CARGO_BUILD_JOBS", "2") .env("CARGO_TERM_COLOR", "never") .env("NO_COLOR", "1"); command } fn run(arguments: &[&str]) -> Output { - let _guard = CARGO_SUBPROCESS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - cargo() - .args([ - "run", - "--quiet", - "-p", - "metabench", - "--profile", - "bench", - "--example", - "metabench_basic", - "--", - ]) - .args(arguments) - .output() - .unwrap() + with_cargo_lock(|| { + cargo() + .args([ + "run", + "--quiet", + "-p", + "metabench", + "--profile", + "bench", + "--example", + "metabench_basic", + "--", + ]) + .args(arguments) + .output() + .unwrap() + }) } fn run_target(target: &str, arguments: &[&std::ffi::OsStr]) -> Output { - let _guard = CARGO_SUBPROCESS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + with_cargo_lock(|| run_target_unlocked(target, arguments)) +} + +fn run_target_unlocked(target: &str, arguments: &[&std::ffi::OsStr]) -> Output { let mut command = cargo(); command.args(["run", "--quiet", "-p", "metabench", "--profile", "bench", "--example", target, "--"]); command.args(arguments); @@ -95,17 +132,77 @@ fn successful_stdout(arguments: &[&str]) -> String { fn gungraun_runner_available() -> bool { let runner = std::env::var_os("GUNGRAUN_RUNNER").unwrap_or_else(|| "gungraun-runner".into()); - Command::new(runner) - .arg("--version") - .output() - .is_ok_and(|output| output.status.success()) + let version = Command::new(runner).arg("--version").output(); + let available = version.as_ref().is_ok_and(|output| output.status.success()); + if std::env::var_os("METABENCH_REQUIRE_NATIVE_CONTRACTS").is_some() { + assert!(available, "required native contracts need the matching gungraun-runner on PATH"); + assert_eq!( + String::from_utf8_lossy(&version.unwrap().stdout).trim(), + "gungraun-runner 0.19.4", + "required native contracts must match the workspace Gungraun version" + ); + assert!( + Command::new("valgrind") + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()), + "required native contracts need Valgrind on PATH" + ); + } + available +} + +#[test] +fn fresh_consumer_resolves_and_compiles_without_the_workspace_lockfile() { + with_cargo_lock(|| { + let directory = tempfile::tempdir().unwrap(); + let dependency = serde_json::to_string(env!("CARGO_MANIFEST_DIR")).unwrap(); + std::fs::write( + directory.path().join("Cargo.toml"), + format!( + r#"[package] +name = "metabench-fresh-consumer" +version = "0.0.0" +edition = "2024" + +[workspace] + +[dependencies] +metabench = {{ path = {dependency} }} +criterion = {{ version = "0.8.2", default-features = false }} +gungraun = "=0.19.4" +"# + ), + ) + .unwrap(); + std::fs::create_dir(directory.path().join("src")).unwrap(); + std::fs::write( + directory.path().join("src/main.rs"), + r#"#[metabench::benchmark(CASE, "consumer", "case")] +fn workload() -> u64 { std::hint::black_box(42) } +fn benchmarks(c: &mut criterion::Criterion) { + c.bench_function("consumer/case", |b| b.iter(|| workload())); +} +metabench::main!(criterion = benchmarks, benchmarks = [CASE],); +"#, + ) + .unwrap(); + let output = cargo() + .current_dir(directory.path()) + .args(["run", "--quiet", "--offline", "--", "--criterion", "--list"]) + .output() + .unwrap(); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + assert!(String::from_utf8_lossy(&output.stdout).contains("consumer/case: benchmark")); + let lockfile = std::fs::read_to_string(directory.path().join("Cargo.lock")).unwrap(); + assert!(!lockfile.contains("name = \"gungraun-summary\"")); + assert_eq!(lockfile.matches("name = \"gungraun-runner\"").count(), 1); + }); } #[cfg(target_os = "linux")] #[test] fn perf_measures_exact_workload_and_writes_metrics() { - use std::os::unix::fs::PermissionsExt as _; - let _guard = CARGO_SUBPROCESS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); let directory = tempfile::tempdir().unwrap(); let perf = directory.path().join("perf"); @@ -236,7 +333,6 @@ fn fake_vtune_directory() -> &'static Path { std::fs::copy(&built, &vtune_path).unwrap(); #[cfg(unix)] { - use std::os::unix::fs::PermissionsExt as _; std::fs::set_permissions(&vtune_path, std::fs::Permissions::from_mode(0o755)).unwrap(); } directory @@ -524,12 +620,747 @@ fn gungraun_listing_is_forwarded() { let stdout = successful_stdout(&["--gungraun", "--list"]); - assert!(stdout.lines().any(|line| { - line.contains("__metabench_group_d23a5d29e30556e2f3f66fae89e4544b::__metabench_benchmark_ecb5d0c1194ff78d70d02d708fc2a852") - })); + assert!( + stdout + .lines() + .any(|line| { line.contains("_checksums") && line.contains("_rolling") }) + ); assert!(!stdout.contains("Running gungraun benchmarks")); } +#[test] +fn setup_calls_do_not_steal_filtered_allocation_measurements() { + let directory = tempfile::tempdir().unwrap(); + let json = directory.path().join("report.json"); + let output = run_target( + "measurement_boundaries", + &[ + "--allocations".as_ref(), + "--criterion-arg=boundaries/allocate".as_ref(), + "--export-json".as_ref(), + json.as_os_str(), + ], + ); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + let report: serde_json::Value = serde_json::from_slice(&std::fs::read(&json).unwrap()).unwrap(); + let entries = report["entries"].as_array().unwrap(); + assert_eq!(entries.len(), 2); + for (entry, size) in entries.iter().zip([128, 32]) { + let metrics = &entry["results"]["alloc_tracker"]["metrics"]; + assert_eq!(metrics["Allocations"]["value"], 1); + assert_eq!(metrics["Allocated bytes"]["value"], size); + } + assert!(!String::from_utf8_lossy(&output.stdout).contains("Warming up")); +} + +#[test] +fn criterion_worker_bypasses_allocation_tracking() { + with_cargo_lock(|| { + let output = cargo() + .env("METABENCH_TEST_REQUIRE_UNTRACKED", "1") + .args([ + "run", + "--quiet", + "-p", + "metabench", + "--profile", + "bench", + "--example", + "measurement_boundaries", + "--", + "--criterion", + "--list", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + assert!(String::from_utf8_lossy(&output.stdout).contains("boundaries/allocate/small")); + }); +} + +#[test] +fn native_probe_cases_reuse_iterators_conditionals_and_lifecycle_hooks() { + with_cargo_lock(|| { + let directory = tempfile::tempdir().unwrap(); + let json = directory.path().join("report.json"); + let lifecycle = directory.path().join("lifecycle.txt"); + let output = cargo() + .env("METABENCH_TEST_LIFECYCLE", &lifecycle) + .args([ + "run", + "--quiet", + "-p", + "metabench", + "--profile", + "bench", + "--example", + "native_probe_contracts", + "--", + "--allocations", + "--export-json", + ]) + .arg(&json) + .output() + .unwrap(); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + let report: serde_json::Value = serde_json::from_slice(&std::fs::read(&json).unwrap()).unwrap(); + let entries = report["entries"].as_array().unwrap(); + assert_eq!(entries.len(), 3); + for (entry, (case, size)) in entries.iter().zip([("conditional", 24), ("sizes_0", 8), ("sizes_1", 16)]) { + assert_eq!(entry["identity"], format!("contracts/iterator/{case}")); + assert_eq!(entry["results"]["alloc_tracker"]["metrics"]["Allocations"]["value"], 1); + assert_eq!(entry["results"]["alloc_tracker"]["metrics"]["Allocated bytes"]["value"], size); + } + let events = std::fs::read_to_string(&lifecycle).unwrap(); + assert_eq!( + events.lines().collect::>(), + [ + "suite setup", + "group setup", + "group teardown", + "suite teardown", + "suite setup", + "group setup", + "case setup", + "case teardown", + "group teardown", + "suite teardown", + "suite setup", + "group setup", + "case setup", + "case teardown", + "group teardown", + "suite teardown", + "suite setup", + "group setup", + "case setup", + "case teardown", + "group teardown", + "suite teardown", + ] + ); + }); +} + +#[test] +fn native_vtune_orders_collection_and_teardown_on_success_and_unwind() { + if !require_fake_vtune_fixture_or_skip() { + return; + } + let path = path_with_fake_vtune(); + with_cargo_lock(|| { + let directory = tempfile::tempdir().unwrap(); + for panic_workload in [false, true] { + let events = directory.path().join("events.log"); + let json = directory.path().join("report.json"); + let markdown = directory.path().join("report.md"); + std::fs::write(&events, b"").unwrap(); + std::fs::write(&json, b"complete existing JSON report").unwrap(); + std::fs::write(&markdown, b"complete existing Markdown report").unwrap(); + let mut command = cargo(); + command + .env("PATH", &path) + .env("METABENCH_TEST_LIFECYCLE", &events) + .env("VTUNE_TEST_COMMAND_LOG", &events) + .env("METABENCH_TEST_ORDERING", "1") + .args([ + "run", + "--quiet", + "-p", + "metabench", + "--profile", + "bench", + "--example", + "native_probe_contracts", + "--", + "--vtune", + "--criterion-arg=contracts/iterator/conditional", + "--criterion-arg=--exact", + "--no-baseline", + "--export-json", + ]) + .arg(&json) + .arg("--export-md") + .arg(&markdown); + if panic_workload { + command.env("METABENCH_TEST_PANIC_WORKLOAD", "1"); + } + let output = command.output().unwrap(); + assert_eq!( + output.status.success(), + !panic_workload, + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let events = std::fs::read_to_string(&events).unwrap(); + let events: Vec<_> = events + .lines() + .map(|event| { + if event.starts_with("resume ") { + "resume" + } else if event.starts_with("pause ") { + "pause" + } else { + event + } + }) + .collect(); + let mut expected = vec![ + "suite setup", + "group setup", + "group teardown", + "suite teardown", + "suite setup", + "group setup", + "case setup", + "resume", + "workload entry", + ]; + if panic_workload { + expected.push("pause"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("native workload panic"), "{stderr}"); + assert!(stderr.contains("worker log:"), "{stderr}"); + assert_eq!(std::fs::read(&json).unwrap(), b"complete existing JSON report"); + assert_eq!(std::fs::read(&markdown).unwrap(), b"complete existing Markdown report"); + } else { + expected.extend(["workload exit", "pause", "case teardown"]); + } + expected.extend(["group teardown", "suite teardown"]); + assert_eq!(events, expected); + if !panic_workload { + let report: serde_json::Value = serde_json::from_slice(&std::fs::read(&json).unwrap()).unwrap(); + assert_eq!(report["entries"].as_array().unwrap().len(), 1); + assert_eq!( + report["entries"][0]["results"]["vtune"]["metrics"]["INST_RETIRED.ANY"]["value"], + 1234.0 + ); + } + } + }); +} + +#[test] +fn native_allocations_include_prestarted_threads_only_inside_the_window() { + let directory = tempfile::tempdir().unwrap(); + let json = directory.path().join("report.json"); + let output = run_target( + "threaded_allocations", + &[ + "--allocations".as_ref(), + "--no-baseline".as_ref(), + "--export-json".as_ref(), + json.as_os_str(), + ], + ); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + let report: serde_json::Value = serde_json::from_slice(&std::fs::read(&json).unwrap()).unwrap(); + assert_eq!(report["entries"].as_array().unwrap().len(), 1); + let metrics = &report["entries"][0]["results"]["alloc_tracker"]["metrics"]; + assert_eq!(metrics["Allocations"]["value"], 4); + assert_eq!(metrics["Allocated bytes"]["value"], 37 + 19 + 83 + 19); +} + +#[test] +fn native_workers_ignore_foreign_collector_environment() { + if !require_fake_vtune_fixture_or_skip() { + return; + } + let path = path_with_fake_vtune(); + with_cargo_lock(|| { + let directory = tempfile::tempdir().unwrap(); + let foreign_control = directory.path().join("foreign-control"); + let foreign_ack = directory.path().join("foreign-ack"); + let foreign_allocations = directory.path().join("foreign-allocations.json"); + std::fs::write(&foreign_control, b"untouched control").unwrap(); + std::fs::write(&foreign_ack, b"bad\n").unwrap(); + std::fs::write(&foreign_allocations, b"untouched allocation artifact").unwrap(); + for selector in ["--allocations", "--vtune", "combined"] { + let json = directory.path().join("report.json"); + let commands = directory.path().join("vtune-commands.log"); + std::fs::write(&commands, b"").unwrap(); + let mut command = cargo(); + command + .env("PATH", &path) + .env("VTUNE_TEST_COMMAND_LOG", &commands) + .env("METABENCH_INTERNAL_PERF_CONTROL", &foreign_control) + .env("METABENCH_INTERNAL_PERF_ACK", &foreign_ack) + .env("METABENCH_INTERNAL_VTUNE_RESULT_DIR", directory.path().join("foreign-vtune")) + .env("METABENCH_INTERNAL_ALLOCATION_ARTIFACT", &foreign_allocations) + .env("METABENCH_INTERNAL_ALLOCATION_IDENTITY", "foreign/identity") + .args([ + "run", + "--quiet", + "-p", + "metabench", + "--profile", + "bench", + "--example", + "native_probe_contracts", + "--", + "--criterion-arg=contracts/iterator/conditional", + "--criterion-arg=--exact", + "--no-baseline", + "--export-json", + ]) + .arg(&json); + if selector == "combined" { + command.args(["--allocations", "--vtune"]); + } else { + command.arg(selector); + } + if selector == "--vtune" { + command.env("METABENCH_TEST_REQUIRE_UNTRACKED", "1"); + } + let output = command.output().unwrap(); + assert!(output.status.success(), "{selector}: {}", String::from_utf8_lossy(&output.stderr)); + assert_eq!(std::fs::read(&foreign_control).unwrap(), b"untouched control"); + assert_eq!(std::fs::read(&foreign_allocations).unwrap(), b"untouched allocation artifact"); + let report: serde_json::Value = serde_json::from_slice(&std::fs::read(&json).unwrap()).unwrap(); + assert_eq!(report["entries"].as_array().unwrap().len(), 1); + if selector != "--vtune" { + let metrics = &report["entries"][0]["results"]["alloc_tracker"]["metrics"]; + assert_eq!(metrics["Allocations"]["value"], 1); + assert_eq!(metrics["Allocated bytes"]["value"], 24); + } + if selector == "--allocations" { + assert_eq!(std::fs::read(&commands).unwrap(), b""); + } else { + assert_eq!(std::fs::read_to_string(&commands).unwrap().lines().count(), 2); + assert_eq!( + report["entries"][0]["results"]["vtune"]["metrics"]["INST_RETIRED.ANY"]["value"], + 1234.0 + ); + } + } + }); +} + +#[cfg(target_os = "linux")] +const FAKE_PERF_CONTROL_SCRIPT: &str = r#"#!/bin/sh +output= +control= +while [ "$#" -gt 0 ]; do + case "$1" in + --output) output=$2; shift 2 ;; + --control) control=${2#fifo:}; shift 2 ;; + --) shift; break ;; + *) shift ;; + esac +done +control_fifo=${control%%,*} +ack_fifo=${control#*,} +case "$FAKE_PERF_FAILURE" in + connection-control) export METABENCH_INTERNAL_PERF_CONTROL="$output.missing" ;; + connection-ack) export METABENCH_INTERNAL_PERF_ACK="$output.missing" ;; + enable-closed|disable-closed) + # A private FIFO lets close produce EOF despite the launcher's original keepalive. + ack_fifo="$output.closed" + mkfifo -m 600 "$ack_fifo" || exit 92 + export METABENCH_INTERNAL_PERF_ACK="$ack_fifo" + ;; +esac +"$@" & +child=$! +if [ "$FAKE_PERF_FAILURE" != connection-control ]; then + exec 3<"$control_fifo" + if [ "$FAKE_PERF_FAILURE" != connection-ack ]; then + exec 4>"$ack_fifo" + IFS= read -r command <&3 + [ "$command" = enable ] || exit 90 + printf '%s\n' enable >> "$FAKE_PERF_EVENTS" + case "$FAKE_PERF_FAILURE" in + enable-malformed) printf 'bad\n' >&4 ;; + enable-closed) exec 4>&- ;; + *) + printf 'ack\n' >&4 + IFS= read -r command <&3 + [ "$command" = disable ] || exit 91 + printf '%s\n' disable >> "$FAKE_PERF_EVENTS" + case "$FAKE_PERF_FAILURE" in + disable-malformed) printf 'bad\n' >&4 ;; + disable-closed) exec 4>&- ;; + *) printf 'ack\n' >&4 ;; + esac + ;; + esac + fi +fi +wait "$child" +status=$? +printf '%s\n' \ + '{"counter-value":"1234","event":"instructions:u"}' \ + '{"counter-value":"567","event":"cycles:u"}' \ + '{"counter-value":"8","event":"branches:u"}' \ + '{"counter-value":"2","event":"branch-misses:u"}' \ + '{"counter-value":"6","event":"cache-references:u"}' \ + '{"counter-value":"1","event":"cache-misses:u"}' > "$output" +exit "$status" +"#; + +#[cfg(target_os = "linux")] +fn bounded_output(command: &Command) -> Output { + let mut bounded = Command::new("timeout"); + bounded + .args(["--kill-after=2s", "20s"]) + .arg(command.get_program()) + .args(command.get_args()); + for (name, value) in command.get_envs() { + if let Some(value) = value { + bounded.env(name, value); + } else { + bounded.env_remove(name); + } + } + bounded.output().unwrap() +} + +#[cfg(target_os = "linux")] +fn build_example(target: &str) -> PathBuf { + let build = cargo() + .args([ + "build", + "--quiet", + "-p", + "metabench", + "--profile", + "bench", + "--example", + target, + "--message-format=json", + ]) + .output() + .unwrap(); + assert!(build.status.success(), "{}", String::from_utf8_lossy(&build.stderr)); + String::from_utf8_lossy(&build.stdout) + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .find_map(|message| { + (message["reason"] == "compiler-artifact" && message["target"]["name"] == target) + .then(|| message["executable"].as_str().map(PathBuf::from)) + .flatten() + }) + .unwrap() +} + +#[cfg(target_os = "linux")] +fn example_command(executable: &Path) -> Command { + let mut command = Command::new(executable); + for (name, value) in cargo().get_envs() { + if let Some(value) = value { + command.env(name, value); + } else { + command.env_remove(name); + } + } + command +} + +#[cfg(target_os = "linux")] +#[test] +fn native_perf_control_failures_are_bounded_and_preserve_reports() { + with_cargo_lock(|| { + let executable = build_example("native_probe_contracts"); + let directory = tempfile::tempdir().unwrap(); + let perf = directory.path().join("perf"); + std::fs::write(&perf, FAKE_PERF_CONTROL_SCRIPT).unwrap(); + std::fs::set_permissions(&perf, std::fs::Permissions::from_mode(0o755)).unwrap(); + let path = std::env::join_paths( + std::iter::once(directory.path().to_owned()).chain(std::env::split_paths(&std::env::var_os("PATH").unwrap_or_default())), + ) + .unwrap(); + for failure in [ + "success", + "enable-malformed", + "enable-closed", + "disable-malformed", + "disable-closed", + "connection-control", + "connection-ack", + ] { + let events = directory.path().join("events.log"); + let json = directory.path().join("report.json"); + let markdown = directory.path().join("report.md"); + std::fs::write(&events, b"").unwrap(); + std::fs::write(&json, b"complete existing JSON report").unwrap(); + std::fs::write(&markdown, b"complete existing Markdown report").unwrap(); + let mut command = example_command(&executable); + command + .env("PATH", &path) + .env("FAKE_PERF_FAILURE", failure) + .env("FAKE_PERF_EVENTS", &events) + .args([ + "--perf", + "--criterion-arg=contracts/iterator/conditional", + "--criterion-arg=--exact", + "--no-baseline", + "--timeout=5s", + "--export-json", + ]) + .arg(&json) + .arg("--export-md") + .arg(&markdown); + let output = bounded_output(&command); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_ne!(output.status.code(), Some(124), "{failure}: outer timeout expired: {stderr}"); + assert_ne!( + output.status.code(), + Some(137), + "{failure}: outer timeout killed the process: {stderr}" + ); + let commands = std::fs::read_to_string(&events).unwrap(); + let expected_commands = if failure.starts_with("connection-") { + "" + } else if failure.starts_with("enable-") { + "enable\n" + } else { + "enable\ndisable\n" + }; + assert_eq!(commands, expected_commands, "{failure}: {stderr}"); + if failure == "success" { + assert!(output.status.success(), "{failure}: {stderr}"); + let report: serde_json::Value = serde_json::from_slice(&std::fs::read(&json).unwrap()).unwrap(); + assert_eq!(report["entries"].as_array().unwrap().len(), 1); + assert_eq!(report["entries"][0]["results"]["perf"]["metrics"]["instructions"]["value"], 1234.0); + } else { + assert!(!output.status.success(), "{failure}: control failure incorrectly succeeded"); + assert!(stderr.contains("failed to control Linux perf counters"), "{failure}: {stderr}"); + assert!(stderr.contains("worker log:"), "{failure}: {stderr}"); + let diagnostic = if failure.ends_with("malformed") { + "unexpected perf acknowledgement" + } else if failure.ends_with("closed") { + "failed to fill whole buffer" + } else { + "No such file or directory" + }; + assert!(stderr.contains(diagnostic), "{failure}: {stderr}"); + assert_eq!(std::fs::read(&json).unwrap(), b"complete existing JSON report"); + assert_eq!(std::fs::read(&markdown).unwrap(), b"complete existing Markdown report"); + } + } + }); +} + +#[test] +fn probe_listing_uses_native_cases_without_criterion_registration() { + let output = run_target("native_probe_contracts", &["--allocations".as_ref(), "--list".as_ref()]); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("contracts/iterator/sizes_0: benchmark"), "{stdout}"); + assert!(stdout.contains("contracts/iterator/sizes_1: benchmark"), "{stdout}"); + assert!(stdout.contains("contracts/iterator/conditional: benchmark"), "{stdout}"); + assert!(!stdout.contains("disabled"), "{stdout}"); +} + +#[test] +fn smoke_and_empty_measurements_preserve_existing_reports() { + let directory = tempfile::tempdir().unwrap(); + let json = directory.path().join("report.json"); + let markdown = directory.path().join("report.md"); + std::fs::write(&json, b"existing JSON").unwrap(); + std::fs::write(&markdown, b"existing Markdown").unwrap(); + for selector in ["--criterion", "--allocations"] { + let output = run_target( + "measurement_boundaries", + &[ + selector.as_ref(), + "--test".as_ref(), + "--export-json".as_ref(), + json.as_os_str(), + "--export-md".as_ref(), + markdown.as_os_str(), + ], + ); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + assert_eq!(std::fs::read(&json).unwrap(), b"existing JSON"); + assert_eq!(std::fs::read(&markdown).unwrap(), b"existing Markdown"); + assert!(!String::from_utf8_lossy(&output.stdout).contains("Warming up")); + } + for selector in ["--criterion", "--allocations"] { + let output = run_target( + "measurement_boundaries", + &[ + selector.as_ref(), + "--criterion-arg=no-such-case".as_ref(), + "--export-json".as_ref(), + json.as_os_str(), + "--export-md".as_ref(), + markdown.as_os_str(), + ], + ); + assert!(!output.status.success()); + assert_eq!(std::fs::read(&json).unwrap(), b"existing JSON"); + assert_eq!(std::fs::read(&markdown).unwrap(), b"existing Markdown"); + } +} + +#[test] +fn failed_native_setup_retains_worker_diagnostics() { + with_cargo_lock(|| { + let output = cargo() + .env("METABENCH_TEST_FAIL_SETUP", "1") + .args([ + "run", + "--quiet", + "-p", + "metabench", + "--profile", + "bench", + "--example", + "measurement_boundaries", + "--", + "--allocations", + "--criterion-arg=small", + "--no-output", + ]) + .output() + .unwrap(); + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("native setup failed"), "{stderr}"); + assert!(stderr.contains("worker log:"), "{stderr}"); + }); +} + +#[test] +fn exact_filter_and_const_workload_support_allocation_probes() { + let directory = tempfile::tempdir().unwrap(); + let json = directory.path().join("report.json"); + let output = run_target( + "measurement_boundaries", + &[ + "--allocations".as_ref(), + "--criterion-arg=boundaries/const".as_ref(), + "--criterion-arg=--exact".as_ref(), + "--criterion-arg=--profile-time=60".as_ref(), + "--criterion-arg=--load-baseline=missing".as_ref(), + "--export-json".as_ref(), + json.as_os_str(), + ], + ); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + let report: serde_json::Value = serde_json::from_slice(&std::fs::read(&json).unwrap()).unwrap(); + assert_eq!(report["entries"].as_array().unwrap().len(), 1); + assert_eq!(report["entries"][0]["identity"], "boundaries/const"); + assert_eq!( + report["entries"][0]["results"]["alloc_tracker"]["metrics"]["Allocations"]["value"], + 0 + ); +} + +#[test] +fn timed_out_worker_reports_captured_diagnostics_and_preserves_baseline() { + with_cargo_lock(|| { + let directory = tempfile::tempdir().unwrap(); + let json = directory.path().join("report.json"); + std::fs::write(&json, b"existing baseline").unwrap(); + let output = cargo() + .env("METABENCH_TEST_TIMEOUT", "1") + .args([ + "run", + "--quiet", + "-p", + "metabench", + "--profile", + "bench", + "--example", + "measurement_boundaries", + "--", + "--allocations", + "--criterion-arg=small", + "--timeout", + "2s", + "--export-json", + ]) + .arg(&json) + .output() + .unwrap(); + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("selected workload is waiting"), "{stderr}"); + assert!(stderr.contains("worker log:"), "{stderr}"); + assert!(stderr.contains("exceeded the 2s timeout"), "{stderr}"); + assert_eq!(std::fs::read(&json).unwrap(), b"existing baseline"); + }); +} + +#[cfg(target_os = "linux")] +struct ReportSnapshot { + path: PathBuf, + original: Option>, +} + +#[cfg(target_os = "linux")] +impl ReportSnapshot { + fn new(path: PathBuf) -> Self { + Self { + original: Self::read(&path), + path, + } + } + + #[expect( + clippy::panic, + reason = "unexpected snapshot I/O must fail the test rather than look like an absent report" + )] + fn read(path: &Path) -> Option> { + match std::fs::read(path) { + Ok(bytes) => Some(bytes), + Err(error) if error.kind() == io::ErrorKind::NotFound => None, + Err(error) => panic!("failed to snapshot {}: {error}", path.display()), + } + } + + fn assert_unchanged(&self) { + assert_eq!(Self::read(&self.path), self.original, "{}", self.path.display()); + } +} + +#[cfg(target_os = "linux")] +impl Drop for ReportSnapshot { + fn drop(&mut self) { + let result = if let Some(bytes) = &self.original { + std::fs::write(&self.path, bytes) + } else { + match std::fs::remove_file(&self.path) { + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + result => result, + } + }; + if let Err(error) = result { + eprintln!("failed to restore smoke-test report {}: {error}", self.path.display()); + } + } +} + +#[cfg(target_os = "linux")] +#[test] +fn default_engine_smoke_checks_do_not_publish_measurements() { + if !gungraun_runner_available() { + return; + } + let directory = tempfile::tempdir().unwrap(); + let json = directory.path().join("report.json"); + std::fs::write(&json, b"existing baseline").unwrap(); + let output = run_target( + "measurement_boundaries", + &["--test".as_ref(), "--export-json".as_ref(), json.as_os_str()], + ); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + assert_eq!(std::fs::read(&json).unwrap(), b"existing baseline"); + assert!(!String::from_utf8_lossy(&output.stdout).contains("Warming up")); + with_cargo_lock(|| { + let root = cargo_target().join("metabench/measurement_boundaries"); + let reports = ["report.json", "report.md"].map(|name| ReportSnapshot::new(root.join(name))); + let output = run_target_unlocked("measurement_boundaries", &[]); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + assert!(!String::from_utf8_lossy(&output.stdout).contains("Warming up")); + for report in &reports { + report.assert_unchanged(); + } + }); +} + #[test] fn criterion_and_allocations_write_parameterized_report() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/metabench_macros/src/lib.rs b/crates/metabench_macros/src/lib.rs index eeeb92d88..a11c66a32 100644 --- a/crates/metabench_macros/src/lib.rs +++ b/crates/metabench_macros/src/lib.rs @@ -180,11 +180,12 @@ use proc_macro::TokenStream; /// - functions with variable argument lists. /// /// Metabench applies `#[inline(never)]` to the benchmark function so every -/// engine measures a stable function boundary. Non-const functions are -/// measured by allocation tracking, Linux `perf`, and Intel `VTune` in -/// addition to Criterion and Gungraun. Benchmark functions declared with -/// `const fn` cannot contain that runtime instrumentation and therefore -/// produce only Criterion and Gungraun measurements. +/// engine measures a stable function boundary, without injecting runtime +/// collector checks into it. Criterion calls the function directly. Separate +/// private adapters collect allocations, Linux `perf`, and Intel `VTune` using +/// the existing native case arguments and setup/teardown hooks. They do not +/// execute Criterion callbacks or infer their inputs. This also supports +/// benchmark functions declared with `const fn`. #[proc_macro_attribute] #[cfg_attr(coverage_nightly, coverage(off))] #[cfg_attr(test, mutants::skip)] diff --git a/crates/metabench_macros/tests/compile.rs b/crates/metabench_macros/tests/compile.rs index 03991d3ff..4eee7b6ed 100644 --- a/crates/metabench_macros/tests/compile.rs +++ b/crates/metabench_macros/tests/compile.rs @@ -8,7 +8,6 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; -use std::sync::OnceLock; use tempfile::TempDir; @@ -48,11 +47,9 @@ const FAIL_CASES: &[(&str, &str)] = &[ ("target_unsupported_option", "unsupported group option"), ]; -fn compile_case(name: &str) -> Output { - let fixture = compile_fixture(); - +fn compile_case(name: &str, fixture: &CompileFixture) -> Output { Command::new(env!("CARGO")) - .args(["check", "--offline", "--quiet", "--manifest-path"]) + .args(["check", "--offline", "--quiet", "--jobs", "2", "--manifest-path"]) .arg(&fixture.manifest) .args(["--bin", name]) .env("CARGO_TARGET_DIR", &fixture.target) @@ -66,24 +63,23 @@ struct CompileFixture { target: PathBuf, } -fn compile_fixture() -> &'static CompileFixture { - static FIXTURE: OnceLock = OnceLock::new(); - - FIXTURE.get_or_init(|| { - let crate_root = Path::new(env!("CARGO_MANIFEST_DIR")); - let workspace_root = crate_root.parent().and_then(Path::parent).unwrap(); - let workspace_root = workspace_root.to_string_lossy().replace('\\', "\\\\"); - let directory = tempfile::tempdir().unwrap(); - let manifest = directory.path().join("Cargo.toml"); - let contents = include_str!("compile_cases.toml").replace("../../../crates", &format!("{workspace_root}/crates")); - fs::write(&manifest, contents).unwrap(); - let target = directory.path().join("target"); - CompileFixture { - _directory: directory, - manifest, - target, - } - }) +fn compile_fixture() -> CompileFixture { + let crate_root = Path::new(env!("CARGO_MANIFEST_DIR")); + let workspace_root = crate_root.parent().and_then(Path::parent).unwrap(); + let target = std::env::var_os("CARGO_TARGET_DIR").map_or_else(|| workspace_root.join("target"), PathBuf::from); + let workspace_root = workspace_root.to_string_lossy().replace('\\', "\\\\"); + let directory = tempfile::Builder::new() + .prefix("metabench-compile-") + .tempdir_in(crate_root) + .unwrap(); + let manifest = directory.path().join("Cargo.toml"); + let contents = include_str!("compile_cases.toml").replace("../../../crates", &format!("{workspace_root}/crates")); + fs::write(&manifest, contents).unwrap(); + CompileFixture { + _directory: directory, + manifest, + target, + } } fn display_output(output: &Output) -> String { @@ -98,8 +94,9 @@ fn display_output(output: &Output) -> String { #[test] #[cfg_attr(miri, ignore)] fn compile_pass_contracts() { + let fixture = compile_fixture(); for name in PASS_CASES { - let output = compile_case(name); + let output = compile_case(name, &fixture); assert!(output.status.success(), "{name} failed to compile:\n{}", display_output(&output)); } } @@ -107,8 +104,9 @@ fn compile_pass_contracts() { #[test] #[cfg_attr(miri, ignore)] fn compile_fail_contracts() { + let fixture = compile_fixture(); for (name, expected) in FAIL_CASES { - let output = compile_case(name); + let output = compile_case(name, &fixture); let stderr = String::from_utf8_lossy(&output.stderr); assert!(!output.status.success(), "{name} unexpectedly compiled"); assert!( diff --git a/crates/metabench_macros/tests/ui/pass/generated_name_collisions.rs b/crates/metabench_macros/tests/ui/pass/generated_name_collisions.rs index daef0b206..4f7f41157 100644 --- a/crates/metabench_macros/tests/ui/pass/generated_name_collisions.rs +++ b/crates/metabench_macros/tests/ui/pass/generated_name_collisions.rs @@ -31,3 +31,43 @@ mod contract { } fn main() {} + +mod collector_names { + use renamed_metabench::benchmark; + + #[benchmark(ALLOCATIONS, "collisions", "allocations")] + #[bench::one(1)] + fn allocations(value: u64) -> u64 { + value + 1 + } + + #[benchmark(PERF, "collisions", "perf")] + #[bench::one(1)] + fn perf(value: u64) -> u64 { + value + 2 + } + + #[benchmark(VTUNE, "collisions", "vtune")] + #[bench::one(1)] + fn vtune(value: u64) -> u64 { + value + 3 + } + + #[benchmark(OUTPUT, "collisions", "output")] + #[bench::one(1)] + fn output(value: u64) -> u64 { + value + 4 + } + + #[benchmark(ARGUMENT, "collisions", "argument")] + #[bench::one(1)] + fn __metabench_argument_0(value: u64) -> u64 { + value + 5 + } + + #[benchmark(DROP, "collisions", "drop")] + #[bench::one(1)] + fn drop(value: u64) -> u64 { + value + 6 + } +} diff --git a/crates/metabench_macros_impl/src/benchmark.rs b/crates/metabench_macros_impl/src/benchmark.rs index aadbc55cd..b834a593c 100644 --- a/crates/metabench_macros_impl/src/benchmark.rs +++ b/crates/metabench_macros_impl/src/benchmark.rs @@ -10,7 +10,7 @@ use syn::{ Signature, Token, parse_quote, parse2, }; -use crate::shared::{metabench_path, parse_ident_value, reject_if_too_complex, support_ident}; +use crate::shared::{metabench_path, native_ident, parse_ident_value, reject_if_too_complex, support_ident}; struct BenchmarkArguments { identity: Ident, @@ -311,25 +311,12 @@ fn expand_benchmark(arguments: BenchmarkArguments, mut function: ItemFn, metaben let callable_attributes = attributes.callable; let conditional_attributes = attributes.conditional; let case_attributes = attributes.cases; - let allocation_tracking = function.sig.constness.is_none(); - if allocation_tracking { - let body = function.block; - let allocation_measurement = format_ident!("_metabench_allocation_measurement", span = Span::mixed_site()); - let perf_measurement = format_ident!("_metabench_perf_measurement", span = Span::mixed_site()); - let vtune_measurement = format_ident!("_metabench_vtune_measurement", span = Span::mixed_site()); - function.block = parse_quote!({ - let #allocation_measurement = #metabench::__private::begin_allocation_measurement(); - let #perf_measurement = #metabench::__private::begin_perf_measurement(); - let #vtune_measurement = #metabench::__private::begin_vtune_measurement(); - #body - }); - } function.attrs.push(syn::parse_quote!(#[inline(never)])); let callable_name = &function.sig.ident; let logical_identity = identity.to_string(); - let adapter_name = support_ident("benchmark", &logical_identity); - let native_group_name = support_ident("group", &logical_identity); + let adapter_name = native_ident("benchmark", &logical_identity); + let native_group_name = native_ident("group", &logical_identity); let (adapter_signature, call_arguments) = adapter_signature(&function.sig, &adapter_name)?; let generic_arguments = function.sig.generics.params.iter().filter_map(|parameter| match parameter { GenericParam::Type(parameter) => Some(¶meter.ident), @@ -337,13 +324,16 @@ fn expand_benchmark(arguments: BenchmarkArguments, mut function: ItemFn, metaben }); let generic_arguments = generic_arguments.collect::>(); let turbofish = (!generic_arguments.is_empty()).then(|| quote!(::<#(#generic_arguments),*>)); - let call = quote!(#callable_name #turbofish (#(#call_arguments),*)); + // Gungraun nests each adapter in its benchmark and wrapper modules, beneath + // our support module. Resolve the original function outside all three. + let call = quote!(super::super::super::#callable_name #turbofish (#(#call_arguments),*)); let native_attribute = if native.is_empty() { quote!(#[#metabench::__private::gungraun::library_benchmark]) } else { quote!(#[#metabench::__private::gungraun::library_benchmark(#native)]) }; let adapter_module = support_ident("native", &logical_identity); + let probe_module = support_ident("probe", &logical_identity); let visibility = &function.vis; Ok(quote! { @@ -357,7 +347,7 @@ fn expand_benchmark(arguments: BenchmarkArguments, mut function: ItemFn, metaben #benchmark_name, stringify!(#native_group_name), stringify!(#adapter_name), - #allocation_tracking, + true, ); #(#conditional_attributes)* @@ -376,6 +366,30 @@ fn expand_benchmark(arguments: BenchmarkArguments, mut function: ItemFn, metaben #call } } + + #(#conditional_attributes)* + #[expect( + clippy::allow_attributes, + clippy::allow_attributes_without_reason, + reason = "the Gungraun case macro emits lint-control attributes" + )] + mod #probe_module { + use super::*; + use #metabench::__private::gungraun; + + #native_attribute + #(#case_attributes)* + #adapter_signature { + let allocations = #metabench::__private::begin_allocation_measurement(); + let perf = #metabench::__private::begin_perf_measurement(); + let vtune = #metabench::__private::begin_vtune_measurement(); + let output = #call; + ::core::mem::drop(vtune); + ::core::mem::drop(perf); + ::core::mem::drop(allocations); + output + } + } }) } @@ -398,9 +412,10 @@ fn expand_benchmark(arguments: BenchmarkArguments, mut function: ItemFn, metaben /// Async, const-generic, and unsafe functions are rejected because the /// generated adapter cannot correctly invoke those adapter shapes or uphold /// an unsafe function's safety contract. Ordinary generic, const, and -/// explicitly ABI-qualified functions are supported. Const functions run -/// under Criterion and Gungraun but do not participate in runtime allocation, -/// Linux perf, or `VTune` tracking. +/// explicitly ABI-qualified functions are supported. Collection controls exist +/// only in a separate private probe adapter, never in the preserved function. +/// Adapter calls resolve directly to the preserved function even when its name +/// matches a generated argument or collector-control local. pub fn benchmark(arguments: TokenStream2, item: TokenStream2) -> Result { reject_if_too_complex(&arguments)?; let arguments = parse2(arguments)?; @@ -412,8 +427,8 @@ pub fn benchmark(arguments: TokenStream2, item: TokenStream2) -> Result>(); + match &callee.path.segments[3].arguments { + syn::PathArguments::None => assert_eq!(expected_types, [] as [String; 0]), + syn::PathArguments::AngleBracketed(arguments) => { + assert!(arguments.colon2_token.is_some()); + let types = arguments + .args + .iter() + .map(|argument| { + let GenericArgument::Type(syn::Type::Path(ty)) = argument else { + panic!("only type parameters should be explicitly forwarded"); + }; + ty.to_token_stream().to_string() + }) + .collect::>(); + assert_eq!(types, expected_types); + } + syn::PathArguments::Parenthesized(_) => panic!("expected a turbofish"), + } + assert_eq!(call.args.len(), input.sig.inputs.len()); + for (index, argument) in call.args.iter().enumerate() { + let expected: Expr = syn::parse_str(&format!("__metabench_argument_{index}")).unwrap(); + assert_same_syntax(argument, &expected); + } + } + + #[test] + fn collector_and_argument_names_cannot_shadow_the_preserved_workload() { + for name in ["allocations", "perf", "vtune", "output", "__metabench_argument_0", "drop"] { + let name = syn::Ident::new(name, proc_macro2::Span::call_site()); + let input: ItemFn = syn::parse_quote! { + fn #name(value: u64) -> u64 { + const BODY_SENTINEL: u64 = 41; + value + BODY_SENTINEL + } + }; + let arguments = syn::parse2::(quote!(IDENTITY, GROUP, "name")).unwrap(); + let output = expand_benchmark(arguments, syn::parse2(input.to_token_stream()).unwrap(), "e!(::metabench)).unwrap(); + let output: syn::File = syn::parse2(output).unwrap(); + let Item::Fn(preserved) = &output.items[0] else { + panic!("expected the preserved workload"); + }; + assert_same_syntax(&preserved.sig, &input.sig); + assert_same_syntax(&preserved.vis, &input.vis); + assert_same_syntax(&preserved.block, &input.block); + let expected_attribute: syn::Attribute = syn::parse_quote!(#[inline(never)]); + assert_eq!(preserved.attrs.len(), 1); + assert_same_syntax(&preserved.attrs[0], &expected_attribute); + let adapters = output + .items + .iter() + .filter_map(|item| { + let Item::Mod(module) = item else { + return None; + }; + module.content.as_ref().unwrap().1.iter().find_map(|item| { + let Item::Fn(function) = item else { + return None; + }; + Some(function) + }) + }) + .collect::>(); + assert_eq!(adapters.len(), 2); + assert_ordinary_adapter(adapters[0], &input); + let Stmt::Local(local) = &adapters[1].block.stmts[3] else { + panic!("expected the captured workload result"); + }; + let expected: Expr = syn::parse_quote!(super::super::super::#name(__metabench_argument_0)); + assert_same_syntax(&local.init.as_ref().unwrap().expr, &expected); + for (statement, name) in adapters[1].block.stmts[4..7].iter().zip(["vtune", "perf", "allocations"]) { + let name = syn::Ident::new(name, proc_macro2::Span::call_site()); + let expected: Stmt = syn::parse_quote!(::core::mem::drop(#name);); + assert_same_syntax(statement, &expected); + } + } + } + + #[test] + fn preserved_workload_and_ordinary_adapter_have_exact_structure() { + let functions = [ + quote! { + fn workload() -> u64 { + const BODY_SENTINEL: u64 = 41; + BODY_SENTINEL + 1 + } + }, + quote! { + pub(crate) fn workload(#[allow(unused)] mut value: u64, _: bool) -> u64 { + const BODY_SENTINEL: u64 = 41; + value += BODY_SENTINEL; + if value > 100 { return value - 1; } + value + 1 + } + }, + quote! { + pub const fn workload(value: u64) -> u64 { + const BODY_SENTINEL: u64 = 41; + value + BODY_SENTINEL + } + }, + quote! { + pub(super) extern "C" fn workload(value: u64) -> u64 { + const BODY_SENTINEL: u64 = 41; + value + BODY_SENTINEL + } + }, + quote! { + fn workload<'a>(value: &'a str) -> &'a str { + let BODY_SENTINEL = value; + BODY_SENTINEL + } + }, + quote! { + pub fn workload<'a, T: Copy, U>(value: &'a T, extra: U) -> (&'a T, U) + where T: Send, U: Clone { + let BODY_SENTINEL = extra.clone(); + (value, BODY_SENTINEL) + } + }, + ]; + for function in functions { + let input: ItemFn = syn::parse2(quote! { + /// Workload documentation. + #[cfg(feature = "enabled")] + #[cfg_attr(unix, cold, bench::one(1), inline(always), cfg_attr(feature = "nested", cfg(all()), benches::many(iter = 1..=2)))] + #[inline(always)] + #[bench::ordinary(1)] + #function + }) + .unwrap(); + let arguments = syn::parse2::(quote!(IDENTITY, GROUP, "name")).unwrap(); + let retained_input = syn::parse2(input.to_token_stream()).unwrap(); + let output = expand_benchmark(arguments, retained_input, "e!(::metabench)).unwrap(); + let output: syn::File = syn::parse2(output).unwrap(); + let preserved = output + .items + .iter() + .filter_map(|item| match item { + Item::Fn(function) if function.sig.ident == "workload" => Some(function), + _ => None, + }) + .collect::>(); + assert_eq!(preserved.len(), 1); + assert_preserved_workload(preserved[0], &input); + + let ordinary_modules = output + .items + .iter() + .filter_map(|item| match item { + Item::Mod(module) if module.ident.to_string().starts_with("__metabench_native_") => Some(module), + _ => None, + }) + .collect::>(); + assert_eq!(ordinary_modules.len(), 1); + let ordinary = ordinary_modules[0]; + let expected_conditions: ItemFn = syn::parse_quote! { + #[cfg(feature = "enabled")] + #[cfg_attr(unix, cfg_attr(feature = "nested", cfg(all())))] + fn expected() {} + }; + assert_eq!(ordinary.attrs.len(), expected_conditions.attrs.len() + 1); + for (actual, expected) in ordinary.attrs.iter().zip(&expected_conditions.attrs) { + assert_same_syntax(actual, expected); + } + let adapters = ordinary + .content + .as_ref() + .unwrap() + .1 + .iter() + .filter_map(|item| match item { + Item::Fn(function) => Some(function), + _ => None, + }) + .collect::>(); + assert_eq!(adapters.len(), 1); + assert_ordinary_adapter(adapters[0], &input); + } + } + + #[test] + fn workload_body_has_no_runtime_collector_checks() { + let expanded = expand( + quote!(IDENTITY, GROUP, "name"), + quote!( + fn workload(value: u64) -> u64 { + value + 1 + } + ), + ) + .unwrap(); + let workload = expanded.split("constIDENTITY").next().unwrap(); + assert!(!workload.contains("begin_allocation_measurement")); + assert!(!workload.contains("begin_perf_measurement")); + assert!(!workload.contains("begin_vtune_measurement")); + assert!(expanded.contains("#[inline(never)]")); + assert!(expanded.contains("value+1")); + let native = expanded.split("mod__metabench_probe_").next().unwrap(); + assert!(!native.contains("begin_allocation_measurement")); + assert!(!native.contains("begin_perf_measurement")); + assert!(!native.contains("begin_vtune_measurement")); + assert_eq!(expanded.matches("begin_allocation_measurement").count(), 1); + assert_eq!(expanded.matches("begin_perf_measurement").count(), 1); + assert_eq!(expanded.matches("begin_vtune_measurement").count(), 1); + } + #[test] fn positional_and_named_forms_expand_equivalently() { let positional = expand( diff --git a/crates/metabench_macros_impl/src/shared.rs b/crates/metabench_macros_impl/src/shared.rs index 70b123fb2..81c1e8de4 100644 --- a/crates/metabench_macros_impl/src/shared.rs +++ b/crates/metabench_macros_impl/src/shared.rs @@ -122,13 +122,40 @@ pub(super) fn support_ident(kind: &str, logical_name: &str) -> Ident { format_ident!("__metabench_{kind}_{hash:032x}", span = Span::mixed_site()) } +pub(super) fn native_ident(kind: &str, logical_name: &str) -> Ident { + let readable = logical_name + .chars() + .take(64) + .map(|character| { + if character.is_ascii_alphanumeric() { + character.to_ascii_lowercase() + } else { + '_' + } + }) + .collect::(); + let hashed = support_ident(kind, logical_name); + format_ident!("{hashed}_{readable}", span = Span::mixed_site()) +} + #[cfg_attr(coverage_nightly, coverage(off))] #[cfg(test)] mod tests { use proc_macro2::TokenStream; use quote::quote; - use super::{MAX_FIELD_TOKENS, MAX_GROUP_DEPTH, parse_ident_value, reject_if_too_complex, support_ident}; + use super::{MAX_FIELD_TOKENS, MAX_GROUP_DEPTH, native_ident, parse_ident_value, reject_if_too_complex, support_ident}; + + #[test] + fn native_names_preserve_readable_identity_without_sanitization_collisions() { + assert!(native_ident("benchmark", "PREPARED_SEND").to_string().ends_with("_prepared_send")); + assert_ne!(native_ident("benchmark", "SEND"), native_ident("benchmark", "send")); + assert_ne!(native_ident("benchmark", "a-b"), native_ident("benchmark", "a_b")); + assert_ne!( + native_ident("benchmark", "a".repeat(100).as_str()), + native_ident("benchmark", format!("{}b", "a".repeat(100)).as_str()) + ); + } #[test] fn identifier_values_accept_only_one_unqualified_segment() { diff --git a/crates/metabench_macros_impl/src/target.rs b/crates/metabench_macros_impl/src/target.rs index 361a1c6d4..cf9a53ad2 100644 --- a/crates/metabench_macros_impl/src/target.rs +++ b/crates/metabench_macros_impl/src/target.rs @@ -9,7 +9,7 @@ use syn::parse::{Parse, ParseStream}; use syn::punctuated::Punctuated; use syn::{Error, Expr, ExprLit, Ident, Lit, LitBool, MetaNameValue, Path, Result, Token, braced, bracketed, parse2, token}; -use crate::shared::{metabench_path, parse_ident_value, reject_if_too_complex, support_ident}; +use crate::shared::{metabench_path, native_ident, parse_ident_value, reject_if_too_complex, support_ident}; fn expect_field(input: ParseStream<'_>, expected: &str) -> Result<()> { let field = input.parse::()?; @@ -167,6 +167,9 @@ impl Parse for Target { } benchmarks.into_iter().map(Group::simple).collect() } else { + if next_field_is(input, "allocator") { + return Err(input.error("`allocator` must follow `benchmarks` or `groups` in metabench::main!")); + } expect_field(input, "groups")?; input.parse::()?; let groups; @@ -233,12 +236,12 @@ fn expand_target(target: Target, metabench: &TokenStream2) -> TokenStream2 { let group_declarations = groups.iter().map(|group| { let name = &group.name; let logical_group = name.to_string(); - let group_name = support_ident("group", &logical_group); + let group_name = native_ident("group", &logical_group); let group_module = support_ident("group_module", &logical_group); let adapter_names = group .benchmarks .iter() - .map(|identity| support_ident("benchmark", &identity.to_string())) + .map(|identity| native_ident("benchmark", &identity.to_string())) .collect::>(); let adapter_imports = group.benchmarks.iter().zip(&adapter_names).map(|(identity, adapter_name)| { let adapter_module = support_ident("native", &identity.to_string()); @@ -270,9 +273,9 @@ fn expand_target(target: Target, metabench: &TokenStream2) -> TokenStream2 { use #group_module::#group_name; } }); - let group_names = groups.iter().map(|group| support_ident("group", &group.name.to_string())); + let group_names = groups.iter().map(|group| native_ident("group", &group.name.to_string())); let identities = groups.iter().flat_map(|group| { - let native_group = support_ident("group", &group.name.to_string()); + let native_group = native_ident("group", &group.name.to_string()); group .benchmarks .iter() @@ -287,6 +290,8 @@ fn expand_target(target: Target, metabench: &TokenStream2) -> TokenStream2 { .join(":"); let identities_name = support_ident("identities", &target_key); let target_module = support_ident("target", &target_key); + let probe_groups = groups.iter().map(|group| expand_probe_group(group, metabench)); + let probes_name = support_ident("probes", &target_key); quote! { #(#group_declarations)* @@ -294,6 +299,7 @@ fn expand_target(target: Target, metabench: &TokenStream2) -> TokenStream2 { const #identities_name: &[#metabench::BenchmarkIdentity] = &[ #(#identities),* ]; + const #probes_name: &[#metabench::__private::ProbeGroup] = &[#(#probe_groups),*]; mod #target_module { use super::*; @@ -305,6 +311,7 @@ fn expand_target(target: Target, metabench: &TokenStream2) -> TokenStream2 { library_benchmark_groups = #(#group_names),* }, identities = #identities_name + , probes = #probes_name #allocator ); @@ -319,6 +326,23 @@ fn expand_target(target: Target, metabench: &TokenStream2) -> TokenStream2 { } } +fn expand_probe_group(group: &Group, metabench: &TokenStream2) -> TokenStream2 { + let native_group = native_ident("group", &group.name.to_string()); + let benchmarks = group.benchmarks.iter().map(|identity| { + let module = support_ident("probe", &identity.to_string()); + let adapter = native_ident("benchmark", &identity.to_string()); + quote!(#metabench::__private::ProbeBenchmark { + identity: #identity.__in_gungraun_group(stringify!(#native_group)), + cases: #module::#adapter::__BENCHES, + }) + }); + quote!(#metabench::__private::ProbeGroup { + benchmarks: &[#(#benchmarks),*], + setup: #native_group::__run_setup, + teardown: #native_group::__run_teardown, + }) +} + /// Generates private native Gungraun groups and the target entry point. pub fn main(input: TokenStream2) -> Result { reject_if_too_complex(&input)?; @@ -354,6 +378,13 @@ mod tests { assert!(simple.contains("allocator=std::alloc::System")); assert!(simple.contains("ONE.__in_gungraun_group")); assert!(simple.contains("TWO.__in_gungraun_group")); + assert!(simple.contains("ProbeGroup{benchmarks:&[")); + let probe = super::support_ident("probe", "ONE"); + let benchmark = super::native_ident("benchmark", "ONE"); + let group = super::native_ident("group", "ONE"); + assert!(simple.contains(&format!("cases:{probe}::{benchmark}::__BENCHES"))); + assert!(simple.contains(&format!("setup:{group}::__run_setup"))); + assert!(simple.contains(&format!("teardown:{group}::__run_teardown"))); let configured = expand(quote! { criterion = { @@ -393,6 +424,12 @@ mod tests { assert!(!output.contains("::metabench::gungraun")); } + #[test] + fn misplaced_allocator_has_an_actionable_order_diagnostic() { + let error = expand(quote!(criterion = benches, allocator = std::alloc::System, benchmarks = [ONE],)).unwrap_err(); + assert!(error.to_string().contains("`allocator` must follow `benchmarks` or `groups`")); + } + #[test] fn malformed_target_shapes_are_rejected_deterministically() { let cases = [ diff --git a/crates/multitude/docs/PERF.md b/crates/multitude/docs/PERF.md index 52ac1b6a9..bfcb227e1 100644 --- a/crates/multitude/docs/PERF.md +++ b/crates/multitude/docs/PERF.md @@ -2,7 +2,7 @@ Generated by [`scripts/perf_report.rs`](../scripts/perf_report.rs). Re-run it to refresh these numbers. -All figures are wall-clock medians measured by Criterion. They are machine-dependent: +All historical figures use wall-clock medians measured by Criterion. They are machine-dependent: the ratios between rows are the durable signal, not the absolute values. This report is a curated set of customer-facing scenarios. The crate also carries a @@ -17,6 +17,14 @@ Benchmark processes were pinned to logical CPU 4. Each comparison group runs in a freshly warmed process. Compared implementations are invoked adjacently to limit host-load and frequency drift. Differential allocation, Serde, and teardown timings are the median of 3 independently warmed paired runs. Compared variants run in the same process, with group order alternated between rounds. +These measurements have not been rerun for the metabench schema 9 migration. +Current `scripts/perf_report.rs` output reads Criterion's primary estimate from +`criterion/time`, not the historical median metric. For differential allocation, +Serde, and teardown, the current script takes the median of those primary +estimates across independently warmed paired runs. That independent-run +aggregation is distinct from Criterion's within-run median used for this +historical snapshot. + ## Arena vs. the system allocator One pass allocates a mixed working set — 1,000 `u64` values, 1,000 32-byte slices, and 1,000 short strings, plus the vectors holding them — and then releases all of it. The `arena` row takes every allocation from one warmed `Arena` and releases the whole generation with a single `Arena::reset`; the system row does the same work with `Box` and `Vec` on the global allocator ([mimalloc](https://github.com/microsoft/mimalloc)) and pays one free per object. Arena warm-up and the source data are built outside the measured region, so only allocation and release traffic is timed. diff --git a/crates/multitude/scripts/perf_report.rs b/crates/multitude/scripts/perf_report.rs index 527b3a239..060640f8c 100755 --- a/crates/multitude/scripts/perf_report.rs +++ b/crates/multitude/scripts/perf_report.rs @@ -362,7 +362,7 @@ fn index_report(report: JsonReport) -> Result { } fn criterion_time_ns(report: &ReportIndex, identity: &str) -> Option { - let metric = report.get(identity)?.results.get("criterion")?.metrics.get("median")?; + let metric = report.get(identity)?.results.get("criterion")?.metrics.get("time")?; let value = metric.value.as_f64(); match metric.unit.as_deref()? { "ps" => Some(value * 1e-3), @@ -383,7 +383,7 @@ fn parse_report(path: &Path, expected: &[(&str, &str)]) -> Result Result Option { let mut values: Vec = crit .iter() @@ -413,6 +413,54 @@ fn lookup_time(crit: &[(String, f64)], key: &str) -> Option { }) } +#[cfg(test)] +mod tests { + use super::{JsonReport, criterion_time_ns, index_report, lookup_time}; + + #[test] + fn schema_nine_primary_estimate_and_units() { + for (value, unit, expected) in [ + (serde_json::json!(1250), "ps", 1.25), + (serde_json::json!(1.25), "ns", 1.25), + (serde_json::json!(1.25), "µs", 1250.0), + (serde_json::json!(1.25), "us", 1250.0), + (serde_json::json!(1.25), "ms", 1_250_000.0), + (serde_json::json!(1.25), "s", 1_250_000_000.0), + ] { + let fixture = serde_json::json!({ + "schema_version": 9, + "entries": [{ + "identity": "example", + "results": {"criterion": {"metrics": { + "time": {"value": value, "unit": unit} + }}} + }] + }); + let report: JsonReport = serde_json::from_value(fixture).unwrap(); + let mut report = index_report(report).unwrap(); + assert_eq!(criterion_time_ns(&report, "example"), Some(expected)); + let metrics = &mut report.get_mut("example").unwrap().results.get_mut("criterion").unwrap().metrics; + let time = metrics.remove("time").unwrap(); + metrics.insert("median".to_string(), time); + assert_eq!(criterion_time_ns(&report, "example"), None); + } + } + + #[test] + fn paired_runs_still_use_median_of_primary_estimates() { + let mut values = vec![ + ("example".to_string(), 9.0), + ("example".to_string(), 1.0), + ("example".to_string(), 3.0), + ("other".to_string(), 100.0), + ]; + assert_eq!(lookup_time(&values, "example"), Some(3.0)); + values.push(("example".to_string(), 5.0)); + assert_eq!(lookup_time(&values, "example"), Some(4.0)); + assert_eq!(lookup_time(&values, "missing"), None); + } +} + fn fmt_ns(ns: Option) -> String { match ns { None => "—".into(), @@ -457,7 +505,7 @@ fn build_report( Re-run it to refresh these numbers.\n\n", ); out.push_str( - "All figures are wall-clock medians measured by Criterion. They are \ + "All figures use primary wall-clock estimates measured by Criterion. They are \ machine-dependent:\nthe ratios between rows are the durable signal, not \ the absolute values.\n\n", ); @@ -471,7 +519,7 @@ fn build_report( out.push_str("## How these numbers were produced\n\n"); let _ = writeln!( out, - "Criterion medians use {criterion_samples} samples, a {criterion_warmup_secs} s \ + "Criterion primary estimates use {criterion_samples} samples, a {criterion_warmup_secs} s \ warm-up, and a {criterion_measurement_secs} s measurement." ); if let Some(cpu) = cpu { @@ -486,7 +534,7 @@ fn build_report( } else { let _ = writeln!( out, - "Differential allocation, Serde, and teardown timings are the median of \ + "Differential allocation, Serde, and teardown timings are the median of the primary estimates from \ {comparison_repetitions} independently warmed paired runs. Compared \ variants run in the same process, with group order alternated between rounds.\n" ); diff --git a/crates/plurality/docs/PERF.md b/crates/plurality/docs/PERF.md index 8d7476c2d..8d69506e3 100644 --- a/crates/plurality/docs/PERF.md +++ b/crates/plurality/docs/PERF.md @@ -2,7 +2,12 @@ Generated by [`scripts/perf_report.rs`](../scripts/perf_report.rs). Re-run it to refresh these numbers. -All figures are wall-clock medians measured by Criterion. They are machine-dependent: the ratios between rows are the durable signal, not the absolute values. +All historical figures are wall-clock medians measured by Criterion. They are machine-dependent: the ratios between rows are the durable signal, not the absolute values. + +These measurements have not been rerun for the metabench schema 9 migration. +Current `scripts/perf_report.rs` output reads Criterion's primary estimate from +`criterion/time`, not the historical median metric. A median across independent +runs, where used, is a separate aggregation of per-run estimates. This report is a curated set of customer-facing scenarios. The crate also carries a larger suite of internal micro-benchmarks, including Callgrind instruction-count suites inside the consolidated `plurality` bench target, which are used for optimization work and are not published here; run them with `cargo bench --bench plurality` in this crate. diff --git a/crates/plurality/scripts/perf_report.rs b/crates/plurality/scripts/perf_report.rs index f2569d736..4cf5a91c1 100755 --- a/crates/plurality/scripts/perf_report.rs +++ b/crates/plurality/scripts/perf_report.rs @@ -28,13 +28,13 @@ serde_json = "1" //! `scripts/perf_report.rs` — full run //! `scripts/perf_report.rs --fast` — quick, lower-fidelity criterion settings -use std::collections::{BTreeMap, btree_map::Entry}; -use std::env; +use std::collections::btree_map::Entry; +use std::collections::BTreeMap; use std::ffi::OsString; use std::fmt::Write as _; -use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, ExitCode}; +use std::{env, fs}; use serde::Deserialize; @@ -176,25 +176,14 @@ fn run(fast: bool) -> Result<(), String> { OsString::from("--export-md"), markdown_path.as_os_str().to_owned(), ]; - for forwarded in [ - "--warm-up-time", - warm, - "--measurement-time", - meas, - "--sample-size", - samples, - ] { + for forwarded in ["--warm-up-time", warm, "--measurement-time", meas, "--sample-size", samples] { args.push(OsString::from("--criterion-arg")); args.push(OsString::from(forwarded)); } println!("==> Running cargo bench --bench plurality"); let mut cmd = Command::new(env::var("CARGO").unwrap_or_else(|_| "cargo".into())); - cmd.current_dir(&crate_dir) - .arg("bench") - .arg("--bench") - .arg("plurality") - .args(&args); + cmd.current_dir(&crate_dir).arg("bench").arg("--bench").arg("plurality").args(&args); let output = cmd .output() .map_err(|e| format!("failed to spawn cargo bench --bench plurality: {e}"))?; @@ -258,7 +247,7 @@ fn index_report(report: JsonReport) -> Result { } fn criterion_time_ns(report: &ReportIndex, identity: &str) -> Option { - let metric = report.get(identity)?.results.get("criterion")?.metrics.get("median")?; + let metric = report.get(identity)?.results.get("criterion")?.metrics.get("time")?; let value = metric.value.as_f64(); match metric.unit.as_deref()? { "ps" => Some(value * 1e-3), @@ -274,6 +263,40 @@ fn crit_per_op(report: &ReportIndex, key: &str) -> Option { criterion_time_ns(report, key).map(|ns| ns / N) } +#[cfg(test)] +mod tests { + use super::{criterion_time_ns, index_report, JsonReport}; + + #[test] + fn schema_nine_primary_estimate_and_units() { + for (value, unit, expected) in [ + (serde_json::json!(1250), "ps", 1.25), + (serde_json::json!(1.25), "ns", 1.25), + (serde_json::json!(1.25), "µs", 1250.0), + (serde_json::json!(1.25), "us", 1250.0), + (serde_json::json!(1.25), "ms", 1_250_000.0), + (serde_json::json!(1.25), "s", 1_250_000_000.0), + ] { + let fixture = serde_json::json!({ + "schema_version": 9, + "entries": [{ + "identity": "example", + "results": {"criterion": {"metrics": { + "time": {"value": value, "unit": unit} + }}} + }] + }); + let report: JsonReport = serde_json::from_value(fixture).unwrap(); + let mut report = index_report(report).unwrap(); + assert_eq!(criterion_time_ns(&report, "example"), Some(expected)); + let metrics = &mut report.get_mut("example").unwrap().results.get_mut("criterion").unwrap().metrics; + let time = metrics.remove("time").unwrap(); + metrics.insert("median".to_string(), time); + assert_eq!(criterion_time_ns(&report, "example"), None); + } + } +} + fn fmt_ns(ns: Option) -> String { match ns { None => "—".into(), @@ -294,7 +317,7 @@ fn build_report(report: &ReportIndex) -> String { let mut out = String::new(); out.push_str("# Plurality Performance Report\n\n"); out.push_str("Generated by [`scripts/perf_report.rs`](../scripts/perf_report.rs). Re-run it to refresh these numbers.\n\n"); - out.push_str("All figures are wall-clock medians measured by Criterion. They are machine-dependent: the ratios between rows are the durable signal, not the absolute values.\n\n"); + out.push_str("All figures are primary wall-clock estimates measured by Criterion. They are machine-dependent: the ratios between rows are the durable signal, not the absolute values.\n\n"); out.push_str("This report is a curated set of customer-facing scenarios. The crate also carries a larger suite of internal micro-benchmarks, including Callgrind instruction-count suites inside the consolidated `plurality` bench target, which are used for optimization work and are not published here; run them with `cargo bench --bench plurality` in this crate.\n\n"); out.push_str("Every measured body allocates one ~32-byte, `Drop`-free value and releases it again. Each pool is pre-warmed with 1,024 slots and every slot is released before measurement starts, so the timed region only ever reuses a slot and never grows the pool. Payload construction is inside the timed region for every implementation, including the guard-returning pools, which write the payload through their guard.\n\n"); diff --git a/crates/routerama/docs/PERF.md b/crates/routerama/docs/PERF.md index 775ae861a..63ea9201a 100644 --- a/crates/routerama/docs/PERF.md +++ b/crates/routerama/docs/PERF.md @@ -6,10 +6,15 @@ Generated by `scripts/perf_report.rs`: **Workload:** one full sweep of the shared request-path lookups (see `benches/common/routes_data.rs`) against each router. Every router is built from the same route table (literal segments plus single-segment `{var}` parameters — the common subset all of them express) in a setup step that is excluded from the measured region. Setup also performs one full unmeasured lookup sweep to initialize lazy matcher and allocator state before measuring the steady-state hot path; `routerama_static` is the compile-time `#[resolver]` router, so it has no construction cost, while `routerama_dynamic` is the run-time router registered from the same table. **Apples-to-apples:** every router is driven to the same *typed* end state — the HTTP method (verb) validated against the request and every captured path variable coerced into its declared type (`u32` parsed, `String` percent-decoded and owned, `&str` borrowed). `routerama` reaches this in one step (a typed enum variant with the method already matched and every field coerced); the third-party routers only *select* a route, so the harness explicitly checks the method and coerces each parameter the same way afterwards. `regex` selects the winner with a `RegexSet` and then re-scans it with the winning `Regex` to capture (two passes), so it does structurally more work — read it as an upper bound for a regex-based router reaching the same end state. -Criterion median is reported (default 30 samples, 1 s warm-up, 2 s measurement; override with `--samples` / `--measurement-time` / `--warm-up-time`). +The historical Criterion median is reported (default 30 samples, 1 s warm-up, 2 s measurement; override with `--samples` / `--measurement-time` / `--warm-up-time`). Instructions, branch misses, and memory accesses are per full lookup sweep. Memory accesses = L1 Hits + LL Hits + RAM Hits (Callgrind D-cache references). +These measurements have not been rerun for the metabench schema 9 migration. +Current `scripts/perf_report.rs` output reads Criterion's primary estimate from +`criterion/time`, not the historical median metric. A median across independent +runs, where used, is a separate aggregation of per-run estimates. + ## `compare_routers` | Resolver | Time (criterion) | Instructions | Branch misses | Mem accesses | diff --git a/crates/routerama/scripts/perf_report.rs b/crates/routerama/scripts/perf_report.rs index 182c82baf..671b00a12 100755 --- a/crates/routerama/scripts/perf_report.rs +++ b/crates/routerama/scripts/perf_report.rs @@ -27,7 +27,8 @@ syn = { version = "2", features = ["full", "parsing"] } //! //! Criterion and Gungraun variants are paired through shared metabench identities. -use std::collections::{BTreeMap, btree_map::Entry}; +use std::collections::BTreeMap; +use std::collections::btree_map::Entry; use std::ffi::OsString; use std::fmt::Write as _; use std::io::Write as _; @@ -254,7 +255,7 @@ fn index_report(report: JsonReport) -> Result { } fn criterion_time_ns(report: &ReportIndex, identity: &str) -> Option { - let metric = report.get(identity)?.results.get("criterion")?.metrics.get("median")?; + let metric = report.get(identity)?.results.get("criterion")?.metrics.get("time")?; let value = metric.value.as_f64(); match metric.unit.as_deref()? { "ps" => Some(value * 1e-3), @@ -279,6 +280,40 @@ fn callgrind_metric(report: &ReportIndex, identity: &str, key: &str) -> Option) -> String { match ns { None => "—".into(), @@ -390,7 +425,7 @@ fn build_report(report: &ReportIndex) -> String { for a regex-based router reaching the same end state. \n", ); out.push_str( - "Criterion median is reported (default 30 samples, 1 s warm-up, 2 s \ + "Criterion's primary wall-clock estimate is reported (default 30 samples, 1 s warm-up, 2 s \ measurement; override with `--samples` / `--measurement-time` / \ `--warm-up-time`). \n", ); @@ -401,7 +436,9 @@ fn build_report(report: &ReportIndex) -> String { out.push_str("| Resolver | Time (criterion) | Instructions | Branch misses | Mem accesses |\n"); out.push_str("|---|---:|---:|---:|---:|\n"); for identity in VARIANTS { - let variant = identity.strip_prefix("compare_routers/").expect("variant identity stays within compare_routers"); + let variant = identity + .strip_prefix("compare_routers/") + .expect("variant identity stays within compare_routers"); let time = criterion_time_ns(report, identity); let instructions = callgrind_metric(report, identity, "Ir"); let branch_misses = callgrind_metric(report, identity, "Bcm"); @@ -549,8 +586,7 @@ fn regenerate_router(crate_dir: &Path) -> Result<(), AppError> { let body = prettyplease::unparse(&file); let out_path = crate_dir.join("benches").join("common").join("bench_router.rs"); - fs::write(&out_path, format!("{ROUTER_HEADER}{body}")) - .map_err(|e| app_err!("writing {}: {e}", out_path.display()))?; + fs::write(&out_path, format!("{ROUTER_HEADER}{body}")).map_err(|e| app_err!("writing {}: {e}", out_path.display()))?; println!("Regenerated {}", out_path.display()); Ok(()) } @@ -648,10 +684,11 @@ fn run(args: &Args) -> Result<(), AppError> { let report: JsonReport = serde_json::from_str(&json).map_err(|e| app_err!("parsing {}: {e}", json_path.display()))?; let report = index_report(report)?; - let expected = VARIANTS - .iter() - .copied() - .chain(QUERY_GROUPS.iter().flat_map(|group| group.variants.iter().map(|variant| variant.identity))); + let expected = VARIANTS.iter().copied().chain( + QUERY_GROUPS + .iter() + .flat_map(|group| group.variants.iter().map(|variant| variant.identity)), + ); let missing: Vec<&str> = expected.filter(|key| criterion_time_ns(&report, key).is_none()).collect(); if !missing.is_empty() { bail!( @@ -665,11 +702,7 @@ fn run(args: &Args) -> Result<(), AppError> { let out_path = crate_dir.join("docs").join("PERF.md"); fs::write(&out_path, &markdown).map_err(|e| app_err!("writing {}: {e}", out_path.display()))?; - println!( - "Wrote {} ({} metabench identities)", - out_path.display(), - report.len(), - ); + println!("Wrote {} ({} metabench identities)", out_path.display(), report.len(),); println!("==> Done. Report written to docs/PERF.md"); Ok(()) } diff --git a/docs/metabench-native-contracts.md b/docs/metabench-native-contracts.md new file mode 100644 index 000000000..ff462310e --- /dev/null +++ b/docs/metabench-native-contracts.md @@ -0,0 +1,45 @@ +# Metabench native contracts + +The repository-specific **Metabench native contracts (Linux)** CI job installs +Valgrind and `gungraun-runner` 0.19.4, matching the workspace's Gungraun version. +It runs the native listing and default-engine smoke contracts and is required +by the stable **Required repository checks** fan-in. These are correctness +checks, not timed performance measurements. + +To run the same required tier locally on Linux: + +```sh +mkdir -p target/metabench-contract-scratch +export TMPDIR="$PWD/target/metabench-contract-scratch" +export CARGO_BUILD_JOBS=2 METABENCH_REQUIRE_NATIVE_CONTRACTS=1 +cargo install --locked --version 0.19.4 gungraun-runner +# Install Valgrind using your Linux distribution's package manager. +gungraun-runner --version +valgrind --version +cargo test --package metabench --test spawned_benchmark --locked gungraun_listing_is_forwarded -- --exact --nocapture +cargo test --package metabench --test spawned_benchmark --locked default_engine_smoke_checks_do_not_publish_measurements -- --exact --nocapture +``` + +`METABENCH_REQUIRE_NATIVE_CONTRACTS` makes missing prerequisites a test failure, +not a successful skip. Without this setting, ordinary local test runs retain +the optional skip when `gungraun-runner` is unavailable. The Linux smoke contract +is not compiled on Windows; Windows' unsupported native-mode behavior is +unchanged. Test output reports each executed contract by name. + +Unit regressions also check collector activation, report boundary values, +generated probe registration, bounded pipe draining, and exact worker cleanup +deadlines. Platform-specific mutation configurations exclude only adapters +that the host does not compile and explicitly documented equivalent flag +mutations; the active platform retains its behavioral checks. + +Windows pipe regressions run in a subprocess with an independent deadline so +a broken nonblocking read fails promptly instead of hanging the test suite. +Reparse-point admission tests exercise file-attribute checks without requiring +symlink privileges, and private-directory creation is checked on every host. + +Scripted retention and drain tests use an in-memory output tail without files +or OS pipes. Private platform tests remain beside their inaccessible internals; +the public worker contracts are exercised by `tests/spawned_benchmark.rs`. +Mutation runs skip only the real-time sustained-output timeout test. The +finite `FakeClock`/`FakeProcess` deadline regressions remain selected, while +ordinary test runs retain the actual process stress scenario.