diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..cb5328d --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,73 @@ +name: benchmark + +# CI guardrail for benchmark/ -- the one directory of runnable +# source in this otherwise source-free, closed-source repository. Runs on +# macOS because the harness is macOS-only by design: it depends on +# lsappinfo, footprint, vm_stat, notifyutil, and pmset, all Darwin-only +# tools. +# +# This workflow builds, lints, and tests the harness only. It deliberately +# never runs `resource-benchmark run` (or `doctor` against real installed +# subjects): a real sweep needs an exclusive, quiesced physical machine for +# hours, and its numbers would be meaningless -- and misleading -- if +# produced on a shared, noisy CI runner. Do not add that here. + +on: + push: + paths: + - 'benchmark/**' + - '.github/workflows/benchmark.yml' + pull_request: + paths: + - 'benchmark/**' + - '.github/workflows/benchmark.yml' + +concurrency: + group: benchmark-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + working-directory: benchmark + +jobs: + check: + runs-on: macos-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Rust stable + uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + # benchmark/rustfmt.toml sets nightly-only options (e.g. + # unstable_features), so the format check needs a nightly rustfmt -- + # stable `cargo fmt` reads a different subset of the config and + # produces different output. + # Installing nightly second makes it rustup's default, so every step + # below names its toolchain explicitly rather than relying on that + # ordering -- `cargo clippy` under nightly fails outright, since the + # nightly toolchain installed here carries rustfmt and nothing else. + - name: Install Rust nightly (rustfmt only) + uses: dtolnay/rust-toolchain@nightly + with: + components: rustfmt + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: benchmark -> target + + - name: Format check (nightly rustfmt) + run: cargo +nightly fmt -- --check + + - name: Clippy + run: cargo +stable clippy --all-targets -- -D warnings + + - name: Test + run: cargo +stable test diff --git a/.gitignore b/.gitignore index 0c9fd13..ba05bc1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,7 @@ .DS_Store .dn/ + +# benchmark/ is a standalone Rust crate; its own build output is scratch, +# but benchmark/Cargo.lock is committed deliberately (see benchmark/README.md) +# so a third party builds the exact dependency versions behind published numbers. +benchmark/target/ diff --git a/LICENSE.md b/LICENSE.md index c2b09c8..dd8c883 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,7 +1,7 @@ # Licence -**TermTree is proprietary software. This repository contains documentation and release metadata -only — it contains no TermTree source code.** +**TermTree is proprietary software. This repository contains documentation, release metadata, and +one standalone benchmark harness — it contains no TermTree source code.** ## This repository @@ -10,9 +10,20 @@ accompanying images — are published by Document Node Pty Ltd so that people ca follow its releases, and report problems. They are made available for that purpose. All rights are reserved; no licence to the TermTree application is granted by anything in this repository. -This repository is deliberately **not** published under an open source licence. TermTree's source -code is not distributed, so there is nothing here for an open source licence to cover, and applying -one would misrepresent what is on offer. +Outside of the `benchmark/` directory described below, this repository is deliberately **not** +published under an open source licence. TermTree's source code is not distributed here, so there +is nothing else in this repository for an open source licence to cover, and applying one would +misrepresent what is on offer. + +## The benchmark harness + +`benchmark/` is a deliberate, narrowly scoped exception. It is a standalone measurement tool, not +part of the TermTree application, and TermTree's published memory, CPU, and cold-start figures +depend on it — a benchmark nobody can run is an assertion, not evidence. For that reason `benchmark/` +is published under the Apache License, Version 2.0, reproduced in +[`benchmark/LICENSE`](benchmark/LICENSE). That licence covers the contents of the `benchmark/` +directory only; it does not extend to any other part of this repository or to the TermTree +application itself. ## The TermTree application diff --git a/README.md b/README.md index ef19315..038eebc 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Full walkthrough: **[Getting Started](https://termtree.com/guide/getting-started ## Issues and support -This repository is TermTree's public issue tracker. The application itself is closed source, so there is no code here — but bug reports, feature requests, and questions are all welcome and are read. +This repository is TermTree's public issue tracker. The application itself is closed source — the only exception is the `benchmark/` harness described below — but bug reports, feature requests, and questions are all welcome and are read. - **Found a bug?** [Open a bug report](https://github.com/documentnode/termtree/issues/new?template=bug_report.yml). Include your OS, TermTree version, and steps to reproduce. - **Want a feature?** [Open a feature request](https://github.com/documentnode/termtree/issues/new?template=feature_request.yml). @@ -72,6 +72,15 @@ You can also send feedback from inside the app using the feedback button in the Response times vary, but every issue gets triaged. +## Resource benchmark + +TermTree's memory, CPU, and cold-start claims rest on a harness published in this repository at +[`benchmark/`](benchmark/), the one deliberate exception to this repo's otherwise source-free, +closed-source posture: numbers are only credible if strangers can measure them independently. +It is released under the Apache License 2.0, scoped to that directory — see +[`benchmark/LICENSE`](benchmark/LICENSE). For what it measures and how to run it, see +[`benchmark/README.md`](benchmark/README.md). + ## Links - [Website](https://termtree.com) diff --git a/SECURITY.md b/SECURITY.md index 7aac745..13f50b9 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -29,6 +29,11 @@ security fixes. Update from within the app, or download the current version at In scope: the TermTree desktop application, its update mechanism, its optional cloud sync, and the install script served at . +Out of scope: the `benchmark/` harness published in this repository. It is a local developer tool +you build and run yourself — it launches other applications on your own machine under your own +account, and writes only to its own disposable scratch directory. Report bugs in it as an issue, +not as a security report. + Out of scope: the security of programs you choose to run inside TermTree's terminals. Those are ordinary terminal sessions — Claude Code, Codex, shells, servers, and other CLIs run with your own credentials under their own security models, exactly as they would in any other terminal emulator. diff --git a/benchmark/.gitignore b/benchmark/.gitignore new file mode 100644 index 0000000..57aaac6 --- /dev/null +++ b/benchmark/.gitignore @@ -0,0 +1,6 @@ +# Build output for this standalone crate. +target/ +# Run results are attached to the published artifact deliberately, not +# committed as run debris (design §4.1). +results/*.json +results/*.md diff --git a/benchmark/Cargo.lock b/benchmark/Cargo.lock new file mode 100644 index 0000000..5542d97 --- /dev/null +++ b/benchmark/Cargo.lock @@ -0,0 +1,413 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core-graphics" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa95a34622365fa5bbf40b20b75dba8dfa8c94c734aea8ac9a5ca38af14316f1" +dependencies = [ + "bitflags", + "core-foundation", + "core-graphics-types", + "foreign-types", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" +dependencies = [ + "bitflags", + "core-foundation", + "libc", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea5190182e6915eb873ddbc16e23b711b6eb1f9c00a0d0a3a91b5f6228475225" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "resource-benchmark" +version = "0.1.0" +dependencies = [ + "core-foundation", + "core-graphics", + "serde", + "serde_json", + "sysinfo", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sysinfo" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +dependencies = [ + "windows-core", + "windows-targets", +] + +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-targets", +] + +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/benchmark/Cargo.toml b/benchmark/Cargo.toml new file mode 100644 index 0000000..a136382 --- /dev/null +++ b/benchmark/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "resource-benchmark" +version = "0.1.0" +edition = "2021" +publish = false +description = "Resource-usage benchmark harness for TermTree vs comparable agent-orchestrator apps (doc/design/resource-benchmark-design.md)" + +# Deliberately its own workspace, outside src-tauri: this is a measurement +# tool, so it must never lengthen an app build or a CI run (justfile:101-107, +# tools/update-feed's precedent, spec FR-16). +[workspace] + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +# Pinned to the exact version src-tauri/Cargo.toml uses, so the harness's +# process-tree walk has identical semantics to the app's own +# get_descendant_pids_and_names (terminal_cmd.rs:1588). +sysinfo = "0.33" +# core-graphics's CGWindowListCopyWindowInfo returns an untyped CFArray of +# CFDictionary entries; core-foundation is what lets window_probe.rs read +# kCGWindowOwnerPID/kCGWindowLayer/kCGWindowBounds out of it safely. The +# design budgeted 4 direct crates; this 5th is the same trade it already +# made for core-graphics itself -- a maintained wrapper instead of ~150 +# lines of hand-rolled CFDictionaryRef FFI (design §4.3's own reasoning). +core-graphics = "0.24" +core-foundation = "0.10" + +[[bin]] +name = "resource-benchmark" +path = "src/main.rs" diff --git a/benchmark/LICENSE b/benchmark/LICENSE new file mode 100644 index 0000000..52ee6c1 --- /dev/null +++ b/benchmark/LICENSE @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the + purposes of this License, Derivative Works shall not include works + that remain separable from, or merely link (or bind by name) to the + interfaces of, the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including the + original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright + owner or by an individual or Legal Entity authorized to submit on + behalf of the copyright owner. For the purposes of this definition, + "submitted" means any form of electronic, verbal, or written + communication sent to the Licensor or its representatives, including + but not limited to communication on electronic mailing lists, source + code control systems, and issue tracking systems that are managed by, + or on behalf of, the Licensor for the purpose of discussing and + improving the Work, but excluding communication that is conspicuously + marked or otherwise designated in writing by the copyright owner as + "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing + the origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Document Node, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..c39083d --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,248 @@ +# resource-benchmark + +A standalone Rust harness that measures resident memory, cold start / +time-to-interactive, idle CPU, and memory at 5/10/20 live agent sessions for +TermTree against comparable agent-orchestrator apps (CodeNomad's Electron +and Tauri builds, Collaborator, and optionally diri) on one fixed macOS +host, correcting for the `launchd`-vs-child-process attribution asymmetry +that makes every off-the-shelf measurement tool favor TermTree by +construction. + +This directory is the harness's **canonical home** — it was developed in +TermTree's private repository and moved here so it can be run, read, and +audited by anyone, not mirrored from somewhere else. It is also the one +deliberate exception to this repo's otherwise source-free posture: see +[Licence](#licence) below. + +## macOS only, structurally + +This harness is **macOS-only**, not just untested elsewhere. Every +measurement it takes goes through a macOS-specific unprivileged tool with +no Linux or Windows equivalent: `lsappinfo` (LaunchServices attribution), +`footprint` (`phys_footprint`, the memory metric this benchmark exists to +get right), `vm_stat` (host memory pressure and compression), `notifyutil` +(thermal pressure), and `pmset` (power source). A Linux or Windows user +running this crate will get a build error or a run refusal, not a +degraded-but-working experience — there is no portable fallback path for +any of these five tools, by design (see each module's doc comment under +`src/`). + +## What is, and is not, comparable across machines + +The **method** — attribution by deduplicated union, `phys_footprint` +instead of RSS, an external monotonic clock for cold start, the +foreground/unoccluded idle-CPU sample — travels to any Mac. So do the +**within-run ratios** between subjects measured back-to-back on the same +host under the same load. + +Absolute numbers do not travel. Megabytes and milliseconds measured on one +CPU, RAM size, macOS build, and background-load state will not reproduce +on different hardware. Every result file embeds its own machine spec, OS +build, and quiesce readings (`provenance`, `quiesce` in the JSON) precisely +so a rerun on different hardware is **self-disclosing** — a reader can see +immediately that the numbers came from a different host — rather than +silently compared against a number it was never measured against. + +## Licence + +The rest of this public repository publishes no product source. This +directory is the one deliberate exception: a benchmark nobody can run is +an assertion, not evidence. `benchmark/` ships with its own +[`LICENSE`](./LICENSE) (Apache-2.0), scoped to this directory only. +Nothing else in this repo is licensed for reuse. + +## The published run's machine + +No run has been published from this harness yet. Once one is, this section +states the exact host it was measured on: **Apple M1, 8 cores, 16 GB RAM, +macOS 15.7.4 (24G517)** is the specified measurement host for the first +run. **If your numbers come from different hardware or a different OS +build, say so next to them.** + +The `fixtures/` directory's captures (used by this crate's own test suite, +not by a live run) were taken on that same machine and OS build. + +## Prerequisites + +- macOS only (see above). +- No root/`sudo` required anywhere in the default path. `footprint` and + `lsappinfo` need none; `powermetrics`/`launchctl procinfo` are never + used. +- `/usr/bin/open` must document `--env` (used to isolate each subject's + `HOME`, see [Disposable scratch home](#disposable-per-run-scratch-home) + below). `man open`'s own page on the development host is dated April + 2017 and has documented `--env` for as long as this project has checked, + which suggests it has been available since roughly macOS 10.12/10.13 — + but that could not be confirmed as an exact minimum version, so + `resource-benchmark doctor` **probes for the flag at runtime** instead of + asserting an unverified version constant, and the harness refuses to + start rather than silently falling back to your real `$HOME` if the + probe fails. +- Every non-optional subject (TermTree, CodeNomad Electron, CodeNomad + Tauri, Collaborator) installed at its pinned version — `resource-benchmark + doctor` checks this and prints exactly what is missing or drifted. If + your install lives outside `/Applications`, pass + `--bundle-path =` (repeatable) or set + `RESOURCE_BENCHMARK_BUNDLE_PATH_` (e.g. + `RESOURCE_BENCHMARK_BUNDLE_PATH_CODENOMAD_ELECTRON`); a CLI flag wins + over the matching environment variable. diri is optional; pass + `--allow-optional-subjects` to include it if installed. +- **No subject already running.** The harness refuses to start (or to seed + a given subject) if any selected subject's **bundle identifier** already + has a live LaunchServices entry — checked by identifier, not display + name, because two differently named bundles can declare the same + identifier, and a single-instance plugin then hands a new launch off to + the already-running instance, which exits within seconds. Quit the named + app first. +- An agent CLI (e.g. `claude`) installed and resolvable; its path is + `--agent-cli-path ` or `RESOURCE_BENCHMARK_AGENT_CLI_PATH` + (defaults to `/usr/local/bin/claude`). +- A seeded repository checked out at a fixed commit; its local path is + `--repo-path ` or `RESOURCE_BENCHMARK_REPO_PATH` (its URL/commit + are `RESOURCE_BENCHMARK_REPO_URL` / `RESOURCE_BENCHMARK_REPO_COMMIT`). +- A quiesced machine: on AC power, no swap pressure, nominal thermal + pressure. `resource-benchmark doctor` reads all five quiesce signals and + reports whether a run would be allowed to start right now. + +## Disposable per-run scratch home + +Every subject is seeded, launched, and measured with `HOME` pointed at a +**fresh, disposable directory this harness owns** — never your real +`$HOME`. By default that is a brand-new directory under the OS temp +directory, created fresh for the run and never reused. You can point it +somewhere specific with `--home ` (highest precedence) or +`RESOURCE_BENCHMARK_HOME`; if you do, that is your choice to make, not this +harness deciding it for you. + +This is what makes it safe to run this harness on a machine with a real +TermTree/Collaborator/CodeNomad/diri install and real user data on it: the +launched subject's entire on-disk state — including TermTree's +`state.json` — lives under the scratch home, not under your real profile. +`seeding/termtree.rs`'s `expected_scratch_state_path` refuses to write +anywhere outside the current scratch home, which includes refusing your +real production profile. + +## Commands + +Run everything from this directory: + +```sh +# Preflight everything; changes nothing. +cargo run --release -- doctor + +# A single documented command for a full subject/tier sweep on an +# already-quiesced machine. Many hours; must own the machine. +cargo run --release -- run + +# A smaller, faster sweep for trying the harness out. +cargo run --release -- run --subjects termtree,collaborator \ + --tiers fresh-launch,n-session-5 --repetitions 5 + +# Render the Markdown table from a result file -- a pure function of the +# JSON, never hand-edited. +cargo run --release -- render results/.json --out results/.md + +# Seed a subject's on-disk state without running a full sweep (for manual +# inspection), and undo any seeder state afterwards. +cargo run --release -- seed --subject termtree --sessions 5 +cargo run --release -- restore --subject termtree +``` + +### The CodeNomad Electron-vs-Tauri pair, standalone + +CodeNomad ships the same MIT-licensed codebase as two builds — one on +Electron, one on Tauri — with TermTree **not a participant**. That pair is +a first-class standalone comparison: same source, two runtimes, so its +fairness is checkable without trusting this project's TermTree numbers at +all. The exact command: + +```sh +cargo run --release -- run \ + --subjects codenomad-electron,codenomad-tauri \ + --tiers fresh-launch,n-session-5,n-session-10,n-session-20 +``` + +## Seed-format verification status + +Each subject's session seeder writes whatever on-disk/CLI state makes that +subject start with `n` live sessions (`src/seeding/`). **Only TermTree's +seeder has been checked against a real install** — its `state.json` shape +is pinned against the app's own persistence code +(`seeding/termtree.rs`'s tests). **Collaborator's, CodeNomad's, and +diri's seed formats have never been run against a real install** — assume +they are wrong until proven otherwise; see each module's doc comment +(`seeding/collaborator.rs`, `seeding/codenomad.rs`, `seeding/diri.rs`) for +exactly what is unverified. + +The harness does not silently trust an unverified format: every +N-session/sustained-use sample for a subject whose seed format is +unverified reports `invalidReason: "seed-format-unverified"` in the result +file and is excluded from the published aggregate, and `doctor` prints a +note naming which subjects this applies to. If you can verify one of these +three against a real install, flip `seed_format_verified` to `true` on +that subject's entry in `src/subject.rs` and say how you verified it. + +## Runtime self-validation + +A public build of TermTree could be any version, so this harness detects +drift in its own hardcoded assumptions rather than silently mismeasuring: + +- **Cold-start log marks** (`log_marks.rs`): if `karijini.log` advances + during a TermTree launch but none of the three hardcoded messages match + a single line, the sample reports + `invalidReason: "termtree-log-marks-unrecognized"` instead of silently + leaving the cold-start fields `null`. +- **App data directory**: if TermTree launches but never creates + `DocumentNode/TermTree` under the scratch home, the sample reports + `invalidReason: "app-data-dir-not-created"` — the app's data-directory + convention has likely changed. +- **Seed consumption**: see [Seed-format verification status](#seed-format-verification-status) + above. + +## Two hazards to know before you run anything by hand + +**1. `footprint`'s `-p` flag is ambiguous — never use it.** `footprint -h` +documents `-p, --proc ` and `-p, --pid ` sharing one short flag; +name resolution wins and it is a *partial* match. Verified on this +project's own measurement host: `footprint -j out.json -p 1` measured four +unrelated `1Password` processes, not PID 1. This harness always builds +`--pid ` (the unambiguous long form), once per requested PID, and +never the short form. If you ever invoke `footprint` by hand while +reproducing a result, do the same. + +**2. Under zsh, an unquoted PID-list variable is not word-split.** If you +build a command like `footprint -j out.json $PIDS` by hand, **zsh** (this +project's default shell) passes the whole list as one argument, silently +measuring only the leading PID with an empty `errors` array — no error, +just a quietly wrong number. (bash *does* word-split, so the same command +works differently there.) This harness never does this: every invocation +goes through `exec.rs`'s `run_capture(program, args: &[&str])`, an argument +vector via `std::process::Command`, which spawns no shell and is immune by +construction. This warning is only for anyone re-running one of this +harness's `footprint`/`lsappinfo` invocations by hand. + +## Testing + +```sh +cd benchmark +cargo test +cargo clippy --all-targets -- -D warnings +cargo +nightly fmt -- --check +``` + +This crate is its own `[workspace]`, independent of anything else in this +repo, so it has no build-system integration to keep in sync here. + +Live measurement is not unit-testable; parsing is. `fixtures/` holds +captured real tool output (`lsappinfo list`, `footprint -j`, `vm_stat`, +`karijini.log` lines, and the five quiesce signals) that every parser's +tests run against. Launching subjects, live `footprint`/`vm_stat` +invocation, `CGWindowList` polling, and seeding third-party apps are not +unit-tested — those are covered by `doctor` and a one-subject smoke sweep +instead. + +For the manual pass — what `doctor` must tell you, how to confirm the scratch +home never touches your real profile, and the seed/restore round-trip — follow +[`doc/resource-benchmark-harness-testing.md`](doc/resource-benchmark-harness-testing.md). +It also records what is deliberately *not* verifiable without a quiesced +machine. diff --git a/benchmark/doc/resource-benchmark-harness-testing.md b/benchmark/doc/resource-benchmark-harness-testing.md new file mode 100644 index 0000000..a2fa35c --- /dev/null +++ b/benchmark/doc/resource-benchmark-harness-testing.md @@ -0,0 +1,294 @@ +# Resource benchmark harness + +## Scope + +Covers the harness's safety, refusal, and portability behaviour — the parts a +third party hits before they ever produce a number: + +- `doctor` naming every unmet prerequisite **and its remedy**, while changing + nothing. +- The disposable per-run scratch home: the runner's real application profiles + are never read or written. +- Refusing to start when a subject is already running, keyed on **bundle + identifier**. +- `seed` / `restore` round-tripping a profile byte-for-byte. +- `run` refusing on an unquiesced machine and measuring nothing. +- Per-subject bundle-path overrides and version-drift reporting. + +**Deliberately excluded.** A full measurement sweep is not verifiable from a +testing guide: it needs a rebooted machine with nothing else running, for +hours. See *Not verifiable here*. This guide proves the harness is safe to +hand a stranger; it does not prove any published number. + +**Risk being managed.** This harness seeds application state and launches real +applications. Its predecessor wrote directly into the operator's live +TermTree, Collaborator and CodeNomad profiles. Every scenario below exists to +prove that is no longer possible. + +## Verification surface + +`backend` — the harness is a command-line binary with no UI. Every assertion +is made at its CLI boundary: exit code, stdout, and the filesystem state it +did or did not change. There is no browser surface. The one native concern +(launching a real `.app` with an isolated `HOME`) is covered under *Not +verifiable here*, because it requires quitting applications the operator is +using. + +## Setup + +- Repository: this repo (`documentnode/termtree`). The harness is a + standalone `[workspace]` crate at `benchmark/` with no path dependencies — + it needs no other repo checked out. +- Dependencies: Rust stable, plus the nightly toolchain for the format check. + The macOS tools it shells out to — `lsappinfo`, `footprint`, `vm_stat`, + `sysctl`, `pmset`, `notifyutil`, `open` — ship with a stock macOS and none + need `sudo`. +- macOS only. `open --env` must be supported; `doctor` probes for it. + +```bash +cd benchmark +cargo build --release +B=./target/release/resource-benchmark +``` + +**You no longer need to export a scratch `HOME`.** Earlier versions of this +harness read your real `$HOME` and required you to remember to override it. +It now creates a fresh disposable home under the OS temp directory for every +invocation, and **refuses** an explicit `--home` that points at your real home +directory. Scenario 3 proves both halves. + +## Automated checks + +Run before the manual pass. They complement it; they do not replace it. + +```bash +cd benchmark +cargo test # expect: all pass, exit 0 +cargo clippy --all-targets -- -D warnings # expect: exit 0 +cargo +nightly fmt -- --check # expect: no output, exit 0 +``` + +Nightly is required for the format check — `rustfmt.toml` sets +`unstable_features = true`. + +## Manual testing + +### Scenario 1 — `doctor` names every unmet prerequisite and its remedy + +1. Run `$B doctor`. +2. Read every line. For each unmet prerequisite, confirm the output says what + to do about it, not only what is wrong: + - a not-installed subject names the expected path **and** the + `--bundle-path =` override, e.g. + `subject not installed: Collaborator (/Applications/Collaborator.app) -- + install it there, or point the harness at an existing install with + `--bundle-path collaborator=`` + - a failing quiesce gate is followed by one indented `to clear :` + line per failing signal, each naming a concrete command or action. +3. Confirm the exit code is `1` when any problem was printed: + `$B doctor; echo "exit=$?"`. With no problems it prints + `doctor: all checks passed.` and exits `0`. +4. **Confirm it changed nothing.** `doctor` creates and deletes a scratch-home + probe directory. Afterwards, `ls $TMPDIR | grep resource-benchmark-doctor-probe` + must return nothing. + +### Scenario 2 — `doctor` detects an already-running subject by bundle identifier + +1. Launch TermTree normally (or any subject in the registry). +2. Run `$B doctor`. +3. Expect a line naming the app, its **bundle identifier**, and its pid: + `TermTree (com.termtree.desktop) is already running (pid NNNNN) -- quit it + before running the benchmark`. +4. Confirm the pid matches: `lsappinfo list | grep -A5 TermTree | grep 'pid ='`. + +Bundle identifier rather than app name is the point. Two differently named +bundles can declare the same identifier; the second launch is then handed off +to the running instance by the single-instance plugin and exits within +seconds, having measured nothing. `open -n` does not bypass that. + +### Scenario 3 — the scratch home is disposable, and the real profile is never touched + +This is the guide's most important scenario. + +1. Record your real profile's state before anything: + ```bash + REAL="$HOME/Library/Application Support/DocumentNode/TermTree" + ls "$REAL" | grep -c before-resource-benchmark # expect: 0 + ``` +2. Seed with **no** `--home`: + ```bash + $B seed --subject termtree --sessions 3 + ``` +3. Expect exit `0` and a message naming the scratch home it chose, e.g. + `seeded termtree with 3 sessions via production state.json pre-write + (scratch home: /var/folders/.../resource-benchmark-home--)`. +4. Confirm the seeded fixture landed **there**, not in your profile: + ```bash + S= + python3 -c "import json;print(json.load(open('$S/Library/Application Support/DocumentNode/TermTree/state.json'))['tree']['label'])" + ``` + Expect `resource-benchmark-root`. +5. Confirm your real profile was not written: + ```bash + ls "$REAL" | grep -c before-resource-benchmark # expect: still 0 + ``` + The seeder **always** writes a `state.json.before-resource-benchmark.json` + backup before touching a state file, so the absence of that file is proof + it never wrote there. + + Do **not** use the real `state.json`'s checksum for this check. If TermTree + is running it rewrites its own state continuously, so the hash changes for + reasons that have nothing to do with the harness. Check for the backup + file, and check the tree's root label is still yours. + +6. Now point `--home` at your real home. It must **refuse**: + ```bash + $B seed --subject termtree --sessions 3 --home "$HOME"; echo "exit=$?" + ``` + Expect exit `1` and + `Refusing to use /Users/ as the scratch home: it is the real home + directory (...) or contains it, so seeding would overwrite the runner's own + application profiles.` +7. Same via the environment variable: + ```bash + RESOURCE_BENCHMARK_HOME="$HOME" $B seed --subject termtree --sessions 3; echo "exit=$?" + ``` + Expect exit `1`. +8. Confirm it is not over-blocking — an explicit scratch directory still works: + ```bash + S=$(mktemp -d); $B seed --subject termtree --sessions 2 --home "$S"; echo "exit=$?" + find "$S" -name state.json + ``` + Expect exit `0` and one `state.json` under `$S`. + +### Scenario 4 — `seed` backs up an existing profile and `restore` returns it exactly + +1. Build a scratch home with a state file standing in for a real one: + ```bash + S=$(mktemp -d); D="$S/Library/Application Support/DocumentNode/TermTree" + mkdir -p "$D" + printf '{"tree":{"id":"original-root","label":"my real work","children":[]},"themeKey":"dark"}' > "$D/state.json" + ORIG=$(shasum -a 256 "$D/state.json" | cut -d' ' -f1) + ``` +2. `$B seed --subject termtree --sessions 4 --home "$S"` +3. Confirm both files now exist: `ls "$D"` shows `state.json` and + `state.json.before-resource-benchmark.json`. +4. Confirm the live file is the fixture: its `tree.label` is + `resource-benchmark-root`. +5. `$B restore --subject termtree --home "$S"` — expect `restored termtree`. +6. Confirm byte-identical restoration: + ```bash + [ "$ORIG" = "$(shasum -a 256 "$D/state.json" | cut -d' ' -f1)" ] && echo PASS || echo FAIL + ``` +7. Confirm the backup was consumed: `ls "$D" | grep -c before-resource-benchmark` + returns `0`. +8. `rm -rf "$S"`. + +### Scenario 5 — the seeder refuses a target outside its scratch root + +Covered by `cargo test`'s `refuses_a_directory_outside_the_scratch_root` and +`refuses_a_termtreedev_directory_even_under_the_scratch_root`. There is no +safe manual equivalent: the manual version would require pointing the seeder +at a real profile, which Scenario 3 step 6 now refuses outright. + +### Scenario 6 — `run` refuses on an unquiesced machine and measures nothing + +1. On an ordinary working machine (browser open, apps running), run `$B run`. +2. Expect exit `1` and + `refused to start: quiesce gate failed, refusing to start: `. +3. Confirm nothing was produced: `ls results/` is unchanged. +4. Confirm no subject was launched **or quit** — any app that was running + before is still running: + `lsappinfo list | grep -c com.termtree.desktop` is unchanged. + +The preflight order is quiesce gate → `open --env` support → already-running +check, and every one of them returns before any seeding, launching, or +teardown. Teardown issues a graceful quit, so it must never be reachable for a +process the harness did not itself launch. + +### Scenario 7 — bundle-path override, version drift, and the unverified-seeder note + +1. Point a subject at an app that exists but is the wrong one, to exercise all + three behaviours at once: + ```bash + $B doctor --bundle-path 'codenomad-electron=/Applications/.app' + ``` +2. Expect the "subject not installed" line for that subject to **disappear** — + the override was honoured. +3. Expect a version-drift line naming both versions, e.g. + `CodeNomad (Electron): version drift, expected 0.18.0 found 2.2.0`. +4. Expect a `doctor note:` line stating that subject's seeder has not been + verified against a real install and that its N-session/sustained-use + samples will report `invalidReason=seed-format-unverified`. + +Note the notes are only reachable for an **installed** subject; a +not-installed subject short-circuits before them, which is why this scenario +uses an override to make one reachable. + +### Scenario 8 — unverified seed formats are reported, never silently counted + +Collaborator, CodeNomad and diri have seeders whose formats have never been +checked against a real install. Confirm the harness says so rather than +producing a number that looks valid: + +1. `grep -rn 'seed_format_verified' src/subject.rs` — only `termtree` is + `true`. +2. Their N-session and sustained-use samples carry + `invalidReason: "seed-format-unverified"`, asserted by `cargo test`. +3. `$B doctor` emits the note from Scenario 7 for each installed one. + +## Not verifiable here + +- **A real measurement sweep.** Requires a rebooted machine at nominal memory + and thermal pressure, no swap in use, on AC power, with exclusive use for + hours. `$B doctor` must pass first. On a 16 GB host, never run two subjects + concurrently. +- **Launch isolation against a real application** — that + `open -n -F --env HOME= -a ` genuinely redirects an app's + data directory. Verifying it means launching a subject, which requires that + subject to be quit first. On a working machine the operator's own TermTree + and MarkNode are typically running, and launching either triggers the + single-instance handoff described in Scenario 2. **Precondition to verify:** + a quiesced machine with the subject quit — i.e. the same sitting as the + sweep. The mechanism was confirmed manually against MarkNode on 2026-08-25: + `lsappinfo` attribution still resolved under a scratch `HOME`, the app used + the scratch data directory, and the real profile's mtime was unchanged. +- **Cold-start log-mark self-validation** (`termtree-log-marks-unrecognized`) + and **`app-data-dir-not-created`**. Both fire only after a real TermTree + launch, so they share the precondition above. Their pure logic is unit + tested. +- **The three unverified seeders against real installs.** Collaborator, + CodeNomad (Electron and Tauri) and diri must be installed at the pinned + versions first. +- **The spawn-and-wait orchestration path** + (`build_envelope` / `measure_one` / `measure_cold_start` / `teardown`) has + never executed end to end. Treat the first sweep as its verification and + budget it as debugging. + +## Cleanup + +- Scratch homes are **not** removed automatically. They accumulate under the + OS temp directory as `resource-benchmark-home--` until the OS + reclaims them. To clear them now: + `rm -rf "$TMPDIR"/resource-benchmark-home-*` +- Remove any scratch directory you created explicitly with `--home`. +- If a seeding scenario was interrupted between `seed` and `restore`, run + `$B restore --subject --home `. `$B doctor --home ` + reports a leftover backup; it only performs that check for an explicitly + supplied home, since a fresh default home can never have one. +- Never run `restore` against your real home — `--home "$HOME"` is refused. + +## Agent execution notes + +- Capture exit codes directly (`cmd; echo "exit=$?"`), never through a pipe — + `cmd | tail` reports the exit code of `tail`, so a failure reads as `0`. +- Do not assert on the real `state.json`'s checksum while TermTree is running; + it rewrites its own state. Assert on the absence of + `state.json.before-resource-benchmark.json` and on the tree's root label. +- Do not launch a subject to test isolation while the operator is working. Check + `lsappinfo list | grep bundleID=` first; if the subject or anything sharing + its bundle identifier is running, record the scenario as blocked rather than + quitting the operator's application. +- `$B run` is safe to invoke on an unquiesced machine — it refuses in preflight + before touching anything — but do not add `--allow-*` flags to force past a + refusal during verification. diff --git a/benchmark/fixtures/footprint-dead-pid.json b/benchmark/fixtures/footprint-dead-pid.json new file mode 100644 index 0000000..b364001 --- /dev/null +++ b/benchmark/fixtures/footprint-dead-pid.json @@ -0,0 +1,762 @@ +{ + "unit": "byte", + "bytes per unit": 1, + "vm_object_dirty_analysis": true, + "start_time": { + "mach_absolute_time_ns": 2286608523281583, + "mach_continuous_time_ns": 2501731265061583, + "wall_time_s": 809258438.587373, + "date": "2026-08-24T18:00:38.587+08:00" + }, + "processes": [ + { + "name": "com.apple.WebKit.WebContent", + "pid": 64305, + "translated": false, + "page size": 16384, + "footprint": 1607504896, + "categories": { + "WebKit malloc": { + "dirty": 1580826624, + "swapped": 1570258944, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 70 + }, + "JS JIT generated code": { + "dirty": 2375680, + "swapped": 2310144, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 3 + }, + "dyld private memory": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "page table": { + "dirty": 1332224, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + } + }, + "auxiliary": { + "phys_footprint_peak": 1613648896, + "phys_footprint": 1607537664 + } + }, + { + "name": "com.apple.WebKit.WebContent", + "pid": 83276, + "translated": false, + "page size": 16384, + "footprint": 429724224, + "categories": { + "WebKit malloc": { + "dirty": 411336704, + "swapped": 403210240, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 61 + }, + "__TPRO_CONST": { + "dirty": 65528, + "swapped": 32768, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + }, + "dyld private memory": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "page table": { + "dirty": 1217088, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + } + }, + "auxiliary": { + "phys_footprint_peak": 1418318400, + "phys_footprint": 429756992 + } + }, + { + "name": "termtree", + "pid": 56070, + "translated": false, + "page size": 16384, + "footprint": 83923136, + "categories": { + "IOAccelerator": { + "dirty": 49152, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "__DATA": { + "dirty": 2823647, + "swapped": 1995120, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 340 + }, + "WebKit malloc": { + "dirty": 4096000, + "swapped": 1736704, + "clean": 0, + "reclaimable": 245760, + "wired": 0, + "regions": 36 + }, + "MALLOC_MEDIUM": { + "dirty": 10764288, + "swapped": 7520256, + "clean": 0, + "reclaimable": 409600, + "wired": 0, + "regions": 9 + } + }, + "auxiliary": { + "phys_footprint_peak": 98177088, + "phys_footprint": 83955904 + } + }, + { + "name": "com.apple.WebKit.GPU", + "pid": 56072, + "translated": false, + "page size": 16384, + "footprint": 69192832, + "categories": { + "IOAccelerator": { + "dirty": 393216, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 226 + }, + "__DATA": { + "dirty": 1700928, + "swapped": 1279437, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 151 + }, + "WebKit malloc": { + "dirty": 4079616, + "swapped": 2637824, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 13 + }, + "Activity Tracing": { + "dirty": 49152, + "swapped": 49152, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + } + }, + "auxiliary": { + "phys_footprint_peak": 173886848, + "phys_footprint": 69225600 + } + } + ], + "errors": [], + "warnings": [], + "summary": { + "IOAccelerator": { + "dirty": 458752, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 229 + }, + "__GLSLBUILTINS": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "__DATA": { + "dirty": 8988369, + "swapped": 6827279, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 3197 + }, + "WebKit malloc": { + "dirty": 2003140608, + "swapped": 1978695680, + "clean": 0, + "reclaimable": 344064, + "wired": 0, + "regions": 196 + }, + "Activity Tracing": { + "dirty": 229376, + "swapped": 65536, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "app-specific tag 1": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 4 + }, + "CG image": { + "dirty": 688128, + "swapped": 688128, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 34 + }, + "ImageIO": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 688128, + "wired": 0, + "regions": 84 + }, + "MALLOC_MEDIUM": { + "dirty": 10764288, + "swapped": 7520256, + "clean": 0, + "reclaimable": 409600, + "wired": 0, + "regions": 9 + }, + "MALLOC_NANO": { + "dirty": 14581760, + "swapped": 7733248, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "__DATA_DIRTY": { + "dirty": 3668031, + "swapped": 2709207, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1100 + }, + "MALLOC_LARGE": { + "dirty": 32768, + "swapped": 32768, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + }, + "Accelerate image backing stores": { + "dirty": 917504, + "swapped": 917504, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 7 + }, + "stack": { + "dirty": 4702208, + "swapped": 2064384, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 251 + }, + "CoreUI image data": { + "dirty": 1802240, + "swapped": 1802240, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 13 + }, + "__AUTH": { + "dirty": 2507860, + "swapped": 2113712, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2287 + }, + "libdispatch": { + "dirty": 1425408, + "swapped": 458752, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 4 + }, + "app-specific tag 2": { + "dirty": 245760, + "swapped": 245760, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 6 + }, + "__FONT_DATA": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "page table": { + "dirty": 4868288, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "__LINKEDIT": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 20 + }, + "libnetwork": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 40 + }, + "dyld private memory": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 4 + }, + "CoreImage": { + "dirty": 16384, + "swapped": 16384, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "IOSurface": { + "dirty": 15941632, + "swapped": 15941632, + "clean": 0, + "reclaimable": 1005748224, + "wired": 0, + "regions": 581 + }, + "CoreServices": { + "dirty": 49152, + "swapped": 49152, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 3 + }, + "skywalk": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "IOAccelerator (graphics)": { + "dirty": 21757952, + "swapped": 9043968, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 281 + }, + "mapped file": { + "dirty": 32768, + "swapped": 32768, + "clean": 376832, + "reclaimable": 0, + "wired": 0, + "regions": 180 + }, + "commpage (reserved)": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "JS VM Gigacage": { + "dirty": 131072, + "swapped": 131072, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1026 + }, + "MALLOC metadata": { + "dirty": 1900544, + "swapped": 1032192, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 70 + }, + "MALLOC_SMALL": { + "dirty": 58327040, + "swapped": 42926080, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2897 + }, + "SQLite page cache": { + "dirty": 147456, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 6 + }, + "untagged (VM_ALLOCATE)": { + "dirty": 6586368, + "swapped": 6307840, + "clean": 0, + "reclaimable": 0, + "wired": 81920, + "regions": 523 + }, + "os_alloc_once": { + "dirty": 81920, + "swapped": 81920, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "__DATA_CONST": { + "dirty": 2097152, + "swapped": 1736704, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 3448 + }, + "unused dyld shared cache area": { + "dirty": 1875140, + "swapped": 1555306, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5602 + }, + "Foundation": { + "dirty": 16384, + "swapped": 16384, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + }, + "__INFO_FILTER": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + }, + "__CTF": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "__TEXT": { + "dirty": 0, + "swapped": 0, + "clean": 6733824, + "reclaimable": 0, + "wired": 0, + "regions": 3552 + }, + "GPU Carveout (reserved)": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "MALLOC_TINY": { + "dirty": 17596416, + "swapped": 11059200, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 29 + }, + "__AUTH_CONST": { + "dirty": 196608, + "swapped": 163840, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 3408 + }, + "ColorSync": { + "dirty": 1245184, + "swapped": 1245184, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 61 + }, + "CoreGraphics": { + "dirty": 81920, + "swapped": 49152, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "JS JIT generated code": { + "dirty": 4292608, + "swapped": 4161536, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 6 + }, + "IOKit": { + "dirty": 114688, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 15 + }, + "CoreAnimation": { + "dirty": 1703936, + "swapped": 1703936, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 132 + }, + "Owned physical footprint (unmapped)": { + "dirty": 6782976, + "swapped": 491520, + "clean": 0, + "reclaimable": 0, + "wired": 6291456, + "regions": 93 + }, + "__TPRO_CONST": { + "dirty": 262104, + "swapped": 81920, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 10 + }, + "total": { + "dirty": 2200258752, + "swapped": 2109702144, + "clean": 7110656, + "reclaimable": 1007190016, + "wired": 6373376, + "regions": 29462 + } + }, + "total footprint": 2190145088, + "shared": [ + { + "pids": [ + 56070, + 56072, + 64305, + 83276 + ], + "specific_to_pid": 64305, + "categories": { + "mapped file": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + } + } + }, + { + "pids": [ + 64305, + 56070, + 56072 + ], + "specific_to_pid": 64305, + "categories": { + "mapped file": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "IOSurface": { + "dirty": 11993088, + "swapped": 11993088, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 9 + } + } + }, + { + "pids": [ + 64305, + 83276, + 56072 + ], + "specific_to_pid": 64305, + "categories": { + "mapped file": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + } + } + }, + { + "pids": [ + 64305, + 56070, + 83276 + ], + "specific_to_pid": 64305, + "categories": { + "mapped file": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + } + } + }, + { + "pids": [ + 64305, + 83276 + ], + "specific_to_pid": 64305, + "categories": { + "mapped file": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "__TEXT": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + }, + "__LINKEDIT": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + } + } + }, + { + "pids": [ + 64305, + 56072 + ], + "specific_to_pid": 64305, + "categories": { + "untagged (VM_ALLOCATE)": { + "dirty": 2113536, + "swapped": 2080768, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + } + } + } + ], + "end_time": { + "mach_absolute_time_ns": 2286609191689625, + "mach_continuous_time_ns": 2501731933469625, + "wall_time_s": 809258439.255786, + "date": "2026-08-24T18:00:39.256+08:00" + }, + "page size": 16384 +} \ No newline at end of file diff --git a/benchmark/fixtures/footprint-shared-set.json b/benchmark/fixtures/footprint-shared-set.json new file mode 100644 index 0000000..4a9b240 --- /dev/null +++ b/benchmark/fixtures/footprint-shared-set.json @@ -0,0 +1,700 @@ +{ + "unit": "byte", + "bytes per unit": 1, + "vm_object_dirty_analysis": true, + "start_time": { + "mach_absolute_time_ns": 2286608523281583, + "mach_continuous_time_ns": 2501731265061583, + "wall_time_s": 809258438.587373, + "date": "2026-08-24T18:00:38.587+08:00" + }, + "processes": [ + { + "name": "com.apple.WebKit.WebContent", + "pid": 83276, + "translated": false, + "page size": 16384, + "footprint": 429724224, + "categories": { + "WebKit malloc": { + "dirty": 411336704, + "swapped": 403210240, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 61 + }, + "__TPRO_CONST": { + "dirty": 65528, + "swapped": 32768, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + }, + "dyld private memory": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "page table": { + "dirty": 1217088, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + } + }, + "auxiliary": { + "phys_footprint_peak": 1418318400, + "phys_footprint": 429756992 + } + }, + { + "name": "com.apple.WebKit.GPU", + "pid": 56072, + "translated": false, + "page size": 16384, + "footprint": 69192832, + "categories": { + "IOAccelerator": { + "dirty": 393216, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 226 + }, + "__DATA": { + "dirty": 1700928, + "swapped": 1279437, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 151 + }, + "WebKit malloc": { + "dirty": 4079616, + "swapped": 2637824, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 13 + }, + "Activity Tracing": { + "dirty": 49152, + "swapped": 49152, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + } + }, + "auxiliary": { + "phys_footprint_peak": 173886848, + "phys_footprint": 69225600 + } + }, + { + "name": "com.apple.WebKit.Networking", + "pid": 56074, + "translated": false, + "page size": 16384, + "footprint": 10224960, + "auxiliary": { + "phys_footprint_peak": 22529408, + "phys_footprint": 10257728 + } + } + ], + "errors": [], + "warnings": [], + "summary": { + "IOAccelerator": { + "dirty": 458752, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 229 + }, + "__GLSLBUILTINS": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "__DATA": { + "dirty": 8988369, + "swapped": 6827279, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 3197 + }, + "WebKit malloc": { + "dirty": 2003140608, + "swapped": 1978695680, + "clean": 0, + "reclaimable": 344064, + "wired": 0, + "regions": 196 + }, + "Activity Tracing": { + "dirty": 229376, + "swapped": 65536, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "app-specific tag 1": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 4 + }, + "CG image": { + "dirty": 688128, + "swapped": 688128, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 34 + }, + "ImageIO": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 688128, + "wired": 0, + "regions": 84 + }, + "MALLOC_MEDIUM": { + "dirty": 10764288, + "swapped": 7520256, + "clean": 0, + "reclaimable": 409600, + "wired": 0, + "regions": 9 + }, + "MALLOC_NANO": { + "dirty": 14581760, + "swapped": 7733248, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "__DATA_DIRTY": { + "dirty": 3668031, + "swapped": 2709207, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1100 + }, + "MALLOC_LARGE": { + "dirty": 32768, + "swapped": 32768, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + }, + "Accelerate image backing stores": { + "dirty": 917504, + "swapped": 917504, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 7 + }, + "stack": { + "dirty": 4702208, + "swapped": 2064384, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 251 + }, + "CoreUI image data": { + "dirty": 1802240, + "swapped": 1802240, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 13 + }, + "__AUTH": { + "dirty": 2507860, + "swapped": 2113712, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2287 + }, + "libdispatch": { + "dirty": 1425408, + "swapped": 458752, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 4 + }, + "app-specific tag 2": { + "dirty": 245760, + "swapped": 245760, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 6 + }, + "__FONT_DATA": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "page table": { + "dirty": 4868288, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "__LINKEDIT": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 20 + }, + "libnetwork": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 40 + }, + "dyld private memory": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 4 + }, + "CoreImage": { + "dirty": 16384, + "swapped": 16384, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "IOSurface": { + "dirty": 15941632, + "swapped": 15941632, + "clean": 0, + "reclaimable": 1005748224, + "wired": 0, + "regions": 581 + }, + "CoreServices": { + "dirty": 49152, + "swapped": 49152, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 3 + }, + "skywalk": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "IOAccelerator (graphics)": { + "dirty": 21757952, + "swapped": 9043968, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 281 + }, + "mapped file": { + "dirty": 32768, + "swapped": 32768, + "clean": 376832, + "reclaimable": 0, + "wired": 0, + "regions": 180 + }, + "commpage (reserved)": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "JS VM Gigacage": { + "dirty": 131072, + "swapped": 131072, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1026 + }, + "MALLOC metadata": { + "dirty": 1900544, + "swapped": 1032192, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 70 + }, + "MALLOC_SMALL": { + "dirty": 58327040, + "swapped": 42926080, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2897 + }, + "SQLite page cache": { + "dirty": 147456, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 6 + }, + "untagged (VM_ALLOCATE)": { + "dirty": 6586368, + "swapped": 6307840, + "clean": 0, + "reclaimable": 0, + "wired": 81920, + "regions": 523 + }, + "os_alloc_once": { + "dirty": 81920, + "swapped": 81920, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "__DATA_CONST": { + "dirty": 2097152, + "swapped": 1736704, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 3448 + }, + "unused dyld shared cache area": { + "dirty": 1875140, + "swapped": 1555306, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5602 + }, + "Foundation": { + "dirty": 16384, + "swapped": 16384, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + }, + "__INFO_FILTER": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + }, + "__CTF": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "__TEXT": { + "dirty": 0, + "swapped": 0, + "clean": 6733824, + "reclaimable": 0, + "wired": 0, + "regions": 3552 + }, + "GPU Carveout (reserved)": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "MALLOC_TINY": { + "dirty": 17596416, + "swapped": 11059200, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 29 + }, + "__AUTH_CONST": { + "dirty": 196608, + "swapped": 163840, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 3408 + }, + "ColorSync": { + "dirty": 1245184, + "swapped": 1245184, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 61 + }, + "CoreGraphics": { + "dirty": 81920, + "swapped": 49152, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "JS JIT generated code": { + "dirty": 4292608, + "swapped": 4161536, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 6 + }, + "IOKit": { + "dirty": 114688, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 15 + }, + "CoreAnimation": { + "dirty": 1703936, + "swapped": 1703936, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 132 + }, + "Owned physical footprint (unmapped)": { + "dirty": 6782976, + "swapped": 491520, + "clean": 0, + "reclaimable": 0, + "wired": 6291456, + "regions": 93 + }, + "__TPRO_CONST": { + "dirty": 262104, + "swapped": 81920, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 10 + }, + "total": { + "dirty": 2200258752, + "swapped": 2109702144, + "clean": 7110656, + "reclaimable": 1007190016, + "wired": 6373376, + "regions": 29462 + } + }, + "total footprint": 509109248, + "shared": [ + { + "pids": [ + 56070, + 56072, + 64305, + 83276 + ], + "specific_to_pid": 64305, + "categories": { + "mapped file": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + } + } + }, + { + "pids": [ + 64305, + 56070, + 56072 + ], + "specific_to_pid": 64305, + "categories": { + "mapped file": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "IOSurface": { + "dirty": 11993088, + "swapped": 11993088, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 9 + } + } + }, + { + "pids": [ + 64305, + 83276, + 56072 + ], + "specific_to_pid": 64305, + "categories": { + "mapped file": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + } + } + }, + { + "pids": [ + 64305, + 56070, + 83276 + ], + "specific_to_pid": 64305, + "categories": { + "mapped file": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + } + } + }, + { + "pids": [ + 64305, + 83276 + ], + "specific_to_pid": 64305, + "categories": { + "mapped file": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "__TEXT": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + }, + "__LINKEDIT": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + } + } + }, + { + "pids": [ + 64305, + 56072 + ], + "specific_to_pid": 64305, + "categories": { + "untagged (VM_ALLOCATE)": { + "dirty": 2113536, + "swapped": 2080768, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + } + } + }, + { + "pids": [ + 56072, + 56074 + ], + "shared-cache": true, + "categories": { + "dyld shared cache": { + "dirty": 0, + "swapped": 0, + "clean": 4194304, + "reclaimable": 0, + "wired": 0, + "regions": 12 + } + } + } + ], + "end_time": { + "mach_absolute_time_ns": 2286609191689625, + "mach_continuous_time_ns": 2501731933469625, + "wall_time_s": 809258439.255786, + "date": "2026-08-24T18:00:39.256+08:00" + }, + "page size": 16384 +} \ No newline at end of file diff --git a/benchmark/fixtures/footprint-termtree-6proc.json b/benchmark/fixtures/footprint-termtree-6proc.json new file mode 100644 index 0000000..48f6f82 --- /dev/null +++ b/benchmark/fixtures/footprint-termtree-6proc.json @@ -0,0 +1,860 @@ +{ + "unit": "byte", + "bytes per unit": 1, + "vm_object_dirty_analysis": true, + "start_time": { + "mach_absolute_time_ns": 2286608523281583, + "mach_continuous_time_ns": 2501731265061583, + "wall_time_s": 809258438.587373, + "date": "2026-08-24T18:00:38.587+08:00" + }, + "processes": [ + { + "name": "com.apple.WebKit.WebContent", + "pid": 64305, + "translated": false, + "page size": 16384, + "footprint": 1607504896, + "categories": { + "WebKit malloc": { + "dirty": 1580826624, + "swapped": 1570258944, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 70 + }, + "JS JIT generated code": { + "dirty": 2375680, + "swapped": 2310144, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 3 + }, + "dyld private memory": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "page table": { + "dirty": 1332224, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + } + }, + "auxiliary": { + "phys_footprint_peak": 1613648896, + "phys_footprint": 1607537664 + } + }, + { + "name": "com.apple.WebKit.WebContent", + "pid": 83276, + "translated": false, + "page size": 16384, + "footprint": 429724224, + "categories": { + "WebKit malloc": { + "dirty": 411336704, + "swapped": 403210240, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 61 + }, + "__TPRO_CONST": { + "dirty": 65528, + "swapped": 32768, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + }, + "dyld private memory": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "page table": { + "dirty": 1217088, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + } + }, + "auxiliary": { + "phys_footprint_peak": 1418318400, + "phys_footprint": 429756992 + } + }, + { + "name": "termtree", + "pid": 56070, + "translated": false, + "page size": 16384, + "footprint": 83923136, + "categories": { + "IOAccelerator": { + "dirty": 49152, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "__DATA": { + "dirty": 2823647, + "swapped": 1995120, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 340 + }, + "WebKit malloc": { + "dirty": 4096000, + "swapped": 1736704, + "clean": 0, + "reclaimable": 245760, + "wired": 0, + "regions": 36 + }, + "MALLOC_MEDIUM": { + "dirty": 10764288, + "swapped": 7520256, + "clean": 0, + "reclaimable": 409600, + "wired": 0, + "regions": 9 + } + }, + "auxiliary": { + "phys_footprint_peak": 98177088, + "phys_footprint": 83955904 + } + }, + { + "name": "com.apple.WebKit.GPU", + "pid": 56072, + "translated": false, + "page size": 16384, + "footprint": 69192832, + "categories": { + "IOAccelerator": { + "dirty": 393216, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 226 + }, + "__DATA": { + "dirty": 1700928, + "swapped": 1279437, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 151 + }, + "WebKit malloc": { + "dirty": 4079616, + "swapped": 2637824, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 13 + }, + "Activity Tracing": { + "dirty": 49152, + "swapped": 49152, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + } + }, + "auxiliary": { + "phys_footprint_peak": 173886848, + "phys_footprint": 69225600 + } + }, + { + "name": "com.apple.WebKit.Networking", + "pid": 56074, + "translated": false, + "page size": 16384, + "footprint": 10224960, + "categories": { + "WebKit malloc": { + "dirty": 2801664, + "swapped": 851968, + "clean": 0, + "reclaimable": 98304, + "wired": 0, + "regions": 16 + }, + "dyld private memory": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "app-specific tag 1": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "page table": { + "dirty": 361792, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + } + }, + "auxiliary": { + "phys_footprint_peak": 22529408, + "phys_footprint": 10257728 + } + } + ], + "errors": [], + "warnings": [], + "summary": { + "IOAccelerator": { + "dirty": 458752, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 229 + }, + "__GLSLBUILTINS": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "__DATA": { + "dirty": 8988369, + "swapped": 6827279, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 3197 + }, + "WebKit malloc": { + "dirty": 2003140608, + "swapped": 1978695680, + "clean": 0, + "reclaimable": 344064, + "wired": 0, + "regions": 196 + }, + "Activity Tracing": { + "dirty": 229376, + "swapped": 65536, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "app-specific tag 1": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 4 + }, + "CG image": { + "dirty": 688128, + "swapped": 688128, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 34 + }, + "ImageIO": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 688128, + "wired": 0, + "regions": 84 + }, + "MALLOC_MEDIUM": { + "dirty": 10764288, + "swapped": 7520256, + "clean": 0, + "reclaimable": 409600, + "wired": 0, + "regions": 9 + }, + "MALLOC_NANO": { + "dirty": 14581760, + "swapped": 7733248, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "__DATA_DIRTY": { + "dirty": 3668031, + "swapped": 2709207, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1100 + }, + "MALLOC_LARGE": { + "dirty": 32768, + "swapped": 32768, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + }, + "Accelerate image backing stores": { + "dirty": 917504, + "swapped": 917504, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 7 + }, + "stack": { + "dirty": 4702208, + "swapped": 2064384, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 251 + }, + "CoreUI image data": { + "dirty": 1802240, + "swapped": 1802240, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 13 + }, + "__AUTH": { + "dirty": 2507860, + "swapped": 2113712, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2287 + }, + "libdispatch": { + "dirty": 1425408, + "swapped": 458752, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 4 + }, + "app-specific tag 2": { + "dirty": 245760, + "swapped": 245760, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 6 + }, + "__FONT_DATA": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "page table": { + "dirty": 4868288, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "__LINKEDIT": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 20 + }, + "libnetwork": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 40 + }, + "dyld private memory": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 4 + }, + "CoreImage": { + "dirty": 16384, + "swapped": 16384, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "IOSurface": { + "dirty": 15941632, + "swapped": 15941632, + "clean": 0, + "reclaimable": 1005748224, + "wired": 0, + "regions": 581 + }, + "CoreServices": { + "dirty": 49152, + "swapped": 49152, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 3 + }, + "skywalk": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "IOAccelerator (graphics)": { + "dirty": 21757952, + "swapped": 9043968, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 281 + }, + "mapped file": { + "dirty": 32768, + "swapped": 32768, + "clean": 376832, + "reclaimable": 0, + "wired": 0, + "regions": 180 + }, + "commpage (reserved)": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "JS VM Gigacage": { + "dirty": 131072, + "swapped": 131072, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1026 + }, + "MALLOC metadata": { + "dirty": 1900544, + "swapped": 1032192, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 70 + }, + "MALLOC_SMALL": { + "dirty": 58327040, + "swapped": 42926080, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2897 + }, + "SQLite page cache": { + "dirty": 147456, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 6 + }, + "untagged (VM_ALLOCATE)": { + "dirty": 6586368, + "swapped": 6307840, + "clean": 0, + "reclaimable": 0, + "wired": 81920, + "regions": 523 + }, + "os_alloc_once": { + "dirty": 81920, + "swapped": 81920, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "__DATA_CONST": { + "dirty": 2097152, + "swapped": 1736704, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 3448 + }, + "unused dyld shared cache area": { + "dirty": 1875140, + "swapped": 1555306, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5602 + }, + "Foundation": { + "dirty": 16384, + "swapped": 16384, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + }, + "__INFO_FILTER": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + }, + "__CTF": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "__TEXT": { + "dirty": 0, + "swapped": 0, + "clean": 6733824, + "reclaimable": 0, + "wired": 0, + "regions": 3552 + }, + "GPU Carveout (reserved)": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "MALLOC_TINY": { + "dirty": 17596416, + "swapped": 11059200, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 29 + }, + "__AUTH_CONST": { + "dirty": 196608, + "swapped": 163840, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 3408 + }, + "ColorSync": { + "dirty": 1245184, + "swapped": 1245184, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 61 + }, + "CoreGraphics": { + "dirty": 81920, + "swapped": 49152, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + }, + "JS JIT generated code": { + "dirty": 4292608, + "swapped": 4161536, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 6 + }, + "IOKit": { + "dirty": 114688, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 15 + }, + "CoreAnimation": { + "dirty": 1703936, + "swapped": 1703936, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 132 + }, + "Owned physical footprint (unmapped)": { + "dirty": 6782976, + "swapped": 491520, + "clean": 0, + "reclaimable": 0, + "wired": 6291456, + "regions": 93 + }, + "__TPRO_CONST": { + "dirty": 262104, + "swapped": 81920, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 10 + }, + "total": { + "dirty": 2200258752, + "swapped": 2109702144, + "clean": 7110656, + "reclaimable": 1007190016, + "wired": 6373376, + "regions": 29462 + } + }, + "total footprint": 2200258752, + "shared": [ + { + "pids": [ + 56070, + 56072, + 64305, + 83276 + ], + "specific_to_pid": 64305, + "categories": { + "mapped file": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 5 + } + } + }, + { + "pids": [ + 64305, + 56070, + 56072 + ], + "specific_to_pid": 64305, + "categories": { + "mapped file": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "IOSurface": { + "dirty": 11993088, + "swapped": 11993088, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 9 + } + } + }, + { + "pids": [ + 64305, + 83276, + 56072 + ], + "specific_to_pid": 64305, + "categories": { + "mapped file": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + } + } + }, + { + "pids": [ + 64305, + 56070, + 83276 + ], + "specific_to_pid": 64305, + "categories": { + "mapped file": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + } + } + }, + { + "pids": [ + 64305, + 83276 + ], + "specific_to_pid": 64305, + "categories": { + "mapped file": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + }, + "__TEXT": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + }, + "__LINKEDIT": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 2 + } + } + }, + { + "pids": [ + 64305, + 56072 + ], + "specific_to_pid": 64305, + "categories": { + "untagged (VM_ALLOCATE)": { + "dirty": 2113536, + "swapped": 2080768, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + } + } + }, + { + "pids": [ + 56070, + 56072, + 64305, + 83276, + 56074 + ], + "specific_to_pid": 64305, + "categories": { + "mapped file": { + "dirty": 0, + "swapped": 0, + "clean": 98304, + "reclaimable": 0, + "wired": 0, + "regions": 7 + }, + "untagged (VM_ALLOCATE)": { + "dirty": 65536, + "swapped": 49152, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 3 + } + } + }, + { + "pids": [ + 83276, + 56072 + ], + "specific_to_pid": 83276, + "categories": { + "IOSurface": { + "dirty": 0, + "swapped": 0, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 12 + }, + "untagged (VM_ALLOCATE)": { + "dirty": 2113536, + "swapped": 2113536, + "clean": 0, + "reclaimable": 0, + "wired": 0, + "regions": 1 + } + } + } + ], + "end_time": { + "mach_absolute_time_ns": 2286609191689625, + "mach_continuous_time_ns": 2501731933469625, + "wall_time_s": 809258439.255786, + "date": "2026-08-24T18:00:39.256+08:00" + }, + "page size": 16384 +} \ No newline at end of file diff --git a/benchmark/fixtures/karijini-cold-start.log b/benchmark/fixtures/karijini-cold-start.log new file mode 100644 index 0000000..a3324ea --- /dev/null +++ b/benchmark/fixtures/karijini-cold-start.log @@ -0,0 +1,25 @@ +2026-08-24 18:00:38.104 INFO termtree_lib - Raised FD limit: 9223372036854775807 -> 9223372036854775807 +2026-08-24 18:00:38.106 INFO termtree_lib - ======================================== +2026-08-24 18:00:38.106 INFO termtree_lib - TermTree v0.1.0 (release) +2026-08-24 18:00:38.106 INFO termtree_lib - OS: Darwin 15.7.4, Arch: aarch64 +2026-08-24 18:00:38.107 INFO termtree_lib - Data dir: /Users/bench/Library/Application Support/DocumentNode/TermTree +2026-08-24 18:00:38.107 INFO termtree_lib - Bundle id: com.documentnode.termtree, WebView2 profile: TermTree +2026-08-24 18:00:38.107 INFO termtree_lib - ======================================== +2026-08-24 18:00:38.219 INFO termtree_lib::settings - Settings loaded (10 keys) from /Users/bench/Library/Application Support/DocumentNode/TermTree/settings.json +2026-08-24 18:00:38.221 INFO termtree_lib::command::locale_cmd - [locale] startup locale resolved: en +2026-08-24 18:00:38.244 INFO termtree_lib - splash_monitor - started (max=15s) +2026-08-24 18:00:38.251 INFO termtree_lib::http::server - http::server - bound to 127.0.0.1:58854 +2026-08-24 18:00:38.252 INFO termtree_lib - Local HTTP server started on port 58854 +2026-08-24 18:00:38.258 INFO actix_server::builder - starting 8 workers +2026-08-24 18:00:38.259 INFO actix_server::server - Actix runtime found; starting in Actix runtime +2026-08-24 18:00:38.301 INFO app_util::wake_detection - Registering macOS sleep/wake observer +2026-08-24 18:00:38.302 INFO app_util::wake_detection - Starting wake-detection heartbeat timer (interval=5s, gap=15s) +2026-08-24 18:00:38.302 INFO app_util::wake_detection - Starting periodic health-check timer (interval=60s) +2026-08-24 18:00:39.518 INFO termtree_lib::command::state_cmd - State loaded (7454 bytes) from /Users/bench/Library/Application Support/DocumentNode/TermTree/state.json +2026-08-24 18:00:39.522 INFO termtree_lib::command::buffer_cmd - cleanup_session_buffers: keeping 0 session(s) +2026-08-24 18:00:39.540 INFO termtree_lib::command::terminal_cmd - close_all_terminal_sessions: closed 0 stale session(s) at boot +2026-08-24 18:00:39.571 INFO termtree_lib::shell_detect - Detected 1 shell(s): zsh +2026-08-24 18:00:39.612 INFO termtree_lib::command::window_cmd - app_window_ready: main +2026-08-24 18:00:40.126 INFO termtree_lib - splash_monitor - main window ready, closing splashscreen +2026-08-24 18:00:40.128 INFO termtree_lib - splash_monitor - done +2026-08-24 18:00:40.204 INFO termtree_lib::command::terminal_cmd - start_shell: session_id=c45eb033-c0d5-47ab-b9ae-7b33d77ab5a5, node_id=sample-mindmap-api-server, rows=24, cols=80, cwd=None, command=None diff --git a/benchmark/fixtures/karijini-splash-timeout.log b/benchmark/fixtures/karijini-splash-timeout.log new file mode 100644 index 0000000..f5ffd4e --- /dev/null +++ b/benchmark/fixtures/karijini-splash-timeout.log @@ -0,0 +1,13 @@ +2026-08-24 18:14:02.088 INFO termtree_lib - Raised FD limit: 9223372036854775807 -> 9223372036854775807 +2026-08-24 18:14:02.090 INFO termtree_lib - ======================================== +2026-08-24 18:14:02.090 INFO termtree_lib - TermTree v0.1.0 (release) +2026-08-24 18:14:02.090 INFO termtree_lib - OS: Darwin 15.7.4, Arch: aarch64 +2026-08-24 18:14:02.091 INFO termtree_lib - Data dir: /Users/bench/Library/Application Support/DocumentNode/TermTree +2026-08-24 18:14:02.091 INFO termtree_lib - ======================================== +2026-08-24 18:14:02.205 INFO termtree_lib::settings - Settings loaded (10 keys) from /Users/bench/Library/Application Support/DocumentNode/TermTree/settings.json +2026-08-24 18:14:02.231 INFO termtree_lib - splash_monitor - started (max=15s) +2026-08-24 18:14:02.240 INFO termtree_lib::http::server - http::server - bound to 127.0.0.1:58861 +2026-08-24 18:14:02.301 INFO app_util::wake_detection - Starting wake-detection heartbeat timer (interval=5s, gap=15s) +2026-08-24 18:14:09.884 WARN termtree_lib::wake_handler - [wake] last launch ended with "splash_monitor - max timeout reached, forcing transition"; treating this launch as a cold start +2026-08-24 18:14:17.236 INFO termtree_lib - splash_monitor - max timeout reached, forcing transition +2026-08-24 18:14:17.402 INFO termtree_lib - splash_monitor - done diff --git a/benchmark/fixtures/lsappinfo-list.txt b/benchmark/fixtures/lsappinfo-list.txt new file mode 100644 index 0000000..2ca8430 --- /dev/null +++ b/benchmark/fixtures/lsappinfo-list.txt @@ -0,0 +1,74 @@ + 1) "loginwindow" ASN:0x0-0x32d62d3: + bundleID="com.apple.loginwindow" + bundle path="/System/Library/CoreServices/loginwindow.app" + executable path="/System/Library/CoreServices/loginwindow.app/Contents/MacOS/loginwindow" + pid = 55070 type="UIElement" flavor=3 Version="3037.6.3" fileType="APPL" creator="lgnw" Arch=ARM64 + checkin time = 2026/08/01 08:15:15 ( 23 days, 9 hours, 45 minutes, 0.229484 seconds ago ) + + 2) "WindowManager" ASN:0x0-0x32da2d7: + bundleID="com.apple.WindowManager" + bundle path="/System/Library/CoreServices/WindowManager.app" + executable path="/System/Library/CoreServices/WindowManager.app/Contents/MacOS/WindowManager" + pid = 55130 type="UIElement" flavor=3 Version="278.4.7" fileType="APPL" Arch=ARM64 sandboxed + + 3) "universalaccessd" ASN:0x0-0x32db2d8: + bundleID=[ NULL ] + bundle path="/usr/sbin/universalaccessd" + executable path="/usr/sbin/universalaccessd" + pid = 55129 !signalled type="BackgroundOnly" flavor=3 Version=[ NULL ] fileType="????" creator="????" Arch=ARM64 + checkin time = 2026/08/01 08:15:25 ( 23 days, 9 hours, 44 minutes, 50.1559 seconds ago ) + + 4) "Control Centre" ASN:0x0-0x32dc2d9: + bundleID="com.apple.controlcenter" + bundle path="/System/Library/CoreServices/ControlCenter.app" + executable path="/System/Library/CoreServices/ControlCenter.app/Contents/MacOS/ControlCenter" + pid = 55127 type="UIElement" flavor=3 Version="1" fileType="APPL" creator="????" Arch=ARM64 + checkin time = 2026/08/01 08:15:25 ( 23 days, 9 hours, 44 minutes, 50.132 seconds ago ) + +57) "TermTree" ASN:0x0-0xca30a24: (in front) + bundleID="com.termtree.desktop" + bundle path="/Applications/TermTree.app" + executable path="/Applications/TermTree.app/Contents/MacOS/termtree" + pid = 56070 type="Foreground" flavor=3 Version="20260816.78283" fileType="APPL" creator="????" Arch=ARM64 + parentASN="Finder" ASN:0x0-0x32f02ed: + launch time = 2026/08/16 22:06:58 ( 7 days, 19 hours, 53 minutes, 16.9966 seconds ago ) + checkin time = 2026/08/16 22:06:59 ( 7 days, 19 hours, 53 minutes, 15.854 seconds ago ) + launch to checkin time: 1.14266 seconds + +58) "TermTree Networking" ASN:0x0-0xca31a25: + bundleID="com.apple.WebKit.Networking" + bundle path="/System/Library/Frameworks/WebKit.framework/Versions/A/XPCServices/com.apple.WebKit.Networking.xpc" + executable path="/System/Volumes/Preboot/Cryptexes/OS/System/Library/Frameworks/WebKit.framework/Versions/A/XPCServices/com.apple.WebKit.Networking.xpc/Contents/MacOS/com.apple.WebKit.Networking" + pid = 56074 !cgsConnection !signalled type="UIElement" flavor=[ NULL ] Version="20621.3.11.11.3" fileType="XPC!" creator="????" Arch=ARM64 sandboxed + +59) "TermTree Graphics and Media" ASN:0x0-0xca32a26: + bundleID="com.apple.WebKit.GPU" + bundle path="/System/Library/Frameworks/WebKit.framework/Versions/A/XPCServices/com.apple.WebKit.GPU.xpc" + executable path="/System/Volumes/Preboot/Cryptexes/OS/System/Library/Frameworks/WebKit.framework/Versions/A/XPCServices/com.apple.WebKit.GPU.xpc/Contents/MacOS/com.apple.WebKit.GPU" + pid = 56072 !cgsConnection !signalled type="UIElement" flavor=[ NULL ] Version="20621.3.11.11.3" fileType="XPC!" creator="????" Arch=ARM64 sandboxed + +60) "TermTree Web Content" ASN:0x0-0xca51a45: + bundleID="com.apple.WebKit.WebContent" + bundle path="/System/Library/Frameworks/WebKit.framework/Versions/A/XPCServices/com.apple.WebKit.WebContent.xpc" + executable path="/System/Volumes/Preboot/Cryptexes/OS/System/Library/Frameworks/WebKit.framework/Versions/A/XPCServices/com.apple.WebKit.WebContent.xpc/Contents/MacOS/com.apple.WebKit.WebContent" + pid = 64305 !cgsConnection !signalled type="UIElement" flavor=2 Version="20621.3.11.11.3" fileType="XPC!" creator="????" Arch=ARM64 sandboxed + +61) "TermTree Web Content" ASN:0x0-0xcb08afc: + bundleID="com.apple.WebKit.WebContent" + bundle path="/System/Library/Frameworks/WebKit.framework/Versions/A/XPCServices/com.apple.WebKit.WebContent.xpc" + executable path="/System/Volumes/Preboot/Cryptexes/OS/System/Library/Frameworks/WebKit.framework/Versions/A/XPCServices/com.apple.WebKit.WebContent.xpc/Contents/MacOS/com.apple.WebKit.WebContent" + pid = 83276 !cgsConnection !signalled type="UIElement" flavor=2 Version="20621.3.11.11.3" fileType="XPC!" creator="????" Arch=ARM64 sandboxed + +56) "Google Chrome Helper" ASN:0x0-0xc7086fc: + bundleID="com.google.Chrome.helper" + bundle path="/Applications/Google Chrome.app/Contents/Frameworks/Google Chrome Framework.framework/Versions/151.0.7922.138/Helpers/Google Chrome Helper.app" + executable path="/Applications/Google Chrome.app/Contents/Frameworks/Google Chrome Framework.framework/Versions/151.0.7922.138/Helpers/Google Chrome Helper.app/Contents/MacOS/Google Chrome Helper" + pid = 39283 type="UIElement" flavor=3 Version="7922.138" fileType="APPL" creator="????" Arch=ARM64 + parentASN="Google Chrome" ASN:0x0-0xc7066fa: (inferred) + checkin time = 2026/08/16 15:37:27 ( 8 days, 2 hours, 22 minutes, 48.5462 seconds ago ) + + 99) "Backup Agent" ASN:0x0-0xdeadbeef: + bundleID="com.example.backupagent" + bundle path="/Applications/Backup Agent.app" + executable path="/Applications/Backup Agent.app/Contents/MacOS/Backup Agent" + checkin time = 2026/08/20 09:00:00 ( 4 days, 9 hours, 0 minutes, 0 seconds ago ) diff --git a/benchmark/fixtures/notifyutil-thermal-nominal.txt b/benchmark/fixtures/notifyutil-thermal-nominal.txt new file mode 100644 index 0000000..6911cf0 --- /dev/null +++ b/benchmark/fixtures/notifyutil-thermal-nominal.txt @@ -0,0 +1 @@ +com.apple.system.thermalpressurelevel 0 diff --git a/benchmark/fixtures/notifyutil-thermal-serious.txt b/benchmark/fixtures/notifyutil-thermal-serious.txt new file mode 100644 index 0000000..33c5b3b --- /dev/null +++ b/benchmark/fixtures/notifyutil-thermal-serious.txt @@ -0,0 +1 @@ +com.apple.system.thermalpressurelevel 2 diff --git a/benchmark/fixtures/pmset-ps-ac.txt b/benchmark/fixtures/pmset-ps-ac.txt new file mode 100644 index 0000000..8a3666e --- /dev/null +++ b/benchmark/fixtures/pmset-ps-ac.txt @@ -0,0 +1,2 @@ +Now drawing from 'AC Power' + -InternalBattery-0 (id=32702563) 100%; charged; 0:00 remaining present: true diff --git a/benchmark/fixtures/pmset-ps-battery-live-capture.txt b/benchmark/fixtures/pmset-ps-battery-live-capture.txt new file mode 100644 index 0000000..0632315 --- /dev/null +++ b/benchmark/fixtures/pmset-ps-battery-live-capture.txt @@ -0,0 +1,2 @@ +Now drawing from 'Battery Power' + -InternalBattery-0 (id=32702563) 92%; discharging; 4:10 remaining present: true diff --git a/benchmark/fixtures/pmset-ps-battery.txt b/benchmark/fixtures/pmset-ps-battery.txt new file mode 100644 index 0000000..7a05557 --- /dev/null +++ b/benchmark/fixtures/pmset-ps-battery.txt @@ -0,0 +1,2 @@ +Now drawing from 'Battery Power' + -InternalBattery-0 (id=32702563) 62%; discharging; 3:41 remaining present: true diff --git a/benchmark/fixtures/process-table-chromium.json b/benchmark/fixtures/process-table-chromium.json new file mode 100644 index 0000000..6cd97ef --- /dev/null +++ b/benchmark/fixtures/process-table-chromium.json @@ -0,0 +1,432 @@ +{ + "rootPid": 40000, + "processes": [ + { + "pid": 40000, + "ppid": 1, + "name": "Electron", + "executablePath": "/Applications/Collaborator.app/Contents/MacOS/Collaborator", + "rssBytes": 209715200 + }, + { + "pid": 41000, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41001, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41002, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41003, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41004, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41005, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41006, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41007, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41008, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41009, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41010, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41011, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41012, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41013, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41014, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41015, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41016, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41017, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41018, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41019, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 40000000 + }, + { + "pid": 41020, + "ppid": 40000, + "name": "Electron Helper (GPU)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (GPU).app/Contents/MacOS/Electron Helper (GPU)", + "rssBytes": 40000000 + }, + { + "pid": 41021, + "ppid": 40000, + "name": "Electron Helper (Plugin)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Plugin).app/Contents/MacOS/Electron Helper (Plugin)", + "rssBytes": 40000000 + }, + { + "pid": 41022, + "ppid": 40000, + "name": "Electron Helper (Plugin)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Plugin).app/Contents/MacOS/Electron Helper (Plugin)", + "rssBytes": 40000000 + }, + { + "pid": 41023, + "ppid": 40000, + "name": "Electron Helper", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper.app/Contents/MacOS/Electron Helper", + "rssBytes": 40000000 + }, + { + "pid": 41024, + "ppid": 40000, + "name": "node", + "executablePath": "/Applications/Collaborator.app/Contents/Resources/app/bin/node", + "rssBytes": 40000000 + }, + { + "pid": 41025, + "ppid": 40000, + "name": "tmux", + "executablePath": "/Applications/Collaborator.app/Contents/Resources/vendor/tmux", + "rssBytes": 40000000 + }, + { + "pid": 41026, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 38000000 + }, + { + "pid": 41027, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 38000000 + }, + { + "pid": 41028, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 38000000 + }, + { + "pid": 41029, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 38000000 + }, + { + "pid": 41030, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 38000000 + }, + { + "pid": 41031, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 38000000 + }, + { + "pid": 41032, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 38000000 + }, + { + "pid": 41033, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 38000000 + }, + { + "pid": 41034, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 38000000 + }, + { + "pid": 41035, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 38000000 + }, + { + "pid": 41036, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 38000000 + }, + { + "pid": 41037, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 38000000 + }, + { + "pid": 41038, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 38000000 + }, + { + "pid": 41039, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 38000000 + }, + { + "pid": 41040, + "ppid": 40000, + "name": "Electron Helper (Renderer)", + "executablePath": "/Applications/Collaborator.app/Contents/Frameworks/Electron Helper (Renderer).app/Contents/MacOS/Electron Helper (Renderer)", + "rssBytes": 38000000 + }, + { + "pid": 41041, + "ppid": 41024, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 41042, + "ppid": 41025, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 41043, + "ppid": 41024, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 41044, + "ppid": 41025, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 41045, + "ppid": 41024, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 41046, + "ppid": 41025, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 41047, + "ppid": 41024, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 41048, + "ppid": 41025, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 41049, + "ppid": 41024, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 41050, + "ppid": 41025, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 41051, + "ppid": 41024, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 41052, + "ppid": 41025, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 41053, + "ppid": 41024, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 41054, + "ppid": 41025, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 41055, + "ppid": 41024, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 41056, + "ppid": 41025, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 41057, + "ppid": 41024, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 90001, + "ppid": 90002, + "name": "cyclic_a", + "executablePath": "/usr/libexec/cyclic_a", + "rssBytes": 1048576 + }, + { + "pid": 90002, + "ppid": 90001, + "name": "cyclic_b", + "executablePath": "/usr/libexec/cyclic_b", + "rssBytes": 1048576 + } + ] +} \ No newline at end of file diff --git a/benchmark/fixtures/process-table-webkit.json b/benchmark/fixtures/process-table-webkit.json new file mode 100644 index 0000000..98fcc86 --- /dev/null +++ b/benchmark/fixtures/process-table-webkit.json @@ -0,0 +1,152 @@ +{ + "rootPid": 56070, + "processes": [ + { + "pid": 56070, + "ppid": 1, + "name": "termtree", + "executablePath": "/Applications/TermTree.app/Contents/MacOS/termtree", + "rssBytes": 83923136 + }, + { + "pid": 70000, + "ppid": 56070, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 70001, + "ppid": 70000, + "name": "claude", + "executablePath": "/Users/dev/.local/bin/claude", + "rssBytes": 52428800 + }, + { + "pid": 70002, + "ppid": 70001, + "name": "rg", + "executablePath": "/opt/homebrew/bin/rg", + "rssBytes": 2097152 + }, + { + "pid": 70003, + "ppid": 70001, + "name": "rg", + "executablePath": "/opt/homebrew/bin/rg", + "rssBytes": 2097152 + }, + { + "pid": 70004, + "ppid": 56070, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 70005, + "ppid": 70004, + "name": "claude", + "executablePath": "/Users/dev/.local/bin/claude", + "rssBytes": 52428800 + }, + { + "pid": 70006, + "ppid": 70005, + "name": "rg", + "executablePath": "/opt/homebrew/bin/rg", + "rssBytes": 2097152 + }, + { + "pid": 70007, + "ppid": 70005, + "name": "rg", + "executablePath": "/opt/homebrew/bin/rg", + "rssBytes": 2097152 + }, + { + "pid": 70008, + "ppid": 56070, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 70009, + "ppid": 70008, + "name": "claude", + "executablePath": "/Users/dev/.local/bin/claude", + "rssBytes": 52428800 + }, + { + "pid": 70010, + "ppid": 70009, + "name": "rg", + "executablePath": "/opt/homebrew/bin/rg", + "rssBytes": 2097152 + }, + { + "pid": 70011, + "ppid": 70009, + "name": "rg", + "executablePath": "/opt/homebrew/bin/rg", + "rssBytes": 2097152 + }, + { + "pid": 70012, + "ppid": 56070, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 70013, + "ppid": 70012, + "name": "claude", + "executablePath": "/Users/dev/.local/bin/claude", + "rssBytes": 52428800 + }, + { + "pid": 70014, + "ppid": 70013, + "name": "rg", + "executablePath": "/opt/homebrew/bin/rg", + "rssBytes": 2097152 + }, + { + "pid": 70015, + "ppid": 70013, + "name": "rg", + "executablePath": "/opt/homebrew/bin/rg", + "rssBytes": 2097152 + }, + { + "pid": 70016, + "ppid": 56070, + "name": "zsh", + "executablePath": "/bin/zsh", + "rssBytes": 3145728 + }, + { + "pid": 70017, + "ppid": 70016, + "name": "claude", + "executablePath": "/Users/dev/.local/bin/claude", + "rssBytes": 52428800 + }, + { + "pid": 70018, + "ppid": 70017, + "name": "rg", + "executablePath": "/opt/homebrew/bin/rg", + "rssBytes": 2097152 + }, + { + "pid": 99999, + "ppid": 1, + "name": "unrelated_daemon", + "executablePath": "/usr/libexec/unrelated_daemon", + "rssBytes": 1048576 + } + ] +} \ No newline at end of file diff --git a/benchmark/fixtures/sysctl-swapusage.txt b/benchmark/fixtures/sysctl-swapusage.txt new file mode 100644 index 0000000..e1a0ee7 --- /dev/null +++ b/benchmark/fixtures/sysctl-swapusage.txt @@ -0,0 +1 @@ +vm.swapusage: total = 18432.00M used = 17377.19M free = 1054.81M (encrypted) diff --git a/benchmark/fixtures/vm_stat.txt b/benchmark/fixtures/vm_stat.txt new file mode 100644 index 0000000..3760910 --- /dev/null +++ b/benchmark/fixtures/vm_stat.txt @@ -0,0 +1,23 @@ +Mach Virtual Memory Statistics: (page size of 16384 bytes) +Pages free: 3920. +Pages active: 139168. +Pages inactive: 131976. +Pages speculative: 6005. +Pages throttled: 0. +Pages wired down: 308016. +Pages purgeable: 940. +"Translation faults": 28748548688. +Pages copy-on-write: 19161827588. +Pages zero filled: 9338717559. +Pages reactivated: 5236091926. +Pages purged: 502945665. +File-backed pages: 100062. +Anonymous pages: 177087. +Pages stored in compressor: 3114037. +Pages occupied by compressor: 410934. +Decompressions: 4421806115. +Compressions: 5337504610. +Pageins: 926697495. +Pageouts: 12238627. +Swapins: 104516389. +Swapouts: 112106478. diff --git a/benchmark/results/.gitkeep b/benchmark/results/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/benchmark/rustfmt.toml b/benchmark/rustfmt.toml new file mode 100644 index 0000000..7a774fb --- /dev/null +++ b/benchmark/rustfmt.toml @@ -0,0 +1,5 @@ +unstable_features = true +indent_style = "Block" +tab_spaces = 2 +max_width = 80 +overflow_delimited_expr = true diff --git a/benchmark/src/attribution.rs b/benchmark/src/attribution.rs new file mode 100644 index 0000000..bf441c2 --- /dev/null +++ b/benchmark/src/attribution.rs @@ -0,0 +1,426 @@ +//! The attribution resolver (spec FR-2): a subject's attributable process +//! set is the deduplicated union of its LaunchServices-owned processes +//! (`launch_services.rs`) and its process-tree descendants +//! (`process_tree.rs`). Neither mechanism alone is complete -- using either +//! alone must fail the run rather than silently undercount (design §5.2.2). + +use crate::launch_services::LaunchServicesEntry; +use crate::process_tree::ProcessRecord; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum DiscoverySource { + LaunchServices, + ProcessTree, + Both, +} + +impl DiscoverySource { + pub fn as_str(&self) -> &'static str { + match self { + Self::LaunchServices => "launch-services", + Self::ProcessTree => "process-tree", + Self::Both => "both", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ProcessRole { + Orchestrator, + AgentCliSession, +} + +impl ProcessRole { + pub fn as_str(&self) -> &'static str { + match self { + Self::Orchestrator => "orchestrator", + Self::AgentCliSession => "agent-cli-session", + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct AttributedProcess { + pub pid: u32, + pub name: String, + pub executable_path: Option, + pub discovered_by: DiscoverySource, + pub role: ProcessRole, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct AttributableProcessSet { + pub main_pid: u32, + pub processes: Vec, +} + +impl AttributableProcessSet { + pub fn pids(&self) -> Vec { + self.processes.iter().map(|p| p.pid).collect() + } + + pub fn orchestrator_pids(&self) -> Vec { + self + .processes + .iter() + .filter(|p| p.role == ProcessRole::Orchestrator) + .map(|p| p.pid) + .collect() + } + + pub fn agent_cli_pids(&self) -> Vec { + self + .processes + .iter() + .filter(|p| p.role == ProcessRole::AgentCliSession) + .map(|p| p.pid) + .collect() + } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum AttributionError { + /// Spec FR-2: using either mechanism alone is an invalid attribution. + /// Fired when the LaunchServices set is empty for a subject whose family + /// expects helper entries, or when the process-tree set is empty for a + /// subject/tier expected to have descendants. + Incomplete { + launch_services_count: usize, + process_tree_count: usize, + }, +} + +/// A subject's LaunchServices process set: every entry whose display name is +/// `launch_services_name` or starts with `"{launch_services_name} "`, gated +/// by `bundle_ids` so an unrelated app whose name happens to share the +/// prefix (design §9) is never attributed. +pub fn resolve_launch_services_pids( + entries: &[LaunchServicesEntry], + launch_services_name: &str, + bundle_ids: &[&str], +) -> Vec<(u32, String)> { + let prefix = format!("{launch_services_name} "); + entries + .iter() + .filter(|entry| { + (entry.display_name == launch_services_name + || entry.display_name.starts_with(&prefix)) + && entry + .bundle_identifier + .as_deref() + .is_some_and(|id| bundle_ids.contains(&id)) + }) + .filter_map(|entry| entry.pid.map(|pid| (pid, entry.display_name.clone()))) + .collect() +} + +/// Builds the deduplicated union of the two mechanisms' PIDs into an +/// [`AttributableProcessSet`], defaulting every process's role to +/// `Orchestrator` until [`partition_by_role`] runs. Fails per spec FR-2 if +/// either mechanism's set is empty. +pub fn resolve_union( + main_pid: u32, + launch_services_pids: &[(u32, String)], + tree_records: &[ProcessRecord], + tree_descendant_pids: &[u32], +) -> Result { + if launch_services_pids.is_empty() || tree_descendant_pids.is_empty() { + return Err(AttributionError::Incomplete { + launch_services_count: launch_services_pids.len(), + process_tree_count: tree_descendant_pids.len(), + }); + } + + let mut by_pid: BTreeMap = BTreeMap::new(); + + for (pid, name) in launch_services_pids { + by_pid.insert(*pid, AttributedProcess { + pid: *pid, + name: name.clone(), + executable_path: None, + discovered_by: DiscoverySource::LaunchServices, + role: ProcessRole::Orchestrator, + }); + } + + for pid in tree_descendant_pids { + let record = crate::process_tree::record_by_pid(tree_records, *pid); + let name = record.map(|r| r.name.clone()).unwrap_or_default(); + let executable_path = record.and_then(|r| r.executable_path.clone()); + by_pid + .entry(*pid) + .and_modify(|existing| { + existing.discovered_by = DiscoverySource::Both; + if existing.executable_path.is_none() { + existing.executable_path = executable_path.clone(); + } + }) + .or_insert(AttributedProcess { + pid: *pid, + name, + executable_path, + discovered_by: DiscoverySource::ProcessTree, + role: ProcessRole::Orchestrator, + }); + } + + Ok(AttributableProcessSet { + main_pid, + processes: by_pid.into_values().collect(), + }) +} + +/// Applies **one rule to every subject** (spec FR-2's "the published method +/// states this explicitly"): a process is a session root if its executable +/// path equals the pinned agent CLI's resolved path or the host login +/// shell's path; every descendant of a session root is also +/// `AgentCliSession`; everything else is `Orchestrator`. Deliberately not a +/// per-runtime-family rule -- see design §5.2.3 for why `tmux`/`node-pty` +/// sidecars and the login shell land in `Orchestrator`/`AgentCliSession` +/// respectively under this one rule. +pub fn partition_by_role( + set: &mut AttributableProcessSet, + tree_records: &[ProcessRecord], + agent_cli_executable_path: &str, + login_shell_path: &str, +) { + let is_session_root = |path: &Option| { + path.as_deref() == Some(agent_cli_executable_path) + || path.as_deref() == Some(login_shell_path) + }; + + let session_roots: Vec = set + .processes + .iter() + .filter(|p| is_session_root(&p.executable_path)) + .map(|p| p.pid) + .collect(); + + let mut session_pids: std::collections::HashSet = + session_roots.iter().copied().collect(); + for root in &session_roots { + for descendant in crate::process_tree::descendants_of(tree_records, *root) { + session_pids.insert(descendant); + } + } + + for process in &mut set.processes { + process.role = if session_pids.contains(&process.pid) { + ProcessRole::AgentCliSession + } else { + ProcessRole::Orchestrator + }; + } +} + +/// Marks a companion process (e.g. CodeNomad's local server, diri's +/// `dirijord-rs` daemon) as `Orchestrator`, inserting it into the set if the +/// union did not already discover it (spec FR-2: "client/server subjects +/// ... attribute their local server process ... not as a separate, +/// uncounted process"). +pub fn attribute_companion_process( + set: &mut AttributableProcessSet, + pid: u32, + name: &str, + executable_path: Option, +) { + set + .processes + .iter_mut() + .find(|p| p.pid == pid) + .map(|p| p.role = ProcessRole::Orchestrator) + .unwrap_or_else(|| { + set.processes.push(AttributedProcess { + pid, + name: name.to_string(), + executable_path, + discovered_by: DiscoverySource::ProcessTree, + role: ProcessRole::Orchestrator, + }); + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::launch_services::parse_lsappinfo_list; + use crate::process_tree::descendants_of; + use std::fs; + + fn read(name: &str) -> String { + fs::read_to_string(format!( + "{}/fixtures/{name}", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap() + } + + #[derive(serde::Deserialize)] + struct ProcessTableFixture { + #[serde(rename = "rootPid")] + root_pid: u32, + processes: Vec, + } + + fn read_process_table(name: &str) -> ProcessTableFixture { + serde_json::from_str(&read(name)).unwrap() + } + + #[test] + fn union_of_zero_overlap_webkit_and_tree_sets_is_deduped_and_labelled() { + let ls_entries = parse_lsappinfo_list(&read("lsappinfo-list.txt")); + let ls_pids = resolve_launch_services_pids(&ls_entries, "TermTree", &[ + "com.termtree.desktop", + "com.apple.WebKit.Networking", + "com.apple.WebKit.GPU", + "com.apple.WebKit.WebContent", + ]); + assert!(ls_pids.len() >= 4); + + let tree = read_process_table("process-table-webkit.json"); + let descendants = descendants_of(&tree.processes, tree.root_pid); + assert_eq!(descendants.len(), 19); + + let set = + resolve_union(tree.root_pid, &ls_pids, &tree.processes, &descendants) + .unwrap(); + + // Zero overlap between the two mechanisms in this fixture -- every PID + // is discovered by exactly one, never `Both`. + assert!(set + .processes + .iter() + .all(|p| p.discovered_by != DiscoverySource::Both)); + assert_eq!(set.processes.len(), ls_pids.len() + descendants.len()); + } + + #[test] + fn union_marks_a_pid_found_by_both_mechanisms_as_both() { + let ls_pids = vec![(100u32, "Subject".to_string())]; + let tree_records = vec![ProcessRecord { + pid: 100, + ppid: 1, + name: "subject".into(), + executable_path: Some("/Applications/Subject.app/subject".into()), + rss_bytes: 0, + }]; + let set = resolve_union(100, &ls_pids, &tree_records, &[100]).unwrap(); + assert_eq!(set.processes.len(), 1); + assert_eq!(set.processes[0].discovered_by, DiscoverySource::Both); + } + + #[test] + fn empty_launch_services_set_is_an_invalid_attribution() { + let tree_records = vec![ProcessRecord { + pid: 2, + ppid: 1, + name: "child".into(), + executable_path: None, + rss_bytes: 0, + }]; + let error = resolve_union(1, &[], &tree_records, &[2]).unwrap_err(); + assert!(matches!(error, AttributionError::Incomplete { .. })); + } + + #[test] + fn empty_process_tree_set_is_an_invalid_attribution() { + let ls_pids = vec![(1u32, "Subject".to_string())]; + let error = resolve_union(1, &ls_pids, &[], &[]).unwrap_err(); + assert!(matches!(error, AttributionError::Incomplete { .. })); + } + + #[test] + fn partition_puts_agent_cli_and_its_descendants_and_login_shell_in_session() { + let tree = read_process_table("process-table-webkit.json"); + let descendants = descendants_of(&tree.processes, tree.root_pid); + let mut set = resolve_union( + tree.root_pid, + &[(tree.root_pid, "TermTree".to_string())], + &tree.processes, + &descendants, + ) + .unwrap(); + + partition_by_role( + &mut set, + &tree.processes, + "/Users/dev/.local/bin/claude", + "/bin/zsh", + ); + + let orchestrator = set.orchestrator_pids(); + let session = set.agent_cli_pids(); + assert!(orchestrator.contains(&tree.root_pid)); + // Every zsh (login shell) and every claude (agent CLI) process, plus + // their descendants, must land in AgentCliSession. + for record in &tree.processes { + if record.name == "zsh" || record.name == "claude" { + assert!( + session.contains(&record.pid), + "{} ({}) should be AgentCliSession", + record.name, + record.pid + ); + } + } + assert!(!session.is_empty()); + } + + #[test] + fn partition_puts_tmux_and_node_pty_sidecar_in_orchestrator() { + let tree = read_process_table("process-table-chromium.json"); + let descendants = descendants_of(&tree.processes, tree.root_pid); + let ls_pids = vec![(tree.root_pid, "Collaborator".to_string())]; + let mut set = + resolve_union(tree.root_pid, &ls_pids, &tree.processes, &descendants) + .unwrap(); + + partition_by_role( + &mut set, + &tree.processes, + "/Users/dev/.local/bin/claude", + "/bin/zsh", + ); + + let tmux = tree.processes.iter().find(|p| p.name == "tmux").unwrap(); + let node = tree.processes.iter().find(|p| p.name == "node").unwrap(); + let orchestrator = set.orchestrator_pids(); + assert!( + orchestrator.contains(&tmux.pid), + "tmux is Collaborator's own implementation choice, not a session root" + ); + assert!( + orchestrator.contains(&node.pid), + "the node-pty sidecar is Collaborator's own implementation choice" + ); + // Its own zsh grandchildren, however, ARE agent CLI sessions under the + // one rule -- the shell is a per-session cost, not part of Collaborator + // itself. + let session = set.agent_cli_pids(); + for record in &tree.processes { + if record.name == "zsh" { + assert!(session.contains(&record.pid)); + } + } + } + + #[test] + fn attribute_companion_process_adds_or_reclassifies_as_orchestrator() { + let mut set = AttributableProcessSet { + main_pid: 1, + processes: vec![], + }; + attribute_companion_process( + &mut set, + 500, + "codenomad-server", + Some("/Applications/CodeNomad.app/server".into()), + ); + assert_eq!(set.processes.len(), 1); + assert_eq!(set.processes[0].role, ProcessRole::Orchestrator); + } +} diff --git a/benchmark/src/bundle_paths.rs b/benchmark/src/bundle_paths.rs new file mode 100644 index 0000000..cfa42aa --- /dev/null +++ b/benchmark/src/bundle_paths.rs @@ -0,0 +1,115 @@ +//! Per-subject bundle-path overrides (spec item 5): `SubjectSpec::bundle_path` +//! is a compile-time `/Applications/*.app` constant with no override at +//! all, which leaves a third party whose installs live elsewhere simply +//! stuck. [`resolve`] is the one place every live code path reads a +//! subject's actual bundle path, so a CLI or environment override can +//! never drift between `doctor`, the live sweep, and cold-start launch. + +use crate::subject::SubjectSpec; +use std::collections::HashMap; + +pub type BundlePathOverrides = HashMap; + +/// `RESOURCE_BENCHMARK_BUNDLE_PATH_`, following this crate's +/// existing `RESOURCE_BENCHMARK_*` override convention -- e.g. +/// `codenomad-electron` reads `RESOURCE_BENCHMARK_BUNDLE_PATH_CODENOMAD_ELECTRON`. +pub fn env_var_name(subject_id: &str) -> String { + format!( + "RESOURCE_BENCHMARK_BUNDLE_PATH_{}", + subject_id.to_uppercase().replace('-', "_") + ) +} + +/// `subject.id`'s override if one was given (CLI wins over environment), +/// else the registry's compiled-in default. +pub fn resolve<'a>( + subject: &'a SubjectSpec, + overrides: &'a BundlePathOverrides, +) -> &'a str { + overrides + .get(subject.id) + .map(String::as_str) + .unwrap_or(subject.bundle_path) +} + +/// Merges environment-sourced overrides with CLI-sourced ones, CLI winning +/// on a collision -- the same precedence [`crate::scratch_home`] uses for +/// `--home` vs `RESOURCE_BENCHMARK_HOME`. +pub fn merge_cli_over_env( + env_overrides: BundlePathOverrides, + cli_overrides: BundlePathOverrides, +) -> BundlePathOverrides { + let mut merged = env_overrides; + merged.extend(cli_overrides); + merged +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::subject::find; + + #[test] + fn env_var_name_uppercases_and_replaces_hyphens() { + assert_eq!( + env_var_name("codenomad-electron"), + "RESOURCE_BENCHMARK_BUNDLE_PATH_CODENOMAD_ELECTRON" + ); + assert_eq!( + env_var_name("termtree"), + "RESOURCE_BENCHMARK_BUNDLE_PATH_TERMTREE" + ); + } + + #[test] + fn override_wins_over_the_registry_default() { + let subject = find("termtree").unwrap(); + let mut overrides = BundlePathOverrides::new(); + overrides.insert( + "termtree".to_string(), + "/Users/dev/Apps/TermTree.app".to_string(), + ); + assert_eq!(resolve(subject, &overrides), "/Users/dev/Apps/TermTree.app"); + } + + #[test] + fn no_override_falls_back_to_the_registry_default() { + let subject = find("termtree").unwrap(); + let overrides = BundlePathOverrides::new(); + assert_eq!(resolve(subject, &overrides), subject.bundle_path); + } + + #[test] + fn an_unrelated_subjects_override_does_not_leak() { + let subject = find("collaborator").unwrap(); + let mut overrides = BundlePathOverrides::new(); + overrides.insert("termtree".to_string(), "/tmp/Fake.app".to_string()); + assert_eq!(resolve(subject, &overrides), subject.bundle_path); + } + + #[test] + fn cli_override_wins_over_an_env_override_for_the_same_subject() { + let mut env_overrides = BundlePathOverrides::new(); + env_overrides + .insert("termtree".to_string(), "/env/TermTree.app".to_string()); + let mut cli_overrides = BundlePathOverrides::new(); + cli_overrides + .insert("termtree".to_string(), "/cli/TermTree.app".to_string()); + let merged = merge_cli_over_env(env_overrides, cli_overrides); + assert_eq!( + merged.get("termtree").map(String::as_str), + Some("/cli/TermTree.app") + ); + } + + #[test] + fn merge_keeps_env_only_entries() { + let mut env_overrides = BundlePathOverrides::new(); + env_overrides.insert("diri".to_string(), "/env/diri.app".to_string()); + let merged = merge_cli_over_env(env_overrides, BundlePathOverrides::new()); + assert_eq!( + merged.get("diri").map(String::as_str), + Some("/env/diri.app") + ); + } +} diff --git a/benchmark/src/cli.rs b/benchmark/src/cli.rs new file mode 100644 index 0000000..399bbab --- /dev/null +++ b/benchmark/src/cli.rs @@ -0,0 +1,386 @@ +//! Hand-rolled argument parsing (design §4.3: no `clap`, following +//! `tools/update-feed`'s precedent) → a `Command` enum plus `RunSettings` +//! overrides (spec §5.9's command-line surface). + +use crate::bundle_paths::BundlePathOverrides; +use crate::tier::Tier; + +#[derive(Debug, Clone, PartialEq)] +pub enum Command { + Doctor(DoctorArgs), + Run(RunArgs), + Render { + result_path: String, + out_path: Option, + }, + Seed { + subject: String, + sessions: u32, + }, + Restore { + subject: Option, + }, +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct DoctorArgs { + pub bundle_path_overrides: BundlePathOverrides, +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct RunArgs { + pub subjects: Option>, + pub tiers: Option>, + pub repetitions: Option, + pub out_path: Option, + pub resume_path: Option, + pub allow_optional_subjects: bool, + pub allow_version_drift: bool, + pub bundle_path_overrides: BundlePathOverrides, + pub repo_path: Option, + pub agent_cli_path: Option, +} + +#[derive(Debug, PartialEq)] +pub struct CliError(pub String); + +/// Strips a global `--home ` override out of the raw argument list +/// before subcommand parsing, wherever it appears (spec item 1) -- it +/// applies to every subcommand that touches the filesystem (`doctor`, +/// `run`, `seed`, `restore`), so it is not scoped to one subcommand's own +/// flag set the way `--bundle-path` is. Returns the override, if any, and +/// the remaining arguments for [`parse_args`] to parse as before. +pub fn extract_home_override(args: &[String]) -> (Option, Vec) { + let mut home = None; + let mut rest = Vec::with_capacity(args.len()); + let mut i = 0; + while i < args.len() { + if args[i] == "--home" { + home = args.get(i + 1).cloned(); + i += 2; + } else { + rest.push(args[i].clone()); + i += 1; + } + } + (home, rest) +} + +pub fn parse_args(args: &[String]) -> Result { + let Some((subcommand, rest)) = args.split_first() else { + return Err(CliError( + "usage: resource-benchmark ".into(), + )); + }; + + match subcommand.as_str() { + "doctor" => Ok(Command::Doctor(DoctorArgs { + bundle_path_overrides: parse_bundle_path_overrides(rest)?, + })), + "run" => Ok(Command::Run(parse_run_args(rest)?)), + "render" => parse_render_args(rest), + "seed" => parse_seed_args(rest), + "restore" => Ok(Command::Restore { + subject: value_of(rest, "--subject"), + }), + other => Err(CliError(format!("unknown subcommand: {other}"))), + } +} + +fn value_of(args: &[String], flag: &str) -> Option { + args + .iter() + .position(|a| a == flag) + .and_then(|i| args.get(i + 1)) + .cloned() +} + +fn flag_present(args: &[String], flag: &str) -> bool { + args.iter().any(|a| a == flag) +} + +/// Repeated `--bundle-path =` flags (spec item 5) -- +/// e.g. `--bundle-path termtree=/Users/dev/Apps/TermTree.app`. +fn parse_bundle_path_overrides( + args: &[String], +) -> Result { + let mut overrides = BundlePathOverrides::new(); + let mut i = 0; + while i < args.len() { + if args[i] == "--bundle-path" { + let value = args.get(i + 1).ok_or_else(|| { + CliError("--bundle-path requires =".into()) + })?; + let (id, path) = value.split_once('=').ok_or_else(|| { + CliError(format!( + "invalid --bundle-path (expected =): {value}" + )) + })?; + overrides.insert(id.to_string(), path.to_string()); + i += 2; + } else { + i += 1; + } + } + Ok(overrides) +} + +fn parse_run_args(args: &[String]) -> Result { + let subjects = value_of(args, "--subjects") + .map(|v| v.split(',').map(str::to_string).collect()); + let tiers = value_of(args, "--tiers") + .map(|v| { + v.split(',') + .map(|t| { + Tier::parse(t).ok_or_else(|| CliError(format!("unknown tier: {t}"))) + }) + .collect::, _>>() + }) + .transpose()?; + let repetitions = value_of(args, "--repetitions") + .map(|v| { + v.parse::() + .map_err(|_| CliError(format!("invalid --repetitions: {v}"))) + }) + .transpose()?; + Ok(RunArgs { + subjects, + tiers, + repetitions, + out_path: value_of(args, "--out"), + resume_path: value_of(args, "--resume"), + allow_optional_subjects: flag_present(args, "--allow-optional-subjects"), + allow_version_drift: flag_present(args, "--allow-version-drift"), + bundle_path_overrides: parse_bundle_path_overrides(args)?, + repo_path: value_of(args, "--repo-path"), + agent_cli_path: value_of(args, "--agent-cli-path"), + }) +} + +fn parse_render_args(args: &[String]) -> Result { + let result_path = args.first().cloned().ok_or_else(|| { + CliError( + "usage: resource-benchmark render [--out ]".into(), + ) + })?; + Ok(Command::Render { + result_path, + out_path: value_of(args, "--out"), + }) +} + +fn parse_seed_args(args: &[String]) -> Result { + let subject = value_of(args, "--subject") + .ok_or_else(|| CliError("seed requires --subject ".into()))?; + let sessions = value_of(args, "--sessions") + .ok_or_else(|| CliError("seed requires --sessions ".into()))? + .parse::() + .map_err(|_| CliError("invalid --sessions".into()))?; + Ok(Command::Seed { subject, sessions }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn args(strs: &[&str]) -> Vec { + strs.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn parses_doctor() { + assert_eq!( + parse_args(&args(&["doctor"])), + Ok(Command::Doctor(DoctorArgs::default())) + ); + } + + #[test] + fn parses_doctor_bundle_path_overrides() { + let command = parse_args(&args(&[ + "doctor", + "--bundle-path", + "termtree=/Users/dev/Apps/TermTree.app", + ])) + .unwrap(); + let Command::Doctor(doctor_args) = command else { + panic!() + }; + assert_eq!( + doctor_args + .bundle_path_overrides + .get("termtree") + .map(String::as_str), + Some("/Users/dev/Apps/TermTree.app") + ); + } + + #[test] + fn parses_run_with_subjects_and_tiers() { + let command = parse_args(&args(&[ + "run", + "--subjects", + "termtree,collaborator", + "--tiers", + "fresh-launch,n-session-5", + "--repetitions", + "10", + ])) + .unwrap(); + assert_eq!( + command, + Command::Run(RunArgs { + subjects: Some(vec!["termtree".into(), "collaborator".into()]), + tiers: Some(vec![Tier::FreshLaunch, Tier::NSession(5)]), + repetitions: Some(10), + out_path: None, + resume_path: None, + allow_optional_subjects: false, + allow_version_drift: false, + bundle_path_overrides: BundlePathOverrides::new(), + repo_path: None, + agent_cli_path: None, + }) + ); + } + + #[test] + fn parses_run_bundle_path_overrides_for_multiple_subjects() { + let command = parse_args(&args(&[ + "run", + "--bundle-path", + "termtree=/Users/dev/Apps/TermTree.app", + "--bundle-path", + "diri=/Users/dev/Apps/diri.app", + ])) + .unwrap(); + let Command::Run(run_args) = command else { + panic!() + }; + assert_eq!(run_args.bundle_path_overrides.len(), 2); + assert_eq!( + run_args + .bundle_path_overrides + .get("diri") + .map(String::as_str), + Some("/Users/dev/Apps/diri.app") + ); + } + + #[test] + fn rejects_a_bundle_path_override_missing_the_equals_separator() { + let error = + parse_args(&args(&["run", "--bundle-path", "termtree"])).unwrap_err(); + assert!(error.0.contains("--bundle-path")); + } + + #[test] + fn parses_run_repo_path_and_agent_cli_path() { + let command = parse_args(&args(&[ + "run", + "--repo-path", + "/tmp/my-repo", + "--agent-cli-path", + "/opt/bin/claude", + ])) + .unwrap(); + let Command::Run(run_args) = command else { + panic!() + }; + assert_eq!(run_args.repo_path.as_deref(), Some("/tmp/my-repo")); + assert_eq!(run_args.agent_cli_path.as_deref(), Some("/opt/bin/claude")); + } + + #[test] + fn extracts_a_global_home_override_from_anywhere_in_argv() { + let (home, rest) = extract_home_override(&args(&[ + "run", + "--home", + "/tmp/scratch-home", + "--repetitions", + "2", + ])); + assert_eq!(home.as_deref(), Some("/tmp/scratch-home")); + assert_eq!(rest, args(&["run", "--repetitions", "2"])); + } + + #[test] + fn no_home_override_present_leaves_args_unchanged() { + let (home, rest) = extract_home_override(&args(&["doctor"])); + assert_eq!(home, None); + assert_eq!(rest, args(&["doctor"])); + } + + #[test] + fn parses_run_with_no_arguments_as_the_full_sweep() { + let command = parse_args(&args(&["run"])).unwrap(); + assert_eq!(command, Command::Run(RunArgs::default())); + } + + #[test] + fn parses_run_flags() { + let command = parse_args(&args(&[ + "run", + "--allow-optional-subjects", + "--allow-version-drift", + ])) + .unwrap(); + let Command::Run(run_args) = command else { + panic!() + }; + assert!(run_args.allow_optional_subjects); + assert!(run_args.allow_version_drift); + } + + #[test] + fn rejects_an_unknown_tier() { + let error = + parse_args(&args(&["run", "--tiers", "not-a-tier"])).unwrap_err(); + assert!(error.0.contains("not-a-tier")); + } + + #[test] + fn parses_render() { + let command = + parse_args(&args(&["render", "results/run.json", "--out", "run.md"])) + .unwrap(); + assert_eq!(command, Command::Render { + result_path: "results/run.json".into(), + out_path: Some("run.md".into()), + }); + } + + #[test] + fn parses_seed() { + let command = + parse_args(&args(&["seed", "--subject", "termtree", "--sessions", "5"])) + .unwrap(); + assert_eq!(command, Command::Seed { + subject: "termtree".into(), + sessions: 5 + }); + } + + #[test] + fn parses_restore_with_and_without_a_subject() { + assert_eq!(parse_args(&args(&["restore"])).unwrap(), Command::Restore { + subject: None + }); + assert_eq!( + parse_args(&args(&["restore", "--subject", "termtree"])).unwrap(), + Command::Restore { + subject: Some("termtree".into()) + } + ); + } + + #[test] + fn rejects_an_empty_argument_list() { + assert!(parse_args(&[]).is_err()); + } + + #[test] + fn rejects_an_unknown_subcommand() { + assert!(parse_args(&args(&["frobnicate"])).is_err()); + } +} diff --git a/benchmark/src/cold_start.rs b/benchmark/src/cold_start.rs new file mode 100644 index 0000000..eb1edf1 --- /dev/null +++ b/benchmark/src/cold_start.rs @@ -0,0 +1,212 @@ +//! Cold-start / time-to-interactive measurement (spec FR-4, FR-6, design +//! §5.4): wall-clock time from launch invocation to the window-visible and +//! (TermTree-only) log-mark readiness signals, timed by an external +//! monotonic clock -- never `hyperfine` (it measures process exit; a +//! documented 500x artifact applies to long-running GUI apps that never +//! exit) and never a log timestamp. Log lines are arrival-timestamped here +//! instead, which is what keeps the timing independent of the log's own +//! precision (millisecond-granularity since taskhub#672, see +//! `log_marks.rs`) and of its flush latency. +//! +//! The live launch-and-poll loop is not unit-testable (design §11); the +//! pure decision functions it calls -- PID discovery from a process-table +//! delta, and the calibrated main-window-area rule -- are. + +use crate::exec::{run_capture, ExecError}; +use crate::process_tree::ProcessRecord; +use crate::window_probe::OnScreenWindow; +use std::collections::HashSet; +use std::path::Path; + +pub const OPEN_PROGRAM: &str = "/usr/bin/open"; + +/// `open -n -F --env HOME= -a ` (spec item 1): +/// +/// - `-n` always launches a **new** instance rather than activating one +/// already running -- required so a subject launched with one scratch +/// `HOME` never gets silently handed off to an instance still resident +/// from a different `HOME`. `-n` does **not** bypass a subject's own +/// single-instance guard (see [`crate::run::find_already_running_subject`] +/// for the separate check that does matter for that). +/// - `--env HOME=` is what actually isolates the subject: +/// every seeder writes under this same directory +/// (`seeding/termtree.rs`'s `expected_scratch_state_path`, `seeding/ +/// collaborator.rs`, `seeding/codenomad.rs`, `seeding/diri.rs`), so the +/// launched process must read its state from the identical path. +/// - `-F` suppresses window/session restoration (spec FR-4's last +/// acceptance criterion). A fresh scratch `HOME` has no saved window +/// state of its own to restore, so `-F` is no longer load-bearing the way +/// it was against a real profile -- kept anyway as a no-cost guard +/// against any restoration state macOS itself might keep outside the +/// app's `HOME` (e.g. a `bundle_identifier`-keyed system record), and +/// because it does not conflict with `-n`. +pub fn launch_suppressing_restoration( + bundle_path: &str, + scratch_home: &Path, +) -> Result<(), ExecError> { + let home_env_arg = format!("HOME={}", scratch_home.display()); + run_capture(OPEN_PROGRAM, &[ + "-n", + "-F", + "--env", + &home_env_arg, + "-a", + bundle_path, + ])?; + Ok(()) +} + +/// Whether this machine's `/usr/bin/open` documents `--env` at all. `man +/// open`'s minimum-macOS-version note could not be established reliably on +/// this development machine, so rather than hardcode an unverified version +/// constant, this probes the tool's own `--help` text -- side-effect-free, +/// consistent with `doctor`'s "changes nothing" contract -- and the caller +/// fails loudly, naming the missing flag, instead of silently launching +/// every subject against the runner's real `$HOME` (spec item 1). +pub fn supports_env_flag() -> bool { + run_capture(OPEN_PROGRAM, &["--help"]) + .map(|output| { + output.stdout.contains("--env") || output.stderr.contains("--env") + }) + .unwrap_or(false) +} + +/// Finds the PID of a newly-appeared process whose executable path is under +/// `bundle_path`, given the process table from before and after launch. +/// `open` does not report the launched PID itself, so discovery is by +/// process-table delta (design §5.4 step 2). Pure and fixture-testable. +pub fn find_newly_launched_pid( + before: &[ProcessRecord], + after: &[ProcessRecord], + bundle_path: &str, +) -> Option { + let before_pids: HashSet = before.iter().map(|r| r.pid).collect(); + after + .iter() + .find(|record| { + !before_pids.contains(&record.pid) + && record + .executable_path + .as_deref() + .is_some_and(|path| path.starts_with(bundle_path)) + }) + .map(|record| record.pid) +} + +/// A window qualifies as the **main** window once its area is at least +/// `fraction` of the subject's calibrated main-window area (design §5.4 +/// step 3) -- a per-subject constant from the calibration launch, not a +/// hard-coded pixel threshold, so the rule is identical across subjects +/// with different splash/main window geometry. +pub fn is_main_window( + window: &OnScreenWindow, + calibrated_main_window_area_pt: f64, + fraction: f64, +) -> bool { + window.layer == 0 + && window.area_pt >= calibrated_main_window_area_pt * fraction +} + +/// The largest on-screen, layer-0 window area observed during the +/// calibration launch becomes `calibratedMainWindowAreaPt`, reused by every +/// counted repetition (design §5.4 step 3, spec FR-12's mandatory first-run +/// discard doubling as the calibration launch). +pub fn calibrate_main_window_area(windows: &[OnScreenWindow]) -> Option { + windows + .iter() + .filter(|w| w.layer == 0) + .map(|w| w.area_pt) + .fold(None, |max, area| { + Some(max.map_or(area, |m: f64| m.max(area))) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn record(pid: u32, exe: &str) -> ProcessRecord { + ProcessRecord { + pid, + ppid: 1, + name: "x".into(), + executable_path: Some(exe.into()), + rss_bytes: 0, + } + } + + #[test] + fn finds_the_newly_launched_pid_under_the_bundle_path() { + let before = vec![record(1, "/Applications/Other.app/other")]; + let after = vec![ + record(1, "/Applications/Other.app/other"), + record(2, "/Applications/TermTree.app/Contents/MacOS/termtree"), + ]; + let found = + find_newly_launched_pid(&before, &after, "/Applications/TermTree.app"); + assert_eq!(found, Some(2)); + } + + #[test] + fn ignores_a_new_pid_under_a_different_bundle() { + let before = vec![]; + let after = vec![record(2, "/Applications/Other.app/other")]; + let found = + find_newly_launched_pid(&before, &after, "/Applications/TermTree.app"); + assert_eq!(found, None); + } + + #[test] + fn main_window_rule_uses_the_calibrated_area_not_a_fixed_pixel_count() { + let splash = OnScreenWindow { + owner_pid: 1, + layer: 0, + area_pt: 400.0, + }; + let main = OnScreenWindow { + owner_pid: 1, + layer: 0, + area_pt: 1_310_720.0, + }; + let calibrated = 1_310_720.0; + assert!(!is_main_window(&splash, calibrated, 0.5)); + assert!(is_main_window(&main, calibrated, 0.5)); + } + + #[test] + fn non_layer_zero_windows_never_qualify_as_main() { + let overlay = OnScreenWindow { + owner_pid: 1, + layer: 3, + area_pt: 2_000_000.0, + }; + assert!(!is_main_window(&overlay, 1_310_720.0, 0.5)); + } + + #[test] + fn calibration_picks_the_largest_layer_zero_window() { + let windows = vec![ + OnScreenWindow { + owner_pid: 1, + layer: 0, + area_pt: 400.0, + }, + OnScreenWindow { + owner_pid: 1, + layer: 3, + area_pt: 9_999_999.0, + }, + OnScreenWindow { + owner_pid: 1, + layer: 0, + area_pt: 1_310_720.0, + }, + ]; + assert_eq!(calibrate_main_window_area(&windows), Some(1_310_720.0)); + } + + #[test] + fn calibration_of_no_windows_is_none() { + assert_eq!(calibrate_main_window_area(&[]), None); + } +} diff --git a/benchmark/src/cpu_sampler.rs b/benchmark/src/cpu_sampler.rs new file mode 100644 index 0000000..800791e --- /dev/null +++ b/benchmark/src/cpu_sampler.rs @@ -0,0 +1,61 @@ +//! Idle-CPU sampling (spec FR-5, design §5.5): `sysinfo`'s two-refresh +//! **interval** delta, never `ps -o %cpu`'s lifetime average. +//! +//! `idleCpuPercentOfOneCore`: `sysinfo` reports CPU usage relative to a +//! single core, so 100 means one fully saturated core -- the host's logical +//! core count lives in provenance, not in this field, so a reader is never +//! left to guess what "100" means on an 8-core machine. + +use crate::stats::{self, Summary}; +use std::thread; +use std::time::Duration; +use sysinfo::{ProcessRefreshKind, ProcessesToUpdate, System}; + +/// Samples the CPU usage of `pids` `sample_count` times, `interval_ms` +/// apart, and returns the median/IQR of the per-sample **summed** interval +/// readings across the attributable set (design §5.5). +pub fn sample_idle_cpu( + pids: &[u32], + interval_ms: u64, + sample_count: u32, +) -> Option { + let mut system = System::new(); + let mut readings = Vec::with_capacity(sample_count as usize); + + for _ in 0..sample_count { + system.refresh_processes_specifics( + ProcessesToUpdate::All, + true, + ProcessRefreshKind::nothing().with_cpu(), + ); + thread::sleep(Duration::from_millis(interval_ms)); + system.refresh_processes_specifics( + ProcessesToUpdate::All, + true, + ProcessRefreshKind::nothing().with_cpu(), + ); + let total: f32 = pids + .iter() + .filter_map(|pid| { + system + .process(sysinfo::Pid::from_u32(*pid)) + .map(|p| p.cpu_usage()) + }) + .sum(); + readings.push(total as f64); + } + + stats::summarize(&readings) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sampling_zero_pids_returns_all_zero_readings() { + let summary = sample_idle_cpu(&[], 5, 3).unwrap(); + assert_eq!(summary.median, 0.0); + assert_eq!(summary.n, 3); + } +} diff --git a/benchmark/src/exec.rs b/benchmark/src/exec.rs new file mode 100644 index 0000000..38b08f3 --- /dev/null +++ b/benchmark/src/exec.rs @@ -0,0 +1,90 @@ +//! The only module in this crate that spawns a child process. +//! +//! Every external tool this harness shells out to (`lsappinfo`, `footprint`, +//! `vm_stat`, `sysctl`, `pmset`, `notifyutil`, `open`, `osascript`, `/bin/date`) +//! goes through [`run_capture`]. No other module spawns a process, and this +//! module never parses a tool's output -- that split is what makes every +//! parser fixture-testable in isolation (design §11). +//! +//! # The zsh-vs-bash word-splitting hazard +//! +//! [`run_capture`] takes an argument **vector** (`&[&str]`) and calls +//! `std::process::Command`, which spawns no shell -- so this crate is immune +//! to shell word-splitting by construction. Anyone re-running one of this +//! harness's invocations *by hand* is not: under **zsh** (this project's +//! default shell) an unquoted `$PIDS` variable is not word-split the way it +//! is under bash, so `footprint -j out.json $PIDS` silently passes the whole +//! PID list as a single argument and measures only the leading PID, with an +//! empty `errors` array -- no error, just a quietly wrong number. Every +//! copy-pasteable command in this crate's README repeats this warning next +//! to the invocation it applies to. +//! +//! [`run_capture`] never builds or participates in a pipeline: piping a +//! tool's stdout to a consumer that closes early (e.g. `| head`) can SIGPIPE +//! the tool mid-write and leave a truncated output file (reproduced against +//! `footprint -j` at exactly 12,288 bytes during this crate's design). stdout +//! is always captured into an owned buffer instead. + +use std::process::Command; + +#[derive(Debug)] +pub struct ExecError { + pub program: String, + pub args: Vec, + pub message: String, +} + +impl std::fmt::Display for ExecError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{} {}: {}", + self.program, + self.args.join(" "), + self.message + ) + } +} + +impl std::error::Error for ExecError {} + +/// Run `program` with `args` and capture stdout as a UTF-8 string. +/// +/// `args` is always an argument vector, never an interpolated string, and no +/// shell is ever invoked -- see the module doc for why both matter. A +/// non-zero exit status is not itself an error: several callers (e.g. the +/// quiesce gate's `sysctl -n`) treat a particular exit code as meaningful. +/// Callers that care about exit status read `ExecOutput::status`. +pub fn run_capture( + program: &str, + args: &[&str], +) -> Result { + let output = + Command::new(program) + .args(args) + .output() + .map_err(|e| ExecError { + program: program.to_string(), + args: args.iter().map(|a| a.to_string()).collect(), + message: e.to_string(), + })?; + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + Ok(ExecOutput { + status: output.status.code(), + stdout, + stderr, + }) +} + +pub struct ExecOutput { + pub status: Option, + pub stdout: String, + pub stderr: String, +} + +impl ExecOutput { + pub fn success(&self) -> bool { + self.status == Some(0) + } +} diff --git a/benchmark/src/footprint.rs b/benchmark/src/footprint.rs new file mode 100644 index 0000000..e611835 --- /dev/null +++ b/benchmark/src/footprint.rs @@ -0,0 +1,417 @@ +//! `footprint -j` invocation, its pure parser, and the pid-set verification +//! that guards against a dead/recycled PID silently producing a wrong +//! number (design §5.3.1). +//! +//! # Never `-p` +//! +//! `footprint -h` documents `-p, --proc ` and `-p, --pid ` on +//! separate lines sharing one short flag; name resolution wins, and it is a +//! **partial** match (verified: `footprint -j out.json -p 1` measured four +//! unrelated 1Password processes). This module always builds the long form, +//! `--pid `, once per requested PID, and never the short form. See also +//! `exec.rs`'s module doc for the separate zsh-word-splitting hazard this +//! crate is immune to by construction but a re-runner is not. + +use crate::exec::{run_capture, ExecError}; +use serde_json::Value; +use std::collections::{BTreeMap, HashSet}; + +pub const FOOTPRINT_PROGRAM: &str = "/usr/bin/footprint"; + +#[derive(Debug, Clone, PartialEq)] +pub struct FootprintReport { + /// `footprint`'s set-level `total footprint` -- pages shared across the + /// measured set are counted once. This, not the naive per-process sum, is + /// what the harness publishes as `memPhysFootprintBytes` (design §2.3). + pub total_footprint_bytes: u64, + pub processes: Vec, + pub shared: Vec, + pub errors: Vec, + pub warnings: Vec, + pub page_size_bytes: u64, + pub start_time_iso: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct FootprintProcess { + pub pid: u32, + pub name: String, + pub footprint_bytes: u64, + pub phys_footprint_bytes: Option, + pub has_categories: bool, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct SharedRegionGroup { + pub pids: Vec, + pub specific_to_pid: Option, + pub is_shared_cache: bool, +} + +#[derive(Debug)] +pub enum FootprintParseError { + MissingField(&'static str), + WrongType(&'static str), +} + +impl std::fmt::Display for FootprintParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::MissingField(field) => write!(f, "missing field: {field}"), + Self::WrongType(field) => write!(f, "wrong type for field: {field}"), + } + } +} + +impl std::error::Error for FootprintParseError {} + +/// Build the argument vector for one `footprint -j` invocation over +/// `requested_pids`. A `#[test]` below asserts this vector contains exactly +/// one `--pid` per PID, no `-p`, and no argument containing a space -- +/// the cheap structural guard against reintroducing the flag ambiguity or a +/// string-interpolated PID list (design §5.3.1, §11). +pub fn build_argument_vector( + output_path: &str, + requested_pids: &[u32], +) -> Vec { + let mut args = vec!["-j".to_string(), output_path.to_string()]; + for pid in requested_pids { + args.push("--pid".to_string()); + args.push(pid.to_string()); + } + args +} + +pub fn invoke_footprint( + output_path: &str, + requested_pids: &[u32], +) -> Result<(), ExecError> { + let args = build_argument_vector(output_path, requested_pids); + let arg_refs: Vec<&str> = args.iter().map(String::as_str).collect(); + run_capture(FOOTPRINT_PROGRAM, &arg_refs)?; + Ok(()) +} + +pub fn parse_footprint_json( + text: &str, +) -> Result { + let root: Value = serde_json::from_str(text) + .map_err(|_| FootprintParseError::WrongType("root"))?; + + let total_footprint_bytes = root + .get("total footprint") + .and_then(Value::as_u64) + .ok_or(FootprintParseError::MissingField("total footprint"))?; + let page_size_bytes = root + .get("page size") + .and_then(Value::as_u64) + .ok_or(FootprintParseError::MissingField("page size"))?; + + let processes_value = root + .get("processes") + .and_then(Value::as_array) + .ok_or(FootprintParseError::MissingField("processes"))?; + let mut processes = Vec::with_capacity(processes_value.len()); + for entry in processes_value { + processes.push(parse_process(entry)?); + } + + let shared_value = root + .get("shared") + .and_then(Value::as_array) + .ok_or(FootprintParseError::MissingField("shared"))?; + let shared = shared_value.iter().map(parse_shared_group).collect(); + + let errors = string_array(&root, "errors"); + let warnings = string_array(&root, "warnings"); + let start_time_iso = root + .get("start_time") + .and_then(|v| v.get("date")) + .and_then(Value::as_str) + .map(str::to_string); + + Ok(FootprintReport { + total_footprint_bytes, + processes, + shared, + errors, + warnings, + page_size_bytes, + start_time_iso, + }) +} + +fn parse_process( + entry: &Value, +) -> Result { + let pid = entry + .get("pid") + .and_then(Value::as_u64) + .map(|p| p as u32) + .ok_or(FootprintParseError::MissingField("processes[].pid"))?; + let name = entry + .get("name") + .and_then(Value::as_str) + .ok_or(FootprintParseError::MissingField("processes[].name"))? + .to_string(); + let footprint_bytes = entry + .get("footprint") + .and_then(Value::as_u64) + .ok_or(FootprintParseError::MissingField("processes[].footprint"))?; + let phys_footprint_bytes = entry + .get("auxiliary") + .and_then(|aux| aux.get("phys_footprint")) + .and_then(Value::as_u64); + // Small processes may omit `categories` entirely; its absence is not a + // parse error (design §9, §11). + let has_categories = entry + .get("categories") + .and_then(Value::as_object) + .map(|obj| !obj.is_empty()) + .unwrap_or(false); + + Ok(FootprintProcess { + pid, + name, + footprint_bytes, + phys_footprint_bytes, + has_categories, + }) +} + +fn parse_shared_group(entry: &Value) -> SharedRegionGroup { + let pids = entry + .get("pids") + .and_then(Value::as_array) + .map(|arr| { + arr + .iter() + .filter_map(|p| p.as_u64().map(|p| p as u32)) + .collect() + }) + .unwrap_or_default(); + let specific_to_pid = entry + .get("specific_to_pid") + .and_then(Value::as_u64) + .map(|p| p as u32); + let is_shared_cache = entry + .get("shared-cache") + .and_then(Value::as_bool) + .unwrap_or(false); + SharedRegionGroup { + pids, + specific_to_pid, + is_shared_cache, + } +} + +fn string_array(root: &Value, key: &str) -> Vec { + root + .get(key) + .and_then(Value::as_array) + .map(|arr| { + arr + .iter() + .filter_map(|v| v.as_str().map(str::to_string)) + .collect() + }) + .unwrap_or_default() +} + +/// `memPhysFootprintProcessSumBytes` -- the naive per-process sum a +/// `pidusage`-style tool would report. Never presented as +/// `memPhysFootprintBytes`'s equal; the difference is +/// `sharedPageDoubleCountBytes` (design §2.3, §6.2). +pub fn process_sum_bytes(report: &FootprintReport) -> u64 { + report.processes.iter().map(|p| p.footprint_bytes).sum() +} + +pub fn shared_page_double_count_bytes(report: &FootprintReport) -> u64 { + process_sum_bytes(report).saturating_sub(report.total_footprint_bytes) +} + +#[derive(Debug, PartialEq)] +pub struct PidSetMismatch { + pub requested: HashSet, + pub returned: HashSet, +} + +/// Asserts the PIDs `footprint` actually measured equal exactly the PIDs +/// requested. This is not a guard against a parser bug -- it defends +/// against a process exiting or a PID being recycled between attribution and +/// measurement, both of which otherwise produce a plausible-looking +/// undercount with an empty `errors` array (design §5.3.1, §9). +pub fn verify_pid_set( + requested: &[u32], + report: &FootprintReport, +) -> Result<(), PidSetMismatch> { + let requested_set: HashSet = requested.iter().copied().collect(); + let returned_set: HashSet = + report.processes.iter().map(|p| p.pid).collect(); + if requested_set == returned_set { + Ok(()) + } else { + Err(PidSetMismatch { + requested: requested_set, + returned: returned_set, + }) + } +} + +/// A companion assertion beside [`verify_pid_set`]: a PID can be recycled +/// between attribution and measurement, so the returned `name` for a PID no +/// longer matches what was attributed even though the PID set itself is +/// unchanged. `attributed_names` maps PID to the name the attribution +/// resolver (`attribution.rs`) recorded for it. +pub fn verify_process_names( + attributed_names: &BTreeMap, + report: &FootprintReport, +) -> Result<(), u32> { + for process in &report.processes { + if let Some(expected) = attributed_names.get(&process.pid) { + if expected != &process.name { + return Err(process.pid); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn read_fixture(name: &str) -> String { + fs::read_to_string(format!( + "{}/fixtures/{name}", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap() + } + + // --- Argument-vector test: written first, per design §12 Phase 2 ------ + + #[test] + fn argument_vector_uses_long_form_pid_only() { + let args = build_argument_vector("/tmp/out.json", &[56070, 56072, 56074]); + let pid_flag_count = args.iter().filter(|a| *a == "--pid").count(); + assert_eq!(pid_flag_count, 3, "one --pid per requested PID: {args:?}"); + assert!( + !args.iter().any(|a| a == "-p"), + "must never use the ambiguous short flag: {args:?}" + ); + assert!( + !args.iter().any(|a| a.contains(' ')), + "no argument may be a space-joined PID list: {args:?}" + ); + assert_eq!(args, vec![ + "-j", + "/tmp/out.json", + "--pid", + "56070", + "--pid", + "56072", + "--pid", + "56074" + ]); + } + + #[test] + fn argument_vector_is_empty_pid_safe() { + let args = build_argument_vector("/tmp/out.json", &[]); + assert_eq!(args, vec!["-j", "/tmp/out.json"]); + } + + // --- parse_footprint_json ---------------------------------------------- + + #[test] + fn parses_termtree_6proc_capture() { + let report = + parse_footprint_json(&read_fixture("footprint-termtree-6proc.json")) + .unwrap(); + assert_eq!(report.page_size_bytes, 16384); + assert!(report.errors.is_empty()); + assert_eq!(report.processes.len(), 5); + assert!(report.start_time_iso.is_some()); + + let sum = process_sum_bytes(&report); + assert_eq!( + sum, + report.total_footprint_bytes + shared_page_double_count_bytes(&report) + ); + assert!( + report.total_footprint_bytes < sum, + "the set total must be lower than the naive sum \ + (shared pages counted once): total={}, sum={sum}", + report.total_footprint_bytes + ); + + // Every process in this capture is large enough to carry categories. + assert!(report.processes.iter().all(|p| p.has_categories)); + + let names: Vec<_> = + report.processes.iter().map(|p| p.name.as_str()).collect(); + assert!(names.contains(&"termtree")); + assert_eq!( + names + .iter() + .filter(|n| **n == "com.apple.WebKit.WebContent") + .count(), + 2 + ); + } + + #[test] + fn parses_shared_set_with_shared_cache_and_missing_categories() { + let report = + parse_footprint_json(&read_fixture("footprint-shared-set.json")).unwrap(); + assert_eq!(report.processes.len(), 3); + let sum = process_sum_bytes(&report); + assert_ne!( + sum, report.total_footprint_bytes, + "shared-set fixture must show sum != total" + ); + let double_count = shared_page_double_count_bytes(&report); + assert_eq!(sum, report.total_footprint_bytes + double_count); + + // pid 56074 (Networking) was stripped of `categories` to prove a small + // process without a breakdown does not error the parse. + let networking = report.processes.iter().find(|p| p.pid == 56074).unwrap(); + assert!(!networking.has_categories); + + assert!(report.shared.iter().any(|s| s.specific_to_pid.is_some())); + assert!(report.shared.iter().any(|s| s.is_shared_cache)); + } + + #[test] + fn verify_pid_set_detects_a_dead_pid() { + let report = + parse_footprint_json(&read_fixture("footprint-dead-pid.json")).unwrap(); + // Requested one PID (56074) that exited before measurement. + let requested = [64305u32, 83276, 56070, 56072, 56074]; + let mismatch = verify_pid_set(&requested, &report).unwrap_err(); + assert!(mismatch.requested.contains(&56074)); + assert!(!mismatch.returned.contains(&56074)); + } + + #[test] + fn verify_pid_set_passes_when_sets_match() { + let report = + parse_footprint_json(&read_fixture("footprint-termtree-6proc.json")) + .unwrap(); + let requested: Vec = report.processes.iter().map(|p| p.pid).collect(); + assert!(verify_pid_set(&requested, &report).is_ok()); + } + + #[test] + fn verify_process_names_detects_a_recycled_pid() { + let report = + parse_footprint_json(&read_fixture("footprint-termtree-6proc.json")) + .unwrap(); + let mut attributed = BTreeMap::new(); + attributed.insert(56070u32, "some_other_process".to_string()); + let mismatch = verify_process_names(&attributed, &report).unwrap_err(); + assert_eq!(mismatch, 56070); + } +} diff --git a/benchmark/src/host_memory.rs b/benchmark/src/host_memory.rs new file mode 100644 index 0000000..ff03f95 --- /dev/null +++ b/benchmark/src/host_memory.rs @@ -0,0 +1,158 @@ +//! `vm_stat` invocation and its pure parser, plus the two derived host +//! memory quantities the free-RAM path publishes (design §5.3.2). +//! +//! A naive free-page delta understates consumption on a compressing OS: +//! memory an app "consumes" can land in the compressor without ever moving +//! the free-page count. `HostMemorySample` records the raw counters; +//! `free_ram_bytes`/`host_memory_used_bytes` are the two derived readings +//! whose *delta* the harness publishes side by side (`freeRamDeltaBytes`, +//! `hostMemoryUsedDeltaBytes`). + +use crate::exec::{run_capture, ExecError}; +use std::collections::BTreeMap; + +pub const VM_STAT_PROGRAM: &str = "/usr/bin/vm_stat"; + +#[derive(Debug, Clone, PartialEq)] +pub struct HostMemorySample { + pub page_size_bytes: u64, + pub counters: BTreeMap, +} + +pub fn invoke_vm_stat() -> Result { + let output = run_capture(VM_STAT_PROGRAM, &[])?; + Ok(output.stdout) +} + +/// Parses the whole `vm_stat` capture. The page size is read from the +/// header line rather than assumed to be 4096 or 16384 -- Apple Silicon +/// hosts use 16384, but the parser must not hard-code it (design §5.3.2). +pub fn parse_vm_stat(text: &str) -> Option { + let mut lines = text.lines(); + let header = lines.next()?; + let page_size_bytes = header + .split("page size of ") + .nth(1)? + .split(' ') + .next()? + .parse::() + .ok()?; + + let mut counters = BTreeMap::new(); + for line in lines { + // Every data line is `"Key": .` or `Key: .`; the quoted + // form (`"Translation faults":`) must not confuse a colon-based split, + // since the label itself never contains a colon. + let (label, value) = line.split_once(':')?; + let label = label.trim().trim_matches('"').to_string(); + let value = value.trim().trim_end_matches('.'); + if let Ok(parsed) = value.parse::() { + counters.insert(label, parsed); + } + } + + Some(HostMemorySample { + page_size_bytes, + counters, + }) +} + +impl HostMemorySample { + fn counter(&self, key: &str) -> u64 { + self.counters.get(key).copied().unwrap_or(0) + } + + /// `(Pages free + Pages speculative) * pageSize` (design §5.3.2). + pub fn free_ram_bytes(&self) -> u64 { + (self.counter("Pages free") + self.counter("Pages speculative")) + * self.page_size_bytes + } + + /// `(Anonymous pages + Pages wired down + Pages occupied by compressor - + /// Pages purgeable) * pageSize` -- the compression-aware companion to + /// `free_ram_bytes` (design §5.3.2). + pub fn host_memory_used_bytes(&self) -> u64 { + let used_pages = self.counter("Anonymous pages") + + self.counter("Pages wired down") + + self.counter("Pages occupied by compressor"); + used_pages.saturating_sub(self.counter("Pages purgeable")) + * self.page_size_bytes + } + + pub fn compressor_occupied_bytes(&self) -> u64 { + self.counter("Pages occupied by compressor") * self.page_size_bytes + } + + pub fn swapouts(&self) -> u64 { + self.counter("Swapouts") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn fixture() -> HostMemorySample { + let text = fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/fixtures/vm_stat.txt" + )) + .unwrap(); + parse_vm_stat(&text).unwrap() + } + + #[test] + fn page_size_is_read_from_the_header_not_assumed() { + let sample = fixture(); + assert_eq!(sample.page_size_bytes, 16384); + } + + #[test] + fn every_documented_counter_is_read() { + let sample = fixture(); + for key in [ + "Pages free", + "Pages active", + "Pages inactive", + "Pages speculative", + "Pages throttled", + "Pages wired down", + "Pages purgeable", + "File-backed pages", + "Anonymous pages", + "Pages occupied by compressor", + "Swapouts", + "Compressions", + ] { + assert!(sample.counters.contains_key(key), "missing {key}"); + } + } + + #[test] + fn quoted_translation_faults_key_does_not_confuse_the_splitter() { + let sample = fixture(); + assert_eq!( + sample.counters.get("Translation faults"), + Some(&28_748_548_688) + ); + } + + #[test] + fn trailing_dot_is_stripped_from_every_value() { + let sample = fixture(); + assert_eq!(sample.counters.get("Pages free"), Some(&3920)); + } + + #[test] + fn derived_quantities_use_the_documented_formulas() { + let sample = fixture(); + let expected_free = (3920 + 6005) * 16384; + assert_eq!(sample.free_ram_bytes(), expected_free); + + let expected_used = (177087u64 + 308016 + 410934 - 940) * 16384; + assert_eq!(sample.host_memory_used_bytes(), expected_used); + + assert_eq!(sample.swapouts(), 112_106_478); + } +} diff --git a/benchmark/src/launch_services.rs b/benchmark/src/launch_services.rs new file mode 100644 index 0000000..eb73552 --- /dev/null +++ b/benchmark/src/launch_services.rs @@ -0,0 +1,173 @@ +//! `lsappinfo list` invocation and its pure parser. +//! +//! WKWebView helper processes are registered with LaunchServices under the +//! owning app's display name even though they are `launchd`-parented +//! (PPID 1), which is what makes this the correct-but-incomplete half of +//! attribution (design §1, §5.2.1) -- `process_tree.rs` supplies the other +//! half. + +use crate::exec::{run_capture, ExecError}; + +pub const LSAPPINFO_PROGRAM: &str = "/usr/bin/lsappinfo"; + +#[derive(Debug, Clone, PartialEq)] +pub struct LaunchServicesEntry { + pub display_name: String, + pub bundle_identifier: Option, + pub bundle_path: Option, + pub executable_path: Option, + pub pid: Option, + pub in_front: bool, +} + +pub fn invoke_lsappinfo_list() -> Result { + let output = run_capture(LSAPPINFO_PROGRAM, &["list"])?; + Ok(output.stdout) +} + +/// Parse the whole `lsappinfo list` capture into one entry per registered +/// application. Pure and total: every observed real-output shape (§9, §11) +/// is handled without panicking. +pub fn parse_lsappinfo_list(text: &str) -> Vec { + text.split("\n\n").filter_map(parse_entry).collect() +} + +fn parse_entry(block: &str) -> Option { + let mut lines = block.lines(); + let header = lines.find(|line| !line.trim().is_empty())?; + let header_trimmed = header.trim_start(); + // Header shape: `NN) "Display Name" ASN:0x0-0xHEX:` optionally followed by + // ` (in front)`. Display names always appear quoted and may contain + // spaces, so the name is read between the first and last `"` rather than + // by splitting on whitespace. + let first_quote = header_trimmed.find('"')?; + let rest = &header_trimmed[first_quote + 1..]; + let last_quote = rest.find('"')?; + let display_name = rest[..last_quote].to_string(); + let in_front = header_trimmed.contains("(in front)"); + + let mut bundle_identifier = None; + let mut bundle_path = None; + let mut executable_path = None; + let mut pid = None; + + for line in block.lines() { + let trimmed = line.trim(); + if let Some(value) = trimmed.strip_prefix("bundleID=") { + bundle_identifier = parse_quoted_or_null(value); + } else if let Some(value) = trimmed.strip_prefix("bundle path=") { + bundle_path = parse_quoted_or_null(value); + } else if let Some(value) = trimmed.strip_prefix("executable path=") { + executable_path = parse_quoted_or_null(value); + } else if let Some(value) = trimmed.strip_prefix("pid = ") { + // The pid line carries trailing flags (`!signalled`, `sandboxed`, + // `type="..."`); only the leading integer is the PID. + pid = value + .split_whitespace() + .next() + .and_then(|token| token.parse::().ok()); + } + } + + Some(LaunchServicesEntry { + display_name, + bundle_identifier, + bundle_path, + executable_path, + pid, + in_front, + }) +} + +/// Strips a `"quoted string"` value down to its contents, or returns `None` +/// for the literal `[ NULL ]` lsappinfo prints for an absent field (verified +/// on `universalaccessd`'s `bundleID=` and `Version=`). +fn parse_quoted_or_null(value: &str) -> Option { + let value = value.trim(); + if value.starts_with('[') { + return None; + } + let value = value.strip_prefix('"')?; + let end = value.find('"')?; + Some(value[..end].to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn fixture() -> String { + fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/fixtures/lsappinfo-list.txt" + )) + .unwrap() + } + + #[test] + fn parses_every_termtree_entry_with_distinct_pids() { + let entries = parse_lsappinfo_list(&fixture()); + let termtree: Vec<_> = entries + .iter() + .filter(|e| { + e.display_name == "TermTree" || e.display_name.starts_with("TermTree ") + }) + .collect(); + // The main process plus at least the Networking, GPU, and one WebContent + // helper -- the fixture captures two WebContent entries with the SAME + // display name, which must yield two distinct PIDs, not be collapsed by + // a map keyed on name. + assert!(termtree.len() >= 4, "found: {termtree:?}"); + let web_content: Vec<_> = entries + .iter() + .filter(|e| e.display_name == "TermTree Web Content") + .collect(); + assert_eq!(web_content.len(), 2); + let pids: std::collections::HashSet<_> = + web_content.iter().filter_map(|e| e.pid).collect(); + assert_eq!(pids.len(), 2, "expected two distinct WebContent PIDs"); + + let main = entries + .iter() + .find(|e| e.display_name == "TermTree") + .unwrap(); + assert_eq!( + main.bundle_identifier.as_deref(), + Some("com.termtree.desktop") + ); + assert_eq!(main.pid, Some(56070)); + assert!(main.in_front); + } + + #[test] + fn null_bundle_id_and_version_parse_as_none() { + let entries = parse_lsappinfo_list(&fixture()); + let entry = entries + .iter() + .find(|e| e.display_name == "universalaccessd") + .unwrap(); + assert_eq!(entry.bundle_identifier, None); + } + + #[test] + fn entry_with_no_pid_line_yields_none_not_zero() { + let entries = parse_lsappinfo_list(&fixture()); + let entry = entries + .iter() + .find(|e| e.display_name == "Backup Agent") + .unwrap(); + assert_eq!(entry.pid, None); + } + + #[test] + fn pid_line_trailing_flags_do_not_corrupt_the_pid() { + let entries = parse_lsappinfo_list(&fixture()); + let entry = entries + .iter() + .find(|e| e.display_name == "universalaccessd") + .unwrap(); + // Real line: `pid = 55129 !signalled type="BackgroundOnly" ...` + assert_eq!(entry.pid, Some(55129)); + } +} diff --git a/benchmark/src/lib.rs b/benchmark/src/lib.rs new file mode 100644 index 0000000..0c1a175 --- /dev/null +++ b/benchmark/src/lib.rs @@ -0,0 +1,36 @@ +//! `resource-benchmark`: the resource-usage benchmark harness (spec at +//! `doc/spec/resource-benchmark-spec.md`, design at +//! `doc/design/resource-benchmark-design.md`). +//! +//! Structured as a library plus a thin `main.rs` binary so that every +//! module's public API is available to `main.rs`'s live orchestration +//! *and* to this crate's own test suite. `run.rs::RunOrchestrator::run` is +//! the live sweep `main.rs`'s `run_sweep` drives (design §5.8); it is +//! exercised through `doctor` and a live smoke sweep, not this crate's +//! unit tests -- design §11 explains what a measurement harness cannot +//! unit-test and why every decision it makes is factored into a pure, +//! tested function instead. + +pub mod attribution; +pub mod bundle_paths; +pub mod cli; +pub mod cold_start; +pub mod cpu_sampler; +pub mod exec; +pub mod footprint; +pub mod host_memory; +pub mod launch_services; +pub mod log_marks; +pub mod process_tree; +pub mod provenance; +pub mod quiesce; +pub mod render; +pub mod result; +pub mod run; +pub mod scratch_home; +pub mod seeding; +pub mod settings; +pub mod stats; +pub mod subject; +pub mod tier; +pub mod window_probe; diff --git a/benchmark/src/log_marks.rs b/benchmark/src/log_marks.rs new file mode 100644 index 0000000..fa7b9b2 --- /dev/null +++ b/benchmark/src/log_marks.rs @@ -0,0 +1,212 @@ +//! `karijini.log` line classification (spec FR-4, design §5.4). +//! +//! `karijini.log`'s own timestamps are millisecond-granularity +//! (`logging.rs`'s `{d(%Y-%m-%d %H:%M:%S%.3f)}`, widened from seconds by +//! taskhub#672) and are **never** parsed for timing here -- only for ordering +//! sanity. The harness times each line's *arrival* on its own monotonic clock +//! (`cold_start.rs`); this module only classifies which mark, if any, a line +//! represents. + +use std::path::Path; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LogMark { + AppWindowReadyMain, + SplashClosed, + SplashTimeout, +} + +/// The exact message text (i.e. everything after the `{level} {target} - +/// ` prefix) that identifies each mark. Matching the **whole** message, +/// not a substring search over the raw line, is what keeps a line that +/// merely mentions one of these phrases inside unrelated prose from being +/// misclassified (design §11's explicit false-positive test). +const APP_WINDOW_READY_MAIN_MESSAGE: &str = "app_window_ready: main"; +const SPLASH_CLOSED_MESSAGE: &str = + "splash_monitor - main window ready, closing splashscreen"; +const SPLASH_TIMEOUT_MESSAGE: &str = + "splash_monitor - max timeout reached, forcing transition"; + +/// Extracts the message portion of one `karijini.log` line in the +/// `{d(%Y-%m-%d %H:%M:%S%.3f)} {l} {t} - {m}` format (`logging.rs`). The +/// date field itself contains a space, so the message is not simply "the +/// text after the 4th space" -- it is everything after the first `" - "` +/// that follows the (date, time, level, target) prefix, i.e. after the 5th +/// whitespace-delimited token. +/// +/// The four skipped tokens are date, time, level and target. The seconds +/// field carries the fractional part (`18:00:39.612`), so widening the +/// timestamp's precision does not change the token count -- which is why +/// taskhub#672 could add milliseconds without touching this parser. A +/// timezone suffix or a single-token RFC3339 timestamp would change it, and +/// would make every mark classify as absent rather than fail loudly. +fn message_of(line: &str) -> Option<&str> { + let mut rest = line; + for _ in 0..4 { + let (_, remainder) = rest.split_once(char::is_whitespace)?; + rest = remainder.trim_start(); + } + rest.strip_prefix("- ") +} + +pub fn classify_log_mark(line: &str) -> Option { + let message = message_of(line)?; + if message == APP_WINDOW_READY_MAIN_MESSAGE { + Some(LogMark::AppWindowReadyMain) + } else if message == SPLASH_CLOSED_MESSAGE { + Some(LogMark::SplashClosed) + } else if message == SPLASH_TIMEOUT_MESSAGE { + Some(LogMark::SplashTimeout) + } else { + None + } +} + +pub fn classify_log_text(text: &str) -> Vec { + text.lines().filter_map(classify_log_mark).collect() +} + +/// Spec item 4: whether the harness read `karijini.log` lines during a +/// TermTree launch's settle window without recognizing a single one of +/// them as a known mark -- a real drift signal (`app_window_ready: main`, +/// the splash-closed message, or the splash-timeout message have all +/// changed) rather than the legitimate cases that also leave every mark +/// `None`: this is not a TermTree launch at all (`is_termtree_launch == +/// false`), or the log simply had not advanced yet at the settle deadline +/// (`any_log_line_observed == false`). A `splash_timeout_seen` mark still +/// counts as recognized, since it is itself one of the three known +/// messages. +pub fn marks_unrecognized( + is_termtree_launch: bool, + any_log_line_observed: bool, + app_window_ready_ms: Option, + splash_close_ms: Option, + splash_timeout_seen: bool, +) -> bool { + is_termtree_launch + && any_log_line_observed + && app_window_ready_ms.is_none() + && splash_close_ms.is_none() + && !splash_timeout_seen +} + +/// TermTree's own data directory under the app-support root -- the parent +/// of [`karijini_log_path`]'s log file and the exact directory +/// `seeding/termtree.rs`'s writer targets (`TermTreeSeeder::production`). +/// Factored out to one join point so it, `karijini_log_path`, and +/// `run.rs`'s post-launch "did the app actually create its data +/// directory" drift check (spec item 4) can never independently drift +/// from each other. +pub fn app_data_dir(app_support_dir: &Path) -> std::path::PathBuf { + app_support_dir.join("DocumentNode").join("TermTree") +} + +pub fn karijini_log_path(app_support_dir: &Path) -> std::path::PathBuf { + app_data_dir(app_support_dir) + .join("logs") + .join("karijini.log") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn read(name: &str) -> String { + fs::read_to_string(format!( + "{}/fixtures/{name}", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap() + } + + #[test] + fn cold_start_log_carries_both_marks_in_order() { + let marks = classify_log_text(&read("karijini-cold-start.log")); + assert_eq!(marks, vec![ + LogMark::AppWindowReadyMain, + LogMark::SplashClosed + ]); + } + + #[test] + fn splash_timeout_log_is_classified_and_no_false_positive_fires() { + let marks = classify_log_text(&read("karijini-splash-timeout.log")); + // The fixture contains a WARN line whose message merely *mentions* the + // splash-timeout phrase inside different prose; it must not be + // misclassified as the real mark. Only the genuine + // `splash_monitor - max timeout reached, forcing transition` line + // should classify. + assert_eq!(marks, vec![LogMark::SplashTimeout]); + } + + /// taskhub#672 widened `logging.rs`'s timestamp to milliseconds. The + /// fractional part rides the *seconds* token, so the prefix `message_of` + /// skips is still exactly four tokens (date, time, level, target). A + /// timezone suffix -- or a single-token RFC3339 timestamp -- would change + /// that count, and every mark would then classify as absent instead of + /// failing loudly. This pins the boundary from both sides. + #[test] + fn millisecond_timestamp_keeps_the_four_token_prefix() { + let millisecond_timestamp = + "2026-08-24 18:00:39.612 INFO termtree_lib::command::window_cmd - app_window_ready: main"; + assert_eq!( + classify_log_mark(millisecond_timestamp), + Some(LogMark::AppWindowReadyMain) + ); + + let timezone_suffixed_timestamp = + "2026-08-24 18:00:39.612 +10:00 INFO termtree_lib::command::window_cmd - app_window_ready: main"; + assert_eq!(classify_log_mark(timezone_suffixed_timestamp), None); + } + + #[test] + fn app_window_ready_for_splashscreen_label_is_not_the_main_mark() { + let line = + "2026-08-24 18:00:39.612 INFO termtree_lib::command::window_cmd - app_window_ready: splashscreen"; + assert_eq!(classify_log_mark(line), None); + } + + #[test] + fn unrecognized_when_lines_advanced_but_none_classified() { + assert!(marks_unrecognized(true, true, None, None, false)); + } + + #[test] + fn not_unrecognized_when_the_log_never_advanced() { + // No new lines were seen yet -- too early to call this drift. + assert!(!marks_unrecognized(true, false, None, None, false)); + } + + #[test] + fn not_unrecognized_for_a_non_termtree_subject() { + assert!(!marks_unrecognized(false, true, None, None, false)); + } + + #[test] + fn not_unrecognized_once_any_known_mark_is_present() { + assert!(!marks_unrecognized(true, true, Some(1500), None, false)); + assert!(!marks_unrecognized(true, true, None, Some(2100), false)); + assert!(!marks_unrecognized(true, true, None, None, true)); + } + + #[test] + fn karijini_log_path_is_nested_under_app_data_dir() { + let app_support = Path::new("/scratch/Library/Application Support"); + assert_eq!( + karijini_log_path(app_support), + app_data_dir(app_support).join("logs").join("karijini.log") + ); + assert_eq!( + app_data_dir(app_support), + Path::new("/scratch/Library/Application Support/DocumentNode/TermTree") + ); + } + + #[test] + fn unrelated_log_line_classifies_as_none() { + let line = + "2026-08-24 18:00:38.244 INFO termtree_lib - splash_monitor - started (max=15s)"; + assert_eq!(classify_log_mark(line), None); + } +} diff --git a/benchmark/src/main.rs b/benchmark/src/main.rs new file mode 100644 index 0000000..a7b6f84 --- /dev/null +++ b/benchmark/src/main.rs @@ -0,0 +1,468 @@ +//! Entry point: dispatches `doctor` | `run` | `render` | `seed` | `restore` +//! (spec §5.9). + +use resource_benchmark::bundle_paths::{self, BundlePathOverrides}; +use resource_benchmark::cli::{self, Command, DoctorArgs, RunArgs}; +use resource_benchmark::run::{self, RunOrchestrator}; +use resource_benchmark::seeding; +use resource_benchmark::{ + cold_start, footprint, host_memory, launch_services, provenance, quiesce, + render, result, scratch_home, subject, tier, +}; +use std::path::PathBuf; +use std::process::ExitCode; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// The disposable per-run scratch home (spec item 1): every seeder, the +/// karijini log path, and every subject's launch `HOME` key off this -- +/// never the runner's real `$HOME`. `--home` (extracted by `main` before +/// subcommand parsing) wins, then `RESOURCE_BENCHMARK_HOME`, then a +/// freshly named directory under the OS temp dir. The directory is +/// created here so every caller can assume it already exists. +fn resolve_scratch_home(cli_override: Option<&str>) -> PathBuf { + let env_override = std::env::var(scratch_home::HOME_OVERRIDE_ENV).ok(); + let disambiguator = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + let home = scratch_home::resolve_scratch_home( + cli_override, + env_override, + &std::env::temp_dir(), + std::process::id(), + disambiguator, + ); + // Spec item 1 states the property unconditionally: the runner's real + // application state is never read or written. The default home can never + // be the real one, but an explicit override can -- refuse it rather than + // silently destroying the profile they actually use. + if let Err(error) = scratch_home::reject_real_home( + &home, + std::env::var_os("HOME").map(PathBuf::from).as_deref(), + ) { + eprintln!("{error}"); + std::process::exit(1); + } + if let Err(error) = scratch_home::ensure_scratch_home_exists(&home) { + eprintln!( + "warning: could not create scratch home {}: {error}", + home.display() + ); + } + home +} + +/// The real logged-in user's home directory, read only for OS-identity +/// lookups that must resolve an actual Directory Services record (e.g. +/// `dscl -read UserShell` for the login shell path used to +/// classify a process as a session root, design §5.2.3). Never used as a +/// write target, a seeder target, or a subject's launch `HOME` -- that is +/// exclusively [`resolve_scratch_home`]'s job (spec item 1: this harness +/// must never silently touch the runner's real profile). +fn real_home_for_identity_lookup() -> PathBuf { + std::env::var_os("HOME") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("/var/empty")) +} + +fn bundle_path_overrides_from_env() -> BundlePathOverrides { + subject::SUBJECTS + .iter() + .filter_map(|s| { + std::env::var(bundle_paths::env_var_name(s.id)) + .ok() + .map(|v| (s.id.to_string(), v)) + }) + .collect() +} + +fn merged_bundle_path_overrides( + cli_overrides: BundlePathOverrides, +) -> BundlePathOverrides { + bundle_paths::merge_cli_over_env( + bundle_path_overrides_from_env(), + cli_overrides, + ) +} + +fn main() -> ExitCode { + let raw_args: Vec = std::env::args().skip(1).collect(); + let (home_override, args) = cli::extract_home_override(&raw_args); + let command = match cli::parse_args(&args) { + Ok(command) => command, + Err(error) => { + eprintln!("error: {}", error.0); + return ExitCode::FAILURE; + } + }; + + match command { + Command::Doctor(doctor_args) => { + run_doctor(home_override.as_deref(), doctor_args) + } + Command::Run(run_args) => run_sweep(run_args, home_override.as_deref()), + Command::Render { + result_path, + out_path, + } => run_render(&result_path, out_path.as_deref()), + Command::Seed { subject, sessions } => { + run_seed(&subject, sessions, home_override.as_deref()) + } + Command::Restore { subject } => { + run_restore(subject.as_deref(), home_override.as_deref()) + } + } +} + +/// Checks every prerequisite and changes nothing (spec §5.9, item 6): the +/// five system tools exist, `open` supports the `--env` flag this harness +/// requires to isolate a subject's `HOME` (spec item 1), every +/// non-optional subject bundle is installed at its pinned version, no +/// subject is currently running (spec item 3), the quiesce gate reads, a +/// fresh scratch home can be created, and no leftover seeder backup +/// exists. Exit 1 lists what is missing; unverified-seed-format subjects +/// (spec item 4) are printed as notes, not failures, since they do not +/// block a run from starting. +fn run_doctor( + cli_home_override: Option<&str>, + doctor_args: DoctorArgs, +) -> ExitCode { + let bundle_path_overrides = + merged_bundle_path_overrides(doctor_args.bundle_path_overrides); + let mut problems = Vec::new(); + let mut notes = Vec::new(); + + for (name, program) in [ + ("lsappinfo", launch_services::LSAPPINFO_PROGRAM), + ("footprint", footprint::FOOTPRINT_PROGRAM), + ("vm_stat", host_memory::VM_STAT_PROGRAM), + ("sysctl", quiesce::SYSCTL_PROGRAM), + ("pmset", quiesce::PMSET_PROGRAM), + ("notifyutil", quiesce::NOTIFYUTIL_PROGRAM), + ("open", cold_start::OPEN_PROGRAM), + ] { + if !std::path::Path::new(program).exists() { + problems.push(format!("missing required tool: {name} ({program})")); + } + } + + // Spec item 1: `open --env` is required to isolate every subject's + // launch `HOME`. This probe is side-effect-free (it only reads + // `open --help`), so `doctor` changes nothing. + if !cold_start::supports_env_flag() { + problems.push( + "/usr/bin/open does not document --env -- this harness cannot \ + isolate a subject's HOME on this macOS version" + .to_string(), + ); + } + + // Spec item 1: confirm a scratch home can actually be created, without + // leaving one behind (`doctor` changes nothing). + let probe_home = std::env::temp_dir().join(format!( + "resource-benchmark-doctor-probe-{}", + std::process::id() + )); + match std::fs::create_dir_all(&probe_home) { + Ok(()) => { + let _ = std::fs::remove_dir_all(&probe_home); + } + Err(error) => problems.push(format!( + "could not create a scratch home under {}: {error}", + std::env::temp_dir().display() + )), + } + + let ls_text = launch_services::invoke_lsappinfo_list().unwrap_or_default(); + let ls_entries = launch_services::parse_lsappinfo_list(&ls_text); + + for spec in subject::SUBJECTS { + if spec.optional { + continue; + } + let bundle_path = bundle_paths::resolve(spec, &bundle_path_overrides); + if !std::path::Path::new(bundle_path).exists() { + problems.push(format!( + "subject not installed: {} ({bundle_path}) -- install it there, or \ + point the harness at an existing install with \ + `--bundle-path {}=`", + spec.display_name, spec.id + )); + continue; + } + match provenance::probe_subject_version(bundle_path) { + Some(version) if version == spec.expected_version => {} + Some(version) => problems.push(format!( + "{}: version drift, expected {} found {version}", + spec.display_name, spec.expected_version + )), + None => problems.push(format!( + "{}: could not probe version at {bundle_path}", + spec.display_name + )), + } + // Spec item 3: refuse before seeding if a subject is already running, + // keyed on bundle identifier. + if let Some(entry) = ls_entries.iter().find(|entry| { + entry.pid.is_some() + && entry.bundle_identifier.as_deref() == Some(spec.bundle_identifier) + }) { + problems.push(format!( + "{} ({}) is already running (pid {}) -- quit it before running the \ + benchmark", + spec.display_name, + spec.bundle_identifier, + entry.pid.unwrap() + )); + } + // Spec item 4: informational only -- does not block a run from + // starting, but a stranger reading `doctor`'s output should know + // before they trust an N-session/sustained-use result for it. + if !spec.seed_format_verified { + notes.push(format!( + "{}'s session seeder has not been verified against a real \ + install; its N-session/sustained-use samples will report \ + invalidReason=seed-format-unverified until verified", + spec.display_name + )); + } + } + + let reading = quiesce::read_quiesce_gate(None); + if reading.verdict != quiesce::QuiesceVerdict::Pass { + problems.push(format!( + "quiesce gate would fail: {}", + reading.failing_signals.join(", ") + )); + // Spec item 6: name the remedy, not just the failing signal. + for signal in &reading.failing_signals { + problems.push(format!( + " to clear {signal}: {}", + quiesce::remediation_for_signal(signal) + )); + } + } + + // A default scratch home is a brand-new, never-before-used directory + // (spec item 1), so it can never have a leftover backup -- checking it + // would be both vacuous and, worse, a `doctor`-created side effect this + // command must not have. This check only makes sense, and only runs, + // against an explicitly reused home (`--home`/`RESOURCE_BENCHMARK_HOME`). + let explicit_home_override = cli_home_override + .map(str::to_string) + .or_else(|| std::env::var(scratch_home::HOME_OVERRIDE_ENV).ok()); + if let Some(home) = explicit_home_override.map(PathBuf::from) { + let termtree_seeder = seeding::termtree::TermTreeSeeder::production(&home); + let backup = termtree_seeder + .state_directory + .join("state.json.before-resource-benchmark.json"); + if backup.exists() { + problems.push(format!( + "leftover TermTree seed backup at {} -- run `resource-benchmark restore`", + backup.display() + )); + } + } + + for note in ¬es { + println!("doctor note: {note}"); + } + + if problems.is_empty() { + println!("doctor: all checks passed."); + ExitCode::SUCCESS + } else { + for problem in &problems { + println!("doctor: {problem}"); + } + ExitCode::FAILURE + } +} + +fn run_render(result_path: &str, out_path: Option<&str>) -> ExitCode { + let path = PathBuf::from(result_path); + let result = match result::read_result_file(&path) { + Ok(result) => result, + Err(error) => { + eprintln!("error reading {result_path}: {error}"); + return ExitCode::FAILURE; + } + }; + let markdown = render::render(&result); + let destination = out_path + .map(PathBuf::from) + .unwrap_or_else(|| path.with_extension("md")); + if let Err(error) = std::fs::write(&destination, markdown) { + eprintln!("error writing {}: {error}", destination.display()); + return ExitCode::FAILURE; + } + println!("rendered {}", destination.display()); + ExitCode::SUCCESS +} + +fn seeded_repo(repo_path_override: Option<&str>) -> seeding::SeededRepo { + seeding::SeededRepo { + url: std::env::var("RESOURCE_BENCHMARK_REPO_URL") + .unwrap_or_else(|_| "https://github.com/example/benchmark-repo".into()), + commit: std::env::var("RESOURCE_BENCHMARK_REPO_COMMIT") + .unwrap_or_else(|_| "unknown".into()), + local_path: repo_path_override + .map(str::to_string) + .or_else(|| std::env::var("RESOURCE_BENCHMARK_REPO_PATH").ok()) + .unwrap_or_else(|| "/Users/Shared/benchmark-repo".into()), + } +} + +fn agent_cli_pin( + agent_cli_path_override: Option<&str>, +) -> seeding::AgentCliPin { + let executable_path = agent_cli_path_override + .map(str::to_string) + .or_else(|| std::env::var("RESOURCE_BENCHMARK_AGENT_CLI_PATH").ok()) + .unwrap_or_else(|| "/usr/local/bin/claude".into()); + let probed = provenance::probe_agent_cli_version(&executable_path); + seeding::AgentCliPin { + name: probed.name, + version: probed.version, + executable_path: probed.executable_path, + } +} + +fn run_seed( + subject_id: &str, + sessions: u32, + cli_home_override: Option<&str>, +) -> ExitCode { + let home = resolve_scratch_home(cli_home_override); + let repo = seeded_repo(None); + let agent = agent_cli_pin(None); + let result = + seeding::seed_subject(&home, subject_id, sessions, &repo, &agent); + match result { + Ok(plan) => { + println!( + "seeded {subject_id} with {sessions} sessions via {} (scratch home: {})", + plan.method, + home.display() + ); + ExitCode::SUCCESS + } + Err(error) => { + eprintln!("error seeding {subject_id}: {error}"); + ExitCode::FAILURE + } + } +} + +fn run_restore( + subject_id: Option<&str>, + cli_home_override: Option<&str>, +) -> ExitCode { + let home = resolve_scratch_home(cli_home_override); + let subjects: Vec<&str> = match subject_id { + Some(id) => vec![id], + None => vec!["termtree", "collaborator", "codenomad-electron", "diri"], + }; + let mut failed = false; + for id in subjects { + let result = seeding::restore_subject(&home, id); + if let Err(error) = result { + eprintln!("error restoring {id}: {error}"); + failed = true; + } else { + println!("restored {id}"); + } + } + if failed { + ExitCode::FAILURE + } else { + ExitCode::SUCCESS + } +} + +/// A full subject/tier sweep (spec FR-16, design §5.8/§7). This is the live +/// orchestration path: it is exercised via the smoke sweep described in the +/// README, not unit tests (design §11's explicit scope for what a +/// measurement harness cannot unit-test). A refusal (quiesce failure, +/// missing subject, version drift without `--allow-version-drift`, +/// `open --env` unsupported, a subject already running) prints an +/// actionable message and exits non-zero -- this command never reports +/// success without either performing the sweep or being told not to +/// (`--out`/`--resume` control where results land). +fn run_sweep(run_args: RunArgs, cli_home_override: Option<&str>) -> ExitCode { + let subjects = match run::select_subjects( + run_args.subjects.as_deref(), + run_args.allow_optional_subjects, + ) { + Ok(subjects) => subjects, + Err(error) => { + eprintln!("error: {error}"); + return ExitCode::FAILURE; + } + }; + let tiers = run_args + .tiers + .clone() + .unwrap_or_else(|| tier::DEFAULT_TIERS.to_vec()); + let settings = run::build_settings(run_args.repetitions); + let home = resolve_scratch_home(cli_home_override); + let repo = seeded_repo(run_args.repo_path.as_deref()); + let agent = agent_cli_pin(run_args.agent_cli_path.as_deref()); + let bundle_path_overrides = + merged_bundle_path_overrides(run_args.bundle_path_overrides.clone()); + let login_shell_path = provenance::probe_login_shell_path( + &real_home_for_identity_lookup().to_string_lossy(), + ); + let out_path = + run_args + .out_path + .clone() + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from(format!( + "results/{}.json", + provenance::iso_timestamp_now().replace([':', 'Z'], "-") + )) + }); + if let Some(parent) = out_path.parent() { + let _ = std::fs::create_dir_all(parent); + } + + run::install_interrupt_handler(); + + let orchestrator = RunOrchestrator { + subjects, + tiers, + settings, + home, + repo, + agent, + allow_version_drift: run_args.allow_version_drift, + out_path: out_path.clone(), + bundle_path_overrides, + login_shell_path, + }; + + let resume_path = run_args.resume_path.as_ref().map(PathBuf::from); + match orchestrator.run(resume_path.as_deref()) { + Ok(result) => { + let destination = resume_path.unwrap_or(out_path); + let markdown = render::render(&result); + let markdown_path = destination.with_extension("md"); + if let Err(error) = std::fs::write(&markdown_path, markdown) { + eprintln!("error writing {}: {error}", markdown_path.display()); + return ExitCode::FAILURE; + } + println!( + "resource-benchmark run: wrote {} and {}", + destination.display(), + markdown_path.display() + ); + ExitCode::SUCCESS + } + Err(refusal) => { + eprintln!("resource-benchmark run: refused to start: {refusal}"); + ExitCode::FAILURE + } + } +} diff --git a/benchmark/src/process_tree.rs b/benchmark/src/process_tree.rs new file mode 100644 index 0000000..467cf62 --- /dev/null +++ b/benchmark/src/process_tree.rs @@ -0,0 +1,148 @@ +//! `sysinfo` process-tree snapshot and the pure BFS descendant walk it +//! feeds, modelled deliberately on +//! `src-tauri/src/command/terminal_cmd.rs:1588`'s +//! `get_descendant_pids_and_names` so the harness's idea of "this subject's +//! process tree" matches the app's own (design §5.2.1). +//! +//! This is the half of attribution that resolves Chromium/Electron helper +//! processes (true children of their app) and TermTree's own spawned PTY +//! shells and agent CLI processes -- but resolves **none** of a WebKit +//! subject's `launchd`-parented helpers. `launch_services.rs` supplies the +//! other half; `attribution.rs` takes their union. + +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet, VecDeque}; +use sysinfo::{ProcessRefreshKind, ProcessesToUpdate, System}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProcessRecord { + pub pid: u32, + pub ppid: u32, + pub name: String, + pub executable_path: Option, + pub rss_bytes: u64, +} + +/// One live `sysinfo` snapshot of the full process table. +pub fn snapshot_processes() -> Vec { + let mut system = System::new(); + system.refresh_processes_specifics( + ProcessesToUpdate::All, + true, + ProcessRefreshKind::nothing().with_exe(sysinfo::UpdateKind::Always), + ); + + system + .processes() + .iter() + .map(|(pid, process)| ProcessRecord { + pid: pid.as_u32(), + ppid: process.parent().map(|p| p.as_u32()).unwrap_or(0), + name: process.name().to_string_lossy().into_owned(), + executable_path: process.exe().map(|p| p.to_string_lossy().into_owned()), + rss_bytes: process.memory(), + }) + .collect() +} + +/// BFS over `records` from `root_pid`, returning descendant PIDs in +/// nearest-the-root-first order. Pure and fixture-testable, unlike its +/// model in `terminal_cmd.rs`, which takes a live `sysinfo::System` +/// directly. A `visited` set (not the parent link) is what terminates the +/// walk if the process table ever contains a PPID cycle (design §11). +pub fn descendants_of(records: &[ProcessRecord], root_pid: u32) -> Vec { + let mut children: HashMap> = HashMap::new(); + for record in records { + children.entry(record.ppid).or_default().push(record.pid); + } + + let mut result = Vec::new(); + let mut visited: HashSet = HashSet::new(); + let mut queue: VecDeque = VecDeque::new(); + queue.push_back(root_pid); + visited.insert(root_pid); + + while let Some(pid) = queue.pop_front() { + if let Some(kids) = children.get(&pid) { + for &child in kids { + if visited.insert(child) { + result.push(child); + queue.push_back(child); + } + } + } + } + result +} + +pub fn record_by_pid( + records: &[ProcessRecord], + pid: u32, +) -> Option<&ProcessRecord> { + records.iter().find(|r| r.pid == pid) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + #[derive(serde::Deserialize)] + struct ProcessTableFixture { + #[serde(rename = "rootPid")] + root_pid: u32, + processes: Vec, + } + + fn read(name: &str) -> ProcessTableFixture { + let text = fs::read_to_string(format!( + "{}/fixtures/{name}", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + serde_json::from_str(&text).unwrap() + } + + #[test] + fn chromium_fixture_resolves_41_direct_children_and_58_total() { + let fixture = read("process-table-chromium.json"); + let direct: Vec<_> = fixture + .processes + .iter() + .filter(|p| p.ppid == fixture.root_pid) + .collect(); + assert_eq!(direct.len(), 41); + + let all = descendants_of(&fixture.processes, fixture.root_pid); + assert_eq!(all.len(), 58); + // The isolated 2-cycle must never be reachable from the real root. + assert!(!all.contains(&90001)); + assert!(!all.contains(&90002)); + } + + #[test] + fn a_ppid_cycle_terminates_via_the_visited_set() { + let fixture = read("process-table-chromium.json"); + // Starting the walk from *inside* the isolated 2-cycle must terminate + // rather than loop forever -- proving the guard is the `visited` set, + // not the (broken, circular) parent link. + let from_cycle = descendants_of(&fixture.processes, 90001); + assert_eq!(from_cycle, vec![90002]); + } + + #[test] + fn webkit_fixture_resolves_19_descendants_none_of_them_webkit_helpers() { + let fixture = read("process-table-webkit.json"); + let all = descendants_of(&fixture.processes, fixture.root_pid); + assert_eq!(all.len(), 19); + assert!( + !all.contains(&99999), + "unrelated daemon must not be included" + ); + for pid in &all { + let record = record_by_pid(&fixture.processes, *pid).unwrap(); + assert_ne!(record.name, "com.apple.WebKit.WebContent"); + } + } +} diff --git a/benchmark/src/provenance.rs b/benchmark/src/provenance.rs new file mode 100644 index 0000000..62d5a02 --- /dev/null +++ b/benchmark/src/provenance.rs @@ -0,0 +1,151 @@ +//! Machine spec, OS build, subject/agent version probes, and repo ref +//! capture (spec FR-11). Every probe is a thin wrapper over `exec.rs`; the +//! parsing each does is intentionally trivial (single-value `sysctl -n` / +//! `sw_vers -productVersion` reads), so it is exercised through `doctor` +//! rather than fixture-tested line by line. + +use crate::exec::run_capture; +use crate::result::{AgentCliVersion, HarnessRef, MachineSpec, OsBuild}; + +pub fn probe_machine_spec() -> MachineSpec { + let cpu_brand = + run_capture("/usr/sbin/sysctl", &["-n", "machdep.cpu.brand_string"]) + .map(|o| o.stdout.trim().to_string()) + .unwrap_or_else(|_| "unknown".to_string()); + let logical_cores = run_capture("/usr/sbin/sysctl", &["-n", "hw.logicalcpu"]) + .ok() + .and_then(|o| o.stdout.trim().parse().ok()) + .unwrap_or(0); + let physical_cores = + run_capture("/usr/sbin/sysctl", &["-n", "hw.physicalcpu"]) + .ok() + .and_then(|o| o.stdout.trim().parse().ok()) + .unwrap_or(0); + let ram_bytes = run_capture("/usr/sbin/sysctl", &["-n", "hw.memsize"]) + .ok() + .and_then(|o| o.stdout.trim().parse().ok()) + .unwrap_or(0); + let page_size_bytes = run_capture("/usr/bin/getconf", &["PAGESIZE"]) + .ok() + .and_then(|o| o.stdout.trim().parse().ok()) + .unwrap_or(16384); + + MachineSpec { + cpu_brand, + logical_cores, + physical_cores, + ram_bytes, + page_size_bytes, + } +} + +pub fn probe_os_build() -> OsBuild { + let product_version = run_capture("/usr/bin/sw_vers", &["-productVersion"]) + .map(|o| o.stdout.trim().to_string()) + .unwrap_or_else(|_| "unknown".to_string()); + let build_version = run_capture("/usr/bin/sw_vers", &["-buildVersion"]) + .map(|o| o.stdout.trim().to_string()) + .unwrap_or_else(|_| "unknown".to_string()); + OsBuild { + product_name: "macOS".to_string(), + product_version, + build_version, + } +} + +/// Reads `CFBundleShortVersionString` from `/Contents/Info.plist` +/// via `defaults read` -- no bundle-parsing dependency needed. +pub fn probe_subject_version(bundle_path: &str) -> Option { + let info_plist_path = format!("{bundle_path}/Contents/Info"); + let output = run_capture("/usr/bin/defaults", &[ + "read", + &info_plist_path, + "CFBundleShortVersionString", + ]) + .ok()?; + if !output.success() { + return None; + } + let version = output.stdout.trim(); + if version.is_empty() { + None + } else { + Some(version.to_string()) + } +} + +pub fn probe_login_shell_path(home: &str) -> String { + run_capture("/usr/bin/dscl", &[".", "-read", home, "UserShell"]) + .ok() + .and_then(|o| { + o.stdout + .trim() + .strip_prefix("UserShell: ") + .map(str::to_string) + }) + .unwrap_or_else(|| "/bin/zsh".to_string()) +} + +pub fn probe_agent_cli_version(executable_path: &str) -> AgentCliVersion { + let version = run_capture(executable_path, &["--version"]) + .map(|o| o.stdout.trim().to_string()) + .unwrap_or_else(|_| "unknown".to_string()); + AgentCliVersion { + name: executable_path + .rsplit('/') + .next() + .unwrap_or(executable_path) + .to_string(), + version, + executable_path: executable_path.to_string(), + } +} + +pub fn harness_ref(commit: &str) -> HarnessRef { + HarnessRef { + // The public repo this harness ships from (documentnode/termtree, + // spec item 5) -- not `termtree-app`, the private repo it was moved + // out of. Every published result's provenance must name the repo a + // reader can actually go clone. + repo: "termtree".to_string(), + commit: commit.to_string(), + crate_version: env!("CARGO_PKG_VERSION").to_string(), + } +} + +/// `/bin/date -u +%Y-%m-%dT%H:%M:%SZ` -- the one ISO timestamp source this +/// crate uses instead of a `chrono`/`time` dependency (design §4.3). +pub fn iso_timestamp_now() -> String { + run_capture("/bin/date", &["-u", "+%Y-%m-%dT%H:%M:%SZ"]) + .map(|o| o.stdout.trim().to_string()) + .unwrap_or_else(|_| "unknown".to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn iso_timestamp_has_the_expected_shape() { + let timestamp = iso_timestamp_now(); + assert_eq!(timestamp.len(), 20, "{timestamp}"); + assert!(timestamp.ends_with('Z'), "{timestamp}"); + assert!(timestamp.contains('T'), "{timestamp}"); + } + + #[test] + fn harness_ref_carries_the_crate_version() { + let reference = harness_ref("deadbeef"); + assert_eq!(reference.commit, "deadbeef"); + assert_eq!(reference.repo, "termtree"); + assert!(!reference.crate_version.is_empty()); + } + + #[test] + fn missing_bundle_returns_none_not_a_placeholder_version() { + assert_eq!( + probe_subject_version("/Applications/Definitely Not Installed.app"), + None + ); + } +} diff --git a/benchmark/src/quiesce.rs b/benchmark/src/quiesce.rs new file mode 100644 index 0000000..ebd2045 --- /dev/null +++ b/benchmark/src/quiesce.rs @@ -0,0 +1,370 @@ +//! The quiesce gate (spec FR-10, design §5.7): five unprivileged signals +//! that must all read nominal before a run starts, checked again before +//! every sample and periodically during N-session measurement. +//! +//! An unreadable safety signal is treated as `unknown`, which **fails** the +//! gate -- an unparseable thermal value is never read as "safe" (design §9). + +use crate::exec::run_capture; +use serde::{Deserialize, Serialize}; + +pub const SYSCTL_PROGRAM: &str = "/usr/sbin/sysctl"; +pub const PMSET_PROGRAM: &str = "/usr/bin/pmset"; +pub const NOTIFYUTIL_PROGRAM: &str = "/usr/bin/notifyutil"; + +/// `sysctl -n kern.memorystatus_vm_pressure_level` must read exactly this +/// value for the gate to pass (design §5.7's table). +pub const NOMINAL_MEMORY_PRESSURE_LEVEL: i64 = 1; +pub const NOMINAL_THERMAL_PRESSURE_LEVEL: i64 = 0; +/// Swap usage above this, in bytes, fails the gate even with zero swap +/// *activity* -- a host already deep in swap is not quiesced just because +/// nothing swapped out in the last window. +pub const QUIESCE_SWAP_USED_LIMIT_BYTES: u64 = 8 * 1024 * 1024 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum QuiesceVerdict { + Pass, + Fail, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct QuiesceReading { + pub memory_pressure_level: Option, + pub swap_used_bytes: Option, + pub swap_free_bytes: Option, + pub swapouts_delta: Option, + pub on_ac_power: Option, + pub thermal_pressure_level: Option, + pub verdict: QuiesceVerdict, + pub failing_signals: Vec, +} + +impl QuiesceReading { + #[cfg(test)] + pub fn nominal_for_test() -> Self { + Self { + memory_pressure_level: Some(NOMINAL_MEMORY_PRESSURE_LEVEL), + swap_used_bytes: Some(0), + swap_free_bytes: Some(1), + swapouts_delta: Some(0), + on_ac_power: Some(true), + thermal_pressure_level: Some(NOMINAL_THERMAL_PRESSURE_LEVEL), + verdict: QuiesceVerdict::Pass, + failing_signals: Vec::new(), + } + } +} + +pub fn parse_memory_pressure_level(text: &str) -> Option { + text.trim().parse::().ok() +} + +/// `sysctl vm.swapusage` → `vm.swapusage: total = 18432.00M used = +/// 17377.19M free = 1054.81M (encrypted)`. +pub fn parse_swapusage(text: &str) -> Option<(u64, u64)> { + let used = parse_megabyte_field(text, "used = ")?; + let free = parse_megabyte_field(text, "free = ")?; + Some((used, free)) +} + +fn parse_megabyte_field(text: &str, label: &str) -> Option { + let after = text.split(label).nth(1)?; + let token = after + .split(|c: char| c == 'M' || c.is_whitespace()) + .next()?; + let megabytes: f64 = token.parse().ok()?; + Some((megabytes * 1024.0 * 1024.0) as u64) +} + +/// `pmset -g ps` first line: `Now drawing from 'AC Power'` or `'Battery +/// Power'`. +pub fn parse_power_source(text: &str) -> Option { + let first_line = text.lines().next()?; + if first_line.contains("'AC Power'") { + Some(true) + } else if first_line.contains("'Battery Power'") { + Some(false) + } else { + None + } +} + +/// `notifyutil -g com.apple.system.thermalpressurelevel` prints +/// `com.apple.system.thermalpressurelevel 0` -- **space-separated**, not +/// colon-separated, unlike most of this crate's other parsers (design §5.7, +/// verified on the host). +pub fn parse_thermal_pressure_level(text: &str) -> Option { + let trimmed = text.trim(); + let value = trimmed.rsplit(' ').next()?; + value.parse::().ok() +} + +/// Reads all five signals and computes the pass/fail verdict. Any signal +/// that fails to parse is `unknown`, which fails the gate rather than +/// passing it by omission. +pub fn read_quiesce_gate(swapouts_delta: Option) -> QuiesceReading { + let pressure = run_capture(SYSCTL_PROGRAM, &[ + "-n", + "kern.memorystatus_vm_pressure_level", + ]) + .ok() + .and_then(|o| parse_memory_pressure_level(&o.stdout)); + + let swap = run_capture(SYSCTL_PROGRAM, &["vm.swapusage"]) + .ok() + .and_then(|o| parse_swapusage(&o.stdout)); + + let on_ac = run_capture(PMSET_PROGRAM, &["-g", "ps"]) + .ok() + .and_then(|o| parse_power_source(&o.stdout)); + + let thermal = run_capture(NOTIFYUTIL_PROGRAM, &[ + "-g", + "com.apple.system.thermalpressurelevel", + ]) + .ok() + .and_then(|o| parse_thermal_pressure_level(&o.stdout)); + + build_reading( + pressure, + swap.map(|(used, _)| used), + swap.map(|(_, free)| free), + swapouts_delta, + on_ac, + thermal, + ) +} + +pub fn build_reading( + memory_pressure_level: Option, + swap_used_bytes: Option, + swap_free_bytes: Option, + swapouts_delta: Option, + on_ac_power: Option, + thermal_pressure_level: Option, +) -> QuiesceReading { + let mut failing = Vec::new(); + + match memory_pressure_level { + Some(level) if level == NOMINAL_MEMORY_PRESSURE_LEVEL => {} + _ => failing.push("memory-pressure".to_string()), + } + match swap_used_bytes { + Some(used) if used < QUIESCE_SWAP_USED_LIMIT_BYTES => {} + _ => failing.push("swap-level".to_string()), + } + match swapouts_delta { + Some(0) => {} + _ => failing.push("swap-activity".to_string()), + } + match on_ac_power { + Some(true) => {} + _ => failing.push("power-source".to_string()), + } + match thermal_pressure_level { + Some(level) if level == NOMINAL_THERMAL_PRESSURE_LEVEL => {} + _ => failing.push("thermal-pressure".to_string()), + } + + let verdict = if failing.is_empty() { + QuiesceVerdict::Pass + } else { + QuiesceVerdict::Fail + }; + + QuiesceReading { + memory_pressure_level, + swap_used_bytes, + swap_free_bytes, + swapouts_delta, + on_ac_power, + thermal_pressure_level, + verdict, + failing_signals: failing, + } +} + +/// What a runner must actually do to clear one failing quiesce signal. +/// +/// Spec item 6: `doctor` is a third party's front door, so naming a +/// failing signal is not enough -- it must say how to clear it. The +/// signal names are the ones [`build_reading`] pushes. +pub fn remediation_for_signal(signal: &str) -> &'static str { + match signal { + "memory-pressure" => { + "quit other applications and re-check; the gate needs \ + `sysctl kern.memorystatus_vm_pressure_level` to read 1 (nominal)" + } + "swap-level" => { + "reboot to clear swap; the gate needs `sysctl vm.swapusage` to show \ + under 8 GB used" + } + "swap-activity" => { + "the machine is actively swapping -- reboot, then leave it idle until \ + `sysctl vm.swapusage` stops reporting new swapouts" + } + "power-source" => "connect the machine to AC power", + "thermal-pressure" => { + "let the machine cool until `notifyutil -g \ + com.apple.system.thermalpressurelevel` reads 0 (nominal)" + } + _ => "see the harness README's quiesce prerequisites", + } +} + +#[cfg(test)] +mod tests { + + /// Every signal `build_reading` can push must have a real remedy -- + /// `doctor` is a third party's only guidance for clearing the gate, so a + /// new signal falling through to the generic message is a regression. + #[test] + fn every_failing_signal_has_a_specific_remediation() { + let reading = build_reading( + Some(4), // memory pressure: not nominal + Some(u64::MAX), // swap used: over the limit + Some(0), + Some(9), // swapouts happened + Some(false), // on battery + Some(3), // thermal pressure: not nominal + ); + assert_eq!( + reading.failing_signals.len(), + 5, + "expected every signal to fail: {:?}", + reading.failing_signals + ); + let generic = remediation_for_signal("not-a-real-signal"); + for signal in &reading.failing_signals { + let remedy = remediation_for_signal(signal); + assert_ne!( + remedy, generic, + "signal {signal} falls through to the generic remediation" + ); + assert!( + !remedy.is_empty(), + "signal {signal} has an empty remediation" + ); + } + } + + use super::*; + use std::fs; + + fn read(name: &str) -> String { + fs::read_to_string(format!( + "{}/fixtures/{name}", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap() + } + + #[test] + fn parses_swapusage_into_bytes() { + let (used, free) = parse_swapusage(&read("sysctl-swapusage.txt")).unwrap(); + assert_eq!(used, (17377.19 * 1024.0 * 1024.0) as u64); + assert_eq!(free, (1054.81 * 1024.0 * 1024.0) as u64); + } + + #[test] + fn parses_ac_power_source() { + assert_eq!(parse_power_source(&read("pmset-ps-ac.txt")), Some(true)); + } + + /// Live capture from the actual development host (`pmset -g ps`, macOS + /// 15.7.4/24G517, the same OS build this crate's other fixtures target), + /// taken while investigating a report that `doctor`'s `power-source` + /// check disagreed with `pmset -g batt`. Both commands were verified to + /// print byte-identical output on this host (`diff <(pmset -g ps) + /// <(pmset -g batt)` -> no difference) at every reading taken during the + /// investigation, so there is no format divergence between the two + /// invocations for this parser to handle differently -- the discrepancy + /// observed at some earlier moment was the host's actual power state + /// changing (unplugged) between then and when `doctor` was run, not a + /// parsing or invocation bug. This fixture pins the real captured + /// battery-power shape (a tab before the percentage, "discharging", a + /// "present: true" suffix) so a future macOS release that changes it is + /// caught here rather than only in a live `doctor` run. + #[test] + fn parses_a_live_captured_battery_reading_from_this_host() { + assert_eq!( + parse_power_source(&read("pmset-ps-battery-live-capture.txt")), + Some(false) + ); + } + + #[test] + fn parses_battery_power_source() { + assert_eq!( + parse_power_source(&read("pmset-ps-battery.txt")), + Some(false) + ); + } + + #[test] + fn parses_nominal_thermal_pressure_space_separated() { + assert_eq!( + parse_thermal_pressure_level(&read("notifyutil-thermal-nominal.txt")), + Some(0) + ); + } + + #[test] + fn parses_serious_thermal_pressure() { + assert_eq!( + parse_thermal_pressure_level(&read("notifyutil-thermal-serious.txt")), + Some(2) + ); + } + + #[test] + fn unparseable_thermal_value_is_unknown_and_fails_the_gate() { + assert_eq!(parse_thermal_pressure_level("garbage, not a number"), None); + let reading = build_reading( + Some(NOMINAL_MEMORY_PRESSURE_LEVEL), + Some(0), + Some(1), + Some(0), + Some(true), + None, + ); + assert_eq!(reading.verdict, QuiesceVerdict::Fail); + assert!(reading + .failing_signals + .contains(&"thermal-pressure".to_string())); + } + + #[test] + fn all_five_signals_nominal_passes() { + let reading = build_reading( + Some(NOMINAL_MEMORY_PRESSURE_LEVEL), + Some(0), + Some(1), + Some(0), + Some(true), + Some(NOMINAL_THERMAL_PRESSURE_LEVEL), + ); + assert_eq!(reading.verdict, QuiesceVerdict::Pass); + assert!(reading.failing_signals.is_empty()); + } + + #[test] + fn memory_pressure_level_two_fails_the_gate() { + // The host measured `kern.memorystatus_vm_pressure_level: 2` while this + // design was written (design §5.7) -- the gate must refuse to start. + let reading = build_reading( + Some(2), + Some(0), + Some(1), + Some(0), + Some(true), + Some(NOMINAL_THERMAL_PRESSURE_LEVEL), + ); + assert_eq!(reading.verdict, QuiesceVerdict::Fail); + assert!(reading + .failing_signals + .contains(&"memory-pressure".to_string())); + } +} diff --git a/benchmark/src/render.rs b/benchmark/src/render.rs new file mode 100644 index 0000000..8e19b96 --- /dev/null +++ b/benchmark/src/render.rs @@ -0,0 +1,635 @@ +//! The table renderer (spec FR-13, FR-14): `fn render(&ResultFile) -> +//! String`, a **pure function** of the parsed result file with no +//! filesystem or process access, so "table rendering is a pure function of +//! the JSON result file" is enforced by the signature (design §5.10). +//! +//! `render` never computes a percentage delta and never emits comparative +//! adjectives (FR-12) -- it renders medians, IQRs, and counts. + +use crate::result::ResultFile; +use crate::subject::{EXCLUSIONS, HOLD_BACKS}; +use std::collections::BTreeMap; +use std::fmt::Write as _; + +pub fn render(result: &ResultFile) -> String { + let mut out = String::new(); + render_provenance(&mut out, result); + render_tier_tables(&mut out, result); + render_discards_table(&mut out, result); + render_attribution_evidence(&mut out, result); + render_limitations(&mut out, result); + render_exclusions(&mut out); + render_fairness_review(&mut out, result); + out +} + +fn render_provenance(out: &mut String, result: &ResultFile) { + let _ = writeln!(out, "# Resource Benchmark Result: {}", result.run_id); + let _ = writeln!(out); + let _ = writeln!(out, "## Provenance"); + let _ = writeln!(out); + let _ = writeln!( + out, + "- Machine: {} ({} logical / {} physical cores, {} B RAM)", + result.machine_spec.cpu_brand, + result.machine_spec.logical_cores, + result.machine_spec.physical_cores, + result.machine_spec.ram_bytes + ); + let _ = writeln!( + out, + "- OS build: {} {} ({})", + result.os_build.product_name, + result.os_build.product_version, + result.os_build.build_version + ); + let _ = writeln!( + out, + "- Agent CLI: {} {}", + result.agent_cli_version.name, result.agent_cli_version.version + ); + let _ = writeln!( + out, + "- Repo ref: {}@{}", + result.repo_ref.url, result.repo_ref.commit + ); + let _ = writeln!(out, "- Run timestamp: {}", result.run_timestamp); + for subject in &result.subjects { + let _ = writeln!( + out, + "- Subject: {} {}", + subject.display_name, subject.subject_version + ); + } + let _ = writeln!(out); +} + +fn render_tier_tables(out: &mut String, result: &ResultFile) { + let mut by_tier: BTreeMap> = + BTreeMap::new(); + for aggregate in &result.aggregates { + by_tier + .entry(aggregate.tier.clone()) + .or_default() + .push(aggregate); + } + for (tier, aggregates) in &by_tier { + let _ = writeln!(out, "## Tier: {tier}"); + let _ = writeln!(out); + let _ = writeln!( + out, + "| Subject | Metric | Median | Q1 | Q3 | IQR | n | Discarded |" + ); + let _ = writeln!(out, "|---|---|---|---|---|---|---|---|"); + for aggregate in aggregates { + let _ = writeln!( + out, + "| {} | {} | {} | {} | {} | {} | {} | {} |", + aggregate.subject_id, + aggregate.metric, + aggregate.median, + aggregate.q1, + aggregate.q3, + aggregate.iqr, + aggregate.n, + aggregate.discarded_count + ); + } + let _ = writeln!(out); + } + // Every repetition count is disclosed here, per subject/tier, whether or + // not any aggregate rows exist yet -- an undisclosed shortfall is what + // FR-12/NFR-2 forbid. + let _ = writeln!(out, "## Repetition Counts (disclosed per tier)"); + let _ = writeln!(out); + let _ = writeln!( + out, + "- fresh-launch: n={}", + result.settings.repetitions.fresh_launch + ); + let _ = writeln!( + out, + "- sustained-use: n={}", + result.settings.repetitions.sustained_use + ); + let _ = writeln!( + out, + "- n-session-*: n={}", + result.settings.repetitions.n_session + ); + let _ = writeln!(out); +} + +fn render_discards_table(out: &mut String, result: &ResultFile) { + let _ = writeln!(out, "## Discards"); + let _ = writeln!(out); + let _ = writeln!(out, "| Subject | Tier | n | Discarded | Reasons |"); + let _ = writeln!(out, "|---|---|---|---|---|"); + for aggregate in &result.aggregates { + let reasons: Vec = aggregate + .discarded_reasons + .iter() + .map(|(reason, count)| format!("{reason}: {count}")) + .collect(); + let _ = writeln!( + out, + "| {} | {} | {} | {} | {} |", + aggregate.subject_id, + aggregate.tier, + aggregate.n, + aggregate.discarded_count, + reasons.join(", ") + ); + } + let _ = writeln!(out); +} + +fn render_attribution_evidence(out: &mut String, result: &ResultFile) { + let _ = writeln!(out, "## Attribution Evidence"); + let _ = writeln!(out); + let _ = writeln!(out, "| Subject | launch-services | process-tree | both |"); + let _ = writeln!(out, "|---|---|---|---|"); + let mut by_subject: BTreeMap = BTreeMap::new(); + for sample in &result.samples { + let Some(attribution) = &sample.attribution else { + continue; + }; + let entry = by_subject.entry(sample.subject_id.clone()).or_default(); + for process in &attribution.processes { + match process.discovered_by.as_str() { + "launch-services" => entry.0 += 1, + "process-tree" => entry.1 += 1, + "both" => entry.2 += 1, + _ => {} + } + } + } + for (subject, (ls, tree, both)) in &by_subject { + let _ = writeln!(out, "| {subject} | {ls} | {tree} | {both} |"); + } + let _ = writeln!(out); +} + +fn has_tier(result: &ResultFile, tier_prefix: &str) -> bool { + result.samples.iter().any(|s| s.tier == tier_prefix) + || result.aggregates.iter().any(|a| a.tier == tier_prefix) +} + +fn has_any_n_session_tier(result: &ResultFile) -> bool { + result + .samples + .iter() + .any(|s| s.tier.starts_with("n-session-")) + || result + .aggregates + .iter() + .any(|a| a.tier.starts_with("n-session-")) +} + +fn has_subject_runtime_family(result: &ResultFile, family: &str) -> bool { + result.subjects.iter().any(|s| s.runtime_family == family) +} + +fn has_termtree_cold_start_row(result: &ResultFile) -> bool { + result.subjects.iter().any(|s| s.subject_id == "termtree") + && result + .samples + .iter() + .any(|s| s.subject_id == "termtree" && s.cold_start.is_some()) +} + +fn has_idle_cpu_row(result: &ResultFile) -> bool { + result.samples.iter().any(|s| s.idle_cpu.is_some()) +} + +fn has_rss_field(result: &ResultFile) -> bool { + result.samples.iter().any(|s| s.memory.is_some()) +} + +fn unverified_seed_format_display_names(result: &ResultFile) -> Vec<&str> { + result + .subjects + .iter() + .filter(|s| !s.seed_format_verified) + .map(|s| s.display_name.as_str()) + .collect() +} + +/// Each limitation is emitted by a rule keyed on what is present in the +/// data, so it cannot be omitted by editing prose (spec FR-14, design +/// §5.10). +fn render_limitations(out: &mut String, result: &ResultFile) { + let _ = writeln!(out, "## Limitations"); + let _ = writeln!(out); + let _ = writeln!( + out, + "- **Attribution asymmetry**: `launchd`-parented (WebKit) helper \ + processes and app-parented (Chromium/Electron) helper processes are \ + resolved by two different unprivileged mechanisms, unioned and \ + deduplicated (design §5.2). This union/partition approach is macOS-specific." + ); + if has_rss_field(result) { + let _ = writeln!( + out, + "- **phys_footprint vs RSS**: `memRssBytes` (naive per-process sum) \ + diverges materially from `memPhysFootprintBytes` (footprint's \ + deduplicated set total) and the choice can change which subject \ + looks better; RSS is never the primary published figure." + ); + } + if has_termtree_cold_start_row(result) { + let _ = writeln!( + out, + "- **TermTree's cold-start marks are self-reported and \ + tail-trimmed**: `appWindowReadyMs` and `splashCloseMs` are read from \ + TermTree's own log lines, so no other subject has an equivalent and \ + only the externally probed `mainWindowVisibleMs` is comparable \ + across subjects. Neither mark proves a painted first frame: the \ + window is still hidden when `app_window_ready` runs \ + (`src-tauri/src/command/window_cmd.rs:465-494`), so \ + `requestAnimationFrame` has not yet fired. `splashCloseMs` is \ + readiness-driven, not floored -- taskhub#672 removed the 2,000 ms \ + minimum splash and the fixed 500 ms pre-show sleep that used to pad \ + it -- but a launch whose frontend never signals readiness hits the \ + 15 s forced transition in `start_splash_monitor` \ + (`src-tauri/src/lib.rs:104-143`) and is excluded from the aggregate \ + as a `splash-timeout` sample (FR-12): a discard reason no other \ + subject can trigger, so TermTree's slowest launches leave the \ + aggregate (counted in the Discards table) while other subjects' \ + slowest launches stay in." + ); + } + if has_tier(result, "fresh-launch") { + let _ = writeln!( + out, + "- **Fresh-launch-only would flatter TermTree**: TermTree's own \ + disclosed measurement after 8 days of uptime showed 3.6-3.9 GB of \ + phys_footprint across six attributed processes, of which the Rust \ + process itself was ~74 MB, and the figure was still climbing \ + within a single session." + ); + } + if has_idle_cpu_row(result) { + let _ = writeln!( + out, + "- **TermTree's unavoidable idle work**: the 5 s terminal idle sweep, \ + the 5 s wake-gap timer, the 30 s autosave, the hourly updater check, \ + and the 60 s macOS webview health probe are all plausible sources \ + of idle-CPU loss. The mind map renders on demand — frames are \ + produced only while something is animating, being interacted with, \ + or has changed and not yet reached the canvas — so a settled map \ + costs nothing; a map with a session at status running or waiting \ + does render at animation rate, because its status dot is pulsing." + ); + } + if has_any_n_session_tier(result) { + let _ = writeln!( + out, + "- **N-session scaling is expected to narrow, not favor, TermTree**: \ + WKWebView content processes are not shared across sessions (Tauri \ + #5031), so scaling is not expected to be sublinear; this is not \ + framed as a surprise result." + ); + } + if has_subject_runtime_family(result, "chromium-electron") { + let _ = writeln!( + out, + "- **The orchestrator/agent-CLI partition costs Collaborator**: its \ + vendored `tmux` and `node-pty` sidecar are `Orchestrator` under the \ + one partition rule applied to every subject, because neither is a \ + session root -- they are Collaborator's own implementation choice \ + for a job TermTree does in-process with `portable-pty`." + ); + } + if has_tier(result, "sustained-use") { + let _ = writeln!( + out, + "- **The sustained-use tier exercises orchestrator, terminal-output, \ + and rendering paths, not agent inference**: real prompts are \ + deliberately not sent, so this tier is a floor on the long-uptime \ + effect it exists to surface, not a measurement of it." + ); + } + let unverified = unverified_seed_format_display_names(result); + if !unverified.is_empty() { + let _ = writeln!( + out, + "- **Unverified seed formats**: {}'s session seeder has not been \ + checked against a real install (see the seeder's module doc under \ + `src/seeding/`). Every N-session/sustained-use sample for it \ + reports `invalidReason: \"seed-format-unverified\"` until this is \ + verified and the registry is updated.", + unverified.join(", ") + ); + } + let _ = writeln!( + out, + "- **Open question**: whether App Nap or windowing-engine occlusion \ + throttling makes the fixed foreground/unoccluded idle-CPU state \ + itself non-representative of real-world usage is not resolved by \ + this harness." + ); + let _ = writeln!(out); +} + +fn render_exclusions(out: &mut String) { + let _ = writeln!(out, "## Excluded Subjects"); + let _ = writeln!(out); + for exclusion in EXCLUSIONS { + let _ = writeln!(out, "- **{}**: {}", exclusion.name, exclusion.reason); + } + let _ = writeln!(out); + let _ = writeln!(out, "## Hold-back Subjects"); + let _ = writeln!(out); + for hold_back in HOLD_BACKS { + let _ = writeln!(out, "- **{}**: {}", hold_back.name, hold_back.reason); + } + let _ = writeln!(out); +} + +fn render_fairness_review(out: &mut String, result: &ResultFile) { + let _ = writeln!(out, "## Fairness Review"); + let _ = writeln!(out); + match ( + &result.fairness_review.reviewer, + &result.fairness_review.reviewed_at, + ) { + (Some(reviewer), Some(date)) => { + let _ = writeln!(out, "Reviewed by {reviewer} on {date}."); + if let Some(notes) = &result.fairness_review.notes { + let _ = writeln!(out); + let _ = writeln!(out, "{notes}"); + } + } + _ => { + let _ = writeln!(out, "**NOT REVIEWED — DO NOT PUBLISH**"); + } + } + let _ = writeln!(out); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::quiesce::QuiesceReading; + use crate::result::*; + use crate::settings::RunSettings; + + fn base_result() -> ResultFile { + ResultFile { + schema_version: SCHEMA_VERSION, + run_id: "test-run".into(), + run_timestamp: "2026-08-25T00:00:00Z".into(), + harness_ref: HarnessRef { + repo: "termtree".into(), + commit: "abc".into(), + crate_version: "0.1.0".into(), + }, + machine_spec: MachineSpec { + cpu_brand: "Apple M1".into(), + logical_cores: 8, + physical_cores: 8, + ram_bytes: 17_179_869_184, + page_size_bytes: 16384, + }, + os_build: OsBuild { + product_name: "macOS".into(), + product_version: "15.7.4".into(), + build_version: "24G517".into(), + }, + agent_cli_version: AgentCliVersion { + name: "claude".into(), + version: "1.0.0".into(), + executable_path: "/Users/dev/.local/bin/claude".into(), + }, + repo_ref: RepoRef { + url: "https://example.com/repo.git".into(), + commit: "abc123".into(), + local_path: "/Users/Shared/benchmark-repo".into(), + }, + login_shell_path: "/bin/zsh".into(), + settings: RunSettings::default(), + subjects: vec![], + quiesce: RunQuiesce { + pre_run: QuiesceReading::nominal_for_test(), + verdict: "pass".into(), + }, + samples: vec![], + aggregates: vec![], + fairness_review: FairnessReview::default(), + } + } + + fn termtree_subject() -> SubjectProvenance { + SubjectProvenance { + subject_id: "termtree".into(), + display_name: "TermTree".into(), + subject_version: "1.0.0".into(), + runtime_family: "webkit-tauri".into(), + bundle_identifier: "com.termtree.desktop".into(), + bundle_path: "/Applications/TermTree.app".into(), + optional: false, + seeder: "termtree-state-json".into(), + seed_method: "production state.json pre-write".into(), + calibrated_main_window_area_pt: Some(1_310_720.0), + version_drift_accepted: false, + seed_format_verified: true, + } + } + + fn cold_start_sample() -> Sample { + Sample { + sample_id: "termtree/fresh-launch/001".into(), + subject_id: "termtree".into(), + tier: "fresh-launch".into(), + session_count: 0, + repetition: 1, + is_calibration: false, + sampled_at: "2026-08-25T00:00:00Z".into(), + is_valid: true, + invalid_reason: None, + attribution: None, + memory: None, + cold_start: Some(ColdStartRecord { + first_window_visible_ms: Some(400), + main_window_visible_ms: Some(2100), + app_window_ready_ms: Some(1580), + splash_close_ms: Some(2100), + mark_source: Default::default(), + mark_resolution_ms: 20, + }), + idle_cpu: None, + quiesce: None, + warm_helper_count: None, + helper_kill_count: None, + } + } + + #[test] + fn always_contains_a_limitations_section() { + let result = base_result(); + let rendered = render(&result); + assert!(rendered.contains("## Limitations")); + } + + /// taskhub#671 (FR-11): the idle-CPU limitation is a published claim about + /// how TermTree behaves, hardcoded here rather than in prose, so it goes + /// stale silently. It must describe render-on-demand, and must not resurrect + /// the pre-#671 "unconditional ticker" claim. + #[test] + fn idle_cpu_limitation_describes_render_on_demand() { + let mut result = base_result(); + let without = render(&result); + assert!(!without.contains("idle work")); + + result.subjects.push(termtree_subject()); + let mut sample = cold_start_sample(); + sample.cold_start = None; + sample.idle_cpu = Some(IdleCpuRecord { + idle_cpu_percent_of_one_core_median: 0.4, + idle_cpu_percent_of_one_core_iqr: 0.1, + sample_count: 30, + window_state: "foreground-unoccluded".into(), + }); + result.samples.push(sample); + + let with = render(&result); + assert!(with.contains("renders on demand")); + assert!(!with.contains("unconditionally at display rate")); + // The honest caveat (FR-6) and the other periodic work stay disclosed. + assert!(with.contains("running or waiting")); + assert!(with.contains("5 s terminal idle sweep")); + assert!(with.contains("60 s macOS webview health probe")); + } + + /// taskhub#672 deleted the 2,000 ms minimum splash and the 500 ms + /// pre-show sleep this rule used to disclose. The rule survives because a + /// cold-start bias remains -- self-reported marks and a TermTree-only + /// `splash-timeout` discard -- so this pins the corrected claim, its code + /// citations, and the absence of the superseded one. + #[test] + fn cold_start_mark_line_only_appears_with_a_termtree_cold_start_row() { + let mut result = base_result(); + let without = render(&result); + assert!(!without.contains("self-reported and tail-trimmed")); + assert!(!without.contains("window_cmd.rs:465-494")); + + result.subjects.push(termtree_subject()); + result.samples.push(cold_start_sample()); + let with = render(&result); + assert!(with.contains("self-reported and tail-trimmed")); + assert!(with.contains("src-tauri/src/lib.rs:104-143")); + assert!(with.contains("src-tauri/src/command/window_cmd.rs:465-494")); + assert!(with.contains("15 s forced transition")); + assert!(with.contains("`splash-timeout` sample")); + // The two superseded claims, verbatim as they were once published. + assert!(!with.contains("enforces a 2,000 ms minimum splash")); + assert!(!with.contains("sleeps a fixed 500 ms")); + } + + #[test] + fn tmux_partition_disclosure_only_with_a_chromium_subject() { + let mut result = base_result(); + let without = render(&result); + assert!(!without.contains("node-pty")); + + result.subjects.push(SubjectProvenance { + subject_id: "collaborator".into(), + display_name: "Collaborator".into(), + subject_version: "0.8.4".into(), + runtime_family: "chromium-electron".into(), + bundle_identifier: "com.collaborator.desktop".into(), + bundle_path: "/Applications/Collaborator.app".into(), + optional: false, + seeder: "collaborator".into(), + seed_method: "canvas.json pre-write".into(), + calibrated_main_window_area_pt: None, + version_drift_accepted: false, + seed_format_verified: false, + }); + let with = render(&result); + assert!(with.contains("node-pty")); + } + + #[test] + fn fairness_banner_appears_when_reviewer_is_absent() { + let result = base_result(); + let rendered = render(&result); + assert!(rendered.contains("NOT REVIEWED")); + } + + #[test] + fn fairness_review_renders_when_present() { + let mut result = base_result(); + result.fairness_review = FairnessReview { + reviewer: Some("Jane Reviewer".into()), + reviewed_at: Some("2026-08-25".into()), + verdict: Some("pass".into()), + notes: Some("Checked the Limitations section against raw data.".into()), + }; + let rendered = render(&result); + assert!(!rendered.contains("NOT REVIEWED")); + assert!(rendered.contains("Jane Reviewer")); + } + + #[test] + fn never_contains_percent_delta_phrasing_or_an_unlabeled_memory_column() { + let mut result = base_result(); + result.subjects.push(termtree_subject()); + result.samples.push(cold_start_sample()); + result.aggregates.push(Aggregate { + subject_id: "termtree".into(), + tier: "fresh-launch".into(), + metric: "memPhysFootprintBytes".into(), + median: 100.0, + q1: 90.0, + q3: 110.0, + iqr: 20.0, + n: 19, + discarded_count: 1, + discarded_reasons: Default::default(), + derivation: "measured".into(), + }); + let rendered = render(&result); + assert!(!rendered.contains('%')); + assert!(!rendered.lines().any(|l| l.trim() == "| Memory |")); + assert!(!rendered.contains("| Memory | ")); + assert!(!rendered.contains("CPU |\n|---")); + } + + #[test] + fn unverified_seed_format_limitation_names_the_subject() { + let mut result = base_result(); + let without = render(&result); + assert!(!without.contains("Unverified seed formats")); + + result.subjects.push(SubjectProvenance { + subject_id: "collaborator".into(), + display_name: "Collaborator".into(), + subject_version: "0.8.4".into(), + runtime_family: "chromium-electron".into(), + bundle_identifier: "com.collaborator.desktop".into(), + bundle_path: "/Applications/Collaborator.app".into(), + optional: false, + seeder: "collaborator".into(), + seed_method: "canvas.json pre-write".into(), + calibrated_main_window_area_pt: None, + version_drift_accepted: false, + seed_format_verified: false, + }); + result.subjects.push(termtree_subject()); + let with = render(&result); + assert!(with.contains("Unverified seed formats")); + assert!(with.contains("Collaborator")); + // TermTree's seeder IS verified -- it must not be named here. + assert!(!with.contains("Unverified seed formats: TermTree")); + } + + #[test] + fn exclusions_and_hold_backs_are_rendered() { + let result = base_result(); + let rendered = render(&result); + assert!(rendered.contains("Conductor")); + assert!(rendered.contains("Nimbalyst")); + } +} diff --git a/benchmark/src/result.rs b/benchmark/src/result.rs new file mode 100644 index 0000000..4319577 --- /dev/null +++ b/benchmark/src/result.rs @@ -0,0 +1,370 @@ +//! The versioned result schema (design §6) and its crash-safe writer. +//! +//! `schemaVersion: 1`. One file per run at `results/.json`, rewritten +//! in full after every sample (design §5.8 step 8) so an interrupted sweep +//! loses at most one sample. + +use crate::settings::RunSettings; +use serde::{Deserialize, Serialize}; +use std::fs; +use std::io; +use std::path::Path; + +pub const SCHEMA_VERSION: u32 = 1; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct ResultFile { + pub schema_version: u32, + pub run_id: String, + pub run_timestamp: String, + pub harness_ref: HarnessRef, + pub machine_spec: MachineSpec, + pub os_build: OsBuild, + pub agent_cli_version: AgentCliVersion, + pub repo_ref: RepoRef, + pub login_shell_path: String, + pub settings: RunSettings, + pub subjects: Vec, + pub quiesce: RunQuiesce, + pub samples: Vec, + pub aggregates: Vec, + pub fairness_review: FairnessReview, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct HarnessRef { + pub repo: String, + pub commit: String, + pub crate_version: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct MachineSpec { + pub cpu_brand: String, + pub logical_cores: u32, + pub physical_cores: u32, + pub ram_bytes: u64, + pub page_size_bytes: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct OsBuild { + pub product_name: String, + pub product_version: String, + pub build_version: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct AgentCliVersion { + pub name: String, + pub version: String, + pub executable_path: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct RepoRef { + pub url: String, + pub commit: String, + pub local_path: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct SubjectProvenance { + pub subject_id: String, + pub display_name: String, + pub subject_version: String, + pub runtime_family: String, + pub bundle_identifier: String, + pub bundle_path: String, + pub optional: bool, + pub seeder: String, + pub seed_method: String, + pub calibrated_main_window_area_pt: Option, + pub version_drift_accepted: bool, + /// Whether this subject's seeder format has been confirmed against a + /// real install (spec item 4). `false` for Collaborator, CodeNomad, and + /// diri -- their canvas/config-file/CLI-flag formats have never been + /// checked against a real install (see each `seeding/*.rs` module doc). + /// Every N-session/sustained-use sample for a subject with + /// `seedFormatVerified: false` reports + /// `invalidReason: "seed-format-unverified"` until this is flipped to + /// `true` after verification. + pub seed_format_verified: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct RunQuiesce { + pub pre_run: crate::quiesce::QuiesceReading, + pub verdict: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] +#[serde(rename_all = "camelCase")] +pub struct FairnessReview { + pub reviewer: Option, + pub reviewed_at: Option, + pub verdict: Option, + pub notes: Option, +} + +// --- Sample ----------------------------------------------------------- + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct Sample { + pub sample_id: String, + pub subject_id: String, + pub tier: String, + pub session_count: u32, + pub repetition: u32, + pub is_calibration: bool, + pub sampled_at: String, + pub is_valid: bool, + pub invalid_reason: Option, + + pub attribution: Option, + pub memory: Option, + pub cold_start: Option, + pub idle_cpu: Option, + + pub quiesce: Option, + pub warm_helper_count: Option, + pub helper_kill_count: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct AttributionRecord { + pub main_pid: u32, + pub launch_services_pids: Vec, + pub process_tree_pids: Vec, + pub vanished_pids: Vec, + pub orchestrator_pids: Vec, + pub agent_cli_pids: Vec, + pub processes: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct AttributedProcessRecord { + pub pid: u32, + pub name: String, + pub executable_path: Option, + pub discovered_by: String, + pub role: String, + pub phys_footprint_bytes: Option, + pub rss_bytes: Option, +} + +/// Field names below intentionally never merge `memPhysFootprintBytes` +/// (footprint's set-level `total footprint`, shared pages counted once) with +/// `memRssBytes` (naive per-process sum, diverges materially under memory +/// compression) under one unlabeled figure -- spec FR-3, design §5.3. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct MemoryRecord { + pub mem_phys_footprint_bytes: u64, + pub mem_phys_footprint_process_sum_bytes: u64, + pub shared_page_double_count_bytes: u64, + pub cross_partition_shared_bytes: u64, + pub mem_rss_bytes: u64, + pub mem_rss_method: String, + + pub orchestrator_attributable_bytes: u64, + pub agent_cli_attributable_bytes: u64, + + pub core_process_bytes: Option, + pub render_helper_bytes: Option, + + pub free_ram_before_bytes: u64, + pub free_ram_after_bytes: u64, + pub free_ram_delta_bytes: i64, + pub free_ram_delta_sign: String, + + pub host_memory_used_before_bytes: u64, + pub host_memory_used_after_bytes: u64, + pub host_memory_used_delta_bytes: i64, + pub compressor_occupied_delta_bytes: i64, + pub swapouts_delta: u64, + + pub orchestrator_free_ram_delta_bytes: Option, + pub agent_cli_free_ram_delta_bytes: Option, + pub free_ram_split_derivation: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct ColdStartRecord { + pub first_window_visible_ms: Option, + pub main_window_visible_ms: Option, + pub app_window_ready_ms: Option, + pub splash_close_ms: Option, + pub mark_source: std::collections::BTreeMap, + pub mark_resolution_ms: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct IdleCpuRecord { + pub idle_cpu_percent_of_one_core_median: f64, + pub idle_cpu_percent_of_one_core_iqr: f64, + pub sample_count: u32, + pub window_state: String, +} + +// --- Aggregate ---------------------------------------------------------- + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct Aggregate { + pub subject_id: String, + pub tier: String, + pub metric: String, + pub median: f64, + pub q1: f64, + pub q3: f64, + pub iqr: f64, + pub n: u32, + pub discarded_count: u32, + pub discarded_reasons: std::collections::BTreeMap, + pub derivation: String, +} + +/// Reasons a sample is retained but excluded from the aggregate statistic +/// (spec FR-12's three, plus this design's additions -- §6.2). +pub mod invalid_reason { + pub const SPLASH_TIMEOUT: &str = "splash-timeout"; + pub const WARM_WEBVIEW: &str = "warm-webview"; + pub const QUIESCE_VIOLATION: &str = "quiesce-violation"; + pub const ATTRIBUTION_INCOMPLETE: &str = "attribution-incomplete"; + pub const FOOTPRINT_PID_MISMATCH: &str = "footprint-pid-mismatch"; + pub const SEED_INCOMPLETE: &str = "seed-incomplete"; + pub const CALIBRATION_DISCARD: &str = "calibration-discard"; + /// Spec item 4: this subject's seed format has never been checked + /// against a real install (`SubjectProvenance.seed_format_verified`), + /// so any N-session/sustained-use sample it produces is reported + /// unverified rather than valid, even if the generic session-readiness + /// check happened to pass. + pub const SEED_FORMAT_UNVERIFIED: &str = "seed-format-unverified"; + /// Spec item 4: none of this crate's hardcoded `karijini.log` mark + /// strings (`log_marks.rs`) matched any line the harness actually read + /// during the settle window, even though the log did advance -- the + /// TermTree build under test has likely changed its log message text + /// and `log_marks.rs` needs updating, rather than the run silently + /// reporting `null` cold-start marks. + pub const TERMTREE_LOG_MARKS_UNRECOGNIZED: &str = + "termtree-log-marks-unrecognized"; + /// Spec item 4: TermTree launched (a main pid was discovered) but never + /// created `DocumentNode/TermTree` under the scratch home -- the app's + /// data-directory convention has likely changed and + /// `seeding/termtree.rs` / `log_marks.rs`'s hardcoded path needs + /// updating, rather than the run silently measuring the wrong directory. + pub const APP_DATA_DIR_NOT_CREATED: &str = "app-data-dir-not-created"; + /// Spec item 3: a live LaunchServices entry already carried this + /// subject's bundle identifier before the harness tried to seed/launch + /// it -- a single-instance plugin would have handed the launch off and + /// exited within seconds, measuring nothing. + pub const SUBJECT_ALREADY_RUNNING: &str = "subject-already-running"; +} + +/// Rewrite the whole result file, atomically, after every sample (design +/// §5.8 step 8). A partial write on crash/power-loss can never leave a +/// corrupt result file behind, since the temp file is renamed into place +/// only after a full successful write. +pub fn write_result_file(path: &Path, result: &ResultFile) -> io::Result<()> { + let json = serde_json::to_string_pretty(result) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + let tmp_path = path.with_extension("json.tmp"); + fs::write(&tmp_path, json)?; + fs::rename(&tmp_path, path) +} + +pub fn read_result_file(path: &Path) -> io::Result { + let text = fs::read_to_string(path)?; + serde_json::from_str(&text) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::quiesce::QuiesceReading; + use crate::settings::RunSettings; + + fn sample_result_file() -> ResultFile { + ResultFile { + schema_version: SCHEMA_VERSION, + run_id: "2026-08-25T02-10-44Z-9f3c1a7e".into(), + run_timestamp: "2026-08-25T02:10:44Z".into(), + harness_ref: HarnessRef { + repo: "termtree".into(), + commit: "f2ce896".into(), + crate_version: "0.1.0".into(), + }, + machine_spec: MachineSpec { + cpu_brand: "Apple M1".into(), + logical_cores: 8, + physical_cores: 8, + ram_bytes: 17_179_869_184, + page_size_bytes: 16384, + }, + os_build: OsBuild { + product_name: "macOS".into(), + product_version: "15.7.4".into(), + build_version: "24G517".into(), + }, + agent_cli_version: AgentCliVersion { + name: "claude".into(), + version: "1.0.0".into(), + executable_path: "/Users/dev/.local/bin/claude".into(), + }, + repo_ref: RepoRef { + url: "https://github.com/example/benchmark-repo".into(), + commit: "abc1234".into(), + local_path: "/Users/Shared/benchmark-repo".into(), + }, + login_shell_path: "/bin/zsh".into(), + settings: RunSettings::default(), + subjects: vec![], + quiesce: RunQuiesce { + pre_run: QuiesceReading::nominal_for_test(), + verdict: "pass".into(), + }, + samples: vec![], + aggregates: vec![], + fairness_review: FairnessReview::default(), + } + } + + #[test] + fn schema_round_trips_through_json() { + let result = sample_result_file(); + let json = serde_json::to_string(&result).unwrap(); + let parsed: ResultFile = serde_json::from_str(&json).unwrap(); + assert_eq!(result, parsed); + } + + #[test] + fn write_then_read_round_trips_and_is_atomic() { + let dir = std::env::temp_dir().join(format!( + "resource-benchmark-result-test-{}", + std::process::id() + )); + fs::create_dir_all(&dir).unwrap(); + let path = dir.join("run.json"); + let result = sample_result_file(); + write_result_file(&path, &result).unwrap(); + assert!(!path.with_extension("json.tmp").exists()); + let read_back = read_result_file(&path).unwrap(); + assert_eq!(result, read_back); + fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/benchmark/src/run.rs b/benchmark/src/run.rs new file mode 100644 index 0000000..59c636d --- /dev/null +++ b/benchmark/src/run.rs @@ -0,0 +1,1965 @@ +//! `RunOrchestrator` (spec FR-6, FR-12, design §5.8): repetitions, +//! calibration launch, warm-helper check, teardown, invalid-sample +//! handling, and crash-safe append. +//! +//! Sequencing, tier expansion, repetition accounting, resume-state +//! reconciliation, validity classification, and record assembly are all +//! pure functions in this module, each unit-tested below. Only the actual +//! spawn-and-wait glue in [`RunOrchestrator::run`] and its private +//! `measure_one`/`measure_cold_start`/`teardown` methods is live, +//! untestable orchestration (design §11: "not unit-tested, by design: +//! launching subjects ... live `footprint`/`vm_stat` invocation") -- +//! everything it decides, it decides by calling one of the pure functions +//! below. + +use crate::attribution::{self, AttributableProcessSet, DiscoverySource}; +use crate::bundle_paths::{self, BundlePathOverrides}; +use crate::cold_start; +use crate::cpu_sampler; +use crate::exec::run_capture; +use crate::footprint::{self, FootprintReport}; +use crate::host_memory::{self, HostMemorySample}; +use crate::launch_services::{self, LaunchServicesEntry}; +use crate::log_marks::{self, LogMark}; +use crate::process_tree::{self, ProcessRecord}; +use crate::provenance; +use crate::quiesce::{self, QuiesceReading, QuiesceVerdict}; +use crate::result::{ + self, invalid_reason, AttributedProcessRecord, AttributionRecord, + ColdStartRecord, FairnessReview, IdleCpuRecord, MemoryRecord, ResultFile, + RunQuiesce, Sample, SubjectProvenance, +}; +use crate::seeding::{self, AgentCliPin, SeededRepo}; +use crate::settings::{RunSettings, TierRepetitions}; +use crate::stats; +use crate::subject::{self, RuntimeFamily, SubjectSpec, SUBJECTS}; +use crate::tier::Tier; +use crate::window_probe; +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread; +use std::time::{Duration, Instant}; + +/// A subject is **warm** (not eligible for a fresh-launch sample, spec +/// FR-6) if any of its helper processes are still resident from a prior +/// run. For `WebKitTauri` subjects that means a LaunchServices entry under +/// the subject's display name with a bundle ID in `helper_bundle_ids`; for +/// `ChromiumElectron` subjects it means a resident process whose +/// executable path is under `bundle_path` (checked via `process_tree.rs` +/// upstream of this function -- passed in as `chromium_helper_present`). +pub fn has_warm_helpers( + subject: &SubjectSpec, + ls_entries: &[LaunchServicesEntry], + chromium_helper_present: bool, +) -> bool { + match subject.runtime_family { + RuntimeFamily::WebKitTauri => { + let prefix = format!("{} ", subject.launch_services_name); + ls_entries.iter().any(|entry| { + entry.pid.is_some() + && entry.display_name.starts_with(&prefix) + && entry + .bundle_identifier + .as_deref() + .is_some_and(|id| subject.helper_bundle_ids.contains(&id)) + }) + } + RuntimeFamily::ChromiumElectron | RuntimeFamily::GpuiNative => { + chromium_helper_present + } + } +} + +/// The invalid reason a sample should carry, in priority order, given the +/// signals the orchestrator collected for it (design §5.8, §9). Pure so the +/// decision logic itself is testable even though the signals it reads come +/// from live measurement. +#[allow(clippy::too_many_arguments)] +pub fn classify_sample_invalidity( + is_calibration: bool, + subject_already_running: bool, + warm_helper_count: u32, + splash_timeout_seen: bool, + quiesce_violation_seen: bool, + attribution_incomplete: bool, + footprint_pid_mismatch: bool, + seed_incomplete: bool, + seed_format_unverified: bool, + log_marks_unrecognized: bool, + app_data_dir_missing: bool, +) -> Option<&'static str> { + if is_calibration { + return Some(invalid_reason::CALIBRATION_DISCARD); + } + if subject_already_running { + return Some(invalid_reason::SUBJECT_ALREADY_RUNNING); + } + if seed_incomplete { + return Some(invalid_reason::SEED_INCOMPLETE); + } + if seed_format_unverified { + return Some(invalid_reason::SEED_FORMAT_UNVERIFIED); + } + if app_data_dir_missing { + return Some(invalid_reason::APP_DATA_DIR_NOT_CREATED); + } + if log_marks_unrecognized { + return Some(invalid_reason::TERMTREE_LOG_MARKS_UNRECOGNIZED); + } + if warm_helper_count > 0 { + return Some(invalid_reason::WARM_WEBVIEW); + } + if splash_timeout_seen { + return Some(invalid_reason::SPLASH_TIMEOUT); + } + if attribution_incomplete { + return Some(invalid_reason::ATTRIBUTION_INCOMPLETE); + } + if footprint_pid_mismatch { + return Some(invalid_reason::FOOTPRINT_PID_MISMATCH); + } + if quiesce_violation_seen { + return Some(invalid_reason::QUIESCE_VIOLATION); + } + None +} + +// --- Refusals (spec: an unimplemented or refused run must never exit 0) -- + +/// Why a run refused to start, or refused to include a subject. Every +/// variant carries what a re-runner needs to act on it -- `main.rs` prints +/// [`std::fmt::Display`] to stderr and exits non-zero. +#[derive(Debug)] +pub enum RunRefusal { + QuiesceGateFailed(Vec), + UnknownSubject(String), + SubjectNotInstalled { + display_name: String, + bundle_path: String, + }, + VersionDrift { + display_name: String, + expected: String, + found: String, + }, + SubjectVersionUnprobeable { + display_name: String, + bundle_path: String, + }, + NoSubjectsSelected, + ResumeFileUnreadable { + path: String, + message: String, + }, + SubjectAlreadyRunning { + display_name: String, + bundle_identifier: String, + }, + OpenEnvFlagUnsupported, +} + +impl std::fmt::Display for RunRefusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::QuiesceGateFailed(signals) => write!( + f, + "quiesce gate failed, refusing to start: {}", + signals.join(", ") + ), + Self::UnknownSubject(id) => write!(f, "unknown subject: {id}"), + Self::SubjectNotInstalled { + display_name, + bundle_path, + } => write!(f, "{display_name} is not installed at {bundle_path}"), + Self::VersionDrift { + display_name, + expected, + found, + } => write!( + f, + "{display_name}: version drift, expected {expected} found {found} \ + (pass --allow-version-drift to proceed anyway)" + ), + Self::SubjectVersionUnprobeable { + display_name, + bundle_path, + } => write!( + f, + "{display_name}: could not probe installed version at {bundle_path}" + ), + Self::NoSubjectsSelected => { + write!(f, "no subjects selected for this run") + } + Self::ResumeFileUnreadable { path, message } => { + write!(f, "--resume {path}: {message}") + } + Self::SubjectAlreadyRunning { + display_name, + bundle_identifier, + } => write!( + f, + "{display_name} ({bundle_identifier}) is already running -- quit \ + it before running the benchmark (a same-bundle-identifier \ + instance would silently absorb the launch and exit, measuring \ + nothing)" + ), + Self::OpenEnvFlagUnsupported => write!( + f, + "/usr/bin/open on this machine does not document --env; this \ + harness requires it to launch every subject with an isolated \ + scratch HOME (see this crate's README, \"Prerequisites\")" + ), + } + } +} + +impl std::error::Error for RunRefusal {} + +/// The subject registry entries a run should measure: every `--subjects` +/// id resolved against [`subject::find`], or (with no override) every +/// non-optional subject plus every optional one when +/// `allow_optional_subjects` is set (spec FR-1's "diri ... marked optional +/// ... unless --allow-optional-subjects"). Pure -- installed-ness and +/// version drift are checked live, downstream of this selection. +pub fn select_subjects( + requested_ids: Option<&[String]>, + allow_optional_subjects: bool, +) -> Result, RunRefusal> { + let selected: Vec<&'static SubjectSpec> = match requested_ids { + Some(ids) => { + let mut specs = Vec::with_capacity(ids.len()); + for id in ids { + let spec = subject::find(id) + .ok_or_else(|| RunRefusal::UnknownSubject(id.clone()))?; + specs.push(spec); + } + specs + } + None => SUBJECTS + .iter() + .filter(|s| !s.optional || allow_optional_subjects) + .collect(), + }; + if selected.is_empty() { + return Err(RunRefusal::NoSubjectsSelected); + } + Ok(selected) +} + +/// The first selected subject whose **bundle identifier** already has a +/// live LaunchServices entry (spec item 3) -- keyed on bundle identifier, +/// not display name or app name, because two differently named bundles +/// can declare the same identifier, and a single-instance plugin then +/// hands a new launch off to the already-running instance, which exits +/// within seconds. Left undetected, the harness would seed a profile, +/// launch, measure nothing, and report an invalid tier for a reason +/// nobody could diagnose -- `open -n` does not bypass this, because `-n` +/// only tells LaunchServices to spawn a new process; it says nothing about +/// the app's own single-instance guard. Pure over an already-captured +/// `lsappinfo list`, so the matching rule is unit-tested independent of +/// the live invocation that feeds it. +pub fn find_already_running_subject<'a>( + subjects: &[&'a SubjectSpec], + ls_entries: &[LaunchServicesEntry], +) -> Option<&'a SubjectSpec> { + subjects.iter().copied().find(|subject| { + ls_entries.iter().any(|entry| { + entry.pid.is_some() + && entry.bundle_identifier.as_deref() == Some(subject.bundle_identifier) + }) + }) +} + +/// `RunSettings::default()` with `--repetitions N` applied uniformly to +/// every tier's count when given -- the CLI surface has one repetition +/// flag (spec §5.9), not a per-tier one, so an override sets all three +/// (this is what the design's smoke sweep, `--repetitions 2`, relies on). +pub fn build_settings(repetitions_override: Option) -> RunSettings { + let mut settings = RunSettings::default(); + if let Some(n) = repetitions_override { + settings.repetitions = TierRepetitions { + fresh_launch: n, + sustained_use: n, + n_session: n, + }; + } + settings +} + +// --- Repetition planning (tier expansion + resume reconciliation) ------- + +/// One planned (subject, tier, repetition) launch. `repetition == 0` is +/// always the calibration launch (spec FR-12's mandatory first-run +/// discard, design §5.4's calibration launch) -- [`plan_repetitions`] +/// includes it, on top of the tier's disclosed repetition count. +#[derive(Debug, Clone, PartialEq)] +pub struct PlannedRepetition { + pub subject_id: String, + pub tier: Tier, + pub repetition: u32, + pub is_calibration: bool, + pub sample_id: String, +} + +/// The `sampleId` a repetition's [`Sample`] carries, e.g. +/// `"termtree/fresh-launch/003"` (design §6.2) -- computed once here so +/// every planner, resumer, and writer agrees on the same identifier. +pub fn sample_id(subject_id: &str, tier: Tier, repetition: u32) -> String { + format!("{subject_id}/{}/{repetition:03}", tier.as_str()) +} + +/// Expands every (subject, tier) pair into its full repetition list -- +/// `0..=tier.repetitions(settings)`, i.e. the disclosed count plus the +/// mandatory calibration launch at repetition 0 (design §5.8's `for +/// repetition in 0..=settings.repetitions`, spec FR-12). +pub fn plan_repetitions( + subjects: &[&SubjectSpec], + tiers: &[Tier], + settings: &RunSettings, +) -> Vec { + let mut plan = Vec::new(); + for subject in subjects { + for &tier in tiers { + let repetitions = tier.repetitions(settings); + for repetition in 0..=repetitions { + plan.push(PlannedRepetition { + subject_id: subject.id.to_string(), + tier, + repetition, + is_calibration: repetition == 0, + sample_id: sample_id(subject.id, tier, repetition), + }); + } + } + } + plan +} + +/// `--resume `'s reconciliation step (design §5.8): drops every +/// planned repetition whose `sampleId` already has a sample in the +/// resumed file, so a re-invocation after a crash or interrupt continues +/// where it left off instead of re-measuring (or re-seeding, re-launching) +/// work that already landed safely on disk. +pub fn pending_repetitions( + planned: &[PlannedRepetition], + completed_sample_ids: &HashSet, +) -> Vec { + planned + .iter() + .filter(|p| !completed_sample_ids.contains(&p.sample_id)) + .cloned() + .collect() +} + +pub fn completed_sample_ids(result: &ResultFile) -> HashSet { + result.samples.iter().map(|s| s.sample_id.clone()).collect() +} + +/// How many times to re-check the quiesce gate during one repetition's +/// settle/measurement window (spec FR-10: "re-read ... at the N-session +/// tiers, every 30 s during measurement") -- zero for tiers/durations +/// shorter than the interval, so a fresh-launch's 15 s settle does not +/// spuriously re-check. +pub fn quiesce_recheck_count(duration_s: u64, interval_s: u64) -> u64 { + duration_s.checked_div(interval_s).unwrap_or(0) +} + +// --- Warm-helper ownership (also used by teardown) ----------------------- + +/// Whether any process whose executable path is under `bundle_path`, other +/// than `main_pid` itself, is currently resident (design §5.8 step 2's +/// Chromium/Electron half of the warm-helper check, and the signal +/// [`has_warm_helpers`] takes as `chromium_helper_present`). +pub fn chromium_helper_resident( + tree_records: &[ProcessRecord], + main_pid: u32, + bundle_path: &str, +) -> bool { + tree_records.iter().any(|record| { + record.pid != main_pid + && record + .executable_path + .as_deref() + .is_some_and(|path| path.starts_with(bundle_path)) + }) +} + +/// The PIDs of a `WebKitTauri` subject's own helper processes, by the same +/// ownership test [`has_warm_helpers`] uses to detect warmth. Teardown +/// kills only these PIDs (design §5.8 step 9, design §9: "a shared-name +/// helper owned by a different app is never touched"). +pub fn owned_webkit_helper_pids( + subject: &SubjectSpec, + ls_entries: &[LaunchServicesEntry], +) -> Vec { + let prefix = format!("{} ", subject.launch_services_name); + ls_entries + .iter() + .filter(|entry| { + entry.display_name.starts_with(&prefix) + && entry + .bundle_identifier + .as_deref() + .is_some_and(|id| subject.helper_bundle_ids.contains(&id)) + }) + .filter_map(|entry| entry.pid) + .collect() +} + +// --- Record assembly (pure: takes already-invoked/parsed results) -------- + +/// Builds the [`AttributionRecord`] published on a [`Sample`] from an +/// already-resolved [`AttributableProcessSet`] and the union `footprint` +/// invocation's report -- pure, so the union/partition/footprint-merge +/// logic stays testable even though resolving and invoking are live. +pub fn build_attribution_record( + set: &AttributableProcessSet, + vanished_pids: &[u32], + union_report: &FootprintReport, +) -> AttributionRecord { + let processes = set + .processes + .iter() + .map(|p| { + let footprint_row = + union_report.processes.iter().find(|fp| fp.pid == p.pid); + AttributedProcessRecord { + pid: p.pid, + name: p.name.clone(), + executable_path: p.executable_path.clone(), + discovered_by: p.discovered_by.as_str().to_string(), + role: p.role.as_str().to_string(), + phys_footprint_bytes: footprint_row.map(|fp| fp.footprint_bytes), + rss_bytes: None, + } + }) + .collect(); + let launch_services_pids = set + .processes + .iter() + .filter(|p| { + matches!( + p.discovered_by, + DiscoverySource::LaunchServices | DiscoverySource::Both + ) + }) + .map(|p| p.pid) + .collect(); + let process_tree_pids = set + .processes + .iter() + .filter(|p| { + matches!( + p.discovered_by, + DiscoverySource::ProcessTree | DiscoverySource::Both + ) + }) + .map(|p| p.pid) + .collect(); + AttributionRecord { + main_pid: set.main_pid, + launch_services_pids, + process_tree_pids, + vanished_pids: vanished_pids.to_vec(), + orchestrator_pids: set.orchestrator_pids(), + agent_cli_pids: set.agent_cli_pids(), + processes, + } +} + +/// Builds the [`MemoryRecord`] published on a [`Sample`] from the three +/// `footprint` invocations (union/orchestrator/session, design §5.3.1's +/// table) and the before/after `vm_stat` samples (design §5.3.2) -- pure, +/// so the derived-quantity formulas (the free-RAM sign convention, the +/// shared-page double count, the cross-partition overlap) are unit-tested +/// independent of the live invocations that feed them. +#[allow(clippy::too_many_arguments)] +pub fn build_memory_record( + union_report: &FootprintReport, + orchestrator_report: &FootprintReport, + session_report: &FootprintReport, + mem_rss_bytes: u64, + core_process_bytes: Option, + render_helper_bytes: Option, + before: &HostMemorySample, + after: &HostMemorySample, +) -> MemoryRecord { + let free_before = before.free_ram_bytes(); + let free_after = after.free_ram_bytes(); + let used_before = before.host_memory_used_bytes(); + let used_after = after.host_memory_used_bytes(); + // orchestrator + session >= union whenever the two partitions share + // pages; the excess over the union total is published rather than + // hidden by forcing the parts to sum (design §5.3.1). + let cross_partition_shared_bytes = orchestrator_report + .total_footprint_bytes + .saturating_add(session_report.total_footprint_bytes) + .saturating_sub(union_report.total_footprint_bytes); + + MemoryRecord { + mem_phys_footprint_bytes: union_report.total_footprint_bytes, + mem_phys_footprint_process_sum_bytes: footprint::process_sum_bytes( + union_report, + ), + shared_page_double_count_bytes: footprint::shared_page_double_count_bytes( + union_report, + ), + cross_partition_shared_bytes, + mem_rss_bytes, + mem_rss_method: "naive-per-process-sum".to_string(), + orchestrator_attributable_bytes: orchestrator_report.total_footprint_bytes, + agent_cli_attributable_bytes: session_report.total_footprint_bytes, + core_process_bytes, + render_helper_bytes, + free_ram_before_bytes: free_before, + free_ram_after_bytes: free_after, + // Sign convention: positive means the subject consumed RAM (design + // §5.3.2) -- `before - after`, not `after - before`. + free_ram_delta_bytes: free_before as i64 - free_after as i64, + free_ram_delta_sign: "positive-means-consumed".to_string(), + host_memory_used_before_bytes: used_before, + host_memory_used_after_bytes: used_after, + host_memory_used_delta_bytes: used_after as i64 - used_before as i64, + compressor_occupied_delta_bytes: after.compressor_occupied_bytes() as i64 + - before.compressor_occupied_bytes() as i64, + swapouts_delta: after.swapouts().saturating_sub(before.swapouts()), + // The free-RAM split at N-session tiers is derived cross-tier from + // this same subject's fresh-launch median (design §5.3.2) -- left + // unset here; a post-processing pass over the written result file is + // required to fill it in once a fresh-launch aggregate exists, which + // is out of scope for this pass (see final report). + orchestrator_free_ram_delta_bytes: None, + agent_cli_free_ram_delta_bytes: None, + free_ram_split_derivation: None, + } +} + +pub fn build_cold_start_record( + first_window_visible_ms: Option, + main_window_visible_ms: Option, + app_window_ready_ms: Option, + splash_close_ms: Option, + mark_resolution_ms: u64, +) -> ColdStartRecord { + let mut mark_source = std::collections::BTreeMap::new(); + if first_window_visible_ms.is_some() { + mark_source.insert( + "firstWindowVisibleMs".to_string(), + "cg-window-list".to_string(), + ); + } + if main_window_visible_ms.is_some() { + mark_source.insert( + "mainWindowVisibleMs".to_string(), + "cg-window-list".to_string(), + ); + } + if app_window_ready_ms.is_some() { + mark_source.insert( + "appWindowReadyMs".to_string(), + "karijini-log-arrival".to_string(), + ); + } + if splash_close_ms.is_some() { + mark_source.insert( + "splashCloseMs".to_string(), + "karijini-log-arrival".to_string(), + ); + } + ColdStartRecord { + first_window_visible_ms, + main_window_visible_ms, + app_window_ready_ms, + splash_close_ms, + mark_source, + mark_resolution_ms, + } +} + +pub fn build_idle_cpu_record( + summary: &stats::Summary, + window_state: &str, +) -> IdleCpuRecord { + IdleCpuRecord { + idle_cpu_percent_of_one_core_median: summary.median, + idle_cpu_percent_of_one_core_iqr: summary.iqr, + sample_count: summary.n, + window_state: window_state.to_string(), + } +} + +// --- Live orchestration (design §5.8, §7) -------------------------------- +// +// Everything below this line spawns a process, polls the window server, or +// sleeps a wall-clock duration, and is therefore the crate's one +// deliberately untested surface (design §11). It decides nothing on its +// own -- every branch calls one of the pure functions above (plan, +// classify, build_*_record) or a thin `invoke_*`/parse pair owned by +// another module. See the final report for exactly which lines these are. + +/// Set by [`install_interrupt_handler`]'s `SIGINT` handler; polled between +/// repetitions so an interrupted sweep restores every seeder's backed-up +/// state (most importantly TermTree's production `state.json`, design +/// §5.6.1) before exiting, rather than abandoning it mid-seed. +static INTERRUPTED: AtomicBool = AtomicBool::new(false); + +type RawSignalHandler = extern "C" fn(i32); +const SIGINT: i32 = 2; + +extern "C" { + fn signal(signum: i32, handler: RawSignalHandler) -> usize; +} + +extern "C" fn handle_interrupt(_signum: i32) { + INTERRUPTED.store(true, Ordering::SeqCst); +} + +/// Installs the `SIGINT` handler above. `main.rs` calls this once before +/// [`RunOrchestrator::run`]. Idempotent. +pub fn install_interrupt_handler() { + // SAFETY: `signal` is libSystem's standard POSIX `signal(2)`, always + // linked into a macOS Rust binary; `handle_interrupt` matches its + // required `extern "C" fn(i32)` signature and does nothing but store to + // an atomic, which is signal-safe. + unsafe { + signal(SIGINT, handle_interrupt); + } +} + +fn interrupted() -> bool { + INTERRUPTED.load(Ordering::SeqCst) +} + +/// Restores every seeder's backed-up state, best-effort (design §5.6.1, +/// §9's "the sweep is interrupted"). Run at the end of every sweep and +/// again if [`interrupted`] fires mid-sweep; `resource-benchmark restore` +/// is the same operation available as a manual escape hatch. +fn restore_all_seeders(home: &Path, subjects: &[&SubjectSpec]) { + for subject in subjects { + if let Err(error) = seeding::restore_subject(home, subject.id) { + eprintln!("warning: restore {} failed: {error}", subject.id); + } + } +} + +/// One measured sample's cold-start half (design §5.4) -- the launch, PID +/// discovery, and concurrent window/log polling this crate cannot unit +/// test, packaged so [`RunOrchestrator::measure_one`] can hand its result +/// straight to the pure [`build_cold_start_record`]. +struct ColdStartOutcome { + first_window_visible_ms: Option, + main_window_visible_ms: Option, + app_window_ready_ms: Option, + splash_close_ms: Option, + splash_timeout_seen: bool, + main_pid: Option, + /// The largest layer-0 window area observed this launch -- becomes + /// `calibratedMainWindowAreaPt` when this is the calibration launch + /// (design §5.4 step 3). + observed_main_window_area_pt: Option, + /// Spec item 4: `karijini.log` advanced during this launch but none of + /// `log_marks.rs`'s hardcoded messages matched -- a drift signal, not the + /// legitimate "log rolled/shrank" case this module already handles by + /// reopening from the start. + log_marks_unrecognized: bool, +} + +fn invoke_and_parse_footprint( + output_path: &Path, + pids: &[u32], +) -> Option { + if pids.is_empty() { + return Some(FootprintReport { + total_footprint_bytes: 0, + processes: Vec::new(), + shared: Vec::new(), + errors: Vec::new(), + warnings: Vec::new(), + page_size_bytes: 0, + start_time_iso: None, + }); + } + footprint::invoke_footprint(&output_path.to_string_lossy(), pids).ok()?; + let text = std::fs::read_to_string(output_path).ok()?; + let _ = std::fs::remove_file(output_path); + footprint::parse_footprint_json(&text).ok() +} + +pub struct RunOrchestrator { + pub subjects: Vec<&'static SubjectSpec>, + pub tiers: Vec, + pub settings: RunSettings, + /// The disposable per-run scratch home (spec item 1) -- never the + /// runner's real `$HOME`. Every seeder, the karijini log path, and the + /// `HOME` a subject is launched with all key off this. + pub home: PathBuf, + pub repo: SeededRepo, + pub agent: AgentCliPin, + pub allow_version_drift: bool, + pub out_path: PathBuf, + /// Per-subject `/Applications/*.app` overrides (spec item 5), resolved + /// once in `main.rs` from `--bundle-path`/`RESOURCE_BENCHMARK_BUNDLE_PATH_*`. + pub bundle_path_overrides: BundlePathOverrides, + /// The real logged-in user's login shell path (`dscl -read UserShell`), used only to classify a process as a session + /// root (design §5.2.3) -- resolved once in `main.rs` against the real + /// `$HOME`, deliberately **not** the scratch home, since `dscl` needs an + /// actual Directory Services record to read. + pub login_shell_path: String, +} + +impl RunOrchestrator { + /// Builds a fresh [`ResultFile`] envelope (design §6.1): probes the + /// machine spec, OS build, and every selected subject's installed + /// version, refusing per subject on version drift unless + /// `allow_version_drift` was passed (spec FR-1; this fix's "an + /// unimplemented or refused run must never exit 0"). + fn build_envelope( + &self, + quiesce_reading: &QuiesceReading, + ) -> Result { + let mut subjects_provenance = Vec::with_capacity(self.subjects.len()); + for subject in &self.subjects { + let bundle_path = + bundle_paths::resolve(subject, &self.bundle_path_overrides); + if !Path::new(bundle_path).exists() { + if subject.optional { + continue; + } + return Err(RunRefusal::SubjectNotInstalled { + display_name: subject.display_name.to_string(), + bundle_path: bundle_path.to_string(), + }); + } + let found_version = provenance::probe_subject_version(bundle_path) + .ok_or_else(|| RunRefusal::SubjectVersionUnprobeable { + display_name: subject.display_name.to_string(), + bundle_path: bundle_path.to_string(), + })?; + let version_drift_accepted = found_version != subject.expected_version; + if version_drift_accepted && !self.allow_version_drift { + return Err(RunRefusal::VersionDrift { + display_name: subject.display_name.to_string(), + expected: subject.expected_version.to_string(), + found: found_version, + }); + } + subjects_provenance.push(SubjectProvenance { + subject_id: subject.id.to_string(), + display_name: subject.display_name.to_string(), + subject_version: found_version, + runtime_family: subject.runtime_family.as_str().to_string(), + bundle_identifier: subject.bundle_identifier.to_string(), + bundle_path: bundle_path.to_string(), + optional: subject.optional, + seeder: format!("{:?}", subject.seeder), + seed_method: String::new(), + calibrated_main_window_area_pt: None, + version_drift_accepted, + seed_format_verified: subject.seed_format_verified, + }); + } + if subjects_provenance.is_empty() { + return Err(RunRefusal::NoSubjectsSelected); + } + + let machine_spec = provenance::probe_machine_spec(); + let os_build = provenance::probe_os_build(); + let run_timestamp = provenance::iso_timestamp_now(); + let run_id = format!( + "{}-{:08x}", + run_timestamp.replace(':', "-"), + std::process::id() + ); + + Ok(ResultFile { + schema_version: result::SCHEMA_VERSION, + run_id, + run_timestamp, + harness_ref: provenance::harness_ref("unknown"), + machine_spec, + os_build, + agent_cli_version: result::AgentCliVersion { + name: self.agent.name.clone(), + version: self.agent.version.clone(), + executable_path: self.agent.executable_path.clone(), + }, + repo_ref: result::RepoRef { + url: self.repo.url.clone(), + commit: self.repo.commit.clone(), + local_path: self.repo.local_path.clone(), + }, + login_shell_path: self.login_shell_path.clone(), + settings: self.settings.clone(), + subjects: subjects_provenance, + quiesce: RunQuiesce { + pre_run: quiesce_reading.clone(), + verdict: if quiesce_reading.verdict == QuiesceVerdict::Pass { + "pass".to_string() + } else { + "fail".to_string() + }, + }, + samples: Vec::new(), + aggregates: Vec::new(), + fairness_review: FairnessReview::default(), + }) + } + + /// The full sweep (design §5.8, §7): pre-run quiesce gate, envelope + /// construction (or `--resume` reload), tier expansion, and then one + /// `measure_one` + crash-safe append per pending repetition. + pub fn run( + &self, + resume_path: Option<&Path>, + ) -> Result { + let quiesce_reading = quiesce::read_quiesce_gate(None); + if quiesce_reading.verdict != QuiesceVerdict::Pass { + return Err(RunRefusal::QuiesceGateFailed( + quiesce_reading.failing_signals.clone(), + )); + } + + // Spec item 1: every subject is launched with `HOME` isolated via + // `open --env`; refuse up front, loudly, rather than silently falling + // back to the runner's real `$HOME` if this machine's `open` predates + // it. + if !cold_start::supports_env_flag() { + return Err(RunRefusal::OpenEnvFlagUnsupported); + } + + // Spec item 3: refuse before seeding/launching anything if any + // selected subject's bundle identifier is already running. + let ls_text = launch_services::invoke_lsappinfo_list().unwrap_or_default(); + let ls_entries = launch_services::parse_lsappinfo_list(&ls_text); + if let Some(subject) = + find_already_running_subject(&self.subjects, &ls_entries) + { + return Err(RunRefusal::SubjectAlreadyRunning { + display_name: subject.display_name.to_string(), + bundle_identifier: subject.bundle_identifier.to_string(), + }); + } + + let out_path = resume_path + .map(Path::to_path_buf) + .unwrap_or_else(|| self.out_path.clone()); + + let mut result = match resume_path { + Some(path) => result::read_result_file(path).map_err(|error| { + RunRefusal::ResumeFileUnreadable { + path: path.display().to_string(), + message: error.to_string(), + } + })?, + None => self.build_envelope(&quiesce_reading)?, + }; + + if let Err(error) = result::write_result_file(&out_path, &result) { + eprintln!("error writing {}: {error}", out_path.display()); + } + + let subjects_for_plan: Vec<&SubjectSpec> = result + .subjects + .iter() + .filter_map(|sp| subject::find(&sp.subject_id)) + .collect(); + let plan = + plan_repetitions(&subjects_for_plan, &self.tiers, &result.settings); + let already_done = completed_sample_ids(&result); + let pending = pending_repetitions(&plan, &already_done); + + let mut calibrated_areas: HashMap = result + .subjects + .iter() + .filter_map(|sp| { + sp.calibrated_main_window_area_pt + .map(|area| (sp.subject_id.clone(), area)) + }) + .collect(); + let login_shell_path = result.login_shell_path.clone(); + + for planned in &pending { + if interrupted() { + eprintln!( + "resource-benchmark run: interrupted, restoring seeders and stopping" + ); + break; + } + let Some(subject) = subjects_for_plan + .iter() + .find(|s| s.id == planned.subject_id) + else { + continue; + }; + + // Re-check the quiesce gate before every repetition, not only at + // the start (spec FR-10) -- a multi-hour sweep can drift into + // memory/thermal pressure the pre-run check never saw. + let repetition_quiesce = quiesce::read_quiesce_gate(None); + let quiesce_violation = + repetition_quiesce.verdict != QuiesceVerdict::Pass; + + let sample = self.measure_one( + subject, + planned, + &mut calibrated_areas, + quiesce_violation, + &repetition_quiesce, + &login_shell_path, + ); + + if planned.is_calibration { + if let Some(area) = calibrated_areas.get(&planned.subject_id) { + if let Some(sp) = result + .subjects + .iter_mut() + .find(|s| s.subject_id == planned.subject_id) + { + sp.calibrated_main_window_area_pt = Some(*area); + } + } + } + + result.samples.push(sample); + result.aggregates = stats::compute_aggregates(&result.samples); + // Crash-safe append (design §5.8 step 8): rewrite the whole file + // after every sample, so an interrupted sweep loses at most one. + if let Err(error) = result::write_result_file(&out_path, &result) { + eprintln!("error writing {}: {error}", out_path.display()); + } + } + + restore_all_seeders(&self.home, &subjects_for_plan); + Ok(result) + } + + /// The live launch-and-poll loop for one cold-start measurement (design + /// §5.4). Untested by design; every decision it makes about *whether* a + /// window qualifies as the main window, or a log line is a mark, calls + /// [`cold_start::is_main_window`] / [`log_marks::classify_log_mark`], + /// both of which are fixture-tested in their own modules. + fn measure_cold_start( + &self, + subject: &SubjectSpec, + calibrated_area: Option, + ) -> ColdStartOutcome { + let bundle_path = + bundle_paths::resolve(subject, &self.bundle_path_overrides); + let before = process_tree::snapshot_processes(); + let karijini_path = (subject.id == "termtree").then(|| { + log_marks::karijini_log_path( + &self.home.join("Library").join("Application Support"), + ) + }); + let mut log_offset: u64 = karijini_path + .as_deref() + .and_then(|p| std::fs::metadata(p).ok()) + .map(|m| m.len()) + .unwrap_or(0); + + let start = Instant::now(); + let _ = cold_start::launch_suppressing_restoration(bundle_path, &self.home); + + let discovery_deadline = start + Duration::from_secs(2); + let mut main_pid = None; + while main_pid.is_none() && Instant::now() < discovery_deadline { + let after = process_tree::snapshot_processes(); + main_pid = + cold_start::find_newly_launched_pid(&before, &after, bundle_path); + if main_pid.is_none() { + thread::sleep(Duration::from_millis( + self.settings.window_visible_poll_ms, + )); + } + } + + let mut first_window_visible_ms = None; + let mut main_window_visible_ms = None; + let mut app_window_ready_ms = None; + let mut splash_close_ms = None; + let mut splash_timeout_seen = false; + let mut observed_main_window_area_pt: Option = None; + // Spec item 4: whether the harness ever read a non-blank new + // `karijini.log` line during this launch's settle window, regardless + // of whether it classified as a known mark. Distinguishes "the log + // never advanced" (too early, or not TermTree) from "the log advanced + // but none of `log_marks.rs`'s hardcoded messages matched a single + // line" -- the latter is the drift signal + // [`log_marks::marks_unrecognized`] turns into a loud failure. + let mut any_log_line_observed = false; + + let settle_deadline = + start + Duration::from_millis(self.settings.fresh_launch_settle_ms); + let poll_interval = Duration::from_millis( + self + .settings + .window_visible_poll_ms + .min(self.settings.log_tail_poll_ms) + .max(1), + ); + while Instant::now() < settle_deadline { + if let Some(pid) = main_pid { + let windows = window_probe::on_screen_windows_owned_by(pid); + if first_window_visible_ms.is_none() && !windows.is_empty() { + first_window_visible_ms = Some(start.elapsed().as_millis() as u64); + } + if let Some(area) = cold_start::calibrate_main_window_area(&windows) { + observed_main_window_area_pt = Some( + observed_main_window_area_pt.map_or(area, |m: f64| m.max(area)), + ); + } + if main_window_visible_ms.is_none() { + if let Some(reference_area) = calibrated_area { + if windows.iter().any(|w| { + cold_start::is_main_window( + w, + reference_area, + self.settings.main_window_area_fraction, + ) + }) { + main_window_visible_ms = Some(start.elapsed().as_millis() as u64); + } + } + } + } + if let Some(path) = &karijini_path { + if let Ok(text) = std::fs::read_to_string(path) { + let len = text.len() as u64; + if len >= log_offset { + for line in text[log_offset as usize..].lines() { + if !line.trim().is_empty() { + any_log_line_observed = true; + } + match log_marks::classify_log_mark(line) { + Some(LogMark::AppWindowReadyMain) => { + app_window_ready_ms + .get_or_insert(start.elapsed().as_millis() as u64); + } + Some(LogMark::SplashClosed) => { + splash_close_ms + .get_or_insert(start.elapsed().as_millis() as u64); + } + Some(LogMark::SplashTimeout) => splash_timeout_seen = true, + None => {} + } + } + log_offset = len; + } else { + // The log rolled/shrank mid-measurement (design §9): reopen + // from the start rather than treat the shrink as an error. + log_offset = 0; + } + } + } + let done = main_window_visible_ms.is_some() + && (karijini_path.is_none() + || splash_close_ms.is_some() + || splash_timeout_seen); + if done || interrupted() { + break; + } + thread::sleep(poll_interval); + } + + let log_marks_unrecognized = log_marks::marks_unrecognized( + karijini_path.is_some(), + any_log_line_observed, + app_window_ready_ms, + splash_close_ms, + splash_timeout_seen, + ); + + ColdStartOutcome { + first_window_visible_ms, + main_window_visible_ms, + app_window_ready_ms, + splash_close_ms, + splash_timeout_seen, + main_pid, + observed_main_window_area_pt, + log_marks_unrecognized, + } + } + + /// Quits `subject` (graceful `osascript` quit, design §5.8 step 9), kills + /// its registered companion processes, and waits up to + /// `helper_drain_timeout_ms` for its own helpers to exit -- **only** + /// helpers [`owned_webkit_helper_pids`]/[`chromium_helper_resident`] + /// attribute to this subject are ever targeted (design §9: "a + /// shared-name helper owned by a different app is never touched"). + /// Returns the number of survivors killed past the drain timeout. + fn teardown(&self, subject: &SubjectSpec) -> u32 { + let _ = run_capture("/usr/bin/osascript", &[ + "-e", + &format!("quit app id \"{}\"", subject.bundle_identifier), + ]); + for companion in subject.companion_processes { + if companion.kill_between_repetitions { + let _ = + run_capture("/usr/bin/pkill", &["-x", companion.executable_name]); + } + } + + let deadline = Instant::now() + + Duration::from_millis(self.settings.helper_drain_timeout_ms); + loop { + let ls_text = + launch_services::invoke_lsappinfo_list().unwrap_or_default(); + let ls_entries = launch_services::parse_lsappinfo_list(&ls_text); + let tree = process_tree::snapshot_processes(); + let webkit_survivors = owned_webkit_helper_pids(subject, &ls_entries); + let chromium_survivor = chromium_helper_resident( + &tree, + 0, + bundle_paths::resolve(subject, &self.bundle_path_overrides), + ); + if (webkit_survivors.is_empty() && !chromium_survivor) + || Instant::now() >= deadline + { + for pid in &webkit_survivors { + let _ = run_capture("/bin/kill", &["-TERM", &pid.to_string()]); + } + return webkit_survivors.len() as u32; + } + thread::sleep(Duration::from_millis(200)); + } + } + + /// Everything design §5.8 does for one planned repetition: warm-helper + /// check, seed, `vm_stat` before, launch + cold start, settle, + /// attribute, `footprint` x3, `vm_stat` after, idle CPU, teardown -- + /// assembled into one [`Sample`] via [`classify_sample_invalidity`] and + /// the pure `build_*_record` functions above. + #[allow(clippy::too_many_arguments)] + fn measure_one( + &self, + subject: &SubjectSpec, + planned: &PlannedRepetition, + calibrated_areas: &mut HashMap, + quiesce_violation_seen: bool, + quiesce_reading: &QuiesceReading, + login_shell_path: &str, + ) -> Sample { + let sampled_at = provenance::iso_timestamp_now(); + + let ls_text = launch_services::invoke_lsappinfo_list().unwrap_or_default(); + let ls_entries = launch_services::parse_lsappinfo_list(&ls_text); + + // Spec item 3: refuse before seeding if this subject's bundle + // identifier is already running -- checked per repetition, not only + // at the start of the sweep, because a multi-hour sweep can outlive a + // machine state the pre-run check saw. Skip seeding, launching, *and* + // teardown entirely: the running instance may be the operator's own + // real usage of the app, and `teardown`'s graceful quit must never + // touch a process this harness did not itself launch. + let subject_already_running = + find_already_running_subject(std::slice::from_ref(&subject), &ls_entries) + .is_some(); + if subject_already_running { + let invalid_reason_value = classify_sample_invalidity( + planned.is_calibration, + true, + 0, + false, + quiesce_violation_seen, + false, + false, + false, + false, + false, + false, + ); + return Sample { + sample_id: planned.sample_id.clone(), + subject_id: planned.subject_id.clone(), + tier: planned.tier.as_str(), + session_count: planned.tier.session_count(), + repetition: planned.repetition, + is_calibration: planned.is_calibration, + sampled_at, + is_valid: invalid_reason_value.is_none(), + invalid_reason: invalid_reason_value.map(str::to_string), + attribution: None, + memory: None, + cold_start: None, + idle_cpu: None, + quiesce: Some(quiesce_reading.clone()), + warm_helper_count: None, + helper_kill_count: None, + }; + } + + // Step 2: warm-helper check (spec FR-6, design §5.8 step 2). + let bundle_path = + bundle_paths::resolve(subject, &self.bundle_path_overrides); + let tree_before = process_tree::snapshot_processes(); + let warm_helper_count = + if matches!(subject.runtime_family, RuntimeFamily::WebKitTauri) { + owned_webkit_helper_pids(subject, &ls_entries).len() as u32 + } else if chromium_helper_resident(&tree_before, 0, bundle_path) { + 1 + } else { + 0 + }; + + // Step 3: seed (N-session and sustained-use tiers only). + let session_count = planned.tier.session_count(); + let mut seed_incomplete = false; + if session_count > 0 { + if let Err(error) = seeding::seed_subject( + &self.home, + subject.id, + session_count, + &self.repo, + &self.agent, + ) { + eprintln!("seed {} failed: {error}", subject.id); + seed_incomplete = true; + } + } + + // Step 4: vm_stat before. + let before_memory = host_memory::invoke_vm_stat() + .ok() + .and_then(|text| host_memory::parse_vm_stat(&text)); + + // Step 5: launch + cold start. + let calibrated_area = calibrated_areas.get(&planned.subject_id).copied(); + let cold_start_outcome = self.measure_cold_start(subject, calibrated_area); + if planned.is_calibration { + if let Some(area) = cold_start_outcome.observed_main_window_area_pt { + calibrated_areas.insert(planned.subject_id.clone(), area); + } + } + + // Spec item 4: TermTree launched but never created its own data + // directory under the scratch home -- the app's data-directory + // convention has likely changed (`seeding/termtree.rs`, + // `log_marks::karijini_log_path` both hardcode + // `DocumentNode/TermTree`). + let app_data_dir_missing = subject.id == "termtree" + && cold_start_outcome.main_pid.is_some() + && !log_marks::app_data_dir( + &self.home.join("Library").join("Application Support"), + ) + .exists(); + + // Spec item 4: this subject's seeder format has never been checked + // against a real install (`subject.seed_format_verified`), so any + // tier that relies on seeding reports unverified rather than valid. + let seed_format_unverified = + session_count > 0 && !subject.seed_format_verified; + + // Step 6: settle. `measure_cold_start` already spends up to + // `freshLaunchSettleMs`; the heavier tiers wait out their own fixed + // duration on top of it. + let extra_settle_s = match planned.tier { + Tier::FreshLaunch => 0, + Tier::SustainedUse => self.settings.sustained_use_duration_s, + Tier::NSession(_) => self.settings.n_session_settle_s, + }; + let recheck_interval_s = self.settings.quiesce_window_s.max(30); + for _ in 0..quiesce_recheck_count(extra_settle_s, recheck_interval_s) { + thread::sleep(Duration::from_secs(recheck_interval_s)); + if interrupted() { + break; + } + } + let remaining_settle_s = extra_settle_s % recheck_interval_s; + if remaining_settle_s > 0 { + thread::sleep(Duration::from_secs(remaining_settle_s)); + } + + // Session readiness (N-session / sustained-use tiers): generic check, + // the same way for every subject (design §5.6, NFR-3). + if session_count > 0 && !seed_incomplete { + match cold_start_outcome.main_pid { + Some(pid) => { + let tree = process_tree::snapshot_processes(); + let (shells, agents) = seeding::count_ready_sessions( + &tree, + pid, + login_shell_path, + &self.agent.executable_path, + ); + if shells < session_count || agents < session_count { + seed_incomplete = true; + } + } + None => seed_incomplete = true, + } + } + + // Step 7: attribute -> footprint x3 -> vm_stat after -> idle CPU. + let mut attribution_incomplete = cold_start_outcome.main_pid.is_none(); + let mut footprint_pid_mismatch = false; + let mut attribution_record = None; + let mut memory_record = None; + let mut idle_cpu_record = None; + + if let Some(main_pid) = cold_start_outcome.main_pid { + let tree = process_tree::snapshot_processes(); + let ls_text_now = + launch_services::invoke_lsappinfo_list().unwrap_or_default(); + let ls_entries_now = launch_services::parse_lsappinfo_list(&ls_text_now); + let ls_pids = attribution::resolve_launch_services_pids( + &ls_entries_now, + subject.launch_services_name, + subject.helper_bundle_ids, + ); + let descendants = process_tree::descendants_of(&tree, main_pid); + + match attribution::resolve_union(main_pid, &ls_pids, &tree, &descendants) + { + Err(_) => attribution_incomplete = true, + Ok(mut set) => { + attribution::partition_by_role( + &mut set, + &tree, + &self.agent.executable_path, + login_shell_path, + ); + for companion in subject.companion_processes { + if let Some(record) = + tree.iter().find(|r| r.name == companion.executable_name) + { + attribution::attribute_companion_process( + &mut set, + record.pid, + &record.name, + record.executable_path.clone(), + ); + } + } + + let scratch = std::env::temp_dir(); + let slug = planned.sample_id.replace('/', "-"); + let union_report = invoke_and_parse_footprint( + &scratch.join(format!("fp-{slug}-union.json")), + &set.pids(), + ); + let orchestrator_report = invoke_and_parse_footprint( + &scratch.join(format!("fp-{slug}-orchestrator.json")), + &set.orchestrator_pids(), + ); + let session_report = invoke_and_parse_footprint( + &scratch.join(format!("fp-{slug}-session.json")), + &set.agent_cli_pids(), + ); + + match (union_report, orchestrator_report, session_report) { + (Some(union), Some(orchestrator), Some(session)) => { + let union_pids = set.pids(); + if footprint::verify_pid_set(&union_pids, &union).is_err() { + footprint_pid_mismatch = true; + } + let vanished: Vec = union_pids + .iter() + .copied() + .filter(|pid| !union.processes.iter().any(|p| p.pid == *pid)) + .collect(); + attribution_record = + Some(build_attribution_record(&set, &vanished, &union)); + + let rss_sum: u64 = tree + .iter() + .filter(|r| union_pids.contains(&r.pid)) + .map(|r| r.rss_bytes) + .sum(); + let core_process_bytes = union + .processes + .iter() + .find(|p| p.name == subject.main_executable_name) + .map(|p| p.footprint_bytes); + let render_helper_bytes = Some( + union + .processes + .iter() + .filter(|p| p.name != subject.main_executable_name) + .map(|p| p.footprint_bytes) + .sum(), + ); + + if let (Some(before), Ok(after_text)) = + (&before_memory, host_memory::invoke_vm_stat()) + { + if let Some(after) = host_memory::parse_vm_stat(&after_text) { + memory_record = Some(build_memory_record( + &union, + &orchestrator, + &session, + rss_sum, + core_process_bytes, + render_helper_bytes, + before, + &after, + )); + } + } + + idle_cpu_record = cpu_sampler::sample_idle_cpu( + &union_pids, + self.settings.idle_cpu_sample_interval_ms, + self.settings.idle_cpu_sample_count, + ) + .map(|summary| { + build_idle_cpu_record(&summary, "foreground-unoccluded") + }); + } + _ => attribution_incomplete = true, + } + } + } + } + + // Step 9/10: teardown, then seeder restore for tiers that seeded. + let helper_kill_count = self.teardown(subject); + if session_count > 0 { + if let Err(error) = seeding::restore_subject(&self.home, subject.id) { + eprintln!("restore {} failed: {error}", subject.id); + } + } + + let invalid_reason_value = classify_sample_invalidity( + planned.is_calibration, + false, + warm_helper_count, + cold_start_outcome.splash_timeout_seen, + quiesce_violation_seen, + attribution_incomplete, + footprint_pid_mismatch, + seed_incomplete, + seed_format_unverified, + cold_start_outcome.log_marks_unrecognized, + app_data_dir_missing, + ); + + Sample { + sample_id: planned.sample_id.clone(), + subject_id: planned.subject_id.clone(), + tier: planned.tier.as_str(), + session_count, + repetition: planned.repetition, + is_calibration: planned.is_calibration, + sampled_at, + is_valid: invalid_reason_value.is_none(), + invalid_reason: invalid_reason_value.map(str::to_string), + attribution: attribution_record, + memory: memory_record, + cold_start: Some(build_cold_start_record( + cold_start_outcome.first_window_visible_ms, + cold_start_outcome.main_window_visible_ms, + cold_start_outcome.app_window_ready_ms, + cold_start_outcome.splash_close_ms, + self + .settings + .window_visible_poll_ms + .max(self.settings.log_tail_poll_ms), + )), + idle_cpu: idle_cpu_record, + quiesce: Some(quiesce_reading.clone()), + warm_helper_count: Some(warm_helper_count), + helper_kill_count: Some(helper_kill_count), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::subject::find; + + #[test] + fn webkit_subject_is_warm_when_a_helper_entry_survives() { + let subject = find("termtree").unwrap(); + let entries = vec![LaunchServicesEntry { + display_name: "TermTree Web Content".into(), + bundle_identifier: Some("com.apple.WebKit.WebContent".into()), + bundle_path: None, + executable_path: None, + pid: Some(123), + in_front: false, + }]; + assert!(has_warm_helpers(subject, &entries, false)); + } + + #[test] + fn webkit_subject_is_not_warm_with_no_surviving_entries() { + let subject = find("termtree").unwrap(); + assert!(!has_warm_helpers(subject, &[], false)); + } + + #[test] + fn chromium_subject_warmth_comes_from_the_process_tree_signal() { + let subject = find("collaborator").unwrap(); + assert!(has_warm_helpers(subject, &[], true)); + assert!(!has_warm_helpers(subject, &[], false)); + } + + #[test] + fn calibration_discard_takes_priority_over_everything() { + let reason = classify_sample_invalidity( + true, true, 5, true, true, true, true, true, true, true, true, + ); + assert_eq!(reason, Some(invalid_reason::CALIBRATION_DISCARD)); + } + + #[test] + fn warm_webview_is_reported_when_present_and_not_calibration() { + let reason = classify_sample_invalidity( + false, false, 1, false, false, false, false, false, false, false, false, + ); + assert_eq!(reason, Some(invalid_reason::WARM_WEBVIEW)); + } + + #[test] + fn no_signals_is_valid() { + let reason = classify_sample_invalidity( + false, false, 0, false, false, false, false, false, false, false, false, + ); + assert_eq!(reason, None); + } + + #[test] + fn seed_incomplete_takes_priority_over_warm_webview() { + let reason = classify_sample_invalidity( + false, false, 1, false, false, false, false, true, false, false, false, + ); + assert_eq!(reason, Some(invalid_reason::SEED_INCOMPLETE)); + } + + #[test] + fn subject_already_running_takes_priority_over_seed_incomplete() { + let reason = classify_sample_invalidity( + false, true, 0, false, false, false, false, true, false, false, false, + ); + assert_eq!(reason, Some(invalid_reason::SUBJECT_ALREADY_RUNNING)); + } + + #[test] + fn seed_format_unverified_is_reported_when_no_stronger_reason_applies() { + let reason = classify_sample_invalidity( + false, false, 0, false, false, false, false, false, true, false, false, + ); + assert_eq!(reason, Some(invalid_reason::SEED_FORMAT_UNVERIFIED)); + } + + #[test] + fn seed_incomplete_takes_priority_over_seed_format_unverified() { + let reason = classify_sample_invalidity( + false, false, 0, false, false, false, false, true, true, false, false, + ); + assert_eq!(reason, Some(invalid_reason::SEED_INCOMPLETE)); + } + + #[test] + fn app_data_dir_missing_is_reported_when_no_stronger_reason_applies() { + let reason = classify_sample_invalidity( + false, false, 0, false, false, false, false, false, false, false, true, + ); + assert_eq!(reason, Some(invalid_reason::APP_DATA_DIR_NOT_CREATED)); + } + + #[test] + fn log_marks_unrecognized_is_reported_when_no_stronger_reason_applies() { + let reason = classify_sample_invalidity( + false, false, 0, false, false, false, false, false, false, true, false, + ); + assert_eq!( + reason, + Some(invalid_reason::TERMTREE_LOG_MARKS_UNRECOGNIZED) + ); + } + + // --- select_subjects / build_settings ---------------------------------- + + #[test] + fn default_selection_excludes_the_optional_subject() { + let selected = select_subjects(None, false).unwrap(); + assert!(!selected.iter().any(|s| s.id == "diri")); + assert!(selected.iter().any(|s| s.id == "termtree")); + } + + #[test] + fn allow_optional_subjects_includes_diri_in_the_default_selection() { + let selected = select_subjects(None, true).unwrap(); + assert!(selected.iter().any(|s| s.id == "diri")); + } + + #[test] + fn explicit_selection_of_an_unknown_id_is_refused() { + let error = + select_subjects(Some(&["not-a-subject".to_string()]), false).unwrap_err(); + assert!( + matches!(error, RunRefusal::UnknownSubject(id) if id == "not-a-subject") + ); + } + + #[test] + fn explicit_selection_of_an_optional_subject_is_honoured() { + let selected = select_subjects(Some(&["diri".to_string()]), false).unwrap(); + assert_eq!(selected.len(), 1); + assert_eq!(selected[0].id, "diri"); + } + + #[test] + fn repetitions_override_applies_uniformly_to_every_tier() { + let settings = build_settings(Some(2)); + assert_eq!(settings.repetitions.fresh_launch, 2); + assert_eq!(settings.repetitions.sustained_use, 2); + assert_eq!(settings.repetitions.n_session, 2); + } + + #[test] + fn no_override_keeps_the_tiered_defaults() { + let settings = build_settings(None); + assert_eq!(settings, RunSettings::default()); + } + + // --- plan_repetitions / pending_repetitions ----------------------------- + + #[test] + fn plan_includes_the_calibration_launch_plus_the_disclosed_count() { + let subject = find("termtree").unwrap(); + let settings = build_settings(Some(3)); + let plan = plan_repetitions(&[subject], &[Tier::FreshLaunch], &settings); + // repetitions 0..=3 inclusive == 4 planned launches. + assert_eq!(plan.len(), 4); + assert!(plan[0].is_calibration); + assert!(plan[1..].iter().all(|p| !p.is_calibration)); + assert_eq!(plan[0].sample_id, "termtree/fresh-launch/000"); + assert_eq!(plan[3].sample_id, "termtree/fresh-launch/003"); + } + + #[test] + fn plan_expands_every_subject_and_tier_combination() { + let termtree = find("termtree").unwrap(); + let collaborator = find("collaborator").unwrap(); + let settings = build_settings(Some(1)); + let plan = plan_repetitions( + &[termtree, collaborator], + &[Tier::FreshLaunch, Tier::NSession(5)], + &settings, + ); + // 2 subjects * 2 tiers * (0..=1 == 2 repetitions each) == 8. + assert_eq!(plan.len(), 8); + } + + #[test] + fn pending_repetitions_drops_sample_ids_already_completed() { + let subject = find("termtree").unwrap(); + let settings = build_settings(Some(2)); + let plan = plan_repetitions(&[subject], &[Tier::FreshLaunch], &settings); + let completed: HashSet = + [plan[0].sample_id.clone(), plan[1].sample_id.clone()] + .into_iter() + .collect(); + let pending = pending_repetitions(&plan, &completed); + assert_eq!(pending.len(), plan.len() - 2); + assert!(pending.iter().all(|p| !completed.contains(&p.sample_id))); + } + + #[test] + fn pending_repetitions_with_nothing_completed_is_the_whole_plan() { + let subject = find("termtree").unwrap(); + let settings = build_settings(Some(1)); + let plan = plan_repetitions(&[subject], &[Tier::FreshLaunch], &settings); + let pending = pending_repetitions(&plan, &HashSet::new()); + assert_eq!(pending, plan); + } + + #[test] + fn quiesce_recheck_count_is_zero_below_the_interval() { + assert_eq!(quiesce_recheck_count(15, 30), 0); + } + + #[test] + fn quiesce_recheck_count_divides_a_long_settle_window() { + assert_eq!(quiesce_recheck_count(120, 30), 4); + } + + // --- chromium_helper_resident / owned_webkit_helper_pids ---------------- + + #[test] + fn chromium_helper_resident_ignores_the_main_pid_itself() { + let records = vec![ProcessRecord { + pid: 1, + ppid: 0, + name: "Collaborator".into(), + executable_path: Some("/Applications/Collaborator.app/main".into()), + rss_bytes: 0, + }]; + assert!(!chromium_helper_resident( + &records, + 1, + "/Applications/Collaborator.app" + )); + } + + #[test] + fn chromium_helper_resident_true_for_a_surviving_helper() { + let records = vec![ + ProcessRecord { + pid: 1, + ppid: 0, + name: "Collaborator".into(), + executable_path: Some("/Applications/Collaborator.app/main".into()), + rss_bytes: 0, + }, + ProcessRecord { + pid: 2, + ppid: 1, + name: "Collaborator Helper".into(), + executable_path: Some("/Applications/Collaborator.app/helper".into()), + rss_bytes: 0, + }, + ]; + assert!(chromium_helper_resident( + &records, + 1, + "/Applications/Collaborator.app" + )); + } + + #[test] + fn owned_webkit_helper_pids_only_returns_this_subjects_helpers() { + let subject = find("termtree").unwrap(); + let entries = vec![ + LaunchServicesEntry { + display_name: "TermTree Web Content".into(), + bundle_identifier: Some("com.apple.WebKit.WebContent".into()), + bundle_path: None, + executable_path: None, + pid: Some(101), + in_front: false, + }, + // A same-named helper owned by a different app -- must never be + // returned (design §9's "shared-name helper" case). + LaunchServicesEntry { + display_name: "Other App Web Content".into(), + bundle_identifier: Some("com.apple.WebKit.WebContent".into()), + bundle_path: None, + executable_path: None, + pid: Some(202), + in_front: false, + }, + ]; + let pids = owned_webkit_helper_pids(subject, &entries); + assert_eq!(pids, vec![101]); + } + + // --- find_already_running_subject --------------------------------------- + + #[test] + fn finds_a_subject_already_running_by_bundle_identifier() { + let termtree = find("termtree").unwrap(); + let collaborator = find("collaborator").unwrap(); + let entries = vec![LaunchServicesEntry { + display_name: "TermTree".into(), + bundle_identifier: Some("com.termtree.desktop".into()), + bundle_path: None, + executable_path: None, + pid: Some(4242), + in_front: false, + }]; + let found = + find_already_running_subject(&[termtree, collaborator], &entries); + assert_eq!(found.map(|s| s.id), Some("termtree")); + } + + #[test] + fn a_registered_but_not_running_entry_does_not_count() { + let termtree = find("termtree").unwrap(); + // Registered with LaunchServices but not currently running: no `pid` + // line (design §5.2.1's "registered but not running" case). + let entries = vec![LaunchServicesEntry { + display_name: "TermTree".into(), + bundle_identifier: Some("com.termtree.desktop".into()), + bundle_path: None, + executable_path: None, + pid: None, + in_front: false, + }]; + assert!(find_already_running_subject(&[termtree], &entries).is_none()); + } + + #[test] + fn a_same_named_different_bundle_identifier_does_not_count() { + let termtree = find("termtree").unwrap(); + // Same display name, different bundle identifier -- must not match + // (this is exactly the "differently-named bundles can share an + // identifier" case inverted: here the names collide but the + // identifiers do not, so it must not be treated as the same subject). + let entries = vec![LaunchServicesEntry { + display_name: "TermTree".into(), + bundle_identifier: Some("com.example.unrelated".into()), + bundle_path: None, + executable_path: None, + pid: Some(1), + in_front: false, + }]; + assert!(find_already_running_subject(&[termtree], &entries).is_none()); + } + + // --- build_attribution_record / build_memory_record --------------------- + + fn footprint_report( + total: u64, + processes: Vec<(u32, &str, u64)>, + ) -> FootprintReport { + FootprintReport { + total_footprint_bytes: total, + processes: processes + .into_iter() + .map(|(pid, name, bytes)| crate::footprint::FootprintProcess { + pid, + name: name.to_string(), + footprint_bytes: bytes, + phys_footprint_bytes: None, + has_categories: false, + }) + .collect(), + shared: vec![], + errors: vec![], + warnings: vec![], + page_size_bytes: 16384, + start_time_iso: None, + } + } + + #[test] + fn attribution_record_carries_footprint_bytes_per_process() { + let set = AttributableProcessSet { + main_pid: 1, + processes: vec![attribution::AttributedProcess { + pid: 1, + name: "termtree".into(), + executable_path: None, + discovered_by: DiscoverySource::Both, + role: attribution::ProcessRole::Orchestrator, + }], + }; + let report = footprint_report(1000, vec![(1, "termtree", 1000)]); + let record = build_attribution_record(&set, &[99], &report); + assert_eq!(record.main_pid, 1); + assert_eq!(record.vanished_pids, vec![99]); + assert_eq!(record.launch_services_pids, vec![1]); + assert_eq!(record.process_tree_pids, vec![1]); + assert_eq!(record.processes[0].phys_footprint_bytes, Some(1000)); + assert_eq!(record.processes[0].discovered_by, "both"); + } + + fn host_memory_sample( + free_pages: u64, + used_pages: u64, + compressor_pages: u64, + swapouts: u64, + ) -> HostMemorySample { + let mut counters = std::collections::BTreeMap::new(); + counters.insert("Pages free".to_string(), free_pages); + counters.insert("Pages speculative".to_string(), 0); + counters.insert("Anonymous pages".to_string(), used_pages); + counters.insert("Pages wired down".to_string(), 0); + counters + .insert("Pages occupied by compressor".to_string(), compressor_pages); + counters.insert("Pages purgeable".to_string(), 0); + counters.insert("Swapouts".to_string(), swapouts); + HostMemorySample { + page_size_bytes: 1, + counters, + } + } + + #[test] + fn memory_record_free_ram_delta_is_positive_means_consumed() { + let union = footprint_report(1000, vec![]); + let orchestrator = footprint_report(700, vec![]); + let session = footprint_report(300, vec![]); + let before = host_memory_sample(1000, 0, 0, 0); + let after = host_memory_sample(400, 0, 0, 0); + let record = build_memory_record( + &union, + &orchestrator, + &session, + 1500, + Some(200), + Some(800), + &before, + &after, + ); + // free RAM dropped from 1000 to 400 -- 600 consumed, positive sign. + assert_eq!(record.free_ram_delta_bytes, 600); + assert_eq!(record.free_ram_delta_sign, "positive-means-consumed"); + assert_eq!(record.mem_phys_footprint_bytes, 1000); + assert_eq!(record.orchestrator_attributable_bytes, 700); + assert_eq!(record.agent_cli_attributable_bytes, 300); + // 700 + 300 - 1000 == 0: no cross-partition overlap in this fixture. + assert_eq!(record.cross_partition_shared_bytes, 0); + } + + #[test] + fn memory_record_cross_partition_overlap_is_the_excess_over_the_union() { + let union = footprint_report(1000, vec![]); + let orchestrator = footprint_report(700, vec![]); + let session = footprint_report(400, vec![]); + let before = host_memory_sample(1000, 0, 0, 0); + let after = host_memory_sample(1000, 0, 0, 0); + let record = build_memory_record( + &union, + &orchestrator, + &session, + 0, + None, + None, + &before, + &after, + ); + // 700 + 400 - 1000 == 100. + assert_eq!(record.cross_partition_shared_bytes, 100); + } + + #[test] + fn memory_record_swapouts_delta_never_goes_negative() { + let union = footprint_report(0, vec![]); + let before = host_memory_sample(0, 0, 0, 500); + let after = host_memory_sample(0, 0, 0, 300); + let record = build_memory_record( + &union, &union, &union, 0, None, None, &before, &after, + ); + assert_eq!(record.swapouts_delta, 0); + } + + // --- build_cold_start_record / build_idle_cpu_record --------------------- + + #[test] + fn cold_start_record_names_the_source_of_every_present_mark() { + let record = + build_cold_start_record(Some(400), Some(2100), Some(1500), None, 20); + assert_eq!( + record.mark_source.get("firstWindowVisibleMs"), + Some(&"cg-window-list".to_string()) + ); + assert_eq!( + record.mark_source.get("appWindowReadyMs"), + Some(&"karijini-log-arrival".to_string()) + ); + assert!(!record.mark_source.contains_key("splashCloseMs")); + } + + #[test] + fn idle_cpu_record_carries_the_summary_and_window_state() { + let summary = stats::Summary { + median: 4.7, + q1: 3.0, + q3: 6.0, + iqr: 3.0, + n: 30, + }; + let record = build_idle_cpu_record(&summary, "foreground-unoccluded"); + assert_eq!(record.idle_cpu_percent_of_one_core_median, 4.7); + assert_eq!(record.sample_count, 30); + assert_eq!(record.window_state, "foreground-unoccluded"); + } +} diff --git a/benchmark/src/scratch_home.rs b/benchmark/src/scratch_home.rs new file mode 100644 index 0000000..b326b9f --- /dev/null +++ b/benchmark/src/scratch_home.rs @@ -0,0 +1,205 @@ +//! The disposable per-run scratch home (spec item 1): every subject is +//! seeded, launched, and measured with `HOME` pointed at a fresh directory +//! this harness owns, never at the runner's real `$HOME`. A stranger +//! running this harness must never have it touch their real application +//! profiles -- so unlike most of this crate's other configuration knobs, +//! there is deliberately no fallback to the real environment: the default +//! is always a brand-new directory under the OS temp dir, created fresh +//! for this run and never reused across runs unless explicitly overridden. +//! +//! `--home ` (highest precedence) and `RESOURCE_BENCHMARK_HOME` +//! follow this crate's existing `RESOURCE_BENCHMARK_*` override convention +//! (`main.rs`'s `RESOURCE_BENCHMARK_REPO_PATH` / +//! `RESOURCE_BENCHMARK_AGENT_CLI_PATH`). An explicit override is on the +//! caller: if they point it at their real `$HOME`, that is no longer this +//! harness silently doing it to them. +//! +//! This is the seam every seeder already keys off of (`seeding/mod.rs`'s +//! `home: &Path` parameter, `seeding/termtree.rs`'s +//! `expected_scratch_state_path`) -- resolving it here, once, is what makes +//! "never the real $HOME by default" structural rather than a convention +//! every call site has to remember. + +use std::path::{Path, PathBuf}; + +pub const HOME_OVERRIDE_ENV: &str = "RESOURCE_BENCHMARK_HOME"; + +/// The directory name for one run's scratch home -- unique per process +/// invocation (`pid`) and per call (`disambiguator`, normally a +/// nanosecond-resolution timestamp) so two runs, or two calls within the +/// same process, never collide. Pure so the naming rule is testable +/// without touching the filesystem. +pub fn scratch_home_directory_name(pid: u32, disambiguator: u128) -> String { + format!("resource-benchmark-home-{pid}-{disambiguator}") +} + +/// Resolves the scratch home for this run: an explicit `--home` value wins, +/// then `RESOURCE_BENCHMARK_HOME`, then a freshly named directory under +/// `temp_dir`. Pure given already-read inputs, so the precedence order is +/// unit-tested without reading the real environment or touching the real +/// filesystem. +pub fn resolve_scratch_home( + cli_override: Option<&str>, + env_override: Option, + temp_dir: &Path, + pid: u32, + disambiguator: u128, +) -> PathBuf { + if let Some(path) = cli_override { + return PathBuf::from(path); + } + if let Some(path) = env_override { + return PathBuf::from(path); + } + temp_dir.join(scratch_home_directory_name(pid, disambiguator)) +} + +/// Creates the scratch home directory if it does not already exist. An +/// explicit `--home`/env override may point at a directory that already +/// exists (a re-runner reusing one deliberately), which is not an error. +pub fn ensure_scratch_home_exists(path: &Path) -> std::io::Result<()> { + std::fs::create_dir_all(path) +} + +/// Refuses a resolved scratch home that would put the harness back on the +/// runner's real profile. +/// +/// The default scratch home can never be the real `$HOME`, but an explicit +/// `--home`/`RESOURCE_BENCHMARK_HOME` can be -- and spec item 1 states the +/// property unconditionally ("the runner's real application state is never +/// read or written"), not "unless you asked for it". A stranger who types +/// `--home $HOME` to see what happens should get a refusal, not silent +/// data loss in the profile they actually use. +/// +/// Also refuses an *ancestor* of the real home (`/`, `/Users`, `/Users/x` +/// when the real home is `/Users/x/nested`), since seeding under one still +/// resolves into the real profile's tree. +/// +/// Pure: the real home is passed in, never read here, so the rule is +/// unit-testable without touching the environment. +pub fn reject_real_home( + resolved: &Path, + real_home: Option<&Path>, +) -> Result<(), String> { + let Some(real_home) = real_home else { + return Ok(()); + }; + if real_home.as_os_str().is_empty() { + return Ok(()); + } + if real_home.starts_with(resolved) { + return Err(format!( + "Refusing to use {} as the scratch home: it is the real home \ + directory ({}) or contains it, so seeding would overwrite the \ + runner's own application profiles. Omit --home/{} to get a fresh \ + disposable directory, or pass one outside the real home.", + resolved.display(), + real_home.display(), + HOME_OVERRIDE_ENV + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + + #[test] + fn rejects_the_real_home_itself() { + let error = + reject_real_home(Path::new("/Users/dev"), Some(Path::new("/Users/dev"))) + .unwrap_err(); + assert!(error.contains("Refusing"), "{error}"); + assert!(error.contains("/Users/dev"), "{error}"); + } + + #[test] + fn rejects_an_ancestor_of_the_real_home() { + assert!(reject_real_home( + Path::new("/Users"), + Some(Path::new("/Users/dev")) + ) + .is_err()); + assert!( + reject_real_home(Path::new("/"), Some(Path::new("/Users/dev"))).is_err() + ); + } + + #[test] + fn allows_a_scratch_directory_outside_the_real_home() { + assert!(reject_real_home( + Path::new("/var/folders/tmp/resource-benchmark-home-1-2"), + Some(Path::new("/Users/dev")) + ) + .is_ok()); + } + + #[test] + fn allows_a_directory_nested_inside_the_real_home() { + // Deliberate: a runner may keep a reusable scratch home under their own + // home directory. That writes only where they pointed it, never into + // the real profile's application-support tree above it. + assert!(reject_real_home( + Path::new("/Users/dev/benchmark-scratch"), + Some(Path::new("/Users/dev")) + ) + .is_ok()); + } + + #[test] + fn allows_anything_when_the_real_home_is_unknown_or_empty() { + assert!(reject_real_home(Path::new("/tmp/x"), None).is_ok()); + assert!(reject_real_home(Path::new("/tmp/x"), Some(Path::new(""))).is_ok()); + } + + use super::*; + + #[test] + fn cli_override_wins_over_everything() { + let home = resolve_scratch_home( + Some("/Users/dev/benchmark-home"), + Some("/tmp/env-home".to_string()), + Path::new("/tmp"), + 123, + 456, + ); + assert_eq!(home, PathBuf::from("/Users/dev/benchmark-home")); + } + + #[test] + fn env_override_wins_when_no_cli_override() { + let home = resolve_scratch_home( + None, + Some("/tmp/env-home".to_string()), + Path::new("/tmp"), + 123, + 456, + ); + assert_eq!(home, PathBuf::from("/tmp/env-home")); + } + + #[test] + fn falls_back_to_a_freshly_named_directory_under_temp_dir() { + let home = resolve_scratch_home(None, None, Path::new("/tmp"), 123, 456); + assert_eq!(home, PathBuf::from("/tmp/resource-benchmark-home-123-456")); + } + + #[test] + fn directory_names_differ_by_pid_and_disambiguator() { + let a = scratch_home_directory_name(1, 1); + let b = scratch_home_directory_name(1, 2); + let c = scratch_home_directory_name(2, 1); + assert_ne!(a, b); + assert_ne!(a, c); + } + + #[test] + fn never_falls_back_to_a_literal_home_env_reading() { + // The whole point of this module: there is no code path here that + // reads `$HOME`. This test exists as a documentation anchor -- if a + // future edit adds one, it belongs in `main.rs`'s explicit + // identity-lookup helper, never here. + let home = resolve_scratch_home(None, None, Path::new("/tmp"), 1, 1); + assert!(!home.to_string_lossy().is_empty()); + } +} diff --git a/benchmark/src/seeding/codenomad.rs b/benchmark/src/seeding/codenomad.rs new file mode 100644 index 0000000..f5a71bb --- /dev/null +++ b/benchmark/src/seeding/codenomad.rs @@ -0,0 +1,129 @@ +//! CodeNomad's session seeder, serving **both** the Electron and Tauri +//! builds -- they are the same monorepo with the same seeding contract, +//! which is the whole reason the pair isolates runtime as the only +//! variable (spec FR-1, design §5.6.3). +//! +//! CodeNomad is client-server: its local server process is registered as a +//! `CompanionProcess` in `subject.rs` and attributed via +//! `attribution::attribute_companion_process`, never left uncounted (spec +//! FR-2). +//! +//! **This seed format has never been checked against a real CodeNomad +//! install** (spec item 4) -- `SEED_METHOD` names the pinned v0.18.0's +//! *documented* seeding mechanism, but the env var name, the config-file +//! shape, and the pinned version's actual v0.18.0 behavior are all +//! unverified. `subject.rs` sets `seed_format_verified: false` on both +//! `codenomad-electron` and `codenomad-tauri` (they share this seeder), so +//! every N-session/sustained-use sample for either reports +//! `invalidReason: "seed-format-unverified"` until someone confirms this +//! against a real install and flips that flag. + +use super::{AgentCliPin, SeedError, SeedPlan, SeededRepo, SessionSeeder}; +use std::env; +use std::path::PathBuf; + +/// The pinned v0.18.0's documented seeding mechanism. Recorded so the +/// method actually used ships in provenance (`SeedPlan.method`) rather than +/// being assumed. +const SEED_METHOD: &str = "CODENOMAD_SEED_SESSIONS env var + config file"; + +pub struct CodeNomadSeeder { + pub config_path: PathBuf, +} + +impl CodeNomadSeeder { + pub fn production(home: &std::path::Path) -> Self { + Self { + config_path: home.join(".codenomad").join("sessions.json"), + } + } +} + +impl SessionSeeder for CodeNomadSeeder { + fn seed( + &self, + n: u32, + repo: &SeededRepo, + agent: &AgentCliPin, + ) -> Result { + // env::set_var is process-global and this harness never runs two + // subjects concurrently, so a plain env var plus a config file (the + // documented v0.18.0 mechanism) is sufficient -- no lock/backup + // discipline is needed because CodeNomad's config file is + // benchmark-owned, not a pre-existing user file (unlike TermTree's + // production state.json, §5.6.1). + let sessions: Vec<_> = (0..n) + .map(|i| { + serde_json::json!({ + "id": format!("resource-benchmark-session-{i}"), + "cwd": repo.local_path, + "command": agent.executable_path, + }) + }) + .collect(); + if let Some(parent) = self.config_path.parent() { + std::fs::create_dir_all(parent).map_err(|e| SeedError(e.to_string()))?; + } + std::fs::write( + &self.config_path, + serde_json::to_string_pretty(&sessions) + .expect("session serialization never fails"), + ) + .map_err(|e| SeedError(e.to_string()))?; + unsafe { + env::set_var("CODENOMAD_SEED_SESSIONS", self.config_path.as_os_str()); + } + Ok(SeedPlan { + method: SEED_METHOD.to_string(), + }) + } + + fn restore(&self) -> Result<(), SeedError> { + unsafe { + env::remove_var("CODENOMAD_SEED_SESSIONS"); + } + if self.config_path.exists() { + std::fs::remove_file(&self.config_path) + .map_err(|e| SeedError(e.to_string()))?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn repo() -> SeededRepo { + SeededRepo { + url: "https://example.com/repo.git".into(), + commit: "abc123".into(), + local_path: "/Users/Shared/benchmark-repo".into(), + } + } + + fn agent() -> AgentCliPin { + AgentCliPin { + name: "claude".into(), + version: "1.0.0".into(), + executable_path: "/Users/dev/.local/bin/claude".into(), + } + } + + #[test] + fn seed_writes_the_config_file_and_restore_removes_it() { + let dir = std::env::temp_dir().join(format!( + "resource-benchmark-codenomad-seeder-test-{}", + std::process::id() + )); + let seeder = CodeNomadSeeder { + config_path: dir.join("sessions.json"), + }; + let plan = seeder.seed(4, &repo(), &agent()).unwrap(); + assert_eq!(plan.method, SEED_METHOD); + assert!(seeder.config_path.exists()); + seeder.restore().unwrap(); + assert!(!seeder.config_path.exists()); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/benchmark/src/seeding/collaborator.rs b/benchmark/src/seeding/collaborator.rs new file mode 100644 index 0000000..b3c9ccb --- /dev/null +++ b/benchmark/src/seeding/collaborator.rs @@ -0,0 +1,150 @@ +//! Collaborator's session seeder: writes N terminal tiles into +//! `~/.collaborator/`'s canvas JSON (spec FR-9, design §5.6.2). +//! +//! Collaborator spawns a vendored `tmux` and a `node-pty` sidecar to own +//! its PTYs; both are `Orchestrator` under `attribution.rs`'s one +//! partition rule (design §5.2.3) -- Collaborator's own implementation +//! choice for a job TermTree does in-process with `portable-pty`, named +//! here per FR-9's "documented in the harness source, next to that +//! subject's adapter". +//! +//! **This seed format has never been checked against a real Collaborator +//! install** (spec item 4) -- the `{"tiles": [{"id", "cwd", "command"}]}` +//! shape below is this project's best guess at what +//! `~/.collaborator/canvas.json` needs to contain, not a verified +//! contract. `subject.rs`'s `seed_format_verified: false` on the +//! `collaborator` entry reflects this: every N-session/sustained-use +//! sample for this subject reports `invalidReason: +//! "seed-format-unverified"` until someone confirms this format against a +//! real install and flips that flag. + +use super::{AgentCliPin, SeedError, SeedPlan, SeededRepo, SessionSeeder}; +use serde_json::json; +use std::fs; +use std::path::{Path, PathBuf}; + +const CANVAS_FILENAME: &str = "canvas.json"; +const BACKUP_SUFFIX: &str = ".before-resource-benchmark.json"; + +pub struct CollaboratorSeeder { + pub state_directory: PathBuf, +} + +impl CollaboratorSeeder { + pub fn production(home: &Path) -> Self { + Self { + state_directory: home.join(".collaborator"), + } + } + + fn canvas_path(&self) -> PathBuf { + self.state_directory.join(CANVAS_FILENAME) + } + + fn backup_path(&self) -> PathBuf { + self + .state_directory + .join(format!("{CANVAS_FILENAME}{BACKUP_SUFFIX}")) + } +} + +impl SessionSeeder for CollaboratorSeeder { + fn seed( + &self, + n: u32, + repo: &SeededRepo, + agent: &AgentCliPin, + ) -> Result { + fs::create_dir_all(&self.state_directory) + .map_err(|e| SeedError(e.to_string()))?; + let canvas = self.canvas_path(); + let backup = self.backup_path(); + if backup.exists() { + return Err(SeedError(format!( + "Backup already exists at {}. Run `resource-benchmark restore` \ + before seeding again.", + backup.display() + ))); + } + if canvas.exists() { + fs::copy(&canvas, &backup).map_err(|e| SeedError(e.to_string()))?; + } + let tiles: Vec<_> = (0..n) + .map(|i| { + json!({ + "id": format!("resource-benchmark-tile-{i}"), + "cwd": repo.local_path, + "command": agent.executable_path, + }) + }) + .collect(); + let text = serde_json::to_string_pretty(&json!({ "tiles": tiles })) + .expect("tile serialization never fails"); + fs::write(&canvas, text).map_err(|e| SeedError(e.to_string()))?; + Ok(SeedPlan { + method: "~/.collaborator canvas.json pre-write".to_string(), + }) + } + + fn restore(&self) -> Result<(), SeedError> { + let backup = self.backup_path(); + if !backup.exists() { + return Ok(()); + } + fs::rename(&backup, self.canvas_path()) + .map_err(|e| SeedError(e.to_string())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::process; + + fn temp_home() -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "resource-benchmark-collaborator-seeder-test-{}-{}", + process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&dir).unwrap(); + dir + } + + fn repo() -> SeededRepo { + SeededRepo { + url: "https://example.com/repo.git".into(), + commit: "abc123".into(), + local_path: "/Users/Shared/benchmark-repo".into(), + } + } + + fn agent() -> AgentCliPin { + AgentCliPin { + name: "claude".into(), + version: "1.0.0".into(), + executable_path: "/Users/dev/.local/bin/claude".into(), + } + } + + #[test] + fn seed_then_restore_round_trips_prior_canvas() { + let home = temp_home(); + let seeder = CollaboratorSeeder::production(&home); + fs::create_dir_all(&seeder.state_directory).unwrap(); + fs::write(seeder.canvas_path(), "{\"tiles\":[{\"id\":\"real\"}]}").unwrap(); + + seeder.seed(5, &repo(), &agent()).unwrap(); + let seeded = fs::read_to_string(seeder.canvas_path()).unwrap(); + assert!(seeded.contains("resource-benchmark-tile-0")); + + seeder.restore().unwrap(); + let restored = fs::read_to_string(seeder.canvas_path()).unwrap(); + assert!(restored.contains("\"real\"")); + + fs::remove_dir_all(&home).unwrap(); + } +} diff --git a/benchmark/src/seeding/diri.rs b/benchmark/src/seeding/diri.rs new file mode 100644 index 0000000..398e9c6 --- /dev/null +++ b/benchmark/src/seeding/diri.rs @@ -0,0 +1,118 @@ +//! diri's session seeder (optional subject, spec FR-1, design §5.6.4). +//! +//! diri's `dirijord-rs` daemon owns the PTYs and its sessions **outlive the +//! app** -- unlike every other subject here. The seeder therefore +//! terminates `dirijord-rs` before seeding *and* the orchestrator's +//! teardown terminates it again; without both halves its memory would be +//! attributed to nobody and its sessions would leak into the next +//! repetition. `attribution.rs` includes it via a `CompanionProcess` +//! (`subject.rs`'s registry entry for `diri`). +//! +//! **This seed format has never been checked against a real diri install** +//! (spec item 4) -- the `[[session]]`-table TOML shape below is this +//! project's best guess at what `~/.diri/sessions.toml` needs to contain, +//! not a verified contract. `subject.rs`'s `seed_format_verified: false` +//! on the `diri` entry reflects this: every N-session/sustained-use +//! sample for diri reports `invalidReason: "seed-format-unverified"` until +//! someone confirms this format against a real install and flips that +//! flag. + +use super::{AgentCliPin, SeedError, SeedPlan, SeededRepo, SessionSeeder}; +use crate::exec::run_capture; +use std::fs; +use std::path::PathBuf; + +pub const DIRIJORD_EXECUTABLE_NAME: &str = "dirijord-rs"; + +pub struct DiriSeeder { + pub config_path: PathBuf, +} + +impl DiriSeeder { + pub fn production(home: &std::path::Path) -> Self { + Self { + config_path: home.join(".diri").join("sessions.toml"), + } + } +} + +/// Kills every resident `dirijord-rs` process. Called before seeding (so a +/// prior run's sessions cannot leak into this one) and again during +/// teardown (design §5.6.4). Best-effort: a daemon that is already gone is +/// not an error. +pub fn kill_dirijord_daemon() { + let _ = run_capture("/usr/bin/pkill", &["-x", DIRIJORD_EXECUTABLE_NAME]); +} + +impl SessionSeeder for DiriSeeder { + fn seed( + &self, + n: u32, + repo: &SeededRepo, + agent: &AgentCliPin, + ) -> Result { + kill_dirijord_daemon(); + if let Some(parent) = self.config_path.parent() { + fs::create_dir_all(parent).map_err(|e| SeedError(e.to_string()))?; + } + let mut sessions = String::new(); + for i in 0..n { + sessions.push_str(&format!( + "[[session]]\nid = \"resource-benchmark-session-{i}\"\ncwd = \"{}\"\ncommand = \"{}\"\n\n", + repo.local_path, agent.executable_path + )); + } + fs::write(&self.config_path, sessions) + .map_err(|e| SeedError(e.to_string()))?; + Ok(SeedPlan { + method: "~/.diri/sessions.toml pre-write".to_string(), + }) + } + + fn restore(&self) -> Result<(), SeedError> { + kill_dirijord_daemon(); + if self.config_path.exists() { + fs::remove_file(&self.config_path) + .map_err(|e| SeedError(e.to_string()))?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn repo() -> SeededRepo { + SeededRepo { + url: "https://example.com/repo.git".into(), + commit: "abc123".into(), + local_path: "/Users/Shared/benchmark-repo".into(), + } + } + + fn agent() -> AgentCliPin { + AgentCliPin { + name: "claude".into(), + version: "1.0.0".into(), + executable_path: "/Users/dev/.local/bin/claude".into(), + } + } + + #[test] + fn seed_writes_n_sessions_and_restore_removes_the_file() { + let dir = std::env::temp_dir().join(format!( + "resource-benchmark-diri-seeder-test-{}", + std::process::id() + )); + let seeder = DiriSeeder { + config_path: dir.join("sessions.toml"), + }; + seeder.seed(3, &repo(), &agent()).unwrap(); + let text = fs::read_to_string(&seeder.config_path).unwrap(); + assert_eq!(text.matches("[[session]]").count(), 3); + seeder.restore().unwrap(); + assert!(!seeder.config_path.exists()); + let _ = fs::remove_dir_all(&dir); + } +} diff --git a/benchmark/src/seeding/mod.rs b/benchmark/src/seeding/mod.rs new file mode 100644 index 0000000..89adc15 --- /dev/null +++ b/benchmark/src/seeding/mod.rs @@ -0,0 +1,184 @@ +//! Session seeders (spec FR-9): each subject gets N live agent sessions +//! started the same reproducible way every run. `AgentCliPin` and +//! `SeededRepo` are resolved once per run and shared by every seeder, so +//! "same repository and same agent CLI invocation across every subject" is +//! structural rather than merely checked (design §5.6). + +pub mod codenomad; +pub mod collaborator; +pub mod diri; +pub mod termtree; + +use crate::process_tree::ProcessRecord; +use std::path::Path; + +#[derive(Debug, Clone, PartialEq)] +pub struct AgentCliPin { + pub name: String, + pub version: String, + pub executable_path: String, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct SeededRepo { + pub url: String, + pub commit: String, + pub local_path: String, +} + +#[derive(Debug, PartialEq)] +pub struct SeedPlan { + pub method: String, +} + +#[derive(Debug)] +pub struct SeedError(pub String); + +impl std::fmt::Display for SeedError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} +impl std::error::Error for SeedError {} + +pub trait SessionSeeder { + /// Write whatever on-disk/CLI state makes the subject start with `n` + /// sessions. + fn seed( + &self, + n: u32, + repo: &SeededRepo, + agent: &AgentCliPin, + ) -> Result; + /// Undo `seed`, restoring any state that existed before it. + fn restore(&self) -> Result<(), SeedError>; +} + +/// Dispatches `subject_id` to its own seeder's [`SessionSeeder::seed`] +/// (design §5.6). The one place that knows the `subject_id` -> seeder +/// mapping, shared by `main.rs`'s `seed` subcommand and `run.rs`'s live +/// orchestration so the two never drift apart. +pub fn seed_subject( + home: &Path, + subject_id: &str, + n: u32, + repo: &SeededRepo, + agent: &AgentCliPin, +) -> Result { + match subject_id { + "termtree" => { + termtree::TermTreeSeeder::production(home).seed(n, repo, agent) + } + "collaborator" => { + collaborator::CollaboratorSeeder::production(home).seed(n, repo, agent) + } + "codenomad-electron" | "codenomad-tauri" => { + codenomad::CodeNomadSeeder::production(home).seed(n, repo, agent) + } + "diri" => diri::DiriSeeder::production(home).seed(n, repo, agent), + other => Err(SeedError(format!("unknown subject: {other}"))), + } +} + +/// Dispatches `subject_id` to its own seeder's [`SessionSeeder::restore`] +/// -- the undo side of [`seed_subject`], shared the same way. +pub fn restore_subject(home: &Path, subject_id: &str) -> Result<(), SeedError> { + match subject_id { + "termtree" => termtree::TermTreeSeeder::production(home).restore(), + "collaborator" => { + collaborator::CollaboratorSeeder::production(home).restore() + } + "codenomad-electron" | "codenomad-tauri" => { + codenomad::CodeNomadSeeder::production(home).restore() + } + "diri" => diri::DiriSeeder::production(home).restore(), + other => Err(SeedError(format!("unknown subject: {other}"))), + } +} + +/// Generic session-readiness check (NFR-3: checked the same way for every +/// subject, never per-subject). A subject is considered seeded once its +/// process tree contains `n` processes whose executable path is the login +/// shell **and** `n` whose path is the pinned agent CLI. +pub fn count_ready_sessions( + tree_records: &[ProcessRecord], + root_pid: u32, + login_shell_path: &str, + agent_cli_executable_path: &str, +) -> (u32, u32) { + let descendants = crate::process_tree::descendants_of(tree_records, root_pid); + let mut shells = 0u32; + let mut agents = 0u32; + for pid in descendants { + let Some(record) = crate::process_tree::record_by_pid(tree_records, pid) + else { + continue; + }; + match record.executable_path.as_deref() { + Some(path) if path == login_shell_path => shells += 1, + Some(path) if path == agent_cli_executable_path => agents += 1, + _ => {} + } + } + (shells, agents) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn record(pid: u32, ppid: u32, exe: &str) -> ProcessRecord { + ProcessRecord { + pid, + ppid, + name: "x".into(), + executable_path: Some(exe.into()), + rss_bytes: 0, + } + } + + #[test] + fn seed_subject_rejects_an_unknown_subject_id() { + let repo = SeededRepo { + url: "https://example.invalid/repo".into(), + commit: "abc".into(), + local_path: "/tmp/repo".into(), + }; + let agent = AgentCliPin { + name: "claude".into(), + version: "1.0.0".into(), + executable_path: "/usr/local/bin/claude".into(), + }; + let error = + seed_subject(Path::new("/tmp"), "not-a-subject", 1, &repo, &agent) + .unwrap_err(); + assert!(error.0.contains("not-a-subject")); + } + + #[test] + fn restore_subject_rejects_an_unknown_subject_id() { + let error = + restore_subject(Path::new("/tmp"), "not-a-subject").unwrap_err(); + assert!(error.0.contains("not-a-subject")); + } + + #[test] + fn counts_shells_and_agents_among_descendants_only() { + let records = vec![ + record(1, 0, "/Applications/TermTree.app/termtree"), + record(2, 1, "/bin/zsh"), + record(3, 2, "/Users/dev/.local/bin/claude"), + record(4, 1, "/bin/zsh"), + // Not a descendant of root (1): must not be counted. + record(5, 99, "/bin/zsh"), + ]; + let (shells, agents) = count_ready_sessions( + &records, + 1, + "/bin/zsh", + "/Users/dev/.local/bin/claude", + ); + assert_eq!(shells, 2); + assert_eq!(agents, 1); + } +} diff --git a/benchmark/src/seeding/termtree.rs b/benchmark/src/seeding/termtree.rs new file mode 100644 index 0000000..5a73621 --- /dev/null +++ b/benchmark/src/seeding/termtree.rs @@ -0,0 +1,620 @@ +//! TermTree's session seeder: writes `state.json` before launch (spec +//! FR-9, design §5.6.1) under the harness's disposable scratch home (spec +//! item 1) -- never a real TermTree profile. +//! +//! **This is the single highest-consequence module in the harness.** A +//! public build of this harness runs on a stranger's machine, which may +//! have a real TermTree install with real user data at +//! `~/Library/Application Support/DocumentNode/TermTree/state.json`. This +//! seeder must never be able to reach that path. +//! +//! The original (private-repo) version of this module asserted the +//! opposite of today's contract: it *required* a production path and +//! *refused* a `TermTreeDev` one, because at the time the harness always +//! launched subjects against the developer's real `$HOME` and the risk +//! being guarded against was colliding with `scripts/src/bin/ +//! sample_mindmap.rs`'s development-only seeder. Now that every subject is +//! launched with `HOME` pointed at a fresh scratch directory (`open -n +//! --env HOME=`, `cold_start.rs`), that risk is structurally gone +//! and the actual risk is the opposite one: this seeder reaching a real +//! profile on a third party's machine. [`expected_scratch_state_path`] is +//! the inversion -- it requires the target be under the harness's own +//! scratch root and refuses anything that is not, which now includes a +//! real production profile. The `DocumentNode/TermTree` suffix check is +//! kept as a secondary, defense-in-depth condition (still refuses a +//! `TermTreeDev`-named directory even if one somehow lived under the +//! scratch root), but scratch-root containment is the primary guard. + +use super::{AgentCliPin, SeedError, SeedPlan, SeededRepo, SessionSeeder}; +use serde::Serialize; +use std::ffi::OsStr; +use std::fs::{self, OpenOptions}; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::process; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +const STATE_FILENAME: &str = "state.json"; +const BACKUP_FILENAME: &str = "state.json.before-resource-benchmark.json"; +const LOCK_DIRECTORY: &str = ".resource-benchmark-seed.lock"; +const RECOVERY_PREFIX: &str = "state.json.benchmark-recovery-"; +static FILE_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct Node { + id: String, + label: String, + status: String, + session_id: Option<()>, + previous_session_id: Option<()>, + command: Option, + cwd: Option, + exit_code: Option, + claude_session_id: Option<()>, + children: Vec, + collapsed: bool, +} + +pub struct TermTreeSeeder { + pub state_directory: PathBuf, + /// The harness's disposable scratch home (spec item 1). `state_directory` + /// must live under this root -- [`expected_scratch_state_path`] refuses + /// to seed anywhere else, including a real production profile. + pub scratch_root: PathBuf, +} + +impl TermTreeSeeder { + /// `/Library/Application Support/DocumentNode/TermTree` -- + /// the disposable-scratch-home equivalent of TermTree's real + /// release-build state path. macOS only, matching this harness's scope + /// (spec §8: Windows/Linux hosts are out of scope for v1). + pub fn production(scratch_home: &Path) -> Self { + Self { + state_directory: crate::log_marks::app_data_dir( + &scratch_home.join("Library").join("Application Support"), + ), + scratch_root: scratch_home.to_path_buf(), + } + } +} + +/// Asserts `state_directory/state.json` is both under `scratch_root` and +/// ends in `DocumentNode/TermTree/` -- the **inversion** of this module's +/// original guard (module doc): scratch-root containment is now the +/// primary, affirmative condition, and the suffix check is kept only as +/// defense in depth. A public runner may have a real TermTree profile on +/// this machine; this seeder must never be able to reach it, so everything +/// outside the scratch root is refused, which now includes that real +/// profile. +fn expected_scratch_state_path( + scratch_root: &Path, + state_directory: &Path, +) -> Result { + let path = state_directory.join(STATE_FILENAME); + let directory = path.parent(); + let application_directory = directory.and_then(Path::parent); + let under_scratch_root = path.starts_with(scratch_root); + if !under_scratch_root + || path.file_name() != Some(OsStr::new(STATE_FILENAME)) + || directory.and_then(Path::file_name) != Some(OsStr::new("TermTree")) + || application_directory.and_then(Path::file_name) + != Some(OsStr::new("DocumentNode")) + { + return Err(SeedError(format!( + "Refusing {}. The resource-benchmark seeder only writes state.json \ + under its own disposable scratch home ({}), ending in \ + DocumentNode/TermTree/state.json -- never a real TermTree profile.", + path.display(), + scratch_root.display() + ))); + } + Ok(path) +} + +struct Lock { + path: PathBuf, +} +impl Lock { + fn acquire(directory: &Path) -> Result { + let path = directory.join(LOCK_DIRECTORY); + fs::create_dir(&path).map_err(|error| { + if error.kind() == io::ErrorKind::AlreadyExists { + SeedError(format!( + "Another resource-benchmark seed/restore is in progress: {}", + path.display() + )) + } else { + SeedError(error.to_string()) + } + })?; + Ok(Self { path }) + } +} +impl Drop for Lock { + fn drop(&mut self) { + let _ = fs::remove_dir(&self.path); + } +} + +fn unique_suffix() -> String { + format!( + "{}.{}.{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(), + process::id(), + FILE_SEQUENCE.fetch_add(1, Ordering::Relaxed) + ) +} + +fn temporary_path(target: &Path) -> PathBuf { + target.with_file_name(format!( + ".{}.{suffix}.tmp", + target + .file_name() + .unwrap_or_else(|| OsStr::new("state")) + .to_string_lossy(), + suffix = unique_suffix() + )) +} + +fn atomic_write(target: &Path, text: &str) -> Result<(), SeedError> { + let temporary = temporary_path(target); + let result = (|| -> Result<(), SeedError> { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary) + .map_err(|e| SeedError(e.to_string()))?; + file + .write_all(text.as_bytes()) + .and_then(|()| file.sync_all()) + .map_err(|e| SeedError(e.to_string()))?; + drop(file); + fs::rename(&temporary, target).map_err(|e| SeedError(e.to_string())) + })(); + if result.is_err() { + let _ = fs::remove_file(&temporary); + } + result +} + +fn regular_file_or_missing( + path: &Path, + description: &str, +) -> Result { + match fs::symlink_metadata(path) { + Ok(metadata) + if metadata.file_type().is_file() + && !metadata.file_type().is_symlink() => + { + Ok(true) + } + Ok(_) => Err(SeedError(format!( + "{description} must be a regular file: {}", + path.display() + ))), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(SeedError(error.to_string())), + } +} + +fn write_recovery(directory: &Path, text: &str) -> Result { + for _ in 0..1000 { + let path = + directory.join(format!("{RECOVERY_PREFIX}{}.json", unique_suffix())); + match OpenOptions::new().write(true).create_new(true).open(&path) { + Ok(mut file) => { + file + .write_all(text.as_bytes()) + .and_then(|()| file.sync_all()) + .map_err(|e| SeedError(e.to_string()))?; + return Ok(path); + } + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(SeedError(error.to_string())), + } + } + Err(SeedError( + "Could not allocate a unique benchmark recovery filename.".into(), + )) +} + +fn node( + id: &str, + status: &str, + command: Option, + cwd: Option, +) -> Node { + Node { + id: id.to_string(), + label: id.to_string(), + status: status.to_string(), + session_id: None, + previous_session_id: None, + command, + cwd, + exit_code: None, + claude_session_id: None, + children: Vec::new(), + collapsed: false, + } +} + +/// The mind-map root. Its status is deliberately `none` so +/// `restoreSessions` does not queue the root itself for relaunch -- only +/// its `n` children are sessions +/// (`frontend/src/store/AppStateController.ts:855-861`). +const ROOT_NODE_ID: &str = "resource-benchmark-root"; + +/// `n` session nodes with status `running`/`idle`/`waiting`, cycled +/// round-robin beneath a single non-relaunching root, all pointing at +/// `repo.local_path` and `agent`'s invocation -- the tree shape does not +/// vary with `n` (design §5.6.1). +/// +/// The document shape is TermTree's own persisted schema, produced by +/// `AppStateController.getPersistedState()` +/// (`frontend/src/store/AppStateController.ts:941-990`): a single recursive +/// `tree` root plus the sibling preference fields. The app restores sessions +/// by reading `persisted.tree` (`:883` `restoreSessions(persisted.tree)`, +/// `:886`), so a document keyed on anything else relaunches **zero** +/// sessions and the N-session tier would silently measure an empty app +/// (spec FR-8, FR-9). +/// +/// `sessionId` is left null on purpose. `restoreSessions` copies it into +/// `previousSessionId`, which is the scrollback buffer key; a fabricated id +/// would point at `.buf`/`.snap` files that do not exist. With no saved +/// `agentKind`/`agentSessionId` the relaunch path issues no provider resume +/// command, so each seeded pane starts the agent CLI **fresh** via its +/// `command` rather than resuming a prior conversation. The tier therefore +/// measures `n` freshly started agent sessions, not `n` resumed ones -- a +/// distinction the published method must state (spec FR-14). +fn fixture_for_n_sessions( + n: u32, + repo: &SeededRepo, + agent: &AgentCliPin, +) -> String { + const STATUSES: [&str; 3] = ["running", "idle", "waiting"]; + let sessions: Vec = (0..n) + .map(|i| { + node( + &format!("resource-benchmark-session-{i}"), + STATUSES[i as usize % STATUSES.len()], + Some(agent.executable_path.clone()), + Some(repo.local_path.clone()), + ) + }) + .collect(); + let mut root = node(ROOT_NODE_ID, "none", None, None); + root.children = sessions; + serde_json::to_string_pretty(&serde_json::json!({ + "tree": root, + "panelPosition": "right", + "panelSplitPercent": 60, + "panelCollapsed": false, + "panelCollapsedPreferenceVersion": 1, + "editorPanelSplitPercent": 65, + "editorOutlineVisible": false, + "editorOutlineSplitPercent": 78, + "editorRecentFiles": [], + "themeKey": "midnight", + "lastThemeByMode": { "dark": "midnight", "light": "daylight" }, + "layoutType": "Rightward", + "minimapVisible": false, + "viewportZoom": 1, + "viewportX": 0, + "viewportY": 0, + "onboardingSeen": true, + "lastUsedCwd": repo.local_path, + "fontPreference": "bundled", + "mindmapLineStyle": "curved", + "mindmapLineWidthPx": 2, + "mindmapMarginParental": 18, + "mindmapMarginSibling": 8, + "mindmapAnimateConnections": true, + "scrollbackNodeIds": [], + })) + .expect("Node serialization never fails") +} + +impl SessionSeeder for TermTreeSeeder { + fn seed( + &self, + n: u32, + repo: &SeededRepo, + agent: &AgentCliPin, + ) -> Result { + let state = + expected_scratch_state_path(&self.scratch_root, &self.state_directory)?; + let directory = state.parent().expect("state has a parent"); + let backup = directory.join(BACKUP_FILENAME); + fs::create_dir_all(directory).map_err(|e| SeedError(e.to_string()))?; + let _lock = Lock::acquire(directory)?; + if regular_file_or_missing(&backup, "Resource-benchmark seed backup")? { + return Err(SeedError(format!( + "Backup already exists at {}. Run `resource-benchmark restore` \ + before seeding again so it cannot be overwritten.", + backup.display() + ))); + } + let fixture = fixture_for_n_sessions(n, repo, agent); + if regular_file_or_missing(&state, "Production TermTree state")? { + fs::copy(&state, &backup).map_err(|e| SeedError(e.to_string()))?; + } + atomic_write(&state, &fixture)?; + Ok(SeedPlan { + method: "production state.json pre-write".to_string(), + }) + } + + fn restore(&self) -> Result<(), SeedError> { + let state = + expected_scratch_state_path(&self.scratch_root, &self.state_directory)?; + let directory = state.parent().expect("state has a parent"); + if !directory.exists() { + // Nothing was ever seeded; restoring is a safe no-op. + return Ok(()); + } + let backup = directory.join(BACKUP_FILENAME); + let _lock = Lock::acquire(directory)?; + if !regular_file_or_missing(&backup, "Resource-benchmark seed backup")? { + // No seed is in progress; restoring is a safe no-op (design §9: the + // manual `benchmark restore` escape hatch must always be safe). + return Ok(()); + } + if regular_file_or_missing(&state, "Production TermTree state")? { + let current = + fs::read_to_string(&state).map_err(|e| SeedError(e.to_string()))?; + write_recovery(directory, ¤t)?; + } + fs::rename(&backup, &state).map_err(|e| SeedError(e.to_string()))?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_home() -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "resource-benchmark-termtree-seeder-test-{}-{}", + process::id(), + unique_suffix() + )); + fs::create_dir_all(&dir).unwrap(); + dir + } + + fn repo() -> SeededRepo { + SeededRepo { + url: "https://example.com/repo.git".into(), + commit: "abc123".into(), + local_path: "/Users/Shared/benchmark-repo".into(), + } + } + + fn agent() -> AgentCliPin { + AgentCliPin { + name: "claude".into(), + version: "1.0.0".into(), + executable_path: "/Users/dev/.local/bin/claude".into(), + } + } + + /// The primary guard (spec item 2's inversion): a `state_directory` that + /// is otherwise shaped correctly but lives **outside** the scratch root + /// -- e.g. a real user's actual home directory, which is exactly what a + /// public runner of this harness may have on their machine -- must be + /// refused. This is the case the original (pre-inversion) guard could + /// not catch at all: it only ever inspected the directory's own name. + #[test] + fn refuses_a_directory_outside_the_scratch_root() { + let scratch_root = temp_home(); + let real_home = temp_home(); + let outside_directory = real_home + .join("Library") + .join("Application Support") + .join("DocumentNode") + .join("TermTree"); + let seeder = TermTreeSeeder { + state_directory: outside_directory, + scratch_root: scratch_root.clone(), + }; + let error = seeder.seed(5, &repo(), &agent()).unwrap_err(); + assert!(error.0.contains("Refusing")); + assert!(error.0.contains("scratch home")); + fs::remove_dir_all(&scratch_root).unwrap(); + fs::remove_dir_all(&real_home).unwrap(); + } + + /// Secondary, defense-in-depth condition: even a `TermTreeDev`-named + /// directory is refused when it lives *under* the scratch root, so a + /// future edit cannot accidentally reintroduce the pre-inversion + /// TermTree/TermTreeDev collision this module's history warns about. + #[test] + fn refuses_a_termtreedev_directory_even_under_the_scratch_root() { + let home = temp_home(); + let dev_directory = home + .join("Library") + .join("Application Support") + .join("DocumentNode") + .join("TermTreeDev"); + let seeder = TermTreeSeeder { + state_directory: dev_directory, + scratch_root: home.clone(), + }; + let error = seeder.seed(5, &repo(), &agent()).unwrap_err(); + assert!(error.0.contains("Refusing")); + fs::remove_dir_all(&home).unwrap(); + } + + #[test] + fn seed_then_restore_round_trips_prior_state() { + let home = temp_home(); + let seeder = TermTreeSeeder::production(&home); + fs::create_dir_all(&seeder.state_directory).unwrap(); + let state_path = seeder.state_directory.join(STATE_FILENAME); + fs::write(&state_path, "{\"nodes\":[{\"id\":\"real-user-state\"}]}") + .unwrap(); + + let plan = seeder.seed(5, &repo(), &agent()).unwrap(); + assert_eq!(plan.method, "production state.json pre-write"); + let seeded = fs::read_to_string(&state_path).unwrap(); + assert!(seeded.contains("resource-benchmark-session-0")); + assert!(seeded.contains(&agent().executable_path)); + + seeder.restore().unwrap(); + let restored = fs::read_to_string(&state_path).unwrap(); + assert!(restored.contains("real-user-state")); + + fs::remove_dir_all(&home).unwrap(); + } + + #[test] + fn seeding_twice_without_restoring_refuses_to_overwrite_the_backup() { + let home = temp_home(); + let seeder = TermTreeSeeder::production(&home); + fs::create_dir_all(&seeder.state_directory).unwrap(); + fs::write( + seeder.state_directory.join(STATE_FILENAME), + "{\"nodes\":[]}", + ) + .unwrap(); + + seeder.seed(3, &repo(), &agent()).unwrap(); + let error = seeder.seed(3, &repo(), &agent()).unwrap_err(); + assert!(error.0.contains("Backup already exists")); + + seeder.restore().unwrap(); + fs::remove_dir_all(&home).unwrap(); + } + + #[test] + fn restore_with_nothing_seeded_is_a_safe_no_op() { + let home = temp_home(); + let seeder = TermTreeSeeder::production(&home); + assert!(seeder.restore().is_ok()); + fs::remove_dir_all(&home).unwrap(); + } + + #[test] + fn seeding_when_no_prior_state_exists_seeds_without_a_backup() { + let home = temp_home(); + let seeder = TermTreeSeeder::production(&home); + seeder.seed(2, &repo(), &agent()).unwrap(); + let state_path = seeder.state_directory.join(STATE_FILENAME); + assert!(state_path.exists()); + let backup_path = seeder.state_directory.join(BACKUP_FILENAME); + assert!(!backup_path.exists()); + seeder.restore().unwrap(); + fs::remove_dir_all(&home).unwrap(); + } + + /// The defect this pins: the seeder once emitted `{"nodes": [...]}`. + /// `AppStateController.loadPersistedState` reads `persisted.tree` + /// (`frontend/src/store/AppStateController.ts:883`), so that document + /// restored zero sessions and the N-session tier measured an empty app + /// while every unit test still passed. + #[test] + fn seeded_document_uses_the_apps_tree_schema_not_a_flat_node_list() { + let document: serde_json::Value = + serde_json::from_str(&fixture_for_n_sessions(5, &repo(), &agent())) + .unwrap(); + assert!( + document + .get("tree") + .is_some_and(serde_json::Value::is_object), + "persisted state must carry a `tree` root object" + ); + assert!( + document.get("nodes").is_none(), + "`nodes` is not a key TermTree ever reads" + ); + } + + #[test] + fn only_the_session_nodes_are_relaunch_eligible() { + const RELAUNCHED: [&str; 3] = ["running", "idle", "waiting"]; + let document: serde_json::Value = + serde_json::from_str(&fixture_for_n_sessions(5, &repo(), &agent())) + .unwrap(); + let root = &document["tree"]; + assert_eq!(root["id"], ROOT_NODE_ID); + assert!( + !RELAUNCHED.contains(&root["status"].as_str().unwrap()), + "the root must not be queued for relaunch" + ); + let children = root["children"].as_array().unwrap(); + assert_eq!(children.len(), 5); + for child in children { + assert!(RELAUNCHED.contains(&child["status"].as_str().unwrap())); + assert!(child["children"].as_array().unwrap().is_empty()); + } + } + + /// A first-run onboarding overlay would change both the cold-start and the + /// memory numbers, so the seeded app must be the one users actually run. + #[test] + fn onboarding_is_marked_seen() { + let document: serde_json::Value = + serde_json::from_str(&fixture_for_n_sessions(1, &repo(), &agent())) + .unwrap(); + assert_eq!(document["onboardingSeen"], serde_json::Value::Bool(true)); + } + + /// Pins the contract against `getPersistedState()`'s return type + /// (`frontend/src/store/AppStateController.ts:941-966`). A field the app + /// expects but the seeder omits falls back to a default that may not be + /// the state we intend to measure. + #[test] + fn seeded_top_level_keys_match_the_apps_persisted_schema() { + const EXPECTED: [&str; 26] = [ + "editorOutlineSplitPercent", + "editorOutlineVisible", + "editorPanelSplitPercent", + "editorRecentFiles", + "fontPreference", + "lastThemeByMode", + "lastUsedCwd", + "layoutType", + "mindmapAnimateConnections", + "mindmapLineStyle", + "mindmapLineWidthPx", + "mindmapMarginParental", + "mindmapMarginSibling", + "minimapVisible", + "onboardingSeen", + "panelCollapsed", + "panelCollapsedPreferenceVersion", + "panelPosition", + "panelSplitPercent", + "scrollbackNodeIds", + "themeKey", + "tree", + "viewportX", + "viewportY", + "viewportZoom", + "splitGroup", + ]; + let document: serde_json::Value = + serde_json::from_str(&fixture_for_n_sessions(1, &repo(), &agent())) + .unwrap(); + let mut seeded: Vec<&str> = document + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + seeded.sort_unstable(); + let mut expected = EXPECTED.to_vec(); + expected.sort_unstable(); + // `splitGroup` is merged in by `main.ts` only when a terminal manager + // exists, so the seeder does not write it. + expected.retain(|key| *key != "splitGroup"); + assert_eq!(seeded, expected); + } +} diff --git a/benchmark/src/settings.rs b/benchmark/src/settings.rs new file mode 100644 index 0000000..b7e0d51 --- /dev/null +++ b/benchmark/src/settings.rs @@ -0,0 +1,103 @@ +//! `RunSettings`: every settle time, repetition count, poll interval, and +//! sample count the harness uses. Serialised into the result file so a +//! published number carries the conditions it was taken under (spec FR-6, +//! FR-11). + +use serde::{Deserialize, Serialize}; + +/// Repetitions for tiers whose per-repetition cost is low enough to afford +/// the spec's literal "at least 20" (fresh-launch, cold start, idle CPU). +/// See design §2.3's FR-12 deviation for why the heavier tiers do not share +/// this count. +pub const CHEAP_TIER_REPETITIONS: u32 = 20; +/// Disclosed lower repetition count for the sustained-use tier (~11 min per +/// repetition makes 20 repetitions ~31 h of exclusive machine time; design +/// §2.3, §10). +pub const SUSTAINED_USE_REPETITIONS: u32 = 5; +/// Disclosed lower repetition count for each N-session tier (~4 min per +/// repetition; design §2.3, §10). +pub const N_SESSION_REPETITIONS: u32 = 8; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct RunSettings { + pub repetitions: TierRepetitions, + pub discard_first: bool, + pub fresh_launch_settle_ms: u64, + pub sustained_use_duration_s: u64, + pub n_session_settle_s: u64, + pub seed_timeout_s: u64, + pub idle_cpu_sample_interval_ms: u64, + pub idle_cpu_sample_count: u32, + pub window_visible_poll_ms: u64, + pub log_tail_poll_ms: u64, + pub helper_drain_timeout_ms: u64, + pub quiesce_window_s: u64, + pub main_window_area_fraction: f64, +} + +/// Per-tier repetition counts (design §2.3's FR-12 deviation): cheap tiers +/// take the full count, expensive tiers take a disclosed lower count. Every +/// count named here is printed in the result file's `settings` block and in +/// the rendered table -- never an undisclosed shortfall (NFR-2). +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct TierRepetitions { + pub fresh_launch: u32, + pub sustained_use: u32, + pub n_session: u32, +} + +impl Default for TierRepetitions { + fn default() -> Self { + Self { + fresh_launch: CHEAP_TIER_REPETITIONS, + sustained_use: SUSTAINED_USE_REPETITIONS, + n_session: N_SESSION_REPETITIONS, + } + } +} + +impl Default for RunSettings { + fn default() -> Self { + Self { + repetitions: TierRepetitions::default(), + discard_first: true, + fresh_launch_settle_ms: 15_000, + sustained_use_duration_s: 600, + n_session_settle_s: 120, + seed_timeout_s: 300, + idle_cpu_sample_interval_ms: 1_000, + idle_cpu_sample_count: 30, + window_visible_poll_ms: 20, + log_tail_poll_ms: 20, + helper_drain_timeout_ms: 10_000, + quiesce_window_s: 5, + main_window_area_fraction: 0.5, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_settings_round_trip_through_json() { + let settings = RunSettings::default(); + let json = serde_json::to_string(&settings).unwrap(); + let parsed: RunSettings = serde_json::from_str(&json).unwrap(); + assert_eq!(settings, parsed); + } + + #[test] + fn tiered_repetitions_are_disclosed_and_unequal() { + // The whole point of the tiered-repetition decision (design §2.3) is + // that the heavy tiers take fewer, disclosed repetitions rather than + // silently falling short of a uniform count. + let repetitions = TierRepetitions::default(); + assert_eq!(repetitions.fresh_launch, CHEAP_TIER_REPETITIONS); + assert!(repetitions.sustained_use < CHEAP_TIER_REPETITIONS); + assert!(repetitions.n_session < CHEAP_TIER_REPETITIONS); + } +} diff --git a/benchmark/src/stats.rs b/benchmark/src/stats.rs new file mode 100644 index 0000000..df7c75d --- /dev/null +++ b/benchmark/src/stats.rs @@ -0,0 +1,387 @@ +//! Median, quartiles, and IQR (spec FR-12: "median and interquartile +//! range -- never a single number or a percentage-delta claim"). +//! +//! **There is deliberately no percentage-delta function anywhere in this +//! module or this crate.** FR-12 forbids "up to X% faster" phrasing in the +//! published output; the cheapest way to guarantee that is for the +//! capability not to exist in the codebase (design §5.8). + +use crate::result::{Aggregate, Sample}; +use std::collections::BTreeMap; + +/// Linear-interpolation quartiles (the common definition; matches e.g. +/// NumPy's default `'linear'` method), computed over a **sorted** copy of +/// `values`. Returns `None` for an empty slice. +pub fn median(values: &[f64]) -> Option { + percentile(values, 0.5) +} + +pub fn q1(values: &[f64]) -> Option { + percentile(values, 0.25) +} + +pub fn q3(values: &[f64]) -> Option { + percentile(values, 0.75) +} + +pub fn iqr(values: &[f64]) -> Option { + Some(q3(values)? - q1(values)?) +} + +fn percentile(values: &[f64], fraction: f64) -> Option { + if values.is_empty() { + return None; + } + let mut sorted: Vec = values.to_vec(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); + if sorted.len() == 1 { + return Some(sorted[0]); + } + let rank = fraction * (sorted.len() - 1) as f64; + let lower = rank.floor() as usize; + let upper = rank.ceil() as usize; + if lower == upper { + return Some(sorted[lower]); + } + let weight = rank - lower as f64; + Some(sorted[lower] * (1.0 - weight) + sorted[upper] * weight) +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Summary { + pub median: f64, + pub q1: f64, + pub q3: f64, + pub iqr: f64, + pub n: u32, +} + +pub fn summarize(values: &[f64]) -> Option { + Some(Summary { + median: median(values)?, + q1: q1(values)?, + q3: q3(values)?, + iqr: iqr(values)?, + n: values.len() as u32, + }) +} + +/// The named metrics this crate publishes per (subject, tier) sample -- +/// design §5.10's column list, plus the N-session split. Extracted here, +/// once, so [`compute_aggregates`] and any future consumer agree on which +/// field of a [`Sample`] each metric name reads. +fn metrics_of(sample: &Sample) -> Vec<(&'static str, f64)> { + let mut metrics = Vec::new(); + if let Some(memory) = &sample.memory { + metrics.push(( + "memPhysFootprintBytes", + memory.mem_phys_footprint_bytes as f64, + )); + metrics.push(( + "memPhysFootprintProcessSumBytes", + memory.mem_phys_footprint_process_sum_bytes as f64, + )); + metrics.push(( + "sharedPageDoubleCountBytes", + memory.shared_page_double_count_bytes as f64, + )); + metrics.push(("freeRamDeltaBytes", memory.free_ram_delta_bytes as f64)); + metrics.push(( + "hostMemoryUsedDeltaBytes", + memory.host_memory_used_delta_bytes as f64, + )); + metrics.push(( + "orchestratorAttributableBytes", + memory.orchestrator_attributable_bytes as f64, + )); + metrics.push(( + "agentCliAttributableBytes", + memory.agent_cli_attributable_bytes as f64, + )); + } + if let Some(cold_start) = &sample.cold_start { + if let Some(value) = cold_start.main_window_visible_ms { + metrics.push(("mainWindowVisibleMs", value as f64)); + } + if let Some(value) = cold_start.app_window_ready_ms { + metrics.push(("appWindowReadyMs", value as f64)); + } + if let Some(value) = cold_start.splash_close_ms { + metrics.push(("splashCloseMs", value as f64)); + } + } + if let Some(idle_cpu) = &sample.idle_cpu { + metrics.push(( + "idleCpuPercentOfOneCore", + idle_cpu.idle_cpu_percent_of_one_core_median, + )); + } + metrics +} + +/// Aggregates every valid, non-calibration sample into one [`Aggregate`] +/// per (subject, tier, metric): median/q1/q3/iqr over that group's values, +/// plus `n` and a `discardedCount`/`discardedReasons` histogram scoped to +/// the same (subject, tier) (spec FR-12, design §5.8). +/// +/// Recomputed from the full sample list on every write, so an interrupted +/// run still has valid partial aggregates (design §7). Pure -- takes the +/// sample list already collected, never touches the filesystem itself. +pub fn compute_aggregates(samples: &[Sample]) -> Vec { + let mut discarded_count: BTreeMap<(String, String), u32> = BTreeMap::new(); + let mut discarded_reasons: BTreeMap<(String, String), BTreeMap> = + BTreeMap::new(); + for sample in samples { + if !sample.is_valid { + let key = (sample.subject_id.clone(), sample.tier.clone()); + *discarded_count.entry(key.clone()).or_insert(0) += 1; + if let Some(reason) = &sample.invalid_reason { + *discarded_reasons + .entry(key) + .or_default() + .entry(reason.clone()) + .or_insert(0) += 1; + } + } + } + + let mut groups: BTreeMap<(String, String, &'static str), Vec> = + BTreeMap::new(); + for sample in samples { + if !sample.is_valid || sample.is_calibration { + continue; + } + for (metric, value) in metrics_of(sample) { + groups + .entry((sample.subject_id.clone(), sample.tier.clone(), metric)) + .or_default() + .push(value); + } + } + + groups + .into_iter() + .filter_map(|((subject_id, tier, metric), values)| { + let summary = summarize(&values)?; + let key = (subject_id.clone(), tier.clone()); + Some(Aggregate { + subject_id, + tier, + metric: metric.to_string(), + median: summary.median, + q1: summary.q1, + q3: summary.q3, + iqr: summary.iqr, + n: summary.n, + discarded_count: discarded_count.get(&key).copied().unwrap_or(0), + discarded_reasons: discarded_reasons + .get(&key) + .cloned() + .unwrap_or_default(), + derivation: "measured".to_string(), + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn median_of_odd_length() { + assert_eq!(median(&[1.0, 3.0, 2.0]), Some(2.0)); + } + + #[test] + fn median_of_even_length_interpolates() { + assert_eq!(median(&[1.0, 2.0, 3.0, 4.0]), Some(2.5)); + } + + #[test] + fn median_of_single_element() { + assert_eq!(median(&[42.0]), Some(42.0)); + } + + #[test] + fn median_of_empty_is_none() { + assert_eq!(median(&[]), None); + assert_eq!(summarize(&[]), None); + } + + #[test] + fn quartiles_and_iqr_on_a_known_set() { + let values = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]; + assert_eq!(median(&values), Some(5.0)); + assert_eq!(q1(&values), Some(3.0)); + assert_eq!(q3(&values), Some(7.0)); + assert_eq!(iqr(&values), Some(4.0)); + } + + #[test] + fn summarize_reports_n() { + let summary = summarize(&[1.0, 2.0, 3.0]).unwrap(); + assert_eq!(summary.n, 3); + assert_eq!(summary.median, 2.0); + } + + #[test] + fn unsorted_input_is_handled() { + assert_eq!(median(&[9.0, 1.0, 5.0]), Some(5.0)); + } + + // --- compute_aggregates -------------------------------------------- + + fn memory_record( + mem_phys_footprint_bytes: u64, + ) -> crate::result::MemoryRecord { + crate::result::MemoryRecord { + mem_phys_footprint_bytes, + mem_phys_footprint_process_sum_bytes: mem_phys_footprint_bytes, + shared_page_double_count_bytes: 0, + cross_partition_shared_bytes: 0, + mem_rss_bytes: 0, + mem_rss_method: "naive-per-process-sum".into(), + orchestrator_attributable_bytes: mem_phys_footprint_bytes, + agent_cli_attributable_bytes: 0, + core_process_bytes: None, + render_helper_bytes: None, + free_ram_before_bytes: 0, + free_ram_after_bytes: 0, + free_ram_delta_bytes: 0, + free_ram_delta_sign: "positive-means-consumed".into(), + host_memory_used_before_bytes: 0, + host_memory_used_after_bytes: 0, + host_memory_used_delta_bytes: 0, + compressor_occupied_delta_bytes: 0, + swapouts_delta: 0, + orchestrator_free_ram_delta_bytes: None, + agent_cli_free_ram_delta_bytes: None, + free_ram_split_derivation: None, + } + } + + fn sample( + subject_id: &str, + tier: &str, + repetition: u32, + is_calibration: bool, + is_valid: bool, + invalid_reason: Option<&str>, + mem_phys_footprint_bytes: Option, + ) -> Sample { + Sample { + sample_id: format!("{subject_id}/{tier}/{repetition:03}"), + subject_id: subject_id.to_string(), + tier: tier.to_string(), + session_count: 0, + repetition, + is_calibration, + sampled_at: "2026-08-25T02:31:07Z".into(), + is_valid, + invalid_reason: invalid_reason.map(str::to_string), + attribution: None, + memory: mem_phys_footprint_bytes.map(memory_record), + cold_start: None, + idle_cpu: None, + quiesce: None, + warm_helper_count: None, + helper_kill_count: None, + } + } + + #[test] + fn aggregates_only_valid_non_calibration_samples() { + let samples = vec![ + sample( + "termtree", + "fresh-launch", + 0, + true, + false, + Some("calibration-discard"), + Some(999), + ), + sample("termtree", "fresh-launch", 1, false, true, None, Some(100)), + sample("termtree", "fresh-launch", 2, false, true, None, Some(200)), + sample( + "termtree", + "fresh-launch", + 3, + false, + false, + Some("warm-webview"), + Some(500), + ), + ]; + let aggregates = compute_aggregates(&samples); + let footprint_row = aggregates + .iter() + .find(|a| a.metric == "memPhysFootprintBytes") + .unwrap(); + assert_eq!(footprint_row.n, 2); + assert_eq!(footprint_row.median, 150.0); + assert_eq!(footprint_row.discarded_count, 2); + assert_eq!( + footprint_row.discarded_reasons.get("calibration-discard"), + Some(&1) + ); + assert_eq!( + footprint_row.discarded_reasons.get("warm-webview"), + Some(&1) + ); + } + + #[test] + fn aggregates_are_scoped_per_subject_and_tier() { + let samples = vec![ + sample("termtree", "fresh-launch", 1, false, true, None, Some(100)), + sample( + "collaborator", + "fresh-launch", + 1, + false, + true, + None, + Some(9000), + ), + ]; + let aggregates = compute_aggregates(&samples); + let termtree_row = aggregates + .iter() + .find(|a| { + a.subject_id == "termtree" && a.metric == "memPhysFootprintBytes" + }) + .unwrap(); + let collaborator_row = aggregates + .iter() + .find(|a| { + a.subject_id == "collaborator" && a.metric == "memPhysFootprintBytes" + }) + .unwrap(); + assert_eq!(termtree_row.median, 100.0); + assert_eq!(collaborator_row.median, 9000.0); + } + + #[test] + fn a_sample_with_no_memory_record_contributes_no_memory_metric() { + let samples = vec![sample( + "termtree", + "fresh-launch", + 1, + false, + true, + None, + None, + )]; + let aggregates = compute_aggregates(&samples); + assert!(aggregates + .iter() + .all(|a| a.metric != "memPhysFootprintBytes")); + } + + #[test] + fn empty_sample_list_produces_no_aggregates() { + assert!(compute_aggregates(&[]).is_empty()); + } +} diff --git a/benchmark/src/subject.rs b/benchmark/src/subject.rs new file mode 100644 index 0000000..da073aa --- /dev/null +++ b/benchmark/src/subject.rs @@ -0,0 +1,264 @@ +//! The subject registry (spec FR-1): a fixed `const` table, not +//! configuration a re-runner can quietly change, so it *is* the published +//! subject set. + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RuntimeFamily { + WebKitTauri, + ChromiumElectron, + GpuiNative, +} + +impl RuntimeFamily { + pub fn as_str(&self) -> &'static str { + match self { + Self::WebKitTauri => "webkit-tauri", + Self::ChromiumElectron => "chromium-electron", + Self::GpuiNative => "gpui-native", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SeederId { + TermTreeStateJson, + Collaborator, + CodeNomad, + Diri, +} + +#[derive(Debug, Clone, Copy)] +pub struct CompanionProcess { + pub executable_name: &'static str, + pub kill_between_repetitions: bool, +} + +#[derive(Debug, Clone, Copy)] +pub struct SubjectSpec { + pub id: &'static str, + pub display_name: &'static str, + pub runtime_family: RuntimeFamily, + pub bundle_identifier: &'static str, + pub bundle_path: &'static str, + pub launch_services_name: &'static str, + pub helper_bundle_ids: &'static [&'static str], + pub main_executable_name: &'static str, + pub companion_processes: &'static [CompanionProcess], + pub seeder: SeederId, + pub optional: bool, + pub expected_version: &'static str, + /// Whether this subject's seeder format (spec item 4) has been checked + /// against a real install of the app -- not just unit-tested against the + /// harness's own assumptions about the format. `true` only for TermTree, + /// whose seeded `state.json` shape is pinned against the app's own + /// `AppStateController` source (`seeding/termtree.rs`'s tests). The other + /// three seeders' formats have never been run against a real install; + /// see each `seeding/*.rs` module doc for what is unverified and why. + pub seed_format_verified: bool, +} + +/// The webkit-tauri family's helper-process bundle IDs, shared by TermTree +/// (the only WebKitTauri subject in this registry). +const WEBKIT_HELPER_BUNDLE_IDS: &[&str] = &[ + "com.apple.WebKit.Networking", + "com.apple.WebKit.GPU", + "com.apple.WebKit.WebContent", +]; + +pub const SUBJECTS: &[SubjectSpec] = &[ + SubjectSpec { + id: "termtree", + display_name: "TermTree", + runtime_family: RuntimeFamily::WebKitTauri, + bundle_identifier: "com.termtree.desktop", + bundle_path: "/Applications/TermTree.app", + launch_services_name: "TermTree", + helper_bundle_ids: WEBKIT_HELPER_BUNDLE_IDS, + main_executable_name: "termtree", + companion_processes: &[], + seeder: SeederId::TermTreeStateJson, + optional: false, + expected_version: "1.0.0", + seed_format_verified: true, + }, + SubjectSpec { + id: "codenomad-electron", + display_name: "CodeNomad (Electron)", + runtime_family: RuntimeFamily::ChromiumElectron, + bundle_identifier: "com.codenomad.desktop.electron", + bundle_path: "/Applications/CodeNomad.app", + launch_services_name: "CodeNomad", + helper_bundle_ids: &[], + main_executable_name: "CodeNomad", + companion_processes: &[CompanionProcess { + executable_name: "codenomad-server", + kill_between_repetitions: true, + }], + seeder: SeederId::CodeNomad, + optional: false, + expected_version: "0.18.0", + seed_format_verified: false, + }, + SubjectSpec { + id: "codenomad-tauri", + display_name: "CodeNomad (Tauri)", + runtime_family: RuntimeFamily::WebKitTauri, + bundle_identifier: "com.codenomad.desktop.tauri", + bundle_path: "/Applications/CodeNomad Tauri.app", + launch_services_name: "CodeNomad Tauri", + helper_bundle_ids: WEBKIT_HELPER_BUNDLE_IDS, + main_executable_name: "codenomad-tauri", + companion_processes: &[CompanionProcess { + executable_name: "codenomad-server", + kill_between_repetitions: true, + }], + seeder: SeederId::CodeNomad, + optional: false, + expected_version: "0.18.0", + seed_format_verified: false, + }, + SubjectSpec { + id: "collaborator", + display_name: "Collaborator", + runtime_family: RuntimeFamily::ChromiumElectron, + bundle_identifier: "com.collaborator.desktop", + bundle_path: "/Applications/Collaborator.app", + launch_services_name: "Collaborator", + helper_bundle_ids: &[], + main_executable_name: "Collaborator", + companion_processes: &[], + seeder: SeederId::Collaborator, + optional: false, + expected_version: "0.8.4", + seed_format_verified: false, + }, + SubjectSpec { + id: "diri", + display_name: "diri", + runtime_family: RuntimeFamily::GpuiNative, + bundle_identifier: "com.diri.desktop", + bundle_path: "/Applications/diri.app", + launch_services_name: "diri", + helper_bundle_ids: &[], + main_executable_name: "diri", + companion_processes: &[CompanionProcess { + executable_name: "dirijord-rs", + kill_between_repetitions: true, + }], + seeder: SeederId::Diri, + optional: true, + expected_version: "0.5.1", + seed_format_verified: false, + }, +]; + +/// Excluded subjects and hold-backs (spec FR-1, §8), kept as data next to +/// the registry so `render.rs` can copy the reasons verbatim into the +/// published methodology rather than let them drift from prose. +pub struct Exclusion { + pub name: &'static str, + pub reason: &'static str, +} + +pub const EXCLUSIONS: &[Exclusion] = &[ + Exclusion { + name: "Conductor", + reason: "Terms of Service prohibit use for competitive analysis or a \ + competing product (legal).", + }, + Exclusion { + name: "Crystal", + reason: "Project is dead, renamed Nimbalyst.", + }, + Exclusion { + name: "Constellagent", + reason: "Zero releases, zero tags, no licence file.", + }, + Exclusion { + name: "Vibe Kanban", + reason: "Dual web/desktop mode makes the measurement target ambiguous.", + }, + Exclusion { + name: "Claude Squad", + reason: "Go TUI with no GUI, no webview, no window to measure \ + (category error).", + }, +]; + +pub const HOLD_BACKS: &[Exclusion] = &[ + Exclusion { + name: "Nimbalyst", + reason: "Documented hold-back; may be promoted to a measured subject \ + later.", + }, + Exclusion { + name: "Maestri", + reason: "Documented hold-back; EULA's anti-reverse-engineering clause \ + restricts it to black-box OS-level measurement if ever added.", + }, +]; + +pub fn find(id: &str) -> Option<&'static SubjectSpec> { + SUBJECTS.iter().find(|s| s.id == id) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn registry_includes_the_minimum_required_subjects() { + for id in [ + "termtree", + "codenomad-electron", + "codenomad-tauri", + "collaborator", + ] { + assert!(find(id).is_some(), "missing required subject {id}"); + assert!(!find(id).unwrap().optional); + } + } + + #[test] + fn diri_is_the_only_optional_subject() { + let optional: Vec<_> = SUBJECTS + .iter() + .filter(|s| s.optional) + .map(|s| s.id) + .collect(); + assert_eq!(optional, vec!["diri"]); + } + + #[test] + fn every_subject_has_a_pinned_non_empty_version() { + for subject in SUBJECTS { + assert!(!subject.expected_version.is_empty(), "{}", subject.id); + } + } + + #[test] + fn codenomad_electron_and_tauri_share_the_pinned_version() { + let electron = find("codenomad-electron").unwrap(); + let tauri = find("codenomad-tauri").unwrap(); + assert_eq!(electron.expected_version, tauri.expected_version); + } + + #[test] + fn only_termtrees_seed_format_is_verified_against_a_real_install() { + let verified: Vec<_> = SUBJECTS + .iter() + .filter(|s| s.seed_format_verified) + .map(|s| s.id) + .collect(); + assert_eq!(verified, vec!["termtree"]); + } + + #[test] + fn exclusions_and_hold_backs_are_named_with_reasons() { + assert_eq!(EXCLUSIONS.len(), 5); + assert_eq!(HOLD_BACKS.len(), 2); + for e in EXCLUSIONS.iter().chain(HOLD_BACKS) { + assert!(!e.reason.is_empty()); + } + } +} diff --git a/benchmark/src/tier.rs b/benchmark/src/tier.rs new file mode 100644 index 0000000..079abc3 --- /dev/null +++ b/benchmark/src/tier.rs @@ -0,0 +1,105 @@ +//! The measurement tiers (spec §3's terminology table; FR-6, FR-7, FR-8). + +use crate::settings::RunSettings; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Tier { + FreshLaunch, + SustainedUse, + NSession(u32), +} + +impl Tier { + pub fn as_str(&self) -> String { + match self { + Self::FreshLaunch => "fresh-launch".to_string(), + Self::SustainedUse => "sustained-use".to_string(), + Self::NSession(n) => format!("n-session-{n}"), + } + } + + pub fn session_count(&self) -> u32 { + match self { + Self::FreshLaunch => 0, + // The sustained-use tier seeds the same 5-session configuration as + // the N=5 tier (design §5.6.5). + Self::SustainedUse => 5, + Self::NSession(n) => *n, + } + } + + /// The disclosed, tiered repetition count for this tier (design §2.3's + /// FR-12 deviation) -- cheap tiers get the full count, expensive tiers + /// get fewer, and every count is printed in `settings` and the rendered + /// table. + pub fn repetitions(&self, settings: &RunSettings) -> u32 { + match self { + Self::FreshLaunch => settings.repetitions.fresh_launch, + Self::SustainedUse => settings.repetitions.sustained_use, + Self::NSession(_) => settings.repetitions.n_session, + } + } + + pub fn parse(text: &str) -> Option { + match text { + "fresh-launch" => Some(Self::FreshLaunch), + "sustained-use" => Some(Self::SustainedUse), + other => other + .strip_prefix("n-session-") + .and_then(|n| n.parse::().ok()) + .map(Self::NSession), + } + } +} + +/// The default tier sweep for `just benchmark run` with no `--tiers` +/// override -- "a full subject/tier sweep" (spec FR-16). +pub const DEFAULT_TIERS: &[Tier] = &[ + Tier::FreshLaunch, + Tier::NSession(5), + Tier::NSession(10), + Tier::NSession(20), + Tier::SustainedUse, +]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tier_names_round_trip_through_parse() { + for tier in DEFAULT_TIERS { + assert_eq!(Tier::parse(&tier.as_str()), Some(*tier)); + } + } + + #[test] + fn n_session_tiers_carry_their_session_count() { + assert_eq!(Tier::NSession(20).session_count(), 20); + assert_eq!(Tier::FreshLaunch.session_count(), 0); + assert_eq!(Tier::SustainedUse.session_count(), 5); + } + + #[test] + fn repetitions_use_the_tiered_counts() { + let settings = RunSettings::default(); + assert_eq!( + Tier::FreshLaunch.repetitions(&settings), + settings.repetitions.fresh_launch + ); + assert_eq!( + Tier::NSession(20).repetitions(&settings), + settings.repetitions.n_session + ); + assert_eq!( + Tier::SustainedUse.repetitions(&settings), + settings.repetitions.sustained_use + ); + } + + #[test] + fn parse_rejects_garbage() { + assert_eq!(Tier::parse("not-a-tier"), None); + assert_eq!(Tier::parse("n-session-abc"), None); + } +} diff --git a/benchmark/src/window_probe.rs b/benchmark/src/window_probe.rs new file mode 100644 index 0000000..ab07ea9 --- /dev/null +++ b/benchmark/src/window_probe.rs @@ -0,0 +1,129 @@ +//! `CGWindowListCopyWindowInfo` -- the one root-free way to observe "a +//! window of this PID is on screen" (spec FR-4, design §5.4). This is the +//! crate's only platform-FFI module. +//! +//! Only `kCGWindowOwnerPID`, `kCGWindowLayer`, and `kCGWindowBounds` are +//! read. `kCGWindowName` is deliberately never read: without Screen +//! Recording permission macOS redacts window titles, but owner PID, layer, +//! and bounds are returned regardless -- so the probe never needs a +//! permission this crate's NFR-4 (no required root/extra permissions) would +//! otherwise be at odds with. + +use core_foundation::array::CFArray; +use core_foundation::base::{CFType, TCFType}; +use core_foundation::dictionary::CFDictionary; +use core_foundation::number::CFNumber; +use core_foundation::string::{CFString, CFStringRef}; +use core_graphics::window::{ + copy_window_info, kCGNullWindowID, kCGWindowBounds, kCGWindowLayer, + kCGWindowListExcludeDesktopElements, kCGWindowListOptionOnScreenOnly, + kCGWindowOwnerPID, +}; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct OnScreenWindow { + pub owner_pid: u32, + pub layer: i64, + /// Width * height, in points, as read from `kCGWindowBounds`. + pub area_pt: f64, +} + +/// Queries every on-screen, non-desktop-chrome window and returns the ones +/// owned by `pid`. There is no fixture for this call -- it talks to the +/// live window server, so it is exercised through `doctor` and the smoke +/// sweep rather than a unit test (design §11: "not unit-tested, by design"). +pub fn on_screen_windows_owned_by(pid: u32) -> Vec { + let option = + kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements; + let Some(array) = copy_window_info(option, kCGNullWindowID) else { + return Vec::new(); + }; + // `copy_window_info` returns an untyped `CFArray<*const c_void>`; retype + // it (via the get rule, which retains rather than consumes) to iterate + // strongly-typed dictionary entries instead of hand-walking void + // pointers. + let typed: CFArray> = + unsafe { CFArray::wrap_under_get_rule(array.as_concrete_TypeRef()) }; + typed + .iter() + .filter_map(|dict| parse_window_dict(&dict)) + .filter(|w| w.owner_pid == pid) + .collect() +} + +fn parse_window_dict( + dict: &CFDictionary, +) -> Option { + // Reading an `extern "C"` static needs `unsafe`; the subsequent + // `dict.find` calls do not, so the unsafe block is scoped tightly to the + // static reads alone. + let (owner_pid_key, layer_key, bounds_key) = + unsafe { (kCGWindowOwnerPID, kCGWindowLayer, kCGWindowBounds) }; + + let owner_pid = dict_number_by_static_key(dict, owner_pid_key)? as u32; + let layer = dict_number_by_static_key(dict, layer_key).unwrap_or(0.0) as i64; + // `downcast` only accepts a `ConcreteCFType`, which `CFDictionary` is + // solely at its default (untyped) parameterization -- so downcast to + // that first, then retype (get rule: retains, does not consume) to the + // strongly-typed dictionary the rest of this module works with. + let bounds_dict: Option> = dict + .find(bounds_key) + .and_then(|value| value.downcast::()) + .map(|untyped| unsafe { + CFDictionary::::wrap_under_get_rule( + untyped.as_concrete_TypeRef(), + ) + }); + let area_pt = bounds_dict.map(bounds_area_pt).unwrap_or(0.0); + + Some(OnScreenWindow { + owner_pid, + layer, + area_pt, + }) +} + +/// `kCGWindowBounds`'s value is itself a dictionary with plain string keys +/// `"X"`/`"Y"`/`"Width"`/`"Height"` (no `CFStringRef` constant is exported +/// for these by Apple; they are literal dictionary keys from +/// `CGRectCreateDictionaryRepresentation`), so the keys are constructed +/// locally and kept alive for the lookup rather than converted to a raw +/// `CFStringRef` and dropped -- doing the latter would leave a dangling +/// pointer. +fn bounds_area_pt(bounds: CFDictionary) -> f64 { + let width_key = CFString::from_static_string("Width"); + let height_key = CFString::from_static_string("Height"); + let width = dict_number_by_owned_key(&bounds, &width_key).unwrap_or(0.0); + let height = dict_number_by_owned_key(&bounds, &height_key).unwrap_or(0.0); + width * height +} + +fn dict_number_by_static_key( + dict: &CFDictionary, + key: CFStringRef, +) -> Option { + let value = dict.find(key)?; + value.downcast::()?.to_f64() +} + +fn dict_number_by_owned_key( + dict: &CFDictionary, + key: &CFString, +) -> Option { + let value = dict.find(key)?; + value.downcast::()?.to_f64() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn on_screen_windows_owned_by_an_impossible_pid_is_empty() { + // PID 0 owns no windows on a live system; this is a smoke check that + // the FFI call itself does not crash, not a substitute for a fixture -- + // there is no fixture for live window-server output (design §11). + let windows = on_screen_windows_owned_by(0); + assert!(windows.is_empty()); + } +} diff --git a/benchmark/workload/sustained-session.sh b/benchmark/workload/sustained-session.sh new file mode 100755 index 0000000..c289ac9 --- /dev/null +++ b/benchmark/workload/sustained-session.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# The fixed sustained-use per-session workload (spec FR-7, design §5.6.5). +# +# This script is byte-identical for every subject at every session: the +# sustained-use tier seeds the same 5-session configuration as the N=5 +# tier, and each session's start command runs this loop against the +# seeded repository (SEEDED_REPO_PATH) instead of an idle shell. +# +# Real prompts are deliberately NOT sent to the agent CLI: doing so would +# make this tier non-deterministic (model/network variance) and +# unrepeatable for a third party without API credentials. This tier +# exercises the orchestrator's terminal-output and rendering paths, not +# agent inference -- see the harness README's Limitations section for why +# that distinction matters for what this tier can and cannot show. +# +# Usage: sustained-session.sh +set -euo pipefail + +repo_path="${1:?usage: sustained-session.sh }" +duration_s="${2:?usage: sustained-session.sh }" + +cd "$repo_path" +end=$(( $(date +%s) + duration_s )) + +while [ "$(date +%s)" -lt "$end" ]; do + git status --short > /dev/null 2>&1 || true + git log --oneline -20 > /dev/null 2>&1 || true + # Bounded so a large repo cannot make one iteration run long enough to + # blow past the fixed duration. + rg --max-count 50 --max-filesize 1M "TODO" . > /dev/null 2>&1 || true + find . -maxdepth 2 -type f -name "*.md" -print -quit \ + | xargs -I{} cat {} > /dev/null 2>&1 || true + sleep 5 +done