dvnold · Oct 7, 2026 · Oct 7, 2026 · Oct 7, 2026 · Oct 7, 2026
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -4,6 +4,7 @@
 
 ## Verification
 
-- [ ] `npm test`
-- [ ] `npm run lint`
-- [ ] Real `fm` smoke test on macOS 27+, if benchmark behavior changed
+- [ ] `npm run check` (lint, tests, package contents)
+- [ ] `actionlint`, if workflows changed
+- [ ] Real `fm` smoke test on macOS 27+ (`fm-bench doctor`, `fm-bench --runs 3`), if benchmark behavior changed
+- [ ] `CHANGELOG.md` and docs updated for user-visible changes
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -10,25 +10,27 @@ concurrency:
   group: ci-${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
-env:
-  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+permissions:
+  contents: read
 
 jobs:
   test:
-    # macOS minutes cost 10x, so the matrix brackets the supported range
-    # (engines: node >=20) instead of testing every LTS in between.
+    # macOS minutes cost 10x, so the matrix covers each Node release line that
+    # is still maintained (engines: node >=22) rather than every minor.
     strategy:
       fail-fast: false
       matrix:
-        node: [20, 24]
+        node: [22, 24, 26]
     runs-on: macos-15
+    timeout-minutes: 15
     name: node ${{ matrix.node }}
     steps:
-      - uses: actions/checkout@v7
-      - uses: actions/setup-node@v7
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
           node-version: ${{ matrix.node }}
-          cache: npm
       - run: npm ci
       - name: Lint, test, and verify package contents
         run: npm run check
@@ -43,8 +45,10 @@ jobs:
           node bin/fm-bench.js --help > /dev/null
           node bin/fm-bench.js legend --json | node -e "const fs=require('fs');const entries=JSON.parse(fs.readFileSync(0,'utf8'));if(!Array.isArray(entries)||entries.length<10)throw new Error('legend --json output looks wrong')"
           node bin/fm-bench.js doctor
-          node bin/fm-bench.js models --format json | node -e "const fs=require('fs');const models=JSON.parse(fs.readFileSync(0,'utf8'));if(!Array.isArray(models)||models.length<1)throw new Error('models --json output looks wrong')"
-          node bin/fm-bench.js --profile quick --runs 1 --no-progress --json | node -e "const fs=require('fs');const report=JSON.parse(fs.readFileSync(0,'utf8'));if(report.version!==require('./package.json').version)throw new Error('report version mismatch');if(!report.results.some(r=>r.ok&&r.firstTokenMs>0))throw new Error('no measured result')"
+          node bin/fm-bench.js models --format json | node -e "const fs=require('fs');const models=JSON.parse(fs.readFileSync(0,'utf8'));if(!Array.isArray(models)||models[0]?.identity!=='AFM 3 Core Advanced')throw new Error('models --json output looks wrong')"
+          node bin/fm-bench.js --profile quick --runs 2 --no-progress --json | node -e "const fs=require('fs');const report=JSON.parse(fs.readFileSync(0,'utf8'));if(report.version!==require('./package.json').version)throw new Error('report version mismatch');if(!report.results.some(r=>r.ok&&r.firstTokenMs>0))throw new Error('no measured result');if(!report.tokenCounter?.calibrated)throw new Error('token counter not calibrated')"
+          FAKE_FM_SCENARIO=legacy-available node bin/fm-bench.js models
+          FAKE_FM_SCENARIO=legacy-available node bin/fm-bench.js --profile quick --no-progress --ci
       - name: macOS version gate without an explicit fm binary
         run: |
           major="$(sw_vers -productVersion | cut -d. -f1)"
@@ -64,3 +68,17 @@ jobs:
           esac
       - name: Publish dry run
         run: npm run publish:dry-run
+
+  # Single required status for branch protection, independent of the matrix.
+  ci:
+    if: always()
+    needs: test
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Require every matrix job to pass
+        run: |
+          if [ "${{ needs.test.result }}" != "success" ]; then
+            echo "test matrix result: ${{ needs.test.result }}"
+            exit 1
+          fi
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -7,96 +7,123 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: "Existing tag to publish, for example v0.5.3"
+        description: "Existing tag to publish, for example v0.8.0"
         required: true
 
-env:
-  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+concurrency:
+  group: release-${{ github.event.inputs.tag || github.ref_name }}
+  cancel-in-progress: false
 
 permissions:
-  contents: write
-  id-token: write
+  contents: read
 
 jobs:
   publish:
     runs-on: macos-15
+    timeout-minutes: 20
+    permissions:
+      # id-token: npm trusted publishing (OIDC) and provenance attestations.
+      contents: write
+      id-token: write
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           fetch-depth: 0
           ref: ${{ github.event.inputs.tag || github.ref }}
+          persist-credentials: false
 
-      - uses: actions/setup-node@v7
+      # Node 24 ships npm 11, which publishes through npm trusted publishing
+      # (OIDC) when the package trusts this workflow, and falls back to
+      # NODE_AUTH_TOKEN otherwise. No dependency cache in a publish job.
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
-          node-version: 22
+          node-version: 24
           registry-url: https://registry.npmjs.org/
-          cache: npm
 
       - run: npm ci
       - name: Lint, test, and verify package contents
         run: npm run check
 
       - name: Resolve release version
         id: version
+        env:
+          INPUT_TAG: ${{ github.event.inputs.tag }}
         run: |
-          if [ -n "${{ github.event.inputs.tag }}" ]; then
-            ver="${{ github.event.inputs.tag }}"
-          else
-            ver="${GITHUB_REF_NAME}"
-          fi
+          ver="${INPUT_TAG:-$GITHUB_REF_NAME}"
           ver="${ver#v}"
           echo "semver=$ver" >> "$GITHUB_OUTPUT"
+          echo "tag=v$ver" >> "$GITHUB_OUTPUT"
           echo "Release version: $ver"
 
       - name: Verify package.json matches tag
+        env:
+          EXPECTED: ${{ steps.version.outputs.semver }}
         run: |
           set -euo pipefail
-          expected="${{ steps.version.outputs.semver }}"
-          actual="$(node --eval "import { readFileSync } from 'node:fs'; console.log(JSON.parse(readFileSync('package.json','utf8')).version)")"
-          if [ "$actual" != "$expected" ]; then
-            echo "package.json version ($actual) does not match tag ($expected)"
+          actual="$(node -p "require('./package.json').version")"
+          if [ "$actual" != "$EXPECTED" ]; then
+            echo "::error::package.json version ($actual) does not match tag ($EXPECTED)"
             exit 1
           fi
 
       - name: Publish to npm
         id: npm
+        continue-on-error: true
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+          VERSION: ${{ steps.version.outputs.semver }}
         run: |
           set -euo pipefail
-          ver="${{ steps.version.outputs.semver }}"
-          if [ -z "${NODE_AUTH_TOKEN:-}" ]; then
-            echo "::warning::NPM_TOKEN is not configured — skipping npm publish. The GitHub release still runs."
-            echo "status=skipped-no-token" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          if npm view "fm-bench@${ver}" version 2>/dev/null | grep -qx "$ver"; then
-            echo "fm-bench@${ver} is already on npm — skipping publish."
+          npm --version
+          if npm view "fm-bench@${VERSION}" version 2>/dev/null | grep -qx "$VERSION"; then
+            echo "fm-bench@${VERSION} is already on npm — skipping publish."
             echo "status=already-published" >> "$GITHUB_OUTPUT"
-          else
-            npm publish --access public --provenance
-            echo "status=published" >> "$GITHUB_OUTPUT"
+            exit 0
           fi
+          npm publish --access public --provenance
+          echo "status=published" >> "$GITHUB_OUTPUT"
 
-      - name: Create GitHub release
+      - name: Create or update GitHub release
         env:
           GH_TOKEN: ${{ github.token }}
-          RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }}
+          TAG: ${{ steps.version.outputs.tag }}
+          VERSION: ${{ steps.version.outputs.semver }}
+          NPM_OUTCOME: ${{ steps.npm.outcome }}
         run: |
           set -euo pipefail
-          tag="$RELEASE_TAG"
-          ver="${tag#v}"
-          notes="$(node scripts/changelog-release-notes.mjs "$ver")"
           notes_file="$(mktemp)"
-          printf '%s' "$notes" >> "$notes_file"
-          printf '\n' >> "$notes_file"
-          if [ "${{ steps.npm.outputs.status }}" = "skipped-no-token" ]; then
-            printf '\nnpm publish status: skipped (NPM_TOKEN not configured in this repository).\n' >> "$notes_file"
+          node scripts/changelog-release-notes.mjs "$VERSION" > "$notes_file"
+          if [ "$NPM_OUTCOME" != "success" ]; then
+            printf '\n> npm publish failed for this tag. See docs/releasing.md to configure trusted publishing, then re-run the Release workflow for %s.\n' "$TAG" >> "$notes_file"
           fi
-          if gh release view "$tag" >/dev/null 2>&1; then
-            echo "GitHub release $tag already exists — updating notes from CHANGELOG.md."
-            gh release edit "$tag" --notes-file "$notes_file" --title "$tag"
+          if gh release view "$TAG" >/dev/null 2>&1; then
+            echo "GitHub release $TAG already exists — updating notes from CHANGELOG.md."
+            gh release edit "$TAG" --notes-file "$notes_file" --title "$TAG"
           else
-            gh release create "$tag" --verify-tag --title "$tag" --notes-file "$notes_file"
+            gh release create "$TAG" --verify-tag --title "$TAG" --notes-file "$notes_file"
           fi
-          rm -f "$notes_file"
+          rm -f "$notes_file"
+
+      - name: Fail the release when npm publish failed
+        if: steps.npm.outcome != 'success'
+        run: |
+          echo "::error::npm publish failed. Configure npm trusted publishing for dvnold/fm-bench (release.yml) or refresh the NPM_TOKEN secret, then re-run this workflow."
+          exit 1
+
+      - name: Verify the published package
+        if: steps.npm.outcome == 'success'
+        env:
+          VERSION: ${{ steps.version.outputs.semver }}
+        run: |
+          set -euo pipefail
+          dir="$(mktemp -d)"
+          cd "$dir"
+          for attempt in 1 2 3 4 5 6; do
+            if npm view "fm-bench@${VERSION}" version >/dev/null 2>&1; then break; fi
+            echo "waiting for the registry (attempt $attempt)"
+            sleep 10
+          done
+          npm init -y >/dev/null
+          npm install "fm-bench@${VERSION}" --no-audit --no-fund
+          test "$(./node_modules/.bin/fm-bench --version)" = "$VERSION"
+          FM_BIN="$GITHUB_WORKSPACE/test/fixtures/fake-fm.mjs" ./node_modules/.bin/fm-bench --profile quick --no-progress --ci
diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml
@@ -12,32 +12,43 @@ concurrency:
   group: version-${{ github.repository }}
   cancel-in-progress: false
 
-env:
-  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
-
 permissions:
+  # actions: dispatch the Release workflow; a tag pushed with GITHUB_TOKEN
+  # does not trigger workflows on its own.
+  actions: write
   contents: write
 
 jobs:
   version:
     runs-on: macos-15
+    timeout-minutes: 15
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           fetch-depth: 0
           ref: main
 
-      - uses: actions/setup-node@v7
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
-          node-version: 22
-          cache: npm
+          node-version: 24
 
       - run: npm ci
       - name: Lint, test, and verify package contents
         run: npm run check
 
-      - run: |
+      - name: Bump, tag, and push
+        env:
+          VERSION_ARG: ${{ inputs.version }}
+        run: |
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          npm version "${{ inputs.version }}" -m "Release %s"
-          git push origin HEAD:main --follow-tags
+          npm version "$VERSION_ARG" -m "Release %s"
+          git push origin HEAD:main --follow-tags
+
+      - name: Start the Release workflow for the new tag
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          tag="v$(node -p "require('./package.json').version")"
+          gh workflow run release.yml --ref main -f tag="$tag"
+          echo "Dispatched Release for $tag"
diff --git a/AGENTS.md b/AGENTS.md
@@ -2,7 +2,7 @@
 
 ## Working on fm-bench
 
-`fm-bench` is a single Node.js CLI (ESM, `bin/fm-bench.js` → `src/`). It has **no runtime npm dependencies**, so `npm install` is effectively a no-op beyond Node itself. Node 20+ is required.
+`fm-bench` is a single Node.js CLI (ESM, `bin/fm-bench.js` → `src/`). It has **no runtime npm dependencies**, so `npm install` is effectively a no-op beyond Node itself. Node 22+ is required.
 
 Standard commands:
 
@@ -11,12 +11,12 @@ Standard commands:
 - Everything: `npm run check` (lint + tests + `npm pack` integrity)
 - Run: `node bin/fm-bench.js <command>` (use the direct path; `npm link` can fail when the global prefix is read-only)
 
-CI runs `npm run check` on Node 20 and 24, plus CLI smoke tests.
+CI runs `npm run check` on Node 22, 24, and 26, plus CLI smoke tests against both `fm` surfaces. Workflows are linted with `actionlint`.
 
 ## Architecture notes
 
 - `src/capabilities.js` — probes `fm --help` / `fm respond --help` and reports what the installed build supports. All subcommand and flag decisions come from here; do not hardcode `fm` subcommand names elsewhere.
-- `src/fm-help.js` — pure parsers for `fm` help/status text (unit tested against captured real output in `test/fixtures/`).
+- `src/fm-help.js` — pure parsers for `fm` help/status text, including the `fm models` list (unit tested against captured real output in `test/fixtures/`).
 - `src/metrics.js` — metric provenance catalogue (`measured` / `proxy` / `derived` / `controlled`) and per-run availability.
 - `src/process.js` — the only place that spawns processes; tracks live children so signals can clean up.
 - `src/bench.js` — orchestration; must not render or print.
@@ -30,6 +30,8 @@ The product benchmarks Apple's `fm` CLI, which only exists on macOS 27+ with App
 FM_BIN=$PWD/test/fixtures/fake-fm.mjs FAKE_FM_SCENARIO=normal node bin/fm-bench.js --profile quick --runs 2
 ```
 
-`FAKE_FM_SCENARIO` selects behaviour: `normal`, `slow`, `malformed`, `fail`, `timeout`, `interrupt`, `hang-help`, `partial`, `short-answer`, `unavailable`, `unavailable-model`, `quota`, `multi-model`, `no-token-count`, `legacy-token-count`, `no-streaming`, `no-model-flag`, `no-models-section`, `token-count-fails`, `help-garbage`, `error-help`.
+`FAKE_FM_SCENARIO` selects behaviour. `normal` mirrors the real macOS 27.2 surface (`models`, `quota-usage`, model identity, one framing token per `count-tokens` count); `legacy-available` mirrors macOS 27.0 (`available`, no quota). Others: `slow`, `malformed`, `fail`, `timeout`, `interrupt`, `hang-help`, `partial`, `short-answer`, `unavailable`, `unavailable-model`, `multi-model` (adds a `pcc` that is unavailable outside Terminal), `license-not-agreed`, `no-token-count`, `legacy-token-count`, `no-streaming`, `no-model-flag`, `no-models-section`, `token-count-fails`, `help-garbage`, `error-help`.
+
+When the real `fm` changes shape, capture its output into `test/fixtures/` (for example `fm --help > test/fixtures/fm-help-macos27.2.txt`), update the fake to match, and add the build to `docs/compatibility.md`.
 
 Commands that need no `fm` at all: `legend`, `validate <report.json>`, `export <report.json>`, `compare <a.json> <b.json>`, `history <dir>`, `--help`, `--version`.