From 544baeb22ed58d47623e8b6e9282bbd573560e78 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 9 Oct 2026 09:23:40 +0200 Subject: [PATCH] ci: validate and test Claude Code mods, and auto-bump the pinned CLI Claude Code loads a hooks module only after its static analysis passes and silently skips one that fails, so a mod that cannot load passed every check here and did nothing on an adopter's machine. - check-claude-mods runs `claude plugin validate --strict` and `claude plugin test` on every plugin whose hooks/hooks.json declares `modules`, with Claude Code pinned in the prek hook env. It is in the manual stage, so commits never fetch the binary; the prek workflow runs it over the whole repo on every PR and push to main. - check-claude-mods-local runs the same check on ordinary commits with the contributor's own Claude Code, skipping when it is absent or older than the pin. - bump-hook-npm-pins.yml moves npm pins in hooks' additional_dependencies (which Dependabot never touches) three times a week, keeping a single draft PR updated in place. The cooldown is 7 days by default and 12 hours for Claude Code, which replaces a bad or vulnerable release within hours. Claude Code is a dev-only tool here: nothing from it ships in a release or a plugin. Generated-by: Claude Opus 5 --- .github/workflows/bump-hook-npm-pins.yml | 235 +++++++++++++++++++++ .github/workflows/pre-commit.yml | 19 +- .pre-commit-config.yaml | 55 +++++ docs/when-to-use-mods.md | 13 +- tools/dev/README.md | 2 + tools/dev/bump-hook-npm-pins.py | 153 ++++++++++++++ tools/dev/check-claude-mods.sh | 103 +++++++++ tools/dev/tests/test_bump_hook_npm_pins.py | 118 +++++++++++ 8 files changed, 695 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/bump-hook-npm-pins.yml create mode 100644 tools/dev/bump-hook-npm-pins.py create mode 100755 tools/dev/check-claude-mods.sh create mode 100644 tools/dev/tests/test_bump_hook_npm_pins.py diff --git a/.github/workflows/bump-hook-npm-pins.yml b/.github/workflows/bump-hook-npm-pins.yml new file mode 100644 index 000000000..c80d36be7 --- /dev/null +++ b/.github/workflows/bump-hook-npm-pins.yml @@ -0,0 +1,235 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Bumps the npm packages pinned in `.pre-commit-config.yaml` hooks' +# `additional_dependencies`, which Dependabot's `pre-commit` ecosystem never +# touches. Today that is the Claude Code CLI behind the `check-claude-mods` +# hook. `tools/dev/bump-hook-npm-pins.py` picks the newest stable release that +# has cleared the package's cooldown — 12 hours for Claude Code, 7 days by +# default. Claude Code is a dev-only tool here: CI runs it to check the mods, +# and nothing from it ships. +# +# Three times a week (Monday, Wednesday, Friday), so a pin trails the newest +# eligible release by a few days at most without a PR every day. Also runnable +# from the Actions tab (*Run workflow*). +# +# ONE PULL REQUEST, UPDATED IN PLACE. Every run targets the same branch, +# `chore/bump-hook-npm-pins`. While a bump PR is open, a later run with a newer +# target replaces that branch's commit with a fresh one on top of current +# `main`, and the same PR now carries the newer bump; bumps never pile up as +# separate PRs. A run whose result matches what the open PR already carries +# changes nothing. +# +# The replacement is built on a throwaway branch first and the PR branch is +# moved onto it only once the signed commit exists, so a failed run never +# leaves the PR pointing at an empty or half-made branch. +# +# SHAPE, as in `bump-dev-version.yml`, whose header explains each choice: +# +# prepare -> signed candidate commit, then the PR branch moved onto it +# open-pr -> open the draft PR, or refresh the open one and return it to draft +# cleanup -> delete the throwaway branch, whatever happened +# +# The PR is a draft because GitHub raises no `pull_request` events for what +# `GITHUB_TOKEN` does, so its checks start only when a maintainer presses +# *Ready for review*. An update puts an already-ready PR back into draft for +# the same reason: the new commit has no checks until it is marked ready +# again. Those checks include the `check-claude-mods` step of `prek`, which +# re-checks every mod against the new pin — the point of the bump. +--- +name: bump hook npm pins + +on: # yamllint disable-line rule:truthy + schedule: + - cron: "17 4 * * 1,3,5" + workflow_dispatch: + +permissions: {} + +env: + BRANCH: chore/bump-hook-npm-pins + +jobs: + prepare: + name: Build the candidate commit + runs-on: ubuntu-slim + timeout-minutes: 10 + permissions: + # Creating the throwaway branch, committing onto it, moving the PR branch. + contents: write + outputs: + updated: ${{ steps.candidate.outputs.updated }} + tmp_branch: ${{ steps.candidate.outputs.tmp_branch }} + bumps: ${{ steps.bump.outputs.bumps }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The commit is created through GitHub's API rather than `git push`, + # so the job never needs the checkout's credentials on disk. + persist-credentials: false + fetch-depth: 1 + + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + enable-cache: true + # A distinct cache slot per job; see `bump-dev-version.yml`. + cache-suffix: "bump-hook-npm-pins" + + - name: Bump the pins + id: bump + run: | + set -euo pipefail + uv run --project tools/dev python tools/dev/bump-hook-npm-pins.py \ + | tee "$RUNNER_TEMP/bumps.txt" + { + echo "bumps<> "$GITHUB_OUTPUT" + + - name: Commit the candidate and move the PR branch onto it + id: candidate + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + HEAD_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + set -euo pipefail + config=.pre-commit-config.yaml + + # Compare against what the bump branch carries, when there is one, + # not against `main`: while a bump PR is open, `main` still has the + # old pins, but the PR may already have exactly these. + if gh api "repos/$REPO/git/ref/heads/$BRANCH" >/dev/null 2>&1; then + gh api -H "Accept: application/vnd.github.raw" \ + "repos/$REPO/contents/$config?ref=$BRANCH" > "$RUNNER_TEMP/current.yaml" + else + git show "HEAD:$config" > "$RUNNER_TEMP/current.yaml" + fi + if cmp -s "$config" "$RUNNER_TEMP/current.yaml"; then + echo "Pins already current on \`main\` or in the open bump PR; nothing to do." \ + >> "$GITHUB_STEP_SUMMARY" + echo "updated=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + { + echo "chore: bump npm pins in prek hooks" + echo + cat "$RUNNER_TEMP/bumps.txt" + echo + echo "Generated by .github/workflows/bump-hook-npm-pins.yml with" + echo "tools/dev/bump-hook-npm-pins.py: the newest stable release past the" + echo "package's cooldown (12 hours for Claude Code, 7 days otherwise)." + } > "$RUNNER_TEMP/bump-message.txt" + + tmp="chore/bump-hook-npm-pins-tmp-$RUN_ID" + gh api "repos/$REPO/git/refs" \ + -f "ref=refs/heads/$tmp" -f "sha=$HEAD_SHA" >/dev/null + # From here on the throwaway branch exists; `cleanup` deletes it. + echo "tmp_branch=$tmp" >> "$GITHUB_OUTPUT" + + # Committed through GitHub's API, not `git commit`: the commit comes + # back signed by GitHub and shows as Verified, with no key material + # in CI. + uv run --project tools/dev python tools/dev/gh-signed-commit.py \ + --branch "$tmp" \ + --expected-head-oid "$HEAD_SHA" \ + --message-file "$RUNNER_TEMP/bump-message.txt" \ + --repo "$REPO" + commit=$(gh api "repos/$REPO/git/ref/heads/$tmp" --jq .object.sha) + + # Point the PR branch at the new commit: create it on the first bump, + # force-move it while a bump PR is open. A force move is right here — + # the branch holds nothing but the previous bump, which this one + # replaces. + if gh api "repos/$REPO/git/ref/heads/$BRANCH" >/dev/null 2>&1; then + gh api -X PATCH "repos/$REPO/git/refs/heads/$BRANCH" \ + -f "sha=$commit" -F force=true >/dev/null + else + gh api "repos/$REPO/git/refs" \ + -f "ref=refs/heads/$BRANCH" -f "sha=$commit" >/dev/null + fi + echo "updated=true" >> "$GITHUB_OUTPUT" + + open-pr: + name: Open or refresh the bump pull request + needs: prepare + if: needs.prepare.outputs.updated == 'true' + runs-on: ubuntu-slim + timeout-minutes: 5 + permissions: + contents: read + pull-requests: write + steps: + - name: Open the pull request, or refresh the open one + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + BUMPS: ${{ needs.prepare.outputs.bumps }} + run: | + set -euo pipefail + { + echo "Bumps the npm packages pinned in \`.pre-commit-config.yaml\` hooks:" + echo + echo '```text' + printf '%s\n' "$BUMPS" + echo '```' + echo + echo "Each is the newest stable release past its cooldown (12 hours for Claude" + echo "Code, 7 days otherwise) — see \`tools/dev/bump-hook-npm-pins.py\`. Later runs" + echo "update this PR in place until it is merged." + echo + echo "The commit is signed by GitHub. GitHub raises no events for anything" + echo "\`GITHUB_TOKEN\` does, so the required checks start out unstarted:" + echo "**press *Ready for review* to start them.** The \`prek\` run includes" + echo "\`check-claude-mods\`, which re-checks every mod against the new pin." + echo + echo "Opened by [\`bump-hook-npm-pins.yml\`](.github/workflows/bump-hook-npm-pins.yml)." + } > "$RUNNER_TEMP/body.md" + + number=$(gh pr list --repo "$REPO" --head "$BRANCH" --state open \ + --json number --jq '.[0].number // empty') + if [[ -n "$number" ]]; then + gh pr edit "$number" --repo "$REPO" --body-file "$RUNNER_TEMP/body.md" + # The new commit has no checks yet; back to draft so marking it + # ready starts them, as for a fresh bump PR. + gh pr ready "$number" --repo "$REPO" --undo || true + echo "Updated #$number with the new bump." >> "$GITHUB_STEP_SUMMARY" + else + url=$(gh pr create --repo "$REPO" --base main --head "$BRANCH" --draft \ + --title "chore: bump npm pins in prek hooks" \ + --body-file "$RUNNER_TEMP/body.md") + echo "Opened $url" >> "$GITHUB_STEP_SUMMARY" + fi + + cleanup: + name: Drop the throwaway branch + needs: [prepare, open-pr] + if: always() && needs.prepare.outputs.tmp_branch != '' + runs-on: ubuntu-slim + timeout-minutes: 5 + permissions: + contents: write + steps: + - name: Delete the throwaway branch + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + TMP_BRANCH: ${{ needs.prepare.outputs.tmp_branch }} + run: gh api "repos/$REPO/git/refs/heads/$TMP_BRANCH" -X DELETE || true diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 8e79414ca..e8fe8ec7b 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -223,13 +223,17 @@ jobs: # matches no hook at all. It warns rather than failing, so the # comma form looks like it worked while skipping nothing. # + # `--skip check-claude-mods-local`: the contributor-side twin of + # `check-claude-mods`, using a local Claude Code the runner does not + # have. The pinned hook runs in its own step below. + # # `--skip lychee`: the link check is whole-repo on every event, # in the dedicated step below. See its comment for why it cannot # ride along with `PREK_SCOPE`. run: >- prek run --show-diff-on-failure --color=always $PREK_SCOPE --skip workspace-pytest --skip identity --skip skill-token-count - --skip lychee + --skip lychee --skip check-claude-mods-local # The link check runs over the whole repository on every event, # `main` and pull request alike — `PREK_SCOPE` does not apply. # @@ -266,3 +270,16 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: prek run lychee --color=always --all-files + # Claude Code mods: `claude plugin validate --strict` and `claude plugin + # test` on every plugin whose `hooks/hooks.json` declares `modules`. The + # hook is `stages: [manual]` so contributors' commits never fetch the + # Claude Code binary it pins, which means the `Run prek` step above never + # runs it — this step is the only place it runs on its own. + # + # Whole repo on both events, like lychee and for a similar reason: the + # pin bump in `.pre-commit-config.yaml` touches no mod file, yet it is + # exactly the change that has to re-check every mod against the new + # version. A no-op (no binary fetched) while the tree has no mod. + - name: Run prek (Claude Code mods — whole repo, both events) + if: ${{ !cancelled() }} + run: prek run check-claude-mods --hook-stage manual --color=always --all-files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7127cf56d..22d459477 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -335,6 +335,61 @@ repos: entry: python3 tools/dev/check-family-plugins.py --fix files: ^(skills/.*/SKILL\.md|plugins/.*|\.claude-plugin/(marketplace|plugin)\.json|\.codex-plugin/plugin\.json|\.agents/plugins/marketplace\.json|(plugin|marketplace)\.json|gemini-extension\.json|apm\.yml|pyproject\.toml)$ pass_filenames: false + # Claude Code mods — `claude plugin validate --strict` and `claude plugin + # test` on every plugin whose `hooks/hooks.json` declares `modules` + # (docs/when-to-use-mods.md). Claude Code loads a hooks module only after + # static analysis passes; a module that fails it never runs, and nothing + # else in the tree notices. A no-op until the tree has a mod. + # + # `language: node` with the CLI pinned in `additional_dependencies`, so + # prek installs it into a cached hook env, the way `lychee` gets its + # toolchain: no local Claude Code needed, and CI needs no install step. + # + # `stages: [manual]`: the hook env is a ~200 MB native binary, which is + # not worth fetching on every contributor's first commit for a check that + # concerns only the few who write mods. CI runs it on every event (its own + # step in `.github/workflows/pre-commit.yml`); locally it is + # `prek run check-claude-mods --hook-stage manual --all-files`. + # + # Claude Code is a dev-only tool here: CI runs it to check the mods, it is + # not a dependency of anything Magpie ships, and nothing from it goes into a + # release or a plugin. + # + # The pin is moved by `.github/workflows/bump-hook-npm-pins.yml` (Dependabot's + # `pre-commit` ecosystem never touches `additional_dependencies`), with a + # 12-hour cooldown rather than the 7 days the rest of the tree uses. A + # cooldown buys time for a bad or compromised release to be withdrawn + # before CI installs it. Claude Code ships several times a day and replaces + # a bad release within hours, so 12 hours already covers that window; and + # a security problem in it is fixed by the next release, so a week-long wait + # would keep CI on the known-bad version instead of protecting it. The mods + # API is also still moving, and adopters auto-update, so the check has to + # track what they actually run. Being dev-only (above), the worst a bad + # version can do is fail or wrongly pass this check. See + # `tools/dev/bump-hook-npm-pins.py`. + - repo: local + hooks: + - id: check-claude-mods + name: check-claude-mods (claude plugin validate --strict / test) + language: node + entry: tools/dev/check-claude-mods.sh + additional_dependencies: ["@anthropic-ai/claude-code@2.1.295"] + files: ^(.*/hooks/.*|.*\.claude-plugin/plugin\.json|tools/dev/check-claude-mods\.sh)$ + pass_filenames: false + stages: [manual] + # The same check on ordinary commits, with the contributor's own Claude + # Code: `language: system`, so nothing is downloaded. It skips itself when + # `claude` is not installed, and when the installed one is older than the + # pin above (an older CLI may not know events a mod uses). A newer one + # runs. CI's pinned run above stays the authoritative result. + - repo: local + hooks: + - id: check-claude-mods-local + name: check-claude-mods-local (local Claude Code, skipped if absent) + language: system + entry: tools/dev/check-claude-mods.sh --local + files: ^(.*/hooks/.*|.*\.claude-plugin/plugin\.json|tools/dev/check-claude-mods\.sh)$ + pass_filenames: false # Every shared prose block a skill carries — the auto-inserted setup # pre-flight, plus any number of declared blocks. It cannot be a hook: on # most harnesses *no code runs at all* when a plugin is installed or diff --git a/docs/when-to-use-mods.md b/docs/when-to-use-mods.md index 3db55f047..b43c1c2cd 100644 --- a/docs/when-to-use-mods.md +++ b/docs/when-to-use-mods.md @@ -177,11 +177,20 @@ The hooks manifest at `hooks/hooks.json` lists the mod modules to load: ``` The module (`hooks/register.ts`) exports a `register(on, options)` function that registers event listeners with the harness. +It imports only its own files, by relative path, and `claude-code`. +Node built-ins such as `node:fs` are not available: reach files, the environment and processes through the mods API (`$.fs`, `$.env`, `$.process`), so the `calls:` line `claude plugin validate` prints is the module's whole surface. +`hooks/hooks.json` can also carry the plugin's settings hooks under `hooks`, next to `modules`; a mod that takes over from a settings hook keeps that hook wired there for sessions where mods do not load. Every mod must include unit tests and pass strict validation before landing: 1. **Static Validation**: Mod modules must pass `claude plugin validate --strict` to verify hooked events and API calls. -2. **Automated Unit Tests**: Hook logic must be tested using `claude plugin test` with corresponding `*.test.ts` test suites. -3. **CI Integration**: Plugin validation and test suites must run alongside standard pre-commit hooks and Python test runners. + Claude Code runs the same analysis when it loads a mod and skips a module that fails it, so a mod that fails here does nothing at all on an adopter's machine. +2. **Automated Unit Tests**: Hook logic must be tested using `claude plugin test` with corresponding `*.test.ts` test suites written against `claude-code/testing`. + A mod with no tests fails the check. +3. **CI Integration**: the `check-claude-mods` prek hook ([`tools/dev/check-claude-mods.sh`](../tools/dev/check-claude-mods.sh)) runs both commands on every plugin whose `hooks/hooks.json` declares `modules`, on every pull request and push to `main`. + prek installs a pinned Claude Code into the hook environment, so neither a local install nor a login is needed. + The hook is in the `manual` stage, so ordinary commits skip it; run it before pushing a mod with `prek run check-claude-mods --hook-stage manual --all-files`. + The pin is bumped three times a week, with a 12-hour cooldown, by [`bump-hook-npm-pins.yml`](../.github/workflows/bump-hook-npm-pins.yml). + Claude Code is a dev-only tool here: CI runs it to check the mods, it is not a dependency of anything Magpie ships, and nothing from it goes into a release or a plugin. ## See also diff --git a/tools/dev/README.md b/tools/dev/README.md index e721286b4..ef1fcdca2 100644 --- a/tools/dev/README.md +++ b/tools/dev/README.md @@ -61,6 +61,8 @@ installable for other members to depend on it. | [`check-duplication.py`](check-duplication.py) | Fails the build on new cross-file near-duplicate prose across `skills/` (recursively), `tools/dev/blocks/*.md`, and `preflight-block.md` — the gate that stops the duplication `check-shared-blocks.py` removes from coming back. Paragraphs over 25 words, normalised to lowercase word tokens and compared across files as sets of 9-grams, scored `\|A ∩ B\| / min(\|A\|, \|B\|)`; fails above 0.50, reports (without failing) everything in `[0.30, 0.50]`, says nothing below that. Reuses `check-shared-blocks.py`'s own `PREFLIGHT_RE` / `DECLARED_RE` marker regexes to blank out generated regions before scoring — blanked rather than deleted, so removing one can never fuse the paragraph before it onto the paragraph after it — and excludes YAML frontmatter and fenced code blocks the same way. A failure names both files, both line numbers, the score, and a snippet of each paragraph, and points at `tools/dev/blocks/.md` as the remedy. | | [`estimate-skill-tokens.py`](estimate-skill-tokens.py) | Estimates each marketplace plugin's **always-on** token cost — the frontmatter `name` + `description` every installed skill advertises on every turn, at ~4 chars/token — and prints it per family. `--check` compares the figures published in `docs/setup/marketplace.md` and `docs/quick-start.md` against the live frontmatter; `check-doc-sync.py` calls it, so an edited description that moves a published number fails the build. The `SKILL.md` body is excluded: it costs nothing until the skill is invoked. | | [`check-family-plugins.py`](check-family-plugins.py) | Validates the marketplace plugins against the skills' `family:` frontmatter — version parity across every ecosystem manifest, Agent Plugins 1.0 conformance, and one well-formed per-family plugin whose `skills/` symlinks match the family exactly. `--fix` regenerates them, which is how the prek hook runs it. | +| [`check-claude-mods.sh`](check-claude-mods.sh) | Runs `claude plugin validate --strict` and `claude plugin test` on every plugin whose `hooks/hooks.json` declares `modules` — the Claude Code mods ([`docs/when-to-use-mods.md`](../../docs/when-to-use-mods.md)). Claude Code skips a hooks module that fails its static analysis, so without this a mod that cannot load passes every other check and does nothing. A mod with no `*.test.ts` fails. Runs as the `check-claude-mods` prek hook, whose `language: node` env carries a pinned Claude Code. The hook is `stages: [manual]`, so commits never fetch the ~200 MB binary: the `prek` workflow runs it in a step of its own, and locally it is `prek run check-claude-mods --hook-stage manual --all-files`. A no-op when the tree has no mods. | +| [`bump-hook-npm-pins.py`](bump-hook-npm-pins.py) | Moves the npm packages pinned in `.pre-commit-config.yaml` hooks' `additional_dependencies` — today the Claude Code CLI behind `check-claude-mods` — to the newest stable release past the package's cooldown: 12 hours for Claude Code, which ships several times a day and replaces a bad release within hours, 7 days otherwise, matching `.github/dependabot.yml`. Claude Code is a dev-only tool here — CI runs it to check the mods, and nothing from it ships. Dependabot's `pre-commit` ecosystem bumps a hook repo's `rev` but never these pins. Never moves a pin backwards. `--check` reports without writing. Run three times a week by [`bump-hook-npm-pins.yml`](../../.github/workflows/bump-hook-npm-pins.yml), which keeps a single draft PR open and updates it in place until it is merged. | | [`bump-dev-version.py`](bump-dev-version.py) | Moves the `.dev` stamp on `project.version` in the root `pyproject.toml` — the single authority every manifest mirrors. Only the edit: `check-family-plugins.py --fix` and `uv lock` still follow, so the three steps read the same whether a human or CI runs them. The stamp is UTC at minute resolution and has to *move* for adopters to pick anything up (`claude plugin update` compares version strings, so a frozen suffix is a silent no-op). Refuses a version with no `.dev` suffix rather than stamping a release. Called by [`bump-dev-version.yml`](../../.github/workflows/bump-dev-version.yml). | | [`gh-signed-commit.py`](gh-signed-commit.py) | Commits the working tree through GitHub's `createCommitOnBranch` mutation instead of `git commit` + `git push`, so the commit is signed by GitHub and shows as **Verified** with no key material in CI. Collects changed and deleted paths from `git status --porcelain -z`, decomposes renames (the mutation has no rename concept), and pins `expectedHeadOid` so a concurrent push fails the call rather than being overwritten. Used by [`bump-dev-version.yml`](../../.github/workflows/bump-dev-version.yml). | | [`check-placeholders.sh`](check-placeholders.sh) | Fails the build on hardcoded project references in skill and tool docs, which must use `` / `` / `` / `` instead. Carries both casings and matches spaced variants. | diff --git a/tools/dev/bump-hook-npm-pins.py b/tools/dev/bump-hook-npm-pins.py new file mode 100644 index 000000000..f53de9544 --- /dev/null +++ b/tools/dev/bump-hook-npm-pins.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Bump the npm packages pinned in ``.pre-commit-config.yaml`` hooks. + +Dependabot's ``pre-commit`` ecosystem bumps a hook repository's ``rev`` but +never the packages a hook pins in ``additional_dependencies``, so a pin such as +``"@anthropic-ai/claude-code@2.1.292"`` would otherwise only move when someone +remembers it. This finds every ``@`` pin there, asks the npm +registry for the newest stable release that has cleared the package's +cooldown, and rewrites the pin in place. + +The cooldown is the same idea as the 7 days in ``.github/dependabot.yml``: a +release gets time to be withdrawn or retagged before it reaches CI. Packages +can shorten it in ``COOLDOWN``. Claude Code gets 12 hours: + +- it ships several times a day and a bad release is replaced within hours, so + 12 hours already covers the window a cooldown exists for; +- a security problem in it is fixed by the next release, so a week-long wait + would hold CI on the known-bad version rather than protect it; +- the mods API is still moving and adopters auto-update, so the check has to + track what they run; +- it is a dev-only tool here — CI runs it to check the mods, and nothing from + it ships — so a bad version can at worst fail or wrongly pass that check. + +Usage: + + bump-hook-npm-pins.py # rewrite pins; print one line per bump + bump-hook-npm-pins.py --check # report only; exit 1 when a bump is due +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import re +import sys +import urllib.parse +import urllib.request +from collections.abc import Callable +from pathlib import Path + +CONFIG = Path(__file__).resolve().parents[2] / ".pre-commit-config.yaml" + +DEFAULT_COOLDOWN = dt.timedelta(days=7) + +#: Per-package overrides of ``DEFAULT_COOLDOWN``. +COOLDOWN = { + "@anthropic-ai/claude-code": dt.timedelta(hours=12), +} + +#: A quoted ``@`` pin, scoped or not. Only exact stable +#: versions match, so a range or a ``cli:`` dependency is left alone. +PIN_RE = re.compile(r'"(?P(?:@[a-z0-9._-]+/)?[a-z0-9._-]+)@(?P\d+\.\d+\.\d+)"') + +STABLE_RE = re.compile(r"^\d+\.\d+\.\d+$") + +REGISTRY = "https://registry.npmjs.org/" + + +def find_pins(text: str) -> dict[str, str]: + """Return ``{package: version}`` for every pin in ``additional_dependencies``.""" + pins: dict[str, str] = {} + for line in text.splitlines(): + if line.strip().startswith("additional_dependencies:"): + for match in PIN_RE.finditer(line): + pins[match["package"]] = match["version"] + return pins + + +def _key(version: str) -> tuple[int, ...]: + return tuple(int(part) for part in version.split(".")) + + +def pick_version(times: dict[str, str], now: dt.datetime, cooldown: dt.timedelta) -> str | None: + """The newest stable version published at least ``cooldown`` before ``now``. + + ``times`` is the registry's ``time`` map: version -> ISO publish time, plus + the ``created`` and ``modified`` bookkeeping keys, which never look like a + version and so drop out. + """ + cutoff = now - cooldown + eligible = [ + version + for version, published in times.items() + if STABLE_RE.match(version) and dt.datetime.fromisoformat(published.replace("Z", "+00:00")) <= cutoff + ] + return max(eligible, key=_key, default=None) + + +def fetch_times(package: str) -> dict[str, str]: + url = REGISTRY + urllib.parse.quote(package, safe="@") + with urllib.request.urlopen(url, timeout=30) as response: + return json.load(response)["time"] + + +def bump( + text: str, + now: dt.datetime, + fetch: Callable[[str], dict[str, str]] = fetch_times, +) -> tuple[str, list[tuple[str, str, str]]]: + """Return the rewritten config and ``(package, old, new)`` for each bump. + + A pin already newer than anything eligible (bumped by hand onto a release + still in its cooldown) is never moved backwards. + """ + bumps = [] + for package, current in find_pins(text).items(): + cooldown = COOLDOWN.get(package, DEFAULT_COOLDOWN) + target = pick_version(fetch(package), now, cooldown) + if target is None or _key(target) <= _key(current): + continue + text = text.replace(f'"{package}@{current}"', f'"{package}@{target}"') + bumps.append((package, current, target)) + return text, bumps + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--check", action="store_true", help="report only; exit 1 when a bump is due") + args = parser.parse_args(argv) + + text = CONFIG.read_text(encoding="utf-8") + new_text, bumps = bump(text, dt.datetime.now(dt.UTC)) + for package, old, new in bumps: + print(f"{package}: {old} -> {new}") + if args.check: + return 1 if bumps else 0 + if bumps: + CONFIG.write_text(new_text, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/dev/check-claude-mods.sh b/tools/dev/check-claude-mods.sh new file mode 100755 index 000000000..8e4bcf050 --- /dev/null +++ b/tools/dev/check-claude-mods.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# check-claude-mods.sh +# +# Validates and tests every Claude Code mod in the repository. +# +# A mod is a plugin whose `hooks/hooks.json` declares a non-empty +# `modules` list (docs/when-to-use-mods.md). For each one this runs: +# +# claude plugin validate --strict static analysis of the +# manifest and hooks module, the same analysis Claude Code runs when +# it loads the mod. A module Claude Code would refuse to load fails +# here instead of silently doing nothing on an adopter's machine. +# claude plugin test the mod's `*.test.ts` / +# `*.test.tsx` files, run with `claude-code/testing`. A mod with no +# tests fails: `claude plugin test` exits 1 when it finds none. +# +# A `hooks/hooks.json` with settings hooks only (no `modules`) is not a mod +# and is skipped, as is the whole run when the tree has no mods at all. +# +# Two hooks run this script: +# +# check-claude-mods `claude` from the prek hook environment +# (`language: node`, pinned in `.pre-commit-config.yaml`). Manual stage; +# CI runs it, and it is the authoritative result. +# check-claude-mods-local `--local`: the contributor's own `claude`, on +# ordinary commits. Skipped when Claude Code is not installed or cannot +# run, and when it is older than the pin, since an older CLI may not +# know events a mod uses and would fail it for nothing. A newer one +# runs: it is what adopters have, and a mod it rejects is worth knowing +# about before the pin catches up. +# +# Neither command needs a login or the network; the configuration directory +# is a throwaway one so the run never reads or writes the user's `~/.claude`. +set -euo pipefail + +repo_root="$(git rev-parse --show-toplevel)" +cd "$repo_root" + +if [ "${1:-}" = "--local" ]; then + pinned="$(sed -n 's/.*"@anthropic-ai\/claude-code@\([0-9.]*\)".*/\1/p' .pre-commit-config.yaml | head -n1)" + if ! command -v claude >/dev/null 2>&1; then + echo "check-claude-mods-local: Claude Code is not installed; skipped (CI runs the pinned $pinned)." + exit 0 + fi + # Some installs wrap the binary and print a banner first; take the line + # that starts with a version. + local_version="$(claude --version 2>/dev/null | grep -Eo '^[0-9]+\.[0-9]+\.[0-9]+' | head -n1 || true)" + if [ -z "$local_version" ]; then + echo "check-claude-mods-local: \`claude --version\` did not run; skipped (CI runs the pinned $pinned)." + exit 0 + fi + if [ "$(printf '%s\n%s\n' "$pinned" "$local_version" | sort -V | head -n1)" != "$pinned" ]; then + echo "check-claude-mods-local: local Claude Code $local_version is older than the pinned $pinned; skipped." + echo " Update Claude Code, or run the pinned one: prek run check-claude-mods --hook-stage manual --all-files" + exit 0 + fi + echo "check-claude-mods-local: using local Claude Code $local_version (CI pins $pinned)." +fi + +mods=() +while IFS= read -r hooks_json; do + if node -e ' + const m = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).modules; + process.exit(Array.isArray(m) && m.length > 0 ? 0 : 1); + ' "$hooks_json"; then + mods+=("$(dirname "$(dirname "$hooks_json")")") + fi +done < <(git ls-files -- '*/hooks/hooks.json') + +[ "${#mods[@]}" -gt 0 ] || exit 0 + +# The git directory is the fallback for an agent sandbox that denies writes +# to the system temp directory but allows them inside the repository. +config_dir="$(mktemp -d 2>/dev/null || mktemp -d "$(git rev-parse --absolute-git-dir)/claude-mods.XXXXXX")" +trap 'rm -rf "$config_dir"' EXIT +export CLAUDE_CONFIG_DIR="$config_dir" +export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 +export DISABLE_AUTOUPDATER=1 + +status=0 +for mod in "${mods[@]}"; do + echo "== $mod" + claude plugin validate --strict "$mod" || status=1 + claude plugin test "$mod" || status=1 +done +exit "$status" diff --git a/tools/dev/tests/test_bump_hook_npm_pins.py b/tools/dev/tests/test_bump_hook_npm_pins.py new file mode 100644 index 000000000..af27ce44c --- /dev/null +++ b/tools/dev/tests/test_bump_hook_npm_pins.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for ``bump-hook-npm-pins.py``. + +What matters is that a bump never lands a release still inside its cooldown, +never picks a pre-release, never moves a pin backwards, and touches only the +quoted ``@`` pins in ``additional_dependencies``. The registry +is replaced by a fake, so nothing here touches the network. +""" + +from __future__ import annotations + +import datetime as dt +import importlib.util +from pathlib import Path +from types import ModuleType + +import pytest + +_SCRIPT = Path(__file__).resolve().parents[1] / "bump-hook-npm-pins.py" + +NOW = dt.datetime(2026, 10, 9, 6, 0, tzinfo=dt.UTC) + +CONFIG = """\ +repos: + - repo: local + hooks: + - id: lychee + additional_dependencies: ["cli:lychee"] + - id: check-claude-mods + language: node + # "@anthropic-ai/claude-code@1.0.0" in a comment is not a pin + additional_dependencies: ["@anthropic-ai/claude-code@2.1.288"] + - id: other + additional_dependencies: ["left-pad@1.0.0", "ranged@^2.0.0"] +""" + +TIMES = { + "@anthropic-ai/claude-code": { + "created": "2025-01-01T00:00:00Z", + "modified": "2026-10-08T18:22:58Z", + "2.1.288": "2026-10-02T18:30:40Z", + "2.1.292": "2026-10-06T17:10:31Z", + "2.1.294": "2026-10-08T03:42:57Z", # 26h old: past the 12-hour cooldown + "2.1.295": "2026-10-08T18:22:58Z", # 11.6h old: inside it + "2.2.0-beta.1": "2026-10-01T00:00:00Z", # pre-release: never picked + }, + "left-pad": { + "1.0.0": "2026-01-01T00:00:00Z", + "1.1.0": "2026-10-05T00:00:00Z", # 4 days old: inside the 7-day default + }, +} + + +def _load() -> ModuleType: + spec = importlib.util.spec_from_file_location("bump_hook_npm_pins", _SCRIPT) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def mod() -> ModuleType: + return _load() + + +def test_finds_only_exact_pins_in_additional_dependencies(mod: ModuleType) -> None: + assert mod.find_pins(CONFIG) == { + "@anthropic-ai/claude-code": "2.1.288", + "left-pad": "1.0.0", + } + + +def test_bumps_to_newest_release_past_its_cooldown(mod: ModuleType) -> None: + new, bumps = mod.bump(CONFIG, NOW, fetch=TIMES.__getitem__) + assert bumps == [("@anthropic-ai/claude-code", "2.1.288", "2.1.294")] + assert '"@anthropic-ai/claude-code@2.1.294"' in new + # The comment and every other line are untouched. + assert '"@anthropic-ai/claude-code@1.0.0" in a comment' in new + assert '"left-pad@1.0.0"' in new + + +def test_default_cooldown_holds_back_a_recent_release(mod: ModuleType) -> None: + later = NOW + dt.timedelta(days=3) + _, bumps = mod.bump(CONFIG, later, fetch=TIMES.__getitem__) + assert ("left-pad", "1.0.0", "1.1.0") in bumps + + +def test_never_moves_a_pin_backwards(mod: ModuleType) -> None: + ahead = CONFIG.replace("claude-code@2.1.288", "claude-code@2.1.295") + new, bumps = mod.bump(ahead, NOW, fetch=TIMES.__getitem__) + assert bumps == [] + assert new == ahead + + +def test_compares_versions_numerically(mod: ModuleType) -> None: + times = {"2.1.9": "2026-01-01T00:00:00Z", "2.1.10": "2026-01-02T00:00:00Z"} + assert mod.pick_version(times, NOW, dt.timedelta(hours=12)) == "2.1.10" + + +def test_nothing_eligible(mod: ModuleType) -> None: + assert mod.pick_version({"1.0.0": "2026-10-09T00:00:00Z"}, NOW, dt.timedelta(hours=12)) is None