From 211c999306065a22a923d352eeee6628e16f5b3f Mon Sep 17 00:00:00 2001 From: Amir Khakshour Date: Wed, 15 Jul 2026 21:13:42 +0200 Subject: [PATCH] feat(readme): advisory minimax README-drift reviewer as a GitHub Action MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Layer 2 of the README-drift stack WITHOUT touching Layer 1: the mechanical `readme-claim-check` job in ci.yml stays the hard, exact, free gate. This new job is ADVISORY — it never fails the PR. - .github/workflows/readme-review.yml — on PRs touching README/lib/bin/db/ recipes/schemas, runs a MiniMax-M3 reviewer (key from the MINIMAX_API_KEY repo secret; self-skips if absent). permissions: contents:read, pull-requests:write; only runs on same-repo PRs. - scripts/readme-review-ci.sh — reuses the existing minimax gatekeeper (cheap $0.005 'is this worth a review' veto) and feeds readme-claim-check --json as EXACT ground truth so the LLM judges SEMANTIC drift, not counts. Posts each suggested_fix as a line-anchored PR `suggestion` block where the excerpt matches a README line, else one consolidated advisory comment. Fail-open at every stage; always exits 0. DRY + MOCK env knobs for local testing. Why advisory not a gate: LLM verdicts are non-deterministic, so a hard LLM gate flakes red; advisory suggestions turn the model's real strength (drafting the fix) into value without the flaky-gate downside. Mechanical exactness stays in Layer 1. Deliberately did NOT LLM-ify readme-claim-check.sh — an LLM is worse at exact counting than `git ls-files | wc -l`, and would add cost/latency/nondeterminism to the one check that is currently free and always right. Requires operator action: add repo secret MINIMAX_API_KEY. Tested locally via MOCK+DRY (line-anchored suggestion + summary fallback + no-key self-skip). --- .github/workflows/readme-review.yml | 56 ++++++++++ scripts/readme-review-ci.sh | 159 ++++++++++++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 .github/workflows/readme-review.yml create mode 100755 scripts/readme-review-ci.sh diff --git a/.github/workflows/readme-review.yml b/.github/workflows/readme-review.yml new file mode 100644 index 00000000..38703c71 --- /dev/null +++ b/.github/workflows/readme-review.yml @@ -0,0 +1,56 @@ +name: README drift review (advisory) + +# Layer 2 of the README-drift stack: a MiniMax-M3 reviewer that posts +# NON-BLOCKING PR suggestions for SEMANTIC claim drift. The hard, exact gate +# stays in Layer 1 (the `readme-claim-check` job in ci.yml). This job never +# fails the PR — it only advises. +# +# Setup: add a repo secret MINIMAX_API_KEY. Without it the job self-skips. + +on: + pull_request: + paths: + - 'README.md' + - 'lib/**' + - 'bin/**' + - 'db/migrations/**' + - 'recipes/**' + - 'schemas/**' + +# least privilege: read the code, comment on the PR. No write to contents. +permissions: + contents: read + pull-requests: write + +concurrency: + group: readme-review-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + readme-review: + name: minimax README drift reviewer (advisory) + runs-on: ubuntu-latest + timeout-minutes: 10 + # only run when the secret exists, so forks / unconfigured repos don't + # show a perpetually-skipped-but-attempted job. + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # need history for the base..head diff + + - name: Fetch PR base for the diff + run: git fetch --no-tags origin "${{ github.base_ref }}" >/dev/null 2>&1 || true + + - name: Install claude CLI + run: npm install -g @anthropic-ai/claude-code >/dev/null 2>&1 || echo "claude install failed — script will self-skip" + + - name: MiniMax README drift review → PR suggestions (advisory) + env: + MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }} + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + MO_BASE_REF: origin/${{ github.base_ref }} + run: bash scripts/readme-review-ci.sh diff --git a/scripts/readme-review-ci.sh b/scripts/readme-review-ci.sh new file mode 100755 index 00000000..aea763df --- /dev/null +++ b/scripts/readme-review-ci.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# readme-review-ci.sh — Layer 2 (minimax) README drift reviewer for CI. +# +# ADVISORY ONLY. Runs on a PR, asks MiniMax-M3 (via the claude CLI + the +# minimax Anthropic-compatible gateway) whether any *semantic* README claim +# drifted, and posts the suggested fixes as NON-BLOCKING PR review comments +# (GitHub `suggestion` blocks where a line can be anchored, else a single +# summary comment). It NEVER fails the PR — the hard gate stays in Layer 1 +# (scripts/readme-claim-check.sh, mechanical/exact/free). +# +# Pipeline (each stage fail-open): +# 1. key + tooling guard → skip if absent +# 2. cheap MiniMax gatekeeper → skip trivial diffs (~$0.005) +# 3. mechanical claim-check --json → EXACT counts as ground truth for the LLM +# 4. one MiniMax reviewer call → JSON {drifted_claims:[{excerpt,reason,fix}]} +# 5. post suggestions to the PR → line-anchored where possible +# +# Env: +# MINIMAX_API_KEY (required; from the GitHub secret) — absent ⇒ skip +# GH_TOKEN GitHub token with pull-requests:write (for posting) +# GH_REPO owner/repo (default: $GITHUB_REPOSITORY) +# PR_NUMBER the PR number (default: from $GITHUB_REF) +# HEAD_SHA PR head sha (for line-anchored review comments) +# MO_BASE_REF base ref for the diff (default: origin/main) +# MO_REVIEW_DRY=1 print the gh calls instead of running them (testing) +# MO_REVIEW_MOCK inject reviewer JSON instead of calling the LLM (testing) +# +# Exit: ALWAYS 0 (advisory). A non-zero would fail the PR; that is Layer 1's job. + +set +e +MINI_ORK_ROOT="${MINI_ORK_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +MO_README="${MO_README:-README.md}" +DRY="${MO_REVIEW_DRY:-0}" + +log() { echo "[readme-review] $*" >&2; } +done_ok() { log "$*"; exit 0; } # advisory: every exit is 0 + +# ── 1. guards (fail-open) ──────────────────────────────────────────────────── +[ -f "$MO_README" ] || done_ok "$MO_README not found — skipping" +if [ -z "${MO_REVIEW_MOCK:-}" ]; then + [ -n "${MINIMAX_API_KEY:-}" ] || done_ok "MINIMAX_API_KEY unset — skipping (advisory only)" + command -v claude >/dev/null 2>&1 || done_ok "claude CLI not found — skipping" +fi +command -v jq >/dev/null 2>&1 || done_ok "jq not found — skipping" + +# ── 2. cheap gatekeeper: is this diff worth a review at all? ────────────────── +if [ -z "${MO_REVIEW_MOCK:-}" ]; then + gk=$(bash "$MINI_ORK_ROOT/scripts/readme-drift-gatekeeper.sh" 2>/dev/null) + gk_rc=$? + gk_verdict=$(printf '%s' "$gk" | jq -r '.verdict // "PANEL_SKIP"' 2>/dev/null) + if [ "$gk_rc" -ne 0 ] || [ "$gk_verdict" != "PANEL_NEEDED" ]; then + done_ok "gatekeeper: no review needed (rc=$gk_rc verdict=${gk_verdict:-none})" + fi + log "gatekeeper: PANEL_NEEDED — running minimax reviewer" +fi + +# ── 3. mechanical ground truth (exact counts — the LLM must NOT re-count) ───── +mech=$(bash "$MINI_ORK_ROOT/scripts/readme-claim-check.sh" --json 2>/dev/null) +[ -n "$mech" ] || mech='{"note":"mechanical check unavailable"}' + +# ── 4. the reviewer call ───────────────────────────────────────────────────── +upstream="${MO_BASE_REF:-origin/main}" +git -C "$MINI_ORK_ROOT" rev-parse "$upstream" >/dev/null 2>&1 || upstream="HEAD~1" +diff=$(git -C "$MINI_ORK_ROOT" diff "$upstream"...HEAD -- lib bin db recipes schemas "$MO_README" 2>/dev/null | head -c 12000) + +read -r -d '' prompt <","reason":"", +"suggested_fix":""}]} +Return {"drifted_claims":[]} if nothing drifted. + +--- MECHANICAL GROUND TRUTH (authoritative; do not re-derive) --- +$mech + +--- git diff (what changed) --- +$diff + +--- README.md (current) --- +$(cat "$MO_README") +EOF + +if [ -n "${MO_REVIEW_MOCK:-}" ]; then + out="$MO_REVIEW_MOCK" # test injection +else + out=$( + source "$MINI_ORK_ROOT/lib/providers/cl_minimax.sh" 2>/dev/null + timeout 150 claude --print --output-format text "$prompt" /dev/null + ) +fi +[ -n "$out" ] || done_ok "reviewer returned nothing — skipping (advisory)" + +# Extract the JSON object even if the model wrapped it in prose/fences. +json=$(printf '%s' "$out" | python3 -c ' +import sys, json, re +s = sys.stdin.read() +m = re.search(r"\{.*\}", s, re.DOTALL) +if not m: + print("{\"drifted_claims\":[]}"); sys.exit(0) +try: + obj = json.loads(m.group(0)) + print(json.dumps(obj)) +except Exception: + print("{\"drifted_claims\":[]}") +' 2>/dev/null) +count=$(printf '%s' "$json" | jq '.drifted_claims | length' 2>/dev/null || echo 0) +[ "${count:-0}" -gt 0 ] 2>/dev/null || done_ok "no semantic drift found — nothing to post" +log "reviewer flagged $count claim(s)" + +# ── 5. post suggestions to the PR (line-anchored where possible) ────────────── +GH_REPO="${GH_REPO:-$GITHUB_REPOSITORY}" +PR_NUMBER="${PR_NUMBER:-$(printf '%s' "${GITHUB_REF:-}" | sed -n 's#refs/pull/\([0-9]*\)/.*#\1#p')}" +HEAD_SHA="${HEAD_SHA:-$(git -C "$MINI_ORK_ROOT" rev-parse HEAD 2>/dev/null)}" + +post_review_comment() { # $1=line $2=body + if [ "$DRY" = "1" ]; then + echo "DRY gh api pulls/$PR_NUMBER/comments path=$MO_README line=$1"; echo "----"; echo "$2"; echo "----" + return 0 + fi + gh api "repos/$GH_REPO/pulls/$PR_NUMBER/comments" \ + -f body="$2" -f commit_id="$HEAD_SHA" -f path="$MO_README" -F line="$1" -f side=RIGHT >/dev/null 2>&1 \ + && log "posted suggestion at $MO_README:$1" \ + || log "could not post line-anchored comment at $1 (will fall back to summary)" +} + +summary_items=() +for i in $(seq 0 $((count - 1))); do + excerpt=$(printf '%s' "$json" | jq -r ".drifted_claims[$i].excerpt // \"\"") + reason=$(printf '%s' "$json" | jq -r ".drifted_claims[$i].reason // \"\"") + fix=$(printf '%s' "$json" | jq -r ".drifted_claims[$i].suggested_fix // \"\"") + [ -n "$excerpt" ] || continue + line=$(grep -n -F -m1 "$excerpt" "$MO_README" 2>/dev/null | head -1 | cut -d: -f1) + if [ -n "$line" ] && [ -n "$fix" ] && [ -n "$PR_NUMBER" ] && [ -n "$HEAD_SHA" ]; then + body=$'```suggestion\n'"$fix"$'\n```\n\n**README drift (minimax):** '"$reason" + post_review_comment "$line" "$body" + else + summary_items+=("- \`${excerpt:0:80}\` — ${reason}"$'\n'" → suggested: ${fix}") + fi +done + +# Any claim we could not anchor to a line → one consolidated advisory comment. +if [ "${#summary_items[@]}" -gt 0 ]; then + body=$'**README drift review (minimax, advisory — not a gate)**\n\nCould not anchor these to a line; please review:\n\n' + for it in "${summary_items[@]}"; do body+="$it"$'\n\n'; done + if [ "$DRY" = "1" ]; then + echo "DRY gh pr comment $PR_NUMBER:"; echo "$body" + elif [ -n "$PR_NUMBER" ]; then + printf '%s' "$body" | gh pr comment "$PR_NUMBER" --repo "$GH_REPO" --body-file - >/dev/null 2>&1 \ + && log "posted summary comment" || log "summary comment post failed (advisory — ignoring)" + fi +fi + +done_ok "review complete ($count claim(s) surfaced)"