feat(plugin): Add Plugin evaluation support across all tiers

.github/workflows/ci.yml

@@ -72,6 +72,27 @@ jobs:
       - name: Lint
         if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}
         run: uv run ruff check .
+      # The Harbor verifier runs these files with the task image's python3, which
+      # can be older than the Python SkillEvaluator itself needs. Compiling catches
+      # syntax; the smoke script also runs code that needs newer runtime features.
+      - name: Compile Harbor verifier files on Python 3.9 and 3.11
+        if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          files=(
+            src/skillevaluator/tier3/harbor/templates/*.py
+            src/skillevaluator/tier3/eval_core/log_converters.py
+            src/skillevaluator/tier3/eval_core/codex_tool_call_normalizer.py
+            src/skillevaluator/evidence.py
+          )
+          for version in 3.9 3.11; do
+            uv run --no-project --python "$version" -- python -I -c \
+              "import sys; [compile(open(p, encoding='utf-8').read(), p, 'exec') for p in sys.argv[1:]]" \
+              "${files[@]}"
+            uv run --no-project --python "$version" --with "idna>=3.10,<4" -- python -I \
+              scripts/ci/smoke_harbor_verifier.py
+          done
       - name: Run tests with coverage
         if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}
         run: >-
@@ -319,6 +340,7 @@ jobs:
           tests/test_results_location_legacy_compatibility.py
           tests/test_tier3_compare.py
           tests/test_tier3_progress.py
+          tests/utils/test_secure_fs_platform.py
           tests/utils/test_secure_fs_selected_reads.py
           tests/test_oss_packaging.py
 

.gitleaks.toml

@@ -10,6 +10,26 @@ targetRules = ["generic-api-key"]
 regexes = ['''^sk-AbCdEf1234567890$''']
 paths = ['''^tests/test_tier3_(progress|result_display)\.py$''']
 
+[[allowlists]]
+description = "Synthetic API key used by plugin hook inline-secret tests"
+condition = "AND"
+targetRules = ["generic-api-key"]
+regexes = ['''^abcd1234secret$''']
+paths = ['''^tests/validators/test_plugin_component_risk\.py$''']
+
+[[allowlists]]
+description = "Synthetic Docker registry login (base64 of a fake user:password) used by the scanner-credential tests"
+condition = "AND"
+targetRules = ["generic-api-key"]
+regexes = ['''^dXNlcjpodW50ZXIy$''']
+paths = ['''^tests/validators/test_audits_endpoints_review_fixes\.py$''']
+
+[[allowlists]]
+description = "Synthetic basic-auth, cloud-key and URL-token fixtures in the first revisions of the security-evidence and MCP-hardening tests; both files now assemble them from split literals"
+condition = "AND"
+commits = ["66ae1d4c1188e2a8d9f0586abe0a895e04b78758", "727826808a2d4c3c2b7c393d67f5339efde28532"]
+paths = ['''^tests/tier3/test_security_finding_evidence\.py$''', '''^tests/validators/test_mcp_hardening\.py$''']
+
 [[allowlists]]
 description = "Synthetic NVIDIA Build token used by streaming redaction tests"
 condition = "AND"
@@ -31,6 +51,27 @@ targetRules = ["generic-api-key"]
 regexes = ['''^cw-key-123456$''']
 paths = ['''^tests/test_harbor_environment_contract\.py$''']
 
+[[allowlists]]
+description = "Synthetic JWT in the first revision of the hook-census label-window test; the test now assembles it from its parts"
+condition = "AND"
+targetRules = ["jwt"]
+commits = ["6dab46f91eb7323b27d0e90ecb7c90f5c38e7da4"]
+paths = ['''^tests/tier3/test_hook_census\.py$''']
+
+[[allowlists]]
+description = "Synthetic deploy token in the first revision of the MCP bundle userinfo test; the test now splits it"
+condition = "AND"
+targetRules = ["generic-api-key"]
+commits = ["79a8fc2eb731bb155bb6b725e3655790eae2fa02"]
+paths = ['''^tests/validators/test_plugin_components\.py$''']
+
+[[allowlists]]
+description = "Synthetic OpenSSH private-key fixtures in the first revision of the report-text redaction tests; the tests now assemble the PEM label"
+condition = "AND"
+targetRules = ["private-key"]
+commits = ["2c5afed9274221fb8a408eeee0601ad53bad9090"]
+paths = ['''^tests/validators/test_url_policy\.py$''']
+
 [[allowlists]]
 description = "Bedrock model ID that follows a credential name in SkillSpector bridge tests"
 condition = "AND"

README.md

@@ -150,8 +150,8 @@ skillevaluator validate ./my-skill
 `--full` remains supported but is unnecessary.
 Use `--tiers 1,2` or `--no-tier3` to skip live evaluation; `--tiers 1` runs
 only the static suite. `--no-autopilot` requires an existing evaluation source.
-Rules, workflows, and plugins retain their applicable validation stages without
-automatically enabling skill evaluation.
+Rules, workflows, and plugins skip automatic Tier 3; see
+[Plugin Evaluation](https://docs.nvidia.com/skills/skillevaluator/plugin-evaluation) for plugins.
 
 Generate and review broader coverage; templates omit negatives unless authored in `EVAL.md`:
 

THIRD_PARTY_NOTICES.md

@@ -1,12 +1,39 @@
 # Third-Party Notices
 
-This file lists the direct third-party Python dependencies declared by this
-distribution. `pyproject.toml` defines the dependency groups, and `uv.lock`
-records the exact resolved dependency set used for this release.
+This file lists direct third-party Python dependencies and vendored test
+fixtures in this distribution. `pyproject.toml` defines the dependency groups,
+and `uv.lock` records the exact resolved dependency set used for this release.
 
 | Dependency group | Packages and licenses |
 | --- | --- |
-| Base | Click (BSD-3-Clause), IDNA (BSD-3-Clause), Jinja2 (BSD-3-Clause), Markdown-It-Py (MIT), Pydantic (MIT), PyYAML (MIT), Rich (MIT) |
+| Base | Click (BSD-3-Clause), IDNA (BSD-3-Clause), Jinja2 (BSD-3-Clause), Markdown-It-Py (MIT), Pydantic (MIT), PyYAML (MIT), regex (Apache-2.0 AND CNRI-Python), Rich (MIT) |
 | LLM | Anthropic (MIT), Boto3 (Apache-2.0), LiteLLM (MIT), OpenAI (Apache-2.0) |
 | Tier 3 | Harbor (Apache-2.0), MCP (MIT), PyJWT (MIT), python-dotenv (BSD-3-Clause) |
 | Security | Bandit (Apache-2.0), pip-audit (Apache-2.0) |
+
+## OpenClaw agent-skills test fixture
+
+`tests/fixtures/openclaw-autoreview/` contains pinned files from
+`openclaw/agent-skills`, commit `2a409d348a4bcf6f15e41e9a20efd0b298a32528`,
+path `skills/autoreview`. The source repository is licensed under the MIT
+License:
+
+Copyright (c) 2026 openclaw
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

docs/agents-and-sandboxes.mdx

@@ -77,6 +77,14 @@ Both bridges support multi-turn tool-call continuation, so agents complete
 real tool loops against Build models. The Claude Code bridge drops
 orchestration and server tools Build cannot execute — `WebFetch`,
 `WebSearch`, `Task`, and similar — before forwarding a request.
+Build accepts tool names of at most 64 characters, so a longer one, such as
+a plugin MCP tool named `mcp__plugin_<plugin>_<server>__<tool>`, reaches the
+model as a stable alias: its first 55 characters and a short hash of the full
+name. The bridges translate the model's calls back, so the agent and its
+trajectory see the real tool name. Build tool messages carry text only, so a
+non-text block in a tool result, such as an image that an MCP tool or a file
+read returns, reaches the model as a short placeholder that names the block
+type.
 
 ### Models on the one-key path
 

docs/benchmark-rollout.mdx

@@ -98,8 +98,9 @@ Efficiency=`skill_efficiency`. A format-only rollout must not silently accept a
 different dataset or agent/model.
 
 A dimension passes at 50%. The overall verdict passes only when every configured
-dimension passes for at least one supported agent; lift is diagnostic evidence
-and does not override that gate.
+dimension passes for at least one supported agent; lift never overrides that
+verdict, but a confirmed regression (a FAIL-band lift whose whole interval is
+below zero) makes the card FAIL.
 
 ## 5. Run the publication gate
 

docs/ci-integration.mdx

@@ -231,7 +231,10 @@ When the exit code is not enough — dashboards, custom thresholds, badge genera
 | Field | Type | Contents |
 | --- | --- | --- |
 | `overall_passed` | boolean | `true` when every check passed |
-| `overall_status` | string | `passed`, `failed`, or `incomplete` (a required scanner produced no evidence) |
+| `overall_status` | string | `passed`, `failed`, or `incomplete`. A check that failed outranks missing evidence: `incomplete` means a required scanner produced no evidence and nothing failed. Every report gives the same headline |
+| `incomplete_scans` | array | Scanners that produced no trustworthy evidence, listed even when `overall_status` is `failed` |
+| `tier_status` | object | Status of each tier that ran, keyed `"1"`, `"2"`, `"3"`: `passed`, `neutral` (Tier 3), `failed`, `incomplete`, or `skipped` |
+| `status_reasons` | array | Why the headline is what it is: what failed, what did not complete, and which advisory tier did not pass |
 | `severity_counts` | object | Totals for `critical`, `high`, `medium`, `low` |
 | `total_errors`, `total_warnings` | number | Aggregate counts across all validators |
 | `skills` | array | Per-skill `{ name, passed, issue_count }` |