From 63987838b698bd3864c38bf66b64d4ef31a7637e Mon Sep 17 00:00:00 2001 From: Christopher Kevin Date: Thu, 1 Oct 2026 13:04:58 -0700 Subject: [PATCH 1/5] feat(plugin): add Codex, Cursor and Agent Plugins v1 manifests and static risk checks Detect and validate .codex-plugin, .cursor-plugin and the Agent Plugins v1 root plugin.json next to the Claude manifest, with per-format component profiles and flagged name/version conflicts. Oversize or non-UTF-8 client manifests no longer hide their hooks or MCP servers. Add Tier 1 static risk checks: hook events, handlers, auto-approve, fetch-and-execute, context injection and hooks.allowed_urls; subagent and command privileges; skill frontmatter hooks and allowed-tools; monitor and LSP commands; permission-bypass flags in every config node. Unreadable or oversize component files fail closed, and whole-tree scans cover declared skill folders. Co-authored-by: Narendran Raghavan Co-Authored-By: Claude Opus 5.5 Signed-off-by: Christopher Kevin --- src/skillevaluator/cli_core.py | 72 +- src/skillevaluator/config/pii_patterns.yaml | 23 +- src/skillevaluator/constants.py | 52 +- .../plugin/intra_plugin_validator.py | 2 +- .../deduplication/plugin/profile.py | 36 +- src/skillevaluator/embedding/registry.py | 12 +- src/skillevaluator/plugin_component_risk.py | 3393 +++++++++++++++++ src/skillevaluator/plugin_components.py | 1385 ++++++- src/skillevaluator/plugin_formats.py | 884 +++++ src/skillevaluator/plugin_manifest.py | 425 ++- src/skillevaluator/tier1/commands.py | 125 +- src/skillevaluator/utils/helpers.py | 141 +- src/skillevaluator/validators/base.py | 24 +- src/skillevaluator/validators/mcp_static.py | 415 +- .../validators/plugin_schema.py | 745 +++- src/skillevaluator/validators/policy.py | 83 +- src/skillevaluator/validators/security.py | 4 +- tests/cli/test_content_type_detection.py | 51 + tests/golden/cli_surface.json | 106 + tests/test_verifier_error_redaction.py | 2 + tests/validators/test_base.py | 29 + tests/validators/test_mcp_static.py | 49 +- .../validators/test_plugin_component_risk.py | 682 ++++ tests/validators/test_plugin_components.py | 158 +- .../test_plugin_hook_risk_bounds.py | 316 ++ .../test_plugin_manifest_formats.py | 787 ++++ .../test_plugin_manifest_review_fixes.py | 828 ++++ .../validators/test_plugin_review_fixes_b.py | 219 ++ .../test_plugin_static_risk_review_fixes.py | 827 ++++ tests/validators/test_policy_plugin_keys.py | 71 + 30 files changed, 11596 insertions(+), 350 deletions(-) create mode 100644 src/skillevaluator/plugin_component_risk.py create mode 100644 src/skillevaluator/plugin_formats.py create mode 100644 tests/validators/test_plugin_component_risk.py create mode 100644 tests/validators/test_plugin_hook_risk_bounds.py create mode 100644 tests/validators/test_plugin_manifest_formats.py create mode 100644 tests/validators/test_plugin_manifest_review_fixes.py create mode 100644 tests/validators/test_plugin_review_fixes_b.py create mode 100644 tests/validators/test_plugin_static_risk_review_fixes.py create mode 100644 tests/validators/test_policy_plugin_keys.py diff --git a/src/skillevaluator/cli_core.py b/src/skillevaluator/cli_core.py index d0be934d..0fae0a66 100644 --- a/src/skillevaluator/cli_core.py +++ b/src/skillevaluator/cli_core.py @@ -19,30 +19,67 @@ CONTENT_TYPE_SKILL, CONTENT_TYPE_UNKNOWN, CONTENT_TYPE_WORKFLOWS, - PLUGIN_CONTAINED_MANIFEST_DIR, + PLUGIN_AGENT_PLUGINS_V1_MANIFEST_FILE, PLUGIN_CONTAINED_MANIFEST_FILE, PLUGIN_MANIFEST_FILES, + PLUGIN_NATIVE_MANIFEST_DIRS, RULES_FILE_EXTENSION, SKILL_MANIFEST_FILE, SKILL_MANIFEST_VARIANTS, WORKFLOWS_MANIFEST_FILE, ) -from skillevaluator.utils.secure_fs import stat_is_link_or_reparse +from skillevaluator.plugin_manifest import agent_plugins_path_opt_in +from skillevaluator.utils.secure_fs import SecurePathError, stat_is_link_or_reparse # --------------------------------------------------------------------------- # Content-type detection # --------------------------------------------------------------------------- +_NATIVE_MANIFEST_DIRS_FOLDED = frozenset(name.casefold() for name in PLUGIN_NATIVE_MANIFEST_DIRS) + + def _is_contained_plugin_manifest(path: Path) -> bool: - """Return whether *path* is a contained-plugin manifest.""" - return path.name == PLUGIN_CONTAINED_MANIFEST_FILE and path.parent.name == PLUGIN_CONTAINED_MANIFEST_DIR + """Return whether *path* is a vendor-directory plugin manifest. + + ``plugin.json`` inside ``.claude-plugin/``, ``.codex-plugin/``, or + ``.cursor-plugin/`` roots a contained plugin at the directory's parent. + Names match without regard to case, as clients on a case-insensitive + filesystem open them. + """ + return ( + path.name.casefold() == PLUGIN_CONTAINED_MANIFEST_FILE + and path.parent.name.casefold() in _NATIVE_MANIFEST_DIRS_FOLDED + ) + + +def _is_agent_plugins_manifest(path: Path) -> bool: + """Return whether a root ``plugin.json`` declares an Agent Plugins ``$schema``. + + A root ``plugin.json`` is an Agent Plugins v1 manifest only when it opts in + with that ``$schema``; any other ``plugin.json`` is not a plugin manifest. + The decision and the bounded, no-follow read are the plugin locator's + (:func:`~skillevaluator.plugin_manifest.agent_plugins_path_opt_in`), so + detection and validation always agree: unparseable or non-UTF-8 JSON that + names the Agent Plugins schema host still marks a plugin, so its error is + reported rather than hidden, and a file over the manifest size bound is + parsed whole up to the lenient bound. Only a file that cannot be read + safely (a link or special file) does not count. + """ + if path.name.casefold() != PLUGIN_AGENT_PLUGINS_V1_MANIFEST_FILE: + return False + try: + return agent_plugins_path_opt_in(path) + except (OSError, SecurePathError, ValueError): + return False def _detect_from_file(path: Path) -> str | None: """Detect content type from a file path.""" if path.name in PLUGIN_MANIFEST_FILES or _is_contained_plugin_manifest(path): return CONTENT_TYPE_PLUGIN + if _is_agent_plugins_manifest(path): + return CONTENT_TYPE_PLUGIN if path.name.upper() == SKILL_MANIFEST_FILE.upper(): return CONTENT_TYPE_SKILL if path.suffix == RULES_FILE_EXTENSION: @@ -68,10 +105,13 @@ def _detect_from_directory(path: Path) -> str | None: for count, entry in enumerate(iterator, start=1): if count > CONTENT_DEDUP_MAX_DISCOVERED_PATHS: return None + folded = entry.name.casefold() interesting = ( entry.name in PLUGIN_MANIFEST_FILES or entry.name in SKILL_MANIFEST_VARIANTS - or entry.name in {WORKFLOWS_MANIFEST_FILE, PLUGIN_CONTAINED_MANIFEST_DIR} + or entry.name == WORKFLOWS_MANIFEST_FILE + or folded == PLUGIN_AGENT_PLUGINS_V1_MANIFEST_FILE + or folded in _NATIVE_MANIFEST_DIRS_FOLDED or entry.name.endswith(RULES_FILE_EXTENSION) ) if not interesting: @@ -80,8 +120,14 @@ def _detect_from_directory(path: Path) -> str | None: non_directory = not stat.S_ISDIR(metadata.st_mode) if entry.name in PLUGIN_MANIFEST_FILES and non_directory: plugin = True - elif entry.name == PLUGIN_CONTAINED_MANIFEST_DIR: - # Presence is enough for auto-detection. The secure plugin + elif folded == PLUGIN_AGENT_PLUGINS_V1_MANIFEST_FILE: + # Only a regular root plugin.json that declares the Agent + # Plugins $schema marks a plugin (bounded, no-follow read). + if stat.S_ISREG(metadata.st_mode) and _is_agent_plugins_manifest(path / entry.name): + plugin = True + elif folded in _NATIVE_MANIFEST_DIRS_FOLDED: + # Presence is enough for auto-detection (any spelling: a + # case-insensitive client opens it). The secure plugin # locator later distinguishes a real contained manifest # from an empty, linked, or malformed marker directory. contained = True @@ -95,8 +141,9 @@ def _detect_from_directory(path: Path) -> str | None: return None # Plugin detection must win before the SKILL.md / nested-structure checks: # a plugin dir may also contain skills/**/SKILL.md, but a plugin manifest at - # the root -- either agent_plugin.yaml/.yml (bundle-reference) or - # .claude-plugin/plugin.json (contained) -- makes it a plugin. + # the root -- agent_plugin.yaml/.yml (bundle-reference), a .claude-plugin/, + # .codex-plugin/, or .cursor-plugin/ plugin.json, or an Agent Plugins root + # plugin.json (contained) -- makes it a plugin. if plugin or contained: return CONTENT_TYPE_PLUGIN if skill: @@ -155,8 +202,9 @@ def detect_content_type(path: Path) -> str: """Auto-detect whether path contains a skill, rules, workflows, or plugin. Detection order: file type -> directory manifests -> path patterns -> nested structure. - A plugin manifest at the root -- agent_plugin.yaml/.yml (bundle-reference) or - .claude-plugin/plugin.json (contained) -- wins over a nested skills tree. + A plugin manifest at the root -- agent_plugin.yaml/.yml (bundle-reference), a + vendor plugin.json (.claude-plugin/, .codex-plugin/, .cursor-plugin/), or an + Agent Plugins root plugin.json (contained) -- wins over a nested skills tree. """ try: metadata = path.lstat() @@ -218,6 +266,8 @@ def resolve_plugin_path(path: Path) -> Path: return path.parent if _is_contained_plugin_manifest(path): return path.parent.parent + if path.name.casefold() == PLUGIN_AGENT_PLUGINS_V1_MANIFEST_FILE: + return path.parent return path diff --git a/src/skillevaluator/config/pii_patterns.yaml b/src/skillevaluator/config/pii_patterns.yaml index 06f85ec0..65c60062 100644 --- a/src/skillevaluator/config/pii_patterns.yaml +++ b/src/skillevaluator/config/pii_patterns.yaml @@ -53,8 +53,13 @@ home_paths: suggestion: "Use a generic path or an environment variable like $HOME" # Email addresses (excluding placeholder/test domains and OData property names) +# The local part is capped at the RFC 5321 limit of 64 characters. Unbounded, every +# start in a long run of local-part characters scanned to the end of the run, so +# one long line took minutes. A match starts at the start of the run or exactly 64 +# characters before the "@", the leftmost start the cap allows. Later starts would +# check the same domain again, so they are not tried. emails: - - pattern: '[A-Za-z0-9._%+-]+@(?!([a-z0-9-]+\.)*example\.com|([a-z0-9-]+\.)*example\.org|([a-z0-9-]+\.)*test\.com|odata\.)[A-Za-z0-9.-]+\.[A-Za-z]{2,}' + - pattern: '(?:(?.*?(?=(?:connection|conn)[-_]?string))(?P(?:connection|conn)[-_]?string.*password=)' severity: critical description: Connection string containing password suggestion: "Remove password from connection string - use environment variable" # JWT tokens (commonly leaked in logs/examples) +# A match starts only at the beginning of a run of JWT characters, so "eyJ-" * n +# is scanned once instead of once per "eyJ". "lead" is the part of the run before +# its first "eyJ" at a word start; "value" is the token that gets reported. jwt_tokens: - - pattern: '\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b' + - pattern: '(?(?:\b|[A-Za-z0-9_-]*?-)(?=eyJ)))(?P=lead)(?PeyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b)' severity: high description: JWT token suggestion: "Remove JWT token - use placeholder like eyJ...REDACTED...xyz" diff --git a/src/skillevaluator/constants.py b/src/skillevaluator/constants.py index 11a1cf04..e2d76061 100644 --- a/src/skillevaluator/constants.py +++ b/src/skillevaluator/constants.py @@ -141,6 +141,45 @@ PLUGIN_MODE = "bundle_reference" PLUGIN_CONTAINED_MANIFEST_TYPE = "claude_plugin_json" PLUGIN_CONTAINED_MODE = "contained" + +# Additional native plugin manifest formats. Each is a contained plugin: its +# components ship inside the plugin root, like ``.claude-plugin/plugin.json``. +PLUGIN_CODEX_MANIFEST_DIR = ".codex-plugin" +PLUGIN_CURSOR_MANIFEST_DIR = ".cursor-plugin" +# Agent Plugins v1 (agent-plugins.org) roots a plugin with ``plugin.json`` at +# the plugin root itself, not inside a vendor directory. +PLUGIN_AGENT_PLUGINS_V1_MANIFEST_FILE = PLUGIN_CONTAINED_MANIFEST_FILE +PLUGIN_CODEX_MANIFEST_TYPE = "codex_plugin_json" +PLUGIN_CURSOR_MANIFEST_TYPE = "cursor_plugin_json" +PLUGIN_AGENT_PLUGINS_V1_MANIFEST_TYPE = "agent_plugins_v1" +# Root-relative manifest paths in discovery precedence order, with the manifest +# type each one selects. When a root holds more than one, the first one wins +# and the others are recorded as additional manifest declarations. +PLUGIN_MANIFEST_PRECEDENCE: tuple[tuple[str, str], ...] = ( + ("agent_plugin.yaml", PLUGIN_MANIFEST_TYPE), + ("agent_plugin.yml", PLUGIN_MANIFEST_TYPE), + (f"{PLUGIN_CONTAINED_MANIFEST_DIR}/{PLUGIN_CONTAINED_MANIFEST_FILE}", PLUGIN_CONTAINED_MANIFEST_TYPE), + (PLUGIN_AGENT_PLUGINS_V1_MANIFEST_FILE, PLUGIN_AGENT_PLUGINS_V1_MANIFEST_TYPE), + (f"{PLUGIN_CODEX_MANIFEST_DIR}/{PLUGIN_CONTAINED_MANIFEST_FILE}", PLUGIN_CODEX_MANIFEST_TYPE), + (f"{PLUGIN_CURSOR_MANIFEST_DIR}/{PLUGIN_CONTAINED_MANIFEST_FILE}", PLUGIN_CURSOR_MANIFEST_TYPE), +) +PLUGIN_MANIFEST_RELATIVE_PATHS: tuple[str, ...] = tuple(path for path, _type in PLUGIN_MANIFEST_PRECEDENCE) +# Vendor directories whose ``plugin.json`` roots a contained plugin at the +# directory's parent. +PLUGIN_NATIVE_MANIFEST_DIRS: tuple[str, ...] = ( + PLUGIN_CONTAINED_MANIFEST_DIR, + PLUGIN_CODEX_MANIFEST_DIR, + PLUGIN_CURSOR_MANIFEST_DIR, +) +# Manifest types whose components are contained in the plugin root. +PLUGIN_CONTAINED_MANIFEST_TYPES = frozenset( + { + PLUGIN_CONTAINED_MANIFEST_TYPE, + PLUGIN_AGENT_PLUGINS_V1_MANIFEST_TYPE, + PLUGIN_CODEX_MANIFEST_TYPE, + PLUGIN_CURSOR_MANIFEST_TYPE, + } +) # Static plugin component inventory bounds (Tier 1 and Tier 3 coverage). PLUGIN_COMPONENT_MAX_ITEMS = 256 # Byte cap for plugin JSON config files (.mcp.json, referenced MCP configs, @@ -163,15 +202,16 @@ # File extensions to scan SCANNABLE_EXTENSIONS = {".py", ".sh", ".yaml", ".yml", ".json", ".md", ".txt"} +# Evaluation output and version-snapshot folder names (a skill's own artifacts). +SCAN_ARTIFACT_DIRS = frozenset({"evals", ".evals", "results", ".results", "versions", ".versions"}) + # Directories to skip at any depth during Tier 1 file-tree walks (scan artifact roots, version snapshots, build caches). +# Plugin skill discovery skips the artifact names only inside a skill: a folder +# with one of those names directly under a plugin's skills/ is searched, and a +# SKILL.md deeper inside one is reported (utils.helpers). SCAN_EXCLUDED_DIRS = frozenset( { - "evals", - ".evals", - "results", - ".results", - "versions", - ".versions", + *SCAN_ARTIFACT_DIRS, "__pycache__", ".git", ".venv", diff --git a/src/skillevaluator/deduplication/plugin/intra_plugin_validator.py b/src/skillevaluator/deduplication/plugin/intra_plugin_validator.py index f6bcd9f0..586979cd 100644 --- a/src/skillevaluator/deduplication/plugin/intra_plugin_validator.py +++ b/src/skillevaluator/deduplication/plugin/intra_plugin_validator.py @@ -56,7 +56,7 @@ def validate(self, plugin_root: Path) -> ValidationResult: self._mark_security_failure(result, str(exc)) return result if located is None or located.manifest_type != PLUGIN_MANIFEST_TYPE: - # Contained (.claude-plugin/plugin.json) or manifest-less plugins + # Contained (Claude, Codex, Cursor, Agent Plugins) or manifest-less plugins # expose no parsed refs, so Check A simply does not apply. result.add_success( "plugin_dep_dedup", diff --git a/src/skillevaluator/deduplication/plugin/profile.py b/src/skillevaluator/deduplication/plugin/profile.py index 004b92e2..27af501c 100644 --- a/src/skillevaluator/deduplication/plugin/profile.py +++ b/src/skillevaluator/deduplication/plugin/profile.py @@ -22,12 +22,13 @@ CONTENT_DEDUP_MAX_TOTAL_BYTES, DESCRIPTION_MAX_LENGTH, NAME_MAX_LENGTH, + PLUGIN_AGENT_PLUGINS_V1_MANIFEST_FILE, PLUGIN_CATALOG_MAX_MEMBER_CHARS, PLUGIN_CATALOG_MAX_MEMBERS, - PLUGIN_CONTAINED_MANIFEST_DIR, PLUGIN_CONTAINED_MANIFEST_FILE, - PLUGIN_CONTAINED_MANIFEST_TYPE, + PLUGIN_CONTAINED_MANIFEST_TYPES, PLUGIN_MANIFEST_FILES, + PLUGIN_NATIVE_MANIFEST_DIRS, SCAN_EXCLUDED_DIRS, SIMILARITY_MAX_DISCOVERED_PATHS, ) @@ -136,14 +137,17 @@ def _ref_member_name(ref: Any) -> str | None: def _load_manifest_data(plugin_root: Path, budget: ProfileByteBudget | None) -> tuple[dict[str, Any], str, str]: located = locate_plugin_manifest(plugin_root) if located is None: - raise PluginProfileError("No plugin manifest (agent_plugin.yaml/.yml or .claude-plugin/plugin.json) found") + raise PluginProfileError( + "No plugin manifest (agent_plugin.yaml/.yml, a .claude-plugin/, .codex-plugin/, or .cursor-plugin/ " + "plugin.json, or an Agent Plugins root plugin.json) found" + ) raw = located.read_text() if budget is not None: budget.consume(raw) try: data: Any = ( load_bounded_json(raw) - if located.manifest_type == PLUGIN_CONTAINED_MANIFEST_TYPE + if located.manifest_type in PLUGIN_CONTAINED_MANIFEST_TYPES else load_bounded_yaml(raw) ) except StructuredDataError as exc: @@ -255,11 +259,21 @@ def discover_plugin_roots(root: Path, *, max_plugins: int) -> list[Path]: """ if locate_plugin_manifest(root) is not None: return [root] - contained = PurePosixPath(PLUGIN_CONTAINED_MANIFEST_DIR) / PLUGIN_CONTAINED_MANIFEST_FILE + + def _is_vendor_manifest(posix: PurePosixPath) -> bool: + return ( + posix.name == PLUGIN_CONTAINED_MANIFEST_FILE + and len(posix.parts) >= 2 + and posix.parts[-2] in PLUGIN_NATIVE_MANIFEST_DIRS + ) def _selected(relative: Path) -> bool: posix = PurePosixPath(relative.as_posix()) - return posix.name in PLUGIN_MANIFEST_FILES or (len(posix.parts) >= 2 and posix.parts[-2:] == contained.parts) + return ( + posix.name in PLUGIN_MANIFEST_FILES + or posix.name == PLUGIN_AGENT_PLUGINS_V1_MANIFEST_FILE + or _is_vendor_manifest(posix) + ) files = discover_secure_files( root, @@ -270,7 +284,15 @@ def _selected(relative: Path) -> bool: plugin_dirs: set[PurePosixPath] = set() for file in files: relative = PurePosixPath(file.relative_path.as_posix()) - plugin_dir = relative.parent if relative.name in PLUGIN_MANIFEST_FILES else relative.parent.parent + plugin_dir = relative.parent.parent if _is_vendor_manifest(relative) else relative.parent + # A root plugin.json roots a plugin only when it is an Agent Plugins manifest. + if not _is_vendor_manifest(relative) and relative.name == PLUGIN_AGENT_PLUGINS_V1_MANIFEST_FILE: + try: + located = locate_plugin_manifest(root / Path(*plugin_dir.parts)) + except PluginManifestPathError: + located = None + if located is None: + continue plugin_dirs.add(plugin_dir) if len(plugin_dirs) > max_plugins: raise ValueError( diff --git a/src/skillevaluator/embedding/registry.py b/src/skillevaluator/embedding/registry.py index f64f8f79..e4372b80 100644 --- a/src/skillevaluator/embedding/registry.py +++ b/src/skillevaluator/embedding/registry.py @@ -34,9 +34,7 @@ CONTENT_TYPE_WORKFLOWS, PLUGIN_CATALOG_MAX_MEMBER_CHARS, PLUGIN_CATALOG_MAX_MEMBERS, - PLUGIN_CONTAINED_MANIFEST_DIR, - PLUGIN_CONTAINED_MANIFEST_FILE, - PLUGIN_MANIFEST_FILES, + PLUGIN_MANIFEST_RELATIVE_PATHS, SIMILARITY_CRITICAL_THRESHOLD, SIMILARITY_DEFAULT_MAX_ENTRIES, SIMILARITY_DEFAULT_MAX_SCALAR_COMPARISONS, @@ -128,9 +126,7 @@ MAX_CATALOG_PLUGIN_MEMBERS = PLUGIN_CATALOG_MAX_MEMBERS MAX_CATALOG_PLUGIN_MEMBER_CHARS = PLUGIN_CATALOG_MAX_MEMBER_CHARS MAX_CATALOG_MEMBER_NAMES = 262_144 -_PLUGIN_CATALOG_MANIFESTS = frozenset( - {*PLUGIN_MANIFEST_FILES, f"{PLUGIN_CONTAINED_MANIFEST_DIR}/{PLUGIN_CONTAINED_MANIFEST_FILE}"} -) +_PLUGIN_CATALOG_MANIFESTS = frozenset(PLUGIN_MANIFEST_RELATIVE_PATHS) @dataclass @@ -508,7 +504,9 @@ def build_from_directory( batch = texts[start : start + batch_size] vectors = [self._client.embed_chunked(batch[0])] if self._full_body else self._client.embed(batch) if len(vectors) != len(batch): - raise ValueError(f"Embedding provider returned {len(vectors)} vectors for {len(batch)} entries in batch") + raise ValueError( + f"Embedding provider returned {len(vectors)} vectors for {len(batch)} entries in batch" + ) for vector in vectors: vector_dimension = _validate_vector(vector, vector_dimension) validated_vectors.append(vector) diff --git a/src/skillevaluator/plugin_component_risk.py b/src/skillevaluator/plugin_component_risk.py new file mode 100644 index 00000000..a329f8ed --- /dev/null +++ b/src/skillevaluator/plugin_component_risk.py @@ -0,0 +1,3393 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Static privilege and hook risk analysis for plugin agents, commands, and hooks. + +Everything here inspects plugin-controlled data (already read through the +bounded, no-follow plugin-root reader) and never runs a hook, an agent, or a +command. It answers two questions: + +* **What does each subagent, command, and skill grant?** Agent frontmatter + (``tools``, ``disallowedTools``, ``model``, ``permissionMode``) and command and + skill frontmatter (``allowed-tools``, ``model``, ``disable-model-invocation``) + are recorded per component. Pre-approved unrestricted or interpreter ``Bash`` + grants, wildcard ``*`` tools, permissive permission modes, and agents that + inherit every tool next to write-capable MCP servers become findings. +* **What can each hook do?** ``hooks.json``, inline hook configs, and skill or + command frontmatter hooks are parsed per the Claude Code hooks reference (event + -> matcher groups -> handlers of type ``command``, ``http``, ``mcp_tool``, + ``prompt``, or ``agent``), with each client's own event names, approval output, + and command keys (:class:`HookDialect`: Cursor, GitHub Copilot). Each handler is + recorded with its event, matcher, handler type, and risk flags; the risky ones + (auto-approval, context injection, fetch-and-execute of remote code, unpinned + package runners, HTTP endpoints, and files outside the plugin root) become + findings. Plugin monitor commands get the same command checks. Every scan is + bounded, and a bound that leaves a handler or a hook script unanalyzed is itself + a finding, so padding cannot hide a hook. + +Findings use the ``PLUGIN_SCHEMA`` category, so a policy overlay can change any +severity with ``severity_overrides`` (``PLUGIN_SCHEMA.``). +""" + +from __future__ import annotations + +import bisect +import functools +import posixpath +import re +import shlex +from collections.abc import Callable, Iterable, Iterator +from dataclasses import dataclass, field +from pathlib import PurePosixPath +from typing import Any +from urllib.parse import parse_qs, unquote, urlparse, urlsplit + +from skillevaluator.constants import ( + CONTENT_DEDUP_MAX_FILE_BYTES, + PLUGIN_AGENT_PLUGINS_V1_MANIFEST_TYPE, + PLUGIN_CURSOR_MANIFEST_TYPE, +) +from skillevaluator.models.result import Finding, Severity +from skillevaluator.utils.redaction import redact_sensitive_text +from skillevaluator.validators.mcp_static import ( + OverrideIssue, + classify_endpoint_host, + classify_mcp_pinning, + host_is_allowlisted, + is_env_reference, + iter_config_strings, + looks_like_inline_secret, + url_ambiguities, + whatwg_url, +) + +CATEGORY = "PLUGIN_SCHEMA" + +# Documented Claude Code hook events (hooks reference). Unknown events are still +# recorded, with the ``unknown_event`` flag, because newer releases add events. +HOOK_EVENTS: frozenset[str] = frozenset( + { + "SessionStart", + "SessionEnd", + "Setup", + "UserPromptSubmit", + "UserPromptExpansion", + "Stop", + "StopFailure", + "PreToolUse", + "PostToolUse", + "PostToolUseFailure", + "PermissionRequest", + "PermissionDenied", + "PostToolBatch", + "PreModelSwitch", + "PostModelSwitch", + "SubagentStart", + "SubagentStop", + "TaskCreated", + "TaskCompleted", + "TeammateIdle", + "WorktreeCreate", + "WorktreeRemove", + "CwdChanged", + "DirectoryAdded", + "FileChanged", + "ConfigChange", + "InstructionsLoaded", + "PreCompact", + "PostCompact", + "Notification", + "MessageDisplay", + "Elicitation", + "ElicitationResult", + } +) +HANDLER_TYPES: tuple[str, ...] = ("command", "http", "mcp_tool", "prompt", "agent") +# Events whose hook output can approve a tool call or a permission prompt. +APPROVAL_EVENTS: frozenset[str] = frozenset({"PreToolUse", "PermissionRequest"}) +# Events whose command stdout (or ``additionalContext``) is injected into the conversation. +CONTEXT_EVENTS: frozenset[str] = frozenset({"UserPromptSubmit", "SessionStart"}) + +MAX_HOOK_EVENTS = 64 +MAX_HOOK_GROUPS = 256 +MAX_HOOK_HANDLERS = 512 +# A hook script is read and scanned whole up to the plugin reader's per-file limit; a larger +# script cannot be read and is reported as unanalyzed (never scanned in part). +MAX_SCRIPT_BYTES = CONTENT_DEDUP_MAX_FILE_BYTES +MAX_SCRIPT_READS = 32 +# File names a script mentions that are checked as possible second-level scripts; more is a finding. +MAX_SCRIPT_REFERENCES = 256 +# Distinct files one shell text runs that are matched against downloads; past it the hook is unanalyzed. +MAX_RUN_SITES = 2048 +MAX_TOOL_ENTRIES = 256 +MAX_MATCHER_CHARS = 256 +MAX_TARGET_CHARS = 200 +MAX_OUTSIDE_REFS = 5 + +_WILDCARD_TOOLS = frozenset({"*", "*(*)", "mcp__*", "mcp__*__*"}) +# Claude Code reads a matcher of only these characters as an exact tool name or '|' list, not a regex. +_EXACT_MATCHER_RE = re.compile(r"[A-Za-z0-9_|]+") +# A looser name list ('Edit, Bash'): one that names Bash is treated as matching Bash. +_NAME_LIST_RE = re.compile(r"[A-Za-z0-9_\- ,|]+") +# A regex matcher that matches all of these tool names (and a shell tool) is treated as "every tool". +_SAMPLE_TOOLS: tuple[str, ...] = ("Bash", "Read", "Write", "Edit", "WebFetch", "mcp__server__tool") +# Tools whose auto-approval skips a prompt that guards writes, network fetches, or MCP side effects. +_SENSITIVE_TOOLS: tuple[str, ...] = ("Write", "Edit", "MultiEdit", "NotebookEdit", "WebFetch", "mcp__server__tool") + +# Bash permission rules, classified the way Claude Code 2.1.x classifies its own allow rules: +# empty or only-'*' content is a bare grant, and an interpreter or wrapper prefix +# ("dangerous_prefix") runs any command. +_TOOL_RULE_RE = re.compile(r"\s*([A-Za-z_][\w-]*)\s*(?:\((.*)\))?\s*", re.DOTALL) +_DANGEROUS_BASH_PREFIXES: tuple[str, ...] = ( + "python", + "python3", + "python2", + "node", + "deno", + "tsx", + "ruby", + "perl", + "php", + "lua", + "npx", + "bunx", + "npm run", + "yarn run", + "pnpm run", + "bun run", + "bash", + "sh", + "ssh", + "zsh", + "fish", + "eval", + "exec", + "env", + "xargs", + "sudo", +) +# 'python -m pkg.module *' is the one option-prefixed interpreter rule Claude Code treats as scoped. +_PYTHON_DOTTED_MODULE_RE = re.compile(r"-m\s+\w+\.[\w.]+(?:\s*:|\s+)") +_ENV_REF_ANYWHERE_RE = re.compile(r"\$\{[A-Za-z_][A-Za-z0-9_]*\}|\$[A-Za-z_][A-Za-z0-9_]*") +# 'user:password@' in a URL authority (through its last '@'); scrubbed from every text a report shows. +_URL_USERINFO_RE = re.compile(r"//[^/?#\s]*@") +_URL_IN_TEXT_RE = re.compile(r"\b[A-Za-z][A-Za-z0-9+.\-]{0,31}://[^\s'\"`<>]{1,2048}") + +# Hook output that approves a tool call or a permission request: Claude Code's and Codex's shapes +# ("permissionDecision": "allow", "decision": "approve", "behavior": "allow"). +_ALLOW_DECISION_RES: tuple[re.Pattern[str], ...] = ( + re.compile(r"permissionDecision\W{0,8}allow\b"), + re.compile(r"\bdecision\W{0,8}approve\b"), + re.compile(r"\bbehavior\W{0,8}allow\b"), +) +# Cursor's approval output: {"permission": "allow"} (beforeShellExecution, beforeMCPExecution, +# beforeReadFile) and {"decision": "allow"} (preToolUse, subagentStart). +_CURSOR_ALLOW_RES: tuple[re.Pattern[str], ...] = ( + re.compile(r"\bpermission\W{0,8}allow\b"), + re.compile(r"\bdecision\W{0,8}allow\b"), +) +_ALLOW_SHAPES: dict[str, tuple[re.Pattern[str], ...]] = {"claude": _ALLOW_DECISION_RES, "cursor": _CURSOR_ALLOW_RES} +# JSON ("\u0061") and shell ("\x61") character escapes, decoded before the allow patterns run. A match only +# starts at the first backslash of a run, so a long run of backslashes with no escape after it stays linear. +_CHAR_ESCAPE_RE = re.compile(r"(?{_INTERPRETER})", re.IGNORECASE) +# A pipeline stage that sources its standard input: '| source /dev/stdin', '| . /dev/fd/0'. +_STAGE_SOURCE_STDIN_RE = re.compile( + r"&?\s*(?:source|\.)\s+[\"']?(?:/dev/stdin|/dev/fd/0|/proc/self/fd/0|-)[\"']?(?:\s|$)" +) +# bash -c "$(curl ...)", sh <(wget ...), eval "$(curl ...)", source <(curl ...), . <(curl ...) +_SUBSTITUTION_RE = re.compile( + rf"(?:{_INTERPRETER}|(?]{{0,128}}<<<\s*[\"']?" + rf"(?:(?:\$\(|`)\s*(?:[{_PATH_CHARS}]{{0,128}}/)?(?P{_FETCHERS})\b|\$\{{?(?P[A-Za-z_]\w{{0,63}})\b)", + re.IGNORECASE | re.MULTILINE, +) +# A variable that holds downloaded content ('code=$(curl -fsSL ...)'), and a command that runs one +# ('eval "$code"', 'bash -c "$code"', 'source "$code"'). +_FETCH_ASSIGNMENT_RE = re.compile( + rf"(?[A-Za-z_]\w{{0,63}})=[\"']?(?:\$\(|`)\s*(?:[{_PATH_CHARS}]{{0,128}}/)?{_FETCHERS}\b", + re.IGNORECASE, +) +_VARIABLE_RUN_RE = re.compile( + rf"(?:(?[A-Za-z_]\w{0,63})\b", + re.IGNORECASE | re.MULTILINE, +) +_VARIABLE_REF_RE = re.compile(r"\$\{?([A-Za-z_]\w{0,63})") +# Python or JavaScript that evaluates a download: exec(urlopen(url).read()), eval(await (await fetch(u)).text()). +_EXEC_FETCHED_RE = re.compile( + r"\b(?:exec|eval)\s*\([^\n]{0,256}?\b(?:urlopen|urlretrieve|requests\.get|httpx\.get|fetch)\s*\(" +) +# iex (iwr ...), Invoke-Expression (New-Object Net.WebClient).DownloadString(...) +_INVOKE_EXPRESSION_RE = re.compile(r"\b(?:iex|invoke-expression)\b", re.IGNORECASE) +_POWERSHELL_FETCH_RE = re.compile(r"\b(?:iwr|irm|invoke-webrequest|invoke-restmethod|downloadstring)\b", re.IGNORECASE) +# A fetch that writes a file: '-o FILE' (also '-fsSLo FILE'), wget's '-O FILE', '--output[-document] FILE', +# '-OutFile FILE', 'tee FILE', or a '> FILE' redirect. curl -O (a URL follows) and wget save under the +# URL's last path segment. +_DOWNLOADER_RE = re.compile(r"\b(?:curl|wget|aria2c|iwr|invoke-webrequest)\b", re.IGNORECASE) +_WGET_RE = re.compile(r"\bwget\b", re.IGNORECASE) +_OUTPUT_FILE_RE = re.compile( + r"(?:\s-outfile|\s-[A-Za-z]{0,8}o|\s--output(?:-document)?|\btee(?:\s+-a)?\s|>{1,2})" + r"(?:\s*=)?\s*[\"']?(?P[^\s;&|'\"<>()`]{1,256})", + re.IGNORECASE, +) +# An archive tool that unpacks a download: 'tar xzf x.tgz -C dir', 'curl ... | tar xz', 'unzip x.zip -d dir'. +_EXTRACTOR_RE = re.compile(r"\b(?:tar|bsdtar|unzip)\b") +# A file the text runs: with an interpreter or 'source' / '.', or as a command. A shell runs any command +# word with a '/' as a file ('./x', '/tmp/x', '&& $DIR/x', 'd/run'), and a variable there runs what it holds +# ('"$TOOL" check'). +_RUN_FILE_RE = re.compile( + rf"(?:{_INTERPRETER}|{_SOURCE_COMMAND})[ \t]+(?:{_OPTION}[ \t]+){{0,4}}+[\"']?" + r"(?P