diff --git a/Discovery/tests/install/recipes/__init__.py b/Discovery/tests/install/recipes/__init__.py new file mode 100644 index 0000000..fbaaf02 --- /dev/null +++ b/Discovery/tests/install/recipes/__init__.py @@ -0,0 +1,59 @@ +"""Fourteen recipe families, one per way something reaches a machine. + +A family is the unit of reuse: adding a tool that installs the way an existing +tool installs is a manifest edit with no code at all. That property is what +keeps a 120-entry inventory cheap to grow, and it is why the family is derived +from the entry's shape rather than declared beside it. + +Three families are implemented: ``declare-mcp``, ``artifact`` and +``npm-global``. Between them they cover 60 of Linux's 105 entries with no +vendor installer, no GUI session and no model downloads - and they are the two +categories where the collector does the most inference, so the cheapest half of +the manifest exercises the most interesting logic. + +The other eleven declare themselves and record ``unimplemented``. That is a +deliberate status: it keeps those entries out of the denominator, and it keeps +them loud. A harness that silently omitted them would report a recall computed +over whatever happened to work. +""" + +from typing import Dict, Type + +from .base import Outcome, Recipe, Unimplemented +from .artifact import ArtifactRecipe +from .declare_mcp import DeclareMcpRecipe +from .npm_global import NpmGlobalRecipe + +#: Families with no implementation yet, and the phase that will bring each in. +#: Named individually rather than defaulted, so adding a family to the manifest +#: without a recipe fails loudly instead of inheriting a stub. +PENDING = { + "app-installer": "P2/P3 - 16 vendors, 16 sets of silent flags", + "service": "P2 - starts a listener and pulls a model", + "channel-variant": "P2 - second installs, after the first ones", + "vscode-ext": "P2 - depends on app-installer", + "scheduler": "P2 - launchd, cron, systemd, schtasks", + "identity": "P3 - baked into the image, not scripted", + "baseline-prereq": "P1 - already in the golden image, verified only", + "runtime-state": "P2 - processes must stay alive across the scan", + "vendor-binary": "P2 - download, chmod, place on PATH", + "non-ai-app": "P2 - reuses app-installer / vscode-ext", + "pipx": "P2 - one command", +} + +REGISTRY: Dict[str, Type[Recipe]] = { + "declare-mcp": DeclareMcpRecipe, + "artifact": ArtifactRecipe, + "npm-global": NpmGlobalRecipe, +} + + +def for_family(family: str) -> Recipe: + """The recipe that executes this family, or one that records why it cannot.""" + implementation = REGISTRY.get(family) + if implementation is None: + return Unimplemented(family, PENDING.get(family, "no recipe registered")) + return implementation() + + +__all__ = ["Outcome", "Recipe", "REGISTRY", "PENDING", "for_family"] diff --git a/Discovery/tests/install/recipes/artifact.py b/Discovery/tests/install/recipes/artifact.py new file mode 100644 index 0000000..45480f2 --- /dev/null +++ b/Discovery/tests/install/recipes/artifact.py @@ -0,0 +1,114 @@ +"""``artifact`` - 24 entries, every one of them a file the collector must notice. + +Skills, commands, hooks, instruction files, a plugin directory, a malformed +bundle, two lookalike scripts and a dangling symlink. No installer, no network: +the whole family is file creation, which is why it is built first alongside +``declare-mcp``. + +Four creation shapes, and the distinction between them matters. ``merge`` is +not ``file``: two hook entries write the same ``settings.json``, and a recipe +that overwrote would silently delete the first and report it as a miss. +""" + +import json +import os +import posixpath +from typing import Any, Dict + +from .. import bodies, writers +from .base import Outcome, Recipe + + +class ArtifactRecipe(Recipe): + family = "artifact" + + def execute(self, context: Any, entry: Any) -> Outcome: + block = entry.create + path = context.path_for(entry) + if not path: + return self._outcome(entry, "failed", + reason="no %s path for %s" % (context.platform, entry.id)) + kind = block.get("kind", "file") + try: + handler = getattr(self, "_" + kind) + except AttributeError: + return self._outcome(entry, "failed", reason="unknown create kind %r" % kind) + + handler(context, entry, path) + if block.get("mode") and context.platform != "win": + (context.driver.sudo if entry.privileged else context.driver.run)( + ["chmod", block["mode"], path]) + + # Verify, always. A write that silently did not land - no permission, a + # missing parent, a transport that does not elevate - would otherwise be + # recorded `installed`, and the scorer would report the collector missing + # a file that was never there. That is the worst failure this harness can + # have: it manufactures a defect in the thing it is measuring. + if not context.driver.exists(path): + return self._outcome(entry, "failed", path=path, + reason="wrote %s but it is not there afterwards" % path) + return self._outcome(entry, "installed", path=path, method="agent_artifact") + + # -- the four shapes ---------------------------------------------- + + def _file(self, context: Any, entry: Any, path: str) -> None: + context.driver.write(path, self._body(context, entry, path), privileged=entry.privileged) + + def _directory(self, context: Any, entry: Any, path: str) -> None: + name = posixpath.basename(path.rstrip("/")) + for filename, content in bodies.plugin(name).items(): + context.driver.write(posixpath.join(path.rstrip("/"), filename), content) + + def _merge(self, context: Any, entry: Any, path: str) -> None: + """A hook joins a settings file rather than replacing it.""" + fragment = bodies.hook(entry.create["event"], context.substitute(entry.create["command"])) + existing: Dict[str, Any] = {} + if context.driver.exists(path): + local = context.scratch_file(path) + context.driver.pull(path, local) + try: + with open(local, encoding="utf-8") as handle: + existing = json.load(handle) + except (OSError, ValueError): + existing = {} + context.driver.write(path, writers.as_json(writers.merge(existing, fragment))) + + def _append(self, context: Any, entry: Any, path: str) -> None: + """AG-12: a key exported from a shell profile that already exists.""" + addition = bodies.shell_export(entry.create["variable"], + context.substitute(entry.create["value"])) + if context.platform == "win": + addition = "\n# %s\n$env:%s = \"%s\"\n" % ( + bodies.MARKER, entry.create["variable"], context.substitute(entry.create["value"])) + existing = "" + if context.driver.exists(path): + local = context.scratch_file(path) + context.driver.pull(path, local) + try: + with open(local, encoding="utf-8") as handle: + existing = handle.read() + except OSError: + existing = "" + context.driver.write(path, existing + addition) + + def _symlink(self, context: Any, entry: Any, path: str) -> None: + """N-09: a link to a target that does not exist, left dangling on purpose.""" + target = entry.create["target"] + context.driver.mkdir(posixpath.dirname(path), privileged=entry.privileged) + if context.platform == "win": + context.driver.shell("New-Item -ItemType SymbolicLink -Force -Path %r -Target %r" + % (path, target)) + elif entry.privileged: + context.driver.sudo(["ln", "-sfn", target, path]) + else: + context.driver.run(["ln", "-sfn", target, path]) + + # -- content ------------------------------------------------------- + + def _body(self, context: Any, entry: Any, path: str) -> str: + name = posixpath.basename(path) + stem = os.path.splitext(name)[0] + body = entry.create.get("body", "instructions") + if body in ("backup_script", "llm_wrapper", "malformed_bundle"): + return getattr(bodies, body)() + return getattr(bodies, body)(stem) diff --git a/Discovery/tests/install/recipes/base.py b/Discovery/tests/install/recipes/base.py new file mode 100644 index 0000000..a3658a5 --- /dev/null +++ b/Discovery/tests/install/recipes/base.py @@ -0,0 +1,74 @@ +"""What every recipe is, and the three answers it may give.""" + +from dataclasses import dataclass, field +from typing import Any, Dict, Optional + +#: The four statuses `manifest.actual.json` records, and what each means to the +#: scorer. Only `installed` is scoreable. The rest leave the denominator - but +#: they leave it *visibly*, because a silently shrinking denominator flatters +#: every recall number computed after it. +STATUSES = ("installed", "unavailable", "failed", "unimplemented") + + +@dataclass +class Outcome: + """What actually happened to one entry, as the runner will record it.""" + + id: str + status: str + catalog_id: Optional[str] = None + family: str = "" + version: Optional[str] = None + path: Optional[str] = None + method: Optional[str] = None + reason: Optional[str] = None + extra: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + row = {"id": self.id, "status": self.status, "family": self.family} + for key in ("catalog_id", "version", "path", "method", "reason"): + value = getattr(self, key) + if value is not None: + row[key] = value + row.update(self.extra) + return row + + +class Recipe: + """One way something reaches a machine. + + ``execute`` installs, verifies and locates in a single call, because the + three are inseparable: a recipe that installed without verifying would let + the runner record a version it never saw, and the scorer would then compare + the collector's answer against the manifest's intention rather than against + the disk. + """ + + family = "" + + def execute(self, context: Any, entry: Any) -> Outcome: + raise NotImplementedError + + # -- helpers shared by the implementations ------------------------- + + def _outcome(self, entry: Any, status: str, **fields: Any) -> Outcome: + return Outcome(id=entry.id, status=status, catalog_id=entry.catalog_id, + family=entry.family, **fields) + + +class Unimplemented(Recipe): + """A family the manifest declares and the harness cannot yet execute. + + Deliberately not an error. The manifest is complete before the harness is, + which is the right way round - the inventory is the specification - and an + entry nobody has automated yet must be visible as exactly that rather than + as a vendor that stopped shipping or a collector that missed something. + """ + + def __init__(self, family: str, reason: str): + self.family = family + self.reason = reason + + def execute(self, context: Any, entry: Any) -> Outcome: + return self._outcome(entry, "unimplemented", + reason="%s recipe not implemented: %s" % (self.family, self.reason)) diff --git a/Discovery/tests/install/recipes/declare_mcp.py b/Discovery/tests/install/recipes/declare_mcp.py new file mode 100644 index 0000000..a967ef4 --- /dev/null +++ b/Discovery/tests/install/recipes/declare_mcp.py @@ -0,0 +1,115 @@ +"""``declare-mcp`` - 27 entries, no installer, the most inference exercised. + +An MCP server exists because a config file says so. So this recipe writes the +config file the host application would have written, in the format that +application uses, at the path it uses on this OS - and nothing else. No package +is fetched, no process is started, nothing is granted network access. + +That is the whole point of building this family first: it covers the two dozen +entries where the collector does the most guessing, and it needs no package +manager, no vendor installer, no GUI session and no network to do it. +""" + +import json +from typing import Any, Dict, List + +from .. import writers +from .base import Outcome, Recipe + +#: How each site spells "here are my MCP servers". A site whose shape is wrong +#: produces a file the collector reads as empty, which would score as a missed +#: site and blame the collector for the harness's own mistake. +SITE_SHAPES = { + "claude-code": ("json", ["mcpServers"]), + "claude-desktop": ("json", ["mcpServers"]), + "cursor": ("json", ["mcpServers"]), + "windsurf": ("json", ["mcpServers"]), + "vscode": ("json", ["servers"]), + "cline": ("json", ["mcpServers"]), + "zed": ("json", ["context_servers"]), + "jetbrains": ("json", ["mcpServers"]), + "opencode": ("json", ["mcp"]), + "codex": ("toml", ["mcp_servers"]), + "goose": ("yaml", ["extensions"]), + "managed-claude-code": ("json", ["mcpServers"]), + "managed-adr": ("json", ["mcpServers"]), + "project": ("json", ["mcpServers"]), +} + + +class DeclareMcpRecipe(Recipe): + family = "declare-mcp" + + def execute(self, context: Any, entry: Any) -> Outcome: + declared = entry.declare + sites = declared.get("sites") or [declared.get("site")] + written: List[str] = [] + + for site in sites: + if site not in SITE_SHAPES: + return self._outcome(entry, "failed", reason="unknown declaration site %r" % site) + path = context.path_for(entry, site=site) + if not path: + return self._outcome(entry, "failed", + reason="no %s path for site %r" % (context.platform, site)) + self._declare_at(context, entry, site, path) + if not context.driver.exists(path): + return self._outcome(entry, "failed", path=path, + reason="declared in %s but the file is not there" % path) + written.append(path) + + # M-SP-02 writes two files on purpose - a managed one and a user one - + # and the *managed* path is the one precedence says wins, so that is the + # path recorded. Recording the user path would make the scorer expect + # the wrong scope for an entry that exists to test scope. + return self._outcome(entry, "installed", path=written[0], method="config", + extra={"paths": written} if len(written) > 1 else {}) + + def _declare_at(self, context: Any, entry: Any, site: str, path: str) -> None: + shape, keys = SITE_SHAPES[site] + document: Dict[str, Any] = {} + for key in reversed(keys): + document = {key: {entry.declare["server_name"]: self._server(context, entry, site)}} \ + if not document else {key: document} + privileged = bool(entry.privileged) or site.startswith("managed-") + if shape == "json": + existing = self._read_json(context, path) + content = writers.as_json(writers.merge(existing, document)) + elif shape == "toml": + content = writers.as_toml(document) + else: + content = writers.as_yaml(document) + context.driver.write(path, content, privileged=privileged) + + def _read_json(self, context: Any, path: str) -> Dict[str, Any]: + """Merge rather than overwrite: nine M-PIN rows share ~/.claude.json. + + A recipe that wrote the file whole would leave one server declared and + eight missing, and the run would report eight misses that never + happened. + """ + if not context.driver.exists(path): + return {} + local = context.scratch_file(path) + context.driver.pull(path, local) + try: + with open(local, encoding="utf-8") as handle: + return json.load(handle) + except (OSError, ValueError): + return {} + + def _server(self, context: Any, entry: Any, site: str) -> Dict[str, Any]: + """The server block, with canaries substituted for their real values.""" + declared = entry.declare + block: Dict[str, Any] = {} + if declared.get("url"): + block["url"] = declared["url"] + block["type"] = declared.get("transport", "sse") + else: + block["command"] = declared["command"] + block["args"] = [context.substitute(arg) for arg in declared.get("args", [])] + for key in ("env", "headers"): + if declared.get(key): + block[key] = {name: context.substitute(value) + for name, value in declared[key].items()} + return block diff --git a/Discovery/tests/install/recipes/npm_global.py b/Discovery/tests/install/recipes/npm_global.py new file mode 100644 index 0000000..1ed14d0 --- /dev/null +++ b/Discovery/tests/install/recipes/npm_global.py @@ -0,0 +1,55 @@ +"""``npm i -g`` - nine CLI agents, one command, one of the cheapest families.""" + +from typing import Any + +from .base import Outcome, Recipe + + +class NpmGlobalRecipe(Recipe): + family = "npm-global" + + def execute(self, context: Any, entry: Any) -> Outcome: + if entry.install.get("unresolved"): + return self._outcome(entry, "unimplemented", + reason="source unresolved: %s" % entry.install["unresolved"]) + package = entry.install["package"] + version = entry.install["version"] + installed = context.driver.run(["npm", "install", "-g", "%s@%s" % (package, version)], + timeout=600) + if not installed.ok: + return self._outcome(entry, "failed", version=version, + reason="npm install failed: %s" % installed.stderr.strip()[:200]) + + binary = entry.install.get("binary") or package.rsplit("/", 1)[-1] + located = context.driver.run(["command", "-v", binary]) + if not located.ok or not located.text(): + # Installed but not on PATH is a harness failure, not a detection + # one: the collector cannot be asked about a binary the shell itself + # cannot find. + return self._outcome(entry, "failed", version=version, + reason="%s not on PATH after install" % binary) + + # The canonical path, because usr-merge and version-manager shims mean + # the first spelling on PATH is often not the file itself - and the + # scorer compares the collector's `install_path` against this one. + path = context.driver.realpath(located.text()) + return self._outcome(entry, "installed", version=self._version(context, entry, version), + path=path, method="npm") + + def _version(self, context: Any, entry: Any, pinned: str) -> str: + """What the machine says, falling back to what we asked for. + + A tool that prints its version differently from the version we pinned is + worth recording as the tool prints it: the collector reads the machine, + not the manifest, and scoring it against the manifest would be scoring + it against something it never saw. + """ + if not entry.verify: + return pinned + result = context.driver.shell(entry.verify) + if not result.ok or not result.text(): + return pinned + for token in result.text().replace("v", " ").split(): + if token[:1].isdigit(): + return token + return pinned