diff --git a/Discovery/tests/report/__init__.py b/Discovery/tests/report/__init__.py new file mode 100644 index 0000000..8d680f4 --- /dev/null +++ b/Discovery/tests/report/__init__.py @@ -0,0 +1,5 @@ +"""Turning a score into something a person can act on.""" + +from .html import render, write + +__all__ = ["render", "write"] diff --git a/Discovery/tests/report/html.py b/Discovery/tests/report/html.py new file mode 100644 index 0000000..5e53a4d --- /dev/null +++ b/Discovery/tests/report/html.py @@ -0,0 +1,249 @@ +"""The scorecard: one page per run. + +The aggregate number is for tracking. The individual rows are what somebody +fixes, so every false positive and every false negative is listed by manifest +id with the evidence the collector recorded - because the snapshot names the +probe, the channel and the path that produced each asset, and a reader should +not have to open two JSON files side by side to find that out. + +Self-contained by construction: one file, no external assets, no network. A +scorecard that needed a CDN would be unreadable on the isolated host that +produced it. +""" + +import html +import json +import os +from typing import Any, Dict, List, Optional + +CATEGORY_LABELS = { + "cli_agent": "AI tools - CLI agents", + "app": "AI tools - apps & browsers", + "extension": "AI tools - extensions", + "model_runtime": "AI tools - model runtimes", + "channel_variant": "AI tools - install variants", + "mcp_server": "MCP servers", + "artifact": "Skills & programmable surface", + "agent": "Agents", + "negative_control": "Negative controls", +} + +_STYLE = """ +:root { color-scheme: light dark; + --bg:#fbfbfa; --fg:#1a1a19; --muted:#6b6b68; --line:#e3e3e0; --card:#fff; + --ok:#177245; --bad:#a8200d; --warn:#8a6100; } +@media (prefers-color-scheme: dark) { :root { + --bg:#151514; --fg:#eeeeec; --muted:#9a9a95; --line:#2e2e2b; --card:#1d1d1b; + --ok:#5bbd88; --bad:#f08a72; --warn:#d9ab54; } } +* { box-sizing: border-box; } +body { margin:0; padding:2.5rem 1.5rem; background:var(--bg); color:var(--fg); + font:15px/1.55 ui-sans-serif,-apple-system,"Segoe UI",Roboto,sans-serif; } +main { max-width: 60rem; margin: 0 auto; } +h1 { font-size:1.6rem; margin:0 0 .25rem; letter-spacing:-.01em; } +h2 { font-size:1.05rem; margin:2.5rem 0 .75rem; letter-spacing:-.01em; } +.sub { color:var(--muted); margin:0 0 2rem; font-size:.9rem; } +.verdict { display:inline-block; padding:.15rem .6rem; border-radius:100px; + font-size:.8rem; font-weight:600; letter-spacing:.02em; } +.pass { background:color-mix(in srgb, var(--ok) 15%, transparent); color:var(--ok); } +.fail { background:color-mix(in srgb, var(--bad) 15%, transparent); color:var(--bad); } +.grid { display:grid; grid-template-columns:repeat(auto-fit,minmax(8rem,1fr)); gap:.75rem; } +.tile { background:var(--card); border:1px solid var(--line); border-radius:.5rem; padding:.85rem 1rem; } +.tile .n { font-size:1.5rem; font-weight:600; font-variant-numeric:tabular-nums; } +.tile .k { color:var(--muted); font-size:.78rem; text-transform:uppercase; letter-spacing:.06em; } +.scroll { overflow-x:auto; } +table { border-collapse:collapse; width:100%; font-size:.88rem; } +th,td { text-align:left; padding:.45rem .6rem; border-bottom:1px solid var(--line); white-space:nowrap; } +th { color:var(--muted); font-weight:500; font-size:.78rem; text-transform:uppercase; letter-spacing:.05em; } +td.n, th.n { text-align:right; font-variant-numeric:tabular-nums; } +code { font:12.5px/1.4 ui-monospace,SFMono-Regular,Menlo,monospace; color:var(--muted); white-space:pre-wrap; } +.id { font:12.5px/1.4 ui-monospace,SFMono-Regular,Menlo,monospace; font-weight:600; } +.bad { color:var(--bad); } .ok { color:var(--ok); } .warn { color:var(--warn); } +.empty { color:var(--muted); font-style:italic; } +""" + + +def render(score: Dict[str, Any]) -> str: + """One HTML page from one ``score.json``.""" + run = score.get("run", {}) + gate = score.get("gate", {}) + parts: List[str] = [ + "", + "", + "ADR Discovery e2e - %s" % _e(run.get("id") or run.get("os", "")), + "
" % _STYLE, + "

ADR Discovery - end-to-end fidelity

", + "

%s · image %s · collector %s" + " · catalog %s

" % ( + _e(run.get("id", "")), _e(run.get("image", "")), _e(run.get("collector", "")), + _e(str(run.get("catalog_version", "")))), + _verdict(gate), + _totals(score), + _denominator(score), + _categories(score), + _fields(score), + _rows("Misses - installed, not reported", score.get("misses", []), _miss_row, + ("id", "name", "category", "expected")), + _rows("Inventions - reported, not installed", score.get("inventions", []), _invention_row, + ("attributed to", "name", "kind", "why it is wrong", "evidence")), + _rows("Duplicates - installed once, reported more than once", + score.get("duplicates", []), _duplicate_row, ("id", "name", "count", "assets")), + _rows("Excluded - not in the denominator", score.get("excluded", []), _excluded_row, + ("id", "name", "reason")), + _crosscutting(score), + "
", + ] + return "\n".join(parts) + + +def write(score: Dict[str, Any], run_dir: str, filename: str = "report.html") -> str: + path = os.path.join(run_dir, filename) + with open(path, "w", encoding="utf-8") as handle: + handle.write(render(score)) + return path + + +# -- sections ---------------------------------------------------------- + + +def _verdict(gate: Dict[str, Any]) -> str: + if gate.get("passed"): + return "

gate passed

" + reasons = ", ".join(_e(reason) for reason in gate.get("reasons", [])) + return "

gate failed %s

" % reasons + + +def _totals(score: Dict[str, Any]) -> str: + totals = score.get("totals", {}) + tiles = [("TP", totals.get("tp"), ""), ("FP", totals.get("fp"), "bad" if totals.get("fp") else ""), + ("FN", totals.get("fn"), "bad" if totals.get("fn") else ""), + ("DUP", totals.get("dup"), "bad" if totals.get("dup") else ""), + ("recall", _ratio(totals.get("recall")), ""), + ("precision", _ratio(totals.get("precision")), "")] + cells = "".join("
%s
%s
" + % (klass, _e(str(value)), _e(label)) for label, value, klass in tiles) + return "
%s
" % cells + + +def _denominator(score: Dict[str, Any]) -> str: + manifest = score.get("manifest", {}) + baseline = score.get("baseline", {}) + rows = [("applicable", manifest.get("applicable"), ""), + ("installed", manifest.get("installed"), ""), + ("unavailable", manifest.get("unavailable"), ""), + ("failed", manifest.get("failed"), "bad" if manifest.get("failed") else ""), + ("unimplemented", manifest.get("unimplemented"), "warn" if manifest.get("unimplemented") else ""), + ("baseline assets", baseline.get("asset_count"), + "ok" if baseline.get("clean") else "bad")] + body = "".join("%s%s" % (_e(k), c, _e(str(v))) + for k, v, c in rows) + return ("

What was in play

" + "%s
stateentries
" % body) + + +def _categories(score: Dict[str, Any]) -> str: + head = ("categoryTPFP" + "FNDUPrecall" + "precision") + rows = [] + for key, values in score.get("by_category", {}).items(): + rows.append("%s%s%s" + "%s%s" + "%s%s" % ( + _e(CATEGORY_LABELS.get(key, key)), values.get("tp", 0), + "bad" if values.get("fp") else "", values.get("fp", 0), + "bad" if values.get("fn") else "", values.get("fn", 0), + "bad" if values.get("dup") else "", values.get("dup", 0), + _ratio(values.get("recall")), _ratio(values.get("precision")))) + return "

Per category

%s%s
" % (head, "".join(rows)) + + +def _fields(score: Dict[str, Any]) -> str: + """Per field, never blended: an average hides which fact is wrong.""" + fields = score.get("fields", {}) + if not fields: + return "" + rows = [] + for name, tally in fields.items(): + wrong = tally.get("wrong", []) + detail = ", ".join("%s (%s → %s)" % (_e(item["id"]), _e(str(item["expected"])), + _e(str(item["observed"]))) for item in wrong[:6]) + rows.append("%s%s%s" + "%s%s" % ( + _e(name), tally.get("checked"), tally.get("correct"), + "bad" if tally.get("accuracy", 1) < 1 else "ok", + _ratio(tally.get("accuracy")), detail or "—")) + return ("

Field accuracy, over true positives

" + "" + "%s
fieldcheckedcorrectaccuracywrong
" % "".join(rows)) + + +def _rows(title: str, rows: List[Dict[str, Any]], formatter: Any, headers: Any) -> str: + if not rows: + return "

%s

none

" % _e(title) + head = "".join("%s" % _e(header) for header in headers) + body = "".join(formatter(row) for row in rows) + return ("

%s (%d)

" + "%s%s
" % (_e(title), len(rows), head, body)) + + +def _miss_row(row: Dict[str, Any]) -> str: + return "%s%s%s%s" % ( + _e(row.get("id", "")), _e(row.get("name", "")), _e(row.get("category", "")), + _e(json.dumps(row.get("expected", {})))) + + +def _invention_row(row: Dict[str, Any]) -> str: + """``matched_on`` is the diagnostic half of an evidence line. + + ``probe/channel path`` says where the collector looked; ``matched_on`` says + which rule fired there, which is what turns a false positive into a + one-line bug report against a specific matcher. + """ + evidence = "; ".join("%s/%s %s [%s]" % (item.get("probe"), item.get("channel"), + item.get("path"), item.get("matched_on")) + for item in row.get("evidence", [])[:3]) + attributed = row.get("attributed_to") + cell = ("%s" % _e(attributed) if attributed + else "unattributed") + return "%s%s%s%s%s" % ( + cell, _e(row.get("name", "")), _e(row.get("kind", "")), + _e(row.get("why_wrong", "")), _e(evidence)) + + +def _duplicate_row(row: Dict[str, Any]) -> str: + paths = "; ".join("%s (%s)" % (asset.get("install_path"), asset.get("install_method")) + for asset in row.get("assets", [])) + return "%s%s%s%s" % ( + _e(row.get("id", "")), _e(row.get("name", "")), row.get("count", 0), _e(paths)) + + +def _excluded_row(row: Dict[str, Any]) -> str: + return "%s%s%s" % ( + _e(row.get("id", "")), _e(row.get("name", "")), _e(row.get("reason", ""))) + + +def _crosscutting(score: Dict[str, Any]) -> str: + canaries = score.get("canaries", {}) + errors = score.get("errors", {}) + queue = score.get("review_queue", {}) + rows = [ + ("canaries", "%d planted, %d leaked" % (canaries.get("planted", 0), canaries.get("leaked", 0)), + "ok" if canaries.get("clean") else "bad"), + ("errors", "%d recorded, %d unexplained" % (errors.get("count", 0), errors.get("unexplained", 0)), + "ok" if not errors.get("unexplained") else "bad"), + ("review queue", "%d of %d expected entries queued, %d items total" + % (queue.get("queued", 0), queue.get("expected", 0), queue.get("size", 0)), + "ok" if queue.get("passed") else "bad"), + ] + body = "".join("%s%s" % (_e(k), c, _e(v)) + for k, v, c in rows) + return ("

Cross-cutting checks

" + "%s
checkresult
" % body) + + +def _ratio(value: Optional[float]) -> str: + return "—" if value is None else ("%.4g" % value) + + +def _e(value: Any) -> str: + return html.escape(str(value), quote=True) diff --git a/Discovery/tests/test_report.py b/Discovery/tests/test_report.py new file mode 100644 index 0000000..50c6a49 --- /dev/null +++ b/Discovery/tests/test_report.py @@ -0,0 +1,78 @@ +"""The scorecard. Tested for the two things a report can get wrong: leaving out +the row somebody needs, and rendering something it was handed as markup.""" + +import json +import os +import unittest + +from . import manifest as manifest_module +from .report import html +from .scoring import score_run + +RECORDED = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "recorded", "synthetic-linux") + + +class Scorecard(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.score = score_run(RECORDED, manifest_module.load()) + cls.page = html.render(cls.score) + + def test_every_miss_is_listed_by_id(self): + """The aggregate number is for tracking; the individual rows are what + somebody fixes.""" + for row in self.score["misses"]: + self.assertIn(row["id"], self.page) + + def test_every_invention_carries_the_evidence_that_produced_it(self): + """A miss can be localized to a probe from the output alone, and the + report should surface that rather than making a reader open two JSON + files side by side.""" + self.assertIn("filesystem", self.page) + self.assertIn("path-contains:mcp", self.page) + + def test_the_gate_verdict_is_visible_without_reading_a_table(self): + self.assertIn("gate failed", self.page) + self.assertIn("duplicates", self.page) + + def test_the_page_is_self_contained(self): + """A scorecard that needed a CDN would be unreadable on the isolated + host that produced it.""" + for external in ("http://", "https://", ""}, + "totals": {"tp": 0, "fp": 0, "fn": 0, "dup": 0}, + "gate": {"passed": True, "reasons": []}}) + self.assertNotIn("none"), 4) + + def test_writing_leaves_a_file_next_to_the_run(self): + import tempfile + with tempfile.TemporaryDirectory() as directory: + path = html.write(self.score, directory) + self.assertTrue(os.path.exists(path)) + with open(path, encoding="utf-8") as handle: + self.assertIn("ADR Discovery", handle.read()) + + def test_no_canary_value_reaches_the_page(self): + """The report is a document that gets shared. A value redacted out of + the snapshot and copied into the scorecard has still left the machine.""" + with open(os.path.join(RECORDED, "canaries.json"), encoding="utf-8") as handle: + planted = json.load(handle) + for name, value in planted.items(): + self.assertNotIn(value, self.page, name) + + +if __name__ == "__main__": + unittest.main()