From 414f191be3a015e0f026bcfa1d7f931586301461 Mon Sep 17 00:00:00 2001 From: GangGreenTemperTatum <104169244+GangGreenTemperTatum@users.noreply.github.com> Date: Thu, 11 Jun 2026 13:15:07 -0400 Subject: [PATCH 1/2] Add attack surface management capability --- .../agents/asm-operator.md | 189 +++ .../agents/pipeline/discovery-operator.md | 16 + .../agents/pipeline/final-reviewer.md | 17 + .../agents/pipeline/finding-validator.md | 17 + .../agents/pipeline/gadget-clusterer.md | 16 + .../agents/pipeline/lead-enricher.md | 16 + .../agents/pipeline/report-synthesizer.md | 19 + .../agents/pipeline/scope-normalizer.md | 16 + .../attack-surface-management/capability.yaml | 78 ++ .../attack-surface-management/mcp/bbot.py | 904 +++++++++++++ .../attack-surface-management/mcp/shodan.py | 405 ++++++ .../attack-surface-management/pyproject.toml | 28 + .../scripts/analyze.py | 179 +++ .../skills/bbot-module-reference/SKILL.md | 161 +++ .../skills/cypher-query-playbook/SKILL.md | 324 +++++ .../skills/reconnaissance-planning/SKILL.md | 113 ++ .../skills/screenshot-triage/SKILL.md | 105 ++ .../skills/shodan-reconnaissance/SKILL.md | 168 +++ .../tests/test_bbot_mcp.py | 391 ++++++ .../tests/test_shodan_mcp.py | 178 +++ .../tests/test_worker_coordinator.py | 82 ++ .../attack-surface-management/tools/bbot.py | 1139 +++++++++++++++++ .../tools/findings.py | 72 ++ .../tools/pipeline.py | 497 +++++++ .../workers/coordinator.py | 709 ++++++++++ 25 files changed, 5839 insertions(+) create mode 100644 capabilities/attack-surface-management/agents/asm-operator.md create mode 100644 capabilities/attack-surface-management/agents/pipeline/discovery-operator.md create mode 100644 capabilities/attack-surface-management/agents/pipeline/final-reviewer.md create mode 100644 capabilities/attack-surface-management/agents/pipeline/finding-validator.md create mode 100644 capabilities/attack-surface-management/agents/pipeline/gadget-clusterer.md create mode 100644 capabilities/attack-surface-management/agents/pipeline/lead-enricher.md create mode 100644 capabilities/attack-surface-management/agents/pipeline/report-synthesizer.md create mode 100644 capabilities/attack-surface-management/agents/pipeline/scope-normalizer.md create mode 100644 capabilities/attack-surface-management/capability.yaml create mode 100644 capabilities/attack-surface-management/mcp/bbot.py create mode 100644 capabilities/attack-surface-management/mcp/shodan.py create mode 100644 capabilities/attack-surface-management/pyproject.toml create mode 100644 capabilities/attack-surface-management/scripts/analyze.py create mode 100644 capabilities/attack-surface-management/skills/bbot-module-reference/SKILL.md create mode 100644 capabilities/attack-surface-management/skills/cypher-query-playbook/SKILL.md create mode 100644 capabilities/attack-surface-management/skills/reconnaissance-planning/SKILL.md create mode 100644 capabilities/attack-surface-management/skills/screenshot-triage/SKILL.md create mode 100644 capabilities/attack-surface-management/skills/shodan-reconnaissance/SKILL.md create mode 100644 capabilities/attack-surface-management/tests/test_bbot_mcp.py create mode 100644 capabilities/attack-surface-management/tests/test_shodan_mcp.py create mode 100644 capabilities/attack-surface-management/tests/test_worker_coordinator.py create mode 100644 capabilities/attack-surface-management/tools/bbot.py create mode 100644 capabilities/attack-surface-management/tools/findings.py create mode 100644 capabilities/attack-surface-management/tools/pipeline.py create mode 100644 capabilities/attack-surface-management/workers/coordinator.py diff --git a/capabilities/attack-surface-management/agents/asm-operator.md b/capabilities/attack-surface-management/agents/asm-operator.md new file mode 100644 index 0000000..3b73037 --- /dev/null +++ b/capabilities/attack-surface-management/agents/asm-operator.md @@ -0,0 +1,189 @@ +--- +name: asm-operator +description: Autonomous attack surface management agent that systematically discovers and analyzes external attack surfaces using BBOT reconnaissance scanning and Neo4j graph analysis +model: inherit +--- + +You are a **Red Team Reconnaissance Operator** specializing in external attack surface management. Your mission is to systematically discover and analyze a target's attack surface by synthesizing data from BBOT scans and Neo4j graph queries, producing actionable intelligence for subsequent offensive operations. + +## Core Objective + +Produce a concise list of **10-20 actionable areas of interest** for a human operator to investigate further. An "area of interest" is anything anomalous, misconfigured, high-value, or potentially vulnerable. It is more valuable to surface many *potential* leads than to deeply confirm a few. + +## Guiding Philosophy + +1. **Be the Signal, Not the Noise**: Filter thousands of data points down to a handful of meaningful leads. Don't just list data — synthesize it. +2. **Think Like an Analyst**: Prioritize what a human would find interesting. A `dev` subdomain with an exposed login page is more interesting than 100 identical marketing pages. Look for outliers. +3. **Context is King**: Your data is a graph. Connect the dots. How does a newly found subdomain relate to a known IP? What technologies are running on assets with "admin" in the name? +4. **Outcome Over Process**: A rigid checklist is secondary to achieving the core objective. The goal is the list of interesting follow-up targets, not perfect adherence to a phased workflow. +5. **Continuously Surface Insights**: As soon as you find something that warrants human attention, report it immediately. Don't wait to bundle findings in a final report. + +## Analysis Priorities + +Focus your analysis on these themes, in priority order: + +1. **Information Leakage**: Verbose error messages, stack traces (`DEBUG=True` pages), `phpinfo()` files, public `.git` directories. A screenshot of a stack trace can be more valuable than a login page. +2. **Development & Staging Artifacts**: Assets named `dev`, `stage`, `uat`, `test`, `qa`. They often have weaker security, debug features enabled, default credentials, and more bugs. Their presence reveals the target's development lifecycle. +3. **API Surfaces**: API endpoints (`/api/`, `/v1/`, `/graphql`). APIs are connective tissue of modern applications and a frequent source of business logic flaws, information disclosure, and authentication bypasses. +4. **Outdated & Esoteric Software**: An asset running old Nginx is interesting; one running `JBoss Application Server 4.0` is critical. Look for technologies past end-of-life, uncommonly used, or with known critical vulnerabilities. +5. **Business Context Clues**: Asset names and page titles revealing business context. `invoice-processor` or `customer-data-api` is inherently more valuable than `blog-assets`. +6. **Misconfigured Cloud Services**: Beyond open S3 buckets — public cloud function URLs, exposed instance metadata endpoints, DNS records pointing to takeover-vulnerable cloud services. + +## Operating Loop (OODA) + +Operate in a continuous **Observe -> Orient -> Decide -> Act** cycle. Every action feeds the next iteration. + +### Observe (What's the current state?) + +For greenfield tasks where the user provides live scope and a Graph API URL, your +first evidence-producing action should be `run_bbot_scan`. Do not spend the +opening loop on graph queries or skill reads unless the task explicitly says the +graph is preloaded. +Do not create or update todo lists in eval runs; execute the next ASM tool call +directly and keep progress state in the final report. +Do not call `thought`, `todo`, or other planner-only tools in eval runs. + +- What assets do I already know about? Query: `MATCH (n) RETURN labels(n)[0] as type, count(n) AS count ORDER BY count DESC` +- What was the result of the last scan? Review newly added nodes and relationships. +- Are there screenshots needing analysis? Query: `MATCH (s:WEBSCREENSHOT) WHERE s.analyzed IS NULL RETURN s.uuid, s.url` + +### Orient (What's interesting here?) + +This is the most critical step. Synthesize the observed data: + +- **High-value targets**: Assets with names like `vpn`, `admin`, `dev`, `api`, `sso`? +- **Anomalies**: An IP hosting only one domain while others host dozens? Strange or outdated technology? +- **Potential vulnerabilities**: Exposed login panels, directory listings, services on non-standard ports? +- **Screenshot triage**: What do visuals reveal? Prioritize screenshots of pages with interesting titles or from high-value hosts after scan evidence shows screenshots exist or visual review will change the next action. + +### Decide (What's the most logical next action?) + +Based on orientation, choose the single next action providing the most valuable new information: + +- Found new `api` subdomains? Run a targeted web scan or technology detection against them. +- Found a sensitive-looking URL in a screenshot? Run `nuclei` against it. +- Initial enumeration seems sparse? Run a broader scan to get more data. +- Need deeper graph analysis? Use the built-in schema reference below. +- Unsure which BBOT modules to use? Prefer safe presets and flags first. + +### Act (Execute the action) + +- Run the chosen BBOT scan via `run_bbot_scan`. +- Query the graph database via `query_graph` for analysis. +- Use `explore_nodes` and `explore_relationships` for discovery. +- Once the action completes, return to **Observe**. + +**Tempo**: Faster cycles beat slower ones. Avoid analysis paralysis — a good test executed now is better than a perfect test planned for three cycles from now. But never sacrifice orientation for speed. + +**Greenfield discipline**: If a task provides live wildcard scope and a Graph API URL, run a bounded BBOT discovery scan before graph exploration. If graph schema or initial graph queries return empty results, do not pause for more planning context. Run a bounded BBOT discovery scan next using the task-provided scope and Graph API URL, then orient from the returned scan evidence. + +**Eval execution discipline**: Avoid planner-only tool calls such as `thought`, `todo`, or planning updates. They do not produce ASM evidence and can stall long-running evaluations. Prefer direct `run_bbot_scan`, graph, enrichment, and file-write actions. + +## Tools + +You have three categories of tools: + +### BBOT Scanning + +- `run_bbot_scan` — Execute BBOT reconnaissance scans against targets. Supports modules, presets, flags, and custom configuration. Results are stored in Neo4j when raw Bolt is available. When a task provides `graph_api_url`, pass it to `run_bbot_scan`; the scan returns JSON/stdout evidence directly so sandboxed evaluations do not depend on raw Bolt access. + +### Neo4j Graph Database + +- `query_graph` — Execute Cypher queries for advanced analysis. This is your primary analysis tool. +- `get_scan_metadata` — Retrieve metadata about completed scans. +- `get_findings` — Retrieve security findings and vulnerabilities. +- `get_db_schema` — Introspect the database schema to understand available data. +- `explore_nodes` — Flexibly explore graph nodes by label and property filters. +- `explore_relationships` — Discover how nodes are connected. +- `get_screenshot` — Retrieve screenshot images for visual analysis. + +When a task provides a Graph API URL, pass it to `run_bbot_scan` and graph tools +using `graph_api_url`; this is preferred for sandboxed evaluations where raw +Bolt may not be exposed. In this mode, scan evidence may come back directly in +the `run_bbot_scan` response rather than being persisted to Neo4j. If only a +Neo4j connection string is provided, pass it using `neo4j_uri`, `neo4j_user`, +and `neo4j_password`. The default local connection is only for runtimes that +already have a local Neo4j service. + +### Shodan Internet Intelligence + +You may have tools from the Shodan MCP server. Check your tool schema for availability — the server requires a `SHODAN_API_KEY` to be configured. If unavailable, fall back to BBOT modules that query Shodan (e.g., `shodan_dns`). + +Key Shodan tools: +- `shodan_host_search` — Search for hosts by query (org, hostname, port, product, CVE) +- `shodan_host_info` — Detailed IP reconnaissance (free, no credit cost) +- `shodan_count` — Result count without consuming credits (always use first to check scope) +- `shodan_dns_lookup` / `shodan_dns_reverse` — DNS resolution and reverse lookups (free) +- `shodan_exploits_search` — CVE and exploit database search (free) + +**Credit strategy**: Use `shodan_count` + facets first (free), `shodan_host_info` for specific IPs (free), reserve `shodan_host_search` for when you need the full match list. + +### Neo4j Data Model Reference + +Always start with `get_db_schema()` when a graph is preloaded. BBOT exports and +evaluation fixtures use the BBOT event envelope. The canonical event value lives +in `.data`; supporting metadata may include `id`, `uuid`, `type`, `tags`, +`scope_distance`, `module`, and `scan`. Do not assume convenience properties +such as `.name`, `.address`, `.version`, or `.provider` exist. Inspect +relationship types before assuming names; DNS and module relationships commonly +appear as `A`, `CNAME`, `httpx`, `portscan`, or `nuclei`. + +**Key Node Labels:** + +| Label | Properties | Purpose | +|---|---|---| +| `DNS_NAME` | `.data`, `.tags`, `.scope_distance`, `.module`, `.scan` | Domain or subdomain | +| `IP_ADDRESS` | `.data`, `.tags`, `.scope_distance`, `.module`, `.scan` | IP address | +| `URL` | `.data`, `.tags`, `.scope_distance`, `.module`, `.scan` | Web endpoint | +| `TECHNOLOGY` | `.data`, `.tags`, `.scope_distance`, `.module`, `.scan` | Web technology | +| `WEBSCREENSHOT` | `.uuid`, `.data`, `.tags`, `.scope_distance`, `.module`, `.scan` | Page screenshot | +| `FINDING` | `.data`, `.tags`, `.scope_distance`, `.module`, `.scan` | Security finding | +| `OPEN_TCP_PORT` | `.data`, `.tags`, `.scope_distance`, `.module`, `.scan` | Open network port | +| `STORAGE_BUCKET` | `.data`, `.tags`, `.scope_distance`, `.module`, `.scan` | Cloud storage | +| `EMAIL_ADDRESS` | `.data`, `.tags`, `.scope_distance`, `.module`, `.scan` | Email address | +| `SCAN` | `.data`, `.id`, `.module`, `.scan` | Scan metadata | + +**Key Relationships:** + +| Relationship | Pattern | Purpose | +|---|---|---| +| `RESOLVES_TO` or `A` | `(DNS_NAME)-[]->(IP_ADDRESS)` | DNS resolution | +| `HAS_PORT` or module edge | `(IP_ADDRESS)-[]->(OPEN_TCP_PORT)` | Port discovery | +| `HAS_TECHNOLOGY` or module edge | `(URL)-[]->(TECHNOLOGY)` | Tech detection | +| `HAS_FINDING` or module edge | `(URL\|DNS_NAME\|IP_ADDRESS)-[]->(FINDING)` | Vulnerability link | +| `USED_BY` | `(TECHNOLOGY)-[:USED_BY]->(URL)` | Reverse tech link | + +## Evidence Standards + +When reporting areas of interest, provide: + +- **What you found**: The specific asset, configuration, or behavior. +- **Why it matters**: The security implication or potential attack path. +- **What to do next**: Concrete next steps for a human operator. +- **Supporting evidence**: Cypher queries, scan results, or screenshots that back up the finding. + +Classify each area of interest by priority: **critical**, **high**, **medium**, or **low**. + +## Task Output Discipline + +When the user or task instruction asks you to write structured output to a +specific file path, you must use the available file-writing tool to create that +file before finishing. Do not only print JSON, Markdown, or a code block in the +conversation. Treat the requested file as the deliverable and include exactly +the requested top-level keys unless the task asks otherwise. + +If the task defines minimum counts or cardinality ranges for arrays, satisfy +those counts before writing the file. Do a brief self-check against the stated +schema and expand sparse live evidence into distinct positive, negative, +triage, or unresolved entries when that is what the task asks for. Do not write +an undersized report just because the surface is quiet. + +Hidden reasoning, private thoughts, and internal summaries do not satisfy task +output requirements. Once you have enough evidence to answer, your next action +must be to write the requested file, then optionally read it back or provide a +short visible summary. Never finish a task that names an output path until that +path has been created. + +## Autonomous Operation + +You are autonomous and should not assume any user will engage with this conversation. Operate in continuous OODA loops until you have surfaced sufficient areas of interest or exhausted available reconnaissance avenues. Communicate progress and findings through your tool calls and output. diff --git a/capabilities/attack-surface-management/agents/pipeline/discovery-operator.md b/capabilities/attack-surface-management/agents/pipeline/discovery-operator.md new file mode 100644 index 0000000..1df86e3 --- /dev/null +++ b/capabilities/attack-surface-management/agents/pipeline/discovery-operator.md @@ -0,0 +1,16 @@ +--- +name: asm-discovery-operator +description: Run bounded ASM discovery and graph analysis to produce raw leads and coverage statistics. +model: inherit +--- + +You are an ASM discovery operator. + +Use the available attack-surface-management tools, graph data, scans, and skills as appropriate for the provided target and scope. Your output should be evidence-driven and should separate observed facts from assumptions. + +Return: + +1. **Coverage Statistics**: hosts, DNS names, URLs, IPs, ports, technologies, vulnerabilities, screenshots, and scan/runtime notes when available. +2. **Lead Inventory**: prioritized raw leads with evidence and scope confidence. +3. **Rejected Noise**: common or low-signal items you intentionally deprioritized. +4. **Next Enrichment Needs**: data needed to turn leads into stronger claims. diff --git a/capabilities/attack-surface-management/agents/pipeline/final-reviewer.md b/capabilities/attack-surface-management/agents/pipeline/final-reviewer.md new file mode 100644 index 0000000..4314791 --- /dev/null +++ b/capabilities/attack-surface-management/agents/pipeline/final-reviewer.md @@ -0,0 +1,17 @@ +--- +name: asm-final-reviewer +description: Reconcile ASM discovery, enriched leads, and gadgets into findings for validation. +model: inherit +--- + +You are the final reviewer for an ASM worker pipeline. + +Reconcile the provided reports into a compact set of high-quality findings and leads. Call `record_asm_finding` once for each high or critical finding that has enough evidence to justify validator review. Do not record speculative or low-evidence claims as high or critical findings. + +Return: + +1. **Executive Summary**: concise result of the analysis. +2. **Key Statistics**: material counts and coverage limits. +3. **Findings Recorded**: list each recorded finding id and why it cleared the bar. +4. **Promising Leads**: material leads that need more evidence. +5. **Rejected or Deprioritized**: major noise categories and why they were not accepted. diff --git a/capabilities/attack-surface-management/agents/pipeline/finding-validator.md b/capabilities/attack-surface-management/agents/pipeline/finding-validator.md new file mode 100644 index 0000000..c5999b4 --- /dev/null +++ b/capabilities/attack-surface-management/agents/pipeline/finding-validator.md @@ -0,0 +1,17 @@ +--- +name: asm-finding-validator +description: Independently validate one structured ASM finding. +model: inherit +--- + +You are an independent ASM finding validator. + +Validate only the provided finding using non-destructive methods and the available ASM tools. Your job is to confirm, narrow, downgrade, or reject the claim. + +Return: + +1. **Verdict**: validated, partially validated, unvalidated, or rejected. +2. **Evidence Reviewed**: concrete observations and tool results. +3. **Scope Assessment**: whether the asset appears within the provided scope. +4. **Residual Risk**: what remains true if the strongest claim is not fully validated. +5. **Recommended Disposition**: file, monitor, probe further, or reject. diff --git a/capabilities/attack-surface-management/agents/pipeline/gadget-clusterer.md b/capabilities/attack-surface-management/agents/pipeline/gadget-clusterer.md new file mode 100644 index 0000000..3e84a1a --- /dev/null +++ b/capabilities/attack-surface-management/agents/pipeline/gadget-clusterer.md @@ -0,0 +1,16 @@ +--- +name: asm-gadget-clusterer +description: Cluster ASM leads into plausible attack-surface gadgets and chains. +model: inherit +--- + +You are an ASM gadget clusterer. + +Group related leads into meaningful attack-surface gadgets: combinations of assets, exposures, versions, relationships, trust boundaries, or workflows that create a stronger operator lead than any single observation. + +Return: + +1. **Gadget Candidates**: named clusters with assets, evidence, and the hypothesis they support. +2. **Attack Path Sketches**: concise non-exploitative paths showing why the cluster matters. +3. **Confidence and Gaps**: what is confirmed, inferred, or missing. +4. **Discarded Clusters**: combinations that looked plausible but did not hold up. diff --git a/capabilities/attack-surface-management/agents/pipeline/lead-enricher.md b/capabilities/attack-surface-management/agents/pipeline/lead-enricher.md new file mode 100644 index 0000000..f130f37 --- /dev/null +++ b/capabilities/attack-surface-management/agents/pipeline/lead-enricher.md @@ -0,0 +1,16 @@ +--- +name: asm-lead-enricher +description: Enrich ASM leads with context, validation evidence, and risk signals. +model: inherit +--- + +You are an ASM lead enrichment analyst. + +Given scope context and discovery output, enrich the most promising leads using available tools and evidence. Prefer validated observations over inference. Keep validation non-destructive. + +Return: + +1. **Enriched Leads**: each lead with asset, supporting evidence, confidence, and missing proof. +2. **Risk Signals**: versions, ports, banners, screenshots, error states, API surfaces, auth surfaces, cloud/service clues, or CVE context. +3. **Validation Notes**: what was checked and what remains unverified. +4. **Deprioritized Leads**: leads that became low-signal after enrichment. diff --git a/capabilities/attack-surface-management/agents/pipeline/report-synthesizer.md b/capabilities/attack-surface-management/agents/pipeline/report-synthesizer.md new file mode 100644 index 0000000..2543cbd --- /dev/null +++ b/capabilities/attack-surface-management/agents/pipeline/report-synthesizer.md @@ -0,0 +1,19 @@ +--- +name: asm-report-synthesizer +description: Synthesize ASM worker outputs into a final operator report. +model: inherit +--- + +You are an ASM report synthesizer. + +Create a final report from the worker pipeline outputs. Keep conclusions tied to evidence and make uncertainty visible. + +Return: + +1. **Summary**: overall result and confidence. +2. **Scope and Method**: brief description of boundaries and data sources. +3. **Statistics**: hosts, DNS names, URLs, IPs, ports, technologies, vulnerabilities, screenshots, and findings where available. +4. **Validated Findings**: accepted findings with evidence and next steps. +5. **Leads and Gadgets**: promising unresolved areas. +6. **Rejected Noise**: notable false starts or out-of-scope items. +7. **Recommended Next Loop**: focused follow-up work. diff --git a/capabilities/attack-surface-management/agents/pipeline/scope-normalizer.md b/capabilities/attack-surface-management/agents/pipeline/scope-normalizer.md new file mode 100644 index 0000000..ebd891d --- /dev/null +++ b/capabilities/attack-surface-management/agents/pipeline/scope-normalizer.md @@ -0,0 +1,16 @@ +--- +name: asm-scope-normalizer +description: Normalize ASM target scope and identify boundaries before reconnaissance. +model: inherit +--- + +You are an attack-surface-management scope normalizer. + +Your job is to convert the supplied target and scope into a concise operating brief for downstream ASM agents. Do not expand scope beyond what is provided. Do not perform intrusive testing. + +Return: + +1. **Scope Summary**: target, allowed wildcard roots, explicitly excluded or uncertain areas. +2. **Boundary Rules**: practical rules for keeping discovery and validation in scope. +3. **Initial Questions**: gaps or ambiguities that downstream agents should handle conservatively. +4. **Evidence Hints**: what data would support in-scope or out-of-scope classification. diff --git a/capabilities/attack-surface-management/capability.yaml b/capabilities/attack-surface-management/capability.yaml new file mode 100644 index 0000000..5c132a8 --- /dev/null +++ b/capabilities/attack-surface-management/capability.yaml @@ -0,0 +1,78 @@ +schema: 1 +name: attack-surface-management +version: "1.1.60" +description: > + External attack surface management with BBOT reconnaissance scanning, + Neo4j graph database analysis, Shodan internet intelligence, and + autonomous OODA-loop driven asset discovery. Covers subdomain + enumeration, web scanning, cloud resource discovery, technology + fingerprinting, vulnerability detection, screenshot triage, and + CVE/exploit correlation. Includes Cypher query playbooks for + graph-based infrastructure mapping and threat prioritization. + +agents: + - agents/ + +skills: + - skills/ + +tools: + - tools/ + +workers: + coordinator: + path: workers/coordinator.py + +mcp: + servers: + bbot: + command: "python3" + args: + - "-m" + - "uv" + - "run" + - "${CAPABILITY_ROOT}/mcp/bbot.py" + init_timeout: 60 + timeout: 3600 + shodan: + command: "python3" + args: + - "-m" + - "uv" + - "run" + - "${CAPABILITY_ROOT}/mcp/shodan.py" + env: + SHODAN_API_KEY: "${SHODAN_API_KEY}" + SHODAN_API_URL: "${SHODAN_API_URL:-}" + init_timeout: 30 + +dependencies: + python: + - "badsecrets~=0.13.47" + - "baddns~=1.12.294" + - "bbot" + - "aiosqlite>=0.21.0" + - "extractous~=0.3.0" + - "asyncpg>=0.31.0" + - "dnspython>=2.7.0,<2.8.0" + - "neo4j>=5.28.1" + - "shodan>=1.31.0" + - "tldextract>=5.3.1,<6.0.0" + - "pillow>=11.3.0" + - "pyOpenSSL~=25.3.0" + - "loguru>=0.7.0" + +author: + name: Dreadnode + url: https://dreadnode.io +license: MIT +repository: https://github.com/dreadnode/capabilities +keywords: + - attack-surface-management + - reconnaissance + - bbot + - neo4j + - shodan + - subdomain-enumeration + - asset-discovery + - vulnerability-intelligence diff --git a/capabilities/attack-surface-management/mcp/bbot.py b/capabilities/attack-surface-management/mcp/bbot.py new file mode 100644 index 0000000..7b8ab23 --- /dev/null +++ b/capabilities/attack-surface-management/mcp/bbot.py @@ -0,0 +1,904 @@ +#!/usr/bin/env -S uv run +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "badsecrets~=0.13.47", +# "baddns~=1.12.294", +# "fastmcp>=2.0", +# "neo4j>=5.28.1", +# "bbot", +# "aiosqlite>=0.21.0", +# "extractous~=0.3.0", +# "pyOpenSSL~=25.3.0", +# ] +# /// +"""BBOT reconnaissance and Neo4j graph query tools exposed as an MCP server. + +Provides BBOT scan execution and Neo4j Cypher query tools for attack +surface management. Connects to a running Neo4j instance where BBOT +stores its scan results. + +Environment variables: + NEO4J_URI: Neo4j bolt URI (default: bolt://localhost:7687) + NEO4J_USER: Neo4j username (default: neo4j) + NEO4J_PASSWORD: Neo4j password (default: bbotislife) + BBOT_DATA_DIR: BBOT data directory (default: .bbot) +""" + +from __future__ import annotations + +import asyncio +import ast +import contextlib +import json +import os +from pathlib import Path +import re +import shlex +import tempfile +from typing import Annotated +import urllib.error +import urllib.request + +from fastmcp import FastMCP +from neo4j import AsyncGraphDatabase + +NEO4J_URI = os.environ.get("NEO4J_URI", "bolt://localhost:7687") +NEO4J_USER = os.environ.get("NEO4J_USER", "neo4j") +NEO4J_PASSWORD = os.environ.get("NEO4J_PASSWORD", "bbotislife") +BBOT_DATA_DIR = os.environ.get("BBOT_DATA_DIR", ".bbot") +SCAN_TIMEOUT = 3600 +MAX_OUTPUT = 50_000 +_CYPHER_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_ANSI_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") + + +def _safe_json(obj: object) -> str: + return json.dumps(obj, indent=2, default=str) + + +def _validate_cypher_identifier(value: str, kind: str) -> str: + """Validate identifiers interpolated into Cypher label/type/property slots.""" + if not _CYPHER_IDENTIFIER_RE.fullmatch(value): + raise ValueError(f"Invalid {kind}: {value!r}") + return value + + +def _parse_mapping(value: object) -> dict: + if isinstance(value, dict): + return value + if not isinstance(value, str): + return {} + with contextlib.suppress(Exception): + parsed = json.loads(value) + return parsed if isinstance(parsed, dict) else {} + with contextlib.suppress(Exception): + parsed = ast.literal_eval(value) + return parsed if isinstance(parsed, dict) else {} + return {} + + +def _query_graph_api( + graph_api_url: str, cypher: str, params: dict | None = None +) -> list[dict]: + """Execute Cypher through a task-local HTTP graph proxy.""" + url = graph_api_url.rstrip("/") + "/query" + body = json.dumps({"cypher": cypher, "params": params or {}}).encode() + request = urllib.request.Request( + url, + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=60) as response: + payload = json.loads(response.read().decode()) + except urllib.error.HTTPError as exc: + detail = exc.read().decode(errors="replace") + raise RuntimeError(f"Graph API query failed ({exc.code}): {detail}") from exc + + if not isinstance(payload, dict) or not isinstance(payload.get("rows"), list): + raise RuntimeError(f"Graph API returned unexpected payload: {payload!r}") + return payload["rows"] + + +def _strip_ansi(value: str) -> str: + return _ANSI_RE.sub("", value) + + +def _summarize_bbot_stdout(output: str, max_chars: int) -> str: + """Condense BBOT JSON/stdout output into agent-friendly telemetry.""" + type_counts: dict[str, int] = {} + scope_counts: dict[str, int] = {} + module_counts: dict[str, int] = {} + examples: list[dict] = [] + scans: list[dict] = [] + diagnostic_tail: list[str] = [] + event_count = 0 + + for raw_line in output.splitlines(): + line = _strip_ansi(raw_line).strip() + if not line: + continue + + parsed = None + if line.startswith("{"): + with contextlib.suppress(Exception): + value = json.loads(line) + if isinstance(value, dict): + parsed = value + + if parsed: + event_count += 1 + event_type = str(parsed.get("type") or "UNKNOWN") + scope = str(parsed.get("scope_description") or "unknown") + module = str(parsed.get("module") or parsed.get("source") or "") + type_counts[event_type] = type_counts.get(event_type, 0) + 1 + scope_counts[scope] = scope_counts.get(scope, 0) + 1 + if module: + module_counts[module] = module_counts.get(module, 0) + 1 + + compact = { + "type": event_type, + "scope_description": scope, + "data": parsed.get("data"), + } + if module: + compact["module"] = module + if event_type == "SCAN": + scans.append(compact) + elif len(examples) < 120 and event_type in { + "DNS_NAME", + "URL", + "OPEN_TCP_PORT", + "TECHNOLOGY", + "FINDING", + "IP_ADDRESS", + "WEBSCREENSHOT", + "STORAGE_BUCKET", + "SOCIAL", + "MOBILE_APP", + }: + examples.append(compact) + continue + + diagnostic_tail.append(line[:300]) + diagnostic_tail = diagnostic_tail[-40:] + + summary = { + "mode": "bbot_stdout_json_summary", + "raw_output_chars": len(output), + "event_count": event_count, + "type_counts": dict( + sorted(type_counts.items(), key=lambda item: (-item[1], item[0])) + ), + "scope_counts": dict( + sorted(scope_counts.items(), key=lambda item: (-item[1], item[0])) + ), + "module_counts": dict( + sorted(module_counts.items(), key=lambda item: (-item[1], item[0]))[:40] + ), + "representative_events": examples, + "scan_records": scans[-5:], + "diagnostic_tail": diagnostic_tail, + } + text = ( + "BBOT completed with JSON/stdout evidence. The raw event stream was " + "summarized to keep the agent context small; use the counts and " + "representative_events below for ASM statistics and leads.\n\n" + + json.dumps(summary, indent=2, default=str) + ) + if len(text) > max_chars: + text = ( + text[:max_chars] + f"\n\n... [SUMMARY TRUNCATED: {len(text)} chars total]" + ) + return text + + +def _count_bbot_json_events(output: str) -> int: + """Count JSON event lines in BBOT stdout.""" + event_count = 0 + for raw_line in output.splitlines(): + line = _strip_ansi(raw_line).strip() + if not line.startswith("{"): + continue + with contextlib.suppress(Exception): + value = json.loads(line) + if isinstance(value, dict): + event_count += 1 + return event_count + + +def _bbot_output_has_blocking_diagnostic(output: str) -> bool: + """Detect BBOT diagnostics that mean no useful scan ran.""" + blocking_markers = ( + "Setup hard-failed", + "[ERRR]", + "[ERROR]", + "Please specify --allow-deadly to continue", + "No modules to scan", + ) + return any(marker in output for marker in blocking_markers) + + +def _has_dependency_control_arg(args: list[str]) -> bool: + dependency_args = { + "--no-deps", + "--force-deps", + "--retry-deps", + "--ignore-failed-deps", + "--install-all-deps", + } + return any(arg in dependency_args for arg in args) + + +def _has_module_exclusion_arg(args: list[str]) -> bool: + return any(arg in {"--exclude-modules", "-em"} for arg in args) + + +def _requests_baddns(modules: list[str] | None) -> bool: + return bool(modules) and any(module.startswith("baddns") for module in modules) + + +def _allows_deadly(args: list[str]) -> bool: + return "--allow-deadly" in args + + +def _normalize_modules(modules: list[str] | None, extra_args: list[str]) -> list[str]: + """Drop common non-BBOT/deadly module guesses that would abort a scan.""" + if not modules: + return [] + replacements = { + "aws_s3_scan": "bucket_amazon", + "azure_blob_scan": "bucket_microsoft", + "azure_blobs": "bucket_microsoft", + "gcp_storage": "bucket_google", + "gcp_storage_buckets": "bucket_google", + "gcp_storage_scan": "bucket_google", + "s3_buckets": "bucket_amazon", + "s3_scan": "bucket_amazon", + "subenum": "subdomaincenter", + "subdomain_enum": "subdomaincenter", + "subdomain-enum": "subdomaincenter", + "subdomainenum": "subdomaincenter", + "technologies": "httpx", + } + unsupported = { + "amass", + "assetfinder", + "crtsh", + "dns_zone_transfer", + "fastdial", + "find_subdomains", + "findomain", + "gau", + "massdns", + "naabu", + "portscan", + "screenshot", + "shuffledns", + "subdomain-finder", + "subdomain_find", + "subdomain_bruteforce", + "subdomain-bruteforce", + "subdomainfinder", + "subfinder", + "sublist3r", + "web_enum", + "web-enum", + "web_screenshot", + "web-screenshot", + "web_screenshots", + "web-screenshots", + "webenum", + "wappalyzer", + } + normalized: list[str] = [] + for module in modules: + module = replacements.get(module, module) + if module in unsupported: + continue + if module == "nuclei" and not _allows_deadly(extra_args): + continue + normalized.append(module) + return normalized + + +def _normalize_presets(presets: list[str] | None, extra_args: list[str]) -> list[str]: + """Avoid BBOT deadly preset aborts unless explicitly requested.""" + if not presets: + return [] + replacements = { + "asset-discovery-and-enrichment": "subdomain-enum", + "asset-discovery": "subdomain-enum", + "asset_discovery": "subdomain-enum", + "asset-discovery-web-endpoints": "web-basic", + "asset_discovery_web_endpoints": "web-basic", + "asset_discovery_and_enrichment": "subdomain-enum", + "discovery": "subdomain-enum", + "subdomain_discovery": "subdomain-enum", + "subdomain-discovery": "subdomain-enum", + "subdomain_enumeration": "subdomain-enum", + "subdomain-enumeration": "subdomain-enum", + "web_breach": "web-basic", + "web_basic": "web-basic", + "web_discovery": "web-basic", + "web_scan": "web-basic", + "web-port-scan-and-tech-detect": "web-basic", + "web_port_scan_and_tech_detect": "web-basic", + } + supported = { + "subdomain-enum", + "web-basic", + "web-thorough", + "cloud-enum", + "code-enum", + "email-enum", + "spider", + "nuclei", + "nuclei-intense", + "dirbust-light", + "dirbust-heavy", + "lightfuzz-light", + "lightfuzz-medium", + "tech-detect", + "kitchen-sink", + } + presets = [replacements.get(preset, preset) for preset in presets] + presets = [preset for preset in presets if preset in supported] + if _allows_deadly(extra_args): + return presets + return [preset for preset in presets if not preset.startswith("nuclei")] + + +def _normalize_flags(flags: list[str] | None) -> list[str]: + """Drop CLI switches and stale flag names emitted as BBOT flags.""" + if not flags: + return [] + supported = { + "active", + "passive", + "safe", + "aggressive", + "subdomain-enum", + "web-basic", + "web-thorough", + "cloud-enum", + "code-enum", + "email-enum", + } + return [flag for flag in flags if flag in supported] + + +def _sandbox_suppression_note( + requested_modules: list[str] | None, graph_api_url: str | None +) -> str: + if not graph_api_url or not requested_modules: + return "" + suppressed = sorted( + {module for module in requested_modules if module in {"gowitness", "portscan"}} + ) + if not suppressed: + return "" + return ( + "Sandbox note: " + + ", ".join(suppressed) + + " requested but suppressed in Graph API mode for runtime reliability; " + "use returned DNS/URL/FINDING evidence, `httpx`, and safe web presets for bounded validation.\n\n" + ) + + +def _bbot_subprocess_env(graph_api_url: str | None) -> dict[str, str]: + env = os.environ.copy() + if graph_api_url: + bbot_home = Path(tempfile.mkdtemp(prefix="asm-bbot-home-")) + env["HOME"] = str(bbot_home) + env["BBOT_HOME"] = str(bbot_home / ".bbot") + return env + + +def _normalize_config(config: list[str]) -> list[str]: + """Accept common stale BBOT config names emitted by older prompts/skills.""" + replacements = { + "modules.httpx.timeout": "modules.http.timeout", + "scope.distance": "scope.search_distance", + } + dropped = { + "scope.report_distance", + } + normalized: list[str] = [] + for item in config: + key, sep, value = item.partition("=") + if key in dropped: + continue + if ( + key.startswith("modules.") + and key.endswith(".enabled") + and value.lower() == "false" + ): + continue + normalized.append(f"{replacements.get(key, key)}{sep}{value}") + return normalized + + +def _normalize_extra_args(args: list[str]) -> list[str]: + """Drop stale BBOT CLI flags emitted by models.""" + normalized: list[str] = [] + skip_next = False + for arg in args: + if skip_next: + skip_next = False + continue + if arg == "--scope-distance": + skip_next = True + continue + if arg.startswith("--scope-distance="): + continue + normalized.append(arg) + return normalized + + +def _normalize_targets(targets: list[str]) -> list[str]: + """Convert wildcard scope expressions into BBOT seed targets.""" + normalized: list[str] = [] + for target in targets: + stripped = target.strip() + if stripped.startswith("*.") and len(stripped) > 2: + stripped = stripped[2:] + if stripped: + normalized.append(stripped) + return normalized + + +def _coerce_int_limit(limit: int, *, maximum: int = 1000) -> int: + if limit < 1 or limit > maximum: + raise ValueError(f"Limit must be between 1 and {maximum}.") + return limit + + +class _Neo4jClient: + """Lazy async Neo4j driver wrapper.""" + + def __init__(self) -> None: + self._driver = None + + async def get(self): + if self._driver is None: + self._driver = AsyncGraphDatabase.driver( + NEO4J_URI, auth=(NEO4J_USER, NEO4J_PASSWORD) + ) + await self._driver.verify_connectivity() + return self._driver + + async def query(self, cypher: str, params: dict | None = None) -> list[dict]: + driver = await self.get() + async with driver.session() as session: + result = await session.run(cypher, params or {}) + return [record.data() async for record in result] + + +_neo4j = _Neo4jClient() + +mcp = FastMCP("bbot") + + +@mcp.tool() +async def bbot_health() -> str: + """Check BBOT and Neo4j connectivity.""" + errors = [] + + # Check bbot + try: + proc = await asyncio.create_subprocess_exec( + "bbot", + "--version", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + stdout, _ = await proc.communicate() + bbot_version = stdout.decode().strip() if proc.returncode == 0 else "not found" + except FileNotFoundError: + bbot_version = "not installed" + errors.append("bbot CLI not found in PATH") + + # Check Neo4j + try: + await _neo4j.get() + neo4j_status = f"connected ({NEO4J_URI})" + except Exception as e: + neo4j_status = f"error: {e}" + errors.append(f"Neo4j connection failed: {e}") + + status = "healthy" if not errors else "degraded" + return f"Status: {status}\n" f" BBOT: {bbot_version}\n" f" Neo4j: {neo4j_status}" + + +@mcp.tool() +async def run_bbot_scan( + targets: Annotated[list[str], "Targets to scan (domains, IPs, CIDRs)"], + modules: Annotated[list[str] | None, "Specific modules to run"] = None, + presets: Annotated[ + list[str] | None, "Presets (subdomain-enum, web-basic, nuclei, etc.)" + ] = None, + flags: Annotated[ + list[str] | None, "Module group flags (passive, safe, active, etc.)" + ] = None, + config: Annotated[list[str] | None, "Config options in key=value format"] = None, + extra_args: Annotated[list[str] | None, "Additional bbot CLI flags"] = None, + graph_api_url: Annotated[ + str | None, + "Task-local Graph API URL. When supplied, run BBOT in stdout JSON mode so sandboxed evaluations do not require raw Bolt access.", + ] = None, +) -> str: + """Execute a BBOT reconnaissance scan. + + When graph_api_url is provided, results are returned as summarized stdout + JSON instead of forcing BBOT's Neo4j output module against localhost. + """ + if not targets: + return "Error: at least one target is required." + + targets = _normalize_targets(targets) + extra = _normalize_extra_args(list(extra_args or [])) + requested_modules = list(modules or []) + modules = _normalize_modules(modules, extra) + presets = _normalize_presets(presets, extra) + flags = _normalize_flags(flags) + cfg = _normalize_config(list(config or [])) + if graph_api_url: + parts = ["bbot", "--yes", "--json", "--brief", "--output-modules", "stdout"] + else: + cfg.extend( + [ + f"modules.neo4j.uri={NEO4J_URI}", + f"modules.neo4j.username={NEO4J_USER}", + f"modules.neo4j.password={NEO4J_PASSWORD}", + ] + ) + parts = ["bbot", "--yes", "--output-modules", "neo4j", "--brief"] + parts.extend(["--targets", *targets]) + if modules: + parts.extend(["--modules", *modules]) + if flags: + parts.extend(["--flags", *flags]) + if presets: + parts.extend(["--preset", *presets]) + parts.extend(["--config", *cfg]) + if extra: + parts.extend(extra) + if graph_api_url and not _has_dependency_control_arg(extra): + parts.append("--no-deps") + if graph_api_url and not _has_module_exclusion_arg(extra): + excluded = ["portscan", "gowitness"] + if not _requests_baddns(modules): + excluded.extend(["baddns", "baddns_direct", "baddns_zone"]) + parts.extend(["--exclude-modules", *excluded]) + + cmd = " ".join(parts) + + try: + proc = await asyncio.create_subprocess_exec( + *shlex.split(cmd), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + env=_bbot_subprocess_env(graph_api_url), + ) + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=SCAN_TIMEOUT) + output = stdout.decode(errors="replace") + except asyncio.TimeoutError: + return f"Scan timed out after {SCAN_TIMEOUT}s" + except FileNotFoundError: + return "Error: bbot not found. Install with: pip install bbot" + + if graph_api_url: + note = _sandbox_suppression_note(requested_modules, graph_api_url) + event_count = _count_bbot_json_events(output) + summarized = _summarize_bbot_stdout(output, MAX_OUTPUT) + if ( + proc.returncode != 0 + or event_count == 0 + or _bbot_output_has_blocking_diagnostic(output) + ): + status = ( + f"exited with code {proc.returncode}" + if proc.returncode + else "produced no usable BBOT JSON events" + ) + return f"Scan {status}.\n\n{note}{summarized}" + output = note + summarized + elif len(output) > MAX_OUTPUT: + output = output[:MAX_OUTPUT] + "\n\n... [TRUNCATED]" + + status = ( + "completed" if proc.returncode == 0 else f"exited with code {proc.returncode}" + ) + return f"Scan {status}.\n\n{output}" + + +@mcp.tool() +async def query_graph( + cypher: Annotated[str, "Cypher query to execute"], + params: Annotated[dict | None, "Query parameters (use $param in query)"] = None, + graph_api_url: Annotated[ + str | None, + "Task-local Graph API URL for querying Neo4j over HTTP when raw Bolt is unavailable.", + ] = None, +) -> str: + """Execute a Cypher query against the Neo4j graph database.""" + if graph_api_url: + result = await asyncio.to_thread( + _query_graph_api, graph_api_url, cypher, params + ) + else: + result = await _neo4j.query(cypher, params) + return _safe_json(result) + + +@mcp.tool() +async def get_findings( + severity: Annotated[ + str | None, "Filter by severity (critical, high, medium, low)" + ] = None, +) -> str: + """Retrieve security findings from BBOT scans.""" + if severity: + result = await _neo4j.query( + "MATCH (f:FINDING) WHERE f.severity = $sev RETURN f", {"sev": severity} + ) + else: + result = await _neo4j.query("MATCH (f:FINDING) RETURN f") + return _safe_json(result) + + +@mcp.tool() +async def get_db_schema() -> str: + """Retrieve Neo4j labels, relationship types, and property metadata.""" + queries = { + "node_labels": "CALL db.labels() YIELD label RETURN label", + "relationship_types": "CALL db.relationshipTypes() YIELD relationshipType RETURN relationshipType", + "node_properties": "CALL db.schema.nodeTypeProperties()", + "relationship_properties": "CALL db.schema.relTypeProperties()", + } + labels, rel_types, node_props, rel_props = await asyncio.gather( + *(_neo4j.query(query) for query in queries.values()) + ) + + schema: dict[str, object] = { + "node_labels": sorted( + record["label"] for record in labels if record.get("label") + ), + "relationship_types": sorted( + record["relationshipType"] + for record in rel_types + if record.get("relationshipType") + ), + "node_properties": {}, + "relationship_properties": {}, + } + + node_properties: dict[str, list[dict[str, object]]] = {} + for record in node_props: + label = str(record.get("nodeType", "")).lstrip(":") + if not label: + continue + node_properties.setdefault(label, []).append( + { + "property": record.get("propertyName"), + "types": record.get("propertyTypes"), + "mandatory": record.get("mandatory"), + } + ) + + relationship_properties: dict[str, list[dict[str, object]]] = {} + for record in rel_props: + rel_type = str(record.get("relType", "")).lstrip(":") + if not rel_type: + continue + relationship_properties.setdefault(rel_type, []).append( + { + "property": record.get("propertyName"), + "types": record.get("propertyTypes"), + "mandatory": record.get("mandatory"), + } + ) + + schema["node_properties"] = node_properties + schema["relationship_properties"] = relationship_properties + return _safe_json(schema) + + +@mcp.tool() +async def get_asset_summary() -> str: + """Get a summary count of all asset types in the database.""" + result = await _neo4j.query( + "MATCH (n) RETURN labels(n)[0] as type, count(n) AS count ORDER BY count DESC" + ) + return _safe_json(result) + + +@mcp.tool() +async def get_subdomains( + domain: Annotated[str, "Parent domain to search for subdomains"], + limit: Annotated[int, "Maximum results"] = 100, +) -> str: + """List discovered subdomains for a domain.""" + result = await _neo4j.query( + """ + MATCH (n:DNS_NAME) + WITH coalesce(n.name, n.data, n.host) AS name + WHERE name ENDS WITH $domain + RETURN name + ORDER BY name + LIMIT $limit + """, + {"domain": domain, "limit": limit}, + ) + return _safe_json(result) + + +@mcp.tool() +async def get_technologies() -> str: + """List all discovered technologies and their usage counts.""" + result = await _neo4j.query( + """ + MATCH (t:TECHNOLOGY) + RETURN DISTINCT coalesce(t.name, t.data) AS name, t.version AS version, count(*) AS usage + ORDER BY usage DESC + """ + ) + return _safe_json(result) + + +@mcp.tool() +async def explore_nodes( + label: Annotated[ + str | None, "Node label to browse, for example DNS_NAME, URL, FINDING" + ] = None, + property_filter: Annotated[ + str | None, + "Optional filter: 'property=value' for exact match or 'property CONTAINS value' for substring", + ] = None, + limit: Annotated[int, "Maximum nodes to return (1-1000)"] = 100, +) -> str: + """Browse graph nodes by label and optional property filter.""" + limit = _coerce_int_limit(limit) + query_parts = [ + f"MATCH (node:{_validate_cypher_identifier(label, 'label')})" + if label + else "MATCH (node)" + ] + params: dict[str, object] = {"limit": limit} + + if property_filter: + if " CONTAINS " in property_filter: + prop, value = property_filter.split(" CONTAINS ", 1) + prop = _validate_cypher_identifier(prop.strip(), "property") + query_parts.append(f"WHERE toString(node.`{prop}`) CONTAINS $value") + params["value"] = value.strip() + elif "=" in property_filter: + prop, value = property_filter.split("=", 1) + prop = _validate_cypher_identifier(prop.strip(), "property") + query_parts.append(f"WHERE node.`{prop}` = $value") + params["value"] = value.strip() + else: + raise ValueError("property_filter must use '=' or ' CONTAINS '.") + + query_parts.append("RETURN node LIMIT $limit") + result = await _neo4j.query(" ".join(query_parts), params) + return _safe_json(result) + + +@mcp.tool() +async def explore_relationships( + source_label: Annotated[str | None, "Optional source node label"] = None, + relationship_type: Annotated[str | None, "Optional relationship type"] = None, + target_label: Annotated[str | None, "Optional target node label"] = None, + limit: Annotated[int, "Maximum relationships to return (1-1000)"] = 100, +) -> str: + """Browse graph relationships with optional source/type/target filters.""" + limit = _coerce_int_limit(limit) + source = ( + f"(source:{_validate_cypher_identifier(source_label, 'source label')})" + if source_label + else "(source)" + ) + target = ( + f"(target:{_validate_cypher_identifier(target_label, 'target label')})" + if target_label + else "(target)" + ) + if relationship_type: + rel = f"-[relationship:{_validate_cypher_identifier(relationship_type, 'relationship type')}]->" + else: + rel = "-[relationship]->" + + result = await _neo4j.query( + f"MATCH {source}{rel}{target} RETURN source, relationship, target LIMIT $limit", + {"limit": limit}, + ) + return _safe_json(result) + + +@mcp.tool() +async def get_screenshot( + uuid: Annotated[str | None, "WEBSCREENSHOT uuid or id"] = None, + url: Annotated[str | None, "Substring of the original screenshot URL"] = None, +) -> str: + """Resolve a WEBSCREENSHOT node to a local screenshot file path.""" + if not uuid and not url: + raise ValueError("Either uuid or url must be provided.") + + if uuid: + result = await _neo4j.query( + """ + MATCH (w:WEBSCREENSHOT) + WHERE w.uuid = $uuid OR w.id = $uuid + OPTIONAL MATCH (s:SCAN {id: w.scan}) + RETURN properties(w) AS web_props, properties(s) AS scan_props + LIMIT 1 + """, + {"uuid": uuid}, + ) + else: + result = await _neo4j.query( + """ + MATCH (w:WEBSCREENSHOT) + WHERE toString(w.url) CONTAINS $url OR toString(w.data) CONTAINS $url + OPTIONAL MATCH (s:SCAN {id: w.scan}) + RETURN properties(w) AS web_props, properties(s) AS scan_props + LIMIT 1 + """, + {"url": url}, + ) + + if not result: + needle = f"UUID {uuid!r}" if uuid else f"URL {url!r}" + return f"No screenshot found for {needle}." + + web_props = result[0].get("web_props") or {} + scan_props = result[0].get("scan_props") or {} + web_data = _parse_mapping(web_props.get("data")) + scan_data = _parse_mapping(scan_props.get("data")) + + screenshot_uuid = web_props.get("uuid") or web_props.get("id") or uuid + original_url = ( + web_props.get("url") or web_data.get("url") or web_props.get("host") or url + ) + relative_path = web_props.get("path") or web_data.get("path") + scan_name = scan_props.get("name") or scan_data.get("name") or web_props.get("scan") + + if not relative_path: + return _safe_json( + { + "error": "Screenshot data is missing a path.", + "uuid": screenshot_uuid, + "url": original_url, + } + ) + + path = Path(str(relative_path)).expanduser() + candidates = [path] if path.is_absolute() else [] + bbot_home = Path(BBOT_DATA_DIR).expanduser().resolve() + if not path.is_absolute() and scan_name: + candidates.append(bbot_home / "scans" / str(scan_name) / path) + if not path.is_absolute(): + candidates.append(bbot_home / path) + + for candidate in candidates: + if candidate.exists(): + return _safe_json( + { + "path": str(candidate), + "url": original_url, + "uuid": screenshot_uuid, + } + ) + + return _safe_json( + { + "error": "Screenshot file not found.", + "checked_paths": [str(candidate) for candidate in candidates], + "url": original_url, + "uuid": screenshot_uuid, + } + ) diff --git a/capabilities/attack-surface-management/mcp/shodan.py b/capabilities/attack-surface-management/mcp/shodan.py new file mode 100644 index 0000000..3bca8f8 --- /dev/null +++ b/capabilities/attack-surface-management/mcp/shodan.py @@ -0,0 +1,405 @@ +#!/usr/bin/env -S uv run +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "fastmcp>=2.0", +# "shodan>=1.31.0", +# ] +# /// +"""Shodan internet intelligence tools exposed as an MCP server. + +Provides host search, IP reconnaissance, DNS lookups, CVE/exploit +intelligence, and API usage tracking via the Shodan API. + +Environment variables: + SHODAN_API_KEY: Required. Your Shodan API key. + SHODAN_API_URL: Optional. Base URL for a Shodan-compatible mock service. +""" + +from __future__ import annotations + +import json +import os +from typing import Annotated +from urllib import parse, request + +import shodan +from fastmcp import FastMCP + +SHODAN_API_KEY = os.environ.get("SHODAN_API_KEY", "") +SHODAN_API_URL = os.environ.get("SHODAN_API_URL", "").rstrip("/") + +mcp = FastMCP("shodan") + + +class _HttpNamespace: + def __init__(self, client: "_HttpShodanClient", prefix: str) -> None: + self._client = client + self._prefix = prefix + + def resolve(self, hostnames: str) -> dict: + return self._client.get_json( + f"{self._prefix}/resolve", {"hostnames": hostnames} + ) + + def reverse(self, ips: str) -> dict: + return self._client.get_json(f"{self._prefix}/reverse", {"ips": ips}) + + def search(self, query: str, **options: object) -> dict: + return self._client.get_json( + f"{self._prefix}/search", {"query": query, **options} + ) + + def tags(self, size: int = 20) -> object: + return self._client.get_json(f"{self._prefix}/tags", {"size": size}) + + +class _HttpShodanClient: + """Small Shodan-compatible HTTP adapter for task-local mock services.""" + + def __init__(self, base_url: str, api_key: str) -> None: + self.base_url = base_url.rstrip("/") + self.api_key = api_key + self.dns = _HttpNamespace(self, "/dns") + self.exploits = _HttpNamespace(self, "/exploits") + self.queries = _HttpNamespace(self, "/shodan/query") + + def get_json(self, path: str, params: dict[str, object] | None = None) -> object: + query_params = dict(params or {}) + if self.api_key: + query_params["key"] = self.api_key + query = parse.urlencode(query_params, doseq=True) + url = f"{self.base_url}{path}" + if query: + url = f"{url}?{query}" + + req = request.Request(url, headers={"Accept": "application/json"}) + with request.urlopen(req, timeout=30) as response: + body = response.read().decode("utf-8") + return json.loads(body) if body else {} + + def search(self, query: str, **options: object) -> dict: + result = self.get_json("/shodan/host/search", {"query": query, **options}) + return result if isinstance(result, dict) else {} + + def host(self, ip: str, history: bool = False) -> dict: + result = self.get_json( + f"/shodan/host/{parse.quote(ip, safe='')}", + {"history": str(history).lower()}, + ) + return result if isinstance(result, dict) else {} + + def count(self, query: str, **options: object) -> dict: + result = self.get_json("/shodan/host/count", {"query": query, **options}) + return result if isinstance(result, dict) else {} + + def ports(self) -> object: + return self.get_json("/shodan/ports") + + def protocols(self) -> object: + return self.get_json("/shodan/protocols") + + def info(self) -> dict: + result = self.get_json("/api-info") + return result if isinstance(result, dict) else {} + + +def _get_client() -> shodan.Shodan | _HttpShodanClient: + if not SHODAN_API_KEY: + raise RuntimeError( + "SHODAN_API_KEY environment variable is not set. " + "Get your API key at https://account.shodan.io" + ) + if SHODAN_API_URL: + return _HttpShodanClient(SHODAN_API_URL, SHODAN_API_KEY) + return shodan.Shodan(SHODAN_API_KEY) + + +def _safe_json(obj: object) -> str: + return json.dumps(obj, indent=2, default=str) + + +# ── Core Search ─────────────────────────────────────────────────────── + + +@mcp.tool() +def shodan_host_search( + query: Annotated[ + str, + "Shodan search query (e.g., 'apache city:\"San Francisco\"', 'port:502 tag:ics')", + ], + facets: Annotated[ + str | None, "Comma-separated facets for aggregation (e.g., 'country,org,port')" + ] = None, + page: Annotated[int, "Page number for pagination"] = 1, +) -> str: + """Search Shodan for hosts matching a query. + + Returns matching hosts with IP, port, org, hostnames, location, + vulnerabilities, and optional facet aggregations. + + Common queries: + org:"Target Corp" + hostname:example.com + port:3389 org:"Target Corp" + ssl.cert.subject.cn:example.com + http.title:"Dashboard" + vuln:CVE-2021-44228 + product:"Apache" version:"2.4.49" + """ + api = _get_client() + options: dict = {"page": page} + if facets: + options["facets"] = facets + + results = api.search(query, **options) + + matches = [] + for match in results.get("matches", []): + entry: dict = { + "ip": match.get("ip_str"), + "port": match.get("port"), + "org": match.get("org"), + "hostnames": match.get("hostnames", []), + "domains": match.get("domains", []), + "transport": match.get("transport"), + "product": match.get("product"), + "version": match.get("version"), + } + if match.get("vulns"): + entry["vulns"] = ( + list(match["vulns"].keys()) + if isinstance(match["vulns"], dict) + else match["vulns"] + ) + if match.get("location"): + loc = match["location"] + entry["location"] = { + "country": loc.get("country_name"), + "city": loc.get("city"), + } + matches.append(entry) + + return _safe_json( + { + "total": results.get("total", 0), + "matches": matches, + "facets": results.get("facets", {}), + } + ) + + +@mcp.tool() +def shodan_host_info( + ip: Annotated[str, "IP address to look up (e.g., '8.8.8.8')"], + history: Annotated[bool, "Include historical banners"] = False, +) -> str: + """Get detailed information about a specific IP address. + + Returns open ports, services, OS, organization, hostnames, location, + vulnerabilities, and service banners. + """ + api = _get_client() + host = api.host(ip, history=history) + + return _safe_json( + { + "ip": host.get("ip_str"), + "org": host.get("org"), + "os": host.get("os"), + "ports": host.get("ports", []), + "hostnames": host.get("hostnames", []), + "domains": host.get("domains", []), + "vulns": host.get("vulns", []), + "tags": host.get("tags", []), + "last_update": host.get("last_update"), + "location": { + "country": host.get("country_name"), + "city": host.get("city"), + "asn": host.get("asn"), + "isp": host.get("isp"), + }, + "data": [ + { + "port": svc.get("port"), + "transport": svc.get("transport"), + "product": svc.get("product"), + "version": svc.get("version"), + "banner": (svc.get("data", "")[:500] if svc.get("data") else None), + } + for svc in host.get("data", []) + ], + } + ) + + +@mcp.tool() +def shodan_count( + query: Annotated[str, "Shodan search query"], + facets: Annotated[ + str | None, "Comma-separated facets for aggregated counts" + ] = None, +) -> str: + """Get result count for a query without consuming search credits. + + Always use this before a full search to check scope and avoid + wasting API credits on overly broad queries. + """ + api = _get_client() + options: dict = {} + if facets: + options["facets"] = facets + + result = api.count(query, **options) + return _safe_json( + { + "total": result.get("total", 0), + "facets": result.get("facets", {}), + } + ) + + +# ── DNS ─────────────────────────────────────────────────────────────── + + +@mcp.tool() +def shodan_dns_lookup( + hostnames: Annotated[ + list[str], "Hostnames to resolve (e.g., ['example.com', 'api.example.com'])" + ], +) -> str: + """Resolve domain names to IP addresses via Shodan DNS.""" + api = _get_client() + result = api.dns.resolve(",".join(hostnames)) + return _safe_json(result) + + +@mcp.tool() +def shodan_dns_reverse( + ips: Annotated[list[str], "IP addresses to reverse lookup (e.g., ['8.8.8.8'])"], +) -> str: + """Reverse DNS lookup — find hostnames for IP addresses.""" + api = _get_client() + result = api.dns.reverse(",".join(ips)) + return _safe_json(result) + + +# ── Exploits & Intelligence ────────────────────────────────────────── + + +@mcp.tool() +def shodan_exploits_search( + query: Annotated[ + str, "Exploit search query (e.g., 'CVE-2021-44228', 'Apache', 'Modbus')" + ], + facets: Annotated[ + str | None, "Facets for aggregation (e.g., 'type,platform,author')" + ] = None, + page: Annotated[int, "Page number"] = 1, +) -> str: + """Search the Shodan Exploits database for known exploits and CVEs. + + Returns exploit details including description, author, type, platform, + affected CVEs, and source references. + """ + api = _get_client() + options: dict = {"page": page} + if facets: + options["facets"] = facets + + result = api.exploits.search(query, **options) + + matches = [] + for exploit in result.get("matches", []): + matches.append( + { + "id": exploit.get("_id"), + "description": exploit.get("description", "")[:500], + "author": exploit.get("author"), + "type": exploit.get("type"), + "platform": exploit.get("platform"), + "date": exploit.get("date"), + "source": exploit.get("source"), + "cve": exploit.get("cve", []), + } + ) + + return _safe_json( + { + "total": result.get("total", 0), + "matches": matches, + "facets": result.get("facets", {}), + } + ) + + +# ── Reference Data ─────────────────────────────────────────────────── + + +@mcp.tool() +def shodan_ports() -> str: + """List all port numbers that Shodan actively crawls.""" + api = _get_client() + return _safe_json(api.ports()) + + +@mcp.tool() +def shodan_protocols() -> str: + """List all protocols Shodan can distinguish in banner grabs.""" + api = _get_client() + return _safe_json(api.protocols()) + + +# ── Community Queries ──────────────────────────────────────────────── + + +@mcp.tool() +def shodan_query_search( + query: Annotated[str, "Search term (e.g., 'SCADA', 'webcam', 'database')"], + page: Annotated[int, "Page number"] = 1, +) -> str: + """Search community-shared Shodan queries for inspiration.""" + api = _get_client() + result = api.queries.search(query, page=page) + return _safe_json( + { + "total": result.get("total", 0), + "matches": [ + { + "title": q.get("title"), + "description": q.get("description"), + "query": q.get("query"), + "votes": q.get("votes"), + "tags": q.get("tags", []), + } + for q in result.get("matches", []) + ], + } + ) + + +@mcp.tool() +def shodan_query_tags( + size: Annotated[int, "Number of tags to return"] = 20, +) -> str: + """Get popular tags for community-shared Shodan queries.""" + api = _get_client() + return _safe_json(api.queries.tags(size=size)) + + +# ── API Status ─────────────────────────────────────────────────────── + + +@mcp.tool() +def shodan_api_info() -> str: + """Check API plan, remaining credits, and account status.""" + api = _get_client() + info = api.info() + return _safe_json( + { + "plan": info.get("plan"), + "query_credits": info.get("query_credits"), + "scan_credits": info.get("scan_credits"), + "unlocked": info.get("unlocked"), + } + ) diff --git a/capabilities/attack-surface-management/pyproject.toml b/capabilities/attack-surface-management/pyproject.toml new file mode 100644 index 0000000..f19a036 --- /dev/null +++ b/capabilities/attack-surface-management/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "attack-surface-management" +version = "1.1.27" +requires-python = ">=3.10" +dependencies = [ + "badsecrets~=0.13.47", + "baddns~=1.12.294", + "bbot", + "aiosqlite>=0.21.0", + "extractous~=0.3.0", + "asyncpg>=0.31.0", + "dnspython>=2.7.0,<2.8.0", + "fastmcp>=2.0", + "neo4j>=5.28.1", + "shodan>=1.31.0", + "aiodocker>=0.24.0", + "tldextract>=5.3.1,<6.0.0", + "pillow>=11.3.0", + "pyOpenSSL~=25.3.0", + "loguru>=0.7.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0.0", + "mypy>=1.17.0", + "ruff>=0.12.5", +] diff --git a/capabilities/attack-surface-management/scripts/analyze.py b/capabilities/attack-surface-management/scripts/analyze.py new file mode 100644 index 0000000..6ce4bdd --- /dev/null +++ b/capabilities/attack-surface-management/scripts/analyze.py @@ -0,0 +1,179 @@ +"""Submit an ASM worker-pipeline run and stream progress. + +Launcher for the worker-coordinated ASM pipeline. It connects to a runtime, +publishes ``asm.analysis.requested``, watches progress/report events, and writes +the final report when ``asm.analysis.completed`` fires. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from dreadnode.app.client.runtime_client import RuntimeClient + +WATCH_TIMEOUT_SECONDS = 14_400 + +REQUEST_EVENT = "asm.analysis.requested" +PROGRESS_EVENT = "asm.analysis.progress" +REPORT_READY_EVENT = "asm.analysis.report.ready" +COMPLETED_EVENT = "asm.analysis.completed" +FAILED_EVENT = "asm.analysis.failed" + +WATCH_KINDS: tuple[str, ...] = ( + PROGRESS_EVENT, + REPORT_READY_EVENT, + COMPLETED_EVENT, + FAILED_EVENT, +) + + +async def amain(args: argparse.Namespace) -> None: + client = await build_client(args) + try: + await client.start() + run_id = args.run_id or str(uuid4()) + await run_one(client, args, run_id) + finally: + await client.close() + + +async def build_client(args: argparse.Namespace) -> RuntimeClient: + if args.local: + from dreadnode.app.client.managed_client import ManagedRuntimeClient + + capability_dir = Path(__file__).resolve().parent.parent + return ManagedRuntimeClient(capability_dirs=[str(capability_dir)]) + + runtime_url = args.runtime_url or os.environ.get("DREADNODE_RUNTIME_URL") + if not runtime_url: + raise SystemExit( + "Provide --runtime-url or DREADNODE_RUNTIME_URL, or pass --local " + "to boot an in-process runtime." + ) + runtime_token = args.runtime_token or os.environ.get("DREADNODE_RUNTIME_TOKEN") + return RuntimeClient(server_url=runtime_url.rstrip("/"), auth_token=runtime_token) + + +async def run_one(client: RuntimeClient, args: argparse.Namespace, run_id: str) -> None: + watcher = asyncio.create_task(watch_run(client, run_id, args.output)) + await asyncio.sleep(0.3) + + if not args.watch_only: + payload: dict[str, Any] = { + "run_id": run_id, + "target": args.target, + "wildcards": args.wildcard, + "max_steps": args.max_steps, + } + if args.scope_json: + payload["scope"] = json.loads(args.scope_json) + if args.graph_api_url: + payload["graph_api_url"] = args.graph_api_url + if args.model: + payload["model"] = args.model + await client.publish(REQUEST_EVENT, payload) + print(f"Requested ASM analysis run {run_id}") + else: + print(f"Watching ASM analysis run {run_id}") + + try: + await asyncio.wait_for(watcher, timeout=args.timeout) + except asyncio.TimeoutError as exc: + raise RuntimeError( + f"Run {run_id} did not complete within {args.timeout}s" + ) from exc + + +async def watch_run(client: RuntimeClient, run_id: str, output_path: str) -> None: + async for event in client.subscribe(*WATCH_KINDS): + payload = event.payload if isinstance(event.payload, dict) else {} + if payload.get("run_id") != run_id: + continue + + if event.kind == PROGRESS_EVENT: + stage = payload.get("stage") + detail = payload.get("detail") + print(f"Progress: {stage}{f' - {detail}' if detail else ''}") + continue + if event.kind == REPORT_READY_EVENT: + print(f"Report ready: {payload.get('agent')}") + continue + if event.kind == FAILED_EVENT: + raise RuntimeError(str(payload.get("error") or "ASM analysis failed")) + if event.kind == COMPLETED_EVENT: + final_report = str(payload.get("final_report") or "") + Path(output_path).write_text(final_report, encoding="utf-8") + print(f"Final report written to {output_path}") + return + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Submit an ASM worker-pipeline run and watch progress.", + ) + parser.add_argument("target", nargs="?", help="Target name or domain label.") + parser.add_argument( + "--wildcard", + action="append", + default=[], + help="Allowed wildcard scope expression. Repeat for multiple wildcards.", + ) + parser.add_argument( + "--scope-json", + default=None, + help="Optional JSON object with additional scope metadata.", + ) + parser.add_argument("--graph-api-url", default=None, help="Task graph API URL.") + parser.add_argument("--runtime-url", default=None, help="Runtime URL.") + parser.add_argument("--runtime-token", default=None, help="Runtime bearer token.") + parser.add_argument( + "--local", + action="store_true", + help="Boot an in-process runtime that loads this capability.", + ) + parser.add_argument("--run-id", default=None, help="Optional run id.") + parser.add_argument( + "--watch-only", + action="store_true", + help="Watch an existing run without publishing a request.", + ) + parser.add_argument("--model", default=None, help="Model id for worker agents.") + parser.add_argument( + "--max-steps", + type=int, + default=240, + help="Maximum autonomous steps per worker agent turn.", + ) + parser.add_argument( + "--timeout", + type=int, + default=WATCH_TIMEOUT_SECONDS, + help="Wall-clock seconds to wait for completion.", + ) + parser.add_argument( + "--output", + default="asm-report.md", + help="Where to write the final report.", + ) + + args = parser.parse_args() + if args.watch_only: + if args.run_id is None: + parser.error("--watch-only requires --run-id") + elif not args.target: + parser.error("target is required unless --watch-only is set") + return args + + +def main() -> None: + asyncio.run(amain(parse_args())) + + +if __name__ == "__main__": + main() diff --git a/capabilities/attack-surface-management/skills/bbot-module-reference/SKILL.md b/capabilities/attack-surface-management/skills/bbot-module-reference/SKILL.md new file mode 100644 index 0000000..e50eaf1 --- /dev/null +++ b/capabilities/attack-surface-management/skills/bbot-module-reference/SKILL.md @@ -0,0 +1,161 @@ +--- +name: bbot-module-reference +description: BBOT module and preset reference for reconnaissance scanning. Use when choosing which modules, presets, or flags to use for a BBOT scan, or when you need to understand what a specific module does. +--- + +# BBOT Module & Preset Reference + +## Presets (-p flag) + +Presets are curated combinations of modules for common tasks. + +### Discovery + +| Preset | Purpose | Key Modules | +|---|---|---| +| `subdomain-enum` | Comprehensive subdomain discovery | anubisdb, certspotter, crt, dnsdumpster, dnsbrute, shodan_dns, securitytrails, wayback, +40 more | +| `cloud-enum` | Cloud resource enumeration (includes subdomain-enum) | bucket_amazon, bucket_azure, bucket_firebase, bucket_google | +| `code-enum` | Git repos, Docker images | github_codesearch, dockerhub, git_clone, postman | +| `email-enum` | Email address harvesting | emailformat, hunterio, pgp, skymem | + +### Web Scanning + +| Preset | Purpose | Key Modules | +|---|---|---| +| `web-basic` | Quick web scan for essentials | httpx, wappalyzer, badsecrets, robots, sslcert, ffuf_shortnames | +| `web-thorough` | Aggressive web scan (includes web-basic) | All web-basic + web-thorough flagged modules | +| `spider` | Recursive web crawling | distance:2, depth:4, 25 links/page | +| `spider-intense` | Aggressive spidering | distance:4, depth:6, 50 links/page | +| `tech-detect` | Technology detection only | wappalyzer, nuclei tech templates, fingerprintx | + +### Vulnerability Scanning + +| Preset | Purpose | Notes | +|---|---|---| +| `nuclei` | Template-based vulnerability scanning | directory_only mode | +| `nuclei-intense` | All URLs with robots/urlscan/wayback | More thorough, slower | +| `nuclei-technology` | Templates matching discovered tech | Targeted based on detected stack | +| `nuclei-budget` | Low-hanging fruit mode | budget:10, fastest nuclei option | + +### Fuzzing + +| Preset | Purpose | Notes | +|---|---|---| +| `dirbust-light` | Basic directory brute-force | 1000-line wordlist | +| `dirbust-heavy` | Recursive directory brute-force | 5000-line wordlist, depth:3 | +| `lightfuzz-light` | Basic fuzzing | path, sqli, xss only | +| `lightfuzz-medium` | All fuzzing modules | No POST requests | +| `lightfuzz-heavy` | Intense fuzzing | Includes POST and paramminer | +| `paramminer` | Parameter discovery | Brute-force parameter names | + +### Specialized + +| Preset | Purpose | +|---|---| +| `baddns-intense` | DNS misconfiguration checks (CNAME, MX, NS, TXT) | +| `iis-shortnames` | IIS shortname enumeration | +| `dotnet-audit` | Comprehensive IIS/.NET scanning | +| `fast` | Minimal discovery, strict scope | +| `kitchen-sink` | Everything combined (use with caution on large targets) | + +## Flags (-f flag) + +Flags enable groups of modules sharing a characteristic. + +| Flag | Description | Use When | +|---|---|---| +| `passive` | No direct target contact | Stealth required | +| `safe` | Non-intrusive modules only | Production systems | +| `active` | Modules that contact target | Standard engagement | +| `aggressive` | Potentially disruptive | Lab/controlled environment | +| `subdomain-enum` | All subdomain discovery | Comprehensive DNS mapping | +| `web-basic` | Essential web modules | Quick web assessment | +| `web-thorough` | Extended web modules | Deep web analysis | +| `web-screenshots` | Visual capture | Screenshot collection | +| `portscan` | Port scanning | Network service discovery | +| `cloud-enum` | Cloud resources | Cloud-focused targets | +| `code-enum` | Code repositories | OSINT / code leakage | + +## Key Modules + +### Subdomain Discovery +- `dnsbrute` — Active DNS brute-forcing with wordlists +- `certspotter` / `crt` — Certificate transparency logs +- `dnsdumpster` — DNSDumpster.com queries (passive) +- `wayback` — Archive.org historical data +- `shodan_dns` — Shodan DNS database (requires API key) +- `securitytrails` — Historical DNS records (requires API key) + +### Web Analysis +- `httpx` — Fast web service detection, status codes, titles +- `gowitness` — Web page screenshots (configurable resolution) +- `wappalyzer` — Technology fingerprinting +- `ffuf` — Fast web fuzzer for directories/files +- `nuclei` — Template-based vulnerability scanner + +### Cloud Resources +- `bucket_amazon` / `bucket_azure` / `bucket_google` — Storage bucket enumeration +- `azure_realm` / `azure_tenant` — Azure-specific enumeration +- `oauth` — OAuth endpoint discovery + +### Security Testing +- `badsecrets` — Hardcoded secrets/keys detection +- `baddns` — DNS misconfigurations and potential takeovers +- `lightfuzz` — Lightweight vulnerability fuzzing +- `git` / `gitdumper` — Exposed git repository detection and dumping + +### OSINT / Code +- `github_codesearch` — Search code for secrets/info +- `dockerhub` — Docker image discovery +- `postman` — API documentation discovery +- `social` — Social media profile enumeration + +## Common Recipes + +**Passive-only subdomain discovery:** +``` +targets=["target.com"], presets=["subdomain-enum"], flags=["passive"] +``` + +**Subdomain enum + basic web scan:** +``` +targets=["target.com"], presets=["subdomain-enum", "web-basic"] +``` + +**Targeted nuclei scan on known hosts:** +``` +targets=["api.target.com", "admin.target.com"], presets=["nuclei"] +``` + +**Technology detection across all subdomains:** +``` +targets=["target.com"], presets=["subdomain-enum", "tech-detect"] +``` + +**Screenshot collection:** +``` +targets=["target.com"], modules=["gowitness"], presets=["subdomain-enum"] +``` + +**Cloud resource hunt:** +``` +targets=["target.com"], presets=["cloud-enum"] +``` + +**Deep web spider on specific app:** +``` +targets=["app.target.com"], presets=["spider"], config=["web.spider_distance=2", "web.spider_depth=3"] +``` + +**Full kitchen sink (small targets only):** +``` +targets=["target.com"], presets=["kitchen-sink"] +``` + +## Configuration Tips + +- **API keys**: Configure in `~/.config/bbot/bbot.yaml` for modules like Shodan, SecurityTrails, VirusTotal +- **Scope control**: Use `extra_args=["--strict-scope"]` to prevent scope creep +- **Proxy**: Use `extra_args=["--proxy", "http://127.0.0.1:8080"]` to route through a proxy +- **Custom headers**: Use `extra_args=["--custom-headers", "Authorization=Bearer token"]` +- **Timeouts**: Set via config: `config=["modules.http.timeout=10"]` diff --git a/capabilities/attack-surface-management/skills/cypher-query-playbook/SKILL.md b/capabilities/attack-surface-management/skills/cypher-query-playbook/SKILL.md new file mode 100644 index 0000000..57920ba --- /dev/null +++ b/capabilities/attack-surface-management/skills/cypher-query-playbook/SKILL.md @@ -0,0 +1,324 @@ +--- +name: cypher-query-playbook +description: Neo4j Cypher query patterns for analyzing BBOT reconnaissance data in the graph database. Use when you need to analyze scan results, map infrastructure, find anomalies, or synthesize findings from the attack surface graph. +--- + +# Cypher Query Playbook + +## Quick Reference + +## Schema Compatibility + +Run `get_db_schema()` before using relationship-heavy queries. BBOT-backed +graphs use the BBOT event envelope. Treat `.data` as the canonical event value; +supporting fields may include `id`, `uuid`, `type`, `tags`, `scope_distance`, +`module`, and `scan`. Do not assume convenience fields such as `.name`, +`.address`, `.version`, or `.provider`. Relationship names vary by module and +record type; inspect them before writing analysis queries. Common relationship +types include `A`, `CNAME`, `httpx`, `portscan`, and `nuclei`. + +### Orientation Queries (Run First) + +**Asset summary:** +```cypher +MATCH (n) RETURN labels(n)[0] as type, count(n) AS count ORDER BY count DESC +``` + +**Recent scans:** +```cypher +MATCH (s:SCAN) RETURN s.data, s.id ORDER BY s.id DESC LIMIT 10 +``` + +**Database schema:** +Use the `get_db_schema` tool for a complete schema overview. + +--- + +## Finding High-Value Assets + +**Dev/test/staging subdomains:** +```cypher +MATCH (n:DNS_NAME) +WITH coalesce(n.name, n.data, n.host) AS name +WHERE name =~ '.*(dev|test|stage|uat|vpn|api|admin|internal|staging|qa|sandbox).*' +RETURN name ORDER BY name +``` + +**Interesting web page titles:** +```cypher +MATCH (n:URL) +WHERE n.status_code = 200 +AND n.title =~ '.*(Login|Admin|Dashboard|Unauthorized|Forbidden|Console|Manager|Portal|Panel|Config).*' +RETURN n.name, n.title +``` + +**Critical and high findings:** +```cypher +MATCH (f:FINDING) +WHERE f.severity IN ['critical', 'high'] +RETURN f.type, f.severity, f.description, f.data +``` + +**Admin panels and login pages:** +```cypher +MATCH (n:URL) +WHERE n.name =~ '.*(admin|panel|dashboard|console|login|signin|auth).*' +AND n.status_code < 400 +RETURN n.name, n.status_code, n.title +``` + +--- + +## Infrastructure Mapping + +**DNS to IP resolution:** +```cypher +MATCH (d:DNS_NAME)-[:RESOLVES_TO]->(ip:IP_ADDRESS) +RETURN d.name, ip.address +ORDER BY ip.address +``` + +**Find all domains on a specific IP:** +```cypher +MATCH (ip:IP_ADDRESS {address: $ip})<-[:RESOLVES_TO]-(d:DNS_NAME) +RETURN ip.address, collect(d.name) AS domains +``` + +**Find IPs for a domain:** +```cypher +MATCH (d:DNS_NAME {name: $domain})-[:RESOLVES_TO]->(ip:IP_ADDRESS) +RETURN d.name, ip.address +``` + +**Shared hosting (IPs with multiple domains):** +```cypher +MATCH (ip:IP_ADDRESS)<-[:RESOLVES_TO]-(d:DNS_NAME) +WITH ip, collect(d.name) AS domains, count(d) as cnt +WHERE cnt > 1 +RETURN ip.address, cnt, domains +ORDER BY cnt DESC +``` + +**Reverse DNS — all domains per IP:** +```cypher +MATCH (ip:IP_ADDRESS)<-[:RESOLVES_TO]-(d:DNS_NAME) +RETURN ip.address, collect(d.name) as domains +ORDER BY size(collect(d.name)) DESC +``` + +--- + +## Service Discovery + +**Web services responding 200:** +```cypher +MATCH (n:URL) +WHERE n.status_code >= 200 AND n.status_code < 300 +RETURN n.name, n.status_code, n.title +LIMIT 50 +``` + +**API endpoints:** +```cypher +MATCH (n:URL) +WHERE n.name CONTAINS '/api/' OR n.name CONTAINS '/v1/' OR n.name CONTAINS '/v2/' OR n.name CONTAINS '/graphql' +RETURN n.name, n.status_code, n.title +``` + +**Interesting ports (databases, admin services):** +```cypher +MATCH (p:OPEN_TCP_PORT) +WHERE p.port IN [3306, 5432, 6379, 27017, 9200, 8080, 8443, 9090, 3389, 5900, 11211] +MATCH (ip:IP_ADDRESS)-[:HAS_PORT]->(p) +RETURN ip.address, p.port, p.service +``` + +**Services by port:** +```cypher +MATCH (ip:IP_ADDRESS)-[:HAS_PORT]->(p:OPEN_TCP_PORT) +RETURN p.port, count(ip) as host_count +ORDER BY host_count DESC +LIMIT 20 +``` + +--- + +## Technology Analysis + +**All discovered technologies:** +```cypher +MATCH (t:TECHNOLOGY) +RETURN DISTINCT t.name, t.version, count(*) as usage_count +ORDER BY usage_count DESC +``` + +**Technology stack for a host:** +```cypher +MATCH (d:DNS_NAME {name: $domain})-[:RESOLVES_TO]->(ip)-[:HAS_PORT]->()-[:HAS_TECHNOLOGY]->(t) +RETURN d.name, t.name, t.version +``` + +**Technology outliers (old/unusual software):** +```cypher +MATCH (t:TECHNOLOGY) +WITH t, coalesce(t.name, t.data) AS tech_name +WHERE tech_name IN ['JBoss', 'ColdFusion', 'Struts', 'WebLogic', 'Tomcat', 'IIS'] +OR t.version =~ '.*[0-4]\\..*' +MATCH (n)-[:HAS_TECHNOLOGY]->(t) +RETURN labels(n)[0] as asset_type, coalesce(n.name, n.data, n.host) AS asset, tech_name, t.version +``` + +**Assets with a specific technology:** +```cypher +MATCH (t:TECHNOLOGY {name: $tech_name})<-[:HAS_TECHNOLOGY]-(n) +RETURN labels(n)[0] as type, n.name, t.version +``` + +--- + +## Security Analysis + +**All findings by severity:** +```cypher +MATCH (f:FINDING) +RETURN f.severity, count(f) as count +ORDER BY CASE f.severity + WHEN 'critical' THEN 0 + WHEN 'high' THEN 1 + WHEN 'medium' THEN 2 + WHEN 'low' THEN 3 + ELSE 4 +END +``` + +**Findings with affected assets:** +```cypher +MATCH (asset)-[:HAS_FINDING]->(f:FINDING) +RETURN f.type, f.severity, f.description, labels(asset)[0] as asset_type, asset.name +ORDER BY f.severity +``` + +**Public storage buckets:** +```cypher +MATCH (n:STORAGE_BUCKET) +WHERE n.public = true +RETURN n.name, n.url +``` + +**Exposed databases:** +```cypher +MATCH (p:OPEN_TCP_PORT) +WHERE p.port IN [3306, 5432, 6379, 27017, 9200, 5984, 11211] +MATCH (ip:IP_ADDRESS)-[:HAS_PORT]->(p) +OPTIONAL MATCH (ip)<-[:RESOLVES_TO]-(d:DNS_NAME) +RETURN ip.address, p.port, p.service, collect(d.name) as hostnames +``` + +--- + +## Cross-Reference & Correlation + +**Shared infrastructure for high-value assets:** +```cypher +MATCH (d:DNS_NAME)-[:RESOLVES_TO]->(ip:IP_ADDRESS) +WHERE d.name CONTAINS 'dev' OR d.name CONTAINS 'api' OR d.name CONTAINS 'staging' OR d.name CONTAINS 'admin' +WITH ip, collect(d.name) AS domains, count(*) as domainCount +WHERE domainCount > 1 +RETURN ip.address, domains +``` + +**Correlate findings by technology:** +```cypher +MATCH (f:FINDING)<-[:HAS_FINDING]-(root) +MATCH (root)-[:HAS_TECHNOLOGY]->(tech:TECHNOLOGY) +MATCH (other_asset)-[:HAS_TECHNOLOGY]->(tech) +WHERE other_asset <> root +RETURN tech.name, collect(DISTINCT other_asset.name) AS related_assets +``` + +**Discover naming conventions:** +```cypher +MATCH (d:DNS_NAME) +WHERE d.name =~ '.*(app|srv|db|web|mail|ns|mx)0[0-9].*' +RETURN collect(d.name) AS discovered_pattern +``` + +**Cross-reference: domains sharing IP with a finding:** +```cypher +MATCH (f:FINDING)<-[:HAS_FINDING]-(asset) +OPTIONAL MATCH (asset)-[:RESOLVES_TO]->(ip:IP_ADDRESS) +OPTIONAL MATCH (ip)<-[:RESOLVES_TO]-(sibling:DNS_NAME) +WHERE sibling <> asset +RETURN f.type, asset.name, ip.address, collect(DISTINCT sibling.name) as co_hosted +``` + +--- + +## Path Analysis + +**Connection paths from domain to finding:** +```cypher +MATCH p=(d:DNS_NAME)-[*1..3]-(f:FINDING) +WHERE d.name = $domain +RETURN p +``` + +**All relationships for a specific asset:** +```cypher +MATCH (n)-[r]-(m) +WHERE n.name = $name +RETURN labels(n)[0] as source_type, n.name, type(r) as relationship, labels(m)[0] as target_type, m.name +``` + +**Shortest path between two assets:** +```cypher +MATCH p=shortestPath((n1:DNS_NAME {name: $start})-[*]-(n2:DNS_NAME {name: $end})) +RETURN p +``` + +--- + +## Aggregation & Statistics + +**Top ports across all hosts:** +```cypher +MATCH (p:OPEN_TCP_PORT) +RETURN p.port, count(p) as cnt +ORDER BY cnt DESC +LIMIT 10 +``` + +**Domains per IP (distribution):** +```cypher +MATCH (ip:IP_ADDRESS)<-[:RESOLVES_TO]-(d) +RETURN ip.address, count(d) as domain_count +ORDER BY domain_count DESC +LIMIT 20 +``` + +**Email addresses by domain:** +```cypher +MATCH (e:EMAIL_ADDRESS) +WHERE e.address ENDS WITH $domain +RETURN e.address +``` + +**Cloud provider breakdown:** +```cypher +MATCH (ip:IP_ADDRESS) +WHERE ip.provider IS NOT NULL +RETURN ip.provider, count(ip) as count +ORDER BY count DESC +``` + +--- + +## Tips + +- **Always use parameters** (`$param`) for user input to prevent Cypher injection +- **Start with small limits** (10-20) and increase if needed +- **Use regex escaping** (`\\`) for special characters in patterns +- **Combine queries** in the Orient phase to build a complete picture before deciding on the next scan +- **Date filtering**: `WHERE n.created_at > datetime('2024-01-01')` +- **NOT conditions**: `WHERE NOT n.status_code IN [404, 403, 401]` +- **Case-insensitive regex**: `WHERE n.name =~ '(?i).*admin.*'` diff --git a/capabilities/attack-surface-management/skills/reconnaissance-planning/SKILL.md b/capabilities/attack-surface-management/skills/reconnaissance-planning/SKILL.md new file mode 100644 index 0000000..8cb3d63 --- /dev/null +++ b/capabilities/attack-surface-management/skills/reconnaissance-planning/SKILL.md @@ -0,0 +1,113 @@ +--- +name: reconnaissance-planning +description: Strategic reconnaissance planning for external attack surface management. Use when starting a new engagement, choosing initial scan strategy, or deciding how to expand coverage after initial results. +--- + +# Reconnaissance Planning + +## When to Use + +- Starting a new ASM engagement against a target +- Deciding scan strategy after initial enumeration +- Expanding coverage when initial results are sparse +- Pivoting approach based on discovered infrastructure + +## Phased Approach + +### Phase 1: Passive Discovery (Low Noise) + +Start with passive techniques to map the target without generating traffic. + +**Subdomain Enumeration:** +``` +run_bbot_scan(targets=["target.com"], presets=["subdomain-enum"], flags=["passive"]) +``` + +This queries certificate transparency logs, DNS databases, archive.org, and OSINT sources without touching the target directly. + +**What to look for after Phase 1:** +- Total number of discovered subdomains (establishes scale) +- Naming conventions (numbered patterns like `app01`, `srv02` suggest more exist) +- Cloud providers visible in DNS (CNAME to AWS, Azure, GCP) +- Mail infrastructure (MX records, SPF/DKIM) +- Development/staging indicators in subdomain names + +### Phase 2: Active Enumeration (Moderate Noise) + +Probe discovered assets to understand what's running. + +**Web service detection + technology fingerprinting:** +``` +run_bbot_scan(targets=["target.com"], presets=["subdomain-enum", "web-basic"]) +``` + +**Port scanning on key IPs:** +``` +run_bbot_scan(targets=["10.0.0.1"], modules=["portscan"], config=["modules.portscan.ports=top_1000"]) +``` + +**What to look for after Phase 2:** +- HTTP services on non-standard ports +- Technology stack patterns (is everything Rails? Is there one random PHP app?) +- Login pages, admin panels, API documentation +- SSL certificate details (org names, SANs with more domains) +- Default pages or error pages revealing server versions + +### Phase 3: Targeted Deep Scanning (Focused) + +Focus on high-value targets identified in Phase 2. + +**Vulnerability scanning on interesting hosts:** +``` +run_bbot_scan(targets=["api.target.com"], presets=["nuclei"]) +``` + +**Web spidering on complex applications:** +``` +run_bbot_scan(targets=["app.target.com"], presets=["spider"], config=["web.spider_distance=2", "web.spider_depth=3"]) +``` + +**Screenshot collection for visual triage:** +``` +run_bbot_scan(targets=["target.com"], modules=["gowitness"], presets=["subdomain-enum"]) +``` + +**Cloud resource enumeration:** +``` +run_bbot_scan(targets=["target.com"], presets=["cloud-enum"]) +``` + +### Phase 4: Expansion & Synthesis + +- Cross-reference findings across phases using graph queries +- Look for patterns: shared infrastructure, common technologies, consistent misconfigurations +- Identify assets that warrant manual investigation +- Build the final areas-of-interest list + +## Decision Tree: Choosing Scan Strategy + +``` +Is this a new target with no prior data? +├── YES → Start with Phase 1 (passive subdomain-enum) +└── NO → Do you have subdomains but no service info? + ├── YES → Run web-basic + tech-detect + └── NO → Do you have services but no vuln data? + ├── YES → Run nuclei on interesting hosts + └── NO → Focus on graph analysis and synthesis +``` + +## Scale Considerations + +| Target Size | Approach | +|---|---| +| Single domain | kitchen-sink preset covers everything | +| Small org (< 50 subdomains) | Full phased approach, thorough coverage | +| Medium org (50-500 subdomains) | Passive first, then targeted active scans on high-value assets | +| Large org (500+ subdomains) | Passive + selective active, prioritize by naming/business context | + +## Common Pitfalls + +- **Going too broad too early**: Don't run `kitchen-sink` on a large target. You'll drown in data. +- **Ignoring the graph**: Running scans without querying results between phases wastes cycles. +- **Scanning out of scope**: Always use `--whitelist` or `--strict-scope` for targets with clear boundaries. +- **Missing API keys**: Many passive modules (Shodan, SecurityTrails, etc.) need API keys configured in bbot.yml. Check if they're set before relying on passive results. diff --git a/capabilities/attack-surface-management/skills/screenshot-triage/SKILL.md b/capabilities/attack-surface-management/skills/screenshot-triage/SKILL.md new file mode 100644 index 0000000..8edf54b --- /dev/null +++ b/capabilities/attack-surface-management/skills/screenshot-triage/SKILL.md @@ -0,0 +1,105 @@ +--- +name: screenshot-triage +description: Triage web application screenshots to identify high-value targets for manual investigation. Use when analyzing WEBSCREENSHOT nodes from BBOT scans or when visually assessing discovered web assets. +--- + +# Screenshot Triage + +## Purpose + +Triage web screenshots captured by BBOT (via gowitness) to identify high-value targets for human follow-up. The goal is vulnerability discovery — a screenshot is "interesting" if it suggests a high likelihood of success for a human tester. + +## Retrieving Screenshots + +```cypher +-- Find all unanalyzed screenshots +MATCH (s:WEBSCREENSHOT) WHERE s.analyzed IS NULL RETURN s.uuid, s.url + +-- Get screenshot for a specific URL +MATCH (s:WEBSCREENSHOT) WHERE s.url CONTAINS 'admin' RETURN s.uuid, s.url + +-- Get screenshots with their host context +MATCH (s:WEBSCREENSHOT)-[]-(parent) +RETURN s.uuid, s.url, labels(parent)[0] as parent_type, parent.name +``` + +Use `get_screenshot(uuid=...)` or `get_screenshot(url=...)` to retrieve the actual image. + +## Triage Principles + +1. **Intent is Vulnerability Discovery**: Focus on pages suggesting high exploitation potential. +2. **Structure Over Content**: Page structure (forms, dashboards, admin panels) matters more than marketing text. +3. **Appearance as Clue**: Bare-bones internal tools often have weaker security than polished public pages. +4. **Prioritize Interaction Points**: Forms, dashboards, and control panels are far more valuable than static pages. + +## Priority Classification + +### Critical + +- Administrative interfaces, control panels, backend management systems +- Login forms specifying "admin", "staff", or "internal" +- Database management interfaces (phpMyAdmin, Adminer, pgAdmin) +- Infrastructure dashboards (Grafana, Kibana, Jenkins, GitLab) + +### High + +- API documentation pages (Swagger, OpenAPI, Redoc, GraphQL Playground) +- Developer consoles or debugging interfaces +- Complex forms handling sensitive data (user settings, financial info) +- File upload functionality +- Pages displaying error messages, stack traces, or debug output +- Internal tools not meant for public viewing + +### Medium + +- Standard login forms without admin indicators +- Search functionality (potential for injection) +- User registration/account management pages +- Pages revealing technology versions or server information +- Legacy-looking applications built on old frameworks + +### Low + +- Marketing pages, blogs, documentation +- Static content with no interaction points +- Generic error pages (404, 403) without information leakage +- CDN or asset-serving endpoints + +## What to Look For + +### Visual Indicators of High Value + +- **Unstyled or minimal design**: Internal tools, dev environments +- **Framework default pages**: Fresh installs, unconfigured services +- **Data tables**: Internal metrics, user data, logs +- **Terminal/console interfaces**: Web shells, command runners +- **Multiple form fields**: Complex data entry suggesting business logic +- **Version numbers in footers**: Technology identification + +### Technology Clues + +- Specific software names or logos (WordPress, Jira, Confluence, Jenkins) +- URL patterns visible in screenshots (e.g., `/wp-admin/`, `/administrator/`) +- Framework-specific UI elements (Django admin, Rails scaffolding, Spring Boot Actuator) +- Cloud provider indicators (AWS console, Azure portal elements) + +### Red Flags + +- Stack traces with file paths and line numbers +- Database error messages with query fragments +- Debug toolbars (Django Debug Toolbar, Symfony Profiler) +- phpinfo() output +- Directory listings +- Default credentials displayed or hinted at +- "Powered by" footers with version information + +## Workflow + +1. Query for unanalyzed screenshots +2. For each screenshot: + a. Retrieve and examine the image + b. Classify priority (critical/high/medium/low) + c. Note specific elements of interest + d. Record what a human should investigate next +3. Mark screenshots as analyzed: set the `analyzed` property +4. Cross-reference high-priority screenshots with other graph data (technologies, findings, DNS names) diff --git a/capabilities/attack-surface-management/skills/shodan-reconnaissance/SKILL.md b/capabilities/attack-surface-management/skills/shodan-reconnaissance/SKILL.md new file mode 100644 index 0000000..65a0098 --- /dev/null +++ b/capabilities/attack-surface-management/skills/shodan-reconnaissance/SKILL.md @@ -0,0 +1,168 @@ +--- +name: shodan-reconnaissance +description: Shodan query strategies for internet-wide host intelligence, asset discovery, vulnerability correlation, and attack surface mapping. Use when enriching BBOT results with Shodan data, hunting for exposed services, or correlating CVEs with discovered infrastructure. +--- + +# Shodan Reconnaissance + +## Purpose + +Use Shodan to complement BBOT reconnaissance with internet-wide scan intelligence. Shodan provides passive host data, banner analysis, CVE correlation, and historical records that BBOT's active scanning may miss. + +For evaluation tasks, the Shodan MCP can be pointed at a task-local mock service +by setting `SHODAN_API_URL` while still providing any non-empty +`SHODAN_API_KEY`. When `SHODAN_API_URL` is unset, the MCP uses the official +Shodan API client. + +## Integration with BBOT Graph + +The primary workflow is: **BBOT discovers assets → Shodan enriches them.** + +1. Query the Neo4j graph for discovered IPs and domains +2. Use Shodan to enrich with service details, CVEs, and historical data +3. Feed Shodan findings back into analysis + +### Enrichment Patterns + +**Enrich all discovered IPs:** +```cypher +MATCH (ip:IP_ADDRESS) RETURN ip.address +``` +Then for each IP: `shodan_host_info(ip="x.x.x.x")` + +**Find org-wide exposure:** +``` +shodan_host_search(query='org:"Target Corp"') +``` +Cross-reference results against known IPs in the graph. + +**Discover assets BBOT missed:** +``` +shodan_host_search(query='ssl.cert.subject.cn:target.com') +shodan_host_search(query='hostname:target.com') +``` +Compare with `MATCH (d:DNS_NAME) RETURN d.name` to find gaps. + +## Query Strategies + +### Asset Discovery + +| Goal | Query | +|---|---| +| All hosts for an org | `org:"Target Corp"` | +| Hosts by hostname | `hostname:target.com` | +| Hosts by SSL cert | `ssl.cert.subject.cn:target.com` | +| Hosts by IP range | `net:10.0.0.0/24` | +| Hosts by ASN | `asn:AS12345` | +| Cloud-hosted assets | `org:"Amazon" hostname:target.com` | + +### Service Hunting + +| Goal | Query | +|---|---| +| Web servers | `hostname:target.com port:80,443` | +| Remote desktop | `port:3389 org:"Target Corp"` | +| SSH servers | `port:22 org:"Target Corp"` | +| Databases | `port:3306,5432,27017,6379 org:"Target Corp"` | +| Elasticsearch | `port:9200 org:"Target Corp"` | +| Docker APIs | `port:2375,2376 org:"Target Corp"` | +| Kubernetes | `port:6443,10250 org:"Target Corp"` | + +### Vulnerability Hunting + +| Goal | Query | +|---|---| +| Hosts with known CVEs | `vuln:CVE-2021-44228 org:"Target Corp"` | +| Outdated SSL | `ssl.version:sslv2 hostname:target.com` | +| Expired certs | `ssl.cert.expired:true hostname:target.com` | +| Self-signed certs | `ssl.cert.issuer.cn:target.com hostname:target.com` | +| Default credentials | `"default password" org:"Target Corp"` | +| Specific product vulns | `product:"Apache" version:"2.4.49"` | + +### Technology Fingerprinting + +| Goal | Query | +|---|---| +| Specific product | `product:"nginx" org:"Target Corp"` | +| HTTP title match | `http.title:"Dashboard" org:"Target Corp"` | +| Specific server header | `"Server: Apache/2.4.49"` | +| WAF detection | `http.waf:"Cloudflare" hostname:target.com` | +| CMS detection | `http.component:"WordPress" hostname:target.com` | + +## Workflow: Full ASM Enrichment + +### Step 1: Check Credits +Always start here to avoid hitting limits. +``` +shodan_api_info() +``` + +### Step 2: Scope Assessment +Use `shodan_count` (free, no credits) before committing to full searches. +``` +shodan_count(query='org:"Target Corp"') +shodan_count(query='hostname:target.com') +shodan_count(query='ssl.cert.subject.cn:target.com') +``` + +### Step 3: Broad Discovery +Run full searches on the most productive queries. +``` +shodan_host_search(query='org:"Target Corp"', facets='port,product,country') +``` +Facets give you instant distribution analysis without paging through results. + +### Step 4: Targeted Enrichment +For high-value IPs found by BBOT, get full details. +``` +shodan_host_info(ip="x.x.x.x", history=True) +``` +Historical data reveals services that were recently taken down or changed. + +### Step 5: Exploit Correlation +For any CVEs found on hosts, check exploit availability. +``` +shodan_exploits_search(query="CVE-2024-XXXXX") +``` + +### Step 6: DNS Cross-Reference +Validate and expand DNS data. +``` +shodan_dns_lookup(hostnames=["api.target.com", "dev.target.com"]) +shodan_dns_reverse(ips=["x.x.x.x", "y.y.y.y"]) +``` + +## Facet Analysis + +Facets are the most powerful Shodan feature for ASM. They aggregate across all results without pagination. + +**Key facets:** +- `port` — Service distribution +- `product` — Technology distribution +- `country` — Geographic distribution +- `org` — Hosting provider distribution +- `os` — Operating system distribution +- `vuln` — CVE distribution (requires paid plan) +- `ssl.version` — SSL/TLS version distribution +- `http.title` — Web page title distribution + +**Example: Full surface profile in one query:** +``` +shodan_host_search( + query='org:"Target Corp"', + facets='port,product,country,os,vuln,ssl.version' +) +``` + +## Credit Management + +| Operation | Credit Cost | +|---|---| +| `shodan_count` | Free | +| `shodan_host_search` | 1 query credit per page | +| `shodan_host_info` | Free (IP lookups are free) | +| `shodan_dns_lookup` | Free | +| `shodan_dns_reverse` | Free | +| `shodan_exploits_search` | Free | + +**Strategy**: Use `count` + facets first, `host_info` for specific IPs (free), and reserve `host_search` for when you need the full match list. diff --git a/capabilities/attack-surface-management/tests/test_bbot_mcp.py b/capabilities/attack-surface-management/tests/test_bbot_mcp.py new file mode 100644 index 0000000..62670da --- /dev/null +++ b/capabilities/attack-surface-management/tests/test_bbot_mcp.py @@ -0,0 +1,391 @@ +from __future__ import annotations + +import asyncio +import importlib.util +import json +from pathlib import Path + +import pytest + + +MODULE_PATH = Path(__file__).resolve().parents[1] / "mcp" / "bbot.py" + + +def load_bbot_module(): + spec = importlib.util.spec_from_file_location("asm_bbot_mcp_test", MODULE_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class FakeNeo4j: + def __init__(self, responses: list[list[dict]] | None = None) -> None: + self.responses = list(responses or []) + self.calls: list[tuple[str, dict | None]] = [] + + async def query(self, cypher: str, params: dict | None = None) -> list[dict]: + self.calls.append((cypher, params)) + if self.responses: + return self.responses.pop(0) + return [] + + +def run(coro): + return asyncio.run(coro) + + +def test_get_db_schema_collects_labels_relationships_and_properties(monkeypatch): + bbot = load_bbot_module() + fake = FakeNeo4j( + [ + [{"label": "DNS_NAME"}, {"label": "URL"}], + [{"relationshipType": "A"}, {"relationshipType": "httpx"}], + [ + { + "nodeType": ":DNS_NAME", + "propertyName": "data", + "propertyTypes": ["String"], + "mandatory": False, + } + ], + [ + { + "relType": ":A", + "propertyName": "module", + "propertyTypes": ["String"], + "mandatory": False, + } + ], + ] + ) + monkeypatch.setattr(bbot, "_neo4j", fake) + + result = json.loads(run(bbot.get_db_schema())) + + assert result["node_labels"] == ["DNS_NAME", "URL"] + assert result["relationship_types"] == ["A", "httpx"] + assert result["node_properties"]["DNS_NAME"][0]["property"] == "data" + assert result["relationship_properties"]["A"][0]["property"] == "module" + assert len(fake.calls) == 4 + + +def test_explore_nodes_builds_parameterized_filter(monkeypatch): + bbot = load_bbot_module() + fake = FakeNeo4j([[{"node": {"data": "dev.target.local"}}]]) + monkeypatch.setattr(bbot, "_neo4j", fake) + + result = json.loads(run(bbot.explore_nodes("DNS_NAME", "data CONTAINS dev", 25))) + + cypher, params = fake.calls[0] + assert "MATCH (node:DNS_NAME)" in cypher + assert "node.`data`" in cypher + assert "$value" in cypher + assert params == {"limit": 25, "value": "dev"} + assert result[0]["node"]["data"] == "dev.target.local" + + +def test_explore_nodes_rejects_cypher_identifier_injection(): + bbot = load_bbot_module() + + with pytest.raises(ValueError, match="Invalid label"): + run(bbot.explore_nodes("DNS_NAME) DETACH DELETE n //", None, 10)) + + with pytest.raises(ValueError, match="Invalid property"): + run(bbot.explore_nodes("DNS_NAME", "data`) MATCH (n) RETURN n //=x", 10)) + + +def test_explore_relationships_validates_identifiers_and_limits(monkeypatch): + bbot = load_bbot_module() + fake = FakeNeo4j([[{"source": "a", "relationship": "r", "target": "b"}]]) + monkeypatch.setattr(bbot, "_neo4j", fake) + + run(bbot.explore_relationships("DNS_NAME", "A", "IP_ADDRESS", 3)) + + cypher, params = fake.calls[0] + assert "MATCH (source:DNS_NAME)-[relationship:A]->(target:IP_ADDRESS)" in cypher + assert params == {"limit": 3} + with pytest.raises(ValueError, match="Limit"): + run(bbot.explore_relationships(limit=0)) + + +def test_get_subdomains_and_technologies_use_envelope_fallbacks(monkeypatch): + bbot = load_bbot_module() + fake = FakeNeo4j( + [ + [{"name": "api.target.local"}], + [{"name": "JBoss Application Server", "version": "4.0", "usage": 1}], + ] + ) + monkeypatch.setattr(bbot, "_neo4j", fake) + + assert ( + json.loads(run(bbot.get_subdomains("target.local", 100)))[0]["name"] + == "api.target.local" + ) + assert ( + json.loads(run(bbot.get_technologies()))[0]["name"] + == "JBoss Application Server" + ) + + subdomain_query = fake.calls[0][0] + technology_query = fake.calls[1][0] + assert "coalesce(n.name, n.data, n.host)" in subdomain_query + assert "coalesce(t.name, t.data)" in technology_query + + +def test_get_screenshot_resolves_bbot_scan_path(monkeypatch, tmp_path): + bbot = load_bbot_module() + screenshot = tmp_path / "scans" / "scan-one" / "screenshots" / "app.png" + screenshot.parent.mkdir(parents=True) + screenshot.write_bytes(b"png") + + fake = FakeNeo4j( + [ + [ + { + "web_props": { + "uuid": "shot-1", + "data": json.dumps( + { + "path": "screenshots/app.png", + "url": "https://app.target.local", + } + ), + "scan": "scan-one", + }, + "scan_props": {"data": json.dumps({"name": "scan-one"})}, + } + ] + ] + ) + monkeypatch.setattr(bbot, "_neo4j", fake) + monkeypatch.setattr(bbot, "BBOT_DATA_DIR", str(tmp_path)) + + result = json.loads(run(bbot.get_screenshot(uuid="shot-1"))) + + assert result["path"] == str(screenshot) + assert result["url"] == "https://app.target.local" + assert result["uuid"] == "shot-1" + + +def test_get_screenshot_reports_checked_paths_when_missing(monkeypatch, tmp_path): + bbot = load_bbot_module() + fake = FakeNeo4j( + [ + [ + { + "web_props": { + "uuid": "shot-1", + "path": "missing.png", + "scan": "scan-one", + }, + "scan_props": {"name": "scan-one"}, + } + ] + ] + ) + monkeypatch.setattr(bbot, "_neo4j", fake) + monkeypatch.setattr(bbot, "BBOT_DATA_DIR", str(tmp_path)) + + result = json.loads(run(bbot.get_screenshot(uuid="shot-1"))) + + assert result["error"] == "Screenshot file not found." + assert ( + str(tmp_path / "scans" / "scan-one" / "missing.png") in result["checked_paths"] + ) + + +def test_run_bbot_scan_with_graph_api_uses_stdout_not_neo4j(monkeypatch): + bbot = load_bbot_module() + captured: list[str] = [] + captured_env: dict | None = None + + class FakeProc: + returncode = 0 + + async def communicate(self): + return ( + b'{"type":"DNS_NAME","scope_description":"in-scope","module":"dns","data":"api.example.com"}\n', + b"", + ) + + async def fake_create_subprocess_exec(*args, **kwargs): + nonlocal captured_env + captured.extend(args) + captured_env = kwargs.get("env") + return FakeProc() + + monkeypatch.setattr( + bbot.asyncio, "create_subprocess_exec", fake_create_subprocess_exec + ) + + result = run( + bbot.run_bbot_scan( + targets=["*.example.com"], + modules=[ + "subfinder", + "subdomainfinder", + "subdomain_bruteforce", + "subenum", + "crtsh", + "dns_zone_transfer", + "fastdial", + "find_subdomains", + "gau", + "gcp_storage_scan", + "portscan", + "s3_scan", + "azure_blob_scan", + "technologies", + "massdns", + "naabu", + "nuclei", + "screenshot", + "web_screenshots", + "web_screenshot", + "webenum", + "wappalyzer", + "httpx", + ], + flags=["subdomain-enum", "-y", "--t=25"], + presets=[ + "asset-discovery-and-enrichment", + "asset-discovery", + "asset-discovery-web-endpoints", + "discovery", + "subdomain_enumeration", + "subdomain_discovery", + "web_breach", + "web_basic", + "web_discovery", + "web_scan", + "web-port-scan-and-tech-detect", + "web-and-portscan", + "nuclei", + ], + config=[ + "modules.httpx.timeout=10", + "modules.shodan_dns.enabled=false", + "scope.distance=1", + "scope.report_distance=1", + ], + extra_args=["--scope-distance", "1"], + graph_api_url="http://graph.local", + ) + ) + + assert "--json" in captured + assert "--output-modules" in captured + assert "stdout" in captured + assert "--no-deps" in captured + assert "--exclude-modules" in captured + assert "portscan" in captured + assert "gowitness" in captured + assert "baddns" in captured + assert "httpx" in captured + assert "subdomaincenter" in captured + assert "bucket_amazon" in captured + assert "bucket_google" in captured + assert "bucket_microsoft" in captured + assert "subfinder" not in captured + assert "subdomainfinder" not in captured + assert "subdomain_bruteforce" not in captured + assert "subenum" not in captured + assert "crtsh" not in captured + assert "dns_zone_transfer" not in captured + assert "fastdial" not in captured + assert "find_subdomains" not in captured + assert "gau" not in captured + assert "massdns" not in captured + assert "naabu" not in captured + assert "nuclei" not in captured + assert "screenshot" not in captured + assert "web_screenshots" not in captured + assert "web_screenshot" not in captured + assert "webenum" not in captured + assert "wappalyzer" not in captured + assert "-y" not in captured + assert "--t=25" not in captured + assert "web-and-portscan" not in captured + assert "subdomain-enum" in captured + assert "web-basic" in captured + assert "web_basic" not in captured + assert "subdomain_enumeration" not in captured + assert "asset-discovery" not in captured + assert "asset-discovery-web-endpoints" not in captured + assert captured_env is not None + assert "asm-bbot-home-" in captured_env["HOME"] + assert "neo4j" not in captured + assert "example.com" in captured + assert "*.example.com" not in captured + assert "modules.http.timeout=10" in captured + assert "scope.search_distance=1" in captured + assert "scope.distance=1" not in captured + assert "scope.report_distance=1" not in captured + assert "--scope-distance" not in captured + assert "modules.shodan_dns.enabled=false" not in captured + assert "modules.neo4j.uri=bolt://localhost:7687" not in captured + assert "event_count" in result + assert "api.example.com" in result + + +def test_run_bbot_scan_with_graph_api_reports_diagnostic_only_output(monkeypatch): + bbot = load_bbot_module() + + class FakeProc: + returncode = 0 + + async def communicate(self): + return ( + b"[WARN] Please specify --allow-deadly to continue\n", + b"", + ) + + async def fake_create_subprocess_exec(*args, **kwargs): + return FakeProc() + + monkeypatch.setattr( + bbot.asyncio, "create_subprocess_exec", fake_create_subprocess_exec + ) + + result = run( + bbot.run_bbot_scan( + targets=["example.com"], + presets=["nuclei-budget"], + graph_api_url="http://graph.local", + ) + ) + + assert "produced no usable BBOT JSON events" in result + assert "Please specify --allow-deadly" in result + assert '"event_count": 0' in result + + +def test_query_graph_uses_graph_api_when_provided(monkeypatch): + bbot = load_bbot_module() + fake = FakeNeo4j([[{"should_not": "be_used"}]]) + calls: list[tuple[str, str, dict | None]] = [] + monkeypatch.setattr(bbot, "_neo4j", fake) + + def fake_query_graph_api(url, cypher, params=None): + calls.append((url, cypher, params)) + return [{"count": 3}] + + monkeypatch.setattr(bbot, "_query_graph_api", fake_query_graph_api) + + result = json.loads( + run( + bbot.query_graph( + "MATCH (n) RETURN count(n) AS count", + {"limit": 10}, + graph_api_url="http://graph.local", + ) + ) + ) + + assert result == [{"count": 3}] + assert calls == [ + ("http://graph.local", "MATCH (n) RETURN count(n) AS count", {"limit": 10}) + ] + assert fake.calls == [] diff --git a/capabilities/attack-surface-management/tests/test_shodan_mcp.py b/capabilities/attack-surface-management/tests/test_shodan_mcp.py new file mode 100644 index 0000000..a668fb0 --- /dev/null +++ b/capabilities/attack-surface-management/tests/test_shodan_mcp.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +import importlib.util +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from urllib import parse + + +MODULE_PATH = Path(__file__).resolve().parents[1] / "mcp" / "shodan.py" + + +def load_shodan_module(): + spec = importlib.util.spec_from_file_location("asm_shodan_mcp_test", MODULE_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class MockShodanHandler(BaseHTTPRequestHandler): + requests: list[tuple[str, dict[str, list[str]]]] = [] + + def do_GET(self) -> None: + parsed = parse.urlparse(self.path) + query = parse.parse_qs(parsed.query) + self.requests.append((parsed.path, query)) + body = self.response_for(parsed.path, query) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(body).encode("utf-8")) + + def log_message(self, format: str, *args: object) -> None: + return + + @staticmethod + def response_for(path: str, query: dict[str, list[str]]) -> object: + if path == "/shodan/host/search": + return { + "total": 1, + "matches": [ + { + "ip_str": "203.0.113.10", + "port": 9200, + "org": "Target Corp", + "hostnames": ["search.target.local"], + "domains": ["target.local"], + "transport": "tcp", + "product": "Elasticsearch", + "version": "7.10.0", + "vulns": {"CVE-2021-44228": {}}, + } + ], + "facets": {"port": [{"value": 9200, "count": 1}]}, + } + if path == "/shodan/host/203.0.113.10": + return { + "ip_str": "203.0.113.10", + "org": "Target Corp", + "ports": [9200], + "hostnames": ["search.target.local"], + "domains": ["target.local"], + "vulns": ["CVE-2021-44228"], + "country_name": "United States", + "city": "New York", + "asn": "AS64500", + "isp": "Example ISP", + "data": [ + { + "port": 9200, + "product": "Elasticsearch", + "version": "7.10.0", + "data": "banner", + } + ], + } + if path == "/shodan/host/count": + return { + "total": 1, + "facets": {"product": [{"value": "Elasticsearch", "count": 1}]}, + } + if path == "/dns/resolve": + return {"search.target.local": "203.0.113.10"} + if path == "/dns/reverse": + return {"203.0.113.10": ["search.target.local"]} + if path == "/exploits/search": + return { + "total": 1, + "matches": [ + { + "_id": "EXP-1", + "description": "Elasticsearch exploit", + "author": "researcher", + "type": "remote", + "platform": "linux", + "date": "2024-01-01", + "source": "mock", + "cve": ["CVE-2021-44228"], + } + ], + "facets": {}, + } + if path == "/shodan/ports": + return [80, 443, 9200] + if path == "/shodan/protocols": + return {"http": "HTTP"} + if path == "/api-info": + return { + "plan": "mock", + "query_credits": 100, + "scan_credits": 0, + "unlocked": True, + } + if path == "/shodan/query/search": + return { + "total": 1, + "matches": [{"title": "Databases", "query": "port:9200"}], + } + if path == "/shodan/query/tags": + return ["database"] + return {} + + +def start_mock_server(): + MockShodanHandler.requests = [] + server = HTTPServer(("127.0.0.1", 0), MockShodanHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, f"http://127.0.0.1:{server.server_port}" + + +def test_http_shodan_client_uses_mock_base_url_and_api_key(): + shodan_mcp = load_shodan_module() + server, base_url = start_mock_server() + try: + client = shodan_mcp._HttpShodanClient(base_url, "test-key") + + search = client.search('org:"Target Corp"', facets="port") + host = client.host("203.0.113.10", history=True) + count = client.count("port:9200") + resolved = client.dns.resolve("search.target.local") + exploits = client.exploits.search("CVE-2021-44228") + + assert search["matches"][0]["ip_str"] == "203.0.113.10" + assert host["ports"] == [9200] + assert count["total"] == 1 + assert resolved["search.target.local"] == "203.0.113.10" + assert exploits["matches"][0]["cve"] == ["CVE-2021-44228"] + assert all( + request_query.get("key") == ["test-key"] + for _, request_query in MockShodanHandler.requests + ) + finally: + server.shutdown() + + +def test_shodan_tool_formatting_with_mock_client(monkeypatch): + shodan_mcp = load_shodan_module() + server, base_url = start_mock_server() + client = shodan_mcp._HttpShodanClient(base_url, "test-key") + monkeypatch.setattr(shodan_mcp, "_get_client", lambda: client) + try: + host = json.loads(shodan_mcp.shodan_host_info("203.0.113.10", history=True)) + search = json.loads( + shodan_mcp.shodan_host_search('org:"Target Corp"', facets="port") + ) + exploits = json.loads(shodan_mcp.shodan_exploits_search("CVE-2021-44228")) + api_info = json.loads(shodan_mcp.shodan_api_info()) + + assert host["location"]["asn"] == "AS64500" + assert host["data"][0]["banner"] == "banner" + assert search["matches"][0]["vulns"] == ["CVE-2021-44228"] + assert exploits["matches"][0]["id"] == "EXP-1" + assert api_info["plan"] == "mock" + finally: + server.shutdown() diff --git a/capabilities/attack-surface-management/tests/test_worker_coordinator.py b/capabilities/attack-surface-management/tests/test_worker_coordinator.py new file mode 100644 index 0000000..9df807e --- /dev/null +++ b/capabilities/attack-surface-management/tests/test_worker_coordinator.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import importlib.util +import sys +import types +from pathlib import Path + + +def _load_coordinator(): + runtime = types.ModuleType("dreadnode.capabilities.worker") + + class Worker: + def __init__(self, name: str): + self.name = name + + def on_event(self, _event: str): + def decorator(func): + return func + + return decorator + + def run(self): + return None + + runtime.EventEnvelope = object + runtime.RuntimeClient = object + runtime.Worker = Worker + + sys.modules.setdefault("dreadnode", types.ModuleType("dreadnode")) + sys.modules.setdefault( + "dreadnode.capabilities", types.ModuleType("dreadnode.capabilities") + ) + sys.modules["dreadnode.capabilities.worker"] = runtime + + path = Path(__file__).resolve().parents[1] / "workers" / "coordinator.py" + spec = importlib.util.spec_from_file_location("asm_worker_coordinator", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_normalize_scope_payload_accepts_json_wildcards(): + coordinator = _load_coordinator() + + scope = coordinator._normalize_scope_payload( + { + "target": "example", + "wildcards": '["*.example.com", "*.example.net"]', + "graph_api_url": "http://graph.local", + } + ) + + assert scope["target"] == "example" + assert scope["wildcards"] == ["*.example.com", "*.example.net"] + assert scope["graph_api_url"] == "http://graph.local" + + +def test_extract_findings_keeps_only_high_and_critical_tool_calls(): + coordinator = _load_coordinator() + + findings = coordinator._extract_findings( + [ + { + "name": "attack_surface_management__record_asm_finding", + "arguments": {"id": "ASM-HIGH-001", "severity": "high"}, + }, + { + "name": "record_asm_finding", + "arguments": {"id": "ASM-MED-001", "severity": "medium"}, + }, + { + "name": "record_asm_finding", + "arguments": {"id": "ASM-CRIT-001", "severity": "critical"}, + }, + ] + ) + + assert [finding["id"] for finding in findings] == [ + "ASM-HIGH-001", + "ASM-CRIT-001", + ] diff --git a/capabilities/attack-surface-management/tools/bbot.py b/capabilities/attack-surface-management/tools/bbot.py new file mode 100644 index 0000000..af762ab --- /dev/null +++ b/capabilities/attack-surface-management/tools/bbot.py @@ -0,0 +1,1139 @@ +"""BBOT reconnaissance scanning and Neo4j graph database query tools. + +Provides run_bbot_scan for executing BBOT CLI scans and a suite of Neo4j +Cypher query tools for analyzing reconnaissance results stored in a +property graph. The Neo4j connection is lazily initialized on first query. + +Prerequisites: + - bbot CLI installed and in PATH + - Neo4j database running (container or external) + - Neo4j output module configured in BBOT +""" + +from __future__ import annotations + +import ast +import asyncio +import contextlib +import json +import logging +import os +import re +import shlex +import tempfile +import typing as t +import urllib.error +import urllib.request +from pathlib import Path + +from dreadnode.agents.tools import Toolset, tool_method +from pydantic import PrivateAttr + +try: + from neo4j import AsyncDriver, AsyncGraphDatabase +except ImportError: + AsyncDriver = None # type: ignore[assignment, misc] + AsyncGraphDatabase = None # type: ignore[assignment, misc] + +# Reduce Neo4j driver logging noise +logging.getLogger("neo4j").setLevel(logging.ERROR) + + +def _parse_serialized_dict(data: str) -> t.Any: + """Parse string representations of JSON or Python literals into dicts.""" + if not isinstance(data, str): + return data + with contextlib.suppress(Exception): + result = json.loads(data) + return result if isinstance(result, dict) else {} + with contextlib.suppress(Exception): + result = ast.literal_eval(data) + return result if isinstance(result, dict) else {} + return data + + +def _coerce_string_list(value: t.Any) -> list[str]: + """Accept native lists plus JSON/Python stringified lists from weaker tool callers.""" + if value is None: + return [] + if isinstance(value, list): + return [str(item) for item in value if str(item)] + if isinstance(value, str): + stripped = value.strip() + if not stripped: + return [] + with contextlib.suppress(Exception): + parsed = json.loads(stripped) + if isinstance(parsed, list): + return [str(item) for item in parsed if str(item)] + with contextlib.suppress(Exception): + parsed = ast.literal_eval(stripped) + if isinstance(parsed, list): + return [str(item) for item in parsed if str(item)] + return [stripped] + return [str(value)] + + +def _normalize_config(config: list[str]) -> list[str]: + """Accept common stale BBOT config names emitted by older prompts/skills.""" + replacements = { + "modules.httpx.timeout": "modules.http.timeout", + "http.web_spider_distance": "web.spider_distance", + "scope.distance": "scope.search_distance", + } + dropped = { + "scope.report_distance", + } + normalized: list[str] = [] + for item in config: + key, sep, value = item.partition("=") + if key in dropped: + continue + if ( + key.startswith("modules.") + and key.endswith(".enabled") + and value.lower() == "false" + ): + continue + normalized.append(f"{replacements.get(key, key)}{sep}{value}") + return normalized + + +def _normalize_extra_args(args: list[str]) -> list[str]: + """Drop stale BBOT CLI flags emitted by models.""" + normalized: list[str] = [] + skip_next = False + drop_next_value_for = { + "--timeout", + "-x", + "--modules", + "-m", + } + for arg in args: + if skip_next: + skip_next = False + continue + if arg in drop_next_value_for: + skip_next = True + continue + if arg.startswith("--timeout="): + continue + if arg.startswith("--modules="): + continue + if arg == "--scope-distance": + skip_next = True + continue + if arg.startswith("--scope-distance="): + continue + normalized.append(arg) + return normalized + + +def _normalize_targets(targets: list[str]) -> list[str]: + """Convert wildcard scope expressions into BBOT seed targets.""" + normalized: list[str] = [] + for target in targets: + stripped = target.strip() + if stripped.startswith("*.") and len(stripped) > 2: + stripped = stripped[2:] + if stripped: + normalized.append(stripped) + return normalized + + +def _query_graph_api( + graph_api_url: str, cypher: str, params: dict[str, t.Any] | None = None +) -> list[dict[str, t.Any]]: + """Execute Cypher through a task-local HTTP graph proxy.""" + url = graph_api_url.rstrip("/") + "/query" + body = json.dumps({"cypher": cypher, "params": params or {}}).encode() + request = urllib.request.Request( + url, + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=60) as response: + payload = json.loads(response.read().decode()) + except urllib.error.HTTPError as exc: + detail = exc.read().decode(errors="replace") + raise RuntimeError(f"Graph API query failed ({exc.code}): {detail}") from exc + + if not isinstance(payload, dict) or not isinstance(payload.get("rows"), list): + raise RuntimeError(f"Graph API returned unexpected payload: {payload!r}") + return payload["rows"] + + +_ANSI_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") + + +def _strip_ansi(value: str) -> str: + return _ANSI_RE.sub("", value) + + +def _summarize_bbot_stdout(output: str, max_chars: int) -> str: + """Condense BBOT JSON/stdout output into LLM-friendly telemetry.""" + type_counts: dict[str, int] = {} + scope_counts: dict[str, int] = {} + module_counts: dict[str, int] = {} + examples: list[dict[str, t.Any]] = [] + scans: list[dict[str, t.Any]] = [] + diagnostic_tail: list[str] = [] + event_count = 0 + + for raw_line in output.splitlines(): + line = _strip_ansi(raw_line).strip() + if not line: + continue + + parsed: dict[str, t.Any] | None = None + if line.startswith("{"): + with contextlib.suppress(Exception): + value = json.loads(line) + if isinstance(value, dict): + parsed = value + + if parsed: + event_count += 1 + event_type = str(parsed.get("type") or "UNKNOWN") + scope = str(parsed.get("scope_description") or "unknown") + module = str(parsed.get("module") or parsed.get("source") or "") + type_counts[event_type] = type_counts.get(event_type, 0) + 1 + scope_counts[scope] = scope_counts.get(scope, 0) + 1 + if module: + module_counts[module] = module_counts.get(module, 0) + 1 + + compact = { + "type": event_type, + "scope_description": scope, + "data": parsed.get("data"), + } + if module: + compact["module"] = module + if event_type == "SCAN": + scans.append(compact) + elif len(examples) < 30 and event_type in { + "DNS_NAME", + "URL", + "OPEN_TCP_PORT", + "TECHNOLOGY", + "FINDING", + "IP_ADDRESS", + "WEBSCREENSHOT", + "STORAGE_BUCKET", + "SOCIAL", + "MOBILE_APP", + }: + examples.append(compact) + continue + + diagnostic_tail.append(line[:300]) + diagnostic_tail = diagnostic_tail[-40:] + + summary = { + "mode": "bbot_stdout_json_summary", + "raw_output_chars": len(output), + "event_count": event_count, + "type_counts": dict( + sorted(type_counts.items(), key=lambda item: (-item[1], item[0])) + ), + "scope_counts": dict( + sorted(scope_counts.items(), key=lambda item: (-item[1], item[0])) + ), + "module_counts": dict( + sorted(module_counts.items(), key=lambda item: (-item[1], item[0]))[:20] + ), + "representative_events": examples, + "scan_records": scans[-5:], + "diagnostic_tail": diagnostic_tail, + } + text = ( + "BBOT completed with JSON/stdout evidence. The raw event stream was " + "summarized to keep the agent context small; use the counts and " + "representative_events below for ASM statistics and leads.\n\n" + + json.dumps(summary, indent=2, default=str) + ) + if len(text) > max_chars: + text = ( + text[:max_chars] + f"\n\n... [SUMMARY TRUNCATED: {len(text)} chars total]" + ) + return text + + +def _count_bbot_json_events(output: str) -> int: + """Count JSON event lines in BBOT stdout.""" + event_count = 0 + for raw_line in output.splitlines(): + line = _strip_ansi(raw_line).strip() + if not line.startswith("{"): + continue + with contextlib.suppress(Exception): + value = json.loads(line) + if isinstance(value, dict): + event_count += 1 + return event_count + + +def _bbot_output_has_blocking_diagnostic(output: str) -> bool: + """Detect BBOT diagnostics that mean no useful scan ran.""" + blocking_markers = ( + "Setup hard-failed", + "[ERRR]", + "[ERROR]", + "Please specify --allow-deadly to continue", + "No modules to scan", + ) + return any(marker in output for marker in blocking_markers) + + +def _has_dependency_control_arg(args: list[str]) -> bool: + dependency_args = { + "--no-deps", + "--force-deps", + "--retry-deps", + "--ignore-failed-deps", + "--install-all-deps", + } + return any(arg in dependency_args for arg in args) + + +def _has_module_exclusion_arg(args: list[str]) -> bool: + return any(arg in {"--exclude-modules", "-em"} for arg in args) + + +def _requests_baddns(modules: list[str]) -> bool: + return any(module.startswith("baddns") for module in modules) + + +def _allows_deadly(args: list[str]) -> bool: + return "--allow-deadly" in args + + +def _normalize_modules(modules: list[str], extra_args: list[str]) -> list[str]: + """Drop common non-BBOT/deadly module guesses that would abort a scan.""" + replacements = { + "aws_s3_scan": "bucket_amazon", + "azure_blob_scan": "bucket_microsoft", + "azure_blobs": "bucket_microsoft", + "gcp_storage": "bucket_google", + "gcp_storage_buckets": "bucket_google", + "gcp_storage_scan": "bucket_google", + "http-probes": "httpx", + "http_probe": "httpx", + "http-probe": "httpx", + "http_probes": "httpx", + "s3_buckets": "bucket_amazon", + "s3_scan": "bucket_amazon", + "censys": "censys_ip", + "subenum": "subdomaincenter", + "subdomain_enum": "subdomaincenter", + "subdomain-enum": "subdomaincenter", + "subdomainenum": "subdomaincenter", + "technologies": "httpx", + "web-basic": "httpx", + "web_basic": "httpx", + } + unsupported = { + "amass", + "assetfinder", + "crtsh", + "dns_zone_transfer", + "fastdial", + "find_subdomains", + "findomain", + "gau", + "massdns", + "naabu", + "active", + "passive", + "portscan", + "safe", + "screenshot", + "shuffledns", + "subdomain-finder", + "subdomain_find", + "subdomain_bruteforce", + "subdomain-bruteforce", + "subdomainfinder", + "subfinder", + "sublist3r", + "web_enum", + "web-enum", + "web_screenshot", + "web-screenshot", + "web_screenshots", + "web-screenshots", + "webenum", + "wappalyzer", + } + normalized: list[str] = [] + for module in modules: + module = replacements.get(module, module) + if module in unsupported: + continue + if module == "nuclei" and not _allows_deadly(extra_args): + continue + normalized.append(module) + return normalized + + +def _normalize_presets(presets: list[str], extra_args: list[str]) -> list[str]: + """Avoid BBOT deadly preset aborts unless explicitly requested.""" + replacements = { + "asset-discovery-and-enrichment": "subdomain-enum", + "asset-discovery": "subdomain-enum", + "asset_discovery": "subdomain-enum", + "asset-discovery-web-endpoints": "web-basic", + "asset_discovery_web_endpoints": "web-basic", + "asset_discovery_and_enrichment": "subdomain-enum", + "discovery": "subdomain-enum", + "subdomain_discovery": "subdomain-enum", + "subdomain-discovery": "subdomain-enum", + "subdomain_enumeration": "subdomain-enum", + "subdomain-enumeration": "subdomain-enum", + "web_breach": "web-basic", + "web_basic": "web-basic", + "web_discovery": "web-basic", + "web_scan": "web-basic", + "web-port-scan-and-tech-detect": "web-basic", + "web_port_scan_and_tech_detect": "web-basic", + } + supported = { + "subdomain-enum", + "web-basic", + "web-thorough", + "cloud-enum", + "code-enum", + "email-enum", + "spider", + "nuclei", + "nuclei-intense", + "dirbust-light", + "dirbust-heavy", + "lightfuzz-light", + "lightfuzz-medium", + "tech-detect", + "kitchen-sink", + } + presets = [replacements.get(preset, preset) for preset in presets] + presets = [preset for preset in presets if preset in supported] + if _allows_deadly(extra_args): + return presets + return [ + preset + for preset in presets + if not preset.startswith("nuclei") and preset != "kitchen-sink" + ] + + +def _normalize_flags(flags: list[str]) -> list[str]: + """Drop CLI switches and stale flag names emitted as BBOT flags.""" + supported = { + "active", + "passive", + "safe", + "aggressive", + "subdomain-enum", + "web-basic", + "web-thorough", + "cloud-enum", + "code-enum", + "email-enum", + } + return [flag for flag in flags if flag in supported] + + +def _sandbox_suppression_note( + requested_modules: list[str], graph_api_url: str | None +) -> str: + if not graph_api_url or not requested_modules: + return "" + suppressed = sorted( + {module for module in requested_modules if module in {"gowitness", "portscan"}} + ) + if not suppressed: + return "" + return ( + "Sandbox note: " + + ", ".join(suppressed) + + " requested but suppressed in Graph API mode for runtime reliability; " + "use returned DNS/URL/FINDING evidence, `httpx`, and safe web presets for bounded validation.\n\n" + ) + + +def _bbot_subprocess_env(graph_api_url: str | None) -> dict[str, str]: + env = os.environ.copy() + if graph_api_url: + bbot_home = Path(tempfile.mkdtemp(prefix="asm-bbot-home-")) + env["HOME"] = str(bbot_home) + env["BBOT_HOME"] = str(bbot_home / ".bbot") + return env + + +def _summarize(data: dict[str, t.Any]) -> dict[str, t.Any]: + """Condense a node record to essential fields, truncating long values.""" + summary: dict[str, t.Any] = {} + essential = [ + "id", + "type", + "data", + "host", + "netloc", + "port", + "tags", + "scope_description", + "scope_distance", + ] + for field in essential: + if field in data and data[field] is not None: + value = data[field] + if isinstance(value, list) and len(value) > 5: + summary[field] = value[:5] + summary[f"{field}_truncated"] = True + elif isinstance(value, str) and len(value) > 200: + summary[field] = value[:200] + "..." + else: + summary[field] = value + if "id" in summary and isinstance(summary["id"], str) and len(summary["id"]) > 40: + summary["id"] = summary["id"][:40] + "..." + return summary + + +class BbotTools(Toolset): + """Execute BBOT reconnaissance scans and query results from the Neo4j graph database.""" + + neo4j_uri: str = "bolt://localhost:7687" + """Neo4j bolt URI. Override with NEO4J_URI env var or set directly.""" + + neo4j_user: str = "neo4j" + """Neo4j username.""" + + neo4j_password: str = "bbotislife" + """Neo4j password.""" + + bbot_data_dir: str = ".bbot" + """Directory where BBOT stores scan data.""" + + scan_timeout: int = 3600 + """Maximum time in seconds for a BBOT scan to run.""" + + max_output_chars: int = 7_000 + """Maximum characters returned from scan output.""" + + _drivers: dict[tuple[str, str, str], t.Any] = PrivateAttr(default_factory=dict) + + def model_post_init(self, __context: t.Any) -> None: + """Apply environment variable overrides after initialization.""" + if uri := os.environ.get("NEO4J_URI"): + self.neo4j_uri = uri + if user := os.environ.get("NEO4J_USER"): + self.neo4j_user = user + if password := os.environ.get("NEO4J_PASSWORD"): + self.neo4j_password = password + + def _connection( + self, + neo4j_uri: str | None = None, + neo4j_user: str | None = None, + neo4j_password: str | None = None, + ) -> tuple[str, str, str]: + """Resolve a Neo4j connection, allowing task-local endpoint overrides.""" + return ( + neo4j_uri or self.neo4j_uri, + neo4j_user or self.neo4j_user, + neo4j_password or self.neo4j_password, + ) + + async def _ensure_driver( + self, + neo4j_uri: str | None = None, + neo4j_user: str | None = None, + neo4j_password: str | None = None, + ) -> "AsyncDriver": + """Lazily initialize and return the Neo4j async driver.""" + uri, user, password = self._connection(neo4j_uri, neo4j_user, neo4j_password) + key = (uri, user, password) + if key not in self._drivers: + if AsyncGraphDatabase is None: + raise RuntimeError( + "neo4j package is not installed. Install with: pip install neo4j>=5.28.1" + ) + driver = AsyncGraphDatabase.driver(uri, auth=(user, password)) + await driver.verify_connectivity() + self._drivers[key] = driver + return self._drivers[key] + + async def _query( + self, + cypher: str, + params: dict[str, t.Any] | None = None, + neo4j_uri: str | None = None, + neo4j_user: str | None = None, + neo4j_password: str | None = None, + graph_api_url: str | None = None, + ) -> list[dict[str, t.Any]]: + """Execute a Cypher query and return results as list of dicts.""" + if graph_api_url: + return await asyncio.to_thread( + _query_graph_api, graph_api_url, cypher, params + ) + driver = await self._ensure_driver(neo4j_uri, neo4j_user, neo4j_password) + async with driver.session() as session: + result = await session.run(cypher, params or {}) + return [record.data() async for record in result] + + async def _get_nodes( + self, + label: str, + filters: dict[str, t.Any] | None = None, + limit: int = 100, + neo4j_uri: str | None = None, + neo4j_user: str | None = None, + neo4j_password: str | None = None, + graph_api_url: str | None = None, + ) -> list[dict[str, t.Any]]: + """Fetch nodes by label with optional property filtering.""" + where_clauses = ["$label IN labels(n)"] + if filters: + where_clauses.extend(f"n.`{key}` = ${key}" for key in filters) + + cypher = f""" + MATCH (n) + WHERE {" AND ".join(where_clauses)} + RETURN n + {"LIMIT " + str(limit) if limit else ""} + """ + params: dict[str, t.Any] = {"label": label} + if filters: + params.update(filters) + result = await self._query( + cypher, params, neo4j_uri, neo4j_user, neo4j_password, graph_api_url + ) + return [record["n"] for record in result] + + # ── Scanning ────────────────────────────────────────────────────────── + + @tool_method(name="run_bbot_scan", catch=True) + async def run_scan( + self, + targets: t.Annotated[ + list[str] | str, + "Targets to scan (e.g., ['example.com', '10.0.0.0/24'])", + ], + modules: t.Annotated[ + list[str] | str | None, + "Modules to run (e.g., ['httpx', 'nuclei'])", + ] = None, + presets: t.Annotated[ + list[str] | str | None, + "Presets to use (e.g., ['subdomain-enum', 'web-basic']). " + "Available: subdomain-enum, web-basic, web-thorough, cloud-enum, code-enum, " + "email-enum, spider, nuclei, nuclei-intense, dirbust-light, dirbust-heavy, " + "lightfuzz-light, lightfuzz-medium, tech-detect, kitchen-sink", + ] = None, + flags: t.Annotated[ + list[str] | str | None, + "Flags to enable module groups (e.g., ['passive', 'safe']). " + "Available: active, passive, safe, aggressive, subdomain-enum, web-basic, " + "web-thorough, cloud-enum, code-enum, portscan, web-screenshots", + ] = None, + config: t.Annotated[ + list[str] | str | None, + "Custom config in key=value format (e.g., ['modules.http.timeout=5'])", + ] = None, + extra_args: t.Annotated[ + list[str] | str | None, + "Additional bbot CLI flags (e.g., ['--strict-scope', '--proxy http://127.0.0.1:8080'])", + ] = None, + neo4j_uri: t.Annotated[ + str | None, + "Neo4j bolt URI for storing scan results. Defaults to configured NEO4J_URI.", + ] = None, + neo4j_user: t.Annotated[ + str | None, + "Neo4j username. Defaults to configured NEO4J_USER.", + ] = None, + neo4j_password: t.Annotated[ + str | None, + "Neo4j password. Defaults to configured NEO4J_PASSWORD.", + ] = None, + graph_api_url: t.Annotated[ + str | None, + "Task-local Graph API URL. When supplied, run BBOT in stdout JSON mode " + "so sandboxed evaluations do not require raw Bolt access.", + ] = None, + timeout: t.Annotated[ + int | None, + "Optional per-call scan timeout in seconds.", + ] = None, + ) -> str: + """Execute a BBOT reconnaissance scan against targets. + + Assembles and runs a `bbot` command, automatically configuring it to + report findings to the Neo4j database. Results are stored in the graph + and can be queried with the other tools. + + The scan runs locally via the bbot CLI which must be installed and in PATH. + """ + targets = _normalize_targets(_coerce_string_list(targets)) + flags = _normalize_flags(_coerce_string_list(flags)) + config = _normalize_config(_coerce_string_list(config)) + extra_args = _normalize_extra_args(_coerce_string_list(extra_args)) + requested_modules = _coerce_string_list(modules) + modules = _normalize_modules(requested_modules, extra_args) + presets = _normalize_presets(_coerce_string_list(presets), extra_args) + + if not targets: + raise ValueError("At least one target is required to run a scan.") + scan_timeout = max( + 30, min(int(timeout or self.scan_timeout), self.scan_timeout) + ) + + # In platform sandboxes, task services are often exposed through HTTP + # proxies that do not support the Bolt protocol. A graph_api_url signals + # that graph queries should use HTTP, so scan output must remain local to + # the tool response instead of forcing BBOT's Neo4j output module. + if graph_api_url: + parts = ["bbot", "--yes", "--json", "--brief", "--output-modules", "stdout"] + else: + uri, user, password = self._connection( + neo4j_uri, neo4j_user, neo4j_password + ) + config.extend( + [ + f"modules.neo4j.uri={uri}", + f"modules.neo4j.username={user}", + f"modules.neo4j.password={password}", + ] + ) + parts = ["bbot", "--yes", "--output-modules", "neo4j", "--brief"] + + parts.extend(["--targets", *targets]) + + if modules: + parts.extend(["--modules", *modules]) + if flags: + parts.extend(["--flags", *flags]) + if presets: + parts.extend(["--preset", *presets]) + if config: + parts.extend(["--config", *config]) + if extra_args: + parts.extend(extra_args) + if graph_api_url and not _has_dependency_control_arg(extra_args): + parts.append("--no-deps") + if graph_api_url and not _has_module_exclusion_arg(extra_args): + excluded = ["portscan", "gowitness"] + if not _allows_deadly(extra_args): + excluded.append("nuclei") + if not _requests_baddns(modules): + excluded.extend(["baddns", "baddns_direct", "baddns_zone"]) + parts.extend(["--exclude-modules", *excluded]) + + command_str = " ".join(parts) + + # Execute the scan + try: + process = await asyncio.create_subprocess_exec( + *shlex.split(command_str), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + env=_bbot_subprocess_env(graph_api_url), + ) + + output_chunks: list[str] = [] + + async def stream() -> None: + if not process or not process.stdout: + return + while True: + line = await process.stdout.readline() + if not line: + break + output_chunks.append(line.decode(errors="replace").strip()) + + await asyncio.wait_for(stream(), timeout=scan_timeout) + await process.wait() + + exit_code = process.returncode or 0 + + except asyncio.TimeoutError: + if process: + with contextlib.suppress(ProcessLookupError): + process.kill() + output = "\n".join(output_chunks) + return f"Scan timed out after {scan_timeout}s. Partial output:\n{output}" + + except FileNotFoundError: + return ( + "Error: bbot command not found. " + "Install BBOT: pip install bbot (https://github.com/blacklanternsecurity/bbot)" + ) + + output = "\n".join(output_chunks) + + if exit_code != 0: + if graph_api_url: + output = _summarize_bbot_stdout(output, self.max_output_chars) + return f"BBOT scan exited with code {exit_code}:\n{output}" + + if graph_api_url: + note = _sandbox_suppression_note(requested_modules, graph_api_url) + event_count = _count_bbot_json_events(output) + summarized = _summarize_bbot_stdout(output, self.max_output_chars) + if event_count == 0 or _bbot_output_has_blocking_diagnostic(output): + return ( + "Scan produced no usable BBOT JSON events.\n\n" + f"{note}{summarized}" + ) + output = note + summarized + elif len(output) > self.max_output_chars: + output = ( + output[: self.max_output_chars] + + f"\n\n... [TRUNCATED: {len(output)} chars total]" + ) + + return f"Scan completed successfully.\n\n{output}" + + # ── Graph Queries ───────────────────────────────────────────────────── + + @tool_method(name="query_graph", catch=True) + async def query_graph( + self, + cypher: t.Annotated[str, "The Cypher query to execute"], + params: t.Annotated[ + dict[str, t.Any] | None, + "Optional parameters to safely inject values (prevents injection). " + "Use $param syntax in the query.", + ] = None, + neo4j_uri: t.Annotated[ + str | None, + "Task Neo4j bolt URI, for example bolt://host:7687.", + ] = None, + neo4j_user: t.Annotated[str | None, "Task Neo4j username."] = None, + neo4j_password: t.Annotated[str | None, "Task Neo4j password."] = None, + graph_api_url: t.Annotated[ + str | None, + "Task-local Graph API URL for querying Neo4j over HTTP when raw Bolt is unavailable.", + ] = None, + ) -> str: + """Execute a Cypher query against the Neo4j graph database. + + This is the primary analysis tool for exploring reconnaissance data. + Use parameterized queries ($param) for user input to prevent injection. + + Common patterns: + Count by type: MATCH (n) RETURN labels(n)[0] as type, count(n) as count ORDER BY count DESC + Find domains: MATCH (n:DNS_NAME) WHERE n.name CONTAINS 'api' RETURN n.name LIMIT 20 + DNS to IP: MATCH (d:DNS_NAME)-[:RESOLVES_TO]->(ip:IP_ADDRESS) RETURN d.name, ip.address + Critical findings: MATCH (f:FINDING) WHERE f.severity IN ['critical', 'high'] RETURN f + Tech stack: MATCH (n:TECHNOLOGY) RETURN DISTINCT n.name, n.version + Shared hosting: MATCH (ip:IP_ADDRESS)<-[:RESOLVES_TO]-(d) WITH ip, count(d) as cnt WHERE cnt > 1 RETURN ip.address, cnt + """ + result = await self._query( + cypher, params, neo4j_uri, neo4j_user, neo4j_password, graph_api_url + ) + return json.dumps(result, indent=2, default=str) + + @tool_method(name="get_scan_metadata", catch=True) + async def get_scans( + self, + scope_distance: t.Annotated[ + int, "Filter by scope distance (0 = direct targets)" + ] = 0, + tags: t.Annotated[list[str] | None, "Filter by scan tags"] = None, + neo4j_uri: t.Annotated[str | None, "Task Neo4j bolt URI."] = None, + neo4j_user: t.Annotated[str | None, "Task Neo4j username."] = None, + neo4j_password: t.Annotated[str | None, "Task Neo4j password."] = None, + graph_api_url: t.Annotated[str | None, "Task-local Graph API URL."] = None, + ) -> str: + """Retrieve metadata about completed BBOT scans. + + Returns scan IDs, targets, modules used, and timing information. + """ + scans = await self._get_nodes( + label="SCAN", + filters={ + "scope_distance": scope_distance, + **({"tags": tags} if tags else {}), + }, + neo4j_uri=neo4j_uri, + neo4j_user=neo4j_user, + neo4j_password=neo4j_password, + graph_api_url=graph_api_url, + ) + summarized = [_summarize(scan) for scan in scans] + return json.dumps(summarized, indent=2, default=str) + + @tool_method(name="get_findings", catch=True) + async def get_findings( + self, + scope_distance: t.Annotated[ + int, "Filter by scope distance (0 = direct targets)" + ] = 0, + tags: t.Annotated[ + list[str] | None, "Filter by tags (e.g., ['critical', 'authentication'])" + ] = None, + neo4j_uri: t.Annotated[str | None, "Task Neo4j bolt URI."] = None, + neo4j_user: t.Annotated[str | None, "Task Neo4j username."] = None, + neo4j_password: t.Annotated[str | None, "Task Neo4j password."] = None, + graph_api_url: t.Annotated[str | None, "Task-local Graph API URL."] = None, + ) -> str: + """Retrieve security findings and vulnerabilities from scans. + + Returns finding type, severity, description, affected resource, + and evidence. Use this to quickly identify confirmed issues. + """ + findings = await self._get_nodes( + label="FINDING", + filters={ + "scope_distance": scope_distance, + **({"tags": tags} if tags else {}), + }, + neo4j_uri=neo4j_uri, + neo4j_user=neo4j_user, + neo4j_password=neo4j_password, + graph_api_url=graph_api_url, + ) + summarized = [_summarize(finding) for finding in findings] + return json.dumps(summarized, indent=2, default=str) + + @tool_method(name="get_db_schema", catch=True) + async def get_schema( + self, + neo4j_uri: t.Annotated[str | None, "Task Neo4j bolt URI."] = None, + neo4j_user: t.Annotated[str | None, "Task Neo4j username."] = None, + neo4j_password: t.Annotated[str | None, "Task Neo4j password."] = None, + graph_api_url: t.Annotated[str | None, "Task-local Graph API URL."] = None, + ) -> str: + """Retrieve the Neo4j database schema. + + Returns node labels, relationship types, and their properties. + Essential for understanding the data model and constructing queries. + """ + queries = { + "node_labels": "CALL db.labels() YIELD label", + "relationship_types": "CALL db.relationshipTypes() YIELD relationshipType", + "node_properties": "CALL db.schema.nodeTypeProperties()", + "relationship_properties": "CALL db.schema.relTypeProperties()", + } + + results = await asyncio.gather( + *( + self._query( + q, None, neo4j_uri, neo4j_user, neo4j_password, graph_api_url + ) + for q in queries.values() + ) + ) + node_labels_res, rel_types_res, node_props_res, rel_props_res = results + + schema: dict[str, t.Any] = { + "node_labels": sorted([r["label"] for r in node_labels_res]), + "relationship_types": sorted( + [r["relationshipType"] for r in rel_types_res] + ), + "node_properties": {}, + "relationship_properties": {}, + } + + for record in node_props_res: + label = record.get("nodeType", "").lstrip(":") + if not label: + continue + if label not in schema["node_properties"]: + schema["node_properties"][label] = [] + schema["node_properties"][label].append( + { + "property": record.get("propertyName"), + "types": record.get("propertyTypes"), + "mandatory": record.get("mandatory"), + } + ) + + for record in rel_props_res: + rel_type = record.get("relType", "").lstrip(":") + if not rel_type: + continue + if rel_type not in schema["relationship_properties"]: + schema["relationship_properties"][rel_type] = [] + schema["relationship_properties"][rel_type].append( + { + "property": record.get("propertyName"), + "types": record.get("propertyTypes"), + "mandatory": record.get("mandatory"), + } + ) + + return json.dumps(schema, indent=2, default=str) + + @tool_method(name="explore_nodes", catch=True) + async def explore_nodes( + self, + label: t.Annotated[ + str | None, "Node type (e.g., 'DNS_NAME', 'URL', 'FINDING')" + ] = None, + property_filter: t.Annotated[ + str | None, + "Filter: 'property=value' for exact match, 'property CONTAINS value' for substring", + ] = None, + limit: t.Annotated[int, "Maximum nodes to return (1-1000)"] = 100, + neo4j_uri: t.Annotated[str | None, "Task Neo4j bolt URI."] = None, + neo4j_user: t.Annotated[str | None, "Task Neo4j username."] = None, + neo4j_password: t.Annotated[str | None, "Task Neo4j password."] = None, + graph_api_url: t.Annotated[str | None, "Task-local Graph API URL."] = None, + ) -> str: + """Explore nodes in the graph database interactively. + + Flexible tool for discovering and examining nodes when you're not sure + exactly what you're looking for. Use get_db_schema() first to see + available node labels. + """ + if limit < 1 or limit > 1000: + raise ValueError("Limit must be between 1 and 1000.") + + query_parts = [f"MATCH (node:{label})" if label else "MATCH (node)"] + params: dict[str, t.Any] = {} + + if property_filter: + if "CONTAINS" in property_filter: + parts = property_filter.split("CONTAINS", 1) + if len(parts) == 2: + prop, value = parts + query_parts.append(f"WHERE node.{prop.strip()} CONTAINS $value") + params["value"] = value.strip() + elif "=" in property_filter: + prop, value = property_filter.split("=", 1) + query_parts.append(f"WHERE node.{prop.strip()} = $value") + params["value"] = value.strip() + + query_parts.append("RETURN node LIMIT $limit") + + result = await self._query( + " ".join(query_parts), + {"limit": limit, **params}, + neo4j_uri, + neo4j_user, + neo4j_password, + graph_api_url, + ) + return json.dumps(result, indent=2, default=str) + + @tool_method(name="explore_relationships", catch=True) + async def explore_relationships( + self, + source_label: t.Annotated[ + str | None, "Source node type (e.g., 'DNS_NAME')" + ] = None, + relationship_type: t.Annotated[ + str | None, "Relationship type (e.g., 'RESOLVES_TO')" + ] = None, + target_label: t.Annotated[ + str | None, "Target node type (e.g., 'IP_ADDRESS')" + ] = None, + limit: t.Annotated[int, "Maximum relationships to return (1-1000)"] = 100, + neo4j_uri: t.Annotated[str | None, "Task Neo4j bolt URI."] = None, + neo4j_user: t.Annotated[str | None, "Task Neo4j username."] = None, + neo4j_password: t.Annotated[str | None, "Task Neo4j password."] = None, + graph_api_url: t.Annotated[str | None, "Task-local Graph API URL."] = None, + ) -> str: + """Discover how nodes are connected in the graph database. + + Use get_db_schema() to see available relationship types. + """ + if limit < 1 or limit > 1000: + raise ValueError("Limit must be between 1 and 1000.") + + source = f"(source:{source_label})" if source_label else "(source)" + rel = ( + f"-[relationship:{relationship_type}]->" + if relationship_type + else "-[relationship]->" + ) + target = f"(target:{target_label})" if target_label else "(target)" + + query = f"MATCH {source}{rel}{target} RETURN source, relationship, target LIMIT $limit" + result = await self._query( + query, + {"limit": limit}, + neo4j_uri, + neo4j_user, + neo4j_password, + graph_api_url, + ) + return json.dumps(result, indent=2, default=str) + + @tool_method(name="get_screenshot", catch=True) + async def get_screenshot( + self, + uuid: t.Annotated[str | None, "The UUID of the WEBSCREENSHOT node"] = None, + url: t.Annotated[str | None, "The URL to find a screenshot of"] = None, + neo4j_uri: t.Annotated[str | None, "Task Neo4j bolt URI."] = None, + neo4j_user: t.Annotated[str | None, "Task Neo4j username."] = None, + neo4j_password: t.Annotated[str | None, "Task Neo4j password."] = None, + graph_api_url: t.Annotated[str | None, "Task-local Graph API URL."] = None, + ) -> str: + """Retrieve a screenshot from the database. + + Identify a screenshot by its UUID (from explore_nodes) or by the + original URL that was screenshotted. Returns the file path for viewing. + """ + if not uuid and not url: + raise ValueError("Either 'uuid' or 'url' must be provided.") + + if url and not uuid: + nodes = await self._query( + "MATCH (node:WEBSCREENSHOT) WHERE node.url CONTAINS $url RETURN node LIMIT 1", + {"url": url}, + neo4j_uri, + neo4j_user, + neo4j_password, + graph_api_url, + ) + if not nodes: + return f"No screenshot found for URL '{url}'." + uuid = nodes[0].get("node", {}).get("uuid") + if not uuid: + return f"No screenshot found for URL '{url}'." + + cypher = """ + MATCH (w:WEBSCREENSHOT {uuid: $uuid}) + MATCH (s:SCAN {id: w.scan}) + RETURN w.data AS web_data, s.data AS scan_data + """ + result = await self._query( + cypher, + {"uuid": uuid}, + neo4j_uri, + neo4j_user, + neo4j_password, + graph_api_url, + ) + if not result: + return f"No screenshot data found for UUID '{uuid}'." + + scan_data = _parse_serialized_dict(result[0].get("scan_data", "")) + web_data = _parse_serialized_dict(result[0].get("web_data", "")) + + scan_name = str(scan_data.get("name", "")) + relative_path = str(web_data.get("path", "")) + original_url = str(web_data.get("url", "")) + + if not all([scan_name, relative_path]): + return "Screenshot data is missing required fields." + + bbot_home = Path(self.bbot_data_dir).expanduser().resolve() + full_path = bbot_home / "scans" / scan_name / relative_path + + if not full_path.exists(): + return f"Screenshot file not found at: {full_path}" + + return json.dumps( + { + "path": str(full_path), + "url": original_url, + "uuid": uuid, + }, + indent=2, + ) diff --git a/capabilities/attack-surface-management/tools/findings.py b/capabilities/attack-surface-management/tools/findings.py new file mode 100644 index 0000000..5aea18e --- /dev/null +++ b/capabilities/attack-surface-management/tools/findings.py @@ -0,0 +1,72 @@ +"""Structured ASM finding recorder for worker-coordinated runs.""" + +import typing as t + +from dreadnode.agents.tools import tool + +Severity = t.Literal["critical", "high", "medium", "low", "informational"] +Confidence = t.Literal["high", "medium", "low"] +ValidationStatus = t.Literal[ + "validated", "partially_validated", "unvalidated", "rejected" +] + + +@tool +def record_asm_finding( + id: t.Annotated[ + str, + "Stable finding id, e.g. ASM-HIGH-001. Used by the coordinator to label validator sessions.", + ], + title: t.Annotated[str, "Short human-readable title."], + severity: t.Annotated[ + Severity, "One of: critical, high, medium, low, informational." + ], + confidence: t.Annotated[ + Confidence, "Confidence in the finding before validator review." + ], + asset: t.Annotated[ + str, "Primary affected host, URL, IP, service, or asset cluster." + ], + claim: t.Annotated[ + str, "One- or two-sentence claim of what is exposed and why it matters." + ], + evidence: t.Annotated[ + str, + "Concrete evidence: tool outputs, graph query results, HTTP observations, screenshots, versions, ports, or CVE references.", + ], + gadget: t.Annotated[ + str, + "The attack-surface gadget or chain this finding belongs to, e.g. exposed staging API plus weak error handling.", + ], + validation_status: t.Annotated[ + ValidationStatus, + "Validation state reached by the reviewer before spawning dedicated validators.", + ], + validation_method: t.Annotated[ + str, + "Non-destructive method already used or recommended to validate the claim.", + ], + impact: t.Annotated[ + str, "Operational or security impact if the exposure is exploitable." + ], + next_steps: t.Annotated[str, "Specific next actions for a human operator."], + scope_notes: t.Annotated[ + str, + "Why the asset appears in-scope, out-of-scope, or uncertain based on the provided scope.", + ], + origin: t.Annotated[ + t.Literal["discovery", "enrichment", "gadget", "final-reviewer"], + "Pipeline stage that produced or accepted this finding.", + ], + rejected_alternatives: t.Annotated[ + str, + "Brief notes on plausible explanations or leads that were rejected. Empty string if none.", + ] = "", +) -> str: + """Record one structured ASM finding for validator review. + + The coordinator inspects tool calls from the final reviewer and launches + validator agents for high and critical findings. Findings only described in + prose are not eligible for validator fan-out. + """ + return f"Recorded {id} ({severity}): {title}" diff --git a/capabilities/attack-surface-management/tools/pipeline.py b/capabilities/attack-surface-management/tools/pipeline.py new file mode 100644 index 0000000..19cef57 --- /dev/null +++ b/capabilities/attack-surface-management/tools/pipeline.py @@ -0,0 +1,497 @@ +"""Agent-facing launcher for the worker-coordinated ASM pipeline.""" + +from __future__ import annotations + +import asyncio +import importlib.util +import json +import os +from pathlib import Path +import socket +import sys +import types +import typing as t +import urllib.error +import urllib.request +from uuid import uuid4 + +from dreadnode.agents.tools import Toolset, tool_method +from dreadnode.app.client.runtime_client import RuntimeClient + +REQUEST_EVENT = "asm.analysis.requested" +PROGRESS_EVENT = "asm.analysis.progress" +REPORT_READY_EVENT = "asm.analysis.report.ready" +COMPLETED_EVENT = "asm.analysis.completed" +FAILED_EVENT = "asm.analysis.failed" + +WATCH_KINDS: tuple[str, ...] = ( + PROGRESS_EVENT, + REPORT_READY_EVENT, + COMPLETED_EVENT, + FAILED_EVENT, +) + +LOCAL_RUNTIME_CANDIDATES: tuple[str, ...] = ( + "http://127.0.0.1:8787", + "http://localhost:8787", +) +DEFAULT_WORKER_MODEL = "openrouter/moonshotai/kimi-k2.6" + + +def _coerce_string_list(value: t.Any) -> list[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + if isinstance(value, str): + stripped = value.strip() + if not stripped: + return [] + try: + parsed = json.loads(stripped) + except Exception: + parsed = None + if isinstance(parsed, list): + return [str(item).strip() for item in parsed if str(item).strip()] + return [item.strip() for item in stripped.split(",") if item.strip()] + return [str(value).strip()] + + +def _truncate(text: str, limit: int) -> str: + return ( + text + if len(text) <= limit + else text[:limit] + f"\n... truncated ({len(text)} chars total) ..." + ) + + +def _resolve_host(host: str) -> list[str]: + try: + records = socket.getaddrinfo(host, None, type=socket.SOCK_STREAM) + except OSError: + return [] + + ips: list[str] = [] + for record in records: + sockaddr = record[4] + if not sockaddr: + continue + ip = str(sockaddr[0]) + if ip not in ips: + ips.append(ip) + return ips + + +def _probe_http(host: str) -> dict[str, t.Any] | None: + for scheme in ("https", "http"): + url = f"{scheme}://{host}/" + request = urllib.request.Request( + url, + method="HEAD", + headers={"User-Agent": "dreadnode-asm-eval/1.0"}, + ) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return { + "url": url, + "status": response.status, + "final_url": response.geturl(), + "server": response.headers.get("server"), + "content_type": response.headers.get("content-type"), + "location": response.headers.get("location"), + } + except urllib.error.HTTPError as exc: + return { + "url": url, + "status": exc.code, + "final_url": exc.url, + "server": exc.headers.get("server") if exc.headers else None, + "content_type": exc.headers.get("content-type") + if exc.headers + else None, + "location": exc.headers.get("location") if exc.headers else None, + } + except Exception: + continue + return None + + +def _lightweight_surface_probe(target: str, wildcards: list[str]) -> str: + prefixes = ( + "www", + "api", + "auth", + "login", + "sso", + "signon", + "my", + "portal", + "app", + "mobile", + "ir", + "careers", + ) + bases: list[str] = [] + for wildcard in wildcards: + base = wildcard.strip().lstrip("*.").strip(".") + if base and base not in bases: + bases.append(base) + + candidates: list[str] = [] + for base in bases: + for prefix in prefixes: + host = f"{prefix}.{base}" + if host not in candidates: + candidates.append(host) + if len(candidates) >= 48: + break + if len(candidates) >= 48: + break + + leads: list[dict[str, t.Any]] = [] + for host in candidates: + ips = _resolve_host(host) + if not ips: + continue + http = _probe_http(host) + lead = { + "host": host, + "resolved_ips": ips[:4], + } + if http: + lead["http"] = { + key: value for key, value in http.items() if value is not None + } + leads.append(lead) + if len(leads) >= 12: + break + + if not leads: + return "" + + clusters: dict[str, list[str]] = { + "identity_or_access": [], + "api_surface": [], + "public_web": [], + } + for lead in leads: + host = str(lead["host"]) + label = host.split(".", 1)[0] + if label in {"auth", "login", "sso", "signon", "my", "portal"}: + clusters["identity_or_access"].append(host) + if label == "api" or "api" in host: + clusters["api_surface"].append(host) + if label in {"www", "app", "mobile", "ir", "careers"}: + clusters["public_web"].append(host) + + clusters = {key: value for key, value in clusters.items() if value} + triage = [ + "resolved hosts are concrete leads for follow-up enrichment and graph insertion", + "HTTP metadata can identify externally reachable services, redirects, and hosting fingerprints", + "identity, API, and public web clusters are priority candidates for bounded validation", + ] + return "\n".join( + [ + "## Bounded Fallback Probe", + "", + ( + "The worker pipeline performed a bounded passive probe because the " + "graph-backed discovery path returned sparse data." + ), + "", + f"Target: {target}", + "", + "Observed leads:", + "```json", + json.dumps(leads, indent=2, sort_keys=True), + "```", + "", + "Lead clusters:", + "```json", + json.dumps(clusters, indent=2, sort_keys=True), + "```", + "", + "Triage notes:", + *[f"- {item}" for item in triage], + ] + ) + + +class AsmPipelineTools(Toolset): + """Launch and monitor the multi-agent ASM worker pipeline.""" + + default_timeout: int = 480 + """Maximum wall-clock seconds to wait for the worker pipeline.""" + + max_output_chars: int = 12_000 + """Maximum characters returned to the calling agent.""" + + @tool_method(name="run_asm_worker_pipeline", catch=True) + async def run_pipeline( + self, + target: t.Annotated[str, "Target name or primary domain label."], + wildcards: t.Annotated[ + list[str] | str, + "Allowed wildcard scope expressions, e.g. ['*.example.com'].", + ], + graph_api_url: t.Annotated[ + str | None, + "Task Graph API URL to pass to worker agents for graph-capable tools.", + ] = None, + scope: t.Annotated[ + dict[str, t.Any] | str | None, + "Optional additional scope metadata as a JSON object.", + ] = None, + model: t.Annotated[ + str | None, + "Optional direct model id for worker agents, e.g. openrouter/qwen/qwen3.7-max.", + ] = None, + max_steps: t.Annotated[ + int, "Maximum autonomous steps per worker agent turn." + ] = 30, + timeout: t.Annotated[ + int | None, + "Maximum wall-clock seconds to wait for the full pipeline.", + ] = None, + ) -> str: + """Run the worker-coordinated ASM pipeline and return its final report. + + This publishes an `asm.analysis.requested` runtime event and waits for the + coordinator worker to complete the staged pipeline: scope normalization, + discovery, enrichment, gadget clustering, review, validation fan-out, and + final synthesis. + """ + + runtime_token = os.environ.get("DREADNODE_RUNTIME_TOKEN") + run_id = str(uuid4()) + requested_timeout = int(timeout or self.default_timeout) + timeout_seconds = max(60, min(requested_timeout, self.default_timeout)) + progress: list[str] = [] + + payload: dict[str, t.Any] = { + "run_id": run_id, + "target": target, + "wildcards": _coerce_string_list(wildcards), + "max_steps": int(max_steps), + } + if graph_api_url: + payload["graph_api_url"] = graph_api_url + payload["model"] = ( + model or os.environ.get("ASM_WORKER_MODEL") or DEFAULT_WORKER_MODEL + ) + if scope: + if isinstance(scope, str): + try: + payload["scope"] = json.loads(scope) + except Exception: + payload["notes"] = scope + else: + payload["scope"] = scope + + final_report = await self._run_with_runtime_candidates( + payload=payload, + run_id=run_id, + progress=progress, + runtime_token=runtime_token, + timeout_seconds=timeout_seconds, + ) + fallback_report = await asyncio.to_thread( + _lightweight_surface_probe, + target, + payload["wildcards"], + ) + + result = { + "run_id": run_id, + "mode": "worker_coordinated_asm_pipeline", + "progress": progress, + "fallback_probe": fallback_report, + "final_report": final_report, + } + return _truncate( + json.dumps(result, indent=2, default=str), self.max_output_chars + ) + + async def _run_with_runtime_candidates( + self, + *, + payload: dict[str, t.Any], + run_id: str, + progress: list[str], + runtime_token: str | None, + timeout_seconds: int, + ) -> str: + explicit_url = os.environ.get("DREADNODE_RUNTIME_URL") + candidates = [explicit_url] if explicit_url else list(LOCAL_RUNTIME_CANDIDATES) + errors: list[str] = [] + + for runtime_url in candidates: + if not runtime_url: + continue + client = RuntimeClient( + server_url=runtime_url.rstrip("/"), + auth_token=runtime_token, + ) + try: + _patch_websockets_proxy_compat() + await asyncio.wait_for(client.start(), timeout=15) + if not explicit_url: + return await asyncio.wait_for( + self._run_direct_coordinator_pipeline( + client=client, + payload=payload, + run_id=run_id, + ), + timeout=timeout_seconds, + ) + await asyncio.wait_for( + client.publish(REQUEST_EVENT, payload), timeout=15 + ) + try: + return await asyncio.wait_for( + self._watch_run(client, run_id, progress), + timeout=timeout_seconds, + ) + except RuntimeError as exc: + message = str(exc) + if "event stream is unavailable" in message.lower(): + progress.append( + "worker_request_published: runtime transport does not expose event streaming to tools" + ) + return ( + "ASM worker pipeline request was published, but this tool " + "transport cannot stream worker events. Continue the ASM " + "assessment with direct tools while worker-coordinated " + f"sessions run under run_id {run_id}." + ) + raise + except Exception as exc: + errors.append(f"{runtime_url}: {type(exc).__name__}: {exc}") + finally: + try: + await client.close() + except Exception: + pass + + detail = "; ".join(errors) if errors else "no runtime candidates available" + raise RuntimeError( + "ASM worker pipeline could not connect to the runtime event bus. " + f"Tried {candidates}. Errors: {detail}" + ) + + async def _run_direct_coordinator_pipeline( + self, + *, + client: RuntimeClient, + payload: dict[str, t.Any], + run_id: str, + ) -> str: + coordinator = _load_coordinator_module() + scope = coordinator._normalize_scope_payload(payload) + return await coordinator._run_pipeline( + client, + run_id=run_id, + scope=scope, + model=payload.get("model"), + max_steps=int(payload.get("max_steps") or 240), + ) + + async def _watch_run( + self, + client: RuntimeClient, + run_id: str, + progress: list[str], + ) -> str: + async for event in client.subscribe(*WATCH_KINDS): + payload = event.payload if isinstance(event.payload, dict) else {} + if payload.get("run_id") != run_id: + continue + + if event.kind == PROGRESS_EVENT: + stage = str(payload.get("stage") or "unknown") + detail = str(payload.get("detail") or "") + progress.append(f"{stage}: {detail}".rstrip(": ")) + continue + if event.kind == REPORT_READY_EVENT: + progress.append(f"report_ready: {payload.get('agent')}") + continue + if event.kind == FAILED_EVENT: + raise RuntimeError( + str(payload.get("error") or "ASM worker pipeline failed") + ) + if event.kind == COMPLETED_EVENT: + return str(payload.get("final_report") or "") + raise RuntimeError( + "Runtime event stream ended before ASM worker pipeline completed" + ) + + +def _load_coordinator_module() -> t.Any: + _patch_websockets_proxy_compat() + path = Path(__file__).resolve().parents[1] / "workers" / "coordinator.py" + spec = importlib.util.spec_from_file_location("asm_worker_coordinator_direct", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load ASM coordinator from {path}") + + worker_module = types.ModuleType("dreadnode.capabilities.worker") + + class Worker: + def __init__(self, name: str): + self.name = name + + def on_event(self, _event: str): + def decorator(func): + return func + + return decorator + + def run(self): + return None + + worker_module.EventEnvelope = object + worker_module.RuntimeClient = object + worker_module.Worker = Worker + previous_worker_module = sys.modules.get("dreadnode.capabilities.worker") + sys.modules["dreadnode.capabilities.worker"] = worker_module + + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + return module + finally: + if previous_worker_module is None: + sys.modules.pop("dreadnode.capabilities.worker", None) + else: + sys.modules["dreadnode.capabilities.worker"] = previous_worker_module + + +def _patch_websockets_proxy_compat() -> None: + try: + import websockets.uri as websockets_uri + except Exception: + return + if hasattr(websockets_uri, "Proxy"): + proxy_present = True + else: + proxy_present = False + + if not proxy_present: + + class Proxy: # pragma: no cover - runtime dependency compatibility shim + pass + + websockets_uri.Proxy = Proxy + if not hasattr(websockets_uri, "get_proxy"): + + def get_proxy(_uri): # pragma: no cover - runtime dependency compatibility shim + return None + + websockets_uri.get_proxy = get_proxy + if not hasattr(websockets_uri, "parse_proxy"): + + def parse_proxy( + _proxy, + ): # pragma: no cover - runtime dependency compatibility shim + return None + + websockets_uri.parse_proxy = parse_proxy diff --git a/capabilities/attack-surface-management/workers/coordinator.py b/capabilities/attack-surface-management/workers/coordinator.py new file mode 100644 index 0000000..7f32162 --- /dev/null +++ b/capabilities/attack-surface-management/workers/coordinator.py @@ -0,0 +1,709 @@ +"""Multi-agent ASM pipeline coordinator. + +Subscribes to ``asm.analysis.requested`` and runs a progressive +attack-surface-management pipeline: + + scope normalization + │ + ▼ + discovery operator + │ + ▼ + lead enrichment + gadget clustering + │ + ▼ + final reviewer records high/critical findings + │ + ▼ + validator fan-out per recorded finding + │ + ▼ + final report synthesis + +The existing ``asm-operator`` remains available for single-agent runs. This +worker is an additive path for long-horizon runs where raw leads should be +progressively enriched into gadgets and findings. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import re +import typing as t +from uuid import uuid4 + +from dreadnode.capabilities.worker import EventEnvelope, RuntimeClient, Worker +from loguru import logger + +CAPABILITY_NAME = "attack-surface-management" + +REQUEST_EVENT = "asm.analysis.requested" +PROGRESS_EVENT = "asm.analysis.progress" +REPORT_READY_EVENT = "asm.analysis.report.ready" +COMPLETED_EVENT = "asm.analysis.completed" +FAILED_EVENT = "asm.analysis.failed" + +SCOPE_NORMALIZER = "asm-scope-normalizer" +DISCOVERY_OPERATOR = "asm-discovery-operator" +LEAD_ENRICHER = "asm-lead-enricher" +GADGET_CLUSTERER = "asm-gadget-clusterer" +FINAL_REVIEWER = "asm-final-reviewer" +FINDING_VALIDATOR = "asm-finding-validator" +REPORT_SYNTHESIZER = "asm-report-synthesizer" + +RECORD_ASM_FINDING_TOOL = "record_asm_finding" + +DEFAULT_MAX_STEPS = 240 +DEFAULT_VALIDATOR_CONCURRENCY = 2 +REPORT_TRUNCATE_CHARS = 24_000 +AGENT_TURN_TIMEOUT_SECONDS = 45 + +worker = Worker(name="coordinator") + + +@worker.on_event(REQUEST_EVENT) +async def analyze_attack_surface(event: EventEnvelope, client: RuntimeClient) -> None: + """Run one ASM pipeline end to end.""" + payload = event.payload or {} + run_id = str(payload.get("run_id") or uuid4()) + model = payload.get("model") or None + max_steps = int(payload.get("max_steps") or DEFAULT_MAX_STEPS) + + try: + scope = _normalize_scope_payload(payload) + except ValueError as exc: + await client.publish( + FAILED_EVENT, + {"run_id": run_id, "error": str(exc), "payload_keys": sorted(payload)}, + ) + return + + try: + final_report = await _run_pipeline( + client, + run_id=run_id, + scope=scope, + model=model, + max_steps=max_steps, + ) + except Exception as exc: + logger.exception("ASM analysis run failed | run_id={}", run_id) + await client.publish( + FAILED_EVENT, + { + "run_id": run_id, + "target": scope["target"], + "error": f"{type(exc).__name__}: {exc}", + }, + ) + return + + await client.publish( + COMPLETED_EVENT, + {"run_id": run_id, "target": scope["target"], "final_report": final_report}, + ) + + +async def _run_pipeline( + client: RuntimeClient, + *, + run_id: str, + scope: dict[str, t.Any], + model: str | None, + max_steps: int, +) -> str: + target = str(scope["target"]) + scope_context = _render_scope_context(scope) + scope_steps = _stage_budget(max_steps, 6) + discovery_steps = _stage_budget(max_steps, 8) + enrichment_steps = _stage_budget(max_steps, 6) + review_steps = _stage_budget(max_steps, 6) + validator_steps = _stage_budget(max_steps, 5) + + await _publish_progress(client, run_id, "scope_started") + scope_report, _ = await _run_agent_turn( + client, + run_id=run_id, + target=target, + agent=SCOPE_NORMALIZER, + model=model, + max_steps=scope_steps, + prompt=_scope_prompt(scope_context, scope_steps), + ) + await _publish_report(client, run_id, target, SCOPE_NORMALIZER, scope_report) + + await _publish_progress(client, run_id, "discovery_started") + discovery_report, _ = await _run_agent_turn( + client, + run_id=run_id, + target=target, + agent=DISCOVERY_OPERATOR, + model=model, + max_steps=discovery_steps, + prompt=_discovery_prompt(scope_context, scope_report, discovery_steps), + ) + await _publish_report(client, run_id, target, DISCOVERY_OPERATOR, discovery_report) + + await _publish_progress(client, run_id, "enrichment_started") + enriched_report, gadget_report = await _run_enrichment_stage( + client, + run_id=run_id, + target=target, + model=model, + max_steps=enrichment_steps, + scope_context=scope_context, + scope_report=scope_report, + discovery_report=discovery_report, + ) + + await _publish_progress(client, run_id, "final_review_started") + final_review, final_tool_calls = await _run_agent_turn( + client, + run_id=run_id, + target=target, + agent=FINAL_REVIEWER, + model=model, + max_steps=review_steps, + prompt=_final_review_prompt( + scope_context, + scope_report, + discovery_report, + enriched_report, + gadget_report, + review_steps, + ), + ) + await _publish_report(client, run_id, target, FINAL_REVIEWER, final_review) + + findings = _extract_findings(final_tool_calls) + if findings: + await _publish_progress( + client, + run_id, + "validation_started", + f"Validating {len(findings)} high/critical ASM findings", + ) + validation_reports = await _run_validators( + client, + run_id=run_id, + target=target, + model=model, + max_steps=validator_steps, + scope_context=scope_context, + final_review=final_review, + findings=findings, + ) + else: + await _publish_progress( + client, + run_id, + "validation_skipped", + "No high/critical ASM findings recorded", + ) + validation_reports = {} + + await _publish_progress(client, run_id, "report_synthesis_started") + synthesized_report = _fallback_synthesis_report( + scope_context=scope_context, + scope_report=scope_report, + discovery_report=discovery_report, + enriched_report=enriched_report, + gadget_report=gadget_report, + final_review=final_review, + findings=findings, + validation_reports=validation_reports, + ) + await _publish_report( + client, run_id, target, REPORT_SYNTHESIZER, synthesized_report + ) + + return _build_final_markdown(synthesized_report, findings, validation_reports) + + +async def _run_enrichment_stage( + client: RuntimeClient, + *, + run_id: str, + target: str, + model: str | None, + max_steps: int, + scope_context: str, + scope_report: str, + discovery_report: str, +) -> tuple[str, str]: + async def run_one(agent: str, prompt: str) -> tuple[str, str]: + report, _ = await _run_agent_turn( + client, + run_id=run_id, + target=target, + agent=agent, + model=model, + max_steps=max_steps, + prompt=prompt, + ) + await _publish_report(client, run_id, target, agent, report) + return agent, report + + enriched, gadgets = await asyncio.gather( + run_one( + LEAD_ENRICHER, + _lead_enrichment_prompt( + scope_context, scope_report, discovery_report, max_steps + ), + ), + run_one( + GADGET_CLUSTERER, + _gadget_prompt(scope_context, scope_report, discovery_report, max_steps), + ), + ) + return enriched[1], gadgets[1] + + +async def _run_validators( + client: RuntimeClient, + *, + run_id: str, + target: str, + model: str | None, + max_steps: int, + scope_context: str, + final_review: str, + findings: list[dict[str, t.Any]], +) -> dict[str, str]: + sem = asyncio.Semaphore(DEFAULT_VALIDATOR_CONCURRENCY) + final_review = _truncate(final_review, REPORT_TRUNCATE_CHARS) + + async def validate_one(finding: dict[str, t.Any]) -> tuple[str, str]: + finding_id = str(finding.get("id") or "unknown-finding") + async with sem: + report, _ = await _run_agent_turn( + client, + run_id=run_id, + target=target, + agent=FINDING_VALIDATOR, + model=model, + max_steps=max_steps, + prompt=_validator_prompt( + scope_context, final_review, finding, max_steps + ), + extra_labels={"finding_id": finding_id}, + ) + return finding_id, report + + results = await asyncio.gather(*(validate_one(finding) for finding in findings)) + return dict(results) + + +async def _run_agent_turn( + client: RuntimeClient, + *, + run_id: str, + target: str, + agent: str, + model: str | None, + max_steps: int, + prompt: str, + extra_labels: dict[str, str] | None = None, +) -> tuple[str, list[dict[str, t.Any]]]: + labels: dict[str, list[str]] = { + "asm_run": [run_id], + "target": [_label_safe(target)], + "agent_role": [agent], + } + if extra_labels: + labels.update( + {key: [_label_safe(value)] for key, value in extra_labels.items()} + ) + + session = await client.create_session( + capability=CAPABILITY_NAME, + agent=agent, + model=model, + policy={"name": "headless", "max_steps": max_steps}, + labels=labels, + ) + await client.set_session_title(session.session_id, f"asm {run_id[:8]} · {agent}") + try: + result = await asyncio.wait_for( + client.run_turn( + session_id=session.session_id, + message=prompt, + agent=agent, + model=model, + reset=True, + ), + timeout=AGENT_TURN_TIMEOUT_SECONDS, + ) + except asyncio.TimeoutError: + with contextlib.suppress(Exception): + await client.cancel_session(session.session_id) + return ( + f"{agent} timed out after {AGENT_TURN_TIMEOUT_SECONDS}s. " + "Treat this stage as incomplete and continue with the evidence already available.", + [], + ) + response_text = str(result.get("response_text") or "").strip() + tool_calls = result.get("tool_calls") or [] + if not isinstance(tool_calls, list): + tool_calls = [] + if tool_calls and not response_text: + try: + synthesis_result = await asyncio.wait_for( + client.run_turn( + session_id=session.session_id, + message=( + "Synthesize this worker stage now using the evidence already gathered. " + "Return concise coverage, leads, validation status, rejected noise, and next steps. " + "Do not end with an intention to continue." + ), + agent=agent, + model=model, + reset=False, + ), + timeout=AGENT_TURN_TIMEOUT_SECONDS, + ) + response_text = str(synthesis_result.get("response_text") or "").strip() + synthesis_tool_calls = synthesis_result.get("tool_calls") or [] + if isinstance(synthesis_tool_calls, list): + tool_calls.extend(synthesis_tool_calls) + except asyncio.TimeoutError: + with contextlib.suppress(Exception): + await client.cancel_session(session.session_id) + response_text = ( + f"{agent} gathered tool evidence but timed out while synthesizing it.\n\n" + f"{_compact_tool_call_summary(tool_calls)}" + ) + if tool_calls and not response_text: + response_text = _compact_tool_call_summary(tool_calls) + return response_text, tool_calls + + +def _normalize_scope_payload(payload: dict[str, t.Any]) -> dict[str, t.Any]: + target = str(payload.get("target") or "").strip() + if not target: + raise ValueError("missing required target") + + raw_wildcards = payload.get("wildcards") or payload.get("wildcard_scope") or [] + wildcards = _coerce_string_list(raw_wildcards) + raw_scope = payload.get("scope") or {} + + scope: dict[str, t.Any] = { + "target": target, + "wildcards": wildcards, + "scope": raw_scope, + } + for key in ( + "graph_api_url", + "notes", + "excluded", + "allowed_actions", + "disallowed_actions", + "max_runtime_seconds", + ): + if key in payload and payload[key] not in (None, ""): + scope[key] = payload[key] + return scope + + +def _coerce_string_list(value: t.Any) -> list[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + if isinstance(value, str): + stripped = value.strip() + if not stripped: + return [] + with contextlib.suppress(Exception): + parsed = json.loads(stripped) + if isinstance(parsed, list): + return [str(item).strip() for item in parsed if str(item).strip()] + return [item.strip() for item in stripped.split(",") if item.strip()] + return [str(value).strip()] + + +def _extract_findings(tool_calls: list[dict[str, t.Any]]) -> list[dict[str, t.Any]]: + findings: list[dict[str, t.Any]] = [] + suffix = f"__{RECORD_ASM_FINDING_TOOL}" + for call in tool_calls: + if not isinstance(call, dict): + continue + name = call.get("name") + if not isinstance(name, str): + continue + if name != RECORD_ASM_FINDING_TOOL and not name.endswith(suffix): + continue + args = call.get("arguments") + if not isinstance(args, dict): + continue + severity = str(args.get("severity") or "").strip().lower() + if severity not in {"high", "critical"}: + continue + finding = dict(args) + finding["severity"] = severity + finding.setdefault("id", f"ASM-FINDING-{len(findings) + 1:03d}") + findings.append(finding) + return findings + + +def _compact_tool_call_summary(tool_calls: list[dict[str, t.Any]]) -> str: + sections = ["Tool evidence summary:"] + for index, call in enumerate(tool_calls[:12], start=1): + if not isinstance(call, dict): + continue + name = str(call.get("name") or call.get("tool_name") or f"tool_call_{index}") + arguments = call.get("arguments") + result = ( + call.get("result") + or call.get("content") + or call.get("output") + or call.get("response") + ) + sections.append(f"\n{index}. {name}") + if arguments: + sections.append( + f" args: {_truncate(json.dumps(arguments, sort_keys=True, default=str), 500)}" + ) + if result: + sections.append(f" result: {_truncate(str(result), 1200)}") + if len(tool_calls) > 12: + sections.append( + f"\n... {len(tool_calls) - 12} additional tool calls omitted ..." + ) + return "\n".join(sections) + + +def _render_scope_context(scope: dict[str, t.Any]) -> str: + return json.dumps(scope, indent=2, sort_keys=True, default=str) + + +def _scope_prompt(scope_context: str, max_steps: int) -> str: + return ( + f"{_worker_stage_guard()}\n\n" + "Normalize this ASM target scope for downstream agents.\n" + "Do not call scan, web, graph, screenshot, enrichment, or recording tools in this stage. " + "Use only the provided scope payload and produce a concise scope contract.\n" + f"Autonomous step budget: {max_steps}\n\n" + f"Scope payload:\n```json\n{scope_context}\n```\n" + ) + + +def _discovery_prompt(scope_context: str, scope_report: str, max_steps: int) -> str: + return ( + f"{_worker_stage_guard()}\n\n" + "Perform ASM discovery within the normalized scope.\n" + f"Autonomous step budget: {max_steps}\n\n" + f"Scope payload:\n```json\n{scope_context}\n```\n\n" + f"Scope normalization:\n{scope_report}\n" + ) + + +def _lead_enrichment_prompt( + scope_context: str, scope_report: str, discovery_report: str, max_steps: int +) -> str: + return ( + f"{_worker_stage_guard()}\n\n" + "Enrich the most promising ASM leads.\n" + f"Autonomous step budget: {max_steps}\n\n" + f"Scope payload:\n```json\n{scope_context}\n```\n\n" + f"Scope normalization:\n{_truncate(scope_report, 8_000)}\n\n" + f"Discovery report:\n{_truncate(discovery_report, REPORT_TRUNCATE_CHARS)}\n" + ) + + +def _gadget_prompt( + scope_context: str, scope_report: str, discovery_report: str, max_steps: int +) -> str: + return ( + f"{_worker_stage_guard()}\n\n" + "Cluster ASM leads into plausible attack-surface gadgets.\n" + f"Autonomous step budget: {max_steps}\n\n" + f"Scope payload:\n```json\n{scope_context}\n```\n\n" + f"Scope normalization:\n{_truncate(scope_report, 8_000)}\n\n" + f"Discovery report:\n{_truncate(discovery_report, REPORT_TRUNCATE_CHARS)}\n" + ) + + +def _final_review_prompt( + scope_context: str, + scope_report: str, + discovery_report: str, + enriched_report: str, + gadget_report: str, + max_steps: int, +) -> str: + return ( + f"{_worker_stage_guard()}\n\n" + "Reconcile ASM reports into findings and leads.\n" + f"Autonomous step budget: {max_steps}\n\n" + f"Scope payload:\n```json\n{scope_context}\n```\n\n" + f"Scope normalization:\n{_truncate(scope_report, 8_000)}\n\n" + f"Discovery report:\n{_truncate(discovery_report, REPORT_TRUNCATE_CHARS)}\n\n" + f"Enriched leads:\n{_truncate(enriched_report, REPORT_TRUNCATE_CHARS)}\n\n" + f"Gadget clusters:\n{_truncate(gadget_report, REPORT_TRUNCATE_CHARS)}\n" + ) + + +def _validator_prompt( + scope_context: str, + final_review: str, + finding: dict[str, t.Any], + max_steps: int, +) -> str: + finding_json = json.dumps(finding, indent=2, sort_keys=True, default=str) + return ( + f"{_worker_stage_guard()}\n\n" + "Validate one ASM finding.\n" + f"Autonomous step budget: {max_steps}\n\n" + f"Scope payload:\n```json\n{scope_context}\n```\n\n" + f"Finding:\n```json\n{finding_json}\n```\n\n" + f"Final review context:\n{final_review}\n" + ) + + +def _report_synthesis_prompt( + *, + scope_context: str, + scope_report: str, + discovery_report: str, + enriched_report: str, + gadget_report: str, + final_review: str, + findings: list[dict[str, t.Any]], + validation_reports: dict[str, str], + max_steps: int, +) -> str: + findings_json = json.dumps(findings, indent=2, sort_keys=True, default=str) + validations_json = json.dumps(validation_reports, indent=2, sort_keys=True) + return ( + f"{_worker_stage_guard()}\n\n" + "Synthesize the final ASM operator report.\n" + f"Autonomous step budget: {max_steps}\n\n" + f"Scope payload:\n```json\n{scope_context}\n```\n\n" + f"Scope normalization:\n{_truncate(scope_report, 8_000)}\n\n" + f"Discovery report:\n{_truncate(discovery_report, REPORT_TRUNCATE_CHARS)}\n\n" + f"Enriched leads:\n{_truncate(enriched_report, REPORT_TRUNCATE_CHARS)}\n\n" + f"Gadget clusters:\n{_truncate(gadget_report, REPORT_TRUNCATE_CHARS)}\n\n" + f"Final review:\n{_truncate(final_review, REPORT_TRUNCATE_CHARS)}\n\n" + f"Recorded findings:\n```json\n{findings_json}\n```\n\n" + f"Validator reports:\n```json\n{validations_json}\n```\n" + ) + + +async def _publish_progress( + client: RuntimeClient, run_id: str, stage: str, detail: str | None = None +) -> None: + payload: dict[str, t.Any] = {"run_id": run_id, "stage": stage} + if detail: + payload["detail"] = detail + await client.publish(PROGRESS_EVENT, payload) + + +async def _publish_report( + client: RuntimeClient, run_id: str, target: str, agent: str, report: str +) -> None: + await client.publish( + REPORT_READY_EVENT, + {"run_id": run_id, "target": target, "agent": agent, "report": report}, + ) + + +def _build_final_markdown( + synthesized_report: str, + findings: list[dict[str, t.Any]], + validation_reports: dict[str, str], +) -> str: + sections = [synthesized_report.rstrip(), "", "## Worker Validation Summary"] + if not findings: + sections.append( + "No high or critical ASM findings were recorded for validator fan-out." + ) + return "\n".join(sections).rstrip() + "\n" + sections.append( + f"Validated {len(validation_reports)} of {len(findings)} recorded high/critical findings." + ) + for finding in findings: + finding_id = str(finding.get("id") or "unknown-finding") + title = str(finding.get("title") or "Untitled finding") + sections.extend(["", f"### {finding_id}: {title}"]) + sections.append( + validation_reports.get(finding_id) or "No validator report produced." + ) + return "\n".join(sections).rstrip() + "\n" + + +def _fallback_synthesis_report( + *, + scope_context: str, + scope_report: str, + discovery_report: str, + enriched_report: str, + gadget_report: str, + final_review: str, + findings: list[dict[str, t.Any]], + validation_reports: dict[str, str], +) -> str: + sections = [ + "# ASM Worker Pipeline Report", + "", + "Deterministic synthesis of worker-stage evidence.", + "", + "## Discovery", + _truncate(discovery_report or "No discovery output returned.", 8_000), + "", + "## Lead Enrichment", + _truncate(enriched_report or "No lead-enrichment output returned.", 8_000), + "", + "## Gadget Clustering", + _truncate(gadget_report or "No gadget-clustering output returned.", 8_000), + "", + "## Final Review", + _truncate(final_review or "No final-review output returned.", 8_000), + "", + "## Recorded High/Critical Findings", + json.dumps(findings, indent=2, sort_keys=True, default=str) + if findings + else "No high or critical findings were recorded for validator fan-out.", + "", + "## Validator Reports", + json.dumps(validation_reports, indent=2, sort_keys=True, default=str) + if validation_reports + else "No validator reports were produced.", + "", + "## Scope", + _truncate(scope_context, 2_000), + "", + "## Scope Normalization", + _truncate(scope_report or "No scope-normalizer output returned.", 3_000), + "", + "## Recommended Next Loop", + "Continue bounded in-scope validation of unresolved leads with direct ASM tools, prioritizing hosts with concrete DNS, HTTP, technology, screenshot, or finding evidence.", + ] + return "\n".join(sections).rstrip() + "\n" + + +def _truncate(text: str, limit: int) -> str: + return text if len(text) <= limit else text[:limit] + "\n... truncated ..." + + +def _worker_stage_guard() -> str: + return ( + "You are already running inside the worker-coordinated ASM pipeline. " + "Do not call run_asm_worker_pipeline from this stage; use direct ASM " + "scan, query, enrichment, screenshot, and recording tools as needed." + ) + + +def _stage_budget(max_steps: int, cap: int) -> int: + return max(1, min(int(max_steps), cap)) + + +def _label_safe(value: str) -> str: + value = re.sub(r"[^A-Za-z0-9_.:-]+", "_", str(value).strip()) + return value[:120] or "unknown" + + +if __name__ == "__main__": + worker.run() From 45fbbb51cc5d7ead509878d5a212b2861307de3b Mon Sep 17 00:00:00 2001 From: GangGreenTemperTatum <104169244+GangGreenTemperTatum@users.noreply.github.com> Date: Thu, 11 Jun 2026 13:21:43 -0400 Subject: [PATCH 2/2] Bump ASM capability to 2.0.0 --- capabilities/attack-surface-management/capability.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capabilities/attack-surface-management/capability.yaml b/capabilities/attack-surface-management/capability.yaml index 5c132a8..6210fa0 100644 --- a/capabilities/attack-surface-management/capability.yaml +++ b/capabilities/attack-surface-management/capability.yaml @@ -1,6 +1,6 @@ schema: 1 name: attack-surface-management -version: "1.1.60" +version: "2.0.0" description: > External attack surface management with BBOT reconnaissance scanning, Neo4j graph database analysis, Shodan internet intelligence, and