diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bb52c0..6cbbb98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Discover agent-facing resources from HTTP and HTML declarations plus bounded + conventional probes, including `llms.txt`, `auth.md`, OAuth metadata, MCP + server cards, A2A agent cards, and Agent Skills indexes. Markdown responses + include a compact navigation appendix. + ## [0.5.0] - 2026-07-14 ### Highlights diff --git a/README.md b/README.md index a776e85..2de08f9 100644 --- a/README.md +++ b/README.md @@ -328,6 +328,8 @@ HTML is automatically converted to markdown: - Code: Fenced blocks and inline backticks - Links: `[text](url)` format - Strips: scripts, styles, iframes, SVGs +- Adds a bounded [Agent resources](docs/agent-discoverability.md) navigation appendix + when discoverable resources are available ## License diff --git a/crates/fetchkit/src/convert.rs b/crates/fetchkit/src/convert.rs index 025afe8..c886e41 100644 --- a/crates/fetchkit/src/convert.rs +++ b/crates/fetchkit/src/convert.rs @@ -1,6 +1,6 @@ //! HTML conversion utilities -use crate::types::{PageLink, PageMetadata}; +use crate::types::{AgentResource, PageLink, PageMetadata}; use url::Url; /// Check if content-type indicates markdown (e.g. `text/markdown`). @@ -734,13 +734,36 @@ pub fn extract_metadata(html: &str) -> PageMetadata { } "link" if !is_closing => { if let Some(rel) = extract_attribute(&tag, "rel") { - if rel == "canonical" { + if rel + .split_ascii_whitespace() + .any(|value| value == "canonical") + { if let Some(href) = extract_attribute(&tag, "href") { if meta.canonical_url.is_none() && !href.is_empty() { meta.canonical_url = Some(href); } } } + if is_agent_link_relation(&rel) { + if let Some(href) = extract_attribute(&tag, "href") { + if !href.is_empty() && meta.agent_resources.len() < 20 { + let media_type = extract_attribute(&tag, "type"); + meta.agent_resources.push(AgentResource { + kind: classify_agent_resource( + &href, + &rel, + media_type.as_deref(), + ), + url: href, + source: "html-link".to_string(), + relation: Some(rel), + media_type, + title: extract_attribute(&tag, "title"), + verified: false, + }); + } + } + } } } "time" if !is_closing => { @@ -870,6 +893,65 @@ fn heading_level(tag_name: &str) -> Option { } /// Extract metadata from a `` tag. +fn is_agent_link_relation(rel: &str) -> bool { + rel.split_ascii_whitespace().any(|value| { + matches!( + value.to_ascii_lowercase().as_str(), + "alternate" + | "service-desc" + | "describedby" + | "authorization_endpoint" + | "mcp" + | "a2a" + | "agent-card" + | "skill" + ) + }) +} + +fn is_agent_metadata_name(name: &str) -> bool { + matches!( + name, + "llms" + | "llms-full" + | "auth" + | "service-desc" + | "api-catalog" + | "mcp" + | "a2a" + | "agent-card" + | "agent-skills" + ) +} + +pub(crate) fn classify_agent_resource(href: &str, rel: &str, media_type: Option<&str>) -> String { + let lower = href.to_ascii_lowercase(); + let rel = rel.to_ascii_lowercase(); + let media_type = media_type.unwrap_or_default().to_ascii_lowercase(); + if lower.ends_with("/llms-full.txt") { + "llms-full-txt" + } else if lower.ends_with("/llms.txt") { + "llms-txt" + } else if lower.ends_with("/auth.md") { + "auth" + } else if lower.contains("oauth") || rel.contains("authorization") { + "oauth" + } else if lower.contains("mcp") || rel.contains("mcp") { + "mcp" + } else if lower.contains("agent") || rel.contains("a2a") || rel.contains("agent") { + "agent-card" + } else if lower.contains("skill") || rel.contains("skill") { + "agent-skills" + } else if rel.contains("service-desc") || media_type.contains("openapi") { + "api-description" + } else if media_type.contains("markdown") { + "markdown" + } else { + "linked-resource" + } + .to_string() +} + fn extract_meta_tag(tag: &str, meta: &mut PageMetadata) { // if let Some(content) = extract_attribute(tag, "content") { @@ -878,7 +960,24 @@ fn extract_meta_tag(tag: &str, meta: &mut PageMetadata) { } // Check name attribute if let Some(name) = extract_attribute(tag, "name") { - match name.to_lowercase().as_str() { + let name_lower = name.to_ascii_lowercase(); + if is_agent_metadata_name(&name_lower) + && meta.agent_resources.len() < 20 + && (content.starts_with('/') + || content.starts_with("http://") + || content.starts_with("https://")) + { + meta.agent_resources.push(AgentResource { + kind: classify_agent_resource(&content, &name_lower, None), + url: content.clone(), + source: "metadata".to_string(), + relation: Some(name_lower.clone()), + media_type: None, + title: None, + verified: false, + }); + } + match name_lower.as_str() { "description" if meta.description.is_none() => { meta.description = Some(content.clone()); } @@ -1653,6 +1752,25 @@ mod tests { ); } + #[test] + fn test_extract_metadata_agent_links() { + let html = r#" + + + + + "#; + let meta = extract_metadata(html); + + assert_eq!(meta.agent_resources.len(), 3); + assert_eq!(meta.agent_resources[0].kind, "markdown"); + assert_eq!(meta.agent_resources[0].url, "/page.md"); + assert_eq!(meta.agent_resources[0].title.as_deref(), Some("Markdown")); + assert_eq!(meta.agent_resources[1].kind, "api-description"); + assert_eq!(meta.agent_resources[2].kind, "mcp"); + assert_eq!(meta.agent_resources[2].source, "metadata"); + } + #[test] fn test_extract_metadata_author() { let html = r#" diff --git a/crates/fetchkit/src/fetchers/default.rs b/crates/fetchkit/src/fetchers/default.rs index 03c17ab..9d7f7fe 100644 --- a/crates/fetchkit/src/fetchers/default.rs +++ b/crates/fetchkit/src/fetchers/default.rs @@ -9,20 +9,21 @@ use crate::client::FetchOptions; use crate::convert::{ - extract_headings, extract_metadata, extract_readable_content, filter_excessive_newlines, - html_to_markdown_with_base_url, html_to_text, is_html, is_markdown_content_type, - is_plain_text_content_type, strip_boilerplate, + classify_agent_resource, extract_headings, extract_metadata, extract_readable_content, + filter_excessive_newlines, html_to_markdown_with_base_url, html_to_text, is_html, + is_markdown_content_type, is_plain_text_content_type, strip_boilerplate, }; use crate::error::FetchError; use crate::fetchers::Fetcher; use crate::file_saver::FileSaver; use crate::transport::{BodyStream, TransportMethod, TransportRequest, TransportResponse}; -use crate::types::{FetchRequest, FetchResponse, HttpMethod, PageQuality}; +use crate::types::{AgentResource, FetchRequest, FetchResponse, HttpMethod, PageQuality}; use crate::DEFAULT_USER_AGENT; use async_trait::async_trait; use bytes::Bytes; use futures::StreamExt; use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, CONTENT_DISPOSITION, LOCATION, USER_AGENT}; +use std::collections::HashSet; use std::time::Duration; use tracing::{debug, error, warn}; use url::Url; @@ -96,6 +97,24 @@ pub(crate) const TRUNCATION_MESSAGE: &str = "\n\n[..content truncated...]"; // THREAT[TM-SSRF-010]: Maximum redirects to follow with IP validation at each hop const MAX_REDIRECTS: usize = 10; +// Agent discovery is deliberately shallow: fixed same-origin paths, no recursion. +const AGENT_RESOURCE_PROBES: &[(&str, &str)] = &[ + ("/llms.txt", "llms-txt"), + ("/llms-full.txt", "llms-full-txt"), + ("/auth.md", "auth"), + ("/.well-known/oauth-authorization-server", "oauth"), + ("/.well-known/openid-configuration", "openid"), + ( + "/.well-known/oauth-protected-resource", + "oauth-protected-resource", + ), + ("/.well-known/api-catalog", "api-catalog"), + ("/.well-known/mcp/server-card.json", "mcp"), + ("/.well-known/agent-card.json", "agent-card"), + ("/.well-known/agent-skills/index.json", "agent-skills"), +]; +const AGENT_PROBE_TIMEOUT: Duration = Duration::from_secs(3); + // THREAT[TM-DOS-001]: Default max body size (10 MB) to prevent memory exhaustion // THREAT[TM-DOS-003]: Also protects against compressed content bombs (gzip bombs) pub(crate) const DEFAULT_MAX_BODY_SIZE: usize = 10 * 1024 * 1024; @@ -215,6 +234,189 @@ fn extract_response_meta(headers: &[(String, String)], url: &str) -> ResponseMet } } +fn resources_from_link_headers(values: &[String], base_url: &str) -> Vec { + let Ok(base_url) = Url::parse(base_url) else { + return Vec::new(); + }; + values + .iter() + .flat_map(|value| split_link_header(value)) + .filter_map(|part| { + let (target, params) = part.split_once('>')?; + let target = target.trim().strip_prefix('<')?; + let relation = link_parameter(params, "rel"); + let media_type = link_parameter(params, "type"); + if !is_agent_link(relation.as_deref(), media_type.as_deref(), target) { + return None; + } + let url = base_url.join(target).ok()?; + Some(AgentResource { + kind: classify_agent_resource( + url.as_str(), + relation.as_deref().unwrap_or_default(), + media_type.as_deref(), + ), + url: url.to_string(), + source: "http-link".to_string(), + relation, + media_type, + title: link_parameter(params, "title"), + verified: false, + }) + }) + .collect() +} + +fn split_link_header(value: &str) -> Vec<&str> { + let mut parts = Vec::new(); + let mut start = 0; + let mut quoted = false; + for (index, ch) in value.char_indices() { + match ch { + '"' => quoted = !quoted, + ',' if !quoted => { + parts.push(value[start..index].trim()); + start = index + 1; + } + _ => {} + } + } + parts.push(value[start..].trim()); + parts +} + +fn link_parameter(params: &str, name: &str) -> Option { + params.split(';').skip(1).find_map(|part| { + let (key, value) = part.trim().split_once('=')?; + key.eq_ignore_ascii_case(name) + .then(|| value.trim().trim_matches('"').to_string()) + }) +} + +fn is_agent_link(relation: Option<&str>, media_type: Option<&str>, target: &str) -> bool { + let relation = relation.unwrap_or_default().to_ascii_lowercase(); + let media_type = media_type.unwrap_or_default().to_ascii_lowercase(); + let target = target.to_ascii_lowercase(); + let relevant_relation = relation.split_ascii_whitespace().any(|value| { + matches!( + value, + "alternate" + | "service-desc" + | "describedby" + | "authorization_endpoint" + | "mcp" + | "a2a" + | "agent-card" + | "skill" + ) + }); + relevant_relation + && (media_type.contains("markdown") + || media_type.contains("json") + || target.contains("llms") + || target.contains("auth.md") + || target.contains("well-known") + || relation != "alternate") +} + +fn normalize_html_resources(resources: &mut Vec, base_url: &str) { + let Ok(base_url) = Url::parse(base_url) else { + resources.clear(); + return; + }; + resources.retain_mut(|resource| match base_url.join(&resource.url) { + Ok(url) if matches!(url.scheme(), "http" | "https") => { + resource.url = url.to_string(); + true + } + _ => false, + }); +} + +fn deduplicate_resources(resources: &mut Vec) { + let mut seen = HashSet::new(); + resources.retain(|resource| seen.insert(resource.url.clone())); + resources.truncate(20); +} + +fn append_agent_resources(content: &mut String, resources: &[AgentResource]) { + content.push_str("\n\n---\n\n## Agent resources\n\n"); + for resource in resources { + let label = resource.title.as_deref().unwrap_or(&resource.kind); + let status = if resource.verified { + "verified" + } else { + "advertised" + }; + content.push_str(&format!( + "- [{label}]({}) — `{}`; {status} via {}\n", + resource.url, resource.kind, resource.source + )); + } +} + +fn probe_content_type_matches(kind: &str, content_type: Option<&str>) -> bool { + let Some(content_type) = content_type else { + return true; + }; + let content_type = content_type.to_ascii_lowercase(); + match kind { + "llms-txt" | "llms-full-txt" | "auth" => { + content_type.starts_with("text/plain") || content_type.contains("markdown") + } + _ => content_type.contains("json"), + } +} + +async fn probe_agent_resources(base_url: &str, options: &FetchOptions) -> Vec { + let Ok(base_url) = Url::parse(base_url) else { + return Vec::new(); + }; + let origin = base_url.origin().ascii_serialization(); + let probes = AGENT_RESOURCE_PROBES + .iter() + .map(|(path, kind)| ((*path).to_string(), (*kind).to_string())) + .collect::>(); + futures::stream::iter(probes) + .map(|(path, kind)| { + let url = Url::parse(&format!("{origin}{path}")); + async move { + let url = url.ok()?; + let mut headers = HeaderMap::new(); + headers.insert(USER_AGENT, HeaderValue::from_static(DEFAULT_USER_AGENT)); + let (response, redirects) = send_request_following_redirects( + url.clone(), + reqwest::Method::HEAD, + headers, + options, + AGENT_PROBE_TIMEOUT, + ) + .await + .ok()?; + if !(200..300).contains(&response.status) || !redirects.is_empty() { + return None; + } + let meta = extract_response_meta(&response.headers, url.as_str()); + if !probe_content_type_matches(&kind, meta.content_type.as_deref()) { + return None; + } + Some(AgentResource { + url: url.to_string(), + kind, + source: "probe".to_string(), + relation: None, + media_type: meta.content_type, + title: None, + verified: true, + }) + } + }) + .buffer_unordered(4) + .filter_map(|resource| async move { resource }) + .collect() + .await +} + #[async_trait] impl Fetcher for DefaultFetcher { fn name(&self) -> &'static str { @@ -270,6 +472,12 @@ impl Fetcher for DefaultFetcher { let status_code = response.status; let final_url = response.url.to_string(); + let link_headers = response + .headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case("link")) + .map(|(_, value)| value.clone()) + .collect::>(); let meta = extract_response_meta(&response.headers, &final_url); // Handle 304 Not Modified (conditional request response) @@ -413,6 +621,24 @@ impl Fetcher for DefaultFetcher { final_content.push_str(TRUNCATION_MESSAGE); } + let mut resources = resources_from_link_headers(&link_headers, &final_url); + if let Some(metadata) = &mut page_metadata { + normalize_html_resources(&mut metadata.agent_resources, &final_url); + resources.append(&mut metadata.agent_resources); + } + if wants_markdown { + resources.extend(probe_agent_resources(&final_url, options).await); + } + deduplicate_resources(&mut resources); + if !resources.is_empty() { + if wants_markdown { + append_agent_resources(&mut final_content, &resources); + } + page_metadata + .get_or_insert_with(Default::default) + .agent_resources = resources; + } + // Compute quality signals let word_count = count_words(&final_content); if let (Some(metadata), Some(method)) = (&mut page_metadata, extraction_method) { @@ -1736,6 +1962,74 @@ mod tests { assert!(response.redirect_chain.is_empty()); } + #[test] + fn test_parse_agent_resources_from_link_headers() { + let resources = resources_from_link_headers( + &[r#"; rel="alternate"; type="text/markdown"; title="LLM index", ; rel="stylesheet""#.to_string()], + "https://example.com/docs/page", + ); + + assert_eq!(resources.len(), 1); + assert_eq!(resources[0].url, "https://example.com/llms.txt"); + assert_eq!(resources[0].kind, "llms-txt"); + assert_eq!(resources[0].source, "http-link"); + assert_eq!(resources[0].title.as_deref(), Some("LLM index")); + assert!(!resources[0].verified); + } + + #[tokio::test] + async fn test_agent_resources_are_probed_and_appended_to_markdown() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/page")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(r#"

Page

"#) + .insert_header("content-type", "text/html") + .insert_header("link", r#"; rel="alternate"; type="text/markdown""#), + ) + .mount(&server) + .await; + Mock::given(method("HEAD")) + .and(path("/llms.txt")) + .respond_with(ResponseTemplate::new(200).insert_header("content-type", "text/markdown")) + .mount(&server) + .await; + Mock::given(method("HEAD")) + .and(path("/auth.md")) + .respond_with(ResponseTemplate::new(200).insert_header("content-type", "text/markdown")) + .mount(&server) + .await; + + let fetcher = DefaultFetcher::new(); + let options = FetchOptions { + enable_markdown: true, + dns_policy: DnsPolicy::allow_all(), + ..Default::default() + }; + let request = FetchRequest::new(format!("{}/page", server.uri())).as_markdown(); + let response = fetcher.fetch(&request, &options).await.unwrap(); + let resources = &response.metadata.as_ref().unwrap().agent_resources; + + assert_eq!( + resources + .iter() + .filter(|resource| resource.kind == "llms-txt") + .count(), + 1 + ); + assert!(resources + .iter() + .any(|resource| resource.kind == "auth" && resource.verified)); + assert!(resources + .iter() + .any(|resource| resource.kind == "api-description")); + let content = response.content.unwrap(); + assert!(content.contains("## Agent resources")); + assert!(content.contains("/auth.md")); + assert!(content.contains("/openapi.json")); + } + #[tokio::test] async fn test_paywall_detection() { let server = MockServer::start().await; diff --git a/crates/fetchkit/src/lib.rs b/crates/fetchkit/src/lib.rs index f5e5789..efc1697 100644 --- a/crates/fetchkit/src/lib.rs +++ b/crates/fetchkit/src/lib.rs @@ -108,8 +108,8 @@ pub use transport::{ TransportResponse, }; pub use types::{ - CrawlPage, CrawlResult, FetchRequest, FetchResponse, HttpMethod, PageLink, PageMetadata, - PageQuality, + AgentResource, CrawlPage, CrawlResult, FetchRequest, FetchResponse, HttpMethod, PageLink, + PageMetadata, PageQuality, }; #[cfg(feature = "bot-auth")] diff --git a/crates/fetchkit/src/types.rs b/crates/fetchkit/src/types.rs index b0783d2..c5a012a 100644 --- a/crates/fetchkit/src/types.rs +++ b/crates/fetchkit/src/types.rs @@ -324,6 +324,28 @@ pub struct PageLink { pub href: String, } +/// An agent-oriented resource advertised by a site or found at a conventional path. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +pub struct AgentResource { + /// Absolute resource URL. + pub url: String, + /// Stable resource kind, such as `llms-txt`, `auth`, or `mcp`. + pub kind: String, + /// Discovery source: `http-link`, `html-link`, `metadata`, or `probe`. + pub source: String, + /// Link relation when the site advertised one. + #[serde(skip_serializing_if = "Option::is_none")] + pub relation: Option, + /// Advertised or returned media type. + #[serde(skip_serializing_if = "Option::is_none")] + pub media_type: Option, + /// Human-readable title when available. + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option, + /// Whether FetchKit confirmed the resource with a request. + pub verified: bool, +} + /// Structured metadata extracted from an HTML page. /// /// All fields are optional — only populated when the corresponding @@ -362,6 +384,10 @@ pub struct PageMetadata { #[serde(skip_serializing_if = "Vec::is_empty", default)] pub links: Vec, + /// Agent-oriented resources advertised by or discovered for the site. + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub agent_resources: Vec, + /// Headings outline (e.g. `["# Title", "## Section 1", "## Section 2"]`) #[serde(skip_serializing_if = "Vec::is_empty", default)] pub headings: Vec, @@ -382,6 +408,7 @@ impl PageMetadata { && self.published_date.is_none() && self.modified_date.is_none() && self.links.is_empty() + && self.agent_resources.is_empty() && self.headings.is_empty() && self.extraction_method.is_none() } diff --git a/docs/agent-discoverability.md b/docs/agent-discoverability.md new file mode 100644 index 0000000..d5acfef --- /dev/null +++ b/docs/agent-discoverability.md @@ -0,0 +1,88 @@ +# Agent resource discovery + +FetchKit enriches regular `GET` responses with advertised resources that help +an agent navigate a site. Markdown `GET` requests additionally probe a bounded +set of conventional resources. Discovery is shallow and descriptive: FetchKit +reports resources but never invokes an advertised API, authentication flow, or +agent endpoint. + +## Discovery sources + +FetchKit combines these sources: + +- HTTP `Link` response headers with agent-relevant relations and Markdown, JSON, + or conventional agent-resource targets. +- HTML `` declarations using `alternate`, `service-desc`, `describedby`, + `authorization_endpoint`, `mcp`, `a2a`, `agent-card`, or `skill` relations. +- HTML `` declarations named `llms`, + `llms-full`, `auth`, `service-desc`, `api-catalog`, `mcp`, `a2a`, + `agent-card`, or `agent-skills`. +- A fixed set of conventional same-origin probes: + + ```text + /llms.txt + /llms-full.txt + /auth.md + /.well-known/oauth-authorization-server + /.well-known/openid-configuration + /.well-known/oauth-protected-resource + /.well-known/api-catalog + /.well-known/mcp/server-card.json + /.well-known/agent-card.json + /.well-known/agent-skills/index.json + ``` + +The list is intentionally explicit. FetchKit does not enumerate the unbounded +`/.well-known/` namespace. Probe requests use `HEAD`, run with bounded +concurrency and a short timeout, and do not recurse into discovered resources. +A probed resource is accepted only after a direct `2xx` response; redirects are +not accepted as verification. + +## Output + +Resources are returned as `PageMetadata.agent_resources`. Each resource has: + +- `url`: normalized absolute URL +- `kind`: stable category such as `llms-txt`, `auth`, `mcp`, or `oauth` +- `source`: `http-link`, `html-link`, `metadata`, or `probe` +- optional `relation`, `media_type`, and `title` +- `verified`: whether a conventional probe confirmed the URL + +Resources advertised by headers or HTML are marked unverified because FetchKit +does not issue an additional request merely to validate each arbitrary target. +Duplicates are removed and output is capped at 20 resources. + +For Markdown requests, FetchKit also appends an `Agent resources` section to the +returned document. Raw HTML is not modified; consumers can use the structured +metadata instead. + +## Security and operational behavior + +All probes use the same URL validation, DNS/IP policy, redirect validation, +proxy policy, and Web Bot Authentication transport rules as the original +request. Probes are restricted to fixed paths on the final response origin and +do not forward request-specific authorization headers. + +Discovery adds up to ten lightweight requests to a Markdown `GET`. Servers that +support `HEAD` can make these inexpensive. Failed, blocked, redirected, or timed +out probes are omitted without failing the requested page fetch. + +## Publishing resources for agents + +Sites get the strongest result by explicitly advertising resources: + +```http +Link: ; rel="alternate"; type="text/markdown"; title="LLM index" +Link: ; rel="service-desc"; type="application/openapi+json" +``` + +or in HTML: + +```html + + +``` + +Conventional resources should return an accurate status and content type for +`HEAD`. Avoid catch-all `200 OK` responses for nonexistent paths, because they +make protocol-level discovery ambiguous. diff --git a/specs/agent-discovery.md b/specs/agent-discovery.md new file mode 100644 index 0000000..e927d1e --- /dev/null +++ b/specs/agent-discovery.md @@ -0,0 +1,29 @@ +# Agent Resource Discovery + +## Abstract + +FetchKit enriches regular fetches with bounded discovery of resources intended +for AI agents. It reads explicit HTTP and HTML advertisements, probes a fixed +set of conventional same-origin paths, exposes typed metadata, and adds compact +navigation links to Markdown output. + +## Requirements + +1. Regular `GET` fetches MUST inspect all final-response `Link` header fields. +2. HTML fetches MUST inspect agent-relevant `` and `` declarations. +3. Markdown `GET` fetches MUST probe `/llms.txt`, `/llms-full.txt`, `/auth.md`, and the + documented fixed set of relevant `/.well-known/` paths on the final origin. +4. FetchKit MUST NOT enumerate arbitrary `/.well-known/` paths or recursively + fetch discovered resources. +5. Probes MUST use existing SSRF, DNS, redirect, proxy, timeout, and request + signing controls. +6. Probe concurrency, timeout, and result count MUST be bounded. +7. Failed discovery MUST NOT fail the requested page fetch. +8. Resources MUST be normalized, classified, deduplicated, and returned as + structured page metadata with source and verification state. +9. Markdown responses MUST end with a compact `Agent resources` section when + at least one resource is found. Raw HTML MUST NOT be modified. +10. Advertised resources MUST NOT be described as verified unless FetchKit + requested and validated that exact resource. +11. Discovery MUST NOT invoke APIs, authorization flows, payment protocols, or + agent capabilities. diff --git a/specs/threat-model.md b/specs/threat-model.md index 49d713f..e4fcf64 100644 --- a/specs/threat-model.md +++ b/specs/threat-model.md @@ -458,3 +458,31 @@ None — all previously open threats have been mitigated. - `specs/fetchers.md` — Pluggable fetcher system - [OWASP SSRF Prevention Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/Server_Side_Request_Forgery_Prevention_Cheat_Sheet.html) - [CWE-918: Server-Side Request Forgery](https://cwe.mitre.org/data/definitions/918.html) + +### TM-DISC-001: Agent discovery request amplification + +**Threat**: A single fetch triggers unbounded secondary requests or recursive +resource traversal. + +**Mitigations**: +- Probe only a fixed, documented path set on the final response origin. +- Never recursively follow discovered resources. +- Bound concurrency, per-probe timeout, and emitted resource count. +- Treat discovery failure as non-fatal. + +**Verification**: Tests assert fixed-path probing, deduplication, and bounded +resource output. + +### TM-DISC-002: Discovery bypasses network policy + +**Threat**: Advertised or conventional resources reach private networks, abuse +redirects, or receive credentials intended for another origin. + +**Mitigations**: +- Route probes through the normal DNS, IP, redirect, proxy, and signing policy. +- Probe fixed same-origin URLs only and reject redirected probes as verified. +- Do not forward request-specific authorization headers. +- Report arbitrary advertised links without fetching them. + +**Verification**: Discovery uses the shared request transport and tests use a +private-address policy override explicitly.