fix(ai_visibility): aggregate batch score, dimensions and signals over the same pages (#785)

CHANGELOG.md

@@ -65,6 +65,7 @@ and verification for each change is in the
 
 ### Fixed
 
+- **AI visibility:** The batch result aggregates overall score, dimension scores and signals over the same page set instead of taking the first page's details; signals are majority outcomes with an `example_url` for their values, and the result no longer depends on page order (#785).
 - **AI transparency:** An image fetch that redirects is counted as failed instead of being dropped silently, and a page whose image fetches all failed reports the check as failed rather than "no AI-generated images".
 - **DNS:** A lookup timeout or SERVFAIL no longer reads as a missing CAA, DNSKEY or SPF record, and an unknown MX answer leaves SPF unknown instead of "no mail".
 - **Security:** A probe that times out, cannot connect or ends in an error status (e.g. a WAF 403) leaves security unmeasured instead of analysing an error page; a rejected `HEAD` falls back to `GET`, and source maps of assets that did not answer make that check partial.

src/ai_visibility/mod.rs

@@ -295,6 +295,9 @@ pub struct AiSignal {
     /// Interpolated values needed to reproduce `detail` in another language
     #[serde(default)]
     pub values: AiSignalValues,
+    /// Batch only: the page whose `values` illustrate this aggregated signal
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub example_url: Option<String>,
 }
 
 impl AiSignal {
@@ -314,6 +317,7 @@ impl AiSignal {
             weight,
             detail,
             values,
+            example_url: None,
         }
     }
 }
@@ -564,27 +568,175 @@ pub fn analyze_ai_visibility(report: &AuditReport) -> AiVisibilityAnalysis {
     }
 }
 
-/// Derive AI visibility for batch mode (average across pages).
+/// Derive AI visibility for batch mode.
+///
+/// Overall score, dimension scores and signals all describe the same page
+/// set: dimension scores are the mean over pages, the overall score is the
+/// weighted average of those dimension means, and each signal is the majority
+/// outcome over the pages that carry it. A signal's interpolated values cannot
+/// be averaged, so they come from an example page named in `example_url`.
+/// Per-page lists (sections, entities, relationships, link suggestions) have
+/// no aggregate and stay empty. Pages are processed in URL order, so the
+/// result does not depend on the order of `reports`.
 pub fn analyze_ai_visibility_batch(reports: &[AuditReport]) -> AiVisibilityAnalysis {
     if reports.is_empty() {
         return empty_analysis();
     }
 
-    let analyses: Vec<AiVisibilityAnalysis> = reports.iter().map(analyze_ai_visibility).collect();
+    let mut pages: Vec<(&str, AiVisibilityAnalysis)> = reports
+        .iter()
+        .map(|r| (r.url.as_str(), analyze_ai_visibility(r)))
+        .collect();
+    pages.sort_by(|a, b| a.0.cmp(b.0));
+
+    let aggregate = |kind: DimensionKind, pick: fn(&AiVisibilityAnalysis) -> &DimensionScore| {
+        aggregate_dimension(kind, pages.iter().map(|(url, a)| (*url, pick(a))).collect())
+    };
+    let readability = aggregate(DimensionKind::Readability, |a| &a.readability.dimension);
+    let citation = aggregate(DimensionKind::Citability, |a| &a.citation.dimension);
+    let chunk_dim = aggregate(DimensionKind::Chunks, |a| &a.chunks.dimension);
+    let kg_dim = aggregate(DimensionKind::KnowledgeGraph, |a| {
+        &a.knowledge_graph.dimension
+    });
+    let policy_dim = aggregate(DimensionKind::Policy, |a| &a.policy.dimension);
+
+    let score = weighted_average(&[
+        (readability.score, 25),
+        (citation.score, 25),
+        (chunk_dim.score, 20),
+        (kg_dim.score, 15),
+        (policy_dim.score, 15),
+    ]);
+
+    let n = pages.len();
+    let majority =
+        |f: fn(&PolicyAnalysis) -> bool| pages.iter().filter(|(_, a)| f(&a.policy)).count() * 2 > n;
+    let mode = |mut values: Vec<String>| {
+        values.sort();
+        let mut best = (String::new(), 0);
+        for v in &values {
+            let count = values.iter().filter(|x| *x == v).count();
+            if count > best.1 {
+                best = (v.clone(), count);
+            }
+        }
+        best.0
+    };
+
+    // Most frequent recommendation; ties go to the earlier entry of this list.
+    let kinds = [
+        chunks::ChunkRecommendationKind::WellSuited,
+        chunks::ChunkRecommendationKind::SplitOversized,
+        chunks::ChunkRecommendationKind::TooLittleStructure,
+        chunks::ChunkRecommendationKind::Mixed,
+    ];
+    let recommendation_kind = kinds
+        .into_iter()
+        .rev()
+        .max_by_key(|k| {
+            pages
+                .iter()
+                .filter(|(_, a)| a.chunks.recommendation_kind == *k)
+                .count()
+        })
+        .expect("kinds is non-empty");
+    let recommendation_counts = pages.iter().fold((0, 0, 0), |acc, (_, a)| {
+        let c = a.chunks.recommendation_counts;
+        (acc.0 + c.0, acc.1 + c.1, acc.2 + c.2)
+    });
+
+    AiVisibilityAnalysis {
+        score,
+        band: crate::registry::FIVE_BAND
+            .label(score as f32, true)
+            .to_string(),
+        readability: ReadabilityAnalysis {
+            dimension: readability,
+        },
+        citation: CitationAnalysis {
+            dimension: citation,
+        },
+        chunks: ChunkAnalysis {
+            dimension: chunk_dim,
+            sections: vec![],
+            recommendation_kind,
+            recommendation_counts,
+            recommendation: chunks::ai_chunk_recommendation(
+                recommendation_kind,
+                recommendation_counts,
+                true,
+            ),
+        },
+        knowledge_graph: KnowledgeGraphAnalysis {
+            dimension: kg_dim,
+            entities: vec![],
+            relationships: vec![],
+            link_suggestions: vec![],
+        },
+        policy: PolicyAnalysis {
+            dimension: policy_dim,
+            blocks_ai_citation: majority(|p| p.blocks_ai_citation),
+            blocks_ai_training: majority(|p| p.blocks_ai_training),
+            blocks_all: majority(|p| p.blocks_all),
+            blocked_ai_bot_count: (pages
+                .iter()
+                .map(|(_, a)| a.policy.blocked_ai_bot_count)
+                .sum::<usize>() as f64
+                / n as f64)
+                .round() as usize,
+            inferred_policy: mode(
+                pages
+                    .iter()
+                    .map(|(_, a)| a.policy.inferred_policy.clone())
+                    .collect(),
+            ),
+        },
+        disclaimer: ai_disclaimer(true),
+    }
+}
+
+/// One dimension over a URL-sorted page set: mean score, majority signals.
+fn aggregate_dimension(kind: DimensionKind, pages: Vec<(&str, &DimensionScore)>) -> DimensionScore {
+    let score = (pages.iter().map(|(_, d)| d.score as f64).sum::<f64>() / pages.len() as f64)
+        .round() as u32;
 
-    let avg_score = analyses.iter().map(|a| a.score).sum::<u32>() / analyses.len() as u32;
+    let mut kinds: Vec<AiSignalKind> = Vec::new();
+    for (_, d) in &pages {
+        for s in &d.signals {
+            if !kinds.contains(&s.kind) {
+                kinds.push(s.kind);
+            }
+        }
+    }
+    kinds.sort_by_key(|k| *k as u32);
 
-    // Use the first report's detailed analysis as template, with averaged scores
-    let mut result = analyses
+    let signals = kinds
         .into_iter()
-        .next()
-        .expect("analyses is non-empty: reports is non-empty, checked above");
-    result.score = avg_score;
-    result.band = crate::registry::FIVE_BAND
-        .label(avg_score as f32, true)
-        .to_string();
-
-    result
+        .map(|k| {
+            let carrying: Vec<(&str, &AiSignal)> = pages
+                .iter()
+                .filter_map(|(url, d)| d.signals.iter().find(|s| s.kind == k).map(|s| (*url, s)))
+                .collect();
+            let present = carrying.iter().filter(|(_, s)| s.present).count() * 2 > carrying.len();
+            let weight =
+                carrying.iter().map(|(_, s)| s.weight).sum::<f32>() / carrying.len() as f32;
+            let (url, example) = carrying
+                .iter()
+                .find(|(_, s)| s.present == present)
+                .expect("a majority outcome is carried by at least one page");
+            let mut signal = AiSignal::new(k, present, weight, example.values.clone());
+            signal.example_url = Some((*url).to_string());
+            signal
+        })
+        .collect();
+
+    DimensionScore {
+        kind,
+        name: ai_dimension_name(kind, true).to_string(),
+        score,
+        label: ai_dimension_label(score, true),
+        signals,
+    }
 }
 
 // ─── Policy analysis ────────────────────────────────────────────────────────
@@ -1308,6 +1460,164 @@ mod tests {
         assert!(analysis.score <= 100);
     }
 
+    /// Three strongly different pages: no SEO data, a rich page with a
+    /// reachable robots.txt, and a thin page with a robots.txt that blocks all.
+    fn different_pages() -> Vec<AuditReport> {
+        use crate::seo::{
+            HeadingStructure, MetaTags, SeoAnalysis, SocialTags, StructuredData, TechnicalSeo,
+        };
+        let seo = |word_count: u32, has_lang: bool, desc: Option<&str>| {
+            let technical = TechnicalSeo {
+                word_count,
+                has_lang,
+                ..Default::default()
+            };
+            let meta = MetaTags {
+                description: desc.map(str::to_string),
+                ..Default::default()
+            };
+            SeoAnalysis {
+                meta,
+                headings: HeadingStructure::default(),
+                technical,
+                social: SocialTags::default(),
+                structured_data: StructuredData::default(),
+                score: 50,
+                content_profile: None,
+                robots: None,
+                page_health: None,
+                serp: None,
+                meta_issues: vec![],
+                image_efficiency: None,
+                indexability: crate::seo::IndexabilityCheck::default(),
+            }
+        };
+
+        let bare = minimal_report();
+
+        let mut rich = minimal_report();
+        rich.url = "https://example.com/rich".into();
+        let mut rich_seo = seo(
+            2400,
+            true,
+            Some("A long and useful description of the page."),
+        );
+        rich_seo.robots = Some(crate::seo::robots::RobotsAudit {
+            fetched: true,
+            ..Default::default()
+        });
+        rich.discoverability.seo = Some(rich_seo);
+
+        let mut thin = minimal_report();
+        thin.url = "https://example.com/thin".into();
+        let mut thin_seo = seo(40, false, None);
+        thin_seo.robots = Some(crate::seo::robots::RobotsAudit {
+            fetched: true,
+            has_wildcard_disallow_all: true,
+            blocks_ai_citation: true,
+            ..Default::default()
+        });
+        thin.discoverability.seo = Some(thin_seo);
+
+        vec![bare, rich, thin]
+    }
+
+    #[test]
+    fn batch_is_invariant_under_page_order_and_consistent_with_its_dimensions() {
+        let reports = different_pages();
+        let singles: Vec<AiVisibilityAnalysis> =
+            reports.iter().map(analyze_ai_visibility).collect();
+        // The pages must actually differ, or the test proves nothing.
+        assert!(singles.iter().any(|a| a.score != singles[0].score));
+        assert!(singles
+            .iter()
+            .any(|a| a.readability.dimension.score != singles[0].readability.dimension.score));
+
+        let permutations: [[usize; 3]; 6] = [
+            [0, 1, 2],
+            [0, 2, 1],
+            [1, 0, 2],
+            [1, 2, 0],
+            [2, 0, 1],
+            [2, 1, 0],
+        ];
+        let json_of = |order: &[usize; 3]| {
+            let ordered: Vec<AuditReport> = order.iter().map(|&i| reports[i].clone()).collect();
+            serde_json::to_string(&analyze_ai_visibility_batch(&ordered)).unwrap()
+        };
+        let expected = json_of(&permutations[0]);
+        for order in &permutations {
+            assert_eq!(json_of(order), expected, "order {order:?}");
+        }
+
+        let batch = analyze_ai_visibility_batch(&reports);
+        let mean = |pick: fn(&AiVisibilityAnalysis) -> u32| {
+            (singles.iter().map(|a| pick(a) as f64).sum::<f64>() / singles.len() as f64).round()
+                as u32
+        };
+        assert_eq!(
+            batch.readability.dimension.score,
+            mean(|a| a.readability.dimension.score)
+        );
+        assert_eq!(
+            batch.citation.dimension.score,
+            mean(|a| a.citation.dimension.score)
+        );
+        assert_eq!(
+            batch.chunks.dimension.score,
+            mean(|a| a.chunks.dimension.score)
+        );
+        assert_eq!(
+            batch.knowledge_graph.dimension.score,
+            mean(|a| a.knowledge_graph.dimension.score)
+        );
+        assert_eq!(
+            batch.policy.dimension.score,
+            mean(|a| a.policy.dimension.score)
+        );
+        assert_eq!(
+            batch.score,
+            weighted_average(&[
+                (batch.readability.dimension.score, 25),
+                (batch.citation.dimension.score, 25),
+                (batch.chunks.dimension.score, 20),
+                (batch.knowledge_graph.dimension.score, 15),
+                (batch.policy.dimension.score, 15),
+            ])
+        );
+        // Same page set as the per-page mean of the overall score (rounding only).
+        assert!(batch.score.abs_diff(mean(|a| a.score)) <= 1);
+    }
+
+    #[test]
+    fn batch_signals_are_majority_outcomes_with_an_example_url() {
+        let reports = different_pages();
+        let batch = analyze_ai_visibility_batch(&reports);
+        for s in batch
+            .readability
+            .dimension
+            .signals
+            .iter()
+            .chain(&batch.policy.dimension.signals)
+        {
+            let url = s
+                .example_url
+                .as_deref()
+                .expect("aggregated signal names an example");
+            assert!(reports.iter().any(|r| r.url == url));
+        }
+        // Wildcard block exists on 1 of 2 pages carrying the signal: not a majority.
+        let wildcard = batch
+            .policy
+            .dimension
+            .signals
+            .iter()
+            .find(|s| s.kind == AiSignalKind::NoWildcardBlock)
+            .unwrap();
+        assert!(!wildcard.present);
+        assert!(!batch.policy.blocks_all);
+    }
+
     #[test]
     fn band_replaces_the_letter_grade_this_module_no_longer_carries() {
         // Plan 29, D2: an indicator of weight 0 gets a band word, not A-F.