Merge dev into main

.github/workflows/checks.yml

@@ -178,5 +178,8 @@ jobs:
         run: pnpm install --frozen-lockfile
       - name: Test
         run: pnpm test
+      # 测量台的判等不依赖任何包，借这个 job 现成的 node 跑，不另开一条流水线
+      - name: Bench scoring tests
+        run: node --test ../scripts/bench/lib.test.mjs
       - name: Build (含类型检查)
         run: pnpm build

README.md

@@ -78,7 +78,7 @@ One Rust binary and one Postgres. Full-text search is embedded in the binary, ve
 | **Agent adjudication** | The agent adjudicates on its own, from common sense and the business documents. Low-confidence calls go to the human review queue, and the decisions people make there are recorded and used to tune the agent, closing the loop. |
 | **Reasoning and derivation** | Ontology axioms compile into rules: transitivity, symmetry, inverses and relation hierarchy derive new facts by forward chaining. Derivation is off by default, since a wrong axiom derives wrong facts. A derived fact is marked as such on the graph, carries validity and confidence like any other, and shows what it was derived from. When it contradicts an asserted fact, the asserted one stands. |
 | **Conflict detection** | Three kinds of conflict, three sets of choices. A new fact that clashes with an older one: close the old, keep both, or reject the new. Data that breaks an axiom (self-loop, asymmetry, transitive cycle, cardinality): retract the fact, relax the axiom, or accept both. The ontology itself is checked first, because violations of a self-contradictory ontology are noise. |
-| **Ontology-driven querying** | Mount a database on a base (Postgres, MySQL and the engines that speak its protocol, Trino for Iceberg / Delta Lake / Hive, Databricks, Snowflake) and chat can query it alongside the documents. The agent proposes how its tables map onto the ontology, and you confirm. The method behind it, [Ontology2SQL](https://github.com/deeplethe/ontology2sql), is state of the art on BIRD Mini-Dev for SQLite and PostgreSQL ([submission](https://github.com/bird-bench/bird-bench.github.io/pull/218)). |
+| **Ontology-driven querying** | Mount a database on a base (Postgres, MySQL and the engines that speak its protocol, ClickHouse, Trino for Iceberg / Delta Lake / Hive, Databricks, Snowflake) and chat can query it alongside the documents. The agent proposes how its tables map onto the ontology, and you confirm. The method behind it, [Ontology2SQL](https://github.com/deeplethe/ontology2sql), is state of the art on BIRD Mini-Dev for SQLite and PostgreSQL ([submission](https://github.com/bird-bench/bird-bench.github.io/pull/218)). |
 | **Multi-user and permissions** | Each knowledge base has its own members and roles: owner, admin, editor and viewer. Open bases are readable by everyone in the deployment, restricted ones only by invitation. The first account registered becomes the system administrator. |
 | **Decision ledger** | Confirming or rejecting a fact, merging or reverting an entity, rebuilding the graph: each leaves a record of who, when, and what the object looked like at the time. The ledger is append-only, and a record outlives its object, even the base it belonged to. |
 | **[Decision intelligence (in development)](#roadmap)** | Record a decision, replay both what was understood and the course it took, and reason over overlaid scenarios. |
@@ -126,7 +126,7 @@ cd web && pnpm install && pnpm dev
 - [ ] **Business rules**: rules written by people over an entity's attribute facts, a threshold or a category set, that classify it as a derived fact with the rule and the premises as its explanation ([#277](https://github.com/deeplethe/utopia/issues/277))
 - [ ] **Execution gate**: checking an agent's calls against ontology rules and symbolic logic
 - [ ] **MaxCompute**: mapping exploration and Ontology2SQL over Alibaba Cloud MaxCompute (Iceberg / Delta Lake via Trino, Databricks and Snowflake are in, awaiting a run against a real cluster)
-- [ ] **More sources**: a ClickHouse driver; a Feishu connector
+- [ ] **More sources**: a Feishu connector
 - [ ] **Agent memory over MCP**: episode writes, the retrieve endpoint, and the MCP server
 - [ ] **Enterprise**: OIDC SSO, backup and restore commands, benchmarks at 100k documents
 

README.zh-CN.md

@@ -127,7 +127,7 @@ cd web && pnpm install && pnpm dev
 - [ ] **业务规则**：由人写下、作用于实体属性事实的规则（阈值、类别集合），把实体归类为一条带前提的派生事实，规则与前提就是它的解释（[#277](https://github.com/deeplethe/utopia/issues/277)）
 - [ ] **执行校验层**：对 Agent 的调用进行本体规则与符号逻辑校验
 - [ ] **问数与映射添加数据湖仓支持**：Iceberg / Delta Lake，以及 Databricks、Snowflake、MaxCompute 的映射探索与 Ontology2SQL 支持
-- [ ] **更多数据源**：ClickHouse 驱动，飞书连接器
+- [ ] **更多数据源**：飞书连接器
 - [ ] **精确到时刻**：在年 / 月 / 日之外加一档 `instant` 精度，给那些本来就带时间戳的来源——现在连接器按 UTC 截到天，跨午夜的事件会差一天
 - [ ] **MCP 上的 Agent 记忆**：补齐 episodes 写入、retrieve 端点与 MCP 服务器
 - [ ] **企业化**：OIDC SSO、备份恢复命令、10 万文档级别的性能基准

crates/utopia-cli/src/main.rs

@@ -82,7 +82,7 @@ struct ManifestDataDir {
 /// not a side effect of a code change.
 // 是迁移文件的**个数**，不是最大的编号（守卫 `schema_version_policy_compares_against_current`
 // 按个数比）：编号有空缺时两者不同——0071 由一个开放 PR 占着，0072 先落，个数是 71
-const CURRENT_SCHEMA_VERSION: u32 = 96;
+const CURRENT_SCHEMA_VERSION: u32 = 97;
 
 fn main() -> anyhow::Result<()> {
     dotenvy::dotenv().ok();

crates/utopia-ingest/src/parsers.rs

@@ -485,6 +485,10 @@ pub(crate) fn docx_xml_to_text(
 
 /// PPTX: extract a:t text in the presentation's logical slide order. Tables under a:tbl
 /// become Markdown grids through the same renderer as DOCX and spreadsheet tables.
+///
+/// 第一页是这份演示的开头，写在任何标题之前；第二页起每页一节（`## Slide N`）。封面上的
+/// 日期说的是整份演示，而一个日期管到哪看它所在的标题（0064）：封面也是一节的话，它的
+/// 日期只管封面，后面每一页的陈述都没有见证。Word 的标题块本来就在第一个标题之前，这里一样
 pub fn pptx(bytes: &[u8]) -> anyhow::Result<String> {
     let mut archive =
         zip::ZipArchive::new(Cursor::new(bytes.to_vec())).context("Failed to unzip pptx")?;
@@ -515,7 +519,12 @@ pub fn pptx(bytes: &[u8]) -> anyhow::Result<String> {
     for (num, name) in slides {
         let xml = pptx_part(&mut archive, &name)?;
         let text = pptx_xml_to_text(&xml)?;
-        if !text.trim().is_empty() {
+        if text.trim().is_empty() {
+            continue;
+        }
+        if num == 1 {
+            out.push_str(&format!("{text}\n"));
+        } else {
             out.push_str(&format!("\n## Slide {num}\n{text}\n"));
         }
     }
@@ -1089,20 +1098,16 @@ fn pptx_xml_to_text(xml: &str) -> anyhow::Result<String> {
                 }
                 "a:tr" if table_depth == 1 => row.clear(),
                 "a:tc" if table_depth == 1 => {
+                    // DrawingML 的合并属性属于 a:tc，后面的 a:tcPr 只描述样式。
                     cell = Some(TableCell {
-                        span: 1,
+                        span: attr(&e, "gridSpan")
+                            .and_then(|v| v.parse().ok())
+                            .unwrap_or(1),
+                        horizontal_merge: truthy(&e, "hMerge"),
+                        vertical_merge: truthy(&e, "vMerge"),
                         ..TableCell::default()
                     });
                 }
-                "a:tcPr" if table_depth == 1 => {
-                    if let Some(c) = cell.as_mut() {
-                        if let Some(span) = attr(&e, "gridSpan").and_then(|v| v.parse().ok()) {
-                            c.span = span;
-                        }
-                        c.horizontal_merge = truthy(&e, "hMerge");
-                        c.vertical_merge = truthy(&e, "vMerge");
-                    }
-                }
                 "a:t" => {
                     if cell.is_some() || table_depth == 0 {
                         in_text = true;
@@ -1119,15 +1124,6 @@ fn pptx_xml_to_text(xml: &str) -> anyhow::Result<String> {
                 "a:tblPr" if table_depth == 1 => {
                     first_is_header = truthy(&e, "firstRow");
                 }
-                "a:tcPr" if table_depth == 1 => {
-                    if let Some(c) = cell.as_mut() {
-                        if let Some(span) = attr(&e, "gridSpan").and_then(|v| v.parse().ok()) {
-                            c.span = span;
-                        }
-                        c.horizontal_merge = truthy(&e, "hMerge");
-                        c.vertical_merge = truthy(&e, "vMerge");
-                    }
-                }
                 "a:br" => match cell.as_mut() {
                     Some(c) => c.text.push(' '),
                     None if table_depth == 0 => out.push('\n'),

crates/utopia-ingest/tests/pptx_breaks.rs

@@ -56,5 +56,5 @@ fn runs_without_a_break_still_join() {
     let text = read(
         r#"<a:p><a:r><a:t>Hel</a:t></a:r><a:r><a:t>lo</a:t></a:r></a:p><a:p><a:r><a:t>world</a:t></a:r></a:p>"#,
     );
-    assert!(text.contains("## Slide 1\nHello\nworld\n"), "{text}");
+    assert!(text.starts_with("Hello\nworld\n"), "{text}");
 }

crates/utopia-ingest/tests/pptx_order.rs

@@ -46,10 +46,12 @@ fn parsed(parts: &Parts) -> String {
 fn logical_order_and_page_numbers_do_not_depend_on_zip_or_part_order() {
     let mut parts = deck();
     let expected = parsed(&parts);
+    // 第一页是开头，没有标题；第二页起每页一节
     assert!(
-        expected.contains("## Slide 1\nSECOND-CREATED & 中 <raw>"),
+        expected.starts_with("SECOND-CREATED & 中 <raw>"),
         "{expected}"
     );
+    assert!(!expected.contains("## Slide 1"), "{expected}");
     assert!(expected.contains("## Slide 2\nFIRST-CREATED"), "{expected}");
     assert!(!expected.contains("UNLISTED"));
     assert!(!expected.contains("NOTES"));
@@ -66,7 +68,7 @@ fn logical_order_and_page_numbers_do_not_depend_on_zip_or_part_order() {
         .replace("rIdA", "rIdB")
         .replace("temp", "rIdA");
     let normal = parsed(&parts);
-    assert!(normal.contains("## Slide 1\nFIRST-CREATED"));
+    assert!(normal.starts_with("FIRST-CREATED"), "{normal}");
     assert!(normal.contains("## Slide 2\nSECOND-CREATED"));
 }
 
@@ -216,3 +218,13 @@ fn empty_presentations_and_manifest_free_legacy_packages_keep_their_behavior() {
     no_root.retain(|(n, _)| n != "_rels/.rels");
     assert_eq!(parsed(&no_root), parsed(&deck()));
 }
+
+/// 封面写在任何标题之前：它说的日期管整份演示，不只管它自己那一页（0064 的范围看标题）。
+/// 第一页没有字的演示没有开头，其余各页照旧各是一节
+#[test]
+fn the_first_slide_opens_the_deck_and_later_slides_are_sections() {
+    let text = parsed(&deck());
+    let first_heading = text.find("## ").expect("later slides are sections");
+    assert!(text[..first_heading].contains("SECOND-CREATED"), "{text}");
+    assert!(text[first_heading..].starts_with("## Slide 2\n"), "{text}");
+}

crates/utopia-ingest/tests/pptx_tables.rs

@@ -31,12 +31,12 @@ fn text_box(text: &str) -> String {
 }
 
 fn cell(text: &str) -> String {
-    cell_with_props(text, "")
+    cell_with_attrs(text, "")
 }
 
-fn cell_with_props(text: &str, props: &str) -> String {
+fn cell_with_attrs(text: &str, attrs: &str) -> String {
     format!(
-        r#"<a:tc><a:txBody><a:bodyPr/><a:lstStyle/><a:p><a:r><a:t>{text}</a:t></a:r></a:p></a:txBody><a:tcPr {props}/></a:tc>"#
+        r#"<a:tc {attrs}><a:txBody><a:bodyPr/><a:lstStyle/><a:p><a:r><a:t>{text}</a:t></a:r></a:p></a:txBody><a:tcPr/></a:tc>"#
     )
 }
 
@@ -48,8 +48,8 @@ fn cell_with_paragraphs(paragraphs: &[&str]) -> String {
     format!(r#"<a:tc><a:txBody><a:bodyPr/><a:lstStyle/>{paragraphs}</a:txBody><a:tcPr/></a:tc>"#)
 }
 
-fn empty_cell(props: &str) -> String {
-    format!(r#"<a:tc><a:txBody><a:bodyPr/><a:lstStyle/><a:p/></a:txBody><a:tcPr {props}/></a:tc>"#)
+fn empty_cell(attrs: &str) -> String {
+    format!(r#"<a:tc {attrs}><a:txBody><a:bodyPr/><a:lstStyle/><a:p/></a:txBody><a:tcPr/></a:tc>"#)
 }
 
 fn row(cells: &[String]) -> String {
@@ -135,7 +135,7 @@ fn a_slide_table_that_is_not_a_grid_of_data_keeps_its_words() {
         let text = parse_slide(&table(
             first_row,
             &[row(&[
-                cell_with_props("Notice for the board", r#"gridSpan="2""#),
+                cell_with_attrs("Notice for the board", r#"gridSpan="2""#),
                 empty_cell(r#"hMerge="1""#),
             ])],
         ));
@@ -186,43 +186,60 @@ fn a_title_and_text_around_a_slide_table_keep_their_lines() {
 
 #[test]
 fn grid_span_and_horizontal_merge_keep_the_grid_shape() {
-    let text = parse_slide(&table(
+    let body = table(
         true,
         &[
             row(&[
-                cell_with_props("Summary", r#"gridSpan="2""#),
-                cell_with_props("COVERED", r#"hMerge="1""#),
+                cell_with_attrs("Summary", r#"gridSpan="2""#),
+                cell_with_attrs("HORIZONTAL-COVERED", r#"hMerge="1""#),
                 cell("Total"),
             ]),
             row(&[cell("Q1"), cell("1,200"), cell("1,350")]),
         ],
-    ));
-
-    assert!(
-        text.contains("|  | Summary | Total |\n| --- | --- | --- |\n| Q1 | 1,200 | 1,350 |"),
-        "{text}"
     );
-    assert!(!text.contains("COVERED"), "{text}");
+
+    // Both XML event forms must leave the merge attributes read from a:tc intact.
+    for properties in ["<a:tcPr/>", "<a:tcPr></a:tcPr>"] {
+        let text = parse_slide(&body.replace("<a:tcPr/>", properties));
+        assert!(!text.contains("HORIZONTAL-COVERED"), "{properties}: {text}");
+        assert!(
+            text.contains("|  | Summary | Total |\n| --- | --- | --- |\n| Q1 | 1,200 | 1,350 |"),
+            "{properties}: {text}"
+        );
+    }
 }
 
 #[test]
 fn a_vertical_merge_continuation_stays_an_empty_cell() {
-    let text = parse_slide(&table(
+    let body = table(
         true,
         &[
             row(&[cell("Quarter"), cell("Revenue"), cell("YoY")]),
-            row(&[cell("Q1"), cell("1,200"), cell("+12%")]),
-            row(&[cell("Q2"), cell("1,350"), empty_cell(r#"vMerge="1""#)]),
+            row(&[
+                cell("Q1"),
+                cell("1,200"),
+                cell_with_attrs("+12%", r#"rowSpan="2""#),
+            ]),
+            row(&[
+                cell("Q2"),
+                cell("1,350"),
+                cell_with_attrs("VERTICAL-COVERED", r#"vMerge="1""#),
+            ]),
             row(&[cell("Q3"), cell("1,410"), cell("+7%")]),
         ],
-    ));
-
-    assert!(
-        text.contains(
-            "| Quarter | Revenue | YoY |\n| --- | --- | --- |\n| Q1 | 1,200 | +12% |\n| Q2 | 1,350 |  |\n| Q3 | 1,410 | +7% |"
-        ),
-        "{text}"
     );
+
+    // An already empty continuation would pass even if vMerge were ignored.
+    for properties in ["<a:tcPr/>", "<a:tcPr></a:tcPr>"] {
+        let text = parse_slide(&body.replace("<a:tcPr/>", properties));
+        assert!(!text.contains("VERTICAL-COVERED"), "{properties}: {text}");
+        assert!(
+            text.contains(
+                "| Quarter | Revenue | YoY |\n| --- | --- | --- |\n| Q1 | 1,200 | +12% |\n| Q2 | 1,350 |  |\n| Q3 | 1,410 | +7% |"
+            ),
+            "{properties}: {text}"
+        );
+    }
 }
 
 #[test]

crates/utopia-llm/src/lib.rs

@@ -114,19 +114,19 @@ pub fn rate_limited(err: &anyhow::Error) -> Option<&RateLimited> {
 /// 实测一篇 32 块的文档在一小时里两类都撞上：502 四次、连接没送到两次。
 pub fn transient(err: &anyhow::Error) -> Option<(&'static str, Option<Duration>)> {
     if let Some(hit) = rate_limited(err) {
-        return Some(("端点限流", hit.retry_after));
+        return Some(("LLM endpoint is rate limiting", hit.retry_after));
     }
     if let Some(hit) = unavailable(err) {
-        return Some(("端点不可用", hit.retry_after));
+        return Some(("LLM endpoint is unavailable", hit.retry_after));
     }
     if err.chain().any(|e| e.is::<Interrupted>()) {
-        return Some(("流断在半路", None));
+        return Some(("LLM response stream was interrupted", None));
     }
     let sending = err
         .chain()
         .find_map(|e| e.downcast_ref::<Unreachable>())
         .filter(|u| !u.0.is_timeout());
-    sending.map(|_| ("请求没送到", None))
+    sending.map(|_| ("LLM request could not be sent", None))
 }
 
 /// 端点开口了又半路没了：流断在一句话中间，既没有 `[DONE]` 也没有 `finish_reason`。
@@ -2122,7 +2122,7 @@ data: {\"choices\":[{\"delta\":{\"content\":\"tail\"},\"finish_reason\":\"stop\"
         server.await.unwrap();
         assert_eq!(
             crate::transient(&err).map(|(w, _)| w),
-            Some("流断在半路"),
+            Some("LLM response stream was interrupted"),
             "{err:#}"
         );
 
@@ -2422,7 +2422,7 @@ data: [DONE]
         assert!(crate::is_unreachable(&err), "{err:#}");
         assert_eq!(
             crate::transient(&err).map(|(w, _)| w),
-            Some("请求没送到"),
+            Some("LLM request could not be sent"),
             "{err:#}"
         );