diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12d3cc2c5..05b2c6d91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,6 +88,12 @@ jobs: failed=$(grep -o '[0-9]* failed' store-tests.log | awk '{s+=$1} END {print s+0}') echo "utopia-store against Postgres: **${passed} passed**, ${failed} failed — a missing database fails this job instead of skipping" >> "$GITHUB_STEP_SUMMARY" + - name: MCP structured reads against Postgres + run: cargo test -p utopia-server api::mcp::tests + env: + UTOPIA_DATABASE_URL: postgres://utopia:utopia@localhost:5432/utopia + UTOPIA_TEST_REQUIRE_DB: "1" + web: runs-on: ubuntu-latest defaults: @@ -109,5 +115,7 @@ jobs: cache-dependency-path: web/pnpm-lock.yaml - name: Install run: pnpm install --frozen-lockfile + - name: Test + run: pnpm test - name: Build (含类型检查) run: pnpm build diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ac35e8816..c8cf95661 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,8 +72,14 @@ jobs: done # 故意不给 UTOPIA_JWT_SECRET：让这一跑覆盖自动生成那条路径。 # 下面的注册请求会签发并携带 token，密钥没生成好就过不了。 + # + # UTOPIA_MIGRATION_URL 显式传空串，而不是不传：compose 的 + # `${VAR:-}` 在变量没设时发出去的就是空串，而这条冒烟测试原本直接 + # docker run、从不传它，于是 #343 全绿发布、用户照 README 起就退出 + # （"relative URL without a base"）。空串必须等同于未设，在这里钉住。 docker run -d --name app --network smoke -p 1516:1516 \ -e UTOPIA_DATABASE_URL=postgres://utopia:utopia@pg:5432/utopia \ + -e UTOPIA_MIGRATION_URL= \ utopia:smoke # 必须打 /api/v1/health 并核对 JSON：健康检查挂在 nest("/api/v1") 里， diff --git a/Cargo.lock b/Cargo.lock index cfb8cb373..5f243ba7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5277,6 +5277,7 @@ dependencies = [ "anyhow", "chrono", "cron", + "futures-util", "pgvector", "serde", "serde_json", diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index be424d07f..92ba67961 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -142,6 +142,61 @@ pub struct Source { pub created_at: DateTime, } +impl Source { + /// 这个来源下的文档要不要进抽取。 + /// + /// **缺省是要。** 只有 config 里 `{"extract": false}` 明说了才不抽——schema + /// 文档就是这样（0035 决定 7）：它是给问数检索表结构的语料，不是事实的来源， + /// 进抽取的结果是每个列名变成一个实体。开关记在来源上而不是文档上，因为 + /// 「只检索、不学习」是这一整个来源的性质；也没有拿来源的名字当规则，那是 + /// 命名约定冒充类型保证（0009）。 + /// + /// 值不是布尔的按没写处理：一个手滑不该让一整个来源静默停抽。 + /// `documents::queue_extraction` 里的 SQL 判的是同一件事，改一处要改两处。 + pub fn extracts(&self) -> bool { + self.config + .get("extract") + .and_then(serde_json::Value::as_bool) + .unwrap_or(true) + } +} + +#[cfg(test)] +mod source_extracts_tests { + use super::Source; + + fn with(config: serde_json::Value) -> Source { + Source { + id: uuid::Uuid::nil(), + kb_id: uuid::Uuid::nil(), + kind: "folder".into(), + name: "x".into(), + config, + icon: None, + sync_interval_minutes: None, + sync_cron: None, + last_sync_at: None, + last_sync_status: "never".into(), + last_sync_error: None, + last_sync_added: 0, + ingest_token: None, + created_at: chrono::Utc::now(), + } + } + + #[test] + fn only_an_explicit_false_turns_extraction_off() { + // 老来源的 config 是 `{}`，watch_folder 的是 `{"path": …}`：都照旧抽取 + assert!(with(serde_json::json!({})).extracts()); + assert!(with(serde_json::json!({ "path": "/x" })).extracts()); + assert!(with(serde_json::json!({ "extract": true })).extracts()); + // 不是布尔的按没写处理，而不是按 false + assert!(with(serde_json::json!({ "extract": "no" })).extracts()); + assert!(with(serde_json::json!({ "extract": 0 })).extracts()); + assert!(!with(serde_json::json!({ "extract": false })).extracts()); + } +} + /// 来源配置里**用来鉴权**的那几个键。凭据只进不出：列表与创建 / 更新的响应都剔掉， /// 更新时客户端没传或传空串就保留库里的原值，审计里也不落。 /// @@ -453,6 +508,19 @@ pub struct RelationTypeView { pub usage: i64, } +/// 一条边上挂的一个属性值（0037）。`value` 与 `entity` 二选一： +/// 金额、比例、日期是字面值；「经 C 撮合」里的 C 是实体（这一格这一刀还不写，位置留着） +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct FactQualifier { + pub qualifier_type_id: Uuid, + pub key: String, + pub label: String, + /// 形状与 `facts.object_value` 一致：{"value": …, "unit": …} + pub value: Option, + pub entity_id: Option, + pub entity_name: Option, +} + /// 抽取未匹配统计（本体扩展建议的信号源）。 #[derive(Debug, Clone, Serialize, sqlx::FromRow)] pub struct OntologyMiss { @@ -550,6 +618,10 @@ pub struct RelationType { /// attribute 专用：text | number | date | bool pub datatype: Option, pub unit: Option, + /// **这条关系的边能带哪些属性**（0037）：指向 kind='attribute' 的行。 + /// `A invested B` 上的「金额」是边自己的属性，不是第二个宾语；金额的 + /// datatype / unit / 换算全复用属性定义，只是它的 domain 是一条关系而不是一个类 + pub qualifiers: Vec, } #[derive(Debug, Clone, Serialize, sqlx::FromRow)] @@ -600,6 +672,9 @@ pub struct GraphEdge { /// **与 `inferred` 不是一回事**，尽管两个词很近：那一位说的是「名字来自原文 /// 而不是本体」，这一位说的是「这条边根本不是谁说的，是引擎推的」 pub derived: bool, + /// 边上的属性（0037）：画布把金额写到边的标签上要靠它 + #[sqlx(skip)] + pub qualifiers: Vec, /// 推它出来的那条规则（`transitive` / `symmetric` / `inverse` / `sub_property`）； /// 断言的边为 None。 /// @@ -634,6 +709,10 @@ pub struct GraphEdge { #[derive(Debug, Clone, Serialize, sqlx::FromRow)] pub struct EntityFact { pub id: Uuid, + pub recorded_at: DateTime, + pub invalidated_at: Option>, + pub supersedes: Option, + pub document_ids: Vec, /// out = 该实体为主语；in = 为宾语 pub direction: String, /// 本体没认下这条关系时回落到原文说法；两者都拿不出时为 None（更早的历史数据长这样） @@ -645,8 +724,13 @@ pub struct EntityFact { pub temporal: Option, pub other_id: Option, pub other_name: Option, + /// 对端实体的类型标签；属性事实没有对端时为 None + pub other_type: Option, /// 字面值宾语（属性事实/问数映射）：{"value":…,"unit":…} 或 {"summary":…} pub object_value: Option, + /// 边上的属性（0037）。不在行里——`fact_qualifiers` 另一张表，加载后按事实 id 补 + #[sqlx(skip)] + pub qualifiers: Vec, pub valid_from: Option>, pub valid_to: Option>, /// 精度描述的是这条事实**有的那些日期**的粒度。两端都没有日期时为 None—— @@ -926,6 +1010,14 @@ pub struct KnowledgeBase { /// 上次推完的时间。**答的是「上次看过没有」，不是「上次改过没有」** pub last_inference_at: Option>, pub ontology_lang: String, + /// 探索从 schema 写的数据描述：一行是什么、键、单位、码值、时间轴、相似列。 + /// 只写 schema 说了的；每次探索重写 + pub data_description: Option, + /// 探索拿不准、需要库的主人答的问题（JSON 字符串数组） + pub data_questions: serde_json::Value, + /// 人写的约定：「测试单不算数」「有效订单是 2/3/4」这类 schema 里没有的规则。 + /// 探索不碰它——量过：宽表语料上问数没有约定 2/18，有 14/18（#520） + pub data_conventions: Option, pub created_at: DateTime, pub updated_at: DateTime, } @@ -1004,6 +1096,35 @@ pub struct MappingRevision { pub changed_at: DateTime, } +/// 一轮映射探索扫了什么、丢了什么、剩下什么（#503）。 +/// +/// **它回答的是覆盖率**：十一条提议对着一张八十列的宽表，与十一条刚好覆盖完 +/// 一个小库，从 `concept_mappings` 里看长得一模一样。分子是 `tables_covered`， +/// 分母是 `tables_scanned`，而 `schema_truncated` 说明覆盖不全是「没看见」 +/// 还是「看见了没提」。 +#[derive(Debug, Clone, Serialize, sqlx::FromRow)] +pub struct ExplorationRun { + pub id: Uuid, + pub started_at: DateTime, + pub finished_at: Option>, + pub sources: Vec, + pub tables_scanned: i32, + pub columns_scanned: i32, + /// schema 文本撞了上限：提示词里没有的表，模型没有机会提 + pub schema_truncated: bool, + /// 这一轮允许提几条（按表数放大） + pub cap: i32, + /// 模型回了几条 / 落库几条。两者之差是被丢掉的，明细在 `dropped` + pub returned: i32, + pub accepted: i32, + /// `{"source": {"n": 12, "example": "…"}, …}`，键见 + /// `utopia_store::exploration_runs::drop_reason` + pub dropped: serde_json::Value, + pub tables_covered: Vec, + /// 跑挂了的那一轮也留一行——失败与「跑了但什么都没提」不是一回事 + pub error: Option, +} + /// 语义层的一条映射：业务概念 → 数据资产定义（见 `docs/decisions/0011`）。 /// /// **字段是列，不是 JSON 里的键。** 从前它是一条 `mapped_to` 事实， @@ -1030,6 +1151,9 @@ pub struct ConceptMapping { /// **状态而不是置信度。** 从前借事实的 confidence 表达「提议 0.6 / 确认 1.0」， /// 那是把二值状态编码成浮点数，还顺带让它落进「低置信事实」那一档 pub status: String, + /// 人从零写的口径记写的人；探索提的为空（#562）。`decided_by` 分不出这件事—— + /// 探索提的经人确认之后同样有 decided_by + pub written_by: Option, } /// 一处公理违规，配好展示所需的三元组文本（见 `axiom_violations`）。 @@ -1097,6 +1221,13 @@ pub struct OntologyDefect { #[derive(Debug, Clone, Serialize, sqlx::FromRow)] pub struct DerivedFactView { pub id: Uuid, + pub predicate_id: Uuid, + pub object_value: Option, + pub rule_id: Option, + pub attribute_rule_id: Option, + pub invalidated_at: Option>, + pub valid_from_precision: Option, + pub valid_to_precision: Option, pub subject_id: Uuid, pub subject: String, /// 字面值结论（业务规则的归类与属性）没有实体宾语（0021） @@ -1139,14 +1270,19 @@ pub struct BlockedDerivation { pub premises: Vec, } -/// 证明的一步：一条断言前提，连同它的证据（0002 R2）。 +/// 证明的一步：一条前提，连同它的证据（0002 R2）。 /// -/// 前提一律是断言（`fact_derivations` 不记派生），所以证明是一条链而不是一棵树： -/// 派生 → 按 `seq` 排好的断言 → 每条断言的原句。叶子就是 chunk。 +/// 前提要么是断言——叶子就是它的原句（chunk）——要么是**另一条派生**（0030）， +/// 那一步的证据是它自己的前提，在 `premises` 里再往下一层。所以证明是一棵树， +/// 深度与推理同一条上限。 #[derive(Debug, Clone, Serialize)] pub struct ProofStep { pub seq: i32, + /// 断言时是 `facts.id`，派生时是 `derived_facts.id`——看 `derived` pub fact_id: Uuid, + /// 这一步自己是推出来的（0030）。**界面要分得出**：一条推出来的前提与 + /// 一条读来的前提在句子上长得一样，而它们能不能追到原文完全不同 + pub derived: bool, pub subject_id: Uuid, pub subject: String, pub predicate_id: Option, @@ -1161,6 +1297,9 @@ pub struct ProofStep { /// 这条前提后来被撤了。派生随之失效，但证明还要读得出「当时靠的是什么」 pub retracted: bool, pub evidence: Vec, + /// 这一步自己的前提，按 `seq`（0030）。断言那一步是空的——它的叶子是 + /// `evidence` 里的原句，不必再往下问 + pub premises: Vec, } /// 一条派生事实的完整证明：它本身，加上按顺序展开到原句的前提。 diff --git a/crates/utopia-extract/src/governor.rs b/crates/utopia-extract/src/governor.rs index 26c8c7377..475e38453 100644 --- a/crates/utopia-extract/src/governor.rs +++ b/crates/utopia-extract/src/governor.rs @@ -1,7 +1,8 @@ //! 治理的第二层（0025 第二刀）：攒批判不定的对，逐条带工具再看一遍。 //! //! 模型能看的东西：一侧的全部事实、一侧的原文片段、台账里人对某个名字的决定、 -//! 库里名字相近的其他实体。结束只有两种：`decide`（same / different + 置信度 + +//! 库里名字相近的其他实体、合并会牵动什么（0028：一致性检查会开出的矛盾、靠着 +//! 一边的派生、点过名的回答、两边的类型是不是一个大类）。结束只有两种：`decide`（same / different + 置信度 + //! 一句理由）或 `defer`（留给人一个具体的问题）。工具定义、提示词与回合的解析 //! 在这里；跑循环、查库的在 server 的 governance 任务里——这里不碰库也不碰模型。 @@ -23,7 +24,7 @@ pub struct EarlierLook<'a> { /// 模型在一个回合里要的事 #[derive(Debug, PartialEq)] pub enum Step { - /// 查一样东西：facts / quotes / ledger / namesakes + /// 查一样东西：facts / quotes / ledger / namesakes / consequences Lookup { tool: String, args: Value, @@ -69,6 +70,10 @@ pub fn tools() -> Value { "name": "namesakes", "description": "Other entities in this base whose name contains the query, with their type and how many facts they carry.", "parameters": query }}, + { "type": "function", "function": { + "name": "consequences", + "description": "What merging A and B would touch: relations that allow one value where the two sides hold different ones, derived facts resting on either side, chat answers that named either side, and whether the two types belong to one family. A merge that would touch any of these is held for a person whatever your confidence.", + "parameters": { "type": "object", "properties": {} } }}, { "type": "function", "function": { "name": "decide", "description": "Give the verdict for this pair.", @@ -105,7 +110,11 @@ pub fn messages(pair: &AdjudicationPair, earlier: &EarlierLook) -> Vec { \n\ You may look things up before answering, at most {MAX_STEPS} lookups: the facts of a \ side, the source passages that mention a side, what people in this base decided about \ - a name, and other entities with a similar name. People's earlier decisions are how the \ + a name, other entities with a similar name, and what merging the two would touch (a \ + relation that allows one value where the sides hold different ones is a contradiction, \ + and a merge that would touch anything outside the graph is held for a person whatever \ + your confidence: prefer to defer with the question that would settle it). People's \ + earlier decisions are how the \ owners of this base want such cases judged; follow them unless the facts of this pair \ clearly differ, and never let one override a contradiction in the facts. Look only for \ what would change your answer.\n\ @@ -174,6 +183,10 @@ pub fn read_step(name: &str, arguments: &str) -> Step { }, _ => Step::Unknown(format!("{name} needs a query")), }, + "consequences" => Step::Lookup { + tool: name.into(), + args: json!({}), + }, "decide" => { let same = match args["verdict"].as_str() { Some("same") => true, @@ -299,7 +312,15 @@ mod tests { .collect(); assert_eq!( names, - ["facts", "quotes", "ledger", "namesakes", "decide", "defer"] + [ + "facts", + "quotes", + "ledger", + "namesakes", + "consequences", + "decide", + "defer" + ] ); } } diff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rs index 0e7358bc7..337666ad7 100644 --- a/crates/utopia-extract/src/lib.rs +++ b/crates/utopia-extract/src/lib.rs @@ -59,6 +59,10 @@ pub struct ExtractedFact { /// 属性事实的字面值（谓词是 attribute 时） #[serde(default)] pub value: Option, + /// **边上的属性**（0037）：`{"amount": "$5 billion", "stake": "20%"}`。 + /// 只对关系事实有意义，key 必须是清单里这条关系声明过的；值照原文写，换算在服务端 + #[serde(default)] + pub qualifiers: Option>, #[serde(default)] pub valid_from: Option, #[serde(default)] @@ -67,6 +71,13 @@ pub struct ExtractedFact { pub confidence: Option, #[serde(default)] pub quote: Option, + /// 引文里逐字点名主语的那几个字（#582）。模型抄，不判断；落库时机器核对它 + /// 是不是 `subject` 那个名字——"Former OpenAI personnel" 不是 OpenAI + #[serde(default)] + pub subject_span: Option, + /// 同上，宾语那一侧 + #[serde(default)] + pub object_span: Option, } /// 提示词里的一条关系。 @@ -83,6 +94,11 @@ pub struct PromptRelation { /// **一律用 key**：模型要输出的就是 key，中文库里 person 的 label 是"人物"， /// 写进签名等于教它输出一个不存在的类型（docs/decisions/0004） pub signature: String, + /// 时间语义（`relation_types.temporal`）：`state` / `event` / `eternal`（0031）。 + /// 只有 event 与 eternal 会在清单里带标记——状态是默认，写出来只多花 token + pub temporal: String, + /// 这条关系的边能带的属性，已排好版：`amount: number $`（0037）。空 = 不带 + pub qualifiers: Vec, } /// Response-scoped reference to a persistent entity; database UUIDs must never enter prompts. @@ -140,15 +156,38 @@ pub fn build_messages( } else { String::new() }; + // 事件与恒常带方括号标记；状态是默认，不标（0031） + let mark = temporal_mark(&r.temporal) + .map(|m| format!(" [{m}]")) + .unwrap_or_default(); + // 边上能带的属性跟在标记后面：`{amount: number $, stake: number %}` + let mark = if r.qualifiers.is_empty() { + mark + } else { + format!("{mark} {{{}}}", r.qualifiers.join(", ")) + }; match (paren.is_empty(), d.is_empty()) { - (false, false) => format!("- {} ({paren}): {d}", r.key), - (false, true) => format!("- {} ({paren})", r.key), - (true, false) => format!("- {}: {d}", r.key), - (true, true) => format!("- {}", r.key), + (false, false) => format!("- {} ({paren}){mark}: {d}", r.key), + (false, true) => format!("- {} ({paren}){mark}", r.key), + (true, false) => format!("- {}{mark}: {d}", r.key), + (true, true) => format!("- {}{mark}", r.key), } }) .collect::>() .join("\n"); + // 标记也只在真有事件或恒常关系时解释一次；全是状态的库，提示词一字不变。 + // 说的是**写什么**而不是「它是什么」：事件的那一刻进 valid_from、valid_to 留空 + // ——不然模型照状态的样子填一个起点，账本就把一次收购读成从那天起一直持续 + let temporal_note = if relations + .iter() + .any(|r| temporal_mark(&r.temporal).is_some()) + { + "\n 3b. A relation marked [event] happens at one moment: put the date it happened in \ + valid_from and leave valid_to null — it has no span and does not end. A relation \ + marked [eternal] holds regardless of time: leave both dates null." + } else { + "" + }; // 记号只在真有签名时解释一次；没有签名的库，提示词一字不变。 // 说明用英文——提示词的**指令语言**是英文，只有 description 跟语料走 // @@ -206,8 +245,7 @@ pub fn build_messages( let attr_rules = if attributes.is_empty() { String::new() } else { - "\n10. Attribute facts carry \"value\" (no \"object\"): number = plain number without \ - thousands separators or unit symbols; date = \"YYYY[-MM[-DD]]\" (a zoned clock time only when the text gives one); bool = true/false; \ + "\n10. Attribute facts carry \"value\" (no \"object\"): number = the figure **as the text writes it, magnitude and currency included** \n (\"86亿元\", \"$5 billion\", \"4,300 人\") — never reduce it to a bare number, the server converts; date = \"YYYY[-MM[-DD]]\" (a zoned clock time only when the text gives one); bool = true/false; \ text = a short string. Only attach an attribute to a subject of its listed class. \ valid_from = when this value took effect, if the text says so." .to_string() @@ -223,7 +261,7 @@ pub fn build_messages( \n\ Output format:\n\ {{\"entities\":[{{\"local_id\":\"e1\",\"name\":\"entity name\",\"type\":\"type key\",\"specific_type\":\"what you would call it\"}}],\n\ - \"facts\":[{{\"subject\":\"subject entity name\",\"subject_ref\":\"e1\",\"predicate\":\"relation key\",\"object\":\"object entity name\",\"object_ref\":\"e2\",\n\ + \"facts\":[{{\"subject\":\"subject entity name\",\"subject_ref\":\"e1\",\"subject_span\":\"the words in quote that name the subject\",\"predicate\":\"relation key\",\"object\":\"object entity name\",\"object_ref\":\"e2\",\"object_span\":\"the words in quote that name the object\",\n\ \"valid_from\":\"2023-01\",\"valid_to\":null,\"confidence\":0.9,\"quote\":\"verbatim supporting quote\"}}]}}\n\ \n\ Rules:\n\ @@ -253,7 +291,7 @@ pub fn build_messages( company\", \"no longer available\", \"until recently\". Use null only for something \ still going on. These are not interchangeable: null asserts it still holds, and \ writing null for a relation the text says is over makes us claim the opposite of \ - the source.\n\ + the source.{temporal_note}\n\ 4. {time_ctx}\n\ 5. quote must be a contiguous excerpt from the source text; every fact needs one.\n\ 6. confidence in 0~1: 0.9 explicitly stated, 0.7 inferred, 0.5 uncertain.\n\ @@ -261,6 +299,32 @@ pub fn build_messages( 8. If no listed relation fits, do not force the nearest one — write the predicate the \ text itself uses, in snake_case (e.g. \"available_on\", \"runs_on\"). A relation \ named after the text is worth more than a listed one that says something false.\n\ + 8a. The same holds for a literal the text states outright — an amount, a share count, \ + a percentage, a capacity, a date, a job title, a ticker. Write it as a fact with \ + \"value\" and no \"object\": {{\"subject\":\"NVIDIA\",\"subject_ref\":\"e1\",\ + \"predicate\":\"purchase_price\",\"value\":\"$11.9 billion\",\"confidence\":0.9,\ + \"quote\":\"...\"}}. Name the predicate after the text when no listed attribute \ + fits — \"purchase_price\", \"job_title\", \"generation_capacity\", \"record_date\". \ + Attach it to the entity the text attaches it to, and keep the literal as written, \ + units and all. **A stated figure left out is the loss that costs most**: the reader \ + came for those numbers, and no later step can recover one that was never written \ + down.\n\ + 8c. A listed relation followed by {{…}} can carry those **qualifiers on the edge**: when the same sentence gives both the other entity and a figure for it — an amount, a stake, a price, a share count — write the relation with its \"object\" and put the figure in \"qualifiers\" keyed exactly as listed, **as written in the text, currency and all** (\"€30 million\", \"15亿元人民币\", never a bare number): {{\"subject\":\"Vega Capital\",\"predicate\":\"invested_in\",\"object\":\"Northwind\", \"qualifiers\":{{\"amount\":\"$5 billion\"}},…}}. Never invent a key that is not listed for that relation, and never drop the figure to keep the edge — a relation without its amount is half the sentence. A relation you name after the text (rule 8) carries its figure the same way — keyed by the listed attribute that fits it, or by the plainest word for it (\"amount\", \"stake\", \"price\") when none does. + 8b. A **listed** relation also takes \"value\" when what the text gives is a \ + string rather than another entity — a job title, a designation, a ticker, a \ + model number. Never invent an entity for a string. And when the text introduces \ + someone by their role — \"X, founder and CEO of Y\", \"Z, co-CEO of W\" — write \ + both facts: the tie to the organization, and the role itself as a value on the \ + person. The tie alone says they are connected; the role is what the sentence \ + was actually telling you.\n\ + 8c. A list of named parties is a list of facts — one per name. \"partners \ + including A, B, C and D\" is four facts, not one; \"advisors A and B\" is two. \ + Do not collapse an enumeration into a summary or into its first member. \ + The same applies to the entities: each named party is its own entity.\n\ + 8d. subject_span and object_span are the exact words in quote that name each side. \ + Copy them; never paraphrase. When the words that do the thing are a description \ + rather than a name — \"former X employees\", \"companies using X\" — the span \ + is that description, whatever you wrote in subject.\n\ 9. The same holds for entity types: if none of the listed types fits, write the type \ the text implies, in snake_case (e.g. \"model\", \"technology\"). Do not fall back \ to a broad listed type such as \"thing\" or \"creative_work\" merely because \ @@ -291,6 +355,16 @@ pub fn build_messages( ] } +/// 清单里给关系带的标记：事件 `[event]`、恒常 `[eternal]`；状态不标。 +/// 认不出的值当状态——数据库的 CHECK 只放这三个进来，这里不再报错 +fn temporal_mark(temporal: &str) -> Option<&'static str> { + match temporal { + "event" => Some("event"), + "eternal" => Some("eternal"), + _ => None, + } +} + /// 已在本文档中出现过的实体，放进提示词的字符预算。 /// /// 超出就截断（保留先出现的）。中文商业文本先出全称、主角先出场，所以 @@ -608,7 +682,7 @@ pub fn build_adjudication_messages(pairs: &[AdjudicationPair]) -> Vec anyhow::Result> Ok(reply.verdicts) } +/// 一个**整体就是一个量**的字符串 → (数值, 单位)。 +/// +/// 判据从严：可选货币符号 + 数字 + 可选量级词 + 可选百分号，此外**一个词都不许有**。 +/// 尾巴上还挂着实词的，含义就不再只是那个数： +/// +/// ```text +/// "$5 billion" → (5e9, Some("$")) +/// "52%" → (52.0, Some("%")) +/// "3.5 million" → (3.5e6, None) +/// "35,000" → (35000.0, None) +/// "900 million weekly active users" → None 后面还有实词 +/// "2025 Atlantic hurricane season" → None 那是一场赛事，不是 2025 +/// "8GW data center" → None +/// "3M" → None 那是一家公司 +/// ``` +/// +/// **量级词只认全写**。单字母后缀（`3M`、`5k`、`2B`）看着省事，代价是把 3M、 +/// K2、B1 这些名字读成数字——一个真实体被读成量值，事实的形状就错了， +/// 而错的那一头是不可逆的：节点没建，名字也没留下。 +/// +/// **单位照抄符号，不猜币种。** `$` 可能是美元、加元、澳元，`¥` 可能是日元或 +/// 人民币。猜出来的 "USD" 是一条没人负责的断言，而原文写的 `$` 是事实。 +pub fn parse_quantity(s: &str) -> Option<(f64, Option)> { + scan_quantity(s, true) +} + +/// 开头是一个量、后面还挂着词的 → 那个量。`"1,250 people"` → (1250, "people")。 +/// +/// **这是给已经知道要什么的地方用的**，与 `parse_quantity` 的严不是一回事。 +/// `parse_quantity` 要判「这串字是不是一个东西」，判错就把一个真实体吃掉， +/// 所以尾巴上有实词一律不认。而这里的调用方手上已经有一条声明了 +/// `datatype = number` 的属性——问的不再是「是不是数」，是「那个数是多少」， +/// 判错的代价只是一个值不对，量级差着好几档。 +pub fn parse_leading_quantity(s: &str) -> Option<(f64, Option)> { + scan_quantity(s, false) +} + +/// 货币：符号、ISO 码、中英文单词，统一成符号。**只认这张表**，认不出的不猜。 +pub fn currency_unit(tok: &str) -> Option<&'static str> { + Some( + match tok.trim_matches(|c: char| c == ',' || c == '.' || c == ';') { + "$" | "USD" | "usd" | "US$" | "dollar" | "dollars" | "美元" => "$", + "€" | "EUR" | "eur" | "euro" | "euros" | "欧元" => "€", + "£" | "GBP" | "gbp" | "pound" | "pounds" | "英镑" => "£", + "¥" | "JPY" | "jpy" | "yen" | "日元" => "¥", + "CNY" | "cny" | "RMB" | "rmb" | "yuan" | "人民币" | "元" | "元人民币" | "人民币元" => { + "¥" + } + "HKD" | "hkd" | "HK$" | "港元" | "港币" => "HK$", + "₩" | "KRW" | "won" | "韩元" => "₩", + "₹" | "INR" | "rupee" | "rupees" | "卢比" => "₹", + _ => return None, + }, + ) +} + +/// 量级词：英文全写，中文千/万/亿。**不认单字母**（`3M` 是一家公司）。 +fn magnitude(tok: &str) -> Option { + Some(match tok { + "thousand" | "千" => 1e3, + "万" => 1e4, + "million" | "百万" => 1e6, + "千万" => 1e7, + "亿" => 1e8, + "billion" | "十亿" => 1e9, + "trillion" | "万亿" => 1e12, + _ => return None, + }) +} + +/// 把 `2亿美元`、`15亿元人民币`、`€30 million`、`30 million euros`、`USD 30m`（不认 m） +/// 这类写法拆成 [前缀货币] 数字 [量级] [后缀货币/单位] [其余]。 +/// `strict` = 整体必须就是一个量：其余部分非空就不认。 +fn scan_quantity(s: &str, strict: bool) -> Option<(f64, Option)> { + let s = s.trim(); + if s.is_empty() { + return None; + } + let (body, percent) = match s.strip_suffix('%') { + Some(b) => (b.trim_end(), true), + None => (s, false), + }; + // 1. 前缀货币：符号紧贴，或 ISO 码/单词后跟空格 + let mut rest = body; + let mut currency: Option<&'static str> = None; + if let Some(c) = rest.chars().next() { + if let Some(u) = currency_unit(&c.to_string()) { + currency = Some(u); + rest = rest[c.len_utf8()..].trim_start(); + } + } + if currency.is_none() { + if let Some((head, tail)) = rest.split_once(char::is_whitespace) { + if let Some(u) = currency_unit(head) { + currency = Some(u); + rest = tail.trim_start(); + } + } + } + // 2. 数字：前导的 [-+0-9.,_] + let num_end = rest + .char_indices() + .find(|(_, c)| !matches!(c, '0'..='9' | '.' | ',' | '_' | '-' | '+')) + .map(|(i, _)| i) + .unwrap_or(rest.len()); + let (num, after) = rest.split_at(num_end); + let cleaned: String = num.chars().filter(|c| !matches!(c, ',' | '_')).collect(); + let mut n: f64 = cleaned.parse().ok()?; + // 3. 数字后面：紧贴或空格隔开的量级词、货币词，逐个吃；吃不动的就是「其余」 + let mut tail = after.trim_start(); + let mut unit: Option = None; + let mut ate_magnitude = false; + loop { + if tail.is_empty() { + break; + } + // 取下一个记号：中文按字（量级/货币词最长两三个字），其它按空白分词 + let (tok, next) = next_token(tail); + if !ate_magnitude { + if let Some(m) = magnitude(tok) { + n *= m; + ate_magnitude = true; + tail = next.trim_start(); + continue; + } + } + if unit.is_none() && currency.is_none() { + if let Some(u) = currency_unit(tok) { + unit = Some(u.to_string()); + tail = next.trim_start(); + continue; + } + } + break; + } + if percent && (currency.is_some() || unit.is_some()) { + return None; + } + if !n.is_finite() { + return None; + } + // 9.2 × 1e8 在二进制浮点里是 919999999.9999999；乘过量级词的数本来就是整数，收回去 + if ate_magnitude && (n - n.round()).abs() < 1e-6 * n.abs().max(1.0) { + n = n.round(); + } + let unit = if percent { + Some("%".to_string()) + } else { + currency.map(str::to_string).or(unit) + }; + if strict { + return tail.is_empty().then_some((n, unit)); + } + // 宽松：其余部分的第一个词当单位（`1,250 people` → people），没有货币时才用 + if unit.is_none() && !tail.is_empty() { + let (tok, _) = next_token(tail); + return Some((n, Some(tok.to_string()))); + } + Some((n, unit)) +} + +/// 下一个记号：ASCII 按空白切；CJK 试最长三字、两字、一字里能认出的量级/货币词， +/// 都认不出就取到下一个空白为止 +fn next_token(s: &str) -> (&str, &str) { + let first = s.chars().next().unwrap_or(' '); + if first.is_ascii() { + let end = s.find(char::is_whitespace).unwrap_or(s.len()); + return (&s[..end], &s[end..]); + } + let idx: Vec = s + .char_indices() + .map(|(i, _)| i) + .chain(std::iter::once(s.len())) + .collect(); + for len in [4usize, 3, 2, 1] { + if idx.len() > len { + let cand = &s[..idx[len]]; + if magnitude(cand).is_some() || currency_unit(cand).is_some() { + return (cand, &s[idx[len]..]); + } + } + } + let end = s.find(char::is_whitespace).unwrap_or(s.len()); + (&s[..end], &s[end..]) +} + /// 属性值按 datatype 归一。失败返回 None——宁缺勿脏，调用方跳过并记日志。 /// number 容忍千分位/空格；date 要求 YYYY[-MM[-DD]] 且保留原精度；bool 宽容 yes/no。 pub fn normalize_attr_value(datatype: &str, raw: &serde_json::Value) -> Option { @@ -689,6 +949,14 @@ pub fn normalize_attr_value(datatype: &str, raw: &serde_json::Value) -> Option() .ok() + // 清洗解不动的再当量解：`$5 billion`、`52%` 这些整体就是数， + // 只是带着符号与量级词。单位不在这里落笔——它随事实走 + // （见 `parse_quantity`），这一档只负责把值变成可比的数 + // 清洗解不动的再当量解。**这一档已经声明了 datatype = number**， + // 问的不是「是不是数」而是「那个数是多少」，所以用宽的那套： + // `$5 billion` → 5e9，`1,250 people` → 1250， + // `42% from customers in Europe` → 42 + .or_else(|| parse_leading_quantity(s).map(|(n, _)| n)) .filter(|f| f.is_finite()) .and_then(serde_json::Number::from_f64) .map(serde_json::Value::Number) @@ -791,9 +1059,47 @@ mod prompt_shape_tests { label: key.replace('_', " "), description: description.into(), signature: signature.into(), + temporal: "state".into(), + qualifiers: vec![], } } + fn timed(key: &str, description: &str, temporal: &str) -> PromptRelation { + PromptRelation { + temporal: temporal.into(), + ..rel(key, description, "") + } + } + + /// 事件与恒常在清单里带标记，说明只出现一次（0031） + #[test] + fn an_event_and_an_eternal_relation_are_marked() { + let rels = vec![ + rel("works_at", "受雇于某个组织。", "person → organization"), + timed("acquired", "One company buys another.", "event"), + timed("capital_of", "", "eternal"), + ]; + let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text"); + let s = &msgs[0].content; + assert!(s.contains("- works_at (person → organization): 受雇于某个组织。")); + assert!(s.contains("- acquired [event]: One company buys another.")); + // 没有描述时括号里是 label，标记跟在括号后面 + assert!(s.contains("- capital_of (capital of) [eternal]")); + assert!(s.contains("A relation marked [event] happens at one moment")); + assert!(s.contains("leave valid_to null")); + } + + /// **全是状态的库，提示词一字不变**：不标、不解释 + #[test] + fn a_base_of_states_pays_nothing_for_the_marks() { + let rels = vec![rel("works_at", "d", "")]; + let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text"); + let s = &msgs[0].content; + assert!(!s.contains("[event]")); + assert!(!s.contains("[eternal]")); + assert!(!s.contains("happens at one moment")); + } + /// 签名进括号，而且**一律是 key**：中文库的 label 是"人物"， /// 写进提示词等于教模型输出一个不存在的类型。 #[test] @@ -923,6 +1229,194 @@ mod prompt_shape_tests { mod tests { use super::*; + /// 边上的属性（0037）：清单里跟在关系后面，回复里挂在事实上。 + #[test] + fn a_relation_lists_its_qualifiers_and_a_fact_carries_them() { + use serde_json::json; + let mut r = PromptRelation { + key: "invested_in".into(), + label: "invested in".into(), + description: "money into a company".into(), + signature: "organization → organization".into(), + temporal: "event".into(), + qualifiers: vec!["amount: number $".into(), "stake: number %".into()], + }; + let msgs = build_messages( + &[], + std::slice::from_ref(&r), + &[], + None, + "a.txt", + &[], + "text", + ); + let prompt = format!("{:?}", msgs); + // 签名、标记、属性清单三段顺序固定：`(签名) [event] {属性}` + assert!(prompt.contains( + "- invested_in (organization → organization) [event] {amount: number $, stake: number %}: money into a company" + ), "{prompt}"); + // 不带属性的关系不多一个花括号 + r.qualifiers.clear(); + let prompt = format!( + "{:?}", + build_messages( + &[], + std::slice::from_ref(&r), + &[], + None, + "a.txt", + &[], + "text" + ) + ); + assert!( + prompt.contains("- invested_in (organization → organization) [event]: money"), + "{prompt}" + ); + assert!(!prompt.contains("[event] {")); + + // 回复：qualifiers 挂在关系事实上；没写的是 None，旧回复不受影响 + let reply = r#"{"entities":[],"facts":[ + {"subject":"Vega","predicate":"invested_in","object":"Northwind", + "qualifiers":{"amount":"$5 billion"},"confidence":0.9}, + {"subject":"Vega","predicate":"invested_in","object":"Kestrel","confidence":0.9} + ]}"#; + let parsed = parse_response(reply).unwrap(); + assert_eq!(parsed.facts.len(), 2); + assert_eq!( + parsed.facts[0] + .qualifiers + .as_ref() + .and_then(|q| q.get("amount")), + Some(&json!("$5 billion")) + ); + assert!(parsed.facts[1].qualifiers.is_none()); + } + + #[test] + fn a_quantity_is_the_whole_string_or_nothing() { + // 整体就是一个量：符号、量级词、千分位都读得动 + assert_eq!(parse_quantity("$5 billion"), Some((5e9, Some("$".into())))); + assert_eq!( + parse_quantity("€1.5 million"), + Some((1.5e6, Some("€".into()))) + ); + assert_eq!(parse_quantity("52%"), Some((52.0, Some("%".into())))); + assert_eq!(parse_quantity("3.5 million"), Some((3.5e6, None))); + assert_eq!(parse_quantity("35,000"), Some((35000.0, None))); + assert_eq!(parse_quantity(" 42 "), Some((42.0, None))); + // 币种：符号、ISO 码、中英文单词，统一成符号；量级：英文全写与中文千万亿 + assert_eq!( + parse_quantity("EUR 30 million"), + Some((3e7, Some("€".into()))) + ); + assert_eq!( + parse_quantity("30 million euros"), + Some((3e7, Some("€".into()))) + ); + assert_eq!( + parse_quantity("USD 5 billion"), + Some((5e9, Some("$".into()))) + ); + assert_eq!(parse_quantity("2亿美元"), Some((2e8, Some("$".into())))); + assert_eq!( + parse_quantity("15亿元人民币"), + Some((1.5e9, Some("¥".into()))) + ); + assert_eq!(parse_quantity("3000万元"), Some((3e7, Some("¥".into())))); + assert_eq!(parse_quantity("1.5亿"), Some((1.5e8, None))); + // 乘过量级的数收成整数：9.2 亿不是 919999999.9999999 + assert_eq!( + parse_quantity("9.2亿元"), + Some((920000000.0, Some("¥".into()))) + ); + assert_eq!( + parse_quantity("$2.5 billion"), + Some((2500000000.0, Some("$".into()))) + ); + + // 尾巴上还有实词：含义不再只是那个数，宁可当实体也不当量 + assert_eq!(parse_quantity("900 million weekly active users"), None); + assert_eq!(parse_quantity("2025 Atlantic hurricane season"), None); + assert_eq!(parse_quantity("$10 billion investment"), None); + assert_eq!(parse_quantity("8GW data center"), None); + // 单字母后缀不认：3M 是一家公司，读成三百万就把一个真实体吃掉了 + assert_eq!(parse_quantity("3M"), None); + assert_eq!(parse_quantity("5k"), None); + // 两个记号撞一起，不是量 + assert_eq!(parse_quantity("$5%"), None); + assert_eq!(parse_quantity(""), None); + assert_eq!(parse_quantity("杭州"), None); + } + + #[test] + fn a_declared_number_reads_past_the_unit() { + // 属性已经声明了 datatype = number，问的是「那个数是多少」。 + // 卡住过的两条都在这里 + assert_eq!( + parse_leading_quantity("1,250 people"), + Some((1250.0, Some("people".into()))) + ); + assert_eq!( + parse_leading_quantity("42% from customers in Europe"), + Some((42.0, Some("%".into()))) + ); + assert_eq!( + parse_leading_quantity("3,400 people worldwide"), + Some((3400.0, Some("people".into()))) + ); + assert_eq!( + parse_leading_quantity("900 million weekly active users"), + Some((9e8, Some("weekly".into()))) + ); + // 整体就是量的仍走严的那套：单位是 `$`，不是 `billion` + assert_eq!( + parse_leading_quantity("$5 billion"), + Some((5e9, Some("$".into()))) + ); + // 开头不是数就还是不认 + // 币种在尾巴上也认；认不出的词才落到「单位是第一个词」 + assert_eq!( + parse_leading_quantity("30 million euros in cash"), + Some((3e7, Some("€".into()))) + ); + assert_eq!( + parse_leading_quantity("15亿元人民币的投资"), + Some((1.5e9, Some("¥".into()))) + ); + assert_eq!( + parse_leading_quantity("30 million francs"), + Some((3e7, Some("francs".into()))) + ); + assert_eq!(parse_leading_quantity("about ten"), None); + assert_eq!(parse_leading_quantity(""), None); + + // **严的那套一点没松**：它要判「是不是一个东西」，判错会吃掉真实体 + assert_eq!(parse_quantity("1,250 people"), None); + assert_eq!(parse_quantity("2025 Atlantic hurricane season"), None); + } + + #[test] + fn a_number_attribute_takes_a_written_quantity() { + use serde_json::json; + // 采纳属性时按 datatype 换算，量也要换得动——否则 `$5 billion` + // 会一路「换不动」，事实永远拿不到谓词 + assert_eq!( + normalize_attr_value("number", &json!("$5 billion")), + Some(json!(5e9)) + ); + assert_eq!( + normalize_attr_value("number", &json!("52%")), + Some(json!(52.0)) + ); + // 原来就认的两种写法不受影响 + assert_eq!( + normalize_attr_value("number", &json!("35,000")), + Some(json!(35000.0)) + ); + assert_eq!(normalize_attr_value("number", &json!("about ten")), None); + } + #[test] fn parse_time_precisions() { assert_eq!(parse_time("2024").unwrap().1, "year"); @@ -1041,6 +1535,26 @@ mod tests { assert!(parse_response(r#"{"facts": [{"subject": "a"#).is_err()); } + /// 片段字段可有可无：老模型输出没有它们，照常解析 + #[test] + fn spans_parse_and_default_to_none() { + let with = parse_response( + r#"{"entities":[],"facts":[{"subject":"OpenAI","predicate":"founded","object":"Anthropic","subject_span":"Former OpenAI personnel","object_span":"Anthropic"}]}"#, + ) + .unwrap(); + assert_eq!( + with.facts[0].subject_span.as_deref(), + Some("Former OpenAI personnel") + ); + assert_eq!(with.facts[0].object_span.as_deref(), Some("Anthropic")); + let without = parse_response( + r#"{"entities":[],"facts":[{"subject":"OpenAI","predicate":"founded","object":"Anthropic"}]}"#, + ) + .unwrap(); + assert!(without.facts[0].subject_span.is_none()); + assert!(without.facts[0].object_span.is_none()); + } + #[test] fn parse_response_with_fence() { let raw = "好的，结果如下：\n```json\n{\"entities\":[{\"name\":\"张三\",\"type\":\"person\"}],\"facts\":[]}\n```"; @@ -1108,6 +1622,8 @@ mod tests { let system = &msgs[0].content; assert!(system.contains("\"local_id\":\"e1\"")); assert!(system.contains("\"subject_ref\":\"e1\"")); + // #578：跟 X 有关的一群人不是 X + assert!(system.contains("subject_span and object_span are the exact words")); assert!(system.contains("unique within this response")); assert!(system.contains("Reuse the same local_id")); assert!(system.contains("permanent identity is proven")); diff --git a/crates/utopia-ingest/src/html.rs b/crates/utopia-ingest/src/html.rs index 9a004965c..2a9895f07 100644 --- a/crates/utopia-ingest/src/html.rs +++ b/crates/utopia-ingest/src/html.rs @@ -1,4 +1,73 @@ //! Canonical HTML conversion shared by file, URL and feed ingestion. +#[cfg(test)] +mod table_tests { + use super::{markdown_from_html, promote_first_row_headers}; + + #[test] + fn a_table_without_headers_still_keeps_its_rows() { + let html = "\ +
RevenueQ2 FY27Q1 FY27
total$96,221$81,615
"; + let md = markdown_from_html(html).expect("converts"); + // 行列关系还在：同一行的三格在同一行文本里 + let row = md + .lines() + .find(|l| l.contains("96,221")) + .expect("the numbers survive"); + assert!(row.contains("81,615"), "同一行的两个数应当在一行里: {md}"); + assert!(row.contains("total"), "行标签应当与它的数在一行: {md}"); + } + + #[test] + fn empty_spacer_columns_do_not_survive() { + let html = "\ +
Revenue$96,221
Margin75.0
"; + let md = markdown_from_html(html).expect("converts"); + let row = md + .lines() + .find(|l| l.contains("96,221")) + .expect("row survives"); + assert!(!row.contains("| |"), "空列应当被砍掉: {row}"); + assert!(row.contains("Revenue"), "有内容的列一个不能少: {row}"); + } + + #[test] + fn a_blank_header_row_gives_way_to_the_real_one() { + let md = super::prune_empty_table_columns( + "| | |\n| --- | --- |\n| a. Tench Coxe | |\n| shares For | 15,411 |", + ); + let first = md.lines().next().expect("a line"); + assert!(first.contains("Tench Coxe"), "表头该是真正的抬头: {md}"); + assert!(md.contains("15,411"), "行不能丢: {md}"); + } + + #[test] + fn a_column_with_any_content_is_kept() { + let md = super::prune_empty_table_columns("| a | | c |\n| --- | --- | --- |\n| | b | |"); + assert!(md.contains("| a | | c |") || md.contains("a"), "{md}"); + assert!(md.contains("b"), "有内容的列不能砍: {md}"); + } + + #[test] + fn a_table_that_already_has_headers_is_untouched() { + let html = "
a
1
"; + assert_eq!(promote_first_row_headers(html), html); + } + + #[test] + fn a_nested_table_does_not_promote_the_outer_row() { + // 内层先被处理；外层因此看得见 ，跳过——不猜哪一行属于谁 + let html = "
inner
"; + let out = promote_first_row_headers(html); + assert_eq!(out.to_ascii_lowercase().matches(" bool { && !has_substantive_prose(markdown) } +/// 没有 `` 的表格，把第一行的 `` 提成 ``。 +/// +/// **理由是一整类文档在解析这一步就把行列关系丢了。** htmd 的表格处理器要求 +/// 表里有显式表头（`` 或 ``），否则整张退回逐格摊平——一格一行。 +/// 而 SEC 的 XBRL 报表一个都没有：实测 NVIDIA 那份财报 11 张表、那份投票结果 +/// 8-K 21 张表，`` 计数都是 0。摊平之后模型看到的是一列孤立标签跟一列 +/// 孤立数字，只能按顺序猜哪个数配哪一列，于是抽出 `NVIDIA net_worth Net income` +/// 这种边，十位董事的四类票数也全部退化成分不出类别的裸数字。 +/// +/// 第一行提成表头是这类表格的实际语义（"Q2 FY27 | Q1 FY27 | Q2 FY26"）， +/// 而且**判据很窄**：整张表一个 ``/`` 都没有时才动。已经有表头的 +/// 表格一个字不改。 +/// +/// 嵌套表格按**从内到外**处理（`` 出现的顺序天然如此）：内层改过之后 +/// 外层就带着 `` 了，于是外层跳过——宁可少改一张，不去猜哪一行属于谁。 +fn promote_first_row_headers(html: &str) -> String { + let lower = html.to_ascii_lowercase(); + let mut out = html.to_string(); + // (开标签结束位置, 表格结束位置)，按闭合顺序 = 从内到外 + let mut opens: Vec = Vec::new(); + let mut spans: Vec<(usize, usize)> = Vec::new(); + let mut i = 0usize; + while i < lower.len() { + let next_open = lower[i..].find(" { + opens.push(o); + i = o + 6; + } + (_, Some(c)) => { + if let Some(o) = opens.pop() { + spans.push((o, c)); + } + i = c + 7; + } + (Some(o), None) => { + opens.push(o); + i = o + 6; + } + (None, None) => break, + } + } + // 位置会随改写移动，所以从后往前改 + spans.sort_by_key(|(o, _)| std::cmp::Reverse(*o)); + for (open, close) in spans { + let seg = &out[open..close.min(out.len())]; + let seg_lower = seg.to_ascii_lowercase(); + if seg_lower.contains(" String { + let mut out = String::with_capacity(row.len()); + let lower = row.to_ascii_lowercase(); + let mut i = 0usize; + while i < row.len() { + if lower[i..].starts_with(" String { + let is_row = |l: &str| { + let t = l.trim(); + t.starts_with('|') && t.ends_with('|') && t.len() > 1 + }; + let cells = |l: &str| -> Vec { + let t = l.trim(); + t[1..t.len() - 1] + .split('|') + .map(|c| c.trim().to_string()) + .collect() + }; + let is_sep = |c: &[String]| { + !c.is_empty() + && c.iter() + .all(|x| !x.is_empty() && x.chars().all(|ch| ch == '-' || ch == ':')) + }; + + let lines: Vec<&str> = markdown.split('\n').collect(); + let mut out: Vec = Vec::with_capacity(lines.len()); + let mut i = 0usize; + while i < lines.len() { + if !is_row(lines[i]) { + out.push(lines[i].to_string()); + i += 1; + continue; + } + let start = i; + while i < lines.len() && is_row(lines[i]) { + i += 1; + } + let rows: Vec> = lines[start..i].iter().map(|l| cells(l)).collect(); + let width = rows.iter().map(Vec::len).max().unwrap_or(0); + let keep: Vec = (0..width) + .map(|c| { + rows.iter() + .any(|r| !is_sep(r) && r.get(c).is_some_and(|x| !x.is_empty())) + }) + .collect(); + // 一列都不剩就原样留着，不去猜 + if !keep.iter().any(|k| *k) { + out.extend(lines[start..i].iter().map(|l| l.to_string())); + continue; + } + let render = |r: &Vec| { + let sep = is_sep(r); + let kept: Vec = (0..width) + .filter(|c| keep[*c]) + .map(|c| { + let v = r.get(c).cloned().unwrap_or_default(); + if sep && v.is_empty() { + "---".to_string() + } else { + v + } + }) + .collect(); + format!("| {} |", kept.join(" | ")) + }; + // **表头整行是空的就让位。** 提上来的第一行有时只是排版用的占位行 + // （SEC 那份投票结果 8-K 的每张表都这样），留着它，模型看到的是 + // 一张列名全空的表——没有信息，还占着「表头」这个位置。下一行顶上， + // 表的第一行才是它真正的抬头（"a. Tench Coxe"）。 + let head_blank = rows.first().is_some_and(|r| { + (0..width) + .filter(|c| keep[*c]) + .all(|c| r.get(c).is_none_or(String::is_empty)) + }); + let body_start = if head_blank && rows.len() > 2 { 2 } else { 0 }; + if body_start == 2 { + out.push(render(&rows[2])); + if let Some(sep) = rows.get(1) { + out.push(render(sep)); + } + for r in &rows[3..] { + out.push(render(r)); + } + } else { + for r in &rows { + out.push(render(r)); + } + } + } + out.join("\n") +} + fn markdown_from_html(html: &str) -> Result { + let html = &promote_first_row_headers(html); let markdown = htmd::HtmlToMarkdown::builder() .skip_tags(vec![ "script", "style", "iframe", "object", "embed", "img", "svg", "math", @@ -252,7 +505,7 @@ fn markdown_from_html(html: &str) -> Result { { return Err(HtmlError::Interstitial); } - normalize_markdown(&markdown) + normalize_markdown(&prune_empty_table_columns(&markdown)) } /// Normalize direct Markdown with the same link policy as HTML conversion. diff --git a/crates/utopia-ingest/src/ontology_rdf.rs b/crates/utopia-ingest/src/ontology_rdf.rs index e926aad20..1f7fd1d4c 100644 --- a/crates/utopia-ingest/src/ontology_rdf.rs +++ b/crates/utopia-ingest/src/ontology_rdf.rs @@ -50,6 +50,10 @@ pub struct OwlProperty { /// `rdfs:subPropertyOf` 的父属性 IRI。多写几条只留第一条—— /// OWL 允许多父，而 R1 的规则一次只升一级，多父要另一套形状 pub sub_property_of: Option, + /// `schema:supersededBy` 的对端 IRI：这条属性已经被另一条取代。schema.org 里 + /// `employees` 让位给 `employee`、`founders` 让位给 `founder`，两条都还在词表里。 + /// 不读这一位，两条都建成关系，抽取时精确 key 各自命中，一个关系永久分成两个（#560） + pub superseded_by: Option, pub domains: Vec, pub ranges: Vec, /// **多条 range 是并集还是交集**。`rdfs:range` 写多条是交集 @@ -229,6 +233,7 @@ pub fn project(bytes: &[u8], format: RdfFormat) -> anyhow::Result // 属性之间的两条关系（不是类型声明，所以单独收） let mut inverse_of: BTreeMap = BTreeMap::new(); let mut sub_property_of: BTreeMap = BTreeMap::new(); + let mut superseded_by: BTreeMap = BTreeMap::new(); let mut asymmetric: BTreeSet = BTreeSet::new(); let mut irreflexive: BTreeSet = BTreeSet::new(); let mut plain_props: BTreeSet = BTreeSet::new(); @@ -315,6 +320,7 @@ pub fn project(bytes: &[u8], format: RdfFormat) -> anyhow::Result || is_schema(p, "rangeIncludes") || p == format!("{OWL}disjointWith") || p == format!("{OWL}inverseOf") + || is_schema(p, "supersededBy") }; for t in &triples { let p = t.predicate.as_str(); @@ -352,6 +358,11 @@ pub fn project(bytes: &[u8], format: RdfFormat) -> anyhow::Result if let Some(o) = &t.object_iri { inverse_of.entry(t.subject.clone()).or_insert(o.clone()); } + } else if is_schema(p, "supersededBy") { + // 取代关系只在 schema.org 词表里出现；多写几条只留第一条 + if let Some(o) = &t.object_iri { + superseded_by.entry(t.subject.clone()).or_insert(o.clone()); + } } else if p == format!("{RDFS}subPropertyOf") { // 从前这条只出现在 `known()` 白名单里——认得出、不报警、**然后扔掉**。 // 导一份带 subPropertyOf 的本体进来，那部分信息当场消失且没有提示 @@ -509,6 +520,7 @@ pub fn project(bytes: &[u8], format: RdfFormat) -> anyhow::Result asymmetric: asymmetric.contains(iri), inverse_of: inverse_of.get(iri).cloned(), sub_property_of: sub_property_of.get(iri).cloned(), + superseded_by: superseded_by.get(iri).cloned(), irreflexive: irreflexive.contains(iri), domains: domains.get(iri).cloned().unwrap_or_default(), ranges: rs, @@ -1480,6 +1492,39 @@ mod against_real_packs { mod property_axiom_tests { use super::*; + /// `schema:supersededBy` 要被读出来：让位的属性带着它让给了谁 + #[test] + fn a_superseded_property_names_its_successor() { + let ttl = r#" +@prefix rdf: . +@prefix rdfs: . +@prefix schema: . +schema:Organization a rdfs:Class ; rdfs:label "Organization" . +schema:Person a rdfs:Class ; rdfs:label "Person" . +schema:employee a rdf:Property ; rdfs:label "employee" ; + schema:domainIncludes schema:Organization ; schema:rangeIncludes schema:Person . +schema:employees a rdf:Property ; rdfs:label "employees" ; + schema:domainIncludes schema:Organization ; schema:rangeIncludes schema:Person ; + schema:supersededBy schema:employee . +"#; + let p = project(ttl.as_bytes(), RdfFormat::Turtle).expect("解析"); + let by = |k: &str| { + p.properties + .iter() + .find(|x| x.key == k) + .unwrap_or_else(|| panic!("没有 {k}")) + }; + assert_eq!( + by("employees").superseded_by.as_deref(), + Some("https://schema.org/employee") + ); + assert!(by("employee").superseded_by.is_none()); + assert!( + !p.unprojected.keys().any(|k| k.contains("supersededBy")), + "认了就不该再算进「暂未投影」" + ); + } + /// `owl:inverseOf` 与 `rdfs:subPropertyOf` 要被读出来。 /// /// 从前 `subPropertyOf` 只出现在 `known()` 白名单里——认得出、不报警、 diff --git a/crates/utopia-llm/src/lib.rs b/crates/utopia-llm/src/lib.rs index 68a1edb41..ded4092b6 100644 --- a/crates/utopia-llm/src/lib.rs +++ b/crates/utopia-llm/src/lib.rs @@ -62,15 +62,15 @@ pub enum ToolStreamItem { } /// **没能从端点拿到一个能解析的回答。** 两种：连不上（DNS、连接、TLS、超时）， -/// 或者连上了但回来的根本不是这个 API（响应体解不成 JSON）。 +/// 或者连上了、状态码说成功、可回来的东西解不成 JSON——根本不是这个 API。 /// /// 合成一类是有意的：从用户那边看这两种是同一件事——"你配的这个地址不是模型 API"， /// 该做的也是同一件事：去看 URL、看代理。第一版只收传输层失败，结果最常见的那种 /// 故障（URL 配错、代理挡在中间回了 HTML）一条告警都不产生， /// 正是"失败无声"本身。 /// -/// **不含**端点干干净净地回了 4xx/5xx：那说明它就是模型 API，只是密钥、 -/// 配额或模型名不对——另一类问题，该找的人也不同。 +/// **不含**端点回的 4xx/5xx，即使错误体不是 JSON：那说明服务器已经回答， +/// 错误体本身就是诊断——另一类问题，该找的人也不同。 /// /// 有类型而不是匹配错误文本：调用链上任何一层加一句 context 都会改文本， /// 而 `anyhow` 的 source 链让 `downcast_ref` 一路都认得出来。 @@ -147,8 +147,9 @@ fn failure( status: reqwest::StatusCode, retry_after: Option, body: &serde_json::Value, + raw: &str, ) -> anyhow::Error { - let detail = err_detail(body); + let detail = err_detail(body, raw); // **欠费要先判，而且不能只看状态码。** // // 402 是标准答案（SiliconFlow 用它），但 OpenAI 的余额耗尽走的是 **429**， @@ -171,6 +172,25 @@ fn failure( anyhow::anyhow!("{kind} request failed ({status}): {detail}") } +/// 把一个非成功状态的响应变成错误（#527）。 +/// +/// **先看状态码，再决定怎么读 body。** 从前成功与失败都先 `resp.json()`，于是一个回了 +/// 纯文本的 400——LM Studio 那句「request (105140 tokens) exceeds the available context」—— +/// 解不成 JSON 就被当成 `Unreachable`，报告者只好架代理抓包才看见它。现在 body 只按 +/// 原文读一次：是 JSON 就按几种常见形状取消息，不是就把原文截一段引出来。 +/// +/// 只有 body **读不出来**（连接半路断了）才仍是 `Unreachable`：那才是传输层的事。 +async fn response_failure( + kind: &str, + status: reqwest::StatusCode, + retry_after: Option, + response: reqwest::Response, +) -> anyhow::Result { + let raw = response.text().await.map_err(Unreachable)?; + let body = serde_json::from_str(&raw).unwrap_or_default(); + Ok(failure(kind, status, retry_after, &body, &raw)) +} + #[derive(Clone)] pub struct LlmClient { http: reqwest::Client, @@ -248,10 +268,10 @@ impl LlmClient { .map_err(Unreachable)?; let status = resp.status(); let retry_after = retry_after_of(resp.headers()); - let body: serde_json::Value = resp.json().await.map_err(Unreachable)?; if !status.is_success() { - return Err(failure("LLM", status, retry_after, &body)); + return Err(response_failure("LLM", status, retry_after, resp).await?); } + let body: serde_json::Value = resp.json().await.map_err(Unreachable)?; log_usage(&self.model, &body); body["choices"][0]["message"]["content"] .as_str() @@ -279,10 +299,10 @@ impl LlmClient { .map_err(Unreachable)?; let status = resp.status(); let retry_after = retry_after_of(resp.headers()); - let body: serde_json::Value = resp.json().await.map_err(Unreachable)?; if !status.is_success() { - return Err(failure("LLM", status, retry_after, &body)); + return Err(response_failure("LLM", status, retry_after, resp).await?); } + let body: serde_json::Value = resp.json().await.map_err(Unreachable)?; let msg = &body["choices"][0]["message"]; if msg.is_null() { anyhow::bail!("Unexpected LLM response shape: {body}"); @@ -336,8 +356,7 @@ impl LlmClient { if !resp.status().is_success() { let status = resp.status(); let retry_after = retry_after_of(resp.headers()); - let body: serde_json::Value = resp.json().await.unwrap_or_default(); - return Err(failure("LLM", status, retry_after, &body)); + return Err(response_failure("LLM", status, retry_after, resp).await?); } let mut bytes = resp.bytes_stream(); @@ -429,8 +448,7 @@ impl LlmClient { if !resp.status().is_success() { let status = resp.status(); let retry_after = retry_after_of(resp.headers()); - let body: serde_json::Value = resp.json().await.unwrap_or_default(); - return Err(failure("LLM", status, retry_after, &body)); + return Err(response_failure("LLM", status, retry_after, resp).await?); } let mut bytes = resp.bytes_stream(); @@ -474,10 +492,10 @@ impl LlmClient { .map_err(Unreachable)?; let status = resp.status(); let retry_after = retry_after_of(resp.headers()); - let body: serde_json::Value = resp.json().await.map_err(Unreachable)?; if !status.is_success() { - return Err(failure("Embedding", status, retry_after, &body)); + return Err(response_failure("Embedding", status, retry_after, resp).await?); } + let body: serde_json::Value = resp.json().await.map_err(Unreachable)?; let data = body["data"] .as_array() .ok_or_else(|| anyhow::anyhow!("Unexpected embedding response shape"))?; @@ -519,12 +537,26 @@ fn log_usage(model: &str, body: &serde_json::Value) { ); } -fn err_detail(body: &serde_json::Value) -> String { +/// 引用原文时最多带多少字。到目前见过的诊断没有一条超过它，而这段字会进日志、 +/// 告警和文档的错误栏，一个回 HTML 页面的代理不该把整页塞进去。按字符数不按字节， +/// 中文错误信息切在字符中间就成了乱码。 +const MAX_ERROR_DETAIL_CHARS: usize = 500; + +/// 从错误体里取一句给人看的话。依次试 OpenAI 的 `error.message`、SiliconFlow 那种 +/// 顶层 `message`、以及 `error` 直接是一个字符串的写法；都不是就引用原文本身。 +/// 「unknown error」只留给 body 真的是空的那种情况——它从前是所有认不出的形状的 +/// 归宿，把最有用的那句话吞掉了（#527）。 +fn err_detail(body: &serde_json::Value, raw: &str) -> String { body["error"]["message"] .as_str() .or_else(|| body["message"].as_str()) - .unwrap_or("unknown error") - .to_string() + .or_else(|| body["error"].as_str()) + .map(String::from) + .or_else(|| { + let raw = raw.trim(); + (!raw.is_empty()).then(|| raw.chars().take(MAX_ERROR_DETAIL_CHARS).collect()) + }) + .unwrap_or_else(|| "unknown error".to_string()) } /// 响应体说不说这是余额问题。 @@ -545,6 +577,239 @@ fn says_out_of_credit(body: &serde_json::Value) -> bool { #[cfg(test)] mod tests { use super::*; + use tokio::io::AsyncWriteExt; + + fn client_at(addr: std::net::SocketAddr) -> LlmClient { + LlmClient::new(&format!("http://{addr}"), None, "m") + } + + fn no_tools() -> serde_json::Value { + serde_json::json!([]) + } + + /// 把请求整个读完：头读到空行，正文按 Content-Length。 + /// + /// **必须先读再答。** 收到的数据还没读就关连接，Windows 会发 RST，客户端那边 + /// 已经到手的响应连同错误一起变成「连接被中止」（os error 10053）——这组测试 + /// 在 Linux 上绿、在 Windows 上红，就是这个原因 + async fn read_request(socket: &mut tokio::net::TcpStream) { + use tokio::io::AsyncReadExt; + let mut buf = Vec::new(); + let mut chunk = [0u8; 1024]; + let header_end = loop { + let n = socket.read(&mut chunk).await.unwrap(); + if n == 0 { + return; + } + buf.extend_from_slice(&chunk[..n]); + if let Some(pos) = buf.windows(4).position(|w| w == b"\r\n\r\n") { + break pos + 4; + } + }; + let head = String::from_utf8_lossy(&buf[..header_end]).to_ascii_lowercase(); + let want: usize = head + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + while buf.len() - header_end < want { + let n = socket.read(&mut chunk).await.unwrap(); + if n == 0 { + break; + } + buf.extend_from_slice(&chunk[..n]); + } + } + + /// 一个只答一次的 HTTP 服务：读完请求，把这份响应原样写回去，关掉。 + /// 用裸 socket 而不是 mock 库，是因为要造的正是「不像模型 API 的回答」—— + /// 纯文本、不完整、什么都行。返回的句柄要 await：它跑完才说明响应真的发出去了 + async fn an_http_response( + status: &str, + content_type: &str, + body: &str, + ) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let status = status.to_string(); + let content_type = content_type.to_string(); + let body = body.to_string(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + read_request(&mut socket).await; + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + socket.write_all(response.as_bytes()).await.unwrap(); + socket.shutdown().await.unwrap(); + }); + (addr, server) + } + + async fn an_http_error(body: &str) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { + an_http_response("502 Bad Gateway", "text/plain", body).await + } + + /// 声称 body 有 100 字节、只发 1 个就挂断：读 body 会半路失败 + async fn an_incomplete_http_error() -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + read_request(&mut socket).await; + socket + .write_all(b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 100\r\nConnection: close\r\n\r\nx") + .await + .unwrap(); + socket.shutdown().await.unwrap(); + }); + (addr, server) + } + + /// 错误里带着端点那句话，且没被判成 Unreachable + async fn assert_diagnosis( + result: anyhow::Result, + diagnosis: &str, + server: tokio::task::JoinHandle<()>, + ) { + server.await.unwrap(); + let error = match result { + Ok(_) => panic!("the endpoint returned an error response"), + Err(error) => error, + }; + assert!(!is_unreachable(&error)); + assert!(error.to_string().contains(diagnosis), "{error:#}"); + } + + /// #527 的正题：一个回纯文本的 502，五条请求路径（对话、工具对话、两种流式、嵌入） + /// 都要把那句话原样带出来，而不是报「unknown error」或「连不上」。 + #[tokio::test] + async fn a_body_that_is_not_json_is_quoted_and_not_called_unreachable() { + let diagnosis = "proxy says the model context is too long"; + let (addr, server) = an_http_error(diagnosis).await; + let client = client_at(addr); + assert_diagnosis( + client + .chat(&[ChatMessage { + role: "user".into(), + content: "hi".into(), + }]) + .await, + diagnosis, + server, + ) + .await; + + let (addr, server) = an_http_error(diagnosis).await; + let client = client_at(addr); + assert_diagnosis(client.chat_tools(&[], &no_tools()).await, diagnosis, server).await; + + let (addr, server) = an_http_error(diagnosis).await; + let client = client_at(addr); + assert_diagnosis( + client.chat_tools_stream(&[], &no_tools()).await, + diagnosis, + server, + ) + .await; + + let (addr, server) = an_http_error(diagnosis).await; + let client = client_at(addr); + assert_diagnosis( + client + .chat_stream(&[ChatMessage { + role: "user".into(), + content: "hi".into(), + }]) + .await, + diagnosis, + server, + ) + .await; + + let (addr, server) = an_http_error(diagnosis).await; + let client = client_at(addr); + assert_diagnosis(client.embed(&["hi".to_string()]).await, diagnosis, server).await; + } + + /// 反面：body 读到一半连接断了，这是传输层的事，仍归 Unreachable—— + /// 别把「引用原文」做过头，把真的连接故障也说成端点的诊断 + #[tokio::test] + async fn a_body_that_cannot_be_read_stays_unreachable() { + let (addr, server) = an_incomplete_http_error().await; + let client = client_at(addr); + let result = client.chat(&[]).await; + server.await.unwrap(); + let error = result.expect_err("the body is incomplete"); + assert!( + is_unreachable(&error), + "body read error lost its source: {error:#}" + ); + } + + /// 成功路径原样：状态码先行没有改变成功响应的解析 + #[tokio::test] + async fn successful_responses_still_parse() { + let (addr, server) = an_http_response( + "200 OK", + "application/json", + r#"{"choices":[{"message":{"content":"ok"}}]}"#, + ) + .await; + let client = client_at(addr); + let result = client.chat(&[]).await; + server.await.unwrap(); + assert_eq!(result.unwrap(), "ok"); + + let (addr, server) = an_http_response( + "200 OK", + "application/json", + r#"{"choices":[{"message":{"content":"ok"}}]}"#, + ) + .await; + let client = client_at(addr); + let result = client.chat_tools(&[], &no_tools()).await; + server.await.unwrap(); + assert_eq!(result.unwrap().content.as_deref(), Some("ok")); + + let (addr, server) = an_http_response( + "200 OK", + "application/json", + r#"{"data":[{"embedding":[1.0]}]}"#, + ) + .await; + let client = client_at(addr); + let result = client.embed(&["hi".to_string()]).await; + server.await.unwrap(); + assert_eq!(result.unwrap(), vec![vec![1.0]]); + } + + /// `{"error": "…"}` 这种把错误直接写成字符串的端点（LM Studio 就是），从前认不出 + #[test] + fn an_error_given_as_a_string_is_forwarded() { + let body = serde_json::json!({ "error": "model is unavailable" }); + assert_eq!( + err_detail(&body, ""), + "model is unavailable", + "a valid error field must not become unknown" + ); + } + + /// OpenAI 形状照旧 + #[test] + fn a_body_with_error_message_still_reads_it() { + let body = serde_json::json!({ "error": { "message": "bad request" } }); + assert_eq!(err_detail(&body, ""), "bad request"); + } + + /// 截断按字符数：501 个「界」截成 500 个，而不是在某个字的字节中间切开 + #[test] + fn a_long_body_is_truncated() { + let raw = "界".repeat(MAX_ERROR_DETAIL_CHARS + 1); + let detail = err_detail(&serde_json::Value::Null, &raw); + assert_eq!(detail.chars().count(), MAX_ERROR_DETAIL_CHARS); + } /// 真发一次注定失败的请求，拿一个货真价实的 `reqwest::Error`。 /// 端口 1 上不会有东西监听，而 127.0.0.1 不走代理。 @@ -692,7 +957,13 @@ mod tests { let body = serde_json::json!({ "error": { "message": "You exceeded your current quota", "code": "insufficient_quota" } }); - let e = failure("LLM", reqwest::StatusCode::TOO_MANY_REQUESTS, None, &body); + let e = failure( + "LLM", + reqwest::StatusCode::TOO_MANY_REQUESTS, + None, + &body, + "", + ); assert!(out_of_credit(&e).is_some(), "该判成欠费"); assert!(rate_limited(&e).is_none(), "不该判成限流"); } @@ -701,7 +972,13 @@ mod tests { #[test] fn a_402_is_a_billing_problem() { let body = serde_json::json!({ "message": "Sorry, your account balance is insufficient" }); - let e = failure("LLM", reqwest::StatusCode::PAYMENT_REQUIRED, None, &body); + let e = failure( + "LLM", + reqwest::StatusCode::PAYMENT_REQUIRED, + None, + &body, + "", + ); assert!(out_of_credit(&e).is_some()); } @@ -709,7 +986,13 @@ mod tests { #[test] fn a_plain_429_is_still_a_rate_limit() { let body = serde_json::json!({ "error": { "message": "TPM limit reached" } }); - let e = failure("LLM", reqwest::StatusCode::TOO_MANY_REQUESTS, None, &body); + let e = failure( + "LLM", + reqwest::StatusCode::TOO_MANY_REQUESTS, + None, + &body, + "", + ); assert!(rate_limited(&e).is_some()); assert!(out_of_credit(&e).is_none()); } diff --git a/crates/utopia-reason/src/rules.rs b/crates/utopia-reason/src/rules.rs index f815ec03e..747d789e0 100644 --- a/crates/utopia-reason/src/rules.rs +++ b/crates/utopia-reason/src/rules.rs @@ -33,6 +33,9 @@ pub enum Op { Lte, Between, In, + /// 反过来的 In。**它判的仍然是一条存在的事实**，所以照样有前提、有区间； + /// 「压根没有这个属性」是另一件事，不在这套里（0029 末节） + NotIn, Present, } @@ -45,6 +48,7 @@ impl Op { Op::Lte => "lte", Op::Between => "between", Op::In => "in", + Op::NotIn => "not_in", Op::Present => "present", } } @@ -57,6 +61,7 @@ impl Op { "lte" => Op::Lte, "between" => Op::Between, "in" => Op::In, + "not_in" => Op::NotIn, "present" => Op::Present, _ => return None, }) @@ -72,11 +77,117 @@ pub enum Operand { /// 类别集合。**逐字匹配**——同一类别换个语言就不算命中，这是 0021 记下的 /// 未决问题，不在这里偷偷做模糊 Set(Vec), + /// 算出来的数（0032）：拿这个实体别的读数算一个门槛，而不是写死一个 + Calc(Expr), None, } +/// 四则。**只有这四个**：再多一个就得回答它对缺读数、对除零、对单位各是什么 +/// 语义，而那是一门语言的开头（0002 / 0032） +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Arith { + Add, + Sub, + Mul, + Div, +} + +impl Arith { + pub fn as_str(self) -> &'static str { + match self { + Arith::Add => "add", + Arith::Sub => "sub", + Arith::Mul => "mul", + Arith::Div => "div", + } + } + pub fn parse(s: &str) -> Option { + Some(match s { + "add" => Arith::Add, + "sub" => Arith::Sub, + "mul" => Arith::Mul, + "div" => Arith::Div, + _ => return None, + }) + } +} + +/// 一个数是怎么算出来的（0032）。叶子要么是这个实体的一条属性读数，要么是 +/// 写死的数；中间是四则。 +/// +/// **树，不是字符串。** 存下来的就是这棵树，界面照着它渲染，求值照着它算—— +/// 两者不会漂移，而这正是 0002 拒掉「用户自定义规则语言」时真正在守的东西。 +#[derive(Debug, Clone, PartialEq)] +pub enum Expr { + /// 这个实体在这个谓词上的读数 + Attr(Uuid), + Const(f64), + Arith { + op: Arith, + l: Box, + r: Box, + }, +} + +/// 一棵算式最深几层。太深的树是「有人在这里写程序」的信号，编译时就该拦下 +pub const MAX_EXPR_DEPTH: usize = 4; + +impl Expr { + /// 这棵树读了哪些谓词，按出现顺序、去重。求值前要为它们各留一个槽位 + pub fn predicates(&self, out: &mut Vec) { + match self { + Expr::Attr(p) => { + if !out.contains(p) { + out.push(*p); + } + } + Expr::Const(_) => {} + Expr::Arith { l, r, .. } => { + l.predicates(out); + r.predicates(out); + } + } + } + + pub fn depth(&self) -> usize { + match self { + Expr::Attr(_) | Expr::Const(_) => 1, + Expr::Arith { l, r, .. } => 1 + l.depth().max(r.depth()), + } + } + + /// 按这一轮选中的读数算一个数。 + /// + /// **算不出来就是 None，不是 0**：读数不在（没记 ≠ 零）、值不是数、除零， + /// 三种情况一律没有值，于是这条规则在这个组合上不出结论（0032）。把它们 + /// 当零，等于在无知的地方填一个确定的答案。 + pub fn eval(&self, bound: &HashMap) -> Option { + Some(match self { + Expr::Attr(p) => num(&bound.get(p)?.value)?, + Expr::Const(n) => *n, + Expr::Arith { op, l, r } => { + let (a, b) = (l.eval(bound)?, r.eval(bound)?); + match op { + Arith::Add => a + b, + Arith::Sub => a - b, + Arith::Mul => a * b, + Arith::Div => { + if b == 0.0 { + return None; + } + a / b + } + } + } + }) + .filter(|n: &f64| n.is_finite()) + } +} + #[derive(Debug, Clone, PartialEq)] pub struct Condition { + /// 同组的条件用「与」连，组与组之间用「或」连（0029）。老规则全是第 0 组 + pub group: i32, pub predicate: Uuid, pub op: Op, pub operand: Operand, @@ -87,6 +198,9 @@ pub struct Condition { pub enum Conclusion { /// 派生归类：结论是一个类，落成 `is_a` 上的字面值 Typing { class: String }, + /// 派生属性：算出来的数（0032）。与上面那支的区别只在值从哪来—— + /// 算式读的那几条读数**一并进前提**，所以结论仍然说得出「凭什么」 + Computed { predicate: Uuid, expr: Expr }, /// 派生属性：某个属性谓词上的一个字面值 Attribute { predicate: Uuid, @@ -98,7 +212,8 @@ pub enum Conclusion { pub struct BusinessRule { pub id: Uuid, pub conclusion: Conclusion, - /// 合取：全部满足才算命中。空条件集永不命中——一条没有判据的规则应当 + /// 条件。**组内合取、组间析取**（0029）：同一组里全部满足才算这一组成立， + /// 任何一组成立这条规则就命中。空条件集永不命中——一条没有判据的规则应当 /// 什么都不推，而不是把整个类都归进去 pub conditions: Vec, } @@ -111,6 +226,9 @@ pub struct RuleHit { pub premises: Vec, pub from: Option, pub to: Option, + /// 算出来的结论值（0032）。**每个组合各算各的**——两条 revenue 三条 cost + /// 就是六个数、六段区间、六行结论；常量结论这里是 None + pub value: Option, } #[derive(Debug, Clone, Default, PartialEq, Eq)] @@ -140,6 +258,11 @@ const MAX_COMBOS: usize = 64; /// 求区间交集，交集非空即一次命中。**所以同一条规则可以在同一个实体上产出 /// 多段区间**——2023 那次读数命中、2025 那次不命中，得到的是两段各自成立的 /// 结论，而不是一行翻来覆去改（0021 决策 4）。 +/// +/// **组内与、组间或**（0029）：上面那一段是一组的算法，整条规则就是把它按组 +/// 跑几遍。封顶按组算——一组展不开不该拖累另一组；两组推出同一区间时按区间 +/// 去重，留先到的那组当证明（多条证明路径对读的人没有区别，全存下来只会让 +/// 证明树跟着路径数长）。 pub fn evaluate( rules: &[BusinessRule], facts: &[AttrFact], @@ -161,50 +284,119 @@ pub fn evaluate( if rule.conditions.is_empty() { continue; } + // 按组切开，组序保持稳定：同一区间被两组同时推出时，留下的是**组序在前** + // 的那条证明，而不是 HashMap 顺序决定的随机一条 + let groups = group_conditions(&rule.conditions); for (subject, own) in &by_subject { - // 每个条件的命中集。任何一个为空，这条规则在这个实体上就不成立 - let mut per_condition: Vec> = Vec::with_capacity(rule.conditions.len()); - let mut satisfiable = true; - for c in &rule.conditions { - let matched: Vec = own - .iter() - .filter(|f| f.predicate == c.predicate && satisfies(c, &f.value)) - .map(|f| f.id) - .collect(); - if matched.is_empty() { - satisfiable = false; - break; + // 同一实体上，一条规则的多个组可能推出同一段区间。去重跨组， + // 因为它们落成的是同一行派生事实 + // 去重的键带上算出来的值（0032）：同一区间上两个不同的值是两行 + let mut seen: Vec<(Option, Option, Option)> = Vec::new(); + // 这条规则在这个实体上有没有哪一组因组合太多没展开完。**按 (规则, 实体) + // 记一次**：报出来的是「这里少推了东西」，不是「少推了几组」 + let mut capped_here = false; + for group in &groups { + // ---- 槽位：这一组要各挑一条读数的那几处。 + // + // 前 group.len() 个是条件自己的谓词（**一个条件一个槽**，与 + // 0032 之前一样：两个写在同一属性上的条件仍然各挑各的读数）； + // 后面是算式里读到、而条件没覆盖的谓词（0032）。算式引用一个 + // 谓词时绑到**第一个**同谓词的槽——`thc > thc * 0.5` 两边说的 + // 就是同一条读数，而不是同一属性的两条 + let mut slots: Vec = group.iter().map(|c| c.predicate).collect(); + let mut extra: Vec = Vec::new(); + for c in group { + if let Operand::Calc(e) = &c.operand { + e.predicates(&mut extra); + } + } + if let Conclusion::Computed { expr, .. } = &rule.conclusion { + expr.predicates(&mut extra); + } + for p in extra { + if !slots.contains(&p) { + slots.push(p); + } } - per_condition.push(matched); - } - if !satisfiable { - continue; - } - - let combos: usize = per_condition.iter().map(|v| v.len()).product(); - if combos > MAX_COMBOS { - report.capped += 1; - continue; - } - // 笛卡尔积。同一区间可能由多个组合得出（同一读数报了两遍）， - // 按区间去重——它们本来就会落成同一行 - let mut seen: Vec<(Option, Option)> = Vec::new(); - for combo in cartesian(&per_condition) { - let Some((from, to)) = validity(&combo, spans) else { + // 每个槽位的候选读数。任何一个为空，这一组在这个实体上就不成立 + let mut per_slot: Vec> = Vec::with_capacity(slots.len()); + let mut satisfiable = true; + for p in &slots { + let matched: Vec = own + .iter() + .filter(|f| f.predicate == *p) + .map(|f| f.id) + .collect(); + if matched.is_empty() { + satisfiable = false; + break; + } + per_slot.push(matched); + } + if !satisfiable { continue; - }; - if seen.contains(&(from, to)) { + } + + // 封顶按组算：一组展不开，不该把另一组也拦下 + let combos: usize = per_slot.iter().map(|v| v.len()).product(); + if combos > MAX_COMBOS { + capped_here = true; continue; } - seen.push((from, to)); - hits.push(RuleHit { - rule: rule.id, - subject: *subject, - premises: combo, - from, - to, - }); + + let by_id: HashMap = own.iter().map(|f| (f.id, *f)).collect(); + + // 笛卡尔积。**条件要等组合定下来才判得了**（0032）：一个算出来的 + // 门槛读的是这一轮选中的那几条读数，判断不再只看一条事实。 + // 同一区间可能由多个组合得出（同一读数报了两遍），按区间去重 + for combo in cartesian(&per_slot) { + // 槽位 → 这一轮选中的那条读数。算式与条件都照着它读 + let mut bound: HashMap = HashMap::new(); + for (i, id) in combo.iter().enumerate() { + let Some(f) = by_id.get(id) else { continue }; + bound.entry(slots[i]).or_insert(f); + } + let holds = group.iter().enumerate().all(|(i, c)| { + combo + .get(i) + .and_then(|id| by_id.get(id)) + .is_some_and(|f| satisfies(c, &f.value, &bound)) + }); + if !holds { + continue; + } + // 算出来的结论：这个组合上算不出数就不出结论（缺读数、 + // 不是数、除零），而不是落一行没有值的派生 + let value = match &rule.conclusion { + Conclusion::Computed { expr, .. } => match expr.eval(&bound) { + Some(v) => Some(v), + None => continue, + }, + _ => None, + }; + let Some((from, to)) = validity(&combo, spans) else { + continue; + }; + // 去重按 (区间, 值)：算出来的结论同一区间可以有不同的值， + // 那是两行，不是一行 + let key = (from, to, value.map(f64::to_bits)); + if seen.contains(&key) { + continue; + } + seen.push(key); + hits.push(RuleHit { + rule: rule.id, + subject: *subject, + premises: combo, + from, + to, + value, + }); + } + } + if capped_here { + report.capped += 1; } } } @@ -212,6 +404,17 @@ pub fn evaluate( (hits, report) } +/// 按 `group` 切成几组，**组序按 group_seq 升序**——两组推出同一区间时， +/// 留下的证明得是稳定的那一条，不能随存储顺序变。 +fn group_conditions(conditions: &[Condition]) -> Vec> { + let mut keys: Vec = conditions.iter().map(|c| c.group).collect(); + keys.sort_unstable(); + keys.dedup(); + keys.into_iter() + .map(|g| conditions.iter().filter(|c| c.group == g).collect()) + .collect() +} + /// 每个条件取一条，穷举组合。调用方已经把上限挡在外面。 fn cartesian(sets: &[Vec]) -> Vec> { let mut out: Vec> = vec![Vec::new()]; @@ -233,9 +436,26 @@ fn cartesian(sets: &[Vec]) -> Vec> { /// /// **类型不对就是不满足，不是报错。** 一个本该是数字的属性被抽成了 /// "十二点三"，这条规则在这个实体上不成立——而不是让整轮物化失败。 -fn satisfies(c: &Condition, value: &serde_json::Value) -> bool { +fn satisfies(c: &Condition, value: &serde_json::Value, bound: &HashMap) -> bool { + // 算出来的门槛：先按这一轮选中的读数算个数，算不出来就是不满足（0032） + if let Operand::Calc(e) = &c.operand { + let Some(n) = e.eval(bound) else { return false }; + let Some(v) = num(value) else { return false }; + return match c.op { + Op::Gt => v > n, + Op::Gte => v >= n, + Op::Lt => v < n, + Op::Lte => v <= n, + // 算式给的是一个数：区间、集合、有记录都不是它能填的位置 + _ => false, + }; + } match (&c.op, &c.operand) { (Op::Present, _) => !value.is_null(), + // 值不在集合里就算满足；**没有值不算**——那是「没记」，不是「不是它」 + (Op::NotIn, Operand::Set(set)) => { + !value.is_null() && text(value).is_some_and(|v| !set.iter().any(|s| s == &v)) + } (Op::Gt, Operand::Num(n)) => num(value).is_some_and(|v| v > *n), (Op::Gte, Operand::Num(n)) => num(value).is_some_and(|v| v >= *n), (Op::Lt, Operand::Num(n)) => num(value).is_some_and(|v| v < *n), @@ -296,11 +516,13 @@ mod tests { }, conditions: vec![ Condition { + group: 0, predicate: id(10), op: Op::Gt, operand: Operand::Num(8.0), }, Condition { + group: 0, predicate: id(11), op: Op::In, operand: Operand::Set(vec!["气测异常".into(), "气测异常后效".into()]), @@ -331,11 +553,13 @@ mod tests { }, conditions: vec![ Condition { + group: 0, predicate: id(10), op: Op::Gt, operand: Operand::Num(8.0), }, Condition { + group: 0, predicate: id(11), op: Op::Present, operand: Operand::None, @@ -357,6 +581,7 @@ mod tests { class: "GasBearingWell".into(), }, conditions: vec![Condition { + group: 0, predicate: id(10), op: Op::Gt, operand: Operand::Num(8.0), @@ -389,11 +614,13 @@ mod tests { }, conditions: vec![ Condition { + group: 0, predicate: id(10), op: Op::Gt, operand: Operand::Num(8.0), }, Condition { + group: 0, predicate: id(11), op: Op::In, operand: Operand::Set(vec!["气测异常".into()]), @@ -412,6 +639,160 @@ mod tests { assert!(hits.is_empty(), "两条前提没有同时成立的时段"); } + /// 两组「或」：任一组成立就命中。第二组的前提里**没有**第一组那条读数—— + /// 一条派生事实带的是真正让它成立的那几条，不是这个实体的全部属性 + #[test] + fn either_group_can_fire_and_carries_only_its_own_premises() { + let rule = BusinessRule { + id: id(90), + conclusion: Conclusion::Typing { + class: "GasBearingWell".into(), + }, + conditions: vec![ + // 第 0 组：全烃 > 8 且 解释 ∈ {气测异常} + Condition { + group: 0, + predicate: id(10), + op: Op::Gt, + operand: Operand::Num(8.0), + }, + Condition { + group: 0, + predicate: id(11), + op: Op::In, + operand: Operand::Set(vec!["气测异常".into()]), + }, + // 第 1 组：综合解释 ∈ {气层} + Condition { + group: 1, + predicate: id(12), + op: Op::In, + operand: Operand::Set(vec!["气层".into()]), + }, + ], + }; + // 只有第二组的那条读数 + let facts = vec![fact(3, 50, 12, json!("气层"))]; + let spans = HashMap::from([(id(3), (Some(100), Some(200)))]); + let (hits, _) = evaluate(&[rule], &facts, &spans); + assert_eq!(hits.len(), 1, "第二组独自成立"); + assert_eq!(hits[0].premises, vec![id(3)]); + assert_eq!((hits[0].from, hits[0].to), (Some(100), Some(200))); + } + + /// 两组同时成立、且推出同一段区间：**一条命中，不是两条**——它们落成的是 + /// 同一行派生事实，留下的证明是组序在前的那一组 + #[test] + fn two_groups_on_the_same_interval_are_one_hit() { + let rule = BusinessRule { + id: id(90), + conclusion: Conclusion::Typing { + class: "GasBearingWell".into(), + }, + conditions: vec![ + Condition { + group: 0, + predicate: id(10), + op: Op::Gt, + operand: Operand::Num(8.0), + }, + Condition { + group: 1, + predicate: id(11), + op: Op::In, + operand: Operand::Set(vec!["气层".into()]), + }, + ], + }; + let facts = vec![fact(1, 50, 10, json!(12.3)), fact(2, 50, 11, json!("气层"))]; + // 同一段区间 + let spans = HashMap::from([ + (id(1), (Some(100), Some(200))), + (id(2), (Some(100), Some(200))), + ]); + let (hits, report) = evaluate(&[rule], &facts, &spans); + assert_eq!(hits.len(), 1); + assert_eq!( + hits[0].premises, + vec![id(1)], + "留下的是组序在前那一组的证明" + ); + assert_eq!(report.hits, 1); + } + + /// 一组展不开不该拖累另一组：**封顶按组算**，而且这个 (规则, 实体) 对 + /// 只报一次「没展完」 + #[test] + fn a_capped_group_does_not_stop_the_other() { + let mut facts: Vec = Vec::new(); + let mut spans = HashMap::new(); + // 第 0 组：同一个属性上 100 条读数，组合数 100 > 64 + for i in 1..=100u8 { + facts.push(fact(i, 50, 10, json!(12.3))); + spans.insert(id(i), (Some(100), Some(200))); + } + // 第 1 组：一条就够 + facts.push(fact(200, 50, 11, json!("气层"))); + spans.insert(id(200), (Some(100), Some(200))); + let rule = BusinessRule { + id: id(90), + conclusion: Conclusion::Typing { + class: "GasBearingWell".into(), + }, + conditions: vec![ + Condition { + group: 0, + predicate: id(10), + op: Op::Gt, + operand: Operand::Num(8.0), + }, + Condition { + group: 1, + predicate: id(11), + op: Op::In, + operand: Operand::Set(vec!["气层".into()]), + }, + ], + }; + let (hits, report) = evaluate(&[rule], &facts, &spans); + assert_eq!(hits.len(), 1, "第二组照样出结论"); + assert_eq!(hits[0].premises, vec![id(200)]); + assert_eq!(report.capped, 1, "(规则, 实体) 只报一次"); + } + + /// `is not one of`：值在集合外算满足；**没有这条读数不算**——那是「没记」， + /// 不是「不是它」 + #[test] + fn not_one_of_needs_a_reading_to_be_true() { + let rule = BusinessRule { + id: id(90), + conclusion: Conclusion::Typing { + class: "NonGas".into(), + }, + conditions: vec![Condition { + group: 0, + predicate: id(11), + op: Op::NotIn, + operand: Operand::Set(vec!["气层".into()]), + }], + }; + let spans = HashMap::from([(id(1), (None, None))]); + let (hit, _) = evaluate( + std::slice::from_ref(&rule), + &[fact(1, 50, 11, json!("水层"))], + &spans, + ); + assert_eq!(hit.len(), 1, "读数在集合外"); + let (miss, _) = evaluate( + std::slice::from_ref(&rule), + &[fact(1, 50, 11, json!("气层"))], + &spans, + ); + assert!(miss.is_empty(), "读数在集合里"); + let (none, _) = evaluate(&[rule], &[], &HashMap::new()); + assert!(none.is_empty(), "没有这条读数：不成立，而不是「不是它」"); + } + /// 数字被抽成字符串照样比得动。**这一条挡的是最难发现的失效**： /// 规则从不报错，只是永远不命中 #[test] @@ -423,6 +804,7 @@ mod tests { value: json!("good"), }, conditions: vec![Condition { + group: 0, predicate: id(10), op: Op::Gte, operand: Operand::Num(12.0), @@ -445,6 +827,7 @@ mod tests { class: "GasBearingWell".into(), }, conditions: vec![Condition { + group: 0, predicate: id(10), op: Op::Gt, operand: Operand::Num(threshold), @@ -477,11 +860,13 @@ mod tests { conclusion: Conclusion::Typing { class: "X".into() }, conditions: vec![ Condition { + group: 0, predicate: id(10), op: Op::Present, operand: Operand::None, }, Condition { + group: 0, predicate: id(11), op: Op::Present, operand: Operand::None, @@ -500,4 +885,182 @@ mod tests { assert_eq!(report.capped, 1, "100 种组合超过上限，要计数"); assert!(hits.is_empty()); } + + fn attr(p: u8) -> Expr { + Expr::Attr(id(p)) + } + fn arith(op: Arith, l: Expr, r: Expr) -> Expr { + Expr::Arith { + op, + l: Box::new(l), + r: Box::new(r), + } + } + + /// 结论是算出来的：margin = revenue − cost。**算式读的两条读数都进前提**， + /// 所以这条结论说得出凭什么，也会在任一条读数变了的时候退场（0032） + #[test] + fn a_computed_conclusion_carries_the_readings_it_read() { + let rule = BusinessRule { + id: id(90), + conclusion: Conclusion::Computed { + predicate: id(12), + expr: arith(Arith::Sub, attr(10), attr(11)), + }, + conditions: vec![Condition { + group: 0, + predicate: id(10), + op: Op::Present, + operand: Operand::None, + }], + }; + let facts = vec![fact(1, 50, 10, json!(300.0)), fact(2, 50, 11, json!(120.0))]; + let spans = HashMap::from([(id(1), (Some(100), None)), (id(2), (Some(100), None))]); + let (hits, _) = evaluate(&[rule], &facts, &spans); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].value, Some(180.0)); + // 条件只提到 revenue，可 cost 也读了——它照样是前提 + assert_eq!(hits[0].premises.len(), 2); + assert!(hits[0].premises.contains(&id(1)) && hits[0].premises.contains(&id(2))); + } + + /// **每个组合各算各的。** 两条 revenue 一条 cost，就是两个数、两段区间、 + /// 两行结论——而不是挑一条算一次（0032 里说的「行数才是真代价」） + #[test] + fn each_combination_of_readings_computes_its_own_value() { + let rule = BusinessRule { + id: id(90), + conclusion: Conclusion::Computed { + predicate: id(12), + expr: arith(Arith::Sub, attr(10), attr(11)), + }, + conditions: vec![Condition { + group: 0, + predicate: id(10), + op: Op::Present, + operand: Operand::None, + }], + }; + let facts = vec![ + fact(1, 50, 10, json!(300.0)), + fact(2, 50, 10, json!(400.0)), + fact(3, 50, 11, json!(120.0)), + ]; + let spans = HashMap::from([ + (id(1), (Some(100), Some(200))), + (id(2), (Some(200), Some(300))), + (id(3), (Some(100), Some(300))), + ]); + let (hits, _) = evaluate(&[rule], &facts, &spans); + assert_eq!(hits.len(), 2, "两条 revenue 各算一个 margin"); + let mut values: Vec = hits.iter().filter_map(|h| h.value).collect(); + values.sort_by(f64::total_cmp); + assert_eq!(values, vec![180.0, 280.0]); + } + + /// 缺一条读数就不出结论——**不是当零**。没记不等于零，在无知的地方填一个 + /// 确定的答案正是这套账本最不该做的事 + #[test] + fn a_missing_reading_computes_nothing_rather_than_zero() { + let rule = BusinessRule { + id: id(90), + conclusion: Conclusion::Computed { + predicate: id(12), + expr: arith(Arith::Sub, attr(10), attr(11)), + }, + conditions: vec![Condition { + group: 0, + predicate: id(10), + op: Op::Present, + operand: Operand::None, + }], + }; + let facts = vec![fact(1, 50, 10, json!(300.0))]; + let spans = HashMap::from([(id(1), (Some(100), None))]); + let (hits, _) = evaluate(&[rule], &facts, &spans); + assert!(hits.is_empty(), "cost 没记，margin 就不该有"); + } + + /// 除零同样什么都不出。一条算不出来的结论与一条不成立的结论一样：没有 + #[test] + fn dividing_by_zero_computes_nothing() { + let rule = BusinessRule { + id: id(90), + conclusion: Conclusion::Computed { + predicate: id(12), + expr: arith(Arith::Div, attr(10), attr(11)), + }, + conditions: vec![Condition { + group: 0, + predicate: id(10), + op: Op::Present, + operand: Operand::None, + }], + }; + let facts = vec![fact(1, 50, 10, json!(300.0)), fact(2, 50, 11, json!(0.0))]; + let spans = HashMap::from([(id(1), (Some(100), None)), (id(2), (Some(100), None))]); + let (hits, _) = evaluate(&[rule], &facts, &spans); + assert!(hits.is_empty()); + } + + /// 门槛也能是算出来的：`revenue > cost × 1.5`。判断要等组合定下来才做得了， + /// 因为门槛读的正是这一轮选中的那条 cost + #[test] + fn a_threshold_can_be_computed_from_another_reading() { + let rule = |factor: f64| BusinessRule { + id: id(90), + conclusion: Conclusion::Typing { + class: "Healthy".into(), + }, + conditions: vec![Condition { + group: 0, + predicate: id(10), + op: Op::Gt, + operand: Operand::Calc(arith(Arith::Mul, attr(11), Expr::Const(factor))), + }], + }; + let facts = vec![fact(1, 50, 10, json!(300.0)), fact(2, 50, 11, json!(120.0))]; + let spans = HashMap::from([(id(1), (Some(100), None)), (id(2), (Some(100), None))]); + + let (hits, _) = evaluate(&[rule(1.5)], &facts, &spans); + assert_eq!(hits.len(), 1, "300 > 120 × 1.5"); + assert_eq!(hits[0].premises.len(), 2, "门槛读的那条也是前提"); + + let (none, _) = evaluate(&[rule(3.0)], &facts, &spans); + assert!(none.is_empty(), "300 不大于 120 × 3"); + } + + /// 同一区间上两个不同的值是**两行**，不是一行。去重的键带上值，否则 + /// 后算出来的那个会被当成重复丢掉 + #[test] + fn two_values_on_one_interval_are_two_hits() { + let rule = BusinessRule { + id: id(90), + conclusion: Conclusion::Computed { + predicate: id(12), + expr: attr(10), + }, + conditions: vec![Condition { + group: 0, + predicate: id(10), + op: Op::Present, + operand: Operand::None, + }], + }; + let facts = vec![fact(1, 50, 10, json!(300.0)), fact(2, 50, 10, json!(400.0))]; + let spans = HashMap::from([(id(1), (Some(100), None)), (id(2), (Some(100), None))]); + let (hits, _) = evaluate(&[rule], &facts, &spans); + assert_eq!(hits.len(), 2, "同一段区间，两个值"); + } + + /// 算式的深度算得对——编译那一侧靠它拦住「有人在这里写程序」 + #[test] + fn depth_counts_the_deepest_branch() { + assert_eq!(attr(10).depth(), 1); + assert_eq!(arith(Arith::Add, attr(10), attr(11)).depth(), 2); + assert_eq!( + arith(Arith::Div, arith(Arith::Sub, attr(10), attr(11)), attr(10)).depth(), + 3 + ); + } } diff --git a/crates/utopia-search/src/lib.rs b/crates/utopia-search/src/lib.rs index 9a30bf0af..a905506a8 100644 --- a/crates/utopia-search/src/lib.rs +++ b/crates/utopia-search/src/lib.rs @@ -183,3 +183,41 @@ pub fn rrf_fuse(lists: &[Vec], limit: usize) -> Vec { ranked.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); ranked.into_iter().take(limit).map(|(id, _)| id).collect() } + +#[cfg(test)] +mod rrf_tests { + use super::rrf_fuse; + + fn ids(list: &[&str]) -> Vec { + list.iter().map(|s| s.to_string()).collect() + } + + /// 一路召回就是那一路的顺序，融合不许重排 + #[test] + fn one_list_comes_back_in_its_own_order() { + let lists = vec![ids(&["a", "b", "c"])]; + assert_eq!(rrf_fuse(&lists, 10), ids(&["a", "b", "c"])); + } + + /// 两路都点名的排最前；只在一路里的按各自名次的倒数相加比大小： + /// a = 1/61 + 1/62，c = 1/63 + 1/61，b = 1/62 + #[test] + fn two_lists_follow_the_reciprocal_rank_arithmetic() { + let lists = vec![ids(&["a", "b", "c"]), ids(&["c", "a"])]; + assert_eq!(rrf_fuse(&lists, 10), ids(&["a", "c", "b"])); + } + + /// 空的那一路不是失败的那一路：它什么分都不加，也不影响别人 + #[test] + fn an_empty_list_contributes_nothing() { + let lists = vec![ids(&["a", "b"]), Vec::new()]; + assert_eq!(rrf_fuse(&lists, 10), ids(&["a", "b"])); + } + + /// 上限在融合之后裁，不是每路各裁 + #[test] + fn the_limit_cuts_after_fusion() { + let lists = vec![ids(&["a", "b"]), ids(&["b", "c"])]; + assert_eq!(rrf_fuse(&lists, 1), ids(&["b"])); + } +} diff --git a/crates/utopia-server/src/adjudication.rs b/crates/utopia-server/src/adjudication.rs index 13ff6eae1..9980c491d 100644 --- a/crates/utopia-server/src/adjudication.rs +++ b/crates/utopia-server/src/adjudication.rs @@ -3,20 +3,29 @@ //! 高置信 same → 自动合并（可回滚），高置信 different → 自动保持分开， //! 其余转人工。未配模型时全部转人工——本任务失败或缺席都不影响抽取与查询。 +use crate::governance::{look_again, Look}; use crate::llm_util; use crate::state::AppState; use sha2::{Digest, Sha256}; use std::collections::HashMap; +use std::sync::{Arc, LazyLock, Mutex}; use utopia_core::models::ReviewItem; use utopia_core::AppError; +use utopia_store::governance as gov; use uuid::Uuid; const BATCH_SIZE: i64 = 12; const AUTO_CONF: f32 = 0.8; const MAX_ROUNDS: usize = 20; +/// 每一百对里有几对**即使裁决器有把握也交给人**（0026）。没有这一份，人裁过的 +/// 语料会收缩成只剩难题：既不再代表一般的判法，也没有样本能量出机器与人的 +/// 一致率。按 id 取样而不是掷骰子：同一对每次跑到的答案一样，才好复现 +const HUMAN_SAMPLE_PCT: u128 = 10; -/// 缓存键：类型 + 双方名字 + 事实摘要（与实体 id 无关——重传文档不重复付费）。 -fn pair_key(item: &ReviewItem) -> String { +/// 缓存键：类型 + 双方名字 + 事实摘要 + 先例（与实体 id 无关——重传文档不重复付费）。 +/// **先例也进键**：答案随先例变（0025 说的正是这个，治理那一路因此干脆不用缓存）； +/// 人又裁了一笔，键就变，旧答案自然作废，不必去清 +fn pair_key(item: &ReviewItem, precedents: &[String]) -> String { let side = |s: &utopia_core::models::ReviewSide| { format!( "{}|{}|{}", @@ -28,11 +37,109 @@ fn pair_key(item: &ReviewItem) -> String { }; let mut sides = [side(&item.left), side(&item.right)]; sides.sort(); - let digest = Sha256::digest(sides.join("##").as_bytes()); + let digest = + Sha256::digest(format!("{}##{}", sides.join("##"), precedents.join("\n")).as_bytes()); digest.iter().map(|b| format!("{b:02x}")).collect() } +/// 这一对是不是抽给人看的那一份（见 HUMAN_SAMPLE_PCT） +fn sampled_for_a_person(item: &ReviewItem) -> bool { + item.id.as_u128() % 100 < HUMAN_SAMPLE_PCT +} + +/// 攒批没定的对要不要带工具再看一遍（0028）：没判决或把握不到线；硬规则拦得住的不看 +/// （大类不同、版本尾巴、含名字的一句话——再看也改不了规则）；有撤回的不看，那是人的事 +fn wants_another_look( + item: &ReviewItem, + p: &gov::Precedents, + same: Option, + conf: f32, +) -> bool { + let unsettled = same.is_none() || conf < AUTO_CONF; + let ruled = gov::types_conflict( + item.left.type_label.as_deref(), + item.right.type_label.as_deref(), + ) || matches!( + gov::name_shape(&item.left.name, &item.right.name), + gov::NameShape::Version | gov::NameShape::Phrase + ); + unsettled && !ruled && p.reverts.is_empty() +} + +/// 一次裁决落地成了什么：第二层的行按它记 applied 还是 proposed +enum Outcome { + Merged(Uuid), + Kept, + Escalated, +} + +/// 第二层看过的一对，记一行 `agent_decisions`（0028）：轨迹、问题、花的调用都在里面。 +/// 预算按这些行算，Agent 队列也从这里读——治理开没开，机器去看过的都看得见 +async fn record_look( + state: &AppState, + kb_id: Uuid, + run_id: Uuid, + item: &ReviewItem, + p: &gov::Precedents, + look: &Look, + outcome: &Outcome, +) -> anyhow::Result<()> { + let action = match look.same { + Some(true) => "merge", + Some(false) => "keep", + None => "unsure", + }; + let (status, merge_id) = match outcome { + Outcome::Merged(id) => ("applied", Some(*id)), + Outcome::Kept => ("applied", None), + Outcome::Escalated => ("proposed", None), + }; + gov::record( + &state.pool, + kb_id, + gov::NewDecision { + run_id, + target_id: item.id, + action, + confidence: look.conf, + reason: look.why.as_deref(), + precedents: gov::precedents_json(p), + status, + merge_id, + question: look.question.as_deref(), + trace: serde_json::Value::Array(look.trace.clone()), + calls: look.calls, + }, + ) + .await?; + Ok(()) +} + +/// 同一个库的裁决一次只跑一个（与 `ontology_index::PER_KB` 同一套理由）。 +/// +/// 入队去重只挡得住「还排着队的」；一批文档陆续抽完时，后一个入队时前一个 +/// 已经是 `running`，于是七个任务同时跑同一个库。它们各自 `pending_adjudications` +/// 拿到**同一批**待裁项：模型调用翻七倍，而两个任务对同一对下判断时，第二个 +/// 撞 `agent_decisions_open_idx` 唯一索引直接失败（2026-09-08 实测：七个任务里 +/// 两个这样挂掉）。 +/// +/// **等而不是跳过**：后到的等前一个跑完再进，进来时重新查一遍待裁项， +/// 通常只剩前一个读完之后才新增的那几对。 +static PER_KB: LazyLock>>>> = + LazyLock::new(Default::default); + +fn lock_for(kb_id: Uuid) -> Arc> { + PER_KB + .lock() + .expect("per-kb adjudication lock table poisoned") + .entry(kb_id) + .or_default() + .clone() +} + pub async fn adjudicate_entities(state: &AppState, kb_id: Uuid) -> anyhow::Result<()> { + let lock = lock_for(kb_id); + let _serial = lock.lock().await; let kb = utopia_store::kbs::get(&state.pool, kb_id).await?; let settings = utopia_store::settings::get(&state.pool, kb.workspace_id).await?; let client = settings.as_ref().and_then(llm_util::chat_client); @@ -52,6 +159,8 @@ pub async fn adjudicate_entities(state: &AppState, kb_id: Uuid) -> anyhow::Resul state.emit_review(kb_id); return Ok(()); }; + // 第二层的行都挂在这一次任务上 + let run_id = Uuid::now_v7(); for _ in 0..MAX_ROUNDS { let items = @@ -60,15 +169,18 @@ pub async fn adjudicate_entities(state: &AppState, kb_id: Uuid) -> anyhow::Resul break; } - // 第一层：裁决缓存 - let mut to_ask: Vec<(ReviewItem, String)> = Vec::new(); + // 第一层：裁决缓存。先例（人在这个库里对这些名字做过什么，连同他们写的理由） + // 先取出来：它既进提示词也进缓存键 + let mut to_ask: Vec<(ReviewItem, String, gov::Precedents, Vec)> = Vec::new(); for item in items { - let key = pair_key(&item); + let p = gov::precedents_for(&state.pool, kb_id, &item).await?; + let precedents = gov::render_lines(&p); + let key = pair_key(&item, &precedents); match utopia_store::resolution::get_verdict(&state.pool, kb_id, &key).await? { Some((same, conf)) => { - apply_verdict(state, kb_id, &item, same, conf, "cached").await? + apply_verdict(state, kb_id, &item, same, conf, "cached", None).await?; } - None => to_ask.push((item, key)), + None => to_ask.push((item, key, p, precedents)), } } if to_ask.is_empty() { @@ -78,27 +190,29 @@ pub async fn adjudicate_entities(state: &AppState, kb_id: Uuid) -> anyhow::Resul // 第二层：攒批 LLM 裁决 let pairs: Vec = to_ask .iter() - .map(|(item, _)| utopia_extract::AdjudicationPair { - left: utopia_extract::AdjudicationSide { - name: item.left.name.clone(), - type_label: item - .left - .type_label - .clone() - .unwrap_or_else(|| "untyped".into()), - facts: item.left.top_facts.clone(), + .map( + |(item, _, _, precedents)| utopia_extract::AdjudicationPair { + left: utopia_extract::AdjudicationSide { + name: item.left.name.clone(), + type_label: item + .left + .type_label + .clone() + .unwrap_or_else(|| "untyped".into()), + facts: item.left.top_facts.clone(), + }, + right: utopia_extract::AdjudicationSide { + name: item.right.name.clone(), + type_label: item + .right + .type_label + .clone() + .unwrap_or_else(|| "untyped".into()), + facts: item.right.top_facts.clone(), + }, + precedents: precedents.clone(), }, - right: utopia_extract::AdjudicationSide { - name: item.right.name.clone(), - type_label: item - .right - .type_label - .clone() - .unwrap_or_else(|| "untyped".into()), - facts: item.right.top_facts.clone(), - }, - precedents: Vec::new(), - }) + ) .collect(); let messages = utopia_extract::build_adjudication_messages(&pairs); // 调用/解析失败 → 任务按退避重试；重试耗尽后行停留在队列里，人工仍可定夺 @@ -112,7 +226,7 @@ pub async fn adjudicate_entities(state: &AppState, kb_id: Uuid) -> anyhow::Resul let by_i: HashMap = verdicts.iter().map(|v| (v.i, v)).collect(); - for (idx, (item, key)) in to_ask.iter().enumerate() { + for (idx, (item, key, p, _)) in to_ask.iter().enumerate() { match by_i.get(&idx) { Some(v) => { let same = match v.verdict.as_str() { @@ -121,6 +235,55 @@ pub async fn adjudicate_entities(state: &AppState, kb_id: Uuid) -> anyhow::Resul _ => None, }; let conf = v.confidence.unwrap_or(0.5).clamp(0.0, 1.0); + // 第二层（0028）：攒批没定的，带工具再看一遍再落地。预算用完或 + // 循环没跑成就照攒批的看法办 + if wants_another_look(item, p, same, conf) { + let earlier = Look::from_batch(same, conf, v.why.clone()); + if let Some(look) = look_again( + state, + kb_id, + &client, + &settings, + item, + &pairs[idx], + &earlier, + ) + .await + { + let outcome = if look.same.is_some() { + utopia_store::resolution::put_verdict( + &state.pool, + kb_id, + key, + look.same, + look.conf, + &model, + ) + .await?; + apply_verdict( + state, + kb_id, + item, + look.same, + look.conf, + "investigated", + look.why.as_deref(), + ) + .await? + } else { + // 问了一个问题，或看了没结论：留给人，卡片上带着建议与问题 + utopia_store::resolution::escalate_review( + &state.pool, + item.id, + "proposed", + ) + .await?; + Outcome::Escalated + }; + record_look(state, kb_id, run_id, item, p, &look, &outcome).await?; + continue; + } + } utopia_store::resolution::put_verdict( &state.pool, kb_id, @@ -130,7 +293,16 @@ pub async fn adjudicate_entities(state: &AppState, kb_id: Uuid) -> anyhow::Resul &model, ) .await?; - apply_verdict(state, kb_id, item, same, conf, "adjudicated").await?; + apply_verdict( + state, + kb_id, + item, + same, + conf, + "adjudicated", + v.why.as_deref(), + ) + .await?; } None => { utopia_store::resolution::escalate_review( @@ -148,6 +320,7 @@ pub async fn adjudicate_entities(state: &AppState, kb_id: Uuid) -> anyhow::Resul Ok(()) } +#[allow(clippy::too_many_arguments)] async fn apply_verdict( state: &AppState, kb_id: Uuid, @@ -155,9 +328,44 @@ async fn apply_verdict( same: Option, conf: f32, via: &str, -) -> anyhow::Result<()> { - match same { + why: Option<&str>, +) -> anyhow::Result { + // 抽给人的那一份：机器有把握也不动手（0026）。理由写进队列那一列， + // 界面会说"这一对是抽样给你的"，而不是让人以为裁决器没把握 + if same.is_some() && conf >= AUTO_CONF && sampled_for_a_person(item) { + let verdict = if same == Some(true) { + "same" + } else { + "different" + }; + utopia_store::resolution::escalate_review( + &state.pool, + item.id, + &format!("escalate_sample|{verdict} {conf:.2}"), + ) + .await?; + return Ok(Outcome::Escalated); + } + let outcome = match same { Some(true) if conf >= AUTO_CONF => { + // 执行闸门（0027）：合并会立刻送出图外的东西——违规、派生、答案——留给人， + // 把握再高也不动手。人看到的是留下的原因，不是「裁决器没把握」 + let impact = utopia_store::execution_gate::impact_of( + &state.pool, + kb_id, + item.left.id, + item.right.id, + ) + .await?; + if let Some(hold) = utopia_store::execution_gate::hold(&impact) { + utopia_store::resolution::escalate_review( + &state.pool, + item.id, + &format!("escalate_impact|{hold}"), + ) + .await?; + return Ok(Outcome::Escalated); + } let (target, source) = utopia_store::resolution::merge_direction(&state.pool, item.left.id, item.right.id) .await?; @@ -172,7 +380,7 @@ async fn apply_verdict( ) .await { - Ok(_) => { + Ok(merge_id) => { utopia_store::resolution::close_review_auto( &state.pool, item.id, @@ -191,9 +399,13 @@ async fn apply_verdict( serde_json::json!({ "left": item.left.name, "right": item.right.name, "score": item.score, "confidence": conf, "via": via, + // 模型的一句理由也留下：机器的行不是先例（0025 决定 1）， + // 但人看 Decisions 时该看得见它凭什么 + "why": why, }), ) .await; + Outcome::Merged(merge_id) } // 同批次连锁合并可能已吞掉其中一方：转人工而不是让任务失败 Err(AppError::Conflict(_)) | Err(AppError::NotFound) => { @@ -203,6 +415,7 @@ async fn apply_verdict( "escalate_entity_changed", ) .await?; + Outcome::Escalated } Err(e) => return Err(e.into()), } @@ -225,9 +438,11 @@ async fn apply_verdict( serde_json::json!({ "left": item.left.name, "right": item.right.name, "score": item.score, "confidence": conf, "via": via, + "why": why, }), ) .await; + Outcome::Kept } _ => { utopia_store::resolution::escalate_review( @@ -236,7 +451,8 @@ async fn apply_verdict( &format!("escalate_unsure|{via} {conf:.2}"), ) .await?; + Outcome::Escalated } - } - Ok(()) + }; + Ok(outcome) } diff --git a/crates/utopia-server/src/api/auth_routes.rs b/crates/utopia-server/src/api/auth_routes.rs index a9a532d46..464a99120 100644 --- a/crates/utopia-server/src/api/auth_routes.rs +++ b/crates/utopia-server/src/api/auth_routes.rs @@ -7,7 +7,6 @@ use serde_json::json; use utopia_core::models::User; use utopia_core::AppError; -use super::kbs::{install_packs, DEFAULT_PACK}; use crate::auth::{self, AuthUser}; use crate::error::ApiResult; use crate::state::AppState; @@ -66,7 +65,7 @@ pub async fn register( let utopia_store::accounts::Registered { user, workspace, - general_kb, + general_kb: _, } = utopia_store::accounts::register( &state.pool, req.email.trim(), @@ -77,23 +76,11 @@ pub async fn register( ) .await?; - /* 首个用户那个 General 库也要有词汇表（#322）。 - 建库对话框里 schema.org 是预勾选的默认（0008、0009），而这条路径绕过了 - 对话框——于是每个部署的第一个库、也就是新用户落地的那个，本体是空的： - 没有 domain/range，抽取的方向就定不住，事实退化成 related_to， - 而这正是 0008 装包要解决的事。 - - **装不上不能挡住注册。** 事务已经提交，账号已经存在；这一步失败只是 - 回到今天的行为（库空着，日后手动装包再重跑类型消解即可，见 0009）， - 让人注册不进来则是把小事变成大事 */ - if let Some(kb_id) = general_kb { - if install_packs(&state, kb_id, user.id, &[DEFAULT_PACK.to_string()]) - .await - .is_err() - { - tracing::warn!(kb_id = %kb_id, pack = DEFAULT_PACK, "默认库的冷启动本体包没装上"); - } - } + // 首个用户那个 General 库**空着起步**（#580）。从前这里给它装 schema.org（#322）， + // 理由是没有 domain/range 事实就没方向；量过之后（#580 的对照）：包给的是实体 + // 类型，不是谓词——装了包八成事实照样没谓词；空库靠 bootstrap 从文档里长出 + // 十几个类、上百条关系，都是语料自己的说法，每块的提示词从 18k tokens 回到 2k。 + // 包还在建库对话框里，要的人一键装 let token = auth::issue_token(&state, user.id)?; let secure = auth::behind_tls(&headers, state.cookie_secure); diff --git a/crates/utopia-server/src/api/chat.rs b/crates/utopia-server/src/api/chat.rs index db1941daa..d582b5164 100644 --- a/crates/utopia-server/src/api/chat.rs +++ b/crates/utopia-server/src/api/chat.rs @@ -29,6 +29,66 @@ const KNOWN_ENTITY_LIMIT: usize = 20; const MAX_HISTORY: usize = 20; const MAX_ROUNDS: usize = 6; +/// 模型说「我去查」却一个工具都没调就结束了一轮时，追问的那句话（#509）。 +/// +/// 实测 DeepSeek-V3 会答「请稍等，我将进行相关搜索」然后什么都不做；再问一次， +/// 还是「稍等，我正在搜索」。提示词第 1、2 条早就禁了这种叙述，所以这不是缺一句 +/// 指令，是模型没听、而循环把它当成了最终答案。守卫放在循环里，不放在提示词里。 +/// +/// 追问只给一次、不流式、只认三种回复：调工具（照常执行）、一个词 DONE（上一句 +/// 本来就是答案：打招呼、问这场对话、拒答，原样收尾）、其他任何文字（还在说空话， +/// 明说没查到证据）。给它 DONE 这条出口，是为了不让「把那句话说短一点」这种 +/// 本就不需要工具的回答被追问成第二个答案。 +/// +/// 措辞把 DONE 的门开得窄：只有问题**不是关于用户数据**时才许说 DONE。实测还有 +/// 一种更坏的停法——不说「稍等」，直接写「以下是我找到的内容」然后凭记忆作答， +/// 库里 steps、sources 全是 0。它没停住，它在撒谎。对它，「若已答完就说 DONE」 +/// 是一条太宽的出口，所以这里明说：关于数据的事实性回答没有工具就不算答。 +const STALL_NUDGE: &str = "(system) Your last message ended the turn without calling any tool, \ + and it cites nothing. An answer about the user's data that was not gathered with a tool is \ + not an answer, whatever the message says it found: call the tool now. Do not describe a \ + plan. Reply with the single word DONE only if the question was not about the user's data \ + at all: a greeting, a question about this transcript, or a refusal."; + +/// 追问后仍不查时补在答案末尾的话。承诺已经流给用户了，收不回来；能做的是 +/// 让文字和空白的轨迹不再互相矛盾——对一个把「每个回答可追溯」当卖点的产品， +/// 一句叙述了从未发生的查证的回答比「不知道」更糟。 +const NO_EVIDENCE_NOTE: &str = + "\n\n(No evidence was gathered for this answer: the model announced a search it did not perform.)"; + +/// 一轮结束、正文非空、整场没调过工具也没有引用：这个答案什么都不站在上面。 +/// 「问这场对话」的消息也满足这两条，所以追问必须便宜、安静，且留有 DONE 出口。 +fn answer_rests_on_nothing(steps: &[serde_json::Value], sources: &[serde_json::Value]) -> bool { + steps.is_empty() && sources.is_empty() +} + +/// 追问之后模型的回复算哪种 +#[derive(Debug, PartialEq, Eq)] +enum AfterNudge { + /// 调了工具：照常执行，接着走 + Tools, + /// 说上一句已经是答案，或者什么都没说：原样收尾 + Done, + /// 又是一段文字：还在说空话 + Stalled, +} + +fn after_nudge(turn: &utopia_llm::AssistantTurn) -> AfterNudge { + if !turn.tool_calls.is_empty() { + return AfterNudge::Tools; + } + let said = turn + .content + .as_deref() + .map(|t| t.trim().trim_matches(|c: char| !c.is_alphanumeric())) + .unwrap_or_default(); + if said.is_empty() || said.eq_ignore_ascii_case("done") { + AfterNudge::Done + } else { + AfterNudge::Stalled + } +} + /// `remember` 曾整个停用过一段（见 `docs/decisions/0015`）：它那时会把一句话直接 /// 变成图上一条活边，实测里「记住 Acme 把总部搬到了深圳」落成的是一条**空谓词、 /// 0.9 置信**的边，而助手宣称的和图里得到的不是一回事。 @@ -139,7 +199,7 @@ pub(super) fn tools_schema(can_write: bool, data_source_names: &[String]) -> ser /// /// 判据直接取自工具表里的 `required`：加一个必填参数，这里自动跟上， /// 不必记得来改第二处。 -fn check_call( +pub(super) fn check_call( tools: &serde_json::Value, name: &str, raw_args: &str, @@ -306,11 +366,18 @@ pub(super) fn base_tools() -> serde_json::Value { always do this for \"who was X in \" questions. \ Conclusions a business rule reached about this entity come back too, \ marked `[rule: ]` with the readings that made them true — take \ - those as given rather than re-deriving them from the readings yourself.", + those as given rather than re-deriving them from the readings yourself. \ + Output is grouped by predicate and cut at `limit`; narrow with predicate, \ + object_type, since / until, or use timeline / neighbors / paths_between.", "parameters": { "type": "object", "properties": { - "entity_id": { "type": "string", "description": "Entity id (uuid) from find_entities." }, + "entity_id": { "type": "string", "description": "Entity id (uuid) from find_entities, or an entity name (the server picks the best match and says which)." }, + "predicate": { "type": "string", "description": "Optional: only facts whose relation name contains this (e.g. 'founder')." }, + "object_type": { "type": "string", "description": "Optional: only facts whose other end is of this type (e.g. 'Person')." }, + "since": { "type": "string", "description": "Optional WORLD-time window start: facts still holding after it." }, + "until": { "type": "string", "description": "Optional WORLD-time window end: facts that had started by it." }, + "limit": { "type": "integer", "description": "How many facts to return (default 80, max 300); the reply says when it is cut." }, "at": { "type": "string", "description": "Optional as-of moment on the WORLD axis (YYYY, YYYY-MM, \ @@ -330,6 +397,62 @@ pub(super) fn base_tools() -> serde_json::Value { } } }, + { + "type": "function", + "function": { + "name": "paths_between", + "description": "How two entities are connected in the knowledge graph: the chains of facts that join them, up to 3 hops, shortest first. THE tool for 'what is the relation between A and B', 'how is X linked to Y', 'who connects A and B'. Both ends take an entity name or an id; with a name the server picks the best match and says which. Pass `at` to require every edge to hold at that moment (world time), `as_of` to read the base as recorded then. Each edge comes with its validity range.", + "parameters": { + "type": "object", + "properties": { + "from": { "type": "string", "description": "One end: entity name, or an id from find_entities." }, + "to": { "type": "string", "description": "The other end: entity name, or an id." }, + "max_hops": { "type": "integer", "description": "Longest chain to consider, 1-3 (default 3)." }, + "at": { "type": "string", "description": "Optional WORLD-time moment (YYYY, YYYY-MM, YYYY-MM-DD): every edge on a path must hold then." }, + "as_of": { "type": "string", "description": "Optional RECORD-time moment: the paths as the base held them then." } + }, + "required": ["from", "to"] + } + } + }, + { + "type": "function", + "function": { + "name": "neighbors", + "description": "The entities linked to one entity, grouped by predicate, one hop. Use it to see what surrounds an entity before deciding where to look next (each further hop is another call); narrow with `predicate` (e.g. 'founder', 'employee') or `object_type` (e.g. 'Person'). Attribute values are not neighbors; entity_facts has them.", + "parameters": { + "type": "object", + "properties": { + "entity": { "type": "string", "description": "Entity name, or an id from find_entities." }, + "predicate": { "type": "string", "description": "Optional: only relations whose name contains this." }, + "object_type": { "type": "string", "description": "Optional: only neighbors of this type (name contains)." }, + "at": { "type": "string", "description": "Optional WORLD-time moment: only links valid then." }, + "as_of": { "type": "string", "description": "Optional RECORD-time moment." }, + "limit": { "type": "integer", "description": "How many to return (default 40, max 300)." } + }, + "required": ["entity"] + } + } + }, + { + "type": "function", + "function": { + "name": "timeline", + "description": "One entity's dated facts in world-time order: THE tool for 'the timeline of X', 'the history of X', 'what happened to X between and '. Only facts with a stated date appear; narrow with `since` / `until` (world time) or `predicate`.", + "parameters": { + "type": "object", + "properties": { + "entity": { "type": "string", "description": "Entity name, or an id from find_entities." }, + "since": { "type": "string", "description": "Optional start of the window (YYYY, YYYY-MM, YYYY-MM-DD)." }, + "until": { "type": "string", "description": "Optional end of the window." }, + "predicate": { "type": "string", "description": "Optional: only relations whose name contains this." }, + "as_of": { "type": "string", "description": "Optional RECORD-time moment: the timeline as the base held it then." }, + "limit": { "type": "integer", "description": "How many to return (default 60, max 300)." } + }, + "required": ["entity"] + } + } + }, { "type": "function", "function": { @@ -391,8 +514,9 @@ pub(super) fn base_tools() -> serde_json::Value { const SYSTEM_PROMPT: &str = "You are the assistant of Utopia, a temporal knowledge platform. \ You have tools: search_chunks (document search) and get_document (the full text of one \ - document found by search), find_entities, entity_facts and changes (a bi-temporal \ - knowledge graph), and search_docs (Utopia's own manual, the Charter).\n\ + document found by search), find_entities, entity_facts, neighbors, timeline, \ + paths_between and changes (a bi-temporal knowledge graph), and search_docs (Utopia's \ + own manual, the Charter).\n\ search_chunks returns short excerpts of the best-matching sections only. When a hit is \ clearly the right document but the excerpt does not carry the answer, read the whole \ document with get_document before saying the knowledge base does not have it.\n\ @@ -415,16 +539,17 @@ const SYSTEM_PROMPT: &str = "You are the assistant of Utopia, a temporal knowled preamble about what you are or are not looking up. Everything below is for messages about \ the user's data.\n\ 1. For factual questions — questions about the user's data, never one about this \ - conversation — ALWAYS gather evidence with tools before answering. Prefer the \ - graph tools for questions about people/organizations/projects and time (\"who was X \ - when\", \"what changed\"), search_chunks for content and detail questions. Combine both \ - when useful.\n\ + conversation — ALWAYS gather evidence with tools before answering. For how two \ + things are related, call paths_between (names are fine); for the history of one \ + thing, timeline; to see what is linked to it, neighbors; for its facts, entity_facts, \ + narrowed with predicate / object_type / since / until. search_chunks answers content \ + and detail questions. Combine both when useful.\n\ 2. Facts carry validity ranges (from → to). For \"as of \" questions pass `at` to \ entity_facts and the server filters to that moment; for history questions omit `at` \ to see the full timeline. For 'what did we know / have on record / believe as of ' or 'before arrived' pass `as_of` — that is the record axis and the ONLY way to answer such a question; do not narrate a plan, call the tool. The two combine: `at` for the date asked about, `as_of` for when. State dates in the answer. Dates in tool output carry their own precision: \ `2023` means the year and `2023-06` the month — never turn them into a specific day; \ - `attested