diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rs index 26e624df..a739a6ea 100644 --- a/crates/utopia-server/src/extraction.rs +++ b/crates/utopia-server/src/extraction.rs @@ -1446,8 +1446,12 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow: } } - // 主宾未在 entities 中声明时先建出来(模型偶尔漏报)。没有 entities 那条 - // 记录就没有类型可依,留空即可——0009 之前这里只能塞 concept + // 主宾没在 entities 里声明时(模型偶尔漏报):库里已经有这个名字的就用它, + // 库里也没有的**不建**(#559)。从前这里一律先建出来、类型留空,结果 + // 一个库里 20% 的实体是 "lawsuit against OpenAI"、"$5 billion"、"March 2024" + // 这样的描述——几乎全部只当过宾语,从没当过主语。漏报的实体多半在 + // 前面的分块或别的文档里已经声明过,按名字找得到;找不到的就是描述。 + // 描述做主语的事实丢掉并记账,做宾语的事实照落,宾语落成字面值 let subject_id = match f.subject_ref.as_deref().map(str::trim) { Some(handle) => match referenced_entity(&ref_entities, handle) { Some(bound) => bound.id, @@ -1465,7 +1469,29 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow: } }, None => { - match no_ref_name_binding(&entity_ids, &handled_by_name, f.subject.trim()) { + let binding = + no_ref_name_binding(&entity_ids, &handled_by_name, f.subject.trim()); + if matches!(binding, NoRefNameBinding::Missing) + && utopia_store::resolution::existing_by_name( + &state.pool, + doc.kb_id, + f.subject.trim(), + ) + .await? + .is_none() + { + drop_signal( + state, + doc.kb_id, + document_id, + utopia_store::extraction_drops::reason::SUBJECT_NOT_DECLARED, + &f.predicate, + Some(f.subject.trim()), + ) + .await; + continue; + } + match binding { NoRefNameBinding::Legacy(id) => id, NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => { touched_names.insert( @@ -1506,28 +1532,117 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow: continue; } }, - None => match no_ref_name_binding(&entity_ids, &handled_by_name, object_name) { - NoRefNameBinding::Legacy(id) => id, - NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => { - touched_names.insert( - utopia_store::resolution::normalize_name(object_name).to_lowercase(), - ); - resolve_bare( + None => { + let binding = no_ref_name_binding(&entity_ids, &handled_by_name, object_name); + if matches!(binding, NoRefNameBinding::Missing) + && utopia_store::resolution::existing_by_name( &state.pool, doc.kb_id, - None, object_name, - ctx, - Some(&chunk.text), - &mut doc_cache, - &handled_by_name, - &mut ambiguous_bare_cache, - &mut needs_adjudication, - &mut human_reviews_found, ) .await? + .is_none() + { + // 没声明、库里也没有:宾语落成字面值。谓词照旧对本体; + // 被动形(`_by`)本该主宾对调,而字面值当不了主语,那条就不给谓词, + // 原词留在证据上 + let predicate_id = match pred_index.lookup(f.predicate.as_str()) { + Some((id, false)) => Some(id), + Some((_, true)) => None, + None => { + let _ = utopia_store::ontology::record_miss( + &state.pool, + doc.kb_id, + "relation_type", + &f.predicate, + Some(&format!("{} → {}", f.subject, object_name)), + ) + .await; + None + } + }; + drop_signal( + state, + doc.kb_id, + document_id, + utopia_store::extraction_drops::reason::OBJECT_UNDECLARED, + &f.predicate, + Some(object_name), + ) + .await; + let literal = serde_json::json!({ "value": object_name }); + if await_nod { + if let utopia_store::pending::Outcome::Proposed(_) = + utopia_store::pending::propose( + &state.pool, + utopia_store::pending::Proposal { + kb_id: doc.kb_id, + subject_id, + predicate_id, + object_id: None, + object_value: Some(&literal), + proposed_predicate: Some(f.predicate.as_str()), + validity, + confidence, + chunk_id: chunk.id, + proposed_by: proposer.user_id, + proposed_token: proposer.token_id, + }, + ) + .await? + { + pending_count += 1; + } + continue; + } + let (fact_id, created) = utopia_store::graph::insert_value_fact( + &state.pool, + doc.kb_id, + subject_id, + predicate_id, + &literal, + validity, + confidence, + ) + .await?; + touched_facts.push(fact_id); + utopia_store::graph::add_evidence( + &state.pool, + fact_id, + chunk.id, + f.quote.as_deref(), + Some(f.predicate.as_str()), + ) + .await?; + if created { + fact_count += 1; + } + continue; } - }, + match binding { + NoRefNameBinding::Legacy(id) => id, + NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => { + touched_names.insert( + utopia_store::resolution::normalize_name(object_name) + .to_lowercase(), + ); + resolve_bare( + &state.pool, + doc.kb_id, + None, + object_name, + ctx, + Some(&chunk.text), + &mut doc_cache, + &handled_by_name, + &mut ambiguous_bare_cache, + &mut needs_adjudication, + &mut human_reviews_found, + ) + .await? + } + } + } }; if subject_id == object_id { continue; diff --git a/crates/utopia-store/src/extraction_drops.rs b/crates/utopia-store/src/extraction_drops.rs index ce6c2a33..770d20c1 100644 --- a/crates/utopia-store/src/extraction_drops.rs +++ b/crates/utopia-store/src/extraction_drops.rs @@ -42,6 +42,10 @@ pub mod reason { /// 守卫放行了、结构却像从句(限定词起头的长串、句中的关系词)。**只记不挡**: /// 实体照常落库,例句留下来——#193 要的是一份跨语料的标注集,再决定哪条升成硬规则 pub const CLAUSE_SUSPECT: &str = "clause_suspect"; + /// 宾语既没在 entities 里声明、库里也没有叫这个名字的东西:事实照落, + /// 宾语落成字面值而不是节点(#559)。记下来是为了量:这一类里有多少 + /// 本该是实体(模型漏报),有多少本来就是描述 + pub const OBJECT_UNDECLARED: &str = "object_undeclared"; } pub async fn record( diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rs index f29ad63a..4d86bd41 100644 --- a/crates/utopia-store/src/resolution.rs +++ b/crates/utopia-store/src/resolution.rs @@ -1001,6 +1001,35 @@ async fn update_profile(pool: &PgPool, id: Uuid, n: i32, ctx: &[f32]) -> AppResu /// /// 没有谓词的事实(`predicate_id IS NULL`,0010)不参与:原话留在证据里, /// 不是本体承认的说法,不该被当成一个人的身份写进后缀。 +/// 库里有没有叫这个名字的实体,**不问类型**、并掉的不算。 +/// +/// 抽取用它判断一个没声明的主宾是不是已知的东西(#559):模型偶尔漏报一个实体 +/// 却在事实里用了它,那时库里多半已经有它;库里也没有的,就不是漏报,是一个 +/// 描述("lawsuit against OpenAI"),不该成节点。同名多个时取事实最多的那个—— +/// 这里只回答「有没有」,谁是谁交给消解 +pub async fn existing_by_name( + pool: &PgPool, + kb_id: Uuid, + raw_name: &str, +) -> AppResult> { + let name = normalize_name(raw_name); + let keys = recall_keys(&name); + let id: Option = sqlx::query_scalar( + "SELECT e.id FROM entities e + WHERE e.kb_id = $1 AND e.merged_into IS NULL + AND (lower(e.canonical_name) = ANY($2) + OR EXISTS (SELECT 1 FROM unnest(e.aliases) a WHERE lower(a) = ANY($2))) + ORDER BY (SELECT count(*) FROM facts f + WHERE f.subject_id = e.id OR f.object_id = e.id) DESC, e.created_at + LIMIT 1", + ) + .bind(kb_id) + .bind(&keys) + .fetch_optional(pool) + .await?; + Ok(id) +} + pub async fn refresh_disambiguators(pool: &PgPool, kb_id: Uuid, name: &str) -> AppResult<()> { let group: Vec<(Uuid,)> = sqlx::query_as( "SELECT id FROM entities diff --git a/crates/utopia-store/tests/an_undeclared_name_is_looked_up.rs b/crates/utopia-store/tests/an_undeclared_name_is_looked_up.rs new file mode 100644 index 00000000..21d0f864 --- /dev/null +++ b/crates/utopia-store/tests/an_undeclared_name_is_looked_up.rs @@ -0,0 +1,104 @@ +//! `existing_by_name`(#559):一个没声明的主宾是不是库里已经有的东西。 +//! 不问类型,认别名,并掉的不算;没有就是没有。 + +use sqlx::PgPool; +use uuid::Uuid; + +struct Fixture { + org: Uuid, + kb: Uuid, + acme: Uuid, + merged: Uuid, +} + +async fn seed(pool: &PgPool) -> anyhow::Result { + let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + let (organization, acme, merged) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'lookup-test')") + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'lookup-test')") + .bind(ws) + .bind(org) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'lookup-test')", + ) + .bind(kb) + .bind(ws) + .execute(pool) + .await?; + sqlx::query("INSERT INTO entity_types (id, kb_id, key, label) VALUES ($1, $2, 'organization', 'Organization')") + .bind(organization) + .bind(kb) + .execute(pool) + .await?; + // 有类型的、带别名的一个;并掉的一个(同名,不该被找到) + sqlx::query( + "INSERT INTO entities (id, kb_id, type_id, canonical_name, aliases) + VALUES ($1, $2, $3, 'Acme Corporation', ARRAY['ACME'])", + ) + .bind(acme) + .bind(kb) + .bind(organization) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO entities (id, kb_id, canonical_name, merged_into) VALUES ($1, $2, 'Nova Labs', $3)", + ) + .bind(merged) + .bind(kb) + .bind(acme) + .execute(pool) + .await?; + Ok(Fixture { + org, + kb, + acme, + merged, + }) +} + +async fn teardown(pool: &PgPool, f: &Fixture) -> anyhow::Result<()> { + sqlx::query("DELETE FROM knowledge_bases WHERE id = $1") + .bind(f.kb) + .execute(pool) + .await?; + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(f.org) + .execute(pool) + .await?; + Ok(()) +} + +#[tokio::test] +async fn a_known_name_is_found_by_any_spelling_and_an_unknown_one_is_not() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let f = seed(&pool).await?; + let run = async { + let find = |name: &'static str| { + let pool = pool.clone(); + async move { utopia_store::resolution::existing_by_name(&pool, f.kb, name).await } + }; + assert_eq!(find("Acme Corporation").await?, Some(f.acme), "原名"); + assert_eq!(find("acme corporation").await?, Some(f.acme), "大小写不论"); + assert_eq!(find("ACME").await?, Some(f.acme), "别名也算"); + assert_eq!( + find("Acme").await?, + Some(f.acme), + "泛用后缀词干互推:Acme ↔ Acme Corporation" + ); + assert_eq!(find("Nova Labs").await?, None, "并掉的不算"); + assert_eq!(find("lawsuit against Acme").await?, None, "描述不是实体"); + let _ = f.merged; + anyhow::Ok(()) + } + .await; + teardown(&pool, &f).await?; + run +}