diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index f3968729..391ec679 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -983,6 +983,14 @@ pub struct KnowledgeBase { /// 上次推完的时间。**答的是「上次看过没有」，不是「上次改过没有」** pub last_inference_at: Option>, pub ontology_lang: String, + /// 探索从 schema 写的数据描述：一行是什么、键、单位、码值、时间轴、相似列。 + /// 只写 schema 说了的；每次探索重写 + pub data_description: Option, + /// 探索拿不准、需要库的主人答的问题（JSON 字符串数组） + pub data_questions: serde_json::Value, + /// 人写的约定：「测试单不算数」「有效订单是 2/3/4」这类 schema 里没有的规则。 + /// 探索不碰它——量过：宽表语料上问数没有约定 2/18，有 14/18（#520） + pub data_conventions: Option, pub created_at: DateTime, pub updated_at: DateTime, } diff --git a/crates/utopia-server/src/api/chat.rs b/crates/utopia-server/src/api/chat.rs index a12121a0..58a9b00c 100644 --- a/crates/utopia-server/src/api/chat.rs +++ b/crates/utopia-server/src/api/chat.rs @@ -625,6 +625,11 @@ pub async fn chat( ) .await?; let workspace_id = kb.workspace_id; + // 数据描述（探索从 schema 写的）与约定（人写的）跟着进 system prompt。 + // **每次都在，不靠检索碰运气**：约定写成一页文档只靠检索也到过 14/18， + // 但那是因为这批问题都在问指标才每题命中（#520） + let data_description = kb.data_description.clone().filter(|s| !s.trim().is_empty()); + let data_conventions = kb.data_conventions.clone().filter(|s| !s.trim().is_empty()); // 注册表在生成器之前取出来：下面那个 `async_stream!` 会把 `state` 整个搬走 let live = state.live.clone(); @@ -657,6 +662,19 @@ pub async fn chat( first, then query. State units and the time range you used in the answer.", ds_names.join(", ") )); + // 描述说的是 schema 里有的（粒度、单位、码值、时间轴），约定说的是 schema 里 + // 没有的（哪些行算数、哪列才是那个数）。后者是问数从 2/18 到 14/18 的那一半 + if let Some(d) = &data_description { + system_prompt.push_str(&format!( + "\nAbout the data (written from the schema; states only what the schema says):\n{d}" + )); + } + if let Some(c) = &data_conventions { + system_prompt.push_str(&format!( + "\nConventions stated by the owner of this base — apply them in every query \ + and every answer (filters, units, which column is the figure):\n{c}" + )); + } if !mappings.is_empty() { system_prompt.push_str( "\nSemantic layer (confirmed definitions — use these instead of guessing from schema):", diff --git a/crates/utopia-server/src/api/kbs.rs b/crates/utopia-server/src/api/kbs.rs index cb75f91d..95d2ad07 100644 --- a/crates/utopia-server/src/api/kbs.rs +++ b/crates/utopia-server/src/api/kbs.rs @@ -51,6 +51,10 @@ pub struct UpdateKbReq { /// 打开那一刻就排一轮，关掉后任务在两簇之间看到就停。见 docs/decisions/0025 #[serde(default)] pub governance: Option, + /// 人写的数据约定（「测试单不算数」这类 schema 里没有的规则），问数与探索的 + /// 提示词都读它。探索生成的描述在另一个字段，PATCH 不了。见 #570 + #[serde(default)] + pub data_conventions: Option, } /// 用户可见的 KB 列表（restricted 库仅矩阵成员与系统管理员可见）。 @@ -110,6 +114,7 @@ pub async fn create( None, None, None, + None, ) .await?; } @@ -191,6 +196,7 @@ pub async fn update( req.inference_interval_minutes, req.auto_type_resolution, req.governance, + req.data_conventions.as_deref().map(str::trim), ) .await?; // 打开开关就自动开始处理：排一轮，同库已排着的不重复 @@ -206,7 +212,13 @@ pub async fn update( "kb.updated", "kb", Some(id), - json!({ "name": req.name, "visibility": req.visibility, "governance": req.governance }), + // 约定改了要留痕：那段文字进每一次问数的提示词，谁什么时候改过得查得到 + json!({ + "name": req.name, + "visibility": req.visibility, + "governance": req.governance, + "data_conventions": req.data_conventions.is_some(), + }), ) .await; Ok(Json(kb)) diff --git a/crates/utopia-server/src/mappings.rs b/crates/utopia-server/src/mappings.rs index 5a6c5bc7..2e7163c8 100644 --- a/crates/utopia-server/src/mappings.rs +++ b/crates/utopia-server/src/mappings.rs @@ -31,6 +31,98 @@ fn proposal_cap(tables: i32) -> i32 { (tables * 3).clamp(12, 60) } +/// 模型回的 JSON 常裹着代码栅栏；剥掉它。 +fn json_body(reply: &str) -> &str { + reply + .trim() + .trim_start_matches("```json") + .trim_start_matches("```") + .trim_end_matches("```") + .trim() +} + +/// 一轮描述最多留几个问题。多了没人答；八个已经是一页 +const MAX_DATA_QUESTIONS: usize = 8; + +/// 模型回的 `{"description": …, "questions": […]}`。描述空的整个不要—— +/// 一个空描述盖掉上一轮的好描述，比没写更糟 +fn parse_description(reply: &str) -> Option<(String, Vec)> { + let v: serde_json::Value = serde_json::from_str(json_body(reply)).ok()?; + let description = v["description"].as_str()?.trim().to_string(); + if description.is_empty() { + return None; + } + let questions = v["questions"] + .as_array() + .map(|qs| { + qs.iter() + .filter_map(|q| q.as_str()) + .map(str::trim) + .filter(|q| !q.is_empty()) + .map(str::to_string) + .take(MAX_DATA_QUESTIONS) + .collect() + }) + .unwrap_or_default(); + Some((description, questions)) +} + +/// 探索写库的数据描述，并列出它拿不准的地方（#570）。 +/// +/// **两半，两个来路。** schema 与注释说了的——一行是什么、键、单位、码值、时间轴、 +/// 哪两列长得像——探索读得出来，这里写；schema 没说的——测试单不算数、有效订单是 +/// 2/3/4、GMV 用实付不用优惠前——探索生成不了，**而且不能猜**：猜出来的约定进了 +/// 提示词，问数会照着算，比没有更糟。所以拿不准的写成问题，人答，答案进 +/// `data_conventions`，这里不碰那一列。 +/// +/// 语言跟本体语言走（0004：生成的文字跟语料走，不跟界面走）。 +async fn describe_data( + state: &AppState, + client: &utopia_llm::LlmClient, + settings: &utopia_core::models::LlmSettings, + kb: &utopia_core::models::KnowledgeBase, + schema_txt: &str, +) -> anyhow::Result<()> { + let lang = if kb.ontology_lang == "zh" { + "Chinese" + } else { + "English" + }; + let prompt = format!( + "You are documenting a database for analysts who will ask questions about it in plain \ + language. Reply with ONLY a JSON object {{\"description\": \"...\", \"questions\": [\"...\"]}}.\n\ + description ({lang}, 10 to 20 short lines): what each table is a table of and what one \ + row is; which columns are keys; the unit of money and quantity columns ONLY where a \ + comment states it; what status or code values mean ONLY where a comment states it; \ + which columns are the time axis; which columns look alike and how they differ (gross vs \ + net, list vs paid). State only what the schema and its comments say. Do not invent \ + business rules, thresholds, or which rows count.\n\ + questions ({lang}, at most {MAX_DATA_QUESTIONS}): the conventions you would need to \ + compute business figures correctly but the schema does not state, each phrased so the \ + owner of the data can answer in one line — which status values count, whether flagged \ + rows (test, internal, gift) are excluded, which of two similar amount columns is the \ + figure, units where no comment states them, whether net figures subtract refunds. Ask \ + only what the schema leaves open.\n\ + Schemas:\n{schema_txt}" + ); + let _permit = llm_util::acquire_chat(state, settings).await; + let reply = client + .chat(&[utopia_llm::ChatMessage { + role: "user".into(), + content: prompt, + }]) + .await?; + let Some((description, questions)) = parse_description(&reply) else { + anyhow::bail!( + "description reply did not parse: {}", + reply.chars().take(120).collect::() + ); + }; + utopia_store::kbs::set_data_description(&state.pool, kb.id, &description, &questions).await?; + tracing::info!(kb_id = %kb.id, lines = description.lines().count(), questions = questions.len(), "数据描述已写"); + Ok(()) +} + /// 探索把 schema 里的量与维度落成 Metric / Dimension 实体，而这两个类不在任何 /// 内置本体包里——0009 之后建库不再自带类。没有它们，下面的 `type_id` 查不到， /// 每条提议都被 `continue` 吞掉，页面只说"已排队"就再无下文（#223）。 @@ -148,6 +240,13 @@ async fn explore(state: &AppState, kb_id: Uuid, run: Uuid) -> anyhow::Result<()> ) .await; + // 先写库的数据描述与拿不准的问题，再提口径。**描述不依赖提议成不成**： + // 提议那一步解析失败整轮报错，描述已经落下了；反过来描述写不出来也不该 + // 拖垮提议——它是顺手的，warn 一句继续 + if let Err(e) = describe_data(state, &client, &settings, &kb, &schema_txt).await { + tracing::warn!(%kb_id, error = %e, "数据描述没写成，提议照常"); + } + // 既有概念（供归并复用，避免重复起名） let existing: Vec<(String,)> = sqlx::query_as( "SELECT e.canonical_name FROM entities e @@ -160,10 +259,20 @@ async fn explore(state: &AppState, kb_id: Uuid, run: Uuid) -> anyhow::Result<()> .await?; let existing_names: Vec = existing.into_iter().map(|(n,)| n).collect(); + // 人写的约定进提议的提示词。schema 里没有「测试单不算数」这句话，探索在宽表上 + // 0/18 正是因为它；有了这句，一条提议才可能长出 FILTER (WHERE is_test = 0)（#570） + let conventions = kb + .data_conventions + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + .unwrap_or("(none stated)"); let prompt = format!( "You are building the semantic layer of a BI system. Given database schemas, propose \ business concepts a user would ask about, each mapped to a concrete definition.\n\ Existing concepts (reuse these names when the meaning matches): {}\n\ + Conventions stated by the owner of this base — apply them in every definition, as \ + filters, unit conversions, or the choice of column: {conventions}\n\ Schemas:\n{}\n\ Reply with ONLY a JSON array, each item:\n\ {{\"name\": \"business concept name\", \"kind\": \"metric\"|\"dimension\", \ @@ -394,7 +503,30 @@ async fn explore(state: &AppState, kb_id: Uuid, run: Uuid) -> anyhow::Result<()> #[cfg(test)] mod tests { - use super::proposal_cap; + use super::{parse_description, proposal_cap}; + + #[test] + fn a_description_is_kept_only_when_it_says_something() { + let (d, qs) = parse_description( + "```json\n{\"description\": \"dw.dwd_ord_dtl: one row per order line.\", \ + \"questions\": [\" Which ord_st values count? \", \"\", \"Exclude is_test = 1?\"]}\n```", + ) + .unwrap(); + assert_eq!(d, "dw.dwd_ord_dtl: one row per order line."); + // 空问题丢掉、首尾空白剥掉，顺序保留 + assert_eq!( + qs, + vec!["Which ord_st values count?", "Exclude is_test = 1?"] + ); + // 空描述整个不要：一个空描述盖掉上一轮的好描述，比没写更糟 + assert!(parse_description("{\"description\": \" \", \"questions\": []}").is_none()); + assert!(parse_description("not json").is_none()); + // 问题没给也行，描述照收 + assert_eq!( + parse_description("{\"description\": \"x\"}").unwrap().1, + Vec::::new() + ); + } #[test] fn a_bigger_schema_gets_a_bigger_cap() { diff --git a/crates/utopia-store/src/kbs.rs b/crates/utopia-store/src/kbs.rs index 20f23bc1..af2baec8 100644 --- a/crates/utopia-store/src/kbs.rs +++ b/crates/utopia-store/src/kbs.rs @@ -73,6 +73,29 @@ pub async fn get(pool: &PgPool, id: Uuid) -> AppResult { .ok_or(AppError::NotFound) } +/// 探索写的数据描述与它拿不准的问题（#570）。 +/// +/// **不碰 `data_conventions`。** 那是人写的；探索每跑一次都重写描述，两样混在 +/// 一个字段里，下一次探索就把人的答案盖了。所以是两列、两条写入路径。 +pub async fn set_data_description( + pool: &PgPool, + id: Uuid, + description: &str, + questions: &[String], +) -> AppResult<()> { + sqlx::query( + "UPDATE knowledge_bases + SET data_description = $2, data_questions = $3, updated_at = now() + WHERE id = $1", + ) + .bind(id) + .bind(description) + .bind(serde_json::json!(questions)) + .execute(pool) + .await?; + Ok(()) +} + #[allow(clippy::too_many_arguments)] pub async fn update( pool: &PgPool, @@ -86,6 +109,7 @@ pub async fn update( inference_interval_minutes: Option, auto_type_resolution: Option, governance: Option, + data_conventions: Option<&str>, ) -> AppResult { // 改语言不回头重写已有的类——它们已经是这个库的数据，可能有人手工调过。 // 这一列往后管的是**新**描述（自动扩本体、AI 建议）写成什么语言 @@ -125,6 +149,8 @@ pub async fn update( -- 从关到开的那一刻记下来：保险丝只数它之后的撤回。SET 右边读的是旧值 governance_since = CASE WHEN $10 IS TRUE AND NOT governance THEN now() ELSE governance_since END, + -- 人写的约定；清空要送空串，送 null 等于不改（与 description 同一约定） + data_conventions = COALESCE($11, data_conventions), updated_at = now() WHERE id = $1 RETURNING *", ) @@ -138,6 +164,7 @@ pub async fn update( .bind(inference_interval_minutes) .bind(auto_type_resolution) .bind(governance) + .bind(data_conventions) .fetch_optional(pool) .await? .ok_or(AppError::NotFound) diff --git a/crates/utopia-store/tests/exploration_describes_the_data.rs b/crates/utopia-store/tests/exploration_describes_the_data.rs new file mode 100644 index 00000000..20d564ea --- /dev/null +++ b/crates/utopia-store/tests/exploration_describes_the_data.rs @@ -0,0 +1,141 @@ +//! 库的数据描述与约定是两列、两条写入路径——打在真库上（#570）。 +//! +//! 探索每跑一次重写描述；人写的约定探索不碰。混在一个字段里，下一次探索就把 +//! 人的答案盖了。这条测试守的就是「各写各的、互不覆盖」。 + +use sqlx::PgPool; +use uuid::Uuid; + +async fn base(pool: &PgPool) -> anyhow::Result { + let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'describe-test')") + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'describe-test')") + .bind(ws) + .bind(org) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'describe-test')", + ) + .bind(kb) + .bind(ws) + .execute(pool) + .await?; + Ok(kb) +} + +#[tokio::test] +async fn the_description_and_the_conventions_do_not_overwrite_each_other() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let kb = base(&pool).await?; + + let run = async { + // 人先写约定 + let no: Option<&str> = None; + utopia_store::kbs::update( + &pool, + kb, + no, + no, + no, + None, + no, + None, + None, + None, + None, + Some("Amounts are in cents.\nis_test = 1 is excluded."), + ) + .await?; + // 探索写描述与问题 + utopia_store::kbs::set_data_description( + &pool, + kb, + "dw.dwd_ord_dtl: one row per order line.", + &["Which ord_st values count?".into()], + ) + .await?; + let got = utopia_store::kbs::get(&pool, kb).await?; + assert_eq!( + got.data_conventions.as_deref(), + Some("Amounts are in cents.\nis_test = 1 is excluded.") + ); + assert_eq!( + got.data_description.as_deref(), + Some("dw.dwd_ord_dtl: one row per order line.") + ); + assert_eq!( + got.data_questions, + serde_json::json!(["Which ord_st values count?"]) + ); + + // 探索再跑一次：描述换了，约定原样 + utopia_store::kbs::set_data_description(&pool, kb, "second run", &[]).await?; + let got = utopia_store::kbs::get(&pool, kb).await?; + assert_eq!(got.data_description.as_deref(), Some("second run")); + assert_eq!(got.data_questions, serde_json::json!([])); + assert!( + got.data_conventions + .as_deref() + .is_some_and(|c| c.contains("is_test")), + "探索不碰人写的约定" + ); + + // 人改约定（PATCH 只送这一项）：描述原样。送 null 等于不改——与 description 同一约定 + utopia_store::kbs::update( + &pool, + kb, + no, + no, + no, + None, + no, + None, + None, + None, + None, + Some("cents only"), + ) + .await?; + let got = utopia_store::kbs::get(&pool, kb).await?; + assert_eq!(got.data_conventions.as_deref(), Some("cents only")); + assert_eq!(got.data_description.as_deref(), Some("second run")); + utopia_store::kbs::update( + &pool, + kb, + Some("renamed"), + no, + no, + None, + no, + None, + None, + None, + None, + no, + ) + .await?; + assert_eq!( + utopia_store::kbs::get(&pool, kb) + .await? + .data_conventions + .as_deref(), + Some("cents only") + ); + Ok::<_, anyhow::Error>(()) + } + .await; + + // 只删知识库，不删 org——用户是软删除的，测试也不该造一个产品里不存在的动作 + sqlx::query("DELETE FROM knowledge_bases WHERE id = $1") + .bind(kb) + .execute(&pool) + .await?; + run +} diff --git a/migrations/0047_exploration_describes_the_data.sql b/migrations/0047_exploration_describes_the_data.sql new file mode 100644 index 00000000..1fe1d27c --- /dev/null +++ b/migrations/0047_exploration_describes_the_data.sql @@ -0,0 +1,18 @@ +-- 探索写一段库的数据描述，并列出它拿不准的地方；约定由人写（#570）。 +-- +-- 量过的事实（#520）：宽表语料上问数没有约定 2/18，约定写成一页散文 14/18。 +-- 约定分两半——结构那半（一行是什么、金额单位、状态码含义、时间轴、哪两列长得像） +-- schema 与注释里读得出来，探索能生成；约定那半（测试单不算数、有效订单是 2/3/4、 +-- GMV 用实付不用优惠前）schema 里没有，探索生成不了，**而且不能让它猜**： +-- 猜出来的约定进了提示词，问数会照着算，比没有更糟。 +-- +-- 所以三个字段，两个来路： +-- data_description 探索生成，每次探索重写。只写 schema 说了的 +-- data_questions 探索生成，它拿不准、需要人答的那几个问题 +-- data_conventions 人写。探索不碰——混在一个字段里，下一次探索就把人的答案盖了 +-- 问数与探索的提示词读前两个（描述）与第三个（约定）；问题清单给页面。 + +ALTER TABLE knowledge_bases + ADD COLUMN data_description TEXT, + ADD COLUMN data_questions JSONB NOT NULL DEFAULT '[]'::jsonb, + ADD COLUMN data_conventions TEXT; diff --git a/scripts/bench/README.md b/scripts/bench/README.md index ed790cb8..ef3d5c33 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -170,8 +170,14 @@ node scripts/bench/ask.mjs --kb --seed # 先把真值写成确认 node scripts/bench/ask.mjs --kb --confirm # 先确认探索提的那些（产品路径） node scripts/bench/ask.mjs --kb --replay # 不重问，拿库里上一轮的回答重判 node scripts/bench/ask.mjs --kb --parallel 4 # 同时问四题：一题 2–6 个模型回合、串行半小时 +node scripts/bench/ask.mjs --kb --conventions # 先把真值里的 conventions 写进库（#570） ``` +`mappings.mjs --fresh --conventions` 同理，只是写在探索之前——探索的提示词也读它。wide 上 +量过的阶梯（chat right）：什么都没有 2/18；探索生成的描述 2/18；描述 + 六条约定 11/18； +约定 + 十八条口径写成一页散文靠检索 14/18；二十七条确认口径进 prompt 17/18。探索覆盖： +没有约定 0/18，有约定 3/18——错的那几条也都带上了 `is_test != 1`，差在选错列。 + **与 `mappings.mjs` 量的不是一回事，一个也推不出另一个。** 口径确认得再准， 答案照样可能错——模型会挑错源、join 错、按错的日期列过滤，或者压根不看语义层， 照着 schema 文档自己写 SQL。反过来，一个一条确认映射都没有的库照样答得出问题， diff --git a/scripts/bench/ask.mjs b/scripts/bench/ask.mjs index 37def27b..3253a984 100644 --- a/scripts/bench/ask.mjs +++ b/scripts/bench/ask.mjs @@ -201,6 +201,12 @@ const main = async () => { const kb = args.kb; if (!kb || kb === true) throw new Error("要一个 --kb "); + // --conventions：把真值文件里的约定写进库（人写约定那条路，#570） + if (args.conventions) { + if (!truth.conventions?.length) throw new Error(`${corpusName} 的真值里没有 conventions`); + await api("PATCH", `/api/v1/kbs/${kb}`, { data_conventions: truth.conventions.join("\n") }); + log(`约定已写进库：${truth.conventions.length} 条`); + } if (args.seed) seedTruth(kb); if (args.confirm) confirmProposals(kb); diff --git a/scripts/bench/mappings.mjs b/scripts/bench/mappings.mjs index 3ca61e62..ed061dbd 100644 --- a/scripts/bench/mappings.mjs +++ b/scripts/bench/mappings.mjs @@ -92,6 +92,13 @@ async function fresh() { log(`kb ${kb}，源 ${dsName} 已挂载，schema 文档 ${mounted.schema_tables ?? "?"} 张表`); if (mounted.schema_error) log(` schema 同步报错：${mounted.schema_error}`); + // --conventions：探索之前把真值文件里的约定写进库，它的提示词会读（#570）。 + // 这是探索在 wide 上从 0/18 动起来的第一个机会：schema 里没有「测试单不算数」 + if (args.conventions) { + if (!truth.conventions?.length) throw new Error(`${corpusName} 的真值里没有 conventions`); + await api("PATCH", `/api/v1/kbs/${kb}`, { data_conventions: truth.conventions.join("\n") }); + log(`约定已写进库：${truth.conventions.length} 条`); + } await api("POST", `/api/v1/kbs/${kb}/data-sources/explore`); log("探索已入队，等提议落库"); let said = "";