diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index d003241a4..f39687291 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -1061,6 +1061,35 @@ pub struct MappingRevision { pub changed_at: DateTime, } +/// 一轮映射探索扫了什么、丢了什么、剩下什么（#503）。 +/// +/// **它回答的是覆盖率**：十一条提议对着一张八十列的宽表，与十一条刚好覆盖完 +/// 一个小库，从 `concept_mappings` 里看长得一模一样。分子是 `tables_covered`， +/// 分母是 `tables_scanned`，而 `schema_truncated` 说明覆盖不全是「没看见」 +/// 还是「看见了没提」。 +#[derive(Debug, Clone, Serialize, sqlx::FromRow)] +pub struct ExplorationRun { + pub id: Uuid, + pub started_at: DateTime, + pub finished_at: Option>, + pub sources: Vec, + pub tables_scanned: i32, + pub columns_scanned: i32, + /// schema 文本撞了上限：提示词里没有的表，模型没有机会提 + pub schema_truncated: bool, + /// 这一轮允许提几条（按表数放大） + pub cap: i32, + /// 模型回了几条 / 落库几条。两者之差是被丢掉的，明细在 `dropped` + pub returned: i32, + pub accepted: i32, + /// `{"source": {"n": 12, "example": "…"}, …}`，键见 + /// `utopia_store::exploration_runs::drop_reason` + pub dropped: serde_json::Value, + pub tables_covered: Vec, + /// 跑挂了的那一轮也留一行——失败与「跑了但什么都没提」不是一回事 + pub error: Option, +} + /// 语义层的一条映射：业务概念 → 数据资产定义（见 `docs/decisions/0011`）。 /// /// **字段是列，不是 JSON 里的键。** 从前它是一条 `mapped_to` 事实， diff --git a/crates/utopia-server/src/api/mapping_routes.rs b/crates/utopia-server/src/api/mapping_routes.rs index 5ee6d3281..9a28616e3 100644 --- a/crates/utopia-server/src/api/mapping_routes.rs +++ b/crates/utopia-server/src/api/mapping_routes.rs @@ -58,10 +58,17 @@ pub async fn list( utopia_store::mappings::page(&state.pool, kb_id, status, needle, limit, offset).await?; let (proposed, confirmed, rejected) = utopia_store::mappings::status_counts(&state.pool, kb_id).await?; + // 上一轮探索的账（#503）。**列表本身答不了「漏了多少」**——十一条提议对着 + // 一张八十列的宽表，与十一条刚好覆盖完一个小库，在 items 里长得一模一样 + let last_run = utopia_store::exploration_runs::recent(&state.pool, kb_id, 1) + .await? + .into_iter() + .next(); Ok(Json(json!({ "items": items, "total": total, "counts": { "proposed": proposed, "confirmed": confirmed, "rejected": rejected }, + "last_run": last_run, }))) } diff --git a/crates/utopia-server/src/mappings.rs b/crates/utopia-server/src/mappings.rs index 12714f486..5a6c5bc73 100644 --- a/crates/utopia-server/src/mappings.rs +++ b/crates/utopia-server/src/mappings.rs @@ -11,10 +11,26 @@ use crate::llm_util; use crate::state::AppState; +use utopia_store::exploration_runs::drop_reason; use uuid::Uuid; const MAX_SCHEMA_CHARS: usize = 12_000; +/// 一轮允许提几条口径。 +/// +/// **上限跟着 schema 的大小走。** 写死的 12 对一个三张表的小库绰绰有余， +/// 对一张八十列的宽表就是覆盖率的天花板：实测 TPC-H 八张表、二十四条口径， +/// 十二条上限之下覆盖率 25%，漏掉的包括那条出现在七条基准查询里的核心收入 +/// 口径（#501）。 +/// +/// 每张表三条是个估计而不是定律——一张事实表值得的口径远多于三条，一张 +/// 码表一条都不值。它只需要比常数强：**规模大的时候不至于一开始就封顶**。 +/// 下限仍是 12，小 schema 不该因此缩水；上限 60 挡住提示词与一轮人工审阅 +/// 的规模。 +fn proposal_cap(tables: i32) -> i32 { + (tables * 3).clamp(12, 60) +} + /// 探索把 schema 里的量与维度落成 Metric / Dimension 实体，而这两个类不在任何 /// 内置本体包里——0009 之后建库不再自带类。没有它们，下面的 `type_id` 查不到， /// 每条提议都被 `continue` 吞掉，页面只说"已排队"就再无下文（#223）。 @@ -48,7 +64,23 @@ pub(crate) async fn ensure_concept_types(pool: &sqlx::PgPool, kb_id: Uuid) -> an Ok(()) } +/// 一轮探索开账、干活、收账（#503）。 +/// +/// **先开行再干活**：跑挂了的那一轮也留一行，因为「失败」与「跑了但一条都没提」 +/// 从前在页面上都是「没有新提议」，而该做的事完全不同。 pub async fn explore_mappings(state: &AppState, kb_id: Uuid) -> anyhow::Result<()> { + let run = utopia_store::exploration_runs::start(&state.pool, kb_id).await?; + match explore(state, kb_id, run).await { + Ok(()) => Ok(()), + Err(e) => { + // 报错路径上的失败不该淹掉它要报的那件事（与 `source_name` 同一条理由） + let _ = utopia_store::exploration_runs::fail(&state.pool, run, &e.to_string()).await; + Err(e) + } + } +} + +async fn explore(state: &AppState, kb_id: Uuid, run: Uuid) -> anyhow::Result<()> { let kb = utopia_store::kbs::get(&state.pool, kb_id).await?; let settings = utopia_store::settings::get(&state.pool, kb.workspace_id) .await? @@ -63,8 +95,18 @@ pub async fn explore_mappings(state: &AppState, kb_id: Uuid) -> anyhow::Result<( ensure_concept_types(&state.pool, kb_id).await?; // 各源 schema（引擎直读，保证新鲜；限量防 prompt 爆炸） + // + // **上限是跨源的一个总数。** 从前那个 `break` 只跳出当前源的列循环， + // 下一个源接着往同一个字符串里追加——`MAX_SCHEMA_CHARS` 读起来像个上限， + // 实际上是「每个源各自超一次」的下限。 let mut schema_txt = String::new(); + let mut tables_scanned = 0i32; + let mut columns_scanned = 0i32; + let mut truncated = false; for ds in &sources { + if truncated { + break; + } let (engine, conn) = utopia_store::datasources::engine_and_conn(&state.pool, ds.id).await?; let cols = crate::query_engine::engine_for(&engine, &conn)? .fetch_schema() @@ -75,8 +117,10 @@ pub async fn explore_mappings(state: &AppState, kb_id: Uuid) -> anyhow::Result<( let key = format!("{}.{}", c.schema, c.table); if key != current { current = key.clone(); + tables_scanned += 1; schema_txt.push_str(&format!("table {key}:\n")); } + columns_scanned += 1; schema_txt.push_str(&format!( " {} {}{}\n", c.column, @@ -85,11 +129,25 @@ pub async fn explore_mappings(state: &AppState, kb_id: Uuid) -> anyhow::Result<( )); if schema_txt.len() > MAX_SCHEMA_CHARS { schema_txt.push_str("(truncated)\n"); + truncated = true; break; } } } + let cap = proposal_cap(tables_scanned); + let source_list: Vec = sources.iter().map(|d| d.name.clone()).collect(); + let _ = utopia_store::exploration_runs::scanned( + &state.pool, + run, + &source_list, + tables_scanned, + columns_scanned, + truncated, + cap, + ) + .await; + // 既有概念（供归并复用，避免重复起名） let existing: Vec<(String,)> = sqlx::query_as( "SELECT e.canonical_name FROM entities e @@ -115,7 +173,7 @@ pub async fn explore_mappings(state: &AppState, kb_id: Uuid) -> anyhow::Result<( \"summary\": \"one line: source + expression, shown to reviewers\", \ \"rationale\": \"why this mapping, citing column comments\"}}\n\ Metrics are aggregatable quantities (use sum/count/avg in expr); dimensions are \ - group-by columns. Propose at most 12, only well-grounded ones.", + group-by columns. Propose at most {cap}, only well-grounded ones.", if existing_names.is_empty() { "(none)".into() } else { @@ -142,23 +200,56 @@ pub async fn explore_mappings(state: &AppState, kb_id: Uuid) -> anyhow::Result<( let source_names: Vec<&str> = sources.iter().map(|d| d.name.as_str()).collect(); let mut accepted = 0usize; - for p in proposals.iter().take(12) { + // 丢弃要计数，还要留一条例子——**光有计数诊断不动**：「十二条源名对不上」 + // 得配上「模型说的是 tpch，挂的是 tpch-2026-09-08」才知道该改什么 + let mut drops: std::collections::BTreeMap<&'static str, (i64, String)> = Default::default(); + let mut note = |reason: &'static str, example: String| { + let e = drops.entry(reason).or_insert((0, example)); + e.0 += 1; + }; + let mut covered: std::collections::BTreeSet = Default::default(); + for p in proposals.iter().take(cap as usize) { let name = p["name"].as_str().map(str::trim).unwrap_or(""); let kind = p["kind"].as_str().unwrap_or(""); - let source = p["source"].as_str().map(str::trim).unwrap_or(""); - if name.is_empty() - || !matches!(kind, "metric" | "dimension") - || !source_names.iter().any(|s| s.eq_ignore_ascii_case(source)) - { + let said = p["source"].as_str().map(str::trim).unwrap_or(""); + if name.is_empty() || !matches!(kind, "metric" | "dimension") { + note(drop_reason::KIND, format!("name={name:?} kind={kind:?}")); continue; } + // **只挂了一个源时，模型说什么都算它。** 没有歧义可言，而对不上的代价是 + // 整条提议消失：实测源叫 `tpch-2026-09-08-12-30`，模型照着 schema 回 + // `tpch`，十二条一条不剩地被吞掉，任务照样 done（#501 跑第一轮时踩的） + let source = if sources.len() == 1 { + source_names[0] + } else { + match source_names + .iter() + .find(|s| s.eq_ignore_ascii_case(said)) + .copied() + { + Some(s) => s, + None => { + note( + drop_reason::SOURCE, + format!("model said {said:?}, mounted: {}", source_names.join(", ")), + ); + continue; + } + } + }; let type_id: Option<(Uuid,)> = sqlx::query_as("SELECT id FROM entity_types WHERE kb_id = $1 AND key = $2") .bind(kb_id) .bind(kind) .fetch_optional(&state.pool) .await?; - let Some((type_id,)) = type_id else { continue }; + let Some((type_id,)) = type_id else { + note( + drop_reason::TYPE, + format!("no entity type {kind:?} in this base"), + ); + continue; + }; // 概念实体：走消解（同名归并；无向量上下文按 v1 兼容归并）。 // 没有块原文可给——这些名字来自数据源的 schema 探索，不是从文档句子里抽的 @@ -179,6 +270,10 @@ pub async fn explore_mappings(state: &AppState, kb_id: Uuid) -> anyhow::Result<( // 断言，是配置**，所以搬去自己的表 let def = &p["definition"]; if !def.is_object() { + note( + drop_reason::DEFINITION, + format!("{name:?} has no definition object"), + ); continue; } let s = |k: &str| { @@ -187,7 +282,12 @@ pub async fn explore_mappings(state: &AppState, kb_id: Uuid) -> anyhow::Result<( .filter(|x| !x.is_empty()) .map(str::to_string) }; - utopia_store::mappings::propose( + // 覆盖率的分子。分母是这一轮扫见的表数——**十一条提议对着八十列的宽表， + // 与十一条刚好覆盖完一个小库，从 `concept_mappings` 里看长得一模一样** + if let Some(t) = s("table") { + covered.insert(t); + } + let (_, written) = utopia_store::mappings::propose( &state.pool, kb_id, resolved.entity_id, @@ -201,12 +301,67 @@ pub async fn explore_mappings(state: &AppState, kb_id: Uuid) -> anyhow::Result<( def["derived"].as_bool().unwrap_or(false), ) .await?; - accepted += 1; + if written { + accepted += 1; + } else { + note( + drop_reason::DECIDED, + format!("{name:?} on {source}: already confirmed or rejected"), + ); + } + } + // 超过上限的那些一条没看，也得记：不然 returned 与 accepted + dropped 对不上， + // 而账本的用处正是让人看出「模型回了六十条、我们只看了三十六条」 + if proposals.len() > cap as usize { + drops.insert( + drop_reason::CAP, + ( + (proposals.len() - cap as usize) as i64, + format!("model returned {}, cap {cap}", proposals.len()), + ), + ); } - tracing::info!(%kb_id, proposals = accepted, "映射探索完成，提议已入审核队列"); + let dropped = serde_json::Value::Object( + drops + .iter() + .map(|(k, (n, ex))| { + ( + (*k).to_string(), + serde_json::json!({ "n": n, "example": ex }), + ) + }) + .collect(), + ); + let covered: Vec = covered.into_iter().collect(); + if let Err(e) = utopia_store::exploration_runs::finish( + &state.pool, + run, + proposals.len() as i32, + accepted as i32, + dropped, + &covered, + ) + .await + { + // 账没记上不该让提议白跑，但也不能装作记上了：留在日志里 + tracing::warn!(%kb_id, run = %run, error = %e, "探索账没记上"); + } + tracing::info!( + %kb_id, + proposals = accepted, + returned = proposals.len(), + tables = tables_scanned, + covered = covered.len(), + truncated, + "映射探索完成，提议已入审核队列" + ); // 一条都没提出来时页面上什么都不会变——Pending 还是 0，而"已排队"那句 - // 早就翻篇了。走告警中心说一声，人才知道该去刷新结构或给列加注释 + // 早就翻篇了。走告警中心说一声，人才知道该去刷新结构或给列加注释。 + // + // **告警要带上是怎么空的。** 从前只说「0 条，这些源」，而「模型一条没回」 + // 与「回了十二条全被源名挡掉」是两件事，该做的动作也不同——前者去给列加注释， + // 后者去看源名。`dropped` 就是这个区别，它现在也在这条告警里 if accepted == 0 { if let Err(e) = utopia_store::alerts::raise( &state.pool, @@ -217,7 +372,14 @@ pub async fn explore_mappings(state: &AppState, kb_id: Uuid) -> anyhow::Result<( min_role: utopia_core::models::Role::Editor, subject_type: None, subject_id: None, - detail: serde_json::json!({ "proposals": 0, "sources": source_names }), + detail: serde_json::json!({ + "proposals": 0, + "sources": source_names, + "returned": proposals.len(), + "dropped": drops.iter().map(|(k, (n, _))| (*k, *n)) + .collect::>(), + "tables_scanned": tables_scanned, + }), }, ) .await @@ -229,3 +391,24 @@ pub async fn explore_mappings(state: &AppState, kb_id: Uuid) -> anyhow::Result<( state.emit_review(kb_id); Ok(()) } + +#[cfg(test)] +mod tests { + use super::proposal_cap; + + #[test] + fn a_bigger_schema_gets_a_bigger_cap() { + // 小库不缩水：从前写死的 12 在这一端是对的 + assert_eq!(proposal_cap(1), 12); + assert_eq!(proposal_cap(4), 12); + // 八张表的 TPC-H：从前 12 条封顶，二十四条真值只覆盖了 6 条（#501） + assert_eq!(proposal_cap(8), 24); + // TPC-DS 二十四张表 + assert_eq!(proposal_cap(24), 60); + // 再大也到此为止：提示词与一轮人工审阅都有自己的上限 + assert_eq!(proposal_cap(300), 60); + // 一张表都没扫见（源连不上、schema 是空的）也不该是 0—— + // **0 条上限会把「连不上」变成「模型什么都没提」**，两件事又混在一起了 + assert_eq!(proposal_cap(0), 12); + } +} diff --git a/crates/utopia-store/src/exploration_runs.rs b/crates/utopia-store/src/exploration_runs.rs new file mode 100644 index 000000000..00c654314 --- /dev/null +++ b/crates/utopia-store/src/exploration_runs.rs @@ -0,0 +1,137 @@ +//! 一轮映射探索的账：扫了多大的东西、丢了什么、覆盖了多少（#503）。 +//! +//! 从前一轮探索只留下两样：`concept_mappings` 里若干行，以及「一条都没提出来」 +//! 时的一条告警。**覆盖率没有任何地方说**——十一条提议对着一张八十列的宽表， +//! 与十一条刚好覆盖完一个小库，页面上一模一样。 +//! +//! 丢弃更是静默的。`explore_mappings` 对每条提议有四处 `continue`，四处都不计数。 +//! 实测踩过一次：源叫 `tpch-2026-09-08-12-30`，模型看着 schema 回的是 `tpch`， +//! 十二条一条不剩地被吞掉，任务照样 `done`。 +//! +//! 与 `extraction_drops` 的分工照那张表自己的话说：那边答「这些事实没落地」， +//! 读者是传文档的人；这边答「这一轮看见了多大的东西、覆盖了多少」，读者是 +//! 管数据源的人，动作是给列加注释或者再跑一轮。 + +use sqlx::PgPool; +use utopia_core::models::ExplorationRun; +use utopia_core::AppResult; +use uuid::Uuid; + +/// 丢弃原因码。前端按这个查文案，所以是稳定契约，不要改字面量 +/// （与 `extraction_drops::reason` 同一约定）。 +pub mod drop_reason { + /// 模型回的 `source` 对不上任何挂载源的名字。**最常踩的一个**：源名带了 + /// 环境后缀或时间戳，而模型照着 schema 回一个短名 + pub const SOURCE: &str = "source"; + /// `kind` 不是 metric / dimension + pub const KIND: &str = "kind"; + /// 概念类查不到——本体里没有 Metric / Dimension，`ensure_concept_types` + /// 之后不该再发生 + pub const TYPE: &str = "type"; + /// `definition` 不是一个对象，或者拿不出可执行的定义 + pub const DEFINITION: &str = "definition"; + /// 模型回的条数超过这一轮的上限，超出的没看。上限按表数定，回得比它多， + /// 多半是模型把列当成了口径——宽表上常见 + pub const CAP: &str = "cap"; + /// 这条概念在这个源上已经有人表过态（确认或拒绝）。提议不覆盖决定， + /// 所以下一轮探索算出同一条时它既不回到待看，也不算这一轮写入的 + pub const DECIDED: &str = "decided"; +} + +/// 开一轮。**先开行再干活**，跑挂了那一轮也有账可查—— +/// 失败与「跑了但什么都没提」在页面上从前都是「没有新提议」。 +pub async fn start(pool: &PgPool, kb_id: Uuid) -> AppResult { + let id = Uuid::now_v7(); + sqlx::query("INSERT INTO mapping_exploration_runs (id, kb_id) VALUES ($1, $2)") + .bind(id) + .bind(kb_id) + .execute(pool) + .await?; + Ok(id) +} + +/// 读完 schema、还没问模型时先记下规模。**问模型可能要一分钟**， +/// 这期间页面上该看得见「正在扫的是多大的东西」。 +pub async fn scanned( + pool: &PgPool, + id: Uuid, + sources: &[String], + tables: i32, + columns: i32, + truncated: bool, + cap: i32, +) -> AppResult<()> { + sqlx::query( + "UPDATE mapping_exploration_runs + SET sources = $2, tables_scanned = $3, columns_scanned = $4, + schema_truncated = $5, cap = $6 + WHERE id = $1", + ) + .bind(id) + .bind(sources) + .bind(tables) + .bind(columns) + .bind(truncated) + .bind(cap) + .execute(pool) + .await?; + Ok(()) +} + +/// 收一轮。`dropped` 的形状见 [`drop_reason`]：每个原因一个 +/// `{"n": 计数, "example": "一条例子"}`——**光有计数诊断不动**， +/// 「十二条源名对不上」要配上「模型说的是 tpch，挂的是 tpch-2026-09-08」 +/// 才知道该改什么。 +pub async fn finish( + pool: &PgPool, + id: Uuid, + returned: i32, + accepted: i32, + dropped: serde_json::Value, + tables_covered: &[String], +) -> AppResult<()> { + sqlx::query( + "UPDATE mapping_exploration_runs + SET finished_at = now(), returned = $2, accepted = $3, + dropped = $4, tables_covered = $5 + WHERE id = $1", + ) + .bind(id) + .bind(returned) + .bind(accepted) + .bind(dropped) + .bind(tables_covered) + .execute(pool) + .await?; + Ok(()) +} + +/// 这一轮挂了。记下来就收摊——**报错路径上的失败不该淹掉它要报的那件事**， +/// 调用方一律 `let _ =`（与 `extraction_drops` 同一约定）。 +pub async fn fail(pool: &PgPool, id: Uuid, error: &str) -> AppResult<()> { + sqlx::query( + "UPDATE mapping_exploration_runs + SET finished_at = now(), error = left($2, 500) WHERE id = $1", + ) + .bind(id) + .bind(error) + .execute(pool) + .await?; + Ok(()) +} + +/// 最近几轮，新的在前。数据映射页读它。 +pub async fn recent(pool: &PgPool, kb_id: Uuid, limit: i64) -> AppResult> { + Ok(sqlx::query_as( + "SELECT id, started_at, finished_at, sources, tables_scanned, columns_scanned, + schema_truncated, cap, returned, accepted, dropped, tables_covered, error + FROM mapping_exploration_runs + WHERE kb_id = $1 + ORDER BY started_at DESC + LIMIT $2", + ) + .bind(kb_id) + .bind(limit) + .fetch_all(pool) + .await?) +} diff --git a/crates/utopia-store/src/lib.rs b/crates/utopia-store/src/lib.rs index 4d51d207c..ec1471e27 100644 --- a/crates/utopia-store/src/lib.rs +++ b/crates/utopia-store/src/lib.rs @@ -11,6 +11,7 @@ pub mod datasources; pub mod db; pub mod documents; pub mod execution_gate; +pub mod exploration_runs; pub mod export; pub mod extraction_drops; pub mod governance; diff --git a/crates/utopia-store/src/mappings.rs b/crates/utopia-store/src/mappings.rs index 6d8332537..f9329feb6 100644 --- a/crates/utopia-store/src/mappings.rs +++ b/crates/utopia-store/src/mappings.rs @@ -32,7 +32,7 @@ pub async fn propose( unit: Option<&str>, summary: Option<&str>, derived: bool, -) -> AppResult { +) -> AppResult<(Uuid, bool)> { // **`DO UPDATE ... WHERE` 不满足时 `RETURNING` 一行都不返回。** // // 这是 Postgres 的实情而不是直觉：条件挡住更新，那一行就不算被这条语句 @@ -50,7 +50,7 @@ pub async fn propose( .fetch_optional(pool) .await?; let id = existing.map(|(i,)| i).unwrap_or_else(Uuid::now_v7); - sqlx::query( + let written = sqlx::query( "INSERT INTO concept_mappings (id, kb_id, concept_id, source, table_name, expr, sql, unit, summary, derived) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) @@ -72,8 +72,11 @@ pub async fn propose( .bind(summary) .bind(derived) .execute(pool) - .await?; - Ok(id) + .await? + .rows_affected(); + // 插入或刷新了算写入；撞上已确认 / 已拒绝的那一行，WHERE 挡下更新，写入数为零—— + // 调用方靠这一位区分「进了待看」与「决定还在」 + Ok((id, written > 0)) } /// 人从零写一条口径（#562）。**落下来就是确认的**：写的人就是表态的人， diff --git a/crates/utopia-store/tests/a_mapping_is_not_a_fact.rs b/crates/utopia-store/tests/a_mapping_is_not_a_fact.rs index 04b53e607..741ebbe49 100644 --- a/crates/utopia-store/tests/a_mapping_is_not_a_fact.rs +++ b/crates/utopia-store/tests/a_mapping_is_not_a_fact.rs @@ -66,9 +66,10 @@ async fn one_concept_one_source_one_mapping() -> anyhow::Result<()> { false, ) }; - let a = p("orders").await?; - let b = p("orders_v2").await?; + let (a, wrote_a) = p("orders").await?; + let (b, wrote_b) = p("orders_v2").await?; assert_eq!(a, b, "同一个 (概念, 源) 该是同一行，不是两行"); + assert!(wrote_a && wrote_b, "第一次插入、第二次刷新，都算写入"); let got = utopia_store::mappings::proposed(&pool, kb, 100, 0).await?; assert_eq!(got.len(), 1, "只该有一条"); @@ -132,7 +133,7 @@ async fn a_rejected_mapping_does_not_come_back() -> anyhow::Result<()> { .execute(&pool) .await?; - let id = utopia_store::mappings::propose( + let (id, _) = utopia_store::mappings::propose( &pool, kb, ent, @@ -147,8 +148,8 @@ async fn a_rejected_mapping_does_not_come_back() -> anyhow::Result<()> { .await?; utopia_store::mappings::decide(&pool, kb, id, "rejected", user).await?; - // 下一轮探索会再次算出同一条——它不该被刷回待看 - utopia_store::mappings::propose( + // 下一轮探索会再次算出同一条——它不该被刷回待看，也不算写入 + let (_, written) = utopia_store::mappings::propose( &pool, kb, ent, @@ -161,6 +162,7 @@ async fn a_rejected_mapping_does_not_come_back() -> anyhow::Result<()> { false, ) .await?; + assert!(!written, "撞上决定的提议不算写入，探索账把它记成 decided"); assert!( utopia_store::mappings::proposed(&pool, kb, 100, 0) .await? diff --git a/crates/utopia-store/tests/an_exploration_says_what_it_covered.rs b/crates/utopia-store/tests/an_exploration_says_what_it_covered.rs new file mode 100644 index 000000000..05572b6ee --- /dev/null +++ b/crates/utopia-store/tests/an_exploration_says_what_it_covered.rs @@ -0,0 +1,124 @@ +//! 一轮映射探索留下的账——打在真库上（#503）。 +//! +//! 两条都是从前做不到的： +//! +//! 1. **覆盖率答得出来。** 从前一轮探索只留下 `concept_mappings` 里若干行， +//! 十一条提议对着八十列的宽表与十一条覆盖完一个小库看起来一模一样。 +//! 2. **跑挂了的那一轮也有账。** 从前失败与「跑了但一条都没提」在页面上 +//! 都是「没有新提议」，而该做的事完全不同。 + +use sqlx::PgPool; +use utopia_store::exploration_runs as runs; +use uuid::Uuid; + +async fn fixture(pool: &PgPool) -> anyhow::Result<(Uuid, Uuid)> { + let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'run-test')") + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'run-test')") + .bind(ws) + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'run-test')") + .bind(kb) + .bind(ws) + .execute(pool) + .await?; + Ok((org, kb)) +} + +#[tokio::test] +async fn a_run_says_what_it_scanned_and_what_it_dropped() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let (_org, kb) = fixture(&pool).await?; + + let run = async { + let id = runs::start(&pool, kb).await?; + runs::scanned( + &pool, + id, + &["warehouse".into()], + 8, + 61, + false, + 24, + ) + .await?; + runs::finish( + &pool, + id, + 12, + 9, + serde_json::json!({ + runs::drop_reason::SOURCE: { "n": 3, "example": "model said \"tpch\", mounted: tpch-2026" } + }), + &["tpch.lineitem".into(), "tpch.orders".into()], + ) + .await?; + + let got = runs::recent(&pool, kb, 10).await?; + assert_eq!(got.len(), 1); + let r = &got[0]; + assert_eq!((r.tables_scanned, r.columns_scanned, r.cap), (8, 61, 24)); + // **回了 12 条、落库 9 条**：两者之差就是被丢掉的，明细在 dropped。 + // 从前这三个数一个都没有，页面上只看得见落库的那 9 条 + assert_eq!((r.returned, r.accepted), (12, 9)); + assert_eq!(r.dropped[runs::drop_reason::SOURCE]["n"], 3); + assert!( + r.dropped[runs::drop_reason::SOURCE]["example"] + .as_str() + .is_some_and(|s| s.contains("mounted")), + "光有计数诊断不动，例子要留住" + ); + assert_eq!(r.tables_covered.len(), 2, "覆盖率的分子"); + assert!(r.finished_at.is_some()); + assert!(r.error.is_none()); + Ok::<_, anyhow::Error>(()) + } + .await; + + // 只删知识库，不删 org——用户是软删除的，测试也不该造一个产品里不存在的动作 + sqlx::query("DELETE FROM knowledge_bases WHERE id = $1") + .bind(kb) + .execute(&pool) + .await?; + run +} + +#[tokio::test] +async fn a_failed_run_is_not_an_empty_one() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let (_org, kb) = fixture(&pool).await?; + + let run = async { + let id = runs::start(&pool, kb).await?; + runs::fail(&pool, id, "Chat model not configured").await?; + + let got = runs::recent(&pool, kb, 10).await?; + assert_eq!(got.len(), 1, "跑挂了的那一轮也该留下一行"); + // 这一行与「跑完了但 accepted = 0」的区别就在这里。两者在页面上 + // 从前都是「没有新提议」，而一个要去配模型，一个要去给列加注释 + assert_eq!(got[0].accepted, 0); + assert!(got[0] + .error + .as_deref() + .is_some_and(|e| e.contains("Chat model"))); + Ok::<_, anyhow::Error>(()) + } + .await; + + sqlx::query("DELETE FROM knowledge_bases WHERE id = $1") + .bind(kb) + .execute(&pool) + .await?; + run +} diff --git a/docs/decisions/0009-no-type-is-a-type.md b/docs/decisions/0009-no-type-is-a-type.md index 38a928d4d..b194bfb03 100644 --- a/docs/decisions/0009-no-type-is-a-type.md +++ b/docs/decisions/0009-no-type-is-a-type.md @@ -7,8 +7,9 @@ same-name entities apart ahead of every heuristic; since #226 same-name entities of kin classes (ancestor, descendant or a shared non-root ancestor) go to Review; `CONFUSABLE_TYPE_KEYS` stays as the fallback when nothing is declared · `metric` / - `dimension` are created on demand by mapping exploration (#231); a semantic-layer pack is - still planned + `dimension` were created on demand by mapping exploration (#231); **they are to retire** + under [0036](0036-exploration-aligns-a-schema-to-the-ontology.md), which found them to be + the same species this record removed - **Written**: 2026-08-30 · condensed into English 2026-09-03 - **Related**: [0008](0008-ontology-packs-as-cold-start.md) makes a real vocabulary the optional start; [0001](0001-ontology-import-and-governance.md) IRI/key split behind the @@ -95,6 +96,15 @@ in the ontology as a class, as if someone had decided it. those classes silently produced zero mappings. 2026-09-03: exploration now creates both as builtin types before exploring (#231); a semantic-layer pack with IRIs is still planned (0016 D2). +- 2026-09-09: #231 fixed the symptom and kept the mistake. `Metric` and `Dimension` are + not classes of things in the world; they are kinds of column, and this record's own + argument — a class that is control flow leaves the ontology — applies to them. Measured on + a flattened order table, exploration filed every column name as an entity of those classes + (28 of 40 concept entities were column names) and proposed 0 of 18 usable definitions. + [0036](0036-exploration-aligns-a-schema-to-the-ontology.md) retires both classes; a + concept becomes an attribute of a real class or a rule over such attributes. The + semantic-layer pack (0016 D2) is superseded by the same record: there is no vocabulary to + pack, only an alignment to propose. ## Open questions diff --git a/docs/decisions/0011-a-mapping-is-not-a-fact.md b/docs/decisions/0011-a-mapping-is-not-a-fact.md index 5e67c2b71..31745e608 100644 --- a/docs/decisions/0011-a-mapping-is-not-a-fact.md +++ b/docs/decisions/0011-a-mapping-is-not-a-fact.md @@ -3,7 +3,9 @@ - **Status**: Implemented · `concept_mappings` table and wiring (#126, the same commit as this record), a standalone Data Mappings page (#140), moved out of the Review queue (#148) · of the three things to rebuild, the Review flow and history are done, the evidence chain - is not · one of two open questions answered (2026-09-02 check) + is not · one of two open questions answered (2026-09-02 check) · **revised in part by + [0036](0036-exploration-aligns-a-schema-to-the-ontology.md)** (2026-09-09): what a mapping + is stands; what a concept is changes, and the table becomes a rendered one - **Written**: 2026-08-31 · condensed into English 2026-09-03 - **Related**: [0009](0009-no-type-is-a-type.md) removes the builtin entity classes, [0010](0010-no-relation-is-no-relation.md) the fallback relation, #125 the other eight @@ -78,6 +80,20 @@ rebuilding all three. rule. - 2026-09-02: we assumed a Review group would be the mapping's home; in use it became its own page (#140, #148). +- 2026-09-09: **the concept a mapping hangs from is the wrong kind of thing.** This record + decided what a mapping is (configuration, its own table, revisions) and did not decide what + a *concept* is; the implementation made it an entity of a builtin class `Metric` or + `Dimension`, with the definition in the side table and nothing on the graph. Two benches + measured the cost: on a flattened order table exploration proposed 0 of 18 usable + definitions and the extractor filed 28 column names as concept entities (#501); a + convention such as "test orders do not count" had nowhere to live, and its absence moved + chat from 17 of 18 right answers to 1 of 18 (#520). + [0036](0036-exploration-aligns-a-schema-to-the-ontology.md) keeps every decision above and + changes the concept: an attribute of a real class, or a rule over such attributes; the + mapping is how a column becomes that attribute's value; `concept_mappings` becomes a + rendered table. Decision 4 (evidence recorded separately) and the second open question + (one concept, several sources) are answered by the alignment itself — the evidence is the + alignment, and a concept with two sources is an attribute two columns feed. ## Open questions diff --git a/docs/decisions/0036-exploration-aligns-a-schema-to-the-ontology.md b/docs/decisions/0036-exploration-aligns-a-schema-to-the-ontology.md new file mode 100644 index 000000000..d4125935d --- /dev/null +++ b/docs/decisions/0036-exploration-aligns-a-schema-to-the-ontology.md @@ -0,0 +1,233 @@ +# 0036 · Exploration aligns a schema to the ontology + +- **Status**: written · decision 7 implemented (#553 → #561: the schema document is + indexed and never extracted, `sources.config.extract`, `graph_status = skipped`; the + column-name entities went from 93 to 12 on the wide bench base) · a definition can be + written by hand (#562 → #563), the door the seeded upper bound simulated · the + conventions-as-prose dead end was measured at 14/18 and revised in place (see Dead ends) · + cuts remaining: #554 alignment, #555 the conversion tree, #556 retiring the two classes · + overturns where a mapping hangs, and keeps what [0011](0011-a-mapping-is-not-a-fact.md) + said about what a mapping is · the two builtin classes `metric` / `dimension` are to + retire (their revision notes are on 0009 and 0011) · the migration that carries the + exploration ledger (#503) is unaffected +- **Written**: 2026-09-09 (conventions in the [README](README.md)) +- **Related**: [0011](0011-a-mapping-is-not-a-fact.md) moved a mapping out of the ledger and + is right that it is configuration; this record moves the *concept* out of the entity table. + [0009](0009-no-type-is-a-type.md), [0010](0010-no-relation-is-no-relation.md) and 0011 are + one line of work, control flow leaving the ontology; this is the fourth step on it. + [0003](0003-ontology-growth-loop.md) is the loop exploration joins, with a schema as the + corpus. [0021](0021-a-rule-reads-attributes-and-concludes-a-type.md), + [0030](0030-a-rule-may-read-what-a-rule-concluded.md) and + [0032](0032-a-rule-computes-what-it-concludes.md) are the rules a definition becomes; + 0032's refusal of aggregation over the graph is what makes the database the place a + definition runs. [0018](0018-the-lakehouse-is-one-protocol-away.md) is why a conversion + cannot be SQL text. The measuring benches that found this: #501, #503, #520. + +> A base mounts an e-commerce order table, fifty-five columns wide, and asks exploration for +> its metrics. Exploration first inserts two classes into the ontology, `Metric` and +> `Dimension`, then reads the schema, then creates twelve entities of those classes — +> `order_amount`, `channel`, `gross_profit` — each with a SQL expression in a side table. +> None of the twelve computes a number the business would recognise (0 of 18 definitions, +> #501). Meanwhile the schema document, ingested as prose, is extracted like any other +> document, and the extractor — shown the two new classes — files every column name as an +> entity: `amt_pay` is now a Metric, `buyer_id` a Dimension, `dw.dim_shop` a Dimension with +> eight facts on it. The graph holds forty concept entities, twelve with a definition, +> twenty-eight that are column names, and nowhere in any of it does it say that the table is +> a table of orders. + +## The problem + +Exploration builds ontology. It should align to one. + +What it builds is the wrong kind of thing. `Metric` and `Dimension` are not classes of +things in the world; they are BI vocabulary for *kinds of column*. The world has orders, +customers and shops. "GMV" is not something that exists; it is an aggregate over an +attribute of orders under a business convention. Making it an entity puts a thing on the +graph that stands for nothing, and 0009, 0010 and 0011 spent three records removing exactly +that species — `concept`, `related_to`, `mapped_to` — from the ontology on the grounds that +they were control flow. The two classes here are the same species, admitted through +`ensure_concept_types` without a proposal, without a person, and (until #231) without even +being noticed. + +Three things then have nowhere to live, and the benches measured each: + +- **A convention.** "Amounts are in cents; test orders are excluded; an order counts at + status 2, 3 or 4." Eighteen definitions in the wide corpus share six such sentences. They + are not an attribute of any Metric, not a field of any mapping row, not a column of + `knowledge_bases`. With them the model answers 17 of 18 questions; without them, 1 of 18 + (#520). It read every comment and still returned ¥45,992,467 for a GMV of ¥32,931,921, + because the comments say what a column *is* and nothing says which rows count. +- **An object whose fields sit in several tables.** `(kb_id, concept_id, source)` was meant + for one number defined two ways; it cannot say that a customer's tier is in the order + table and their name in the CRM. There is no key, no join, no class. +- **The connection between the graph and the database.** A concept entity has zero facts + (`facts_on_it = 0` for all twelve); its relation to `dwd_ord_dtl` exists only in the side + table. Rules (0021–0032) read facts and cannot see a mapping; chat reads mappings and + cannot see a rule. Two paths, no seam. + +## Dead ends + +- **Keep the two classes and stop the extractor from seeing them.** Treats the symptom. The + column-name entities are the visible half; the invisible half is that `order_amount` is + still a thing that does not exist. +- **A "conventions" text box on the base, read into both prompts.** Proposed during this + investigation, and it would move the wide corpus from 6% — the model does read prose. + Rejected because it is a place to put sentences instead of a place to put the thing the + sentences describe: a filter on an attribute, a unit on a column. Once those have a home + the box is redundant, and a box that stays is where the next undocumented convention goes + instead of into the ontology. + + > **Revised 2026-09-09, after measuring it.** The paragraph above rejected the box on + > principle and gave no number. The number: a one-page markdown of the six conventions, + > dropped into the "Data schemas" folder (retrieval only, after #553) and reached through + > `search_chunks`, takes the wide corpus from **2/18 to 14/18** (#520). Zero code, zero + > mappings. That is two-thirds of the gap to the seeded 17/18, and it says the cheap thing + > should exist — a person needs somewhere to write conventions *today*, and a document + > already works. What the four misses show is the reason the principle still stands: two + > of them are the page contradicting itself (its "refund total" counted fully-refunded + > orders GMV never held; "valid" and "paid" were both defined and the model divided by the + > wrong one). Prose lets two definitions be written that do not compose; an expression + > over attributes cannot be. So the order of work follows: a written definition (#562) now, + > the alignment and rules for what prose cannot make unambiguous. The dead end is not the + > box; it is the box as the *only* place. +- **Natural-language mappings.** Not executable, therefore not verifiable: the bench cannot + run "divide by a hundred". Every question re-translates the sentence, and #520's baseline + is the record of how that goes. +- **SQL text as the definition** (today's `expr` and `sql` columns). Executable and + verifiable, and wrong on two counts 0032 already made about rule expressions: it binds to a + dialect (`FILTER (WHERE …)` is Postgres; there are five engines), and it can name a column + that does not exist, so the error moves from unfillable to undiscovered. +- **Exploration proposing the definitions themselves** (GMV, refund rate). The schema does + not contain them — 0 of 18 on the wide corpus, and the model said in its own words that it + was searching for a definition of "valid order" and found none. What it would propose is + `sum(amt_pay)`: runs, looks right, off by a hundred and includes the test orders. That is + precisely what automatic confirmation (#504) must never ship. + +## Decisions + +### 1. Exploration produces an alignment proposal per table + +For each table it reads: the class it is a table *of*, the attributes its columns carry, the +relations its foreign keys are, and for each aligned column an expression that turns the +column into the attribute's value. It finds existing classes and attributes first and +proposes new ones only where none fits — 0003's loop with a schema in place of a corpus. + +It goes through `ontology_proposals`, which already has the four sections this needs +(`entity_types`, `attribute_types`, `relation_types`, `map_to`), and a person adopts it. **A +table's alignment is adopted or rejected as one thing.** Adopting `Order` while rejecting +every attribute of it is not a decision anyone means to make, so the page does not offer it. +Adoption writes the ontology rows and the alignment rows together. + +### 2. Not every column is an attribute, and exploration must say which class each one belongs to + +A wide table is several classes flattened into one. The alignment un-flattens it: + +| column | is | +|---|---| +| `amt_pay`, `qty`, `ord_st`, `dt_crt` | attributes of `Order` | +| `buyer_id`, `shop_id` | relations, `Order → Customer` and `Order → Shop` | +| `shop_nm`, `prov`, `buyer_lvl` | attributes of `Shop`, `Address`, `Customer`, flattened in | +| `ver`, `etl_dt`, `rmk`, `price_old` | nothing; left out | + +This is the judgement #502 was reaching for (a key is not a quantity) with the rest of it: +a key is a relation, a quantity is an attribute, and a flattened column is somebody else's +attribute. It is harder than what exploration does today and it is the whole value of doing +it. + +### 3. A conversion is a tree, the same tree as a rule expression + +`amt_pay / 100`, `CASE chnl WHEN 1 THEN 'app' … END`, `date_trunc('month', dt_crt)` are +stored as the expression tree 0032 defined — `{attr} | {const} | {op, l, r}` — extended with +the node kinds alignment needs and rules do not yet: a cast, a case, a date truncation. The +extension is driven by corpora, not designed ahead: the wide corpus needs those three, and +the next corpus says what else. + +A tree is engine-neutral and is rendered to each dialect at query time. A tree's leaves are +attribute ids, so it cannot reference a column that does not exist; the failure is at save +time, where a person is looking. + +**Input may be text.** 0032 allowed "a box that parses into this same tree", and this is +the case for it: a person and a model both write `amt_pay / 100` more readily than they +compose it from pickers. `sqlparser` is already a dependency (the read-only gate in +`query_engine` uses it); it parses the text into the tree, and what it cannot parse into a +supported node is refused. What 0032 forbade — storing the string and evaluating it at run +time — stays forbidden. + +The natural-language column stays, as explanation: `summary` is what a reviewer reads and +what the chat prompt quotes. The tree is the definition, the sentence is the description, +and neither replaces the other. + +### 4. A definition is a rule over aligned attributes, and a person writes it + +GMV is `sum(Order.paid_amount) where Order.is_valid`. It is not an entity and not a row in +a mapping table; it is a rule of the kind 0021 built and 0032 made computable, over +attributes the alignment made real. Exploration prepares the attributes; it does not propose +the rule (dead ends, above). + +A convention that many definitions share is **one rule that the others read** (0030): +`Order.is_valid ← status ∈ {paid, shipped, done} ∧ ¬is_test`, written once, inherited by +GMV, net sales, order count, refund rate. This is where "test orders don't count" lives — a +condition on an attribute, with a name, that a person wrote and can change in one place. + +The unit conversion lives on the alignment (`amt_pay / 100`), and the attribute carries the +unit (`relation_types.unit = 'CNY'`). 0032 noted that nothing checks units when an +expression is written; this record gives units a place to come from, so that check has +something to read. + +### 5. `Metric` and `Dimension` retire; `concept_mappings` becomes derived + +`ensure_concept_types` goes. No exploration inserts a class without a proposal. + +`concept_mappings` is not dropped: its status, revisions, audit stream and page (0011 §2–3) +are the review flow this record still needs. What changes is where a row comes from. Today a +row is the definition; after this, a row is **rendered** from an alignment and a rule — the +rule, the table it aligns to, the column expressions, the source's dialect — into the `sql` +it holds now. The bench (#501, #520) reads the same table it reads today and scores the same +way. A row that a person edited by hand and a row that was rendered are distinguishable, +because a rendered one names the rule it came from. + +### 6. Cold start: exploration proposes the class it cannot find + +A new base has no `Order`. Exploration proposes it, through the same proposals and the same +adoption, which is the difference from today: `ensure_concept_types` writes; a proposal +waits. A base on an ontology pack (0008) will more often find `Order` already there and +align to it, which is the case this record is written for. + +### 7. The schema document is a search corpus, not a source of facts + +Ingesting a schema as markdown gives chat something to retrieve (`search_chunks` finds the +table). Extracting it as if it were prose is what produced twenty-eight column-name +entities. The document keeps its place in retrieval and leaves the extraction path. This is +the one decision here that is a bug fix and can land on its own. + +## Open questions + +- **Where a rule over aligned attributes runs.** 0032 refused aggregation on the graph + because the graph is open-world and a sum asserts completeness. A database table is + closed-world: the table *is* all the rows, so `sum(Order.paid_amount)` is an honest + statement there. The alignment therefore gives a rule a place it can legitimately be + summed — but that place is another executor, translating the tree and the rule to a + dialect and running it at the source. Whether to build that, or to stop at rendering + `concept_mappings` rows for chat to use, is the largest decision this record does not + make. Today the two paths (documents → facts → rules; database → schema document → model + writes SQL) are separate, and this record only says where the seam would be. +- **Matching a column to an existing attribute.** By name? By sampled values (#502)? By + asking the model to say, with the attribute list in the prompt? The wide corpus, where + `amt_pay` must land on `Order.paid_amount` and `buyer_lvl` on `Customer.tier`, is the test. +- **What the tree needs beyond four operators**, and whether `case` over a code column is + a conversion (alignment) or a dimension (a rule that names groups). The wide corpus has + both readings of `chnl`. +- **The page.** How a person reviews a table's alignment, edits one column's expression, + and sees which rules a change reaches. Capability first, interface after, as usual. +- **`derived`** on `concept_mappings` (0011 §1) was for "conversion rate = orders / visits". + Under this record that is a rule reading two rules, and the flag has no meaning. It is + kept until the rendering exists and dropped with the migration that consolidates it. + +## What this does to 0011 + +0011's argument stands: a mapping is not an assertion about the world; it is configuration; +it lives outside the ledger and may be edited in place. This record accepts all of that and +changes one thing 0011 did not decide but its implementation assumed — that the *concept* a +mapping hangs from is an entity of a class called Metric. The concept is an attribute of a +real class, or a rule over such attributes, and the mapping is how a column becomes that +attribute's value. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index a6b4b5d5a..c35d78677 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -32,9 +32,9 @@ The test for writing one: if someone (including us) looks at a piece of code in | 0006 | [Ontology scale and the extraction prompt](0006-ontology-scale-and-the-prompt.md) | Built · the character budget (24,000) and per-chunk retrieval live, values untested · answer keys still hand-filled | | 0007 | [Counting decides what becomes a relation](0007-who-decides-what-becomes-a-relation.md) | Built · adoption decided by counting (`MIN_DOCS = 2`, `MIN_SIGNALS = 3`), proposals persist (#112) · narrative verbs and `_by` folding still open | | 0008 | [Ontology packs as the cold start](0008-ontology-packs-as-cold-start.md) | Built · five packs embedded, multi-select at creation, schema.org by default · three open questions stay open; Chinese labels got worse | -| 0009 | [An undecided type stays empty](0009-no-type-is-a-type.md) | Implemented · `type_id` nullable, builtin classes gone · kin classes go to Review (#226), declared `disjointWith` keeps them apart (0016 B3) · `metric` / `dimension` builtin on demand (#231) | +| 0009 | [An undecided type stays empty](0009-no-type-is-a-type.md) | Implemented · `type_id` nullable, builtin classes gone · kin classes go to Review (#226), declared `disjointWith` keeps them apart (0016 B3) · `metric` / `dimension` builtin on demand (#231) · `metric` / `dimension` to retire under 0036 | | 0010 | [An unnamed relation stays empty](0010-no-relation-is-no-relation.md) | Implemented · `predicate_id` nullable, `related_to` gone, wording recovered by `fact_surface_predicate` · follow-ups done with 0011 | -| 0011 | [A mapping is configuration](0011-a-mapping-is-not-a-fact.md) | Implemented (#126 / #140 / #148) · Review flow and revision history rebuilt · the evidence chain not built | +| 0011 | [A mapping is configuration](0011-a-mapping-is-not-a-fact.md) | Implemented (#126 / #140 / #148) · Review flow and revision history rebuilt · the evidence chain not built · revised in part by 0036: the concept becomes an attribute or a rule, the table becomes a rendered one | | 0012 | [The ontology is a contract](0012-the-ontology-is-a-contract-not-a-suggestion.md) | Implemented · violation rate 57% → 4%, reversals 39 → 0 · guard extended to adoption and merge (#190 / #196) · reified-shell filter at pack import open | | 0013 | [A source hands over its history](0013-a-source-should-hand-over-its-history.md) | Implemented for GitHub, Jira and Notion (#134 / #135 / #213) · Feishu and Confluence not started · `instant` precision not triggered | | 0014 | [Identity from the person, scope from the token](0014-identity-from-the-person-scope-from-the-token.md) | Implemented (#180) · five read-only MCP tools over Streamable HTTP · tokens page at `/account/tokens` · `can_write` still hard-coded false | @@ -56,9 +56,10 @@ The test for writing one: if someone (including us) looks at a piece of code in | 0030 | [A rule may read what a rule concluded](0030-a-rule-may-read-what-a-rule-concluded.md) | Cuts 1–2 · A rule could not read another rule's conclusion (`attribute_facts()` reads `facts`, subject scope is `entities.type_id`, one pass), so `A → B → C` has to be written as one flattened rule. The answer is the fixed point the axiom side already runs, **inside one call and in memory** — which answers 0013's objection instead of waiving it, since a run stays a pure function of the asserted facts. A derived typing enters as a **premise**, not a filter, so the conclusion narrows to the window the entity was actually a `B`; a rule may read its own conclusions because the derivable space is finite; `fact_derivations` gains `premise_derived_id` and the proof becomes a tree; retirement cascades for free because recompute is total. The page showing which rules feed which is next | | 0031 | [An event holds at the moment it names](0031-an-event-holds-at-the-moment-it-names.md) | Implemented (#486) · `Validity::under` normalises every write by the predicate's `temporal` (an event is one moment written at both ends, an eternal fact has no dates), `world_axis` and `read_span` read an event as the bucket it names and an undated event at no moment, an eternal fact at every moment · the prompt marks `[event]` / `[eternal]` and says what to write · no schema change, old rows read correctly · the panel's point rendering and the ontology hint are the UI cut | | 0032 | [A rule computes what it concludes](0032-a-rule-computes-what-it-concludes.md) | Cuts 1–2 · A rule's conclusion was a **constant** (`attribute_rules.conclude_value` is a JSONB literal), so a criterion about a number nobody wrote down — net pay, a margin, a ratio — gets computed elsewhere and typed back in as an assertion, losing the readings it stood on. An operand on either side becomes an expression tree over attributes: its premises are the readings it touched and its interval is their intersection, so retirement needs no new machinery. **Picked, not typed** — not because a parsed string would break the display (it would not; that reason is wrong and recorded as wrong) but because a picker cannot compose an expression over a predicate that does not exist, and because a text box grows a grammar on request. A value may be reached **across a relation** (a path has definite premises and an interval; a to-many path is just a cartesian product the evaluator already walks), while **aggregation is refused**: a `sum` asserts *these are all the readings*, a completeness claim this base cannot hold — and while recompute still retires it correctly, its proof cannot say why it was withdrawn, and nothing incremental could ever wake it. Row count, unit checking and division by zero are the costs | -| 0035 | [A vector index is built by a job](0035-a-vector-index-is-built-by-a-job.md) | Implemented · a partial HNSW index per dimension on `chunks.embedding` and `entities.profile_embedding`, requested by the first write of that dimension and built by a `build_vector_index` job outside any transaction · the two nearest-neighbour reads write the dimension as a literal, cast both sides and set `hnsw.iterative_scan = relaxed_order` · type resolution gathers its neighbours eight at a time in order and remembers descendant sets per batch (#512, #514) · dimensions above 2000 stay on the exact path | | 0033 | [RSS summaries are scoped to the source being listed](0033-rss-source-summaries-are-source-scoped.md) | Query implemented for #417 · source- and generation-scoped lateral aggregation preserves the flat API; nested `rss_full_content` public contract pending; baseline rows are intentionally outside the five work counts | | 0034 | [An action is a declared call](0034-an-action-is-a-declared-call.md) | Cut 1 · the record. A base can conclude but cannot act: nothing holds a call it may make, a typed parameter, or what it sent. An action is **data** (method, URL, headers, a sealed auth block, a JSON body template, scalar parameters with bounds or a value set), substituted and never scripted, so the page renders what runs and an unknown placeholder is refused at save. Registered once for the deployment and **granted** to a base, one layer where warehouses have two because nothing is mounted. Every run is a row read by two log pages. The reach is the operator's, with no placeholder in the host. Rules fire actions in the next record | +| 0035 | [A vector index is built by a job](0035-a-vector-index-is-built-by-a-job.md) | Implemented · a partial HNSW index per dimension on `chunks.embedding` and `entities.profile_embedding`, requested by the first write of that dimension and built by a `build_vector_index` job outside any transaction · the two nearest-neighbour reads write the dimension as a literal, cast both sides and set `hnsw.iterative_scan = relaxed_order` · type resolution gathers its neighbours eight at a time in order and remembers descendant sets per batch (#512, #514) · dimensions above 2000 stay on the exact path | +| 0036 | [Exploration aligns a schema to the ontology](0036-exploration-aligns-a-schema-to-the-ontology.md) | Written, not implemented · exploration proposes an alignment per table (the class it is a table of, its attributes, its relations, a conversion tree per column) through `ontology_proposals`, adopted as one thing · a definition is a rule over aligned attributes, written by a person; a shared convention is one rule others read · conversions are 0032's expression tree, text parsed by `sqlparser`, never stored as SQL · `Metric` / `Dimension` retire and `concept_mappings` becomes rendered · the schema document leaves extraction · where a rule runs against the source stays open | ## Not a decision record diff --git a/migrations/0044_an_exploration_says_what_it_covered.sql b/migrations/0044_an_exploration_says_what_it_covered.sql new file mode 100644 index 000000000..dad89a3d6 --- /dev/null +++ b/migrations/0044_an_exploration_says_what_it_covered.sql @@ -0,0 +1,51 @@ +-- 一次映射探索扫了什么、丢了什么、剩下什么（#503）。 +-- +-- 从前一轮探索只留下两样东西：`concept_mappings` 里若干行，以及一条 +-- 「一条都没提出来」的告警。**十一条提议对着一张八十列的宽表，与十一条 +-- 对着一个刚好覆盖完的小库，页面上长得一模一样**——覆盖率没有任何地方说。 +-- +-- 而丢弃是静默的。`explore_mappings` 对每条提议有四处 `continue`：源名对不上、 +-- kind 不是 metric/dimension、概念类查不到、definition 不是对象。四处都不计数。 +-- 实测踩过一次：源叫 `tpch-2026-09-08-12-30`，模型看着 schema 回的 `source` +-- 是 `tpch`，十二条提议一条不剩地被吞掉，任务照样 done。 +-- +-- 所以把一轮探索本身记下来。**它不是审计事件**：审计答的是「谁做了什么」， +-- 这张表答的是「这一轮看见了多大的东西、覆盖了多少」，是下一次要不要再跑、 +-- 要不要给列加注释的依据。 + +CREATE TABLE mapping_exploration_runs ( + id UUID PRIMARY KEY, + kb_id UUID NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + finished_at TIMESTAMPTZ, + + -- 这一轮读了哪些源 + sources TEXT[] NOT NULL DEFAULT '{}', + tables_scanned INTEGER NOT NULL DEFAULT 0, + columns_scanned INTEGER NOT NULL DEFAULT 0, + -- schema 文本撞了上限：**提示词里没有的表，模型没有机会提**。 + -- 覆盖率低的时候这一位说明是「没看见」还是「看见了没提」 + schema_truncated BOOLEAN NOT NULL DEFAULT FALSE, + + -- 这一轮允许提几条（按表数放大，不再是写死的 12） + cap INTEGER NOT NULL DEFAULT 0, + -- 模型回了几条 / 落库几条。两者之差就是被丢掉的 + returned INTEGER NOT NULL DEFAULT 0, + accepted INTEGER NOT NULL DEFAULT 0, + -- 丢弃分类：{"source": n, "kind": n, "type": n, "definition": n}。 + -- 分开数是因为该做的事不同——源名对不上要改源名或改提示词， + -- 概念类查不到是本体缺了类，definition 不成形是模型没听懂格式 + dropped JSONB NOT NULL DEFAULT '{}'::jsonb, + + -- 有至少一条提议落在上面的表，`schema.table` 原样。覆盖率的分子； + -- 分母是 tables_scanned + tables_covered TEXT[] NOT NULL DEFAULT '{}', + + -- 跑挂了的那一轮也留一行：**失败与「跑了但什么都没提」不是一回事**， + -- 而从前两者在页面上都是「没有新提议」 + error TEXT +); + +-- 页面只要最近那几轮 +CREATE INDEX mapping_exploration_runs_idx + ON mapping_exploration_runs (kb_id, started_at DESC); diff --git a/scripts/bench/README.md b/scripts/bench/README.md index 4d3427265..ed790cb8f 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -120,6 +120,90 @@ node scripts/bench/govern.mjs --kb --score --stuck # 连留给人的那 同一份语料上的对照（2026-09-06，闸门还是「没先例不合」那版）：治理关着老裁决器自动合 146、留 50 给人；治理开着一对不合、留 200。这个数就是记录 0025 决定 4 修订的起因。 +## 映射的测量台（#501） + +`mappings.mjs` 量的是探索从数据库 schema 提议的口径，对不对、漏了多少。 + +``` +node scripts/bench/mappings.mjs --fresh # 新库 → 挂源 → 探索 → 打分 +node scripts/bench/mappings.mjs --fresh --no-comments # 同上，语料不带列注释 +node scripts/bench/mappings.mjs --kb --score # 只打分，不动库 +``` + +**打分不看名字，看数。** 治理那边真值按两个名字键，因为它判的是二分类；这里概念名 +是模型自己起的，「Revenue」与「已支付 GMV」按名字对不上任何一条。所以真值一条是 +「一个业务口径 + 一条 gold SQL」，打分把提议真跑一遍，跟 gold 的结果比数——数一样 +就是同一个口径。省掉逐条手标，判据还是客观的。 + +四栏分开读：`covered` 是真值里被算出来的有几条（漏没漏）；`right` 是提议里跑得通 +且对上某条真值的；`wrong` 是**跑得通但一条都对不上**的，附它算出的数与最接近的真值； +`broken` 是跑不通的。**要看的是 `wrong`。** 跑不通的提议无害，它失败得很响，人一眼 +看得见；跑得通而算错的才是全部风险——问数会拿它印出一个看起来完全正常的数字。这一栏 +也是 #504 能不能默认开的唯一依据。 + +`traps_hit` 数的是落到陷阱列上的提议：整数外键求和、`o_shippriority` 这种恒为 0 的列、 +把一段自由文本当维度。它们都跑得通。 + +语料在 `schemas/.sql`，真值在 `truth/.mappings.json`。 + +- **口径不是我们定的。** TPC-H 的 22 条查询里写死了「收入」在这个 schema 上就是 + `sum(l_extendedprice * (1 - l_discount))`，真值从那儿抄。自己拟一份口径来量自己的 + 探索，量出来的是自我一致。 +- **行是 `generate_series` 造的，不按 TPC 的生成规范。** 探索只读 schema 不读数据， + 打分时 gold 与提议跑在同一批行上，比的是两个数一不一样——那份规范值钱的是 schema + 与查询，不是它的数据分布。`setseed` 固定，两轮之间语料不变。 +- **列注释单独一个文件，因为它是一个自变量。** 真 TPC-H 一条注释都没有，而真实库里 + 注释是探索最主要的线索（提示词专门要求 citing column comments）。加载与不加载各跑 + 一轮，两个分数之差就是注释值多少分。注释只描述列，不描述口径——写「折后收入 = + extendedprice × (1 - discount)」等于把 gold SQL 抄给模型。 +- **每一组一个新库**在这里还多一层理由：`propose` 的 `ON CONFLICT … WHERE status = + 'proposed'` 让第二轮探索继承第一轮的行，同一个库上跑两次，第二次的分不是第二次的。 + +## 问数的测量台（#520） + +`ask.mjs` 量的是端到端：人问一句话，拿回来的数对不对。 + +``` +node scripts/bench/ask.mjs --kb # 跑全部问题 +node scripts/bench/ask.mjs --kb --only disc_revenue +node scripts/bench/ask.mjs --kb --seed # 先把真值写成确认口径（上界） +node scripts/bench/ask.mjs --kb --confirm # 先确认探索提的那些（产品路径） +node scripts/bench/ask.mjs --kb --replay # 不重问，拿库里上一轮的回答重判 +node scripts/bench/ask.mjs --kb --parallel 4 # 同时问四题：一题 2–6 个模型回合、串行半小时 +``` + +**与 `mappings.mjs` 量的不是一回事，一个也推不出另一个。** 口径确认得再准， +答案照样可能错——模型会挑错源、join 错、按错的日期列过滤，或者压根不看语义层， +照着 schema 文档自己写 SQL。反过来，一个一条确认映射都没有的库照样答得出问题， +而且有时是对的。 + +判分材料只有两样，因为 `query_data` 记得很少（step 里只有源名与用途）：助手最后 +那段话，以及 `tool_exchange` 里**模型真正跑过的 SQL**。后者是有用的那个—— +把它跑过的 SQL 重跑一遍，跟 gold 比数。SQL 很短，逃得过 `tool_exchange` 的截断， +而它的输出逃不过。 + +于是两栏而不是一栏：`sql_right` 是它跑的那条算的就是问题问的数，`answer_right` +是它印出来的数就是那个数。**两者会分家**——查对了却把单位说错、四舍五入错、 +或者转述成另一个数字，在只看 SQL 的分里是对的，而人读到的答案是错的。判等用 +`lib.mjs` 的 `roughly`（比 `same` 松两个量级）：模型说「约 8.63 亿」与 +862793473.48 是同一个答案，判成错的话，量的就不是问数准不准，是模型肯不肯把 +小数点后八位抄全。 + +问题在 `truth/tpch.questions.json`，**按口径的 id 键**，与 `tpch.mappings.json` +共用 gold SQL。这不只是省事：一道题答错时，脚本据此说得出它需要的那条口径**有没有 +确认映射**——`mapped` / `UNMAPPED`。闭环就在这一句上： + +1. 跑一轮，看哪些错； +2. 错的题里，口径没配映射的 → 覆盖率的问题，去探索或手写； +3. 配了还错的 → 提示词、工具或口径本身的问题； +4. 再跑。 + +`mapped_definitions` 这一栏满了，「映射全」就有了确定的意思，剩下的错都不再是 +覆盖率的事。 + +`lib.mjs` 是两个测量台共用的地基——**判等必须是同一份**，各写一份 `same()` +迟早漂移，而一旦漂移，「提议对了几条」与「答案对了几条」就不是同一把尺子量出来的。 + ## 读数怎么算 - `prompt_tokens_est` 是**本体段**的估算，不是整个提示词。实测 4.0 字符 ≈ 1 token diff --git a/scripts/bench/ask.mjs b/scripts/bench/ask.mjs new file mode 100644 index 000000000..37def27bd --- /dev/null +++ b/scripts/bench/ask.mjs @@ -0,0 +1,320 @@ +#!/usr/bin/env node +// 问数的端到端测量台（#520）：人问一句话，拿回来的数对不对。 +// +// **与 `mappings.mjs` 量的不是一回事，而且一个推不出另一个。** 口径确认得 +// 再准，答案照样可能错——模型会挑错源、join 错、按错的日期列过滤，或者压根 +// 不看语义层、照着 schema 文档自己写 SQL。反过来，一个一条确认映射都没有的 +// 库照样答得出问题，靠的是 schema 文档，而且有时是对的。 +// +// 判分材料只有两样，因为 `query_data` 记得很少（step 里只有源名与用途）： +// 1. 助手最后那段话； +// 2. `tool_exchange` 里的工具调用——**模型真正跑过的 SQL**。 +// 后者是有用的那个：**把它跑过的 SQL 重跑一遍，跟 gold 的结果比数**。SQL 很短， +// 逃得过 `tool_exchange` 的截断，而它的输出逃不过。 +// +// 两栏而不是一栏： +// sql_right 它跑的那条 SQL 算的是问题问的那个数 +// answer_right 它印出来的数就是那个数 +// 两者会分家。查对了却把单位说错、四舍五入错、或者转述成另一个数字的模型， +// 在只看 SQL 的分里是对的，而人读到的答案是错的。 +// +// 用法： +// node scripts/bench/ask.mjs --kb # 跑全部问题 +// node scripts/bench/ask.mjs --kb --only disc_revenue +// node scripts/bench/ask.mjs --kb --confirm # 先确认探索提的那些（产品路径） +// node scripts/bench/ask.mjs --kb --seed # 先把真值写成确认口径（上界） +// node scripts/bench/ask.mjs --kb --replay # 不重问，拿库里上一轮的回答重判 +// node scripts/bench/ask.mjs --kb --parallel 4 # 同时问四题（缺省一题一题来） +// +// 环境变量见 lib.mjs。 + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { + BASE, api, login, psql, value, firstRow, same, roughly, log, parseArgs, cookieHeader, +} from "./lib.mjs"; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const args = parseArgs(process.argv); +const corpusName = args.corpus || "tpch"; +const truth = JSON.parse(fs.readFileSync(path.join(HERE, "truth", `${corpusName}.mappings.json`), "utf8")); +const qs = JSON.parse(fs.readFileSync(path.join(HERE, "truth", `${corpusName}.questions.json`), "utf8")); +const CORPUS_DB = process.env.BENCH_CORPUS_DB || `bench_${corpusName}`; + +/// 一轮问答。SSE 帧是 `event: X\ndata: {...}\n\n`。 +async function ask(kb, message) { + const res = await fetch(`${BASE}/api/v1/kbs/${kb}/chat`, { + method: "POST", + headers: { "content-type": "application/json", cookie: cookieHeader() }, + body: JSON.stringify({ message }), + }); + if (!res.ok) throw new Error(`chat -> ${res.status} ${(await res.text()).slice(0, 200)}`); + const reader = res.body.getReader(); + const dec = new TextDecoder(); + let buf = "", text = "", conversation = null, error = null; + const steps = []; + for (;;) { + const { done, value: chunk } = await reader.read(); + if (done) break; + buf += dec.decode(chunk, { stream: true }); + let i; + while ((i = buf.indexOf("\n\n")) >= 0) { + const frame = buf.slice(0, i); + buf = buf.slice(i + 2); + const ev = /^event: ?(.*)$/m.exec(frame)?.[1]; + const data = frame + .split("\n") + .filter((l) => l.startsWith("data:")) + .map((l) => l.slice(5).replace(/^ /, "")) + .join("\n"); + try { + if (ev === "conversation") conversation = JSON.parse(data).id; + else if (ev === "delta") text += JSON.parse(data).text ?? ""; + else if (ev === "step") steps.push(JSON.parse(data)); + else if (ev === "error") error = data; + } catch { + /* 半帧或非 JSON：下一帧再说 */ + } + } + } + return { conversation, text, steps, error }; +} + +/// 模型跑过的 SQL。**递归找**，因为工具调用的 `arguments` 本身是一段 JSON +/// 字符串，而这一层的形状随模型端而变（有的给 tool_calls，有的给 function_call）。 +function sqlsIn(node, out = []) { + if (typeof node === "string") { + const s = node.trim(); + if (s.startsWith("{") || s.startsWith("[")) { + try { sqlsIn(JSON.parse(s), out); } catch { /* 就是一段普通文本 */ } + } + return out; + } + if (Array.isArray(node)) { for (const n of node) sqlsIn(n, out); return out; } + if (node && typeof node === "object") { + for (const [k, v] of Object.entries(node)) { + if (k === "sql" && typeof v === "string" && v.trim()) out.push(v.trim()); + else sqlsIn(v, out); + } + } + return out; +} + +/// 上一轮问过的答案，从库里读回来重判。 +/// +/// **问一轮很贵**（二十四题四十分钟，还有模型的账），而判分的口径会改—— +/// 头一轮就改了两次。会话本来就完整存在库里，所以重判不该重问：改了判据 +/// 拿旧回答重跑一遍，才谈得上多轮迭代。 +/// +/// 同一个问题问过多次时取最新的一次。 +function replayFromDb(kb) { + const rows = JSON.parse(psql(`SELECT coalesce(json_agg(x), '[]') FROM ( + SELECT c.id, + (SELECT content FROM conversation_messages + WHERE conversation_id = c.id AND role = 'user' + ORDER BY created_at LIMIT 1) AS ask, + (SELECT content FROM conversation_messages + WHERE conversation_id = c.id AND role = 'assistant' + ORDER BY created_at DESC LIMIT 1) AS said, + (SELECT coalesce(json_agg(tool_exchange), '[]') FROM conversation_messages + WHERE conversation_id = c.id AND role = 'assistant') AS ex + FROM conversations c + WHERE c.kb_id = '${kb}' + ORDER BY c.created_at DESC) x`)); + const byAsk = new Map(); + for (const r of rows) if (r.ask && !byAsk.has(r.ask.trim())) byAsk.set(r.ask.trim(), r); + return byAsk; +} + +/// 把真值全部写成确认过的口径（`--seed`）。 +/// +/// **这是上界，不是产品路径。** 探索提不提得出这些口径是 #501 的事；这里 +/// 直接把答案配进语义层，问的是另一个问题：**口径完备时，问数能到多少？** +/// 基线（一条确认映射都没有，模型照着 schema 文档自己写 SQL）与这一轮之差， +/// 就是语义层在这份语料上值多少分。 +/// +/// 概念实体按名字复用探索建好的那些；`Metric` 类由探索的 `ensure_concept_types` +/// 建下，所以这一步要在跑过一轮探索的库上做。 +function seedTruth(kb) { + const typeId = psql(`SELECT id FROM entity_types WHERE kb_id = '${kb}' AND key = 'metric'`); + if (!typeId) throw new Error("这个库还没有 Metric 类——先跑一轮探索"); + let n = 0; + for (const m of [...truth.metrics, ...(truth.plausible ?? [])]) { + // **概念名加个记号，别撞上探索建的实体。** 头一次跑没加，`Average order + // value` 正好与探索起的名字同名，`ON CONFLICT … DO UPDATE` 就把那条提议的 + // sql 改成了 gold——上界那一轮顺手污染了产品路径那一轮的语料 + const label = `${m.label} (truth)`.replace(/'/g, "''"); + const gold = m.gold.replace(/'/g, "''"); + const summary = `${m.label} — ${m.from ?? "bench truth"}`.replace(/'/g, "''"); + const ent = psql(` + WITH found AS (SELECT id FROM entities + WHERE kb_id = '${kb}' AND lower(canonical_name) = lower('${label}') + AND merged_into IS NULL LIMIT 1), + made AS (INSERT INTO entities (id, kb_id, type_id, canonical_name, aliases) + SELECT gen_random_uuid(), '${kb}', '${typeId}', '${label}', '{}' + WHERE NOT EXISTS (SELECT 1 FROM found) RETURNING id) + SELECT id FROM found UNION ALL SELECT id FROM made`); + psql(` + INSERT INTO concept_mappings (id, kb_id, concept_id, source, table_name, sql, summary, status) + VALUES (gen_random_uuid(), '${kb}', '${ent}', '${corpusName}', NULL, '${gold}', '${summary}', 'confirmed') + ON CONFLICT (kb_id, concept_id, source) + DO UPDATE SET sql = EXCLUDED.sql, summary = EXCLUDED.summary, + status = 'confirmed', updated_at = now()`); + n++; + } + log(`真值已写成 ${n} 条确认口径`); +} + +/// 把探索提出的口径全部确认（`--confirm`），模拟人审完一轮。 +/// +/// **这是产品路径上那一格。** `--seed` 把真值直接配进去，量的是上界； +/// 什么都不配，量的是下界（模型照着 schema 文档自己写 SQL）。真实的库 +/// 落在中间：探索提多少、人确认多少，问数就拿到多少。 +/// +/// 全部确认而不是只确认对的那些，因为这一轮 tpch 上探索的 `wrong` 是 0 +/// （#501），一个照着看的人会把它们都点过——**审阅者不知道哪条是对的， +/// 那正是他要判断的事**。 +function confirmProposals(kb) { + const before = Number(psql(`SELECT count(*) FROM concept_mappings + WHERE kb_id = '${kb}' AND status = 'proposed'`)); + psql(`UPDATE concept_mappings + SET status = 'confirmed', decided_at = now(), + decided_by = (SELECT id FROM users ORDER BY created_at LIMIT 1) + WHERE kb_id = '${kb}' AND status = 'proposed'`); + log(`探索提议已确认 ${before} 条`); +} + +/// 答案文本里的数字。千分位逗号去掉；引用标记 `[3]` 也会被算进来， +/// 但它撞上一条真值的概率可以忽略——真值里最小的是 1.21 +function numbersIn(text) { + const out = []; + for (const m of text.matchAll(/-?\d[\d,]*\.?\d*/g)) { + const n = Number(m[0].replace(/,/g, "")); + if (Number.isFinite(n)) out.push(n); + } + return out; +} + +const main = async () => { + await login(); + const kb = args.kb; + if (!kb || kb === true) throw new Error("要一个 --kb "); + + if (args.seed) seedTruth(kb); + if (args.confirm) confirmProposals(kb); + + // 真值：单值口径 + 那些站得住但 22 条查询没说的 + const gold = new Map( + [...truth.metrics, ...(truth.plausible ?? [])].map((m) => [m.id, { ...m, value: value(CORPUS_DB, m.gold).n }]), + ); + + // **这个问题需要的口径，有没有一条确认过的映射？** 闭环就在这一句上： + // 答错的题分两种，一种是口径没配（去补映射），一种是配了还错（去看提示词 + // 或工具）。两种该做的事完全不同，而从前它们在结果里长得一样 + const confirmed = JSON.parse(psql(`SELECT coalesce(json_agg(x), '[]') FROM ( + SELECT m.table_name, m.expr, m.sql FROM concept_mappings m + WHERE m.kb_id = '${kb}' AND m.status = 'confirmed') x`)); + const mapped = new Set(); + for (const m of confirmed) { + const sql = m.sql?.trim() + || (m.expr && m.table_name + ? `SELECT ${m.expr} FROM ${m.table_name.includes(".") ? m.table_name : `${truth.db_schema}.${m.table_name}`}` + : null); + if (!sql) continue; + const got = value(CORPUS_DB, sql); + if (got.n === undefined) continue; + for (const [id, g] of gold) if (same(g.value, got.n)) mapped.add(id); + } + + // --replay：不重问，拿库里上一轮的回答重判 + const replay = args.replay ? replayFromDb(kb) : null; + const questions = qs.questions.filter((q) => (args.only && args.only !== true ? q.id === args.only : true)); + const c = { right: 0, sql_only: 0, answer_only: 0, wrong: 0, no_sql: 0, failed: 0 }; + const rows = []; + // **一题一个会话，互不共享状态，所以能并发问**（`--parallel N`，缺省 1）。 + // 串行时一题 2–6 个模型回合、平均一分半，十八题半小时；服务端的 + // `model_concurrency` 缺省 10，闸门不是我们。上限别开太高：模型端会限流 + async function one(q, i) { + const g = gold.get(q.id); + if (!g || !Number.isFinite(g.value)) { log(`跳过 ${q.id}：真值算不出来`); return; } + let r; + if (replay) { + const prev = replay.get(q.ask.trim()); + if (!prev) { log(`跳过 ${q.id}：库里没有问过这一句`); return; } + r = { conversation: prev.id, text: prev.said ?? "", exchange: prev.ex ?? [] }; + } else { + try { + r = await ask(kb, q.ask); + } catch (e) { + c.failed++; + rows.push({ i, text: `FAILED ${q.id} — ${String(e.message).slice(0, 120)}` }); + return; + } + r.exchange = r.conversation + ? JSON.parse(psql(`SELECT coalesce(json_agg(tool_exchange), '[]') FROM conversation_messages + WHERE conversation_id = '${r.conversation}' AND role = 'assistant'`)) + : []; + } + const sqls = sqlsIn(r.exchange); + // **第一行的每一列都算数。** 模型问「平均行金额」跑的是 + // `SELECT COUNT(*), AVG(l_extendedprice), MIN(…), MAX(…)`，只看第一列 + // 拿到的是行数，一条完全正确的查询会被判成错的 + const ran = sqls.map((s) => ({ s, ns: firstRow(CORPUS_DB, s.replace(/;\s*$/, "")).ns ?? [] })); + // **这里用 `roughly` 而不是 `same`。** 模型会自己 `ROUND(…, 2)`——那是 + // 它的格式选择，不是另一个口径；594.75 与 594.74915 的相对误差刚好越过 + // `same` 的 1e-6，于是一条完全正确的查询被判成算了别的东西。 + // `mapped` 那边仍然用 `same`，因为它是在二十七条真值里挑中一条，认错了 + // 就是认错了 + const sqlRight = ran.some((x) => x.ns.some((n) => roughly(n, g.value))); + const answerRight = numbersIn(r.text).some((n) => roughly(n, g.value)); + + let verdict; + if (sqlRight && answerRight) { verdict = "RIGHT"; c.right++; } + else if (sqlRight) { verdict = "SQL-ONLY"; c.sql_only++; } + else if (answerRight) { verdict = "ANSWER-ONLY"; c.answer_only++; } + else if (sqls.length === 0) { verdict = "NO-SQL"; c.no_sql++; } + else { verdict = "WRONG"; c.wrong++; } + + const flag = mapped.has(q.id) ? "mapped" : "UNMAPPED"; + log(`${verdict.padEnd(11)} ${q.id} (${flag})`); + if (verdict !== "RIGHT") { + rows.push({ i, text: + `${verdict} ${q.id} (${flag}) truth ${g.value}\n` + + ` asked: ${q.ask}\n` + + (ran.length + ? ran.map((x) => ` ran: ${x.s.replace(/\s+/g, " ").slice(0, 150)} → ${x.ns.join(", ")}`).join("\n") + : " ran: (没跑任何 SQL)") + + `\n said: ${r.text.replace(/\s+/g, " ").slice(0, 200)}`, + }); + } + } + const parallel = Math.max(1, Math.min(8, Number(args.parallel) || 1)); + let next = 0; + await Promise.all(Array.from({ length: parallel }, async () => { + while (next < questions.length) { + const i = next++; + await one(questions[i], i); + } + })); + // 并发之后完成顺序是乱的；报告按题目顺序排，两轮之间才好对着看 + rows.sort((a, b) => a.i - b.i); + + const n = questions.length; + const pct = (a) => (n ? `${Math.round((a / n) * 100)}%` : "—"); + console.log(JSON.stringify({ + kb, corpus: corpusName, questions: n, + right: `${c.right}/${n} (${pct(c.right)})`, + sql_only: c.sql_only, + answer_only: c.answer_only, + wrong: c.wrong, + no_sql: c.no_sql, + failed: c.failed, + // 口径有确认映射的题占多少。**「映射全」有了确定的意思**：这一栏满了， + // 剩下的错就都不是覆盖率的问题 + mapped_definitions: `${[...gold.keys()].filter((id) => mapped.has(id)).length}/${gold.size}`, + }, null, 2)); + if (rows.length) console.log("\n" + rows.map((r) => r.text).join("\n\n")); +}; + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/scripts/bench/lib.mjs b/scripts/bench/lib.mjs new file mode 100644 index 000000000..21fdda4be --- /dev/null +++ b/scripts/bench/lib.mjs @@ -0,0 +1,134 @@ +//! 两个映射测量台共用的地基：`mappings.mjs` 量提议，`ask.mjs` 量答案。 +//! +//! **判等必须是同一份。** 两边都在做「跑一条 SQL，跟 gold 的结果比数」， +//! 各写一份 `same()` 迟早漂移，而一旦漂移，「提议对了几条」与「答案对了几条」 +//! 就不是同一把尺子量出来的，放在一起比较就没有意义。 + +import { execFileSync } from "node:child_process"; + +export const BASE = process.env.BENCH_BASE || "http://127.0.0.1:8322"; +export const EMAIL = process.env.BENCH_EMAIL || "bench@test.local"; +export const PASSWORD = process.env.BENCH_PASSWORD || "benchbench123"; +const PSQL = process.env.BENCH_PSQL || "docker exec -e PGPASSWORD=utopia landscapebi-db-1 psql -U utopia -d utopia -tAc"; +/// 应用库与语料库都不是 BENCH_PSQL 里写的那个：**测量台跑在自己的库上** +/// （bench/README 的第一条规则），而 BENCH_PSQL 与 govern.mjs 共用，指着开发库 +export const APP_DB = process.env.BENCH_APP_DB || "utopia_mapbench"; + +export function parseArgs(argv) { + return Object.fromEntries( + argv.slice(2).reduce((acc, cur, i, arr) => { + if (cur.startsWith("--")) acc.push([cur.slice(2), arr[i + 1]?.startsWith("--") ? true : (arr[i + 1] ?? true)]); + return acc; + }, []), + ); +} + +let cookie = ""; +export async function api(method, url, body) { + const init = { method, headers: {} }; + if (cookie) init.headers.cookie = cookie; + if (body !== undefined) { init.headers["content-type"] = "application/json"; init.body = JSON.stringify(body); } + const r = await fetch(BASE + url, init); + for (const c of r.headers.getSetCookie?.() ?? []) cookie = c.split(";")[0]; + const text = await r.text(); + if (!r.ok) throw new Error(`${method} ${url} -> ${r.status} ${text.slice(0, 200)}`); + return text ? JSON.parse(text) : null; +} +export const cookieHeader = () => cookie; + +/// 登录；账号不在就按首用户注册（新库上测量台该能从头跑起来， +/// 而首个注册的人建 org 与 workspace 并且是 admin——注册数据源要 admin） +export async function login() { + try { + await api("POST", "/api/v1/auth/login", { email: EMAIL, password: PASSWORD }); + } catch { + await api("POST", "/api/v1/auth/register", { + email: EMAIL, password: PASSWORD, display_name: "bench", org_name: "bench", + }); + } +} + +/// 把命令行里的 `-d <库>` 换成指定的库；命令行里本来没写 `-d`（PGDATABASE 那种写法） +/// 就插在 SQL 前面——**不能是「换不到就算了」**：那样每条 onDb() 都落到命令的默认库， +/// 而 DROP SCHEMA 与模型写的 SQL 都走 onDb() +function withDb(cmdline, db) { + const parts = cmdline.split(" "); + const i = parts.indexOf("-d"); + if (i >= 0 && i + 1 < parts.length) parts[i + 1] = db; + else parts.splice(parts.length - 1, 0, "-d", db); + return parts.join(" "); +} +function run(cmdline, sql) { + const parts = cmdline.split(" "); + return execFileSync(parts[0], [...parts.slice(1), sql], { encoding: "utf8", maxBuffer: 64 << 20 }).trim(); +} +export const psql = (sql) => run(withDb(PSQL, APP_DB), sql); +export const onDb = (db, sql) => run(withDb(PSQL, db), sql); +export const num = (sql) => Number(psql(sql) || 0); + +/// 一条 SQL 第一行里的所有数字。 +/// +/// **模型不写单列查询。** 问「平均行金额」，它跑的是 +/// `SELECT COUNT(*), AVG(l_extendedprice), MIN(...), MAX(...)`——一次把上下文 +/// 都查出来。只看第一列就拿到 60000（行数），把一条完全正确的查询判成错的； +/// 头一轮基线上五道题栽在这里，全被记成「数对了但 SQL 不对」。 +export function firstRow(db, sql) { + try { + const out = onDb(db, `SET statement_timeout = '20s'; ${sql}`); + const first = out.split("\n").map((l) => l.trim()).filter((l) => l !== "" && l !== "SET")[0]; + if (first === undefined) return { empty: true }; + const ns = first.split("|").map((x) => Number(x)).filter((n) => Number.isFinite(n)); + return { ns }; + } catch (e) { + return { error: String(e.stderr || e.message).split("\n").filter((l) => l.trim())[0]?.slice(0, 120) }; + } +} + +/// 一条 SQL 跑出来的第一个值。口径的定义是单值的，用这个； +/// 判模型跑过的 SQL 用 [`firstRow`]。 +export function value(db, sql) { + try { + const out = onDb(db, `SET statement_timeout = '20s'; ${sql}`); + // **命令标签也走 stdout。** `SET statement_timeout` 先打一行 `SET`， + // 而 -tA 下数据行不带标签——不滤掉它，每条读到的第一行都是 `SET` + const first = out.split("\n").map((l) => l.trim()).filter((l) => l !== "" && l !== "SET")[0]; + if (first === undefined) return { empty: true }; + return { n: Number(String(first).split("|")[0]) }; + } catch (e) { + return { error: String(e.stderr || e.message).split("\n").filter((l) => l.trim())[0]?.slice(0, 120) }; + } +} + +/// 两个数是不是同一个口径算出来的。 +/// +/// **容差要松到吃得下小数舍入，紧到分得开两个口径**——tpch 里 charge 与 +/// order_total 是同一个业务量的两条算法，差在分位上，判成同一条是对的； +/// 而 disc_revenue 与 discount_given 差着一个数量级。 +export const same = (a, b) => { + if (!Number.isFinite(a) || !Number.isFinite(b)) return false; + if (a === b) return true; + return Math.abs(a - b) <= 1e-6 * Math.max(Math.abs(a), Math.abs(b), 1); +}; + +/// 人读答案时的同一个数。**比 `same` 松两个量级**：模型说「约 8.63 亿」 +/// 与 862793473.48 是同一个答案，而把它判成错的话，量的就不是问数准不准， +/// 是模型肯不肯把小数点后八位抄全。 +export const roughly = (a, b) => { + if (!Number.isFinite(a) || !Number.isFinite(b)) return false; + if (same(a, b)) return true; + return Math.abs(a - b) <= 5e-3 * Math.max(Math.abs(a), Math.abs(b), 1); +}; + +export const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +export const log = (...a) => console.error(new Date().toISOString().slice(11, 19), ...a); + +export async function until(fn, everyMs, stallMs) { + let deadline = Date.now() + stallMs, last = null; + for (;;) { + const r = await fn(); + if (r === true) return; + if (typeof r === "number" && r !== last) { last = r; deadline = Date.now() + stallMs; } + if (Date.now() > deadline) throw new Error(`等超时：${Math.round(stallMs / 60000)} 分钟没有任何进展`); + await sleep(everyMs); + } +} diff --git a/scripts/bench/mappings.mjs b/scripts/bench/mappings.mjs new file mode 100644 index 000000000..3ca61e62a --- /dev/null +++ b/scripts/bench/mappings.mjs @@ -0,0 +1,225 @@ +#!/usr/bin/env node +// 映射探索的测量台（#501）：agent 从 schema 提议的口径，对不对、漏了多少。 +// +// **打分不看名字，看数。** 治理那边的真值按两个名字键，因为它判的是二分类； +// 这里概念名是模型自己起的，「Revenue」与「已支付 GMV」按名字对不上任何一条。 +// 所以真值一条是「一个业务口径 + 一条 gold SQL」，打分把提议真跑一遍， +// 跟 gold 的结果比数——数一样就是同一个口径，叫什么无所谓。 +// +// 四栏，含义分开： +// covered 真值 N 条，被至少一条提议算出来的有几条（漏没漏） +// right 提议 K 条，跑得通且对上某条真值的有几条 +// wrong **跑得通但一条都对不上**，附它算出的数与最接近的真值 +// broken 跑不通（列不存在、语法错） +// +// wrong 是决定 #504 能不能默认开的那个数。跑不通的提议无害，它失败得很响； +// 跑得通而算错的才是全部风险——问数会拿它印出一个看起来完全正常的数字。 +// +// 用法： +// node scripts/bench/mappings.mjs --fresh # 新库 → 挂源 → 探索 → 打分 +// node scripts/bench/mappings.mjs --fresh --no-comments # 同上，但语料不带列注释 +// node scripts/bench/mappings.mjs --kb --score # 只打分，不动库 +// +// 环境变量：BENCH_BASE（默认 http://127.0.0.1:8322）、BENCH_EMAIL / BENCH_PASSWORD、 +// BENCH_PSQL（默认 docker exec … psql -d utopia -tAc）、 +// BENCH_CORPUS_CONN（服务端用来连语料库的连接串）。 + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +// **判等与连库的那几样住在 lib.mjs，两个测量台共用。** 各写一份 `same()` +// 迟早漂移，而一旦漂移，「提议对了几条」与「答案对了几条」就不是同一把尺子 +// 量出来的（#520） +import { api, login, psql, onDb, num, value, same, log, until, sleep, parseArgs } from "./lib.mjs"; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const args = parseArgs(process.argv); +const corpusName = args.corpus || "tpch"; +const truth = JSON.parse(fs.readFileSync(path.join(HERE, "truth", `${corpusName}.mappings.json`), "utf8")); +const CORPUS_DB = process.env.BENCH_CORPUS_DB || `bench_${corpusName}`; +const CORPUS_CONN = process.env.BENCH_CORPUS_CONN || `postgres://utopia:utopia@localhost:5432/${CORPUS_DB}`; +const corpusPsql = (sql) => onDb(CORPUS_DB, `SET statement_timeout = '20s'; ${sql}`); + +// ---------- 语料 ---------- +// +// **每一组一个新库**（bench/README 的第一条规则），这里还多一层理由： +// `propose` 的 `ON CONFLICT … WHERE status = 'proposed'` 让第二轮探索 +// 继承第一轮的行，同一个库上跑两次，第二次的分不是第二次的。 +function loadCorpus() { + const db = CORPUS_DB; + const exists = onDb("postgres", `SELECT 1 FROM pg_database WHERE datname='${db}'`); + if (!exists) { + onDb("postgres", `CREATE DATABASE ${db}`); + log(`建库 ${db}`); + } + corpusPsql(fs.readFileSync(path.join(HERE, "schemas", `${corpusName}.sql`), "utf8")); + // 行可以单独一个文件：`wide` 的建表与造数各自都不短，放一起读不动 + const data = path.join(HERE, "schemas", `${corpusName}.data.sql`); + if (fs.existsSync(data)) corpusPsql(fs.readFileSync(data, "utf8")); + log(`${corpusName} 表与行就位`); + // 注释是自变量：不加载就是一份同构但没有注释的语料，两轮之差即注释值多少分 + if (!args["no-comments"]) { + corpusPsql(fs.readFileSync(path.join(HERE, "schemas", `${corpusName}.comments.sql`), "utf8")); + log("列注释已加载"); + } else { + log("列注释**未**加载（--no-comments）"); + } +} + +// ---------- 新库 + 挂源 + 探索 ---------- +async function fresh() { + loadCorpus(); + const ws = (await api("GET", "/api/v1/workspaces"))[0].id; + const stamp = new Date().toISOString().slice(0, 16).replace(/[:T]/g, "-"); + const label = args.label || (args["no-comments"] ? "no-comments" : "commented"); + const kb = (await api("POST", `/api/v1/workspaces/${ws}/kbs`, { name: `mappings ${corpusName} ${label} ${stamp}` })).id; + await sleep(4000); + + // **源名就叫语料名，各轮复用同一个源。** + // + // 第一轮给它起了 `tpch-2026-09-08-12-41`（想让各轮的源并存着比），结果是 + // 零提议：`explore_mappings` 拿模型回的 `source` 去比挂载源的名字 + // （`eq_ignore_ascii_case`），而模型看着 schema 回的是 `tpch`，对不上就 + // 整条 `continue`，十二条一条不剩。任务照样 done，页面上只有一条 + // 「没提出来」的告警。**源名是提议能不能落地的隐性依赖**，真实的库里 + // 一样会踩——它该进 #503 的覆盖率报告，而不是靠人猜。 + const dsName = corpusName; + const existing = (await api("GET", "/api/v1/admin/data-sources")).data_sources.find((d) => d.name === dsName); + const ds = existing?.id + ?? (await api("POST", "/api/v1/admin/data-sources", { name: dsName, conn_string: CORPUS_CONN })).id; + await api("PUT", `/api/v1/admin/data-sources/${ds}/grants/${ws}`); + const mounted = await api("PUT", `/api/v1/kbs/${kb}/data-sources/${ds}`); + log(`kb ${kb}，源 ${dsName} 已挂载，schema 文档 ${mounted.schema_tables ?? "?"} 张表`); + if (mounted.schema_error) log(` schema 同步报错：${mounted.schema_error}`); + + await api("POST", `/api/v1/kbs/${kb}/data-sources/explore`); + log("探索已入队，等提议落库"); + let said = ""; + await until(async () => { + const row = psql(`SELECT status || E'\\t' || attempts || E'\\t' || coalesce(left(last_error, 160), '') + FROM jobs WHERE kind='explore_mappings' AND payload->>'kb_id'='${kb}' + ORDER BY id DESC LIMIT 1`); + const [status, attempts, err] = (row || "queued\t0\t").split("\t"); + // **任务重试期间就把错话说出来。** 头一轮等满十分钟拿到的是「没有任何进展」， + // 而真正该看见的是「模型密钥解不开」——它第一次失败时就已经写在 last_error 里了 + if (err && err !== said) { said = err; log(` 第 ${attempts} 次失败：${err}`); } + if (status === "done" || status === "failed") return true; + return num(`SELECT count(*) FROM concept_mappings WHERE kb_id='${kb}'`); + }, 5000, 600000); + return kb; +} + +// ---------- 打分 ---------- +const qualify = (t) => (t && t.includes(".") ? t : `${truth.db_schema}.${t}`); + +// 提议怎么变成一条能跑的 SQL：给了 sql 就用 sql，否则 expr + table 拼一条。 +// 两者都没有就只有 table_name——那不是一个可执行的口径，算 broken。 +function proposalSql(m) { + if (m.sql && m.sql.trim()) return m.sql.trim().replace(/;\s*$/, ""); + if (m.expr && m.table_name) return `SELECT ${m.expr} FROM ${qualify(m.table_name)}`; + return null; +} + + +function score(kb) { + const rows = JSON.parse(psql(`SELECT coalesce(json_agg(x), '[]') FROM ( + SELECT m.id, e.canonical_name AS concept, t.key AS kind, m.source, m.table_name, + m.expr, m.sql, m.unit, m.status + FROM concept_mappings m + JOIN entities e ON e.id = m.concept_id + JOIN entity_types t ON t.id = e.type_id + WHERE m.kb_id = '${kb}' ORDER BY e.canonical_name) x`)); + + const gold = truth.metrics.map((m) => ({ ...m, value: value(CORPUS_DB, m.gold).n })); + // 22 条查询没说、但读得懂这个 schema 的人不会反对的口径（退货率、客均余额）。 + // **单独一栏，不算对也不算错**——头一轮把退货率记成 wrong，而它没有任何毛病， + // 错的是真值不全。govern.mjs 的 `unlabeled` 是同一件事 + const plausible = (truth.plausible || []).map((m) => ({ ...m, value: value(CORPUS_DB, m.gold).n })); + const dimCols = new Set(truth.dimensions.map((d) => d.column.toLowerCase())); + // 列名不带表限定的那一份：同一个维度可能从事实表读，也可能从维表读 + // （`shop_nm` 两边都有），而两种都对 + const dimNames = new Set([...dimCols].map((c) => c.split(".").pop())); + // 陷阱分角色。**同一列在两个角色下不是同一件事**：`p_size` 当维度是对的 + // （Q16 就按它分组），当指标求和才没有意义；`l_comment` 反过来。 + // 第三轮上这条把一条正确的维度提议记成了踩陷阱 + const trapFor = (role) => + new Map( + truth.traps + .filter((t) => (t.as ?? "both") === "both" || t.as === role) + .map((t) => [t.column.toLowerCase(), t.why]), + ); + const dimTraps = trapFor("dimension"); + const trapCols = trapFor("metric"); + + const c = { right: 0, wrong: 0, broken: 0, traps: 0, plausible: 0, dim_right: 0, dim_wrong: 0 }; + const hit = new Set(), wrong = [], broken = [], fair = []; + + for (const m of rows) { + // 维度没有数可比：判它指的那一列在不在真值的 group-by 集合里， + // 以及有没有落到陷阱列上（把一个键或一段自由文本当成维度） + if (m.kind === "dimension") { + // **维度不一定是一个裸列名。** 模型给渠道的定义是 + // `CASE chnl WHEN 1 THEN 'APP' … END`——把码翻译成人看的名字，这正是 + // 一个维度该做的事。只认裸列名的话，三条正确的提议被判成错的（wide 第一轮）。 + // 所以从表达式里把标识符抽出来，命中任一真值维度列即算数 + const table = qualify(m.table_name).toLowerCase(); + const ids = (m.expr || "").toLowerCase().match(/[a-z_][a-z0-9_]*/g) ?? []; + const col = ids.find((id) => dimCols.has(`${table}.${id}`) || dimNames.has(id)) + ?? `${table}.${(m.expr || "").replace(/[^\w.]/g, "")}`.toLowerCase(); + if (dimCols.has(col) || dimNames.has(col.split(".").pop())) c.dim_right++; + else { c.dim_wrong++; wrong.push(`dim "${m.concept}" → ${col}${dimTraps.has(col) ? ` **trap: ${dimTraps.get(col)}**` : ""}`); } + if (dimTraps.has(col)) c.traps++; + continue; + } + const sql = proposalSql(m); + if (!sql) { c.broken++; broken.push(`"${m.concept}" 没有可执行的定义（只有 table_name=${m.table_name}）`); continue; } + const got = value(CORPUS_DB, sql); + if (got.error) { c.broken++; broken.push(`"${m.concept}" → ${got.error}`); continue; } + if (got.empty) { c.broken++; broken.push(`"${m.concept}" 返回空`); continue; } + + const matched = gold.filter((g) => same(g.value, got.n)); + if (matched.length) { c.right++; matched.forEach((g) => hit.add(g.id)); continue; } + const plaus = plausible.find((g) => same(g.value, got.n)); + if (plaus) { c.plausible++; fair.push(`"${m.concept}" = ${plaus.id} (${plaus.label})`); continue; } + + c.wrong++; + // 最接近的真值：告诉人这条错在哪个方向，而不是只说它错了 + const near = gold + .filter((g) => Number.isFinite(g.value)) + .sort((a, b) => Math.abs(a.value - got.n) - Math.abs(b.value - got.n))[0]; + const trap = [...trapCols.keys()].find((k) => (m.expr || "").toLowerCase().includes(k.split(".").pop())); + if (trap) c.traps++; + wrong.push( + `"${m.concept}" = ${sql}\n 算出 ${got.n}，最近的真值 ${near?.id} = ${near?.value}` + + (trap ? `\n **trap: ${trapCols.get(trap)}**` : ""), + ); + } + + const pct = (a, b) => (b ? `${Math.round((a / b) * 100)}%` : "—"); + const missed = gold.filter((g) => !hit.has(g.id)); + // **那一轮带没带注释，问库名而不是问命令行参数。** `--score` 重打一次分时 + // 命令行上没有 `--no-comments`，照参数写就把不带注释的那轮报成带注释的 + const kbName = psql(`SELECT name FROM knowledge_bases WHERE id = '${kb}'`); + const out = { + kb, corpus: corpusName, + comments: !kbName.includes("no-comments"), + proposals: rows.length, + metrics: { right: c.right, plausible: c.plausible, wrong: c.wrong, broken: c.broken }, + dimensions: { right: c.dim_right, wrong: c.dim_wrong }, + covered: `${hit.size}/${gold.length} (${pct(hit.size, gold.length)})`, + traps_hit: c.traps, + }; + console.log(JSON.stringify(out, null, 2)); + if (fair.length) console.log("\nPLAUSIBLE — 22 条查询没说，但站得住的口径\n " + fair.join("\n ")); + if (wrong.length) console.log("\nWRONG — 跑得通，算的不是任何一条真值\n " + wrong.join("\n ")); + if (broken.length) console.log("\nBROKEN — 跑不通（无害，人一眼看得见）\n " + broken.join("\n ")); + if (missed.length) console.log("\nMISSED — 真值里没人提的口径\n " + missed.map((g) => `${g.id} (${g.label})`).join("\n ")); +} + +// ---------- 主流程 ---------- +const main = async () => { + await login(); + const kb = args.kb && args.kb !== true ? args.kb : await fresh(); + score(kb); +}; +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/scripts/bench/schemas/tpch.comments.sql b/scripts/bench/schemas/tpch.comments.sql new file mode 100644 index 000000000..eb6d0891f --- /dev/null +++ b/scripts/bench/schemas/tpch.comments.sql @@ -0,0 +1,70 @@ +-- TPC-H 的列注释。**单独一个文件，因为它是一个自变量。** +-- +-- 真 TPC-H 一条注释都没有，而真实库里注释是探索最主要的线索—— +-- `explore_mappings` 把它原样拼进 prompt（`-- {comment}`），提示词还专门 +-- 要求「citing column comments」。加载与不加载各跑一轮，两个分数之差 +-- 就是注释值多少分。 +-- +-- **写法上守一条线：注释描述列，不描述口径。** 写「折后收入 = +-- extendedprice × (1 - discount)」等于把真值的 gold SQL 抄给模型， +-- 量出来的就不是它能不能读懂 schema 了。所以只说这一列是什么、 +-- 什么单位、取值域多大——真实的 DBA 也就写到这个份上。 +-- +-- 几列故意不写：地址、电话、以及 *_comment 那几列里除 l_comment 外的。 +-- 真实库不会每列都有注释，而「没注释的列会被怎么处理」本身要量。 + +SET search_path TO tpch; + +COMMENT ON TABLE region IS 'Sales regions; five rows, the top of the geography hierarchy'; +COMMENT ON COLUMN region.r_name IS 'Region name (AFRICA, AMERICA, ASIA, EUROPE, MIDDLE EAST)'; + +COMMENT ON TABLE nation IS 'Countries, each belonging to one region'; +COMMENT ON COLUMN nation.n_name IS 'Country name, uppercase'; +COMMENT ON COLUMN nation.n_regionkey IS 'The region this country belongs to'; + +COMMENT ON TABLE supplier IS 'Suppliers of parts'; +COMMENT ON COLUMN supplier.s_nationkey IS 'Country the supplier is based in'; +COMMENT ON COLUMN supplier.s_acctbal IS 'Account balance in USD; may be negative'; + +COMMENT ON TABLE customer IS 'Customers placing orders'; +COMMENT ON COLUMN customer.c_nationkey IS 'Country the customer is based in'; +COMMENT ON COLUMN customer.c_acctbal IS 'Account balance in USD; may be negative'; +COMMENT ON COLUMN customer.c_mktsegment IS 'Market segment: AUTOMOBILE, BUILDING, FURNITURE, MACHINERY or HOUSEHOLD'; + +COMMENT ON TABLE part IS 'Parts catalogue'; +COMMENT ON COLUMN part.p_mfgr IS 'Manufacturer, Manufacturer#1..5'; +COMMENT ON COLUMN part.p_brand IS 'Brand, Brand#11..55; a brand belongs to one manufacturer'; +COMMENT ON COLUMN part.p_type IS 'Material and finish, space separated, e.g. PROMO BRUSHED STEEL'; +COMMENT ON COLUMN part.p_size IS 'Nominal size, 1..50, unitless'; +COMMENT ON COLUMN part.p_container IS 'Packaging, e.g. SM CASE, JUMBO BOX'; +COMMENT ON COLUMN part.p_retailprice IS 'List price per unit in USD'; + +COMMENT ON TABLE partsupp IS 'Which supplier can supply which part, and on what terms'; +COMMENT ON COLUMN partsupp.ps_availqty IS 'Units this supplier currently holds of this part'; +COMMENT ON COLUMN partsupp.ps_supplycost IS 'What we pay this supplier per unit, USD'; + +COMMENT ON TABLE orders IS 'Customer orders; one row per order, line detail in lineitem'; +COMMENT ON COLUMN orders.o_custkey IS 'Customer who placed the order'; +COMMENT ON COLUMN orders.o_orderstatus IS 'O = all lines still open, F = all lines fulfilled, P = partially fulfilled'; +COMMENT ON COLUMN orders.o_totalprice IS 'Order total in USD, taxed and discounted, equal to the sum over its lines'; +COMMENT ON COLUMN orders.o_orderdate IS 'Date the order was placed'; +COMMENT ON COLUMN orders.o_orderpriority IS 'Priority, 1-URGENT .. 5-LOW'; +COMMENT ON COLUMN orders.o_clerk IS 'Clerk who took the order'; + +COMMENT ON TABLE lineitem IS 'Order lines; the fact table. One row per part on an order'; +COMMENT ON COLUMN lineitem.l_orderkey IS 'The order this line belongs to'; +COMMENT ON COLUMN lineitem.l_partkey IS 'The part sold on this line'; +COMMENT ON COLUMN lineitem.l_suppkey IS 'The supplier that supplied it'; +COMMENT ON COLUMN lineitem.l_linenumber IS 'Position of this line within its order, 1-based'; +COMMENT ON COLUMN lineitem.l_quantity IS 'Units sold on this line'; +COMMENT ON COLUMN lineitem.l_extendedprice IS 'Line amount in USD before discount and tax: units × the part list price'; +COMMENT ON COLUMN lineitem.l_discount IS 'Discount granted on this line as a fraction, 0.00 to 0.10'; +COMMENT ON COLUMN lineitem.l_tax IS 'Tax rate applied to this line as a fraction, 0.00 to 0.08'; +COMMENT ON COLUMN lineitem.l_returnflag IS 'R = returned by the customer, A = accepted, N = not yet settled'; +COMMENT ON COLUMN lineitem.l_linestatus IS 'F = fulfilled, O = still open'; +COMMENT ON COLUMN lineitem.l_shipdate IS 'Date the line shipped'; +COMMENT ON COLUMN lineitem.l_commitdate IS 'Date we committed to the customer'; +COMMENT ON COLUMN lineitem.l_receiptdate IS 'Date the customer received it; later than commitdate means late'; +COMMENT ON COLUMN lineitem.l_shipmode IS 'Carrier mode: REG AIR, AIR, RAIL, SHIP, TRUCK, MAIL or FOB'; +COMMENT ON COLUMN lineitem.l_shipinstruct IS 'Handling instruction on the shipment'; +COMMENT ON COLUMN lineitem.l_comment IS 'Free-text remark typed by staff; no analytical meaning'; diff --git a/scripts/bench/schemas/tpch.sql b/scripts/bench/schemas/tpch.sql new file mode 100644 index 000000000..175a62138 --- /dev/null +++ b/scripts/bench/schemas/tpch.sql @@ -0,0 +1,254 @@ +-- TPC-H on Postgres：映射测量台的第一份语料（#501） +-- +-- **数据不按 TPC-H 的生成规范来。** 那份规范值钱的地方是 schema 与 22 条查询—— +-- 查询里写死了「收入」在这个 schema 上就是 sum(l_extendedprice * (1 - l_discount))， +-- 真值从那里抄，不是谁拍的脑袋。而行是怎么长出来的对打分毫无影响：探索只读 +-- schema 不读数据，打分时 gold 与提议跑在同一批行上，比的是两个数一不一样。 +-- 所以用 generate_series 造几万行，跑得快、装得下、可重放。 +-- +-- 表与列名照规范原样，包括 *_comment 那几列——它们是**数据列**（一段随机文本）， +-- 与我们另外加的列注释是两回事。留着，因为真实库里也有这种名字唬人的列， +-- 探索会不会把 l_comment 当维度提出来是要量的事情之一。 +-- +-- 列注释单独在 tpch.comments.sql 里，不加载就是一份没有注释的同构语料。 +-- 真实库里注释是模型最主要的线索，分开就能量出它值多少分。 + +DROP SCHEMA IF EXISTS tpch CASCADE; +CREATE SCHEMA tpch; +SET search_path TO tpch; + +CREATE TABLE region ( + r_regionkey INTEGER PRIMARY KEY, + r_name CHAR(25) NOT NULL, + r_comment VARCHAR(152) +); + +CREATE TABLE nation ( + n_nationkey INTEGER PRIMARY KEY, + n_name CHAR(25) NOT NULL, + n_regionkey INTEGER NOT NULL REFERENCES region(r_regionkey), + n_comment VARCHAR(152) +); + +CREATE TABLE supplier ( + s_suppkey INTEGER PRIMARY KEY, + s_name CHAR(25) NOT NULL, + s_address VARCHAR(40) NOT NULL, + s_nationkey INTEGER NOT NULL REFERENCES nation(n_nationkey), + s_phone CHAR(15) NOT NULL, + s_acctbal DECIMAL(15,2) NOT NULL, + s_comment VARCHAR(101) +); + +CREATE TABLE customer ( + c_custkey INTEGER PRIMARY KEY, + c_name VARCHAR(25) NOT NULL, + c_address VARCHAR(40) NOT NULL, + c_nationkey INTEGER NOT NULL REFERENCES nation(n_nationkey), + c_phone CHAR(15) NOT NULL, + c_acctbal DECIMAL(15,2) NOT NULL, + c_mktsegment CHAR(10) NOT NULL, + c_comment VARCHAR(117) +); + +CREATE TABLE part ( + p_partkey INTEGER PRIMARY KEY, + p_name VARCHAR(55) NOT NULL, + p_mfgr CHAR(25) NOT NULL, + p_brand CHAR(10) NOT NULL, + p_type VARCHAR(25) NOT NULL, + p_size INTEGER NOT NULL, + p_container CHAR(10) NOT NULL, + p_retailprice DECIMAL(15,2) NOT NULL, + p_comment VARCHAR(23) +); + +CREATE TABLE partsupp ( + ps_partkey INTEGER NOT NULL REFERENCES part(p_partkey), + ps_suppkey INTEGER NOT NULL REFERENCES supplier(s_suppkey), + ps_availqty INTEGER NOT NULL, + ps_supplycost DECIMAL(15,2) NOT NULL, + ps_comment VARCHAR(199), + PRIMARY KEY (ps_partkey, ps_suppkey) +); + +CREATE TABLE orders ( + o_orderkey INTEGER PRIMARY KEY, + o_custkey INTEGER NOT NULL REFERENCES customer(c_custkey), + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMAL(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + o_comment VARCHAR(79) +); + +CREATE TABLE lineitem ( + l_orderkey INTEGER NOT NULL REFERENCES orders(o_orderkey), + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity DECIMAL(15,2) NOT NULL, + l_extendedprice DECIMAL(15,2) NOT NULL, + l_discount DECIMAL(15,2) NOT NULL, + l_tax DECIMAL(15,2) NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44), + PRIMARY KEY (l_orderkey, l_linenumber), + FOREIGN KEY (l_partkey, l_suppkey) REFERENCES partsupp(ps_partkey, ps_suppkey) +); + +-- ---------------------------------------------------------------- 行 +-- +-- setseed 让同一份语料每次生成得一模一样：**两轮探索的分数要可比**， +-- 语料自己先不能变（bench/README 那条「每一组一个新库」的同一个理由）。 + +SELECT setseed(0.42); + +INSERT INTO region VALUES + (0, 'AFRICA', 'lar deposits. blithely final packages cajole'), + (1, 'AMERICA', 'hs use ironic, even requests'), + (2, 'ASIA', 'ges. thinly even pinto beans ca'), + (3, 'EUROPE', 'ly final courts cajole furiously final excuse'), + (4, 'MIDDLE EAST', 'uickly special accounts cajole carefully blithely close requests'); + +INSERT INTO nation (n_nationkey, n_name, n_regionkey, n_comment) +SELECT * FROM (VALUES + (0,'ALGERIA',0),(1,'ARGENTINA',1),(2,'BRAZIL',1),(3,'CANADA',1),(4,'EGYPT',4), + (5,'ETHIOPIA',0),(6,'FRANCE',3),(7,'GERMANY',3),(8,'INDIA',2),(9,'INDONESIA',2), + (10,'IRAN',4),(11,'IRAQ',4),(12,'JAPAN',2),(13,'JORDAN',4),(14,'KENYA',0), + (15,'MOROCCO',0),(16,'MOZAMBIQUE',0),(17,'PERU',1),(18,'CHINA',2),(19,'ROMANIA',3), + (20,'SAUDI ARABIA',4),(21,'VIETNAM',2),(22,'RUSSIA',3),(23,'UNITED KINGDOM',3), + (24,'UNITED STATES',1) +) AS v(k, n, r), LATERAL (SELECT 'final accounts wake ' || n) AS c(cm); + +INSERT INTO supplier +SELECT i, + 'Supplier#' || lpad(i::text, 9, '0'), + lpad(md5(i::text), 20, 'x'), + (random() * 24)::int, + '27-' || lpad(((random() * 899 + 100))::int::text, 3, '0') || '-' || + lpad(((random() * 899 + 100))::int::text, 3, '0') || '-' || + lpad(((random() * 8999 + 1000))::int::text, 4, '0'), + round((random() * 10000 - 1000)::numeric, 2), + 'each slyly above the careful' +FROM generate_series(1, 100) AS i; + +INSERT INTO customer +SELECT i, + 'Customer#' || lpad(i::text, 9, '0'), + lpad(md5(i::text || 'a'), 25, 'y'), + (random() * 24)::int, + '25-' || lpad(((random() * 899 + 100))::int::text, 3, '0') || '-' || + lpad(((random() * 899 + 100))::int::text, 3, '0') || '-' || + lpad(((random() * 8999 + 1000))::int::text, 4, '0'), + round((random() * 10000 - 1000)::numeric, 2), + (ARRAY['AUTOMOBILE','BUILDING','FURNITURE','MACHINERY','HOUSEHOLD'])[1 + (random() * 4)::int], + 'requests wake fluffily' +FROM generate_series(1, 1500) AS i; + +INSERT INTO part +SELECT i, + (ARRAY['almond','antique','blush','burnished','cornflower'])[1 + (random() * 4)::int] || ' ' || + (ARRAY['azure','chocolate','dim','frosted','ghost'])[1 + (random() * 4)::int] || ' ' || + (ARRAY['lace','metallic','powder','rose','steel'])[1 + (random() * 4)::int], + 'Manufacturer#' || (1 + (random() * 4)::int), + 'Brand#' || (1 + (random() * 4)::int) || (1 + (random() * 4)::int), + (ARRAY['STANDARD','SMALL','MEDIUM','LARGE','ECONOMY','PROMO'])[1 + (random() * 5)::int] || ' ' || + (ARRAY['ANODIZED','BURNISHED','PLATED','POLISHED','BRUSHED'])[1 + (random() * 4)::int] || ' ' || + (ARRAY['TIN','NICKEL','BRASS','STEEL','COPPER'])[1 + (random() * 4)::int], + 1 + (random() * 49)::int, + (ARRAY['SM','LG','MED','JUMBO','WRAP'])[1 + (random() * 4)::int] || ' ' || + (ARRAY['CASE','BOX','BAG','JAR','PKG'])[1 + (random() * 4)::int], + round((90 + i % 2000 * 0.5 + random() * 10)::numeric, 2), + 'final deposits' +FROM generate_series(1, 2000) AS i; + +-- 每个零件四个供应商（规范是 SUPPLIER_PER_PART = 4） +INSERT INTO partsupp +SELECT p.p_partkey, + 1 + ((p.p_partkey * 7 + s.n * 23) % 100), + (random() * 9999)::int, + round((random() * 1000 + 1)::numeric, 2), + 'careful accounts sleep' +FROM part p CROSS JOIN generate_series(0, 3) AS s(n) +ON CONFLICT DO NOTHING; + +INSERT INTO orders +SELECT i, + 1 + (random() * 1499)::int, + 'O', + 0, -- 明细生成完再回填 + DATE '1992-01-01' + (random() * 2520)::int, -- 1992-01-01 .. 1998-12-31 + (ARRAY['1-URGENT','2-HIGH','3-MEDIUM','4-NOT SPECIFIED','5-LOW'])[1 + (random() * 4)::int], + 'Clerk#' || lpad((1 + (random() * 99)::int)::text, 9, '0'), + 0, + 'slyly special requests' +FROM generate_series(1, 15000) AS i; + +-- 明细。**数值关系要立得住**：l_extendedprice = l_quantity × p_retailprice， +-- 折扣 0–0.10、税 0–0.08，都照规范的取值域——Q1 的 disc_price 与 charge +-- 两条口径全建在这三列上，随便填的话真值算出来的数没有意义。 +-- +-- returnflag / linestatus 按 shipdate 定，也是规范的规则：1995-06-17 之前 +-- 发的是已结（F，退货 R 或已收 A），之后是在途（O，N）。Q1 正是按这两列分组。 +INSERT INTO lineitem +WITH ps AS ( + SELECT row_number() OVER (ORDER BY ps_partkey, ps_suppkey) AS rn, ps_partkey, ps_suppkey + FROM partsupp +), +n AS (SELECT count(*) AS c FROM ps), +raw AS ( + SELECT o.o_orderkey, + o.o_orderdate, + ln::int AS l_linenumber, + 1 + (random() * (n.c - 1))::int AS ps_rn, + round((1 + random() * 49)::numeric, 2) AS l_quantity, + round((random() * 0.10)::numeric, 2) AS l_discount, + round((random() * 0.08)::numeric, 2) AS l_tax, + o.o_orderdate + (1 + random() * 120)::int AS l_shipdate + FROM orders o + CROSS JOIN n + CROSS JOIN LATERAL generate_series(1, 1 + (random() * 5)::int) AS ln +) +SELECT r.o_orderkey, ps.ps_partkey, ps.ps_suppkey, r.l_linenumber, + r.l_quantity, + round(r.l_quantity * p.p_retailprice, 2), + r.l_discount, + r.l_tax, + CASE WHEN r.l_shipdate <= DATE '1995-06-17' + THEN (ARRAY['R','A'])[1 + (random() * 1)::int] ELSE 'N' END, + CASE WHEN r.l_shipdate <= DATE '1995-06-17' THEN 'F' ELSE 'O' END, + r.l_shipdate, + r.o_orderdate + (1 + random() * 90)::int, + r.l_shipdate + (1 + random() * 30)::int, + (ARRAY['DELIVER IN PERSON','COLLECT COD','NONE','TAKE BACK RETURN'])[1 + (random() * 3)::int], + (ARRAY['REG AIR','AIR','RAIL','SHIP','TRUCK','MAIL','FOB'])[1 + (random() * 6)::int], + 'carefully ironic deposits' +FROM raw r +JOIN ps ON ps.rn = r.ps_rn +JOIN part p ON p.p_partkey = ps.ps_partkey; + +-- 订单总额回填：含税折后价之和，与规范一致。o_totalprice 与 lineitem +-- 对得上，「订单均价」这类口径才有唯一答案 +UPDATE orders o + SET o_totalprice = t.total, + o_orderstatus = t.status + FROM ( + SELECT l_orderkey, + round(sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)), 2) AS total, + CASE WHEN count(*) FILTER (WHERE l_linestatus = 'O') = 0 THEN 'F' + WHEN count(*) FILTER (WHERE l_linestatus = 'F') = 0 THEN 'O' + ELSE 'P' END AS status + FROM lineitem GROUP BY l_orderkey + ) t + WHERE o.o_orderkey = t.l_orderkey; + +ANALYZE; diff --git a/scripts/bench/schemas/wide.comments.sql b/scripts/bench/schemas/wide.comments.sql new file mode 100644 index 000000000..b16f67250 --- /dev/null +++ b/scripts/bench/schemas/wide.comments.sql @@ -0,0 +1,63 @@ +-- `wide` 的列注释——**写成真实数仓里那种注释**，而不是 TPC-H 那种教科书注释。 +-- +-- 真实的宽表注释有三个特点，这里都照做： +-- +-- 1. **只有一部分列有。** 建表的人给自己关心的写了，剩下的没写。 +-- 2. **说的是字段是什么，不是口径是什么。** 「实付金额（分）」是列的事实； +-- 「GMV 要排掉测试单且只算已支付」是业务约定，它住在文档、周会和某个人 +-- 脑子里，不住在 `COMMENT ON COLUMN` 里。 +-- 3. **状态码列出取值，但不说哪些算数。** 「1 创建 2 已付 3 已发 4 完成 +-- 5 关闭 6 退款」——看得见六个值，看不出「有效订单是 2/3/4」。 +-- +-- 所以带上这份注释之后，模型该能答对单位，仍然答不对口径。**那正是语义层 +-- 要填的那一格**：注释解决列的歧义，口径解决业务的歧义，两件事。 + +SET search_path TO dw; + +COMMENT ON TABLE dwd_ord_dtl IS '订单明细宽表，一行一个订单商品行，T+1 由 ods 层打平生成'; + +COMMENT ON COLUMN dwd_ord_dtl.ord_id IS '订单号'; +COMMENT ON COLUMN dwd_ord_dtl.ord_ln IS '订单内行号'; +COMMENT ON COLUMN dwd_ord_dtl.buyer_id IS '买家 ID'; +COMMENT ON COLUMN dwd_ord_dtl.shop_id IS '店铺 ID，关联 dim_shop'; +COMMENT ON COLUMN dwd_ord_dtl.item_id IS '商品 ID'; +COMMENT ON COLUMN dwd_ord_dtl.cat_id IS '叶子类目 ID'; + +COMMENT ON COLUMN dwd_ord_dtl.amt_total IS '订单金额（分），商品金额 + 运费，优惠前'; +COMMENT ON COLUMN dwd_ord_dtl.amt_pay IS '实付金额（分）'; +COMMENT ON COLUMN dwd_ord_dtl.amt_item IS '商品金额（分），单价 × 件数'; +COMMENT ON COLUMN dwd_ord_dtl.amt_frght IS '运费（分）'; +COMMENT ON COLUMN dwd_ord_dtl.amt_disc IS '活动优惠（分）'; +COMMENT ON COLUMN dwd_ord_dtl.amt_cpn IS '优惠券抵扣（分）'; +COMMENT ON COLUMN dwd_ord_dtl.amt_rfnd IS '退款金额（分）'; +COMMENT ON COLUMN dwd_ord_dtl.amt_cost IS '商品成本（分）'; +COMMENT ON COLUMN dwd_ord_dtl.amt_pt IS '积分抵扣（积分数，非金额）'; +COMMENT ON COLUMN dwd_ord_dtl.price_old IS '旧价格字段，2023 年迁移后不再写入'; + +COMMENT ON COLUMN dwd_ord_dtl.qty IS '件数'; +COMMENT ON COLUMN dwd_ord_dtl.qty_rfnd IS '退货件数'; + +COMMENT ON COLUMN dwd_ord_dtl.ord_st IS '订单状态：1 创建 2 已付 3 已发货 4 已完成 5 已关闭 6 已退款'; +COMMENT ON COLUMN dwd_ord_dtl.pay_st IS '支付状态：0 未支付 1 已支付 2 部分退款 3 全额退款'; +COMMENT ON COLUMN dwd_ord_dtl.is_test IS '测试单标记'; +COMMENT ON COLUMN dwd_ord_dtl.is_1st IS '是否该买家首单'; +COMMENT ON COLUMN dwd_ord_dtl.flg_r IS '退货标记'; +COMMENT ON COLUMN dwd_ord_dtl.chnl IS '渠道：1 APP 2 站内 H5 3 小程序 4 三方平台'; +COMMENT ON COLUMN dwd_ord_dtl.pay_mtd IS '支付方式：1 支付宝 2 微信 3 银行卡 4 其他'; + +COMMENT ON COLUMN dwd_ord_dtl.dt_crt IS '下单时间'; +COMMENT ON COLUMN dwd_ord_dtl.dt_pay IS '支付时间'; +COMMENT ON COLUMN dwd_ord_dtl.stat_dt IS '统计日期（分区字段），取下单日期'; +COMMENT ON COLUMN dwd_ord_dtl.etl_dt IS 'ETL 写入时间'; + +COMMENT ON COLUMN dwd_ord_dtl.shop_nm IS '店铺名称'; +COMMENT ON COLUMN dwd_ord_dtl.cat_nm IS '叶子类目名称'; +COMMENT ON COLUMN dwd_ord_dtl.cat_nm_l1 IS '一级类目名称'; +COMMENT ON COLUMN dwd_ord_dtl.prov IS '收货省份'; +COMMENT ON COLUMN dwd_ord_dtl.city IS '收货城市'; +COMMENT ON COLUMN dwd_ord_dtl.buyer_lvl IS '买家等级 1-5'; +COMMENT ON COLUMN dwd_ord_dtl.ver IS 'schema 版本号'; + +COMMENT ON TABLE dim_shop IS '店铺维表'; +COMMENT ON COLUMN dim_shop.shop_nm IS '店铺名称'; +COMMENT ON COLUMN dim_shop.lvl IS '店铺等级 1-5'; diff --git a/scripts/bench/schemas/wide.data.sql b/scripts/bench/schemas/wide.data.sql new file mode 100644 index 000000000..5bf97ee36 --- /dev/null +++ b/scripts/bench/schemas/wide.data.sql @@ -0,0 +1,146 @@ +-- `wide` 语料的行。DDL 在 wide.sql，两者一起加载。 +-- +-- setseed 固定：**两轮的分数要可比，语料自己先不能变**。 + +SET search_path TO dw; +SELECT setseed(0.73); + +INSERT INTO dim_shop +SELECT i, + (ARRAY['京东自营','天猫旗舰','拼多多','唯品会','苏宁','小米','华为','安踏'])[1 + (random()*7)::int] + || (ARRAY['旗舰店','专营店','官方店','工厂店'])[1 + (random()*3)::int] || i, + 100000 + (random()*899)::bigint, + (ARRAY['广东','浙江','江苏','上海','北京','山东','四川','福建'])[1 + (random()*7)::int], + 1 + (random()*4)::int, + DATE '2019-01-01' + (random()*2000)::int +FROM generate_series(1, 200) AS i; + +INSERT INTO dwd_ord_dtl ( + ord_id, ord_ln, buyer_id, seller_id, shop_id, item_id, sku_id, cat_id, cat_id_l1, + addr_id, pay_id, promo_id, cpn_id, lgst_id, + amt_total, amt_pay, amt_item, amt_frght, amt_disc, amt_cpn, amt_rfnd, amt_tax, + amt_cost, amt_pt, price_old, qty, qty_rfnd, wt_g, + ord_st, pay_st, lgst_st, rfnd_st, is_test, is_gift, is_presale, is_1st, flg_r, + chnl, pay_mtd, dt_crt, dt_pay, dt_shp, dt_fin, dt_rfnd, stat_dt, + shop_nm, cat_nm, cat_nm_l1, item_nm, brand_nm, prov, city, buyer_lvl, src, dev, + ver, rmk, ext +) +WITH base AS ( + SELECT i, + (i - 1) / 2 + 1 AS ord_id, + ((i - 1) % 2 + 1)::smallint AS ord_ln, + 1 + (random() * 4999)::int AS buyer_id, + 1 + (random() * 199)::int AS shop_id, + 1 + (random() * 49)::int AS cat_id, + 1 + (random() * 4)::int AS qty, + 1000 + (random() * 49000)::bigint AS unit_price, + random() AS r_test, + random() AS r_pay, + random() AS r_disc, + random() AS r_cpn, + random() AS r_rfnd, + random() AS r_chnl, + random() AS r_1st, + random() AS r_misc, + DATE '2025-01-01' + (random() * 364)::int AS d_crt + FROM generate_series(1, 60000) AS i +), +money AS ( + SELECT *, + unit_price * qty AS amt_item, + (unit_price * qty * r_disc * 0.2)::bigint AS amt_disc, + CASE WHEN r_cpn < 0.3 THEN (500 + r_cpn * 6000)::bigint ELSE 0 END AS amt_cpn, + -- 满 99 元包邮：一条真实的业务规则，schema 里同样一个字都没有 + CASE WHEN unit_price * qty >= 9900 THEN 0 + ELSE (500 + r_misc * 1000)::bigint END AS amt_frght, + CASE WHEN r_test < 0.02 THEN 1 ELSE 0 END AS is_test, + -- 未付 15% / 已付 78% / 部分退 4% / 全退 3% + CASE WHEN r_pay < 0.15 THEN 0 + WHEN r_pay < 0.93 THEN 1 + WHEN r_pay < 0.97 THEN 2 + ELSE 3 END AS pay_st + FROM base +), +calc AS ( + SELECT *, + amt_item + amt_frght AS amt_total, + GREATEST(amt_item - amt_disc - amt_cpn + amt_frght, 0) AS amt_pay + FROM money +) +SELECT ord_id, ord_ln, buyer_id, + 100000 + (calc.shop_id % 900)::bigint AS seller_id, + calc.shop_id, + 900000 + cat_id * 1000 + (r_misc * 999)::int AS item_id, + 9000000 + (r_misc * 99999)::bigint AS sku_id, + cat_id, + 1 + cat_id % 8 AS cat_id_l1, + 500000 + buyer_id AS addr_id, + CASE WHEN pay_st > 0 THEN 7000000 + i ELSE NULL END AS pay_id, + CASE WHEN r_disc > 0.6 THEN 300 + (r_disc * 40)::int ELSE NULL END AS promo_id, + CASE WHEN amt_cpn > 0 THEN 800 + (r_cpn * 90)::int ELSE NULL END AS cpn_id, + CASE WHEN pay_st > 0 THEN 6000000 + i ELSE NULL END AS lgst_id, + + amt_total, amt_pay, amt_item, amt_frght, amt_disc, amt_cpn, + -- 退款只发生在部分退与全退上 + CASE WHEN pay_st = 3 THEN amt_pay + WHEN pay_st = 2 THEN (amt_pay * (0.2 + r_rfnd * 0.4))::bigint + ELSE 0 END AS amt_rfnd, + (amt_pay * 0.06)::bigint AS amt_tax, + (amt_item * (0.5 + r_misc * 0.25))::bigint AS amt_cost, + (r_misc * 500)::bigint AS amt_pt, + NULL::bigint AS price_old, + qty, + CASE WHEN pay_st = 3 THEN qty + WHEN pay_st = 2 THEN GREATEST((qty * 0.5)::int, 1) + ELSE 0 END AS qty_rfnd, + (100 + r_misc * 4900)::int AS wt_g, + + -- 状态白名单：**有效订单是 2/3/4**，而 1 与 5 从没付过钱、6 已全退 + (CASE WHEN pay_st = 0 THEN (CASE WHEN r_misc < 0.6 THEN 1 ELSE 5 END) + WHEN pay_st = 1 THEN (CASE WHEN r_misc < 0.2 THEN 2 + WHEN r_misc < 0.5 THEN 3 ELSE 4 END) + WHEN pay_st = 2 THEN 4 + ELSE 6 END)::smallint AS ord_st, + pay_st::smallint, + (CASE WHEN pay_st = 0 THEN 0 WHEN r_misc < 0.3 THEN 1 ELSE 2 END)::smallint AS lgst_st, + (CASE WHEN pay_st = 2 THEN 1 WHEN pay_st = 3 THEN 2 ELSE 0 END)::smallint AS rfnd_st, + is_test::smallint, + (CASE WHEN r_misc < 0.03 THEN 1 ELSE 0 END)::smallint AS is_gift, + (CASE WHEN r_misc > 0.95 THEN 1 ELSE 0 END)::smallint AS is_presale, + (CASE WHEN r_1st < 0.18 THEN 1 ELSE 0 END)::smallint AS is_1st, + (CASE WHEN pay_st IN (2, 3) THEN 1 ELSE 0 END)::smallint AS flg_r, + (1 + floor(r_chnl * 4)::int)::smallint AS chnl, + (1 + floor(r_misc * 4)::int)::smallint AS pay_mtd, + + d_crt::timestamptz + ((r_misc * 86400)::int || ' seconds')::interval AS dt_crt, + CASE WHEN pay_st > 0 THEN d_crt::timestamptz + (((r_misc + 0.1) * 86400)::int || ' seconds')::interval END AS dt_pay, + CASE WHEN pay_st > 0 AND r_misc > 0.3 THEN d_crt::timestamptz + INTERVAL '1 day' END AS dt_shp, + CASE WHEN pay_st IN (1, 2) AND r_misc > 0.5 THEN d_crt::timestamptz + INTERVAL '4 days' END AS dt_fin, + CASE WHEN pay_st IN (2, 3) THEN d_crt::timestamptz + INTERVAL '9 days' END AS dt_rfnd, + d_crt AS stat_dt, + + s.shop_nm, + (ARRAY['手机','笔记本','平板','耳机','智能手表','空调','冰箱','洗衣机','面膜','口红', + '洗发水','牛奶','坚果','咖啡','运动鞋','卫衣','连衣裙','背包','图书','玩具'])[1 + cat_id % 20] + AS cat_nm, + (ARRAY['数码','家电','美妆','食品','服饰','运动','图书','母婴'])[1 + cat_id % 8] AS cat_nm_l1, + (ARRAY['旗舰','轻薄','便携','家用','专业','入门'])[1 + floor(r_misc * 6)::int] || '款商品' + || (900000 + cat_id * 1000) AS item_nm, + (ARRAY['华为','小米','苹果','三星','美的','海尔','欧莱雅','雀巢',NULL])[1 + floor(r_misc * 9)::int] AS brand_nm, + (ARRAY['广东','浙江','江苏','山东','河南','四川','湖北','湖南','河北','福建', + '安徽','陕西','江西','辽宁','重庆','北京','上海','天津','广西','云南'])[1 + floor(r_chnl * 20)::int] AS prov, + (ARRAY['深圳','广州','杭州','南京','济南','郑州','成都','武汉','长沙','石家庄', + '福州','合肥','西安','南昌','沈阳','重庆','北京','上海','天津','南宁'])[1 + floor(r_chnl * 20)::int] AS city, + (1 + floor(r_1st * 5)::int)::smallint AS buyer_lvl, + (ARRAY['search','feed','push','ad','direct','share'])[1 + floor(r_chnl * 6)::int] AS src, + (ARRAY['ios','android','pc','h5'])[1 + floor(r_misc * 4)::int] AS dev, + + 1::smallint AS ver, + CASE WHEN r_misc < 0.05 THEN '客服备注：' || (r_misc * 1000)::int END AS rmk, + '{"ab":"' || (1 + (r_misc * 3)::int) || '"}' AS ext +FROM calc +JOIN dim_shop s ON s.shop_id = calc.shop_id; + +CREATE INDEX ON dwd_ord_dtl (stat_dt); +CREATE INDEX ON dwd_ord_dtl (ord_st, pay_st); +ANALYZE; diff --git a/scripts/bench/schemas/wide.sql b/scripts/bench/schemas/wide.sql new file mode 100644 index 000000000..9c1c5c1ee --- /dev/null +++ b/scripts/bench/schemas/wide.sql @@ -0,0 +1,114 @@ +-- 一张打平的电商订单宽表：映射测量台的第二份语料（#501 / #520）。 +-- +-- **TPC-H 量不出语义层是干什么的。** 八张表、诚实的列名、每个分析列一条注释， +-- 模型光看 schema 文档就答对 24 题里的 23 题，语义层只剩一题的空间（#520 第一轮）。 +-- 那不是语义层没用，是那份 schema 自己把答案写在了列名上。 +-- +-- 这一份反过来造：**每一条口径都配一个「模型会自然猜错」的做法**。 +-- +-- 1. **金额单位是分。** `amt_pay` 是 5980 而不是 59.80。天真的 `sum(amt_pay)` +-- 跑得通、数好看、错一百倍。 +-- 2. **列名是内部黑话。** `amt_*` / `dt_*` / `ord_st` / `chnl` / `flg_*`—— +-- 真实数仓的样子，而不是 `l_extendedprice`。 +-- 3. **口径是业务约定，不在列名里。** 「GMV」要排掉测试单、只算已支付； +-- 「有效订单」有一张状态白名单；「净销售额」要扣退款。这些约定 schema +-- 一个字都没写，一个没被告知的人（或模型）不可能猜对。 +-- 4. **相似列成对出现。** `amt_total`（含运费与优惠前）与 `amt_pay`（实付） +-- 差着运费和优惠，选错一个数就错，而两个都跑得通。 +-- 5. **陷阱列。** 整数外键 sum 得动；`ver` 恒为 1；`price_old` 是废弃列全 NULL； +-- `is_test` 求和是「测试单数」，看着像个指标。 +-- +-- join 打平是宽表的定义，也是它没有外键的原因——#502 的 `fetch_keys` 在这种 +-- 表上一无所获，只有抽样与基数分得出 `buyer_id` 是键而 `qty` 是数。 + +DROP SCHEMA IF EXISTS dw CASCADE; +CREATE SCHEMA dw; +SET search_path TO dw; + +-- 订单明细宽表。一行 = 一个订单里的一个商品行 +CREATE TABLE dwd_ord_dtl ( + -- 键。**全是整数，全都 sum 得动** + id BIGSERIAL PRIMARY KEY, + ord_id BIGINT NOT NULL, + ord_ln SMALLINT NOT NULL, + buyer_id BIGINT NOT NULL, + seller_id BIGINT NOT NULL, + shop_id INTEGER NOT NULL, + item_id BIGINT NOT NULL, + sku_id BIGINT NOT NULL, + cat_id INTEGER NOT NULL, + cat_id_l1 INTEGER NOT NULL, + addr_id BIGINT NOT NULL, + pay_id BIGINT, + promo_id INTEGER, + cpn_id INTEGER, + lgst_id BIGINT, + + -- 金额，**单位是分**。天真的 sum 会给出一个大一百倍的数 + amt_total BIGINT NOT NULL, + amt_pay BIGINT NOT NULL, + amt_item BIGINT NOT NULL, + amt_frght BIGINT NOT NULL, + amt_disc BIGINT NOT NULL, + amt_cpn BIGINT NOT NULL, + amt_rfnd BIGINT NOT NULL, + amt_tax BIGINT NOT NULL, + amt_cost BIGINT NOT NULL, + amt_pt BIGINT NOT NULL, + -- 废弃列：迁移之后再没写过，全是 NULL + price_old BIGINT, + + qty INTEGER NOT NULL, + qty_rfnd INTEGER NOT NULL, + wt_g INTEGER, + + -- 状态与标志。**口径的白名单藏在这里，而 schema 不说** + ord_st SMALLINT NOT NULL, + pay_st SMALLINT NOT NULL, + lgst_st SMALLINT NOT NULL, + rfnd_st SMALLINT NOT NULL, + is_test SMALLINT NOT NULL, + is_gift SMALLINT NOT NULL, + is_presale SMALLINT NOT NULL, + is_1st SMALLINT NOT NULL, + flg_r SMALLINT NOT NULL, + chnl SMALLINT NOT NULL, + pay_mtd SMALLINT NOT NULL, + + -- 时间 + dt_crt TIMESTAMPTZ NOT NULL, + dt_pay TIMESTAMPTZ, + dt_shp TIMESTAMPTZ, + dt_fin TIMESTAMPTZ, + dt_rfnd TIMESTAMPTZ, + stat_dt DATE NOT NULL, + + -- 打平进来的维度 + shop_nm VARCHAR(60) NOT NULL, + cat_nm VARCHAR(40) NOT NULL, + cat_nm_l1 VARCHAR(40) NOT NULL, + item_nm VARCHAR(80) NOT NULL, + brand_nm VARCHAR(40), + prov VARCHAR(20) NOT NULL, + city VARCHAR(30) NOT NULL, + buyer_lvl SMALLINT NOT NULL, + src VARCHAR(20) NOT NULL, + dev VARCHAR(20), + + -- 噪声：恒定、自由文本、ETL 元数据 + ver SMALLINT NOT NULL DEFAULT 1, + rmk VARCHAR(200), + ext VARCHAR(200), + etl_dt TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- 店铺维表。**只有它有一条外键关系**，而宽表这边没有约束声明—— +-- 打平之后 shop_id 与 shop_nm 都在事实表里，维表是留着的那份原始记录 +CREATE TABLE dim_shop ( + shop_id INTEGER PRIMARY KEY, + shop_nm VARCHAR(60) NOT NULL, + seller_id BIGINT NOT NULL, + prov VARCHAR(20) NOT NULL, + lvl SMALLINT NOT NULL, + open_dt DATE NOT NULL +); diff --git a/scripts/bench/truth/tpch.mappings.json b/scripts/bench/truth/tpch.mappings.json new file mode 100644 index 000000000..294e45cbe --- /dev/null +++ b/scripts/bench/truth/tpch.mappings.json @@ -0,0 +1,78 @@ +{ + "corpus": "tpch", + "db_schema": "tpch", + "notes": "Definitions come from the TPC-H specification's 22 queries rather than from anyone's reading of this schema. Scoring runs `gold` and the proposal against the same rows and compares the numbers, so what the model named its concept does not matter. `near` marks a definition that is one edit away from another and is the interesting way to be wrong. `keys` is what #502 has to recover; `traps` are columns that aggregate without error and mean nothing. `plausible` holds definitions the benchmark's queries do not state but a reader of this schema would accept — they are counted in their own column, neither right nor wrong, the way govern.mjs counts `unlabeled`. Entries are added there only after a run proposed them, with `seen` recording which run, so the file stays a statement about the schema rather than a record of what the explorer happened to say.", + "metrics": [ + { "id": "line_count", "label": "Order lines", "from": "Q1 count_order", "gold": "SELECT count(*) FROM tpch.lineitem" }, + { "id": "sum_qty", "label": "Units sold", "from": "Q1 sum_qty", "gold": "SELECT sum(l_quantity) FROM tpch.lineitem" }, + { "id": "base_price", "label": "Gross sales", "from": "Q1 sum_base_price", "gold": "SELECT sum(l_extendedprice) FROM tpch.lineitem" }, + { "id": "disc_revenue", "label": "Revenue (discounted)", "from": "Q1 sum_disc_price; the revenue of Q3, Q5, Q7, Q8, Q9, Q15, Q19", "gold": "SELECT sum(l_extendedprice * (1 - l_discount)) FROM tpch.lineitem", "central": true }, + { "id": "charge", "label": "Revenue including tax", "from": "Q1 sum_charge", "gold": "SELECT sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) FROM tpch.lineitem" }, + { "id": "avg_qty", "label": "Average units per line", "from": "Q1 avg_qty", "gold": "SELECT avg(l_quantity) FROM tpch.lineitem" }, + { "id": "avg_price", "label": "Average line amount", "from": "Q1 avg_price", "gold": "SELECT avg(l_extendedprice) FROM tpch.lineitem" }, + { "id": "avg_disc", "label": "Average discount rate", "from": "Q1 avg_disc", "gold": "SELECT avg(l_discount) FROM tpch.lineitem" }, + { "id": "discount_given", "label": "Revenue lost to discount", "from": "Q6 revenue", "gold": "SELECT sum(l_extendedprice * l_discount) FROM tpch.lineitem", "near": "disc_revenue", "note": "Q6 calls this revenue as well, and it is one character from disc_revenue. A proposal computing this under the name of revenue is exactly the failure automatic confirmation would ship." }, + { "id": "returned_revenue", "label": "Revenue of returned lines", "from": "Q10", "gold": "SELECT sum(l_extendedprice * (1 - l_discount)) FROM tpch.lineitem WHERE l_returnflag = 'R'", "near": "disc_revenue" }, + { "id": "late_lines", "label": "Late deliveries", "from": "Q12, receipt after commit", "gold": "SELECT count(*) FROM tpch.lineitem WHERE l_receiptdate > l_commitdate" }, + { "id": "order_count", "label": "Orders", "from": "Q4, Q13", "gold": "SELECT count(*) FROM tpch.orders" }, + { "id": "order_total", "label": "Booked order value", "from": "Q18 o_totalprice", "gold": "SELECT sum(o_totalprice) FROM tpch.orders" }, + { "id": "avg_order_value", "label": "Average order value", "from": "derived from Q18", "gold": "SELECT avg(o_totalprice) FROM tpch.orders" }, + { "id": "urgent_orders", "label": "High priority orders", "from": "Q12 high_line_count", "gold": "SELECT count(*) FROM tpch.orders WHERE o_orderpriority IN ('1-URGENT', '2-HIGH')" }, + { "id": "customer_count", "label": "Customers", "from": "Q22 numcust", "gold": "SELECT count(*) FROM tpch.customer" }, + { "id": "customer_balance", "label": "Customer balance", "from": "Q22 totacctbal", "gold": "SELECT sum(c_acctbal) FROM tpch.customer" }, + { "id": "positive_balance", "label": "Customer balance in credit", "from": "Q22, c_acctbal > 0", "gold": "SELECT sum(c_acctbal) FILTER (WHERE c_acctbal > 0) FROM tpch.customer", "near": "customer_balance" }, + { "id": "inventory_value", "label": "Stock value at cost", "from": "Q11 value", "gold": "SELECT sum(ps_supplycost * ps_availqty) FROM tpch.partsupp", "note": "Two columns multiplied. A proposal summing either one on its own runs fine and is wrong." }, + { "id": "avail_qty", "label": "Units in stock", "from": "Q20", "gold": "SELECT sum(ps_availqty) FROM tpch.partsupp", "near": "inventory_value" }, + { "id": "min_supply_cost", "label": "Best supply cost", "from": "Q2", "gold": "SELECT min(ps_supplycost) FROM tpch.partsupp" }, + { "id": "supplier_count", "label": "Suppliers", "from": "Q16 supplier_cnt", "gold": "SELECT count(*) FROM tpch.supplier" }, + { "id": "part_count", "label": "Parts", "from": "catalogue size", "gold": "SELECT count(*) FROM tpch.part" }, + { "id": "avg_retail_price", "label": "Average list price", "from": "Q17", "gold": "SELECT avg(p_retailprice) FROM tpch.part" } + ], + "plausible": [ + { "id": "return_rate", "label": "Return rate", "gold": "SELECT sum(CASE WHEN l_returnflag = 'R' THEN 1.0 ELSE 0.0 END) / count(*) * 100 FROM tpch.lineitem", "seen": "run 1, commented" }, + { "id": "avg_customer_balance", "label": "Average customer balance", "gold": "SELECT avg(c_acctbal) FROM tpch.customer", "seen": "run 1, uncommented" }, + { "id": "returned_lines", "label": "Returned lines", "gold": "SELECT count(*) FROM tpch.lineitem WHERE l_returnflag = 'R'", "seen": "run 3, commented" } + ], + "dimensions": [ + { "column": "tpch.lineitem.l_returnflag", "from": "Q1 group by" }, + { "column": "tpch.lineitem.l_linestatus", "from": "Q1 group by" }, + { "column": "tpch.lineitem.l_shipmode", "from": "Q12 group by" }, + { "column": "tpch.lineitem.l_shipinstruct", "from": "Q19 filter" }, + { "column": "tpch.lineitem.l_shipdate", "from": "the time axis of Q1, Q3, Q6, Q7" }, + { "column": "tpch.orders.o_orderpriority", "from": "Q4 group by" }, + { "column": "tpch.orders.o_orderstatus", "from": "order state" }, + { "column": "tpch.orders.o_orderdate", "from": "the time axis of Q3, Q5, Q8, Q9" }, + { "column": "tpch.customer.c_mktsegment", "from": "Q3 filter" }, + { "column": "tpch.nation.n_name", "from": "Q5, Q7, Q8, Q9 group by" }, + { "column": "tpch.region.r_name", "from": "Q5, Q8 filter" }, + { "column": "tpch.part.p_brand", "from": "Q16, Q17, Q19 group by" }, + { "column": "tpch.part.p_mfgr", "from": "Q2 filter" }, + { "column": "tpch.part.p_type", "from": "Q8, Q9, Q14 filter" }, + { "column": "tpch.part.p_size", "from": "Q16 group by" }, + { "column": "tpch.part.p_container", "from": "Q17, Q19 filter" } + ], + "keys": [ + { "column": "tpch.lineitem.l_orderkey", "references": "tpch.orders.o_orderkey" }, + { "column": "tpch.lineitem.l_partkey", "references": "tpch.partsupp.ps_partkey" }, + { "column": "tpch.lineitem.l_suppkey", "references": "tpch.partsupp.ps_suppkey" }, + { "column": "tpch.orders.o_custkey", "references": "tpch.customer.c_custkey" }, + { "column": "tpch.customer.c_nationkey", "references": "tpch.nation.n_nationkey" }, + { "column": "tpch.supplier.s_nationkey", "references": "tpch.nation.n_nationkey" }, + { "column": "tpch.nation.n_regionkey", "references": "tpch.region.r_regionkey" }, + { "column": "tpch.partsupp.ps_partkey", "references": "tpch.part.p_partkey" }, + { "column": "tpch.partsupp.ps_suppkey", "references": "tpch.supplier.s_suppkey" } + ], + "traps": [ + { "column": "tpch.lineitem.l_orderkey", "why": "integer key; sums cleanly and means nothing" }, + { "column": "tpch.lineitem.l_partkey", "why": "integer key" }, + { "column": "tpch.lineitem.l_suppkey", "why": "integer key" }, + { "column": "tpch.lineitem.l_linenumber", "why": "a position within its order; summing it counts nothing" }, + { "column": "tpch.orders.o_custkey", "why": "integer key" }, + { "column": "tpch.orders.o_shippriority", "why": "always 0 in this corpus; a metric built on it is silently empty" }, + { "column": "tpch.customer.c_nationkey", "why": "integer key" }, + { "column": "tpch.supplier.s_nationkey", "why": "integer key" }, + { "column": "tpch.nation.n_regionkey", "why": "integer key" }, + { "column": "tpch.part.p_size", "as": "metric", "why": "a unitless nominal size; summing it means nothing, while grouping by it is what Q16 does" }, + { "column": "tpch.lineitem.l_comment", "as": "dimension", "why": "free text; a dimension built on it has one group per row" } + ] +} diff --git a/scripts/bench/truth/tpch.questions.json b/scripts/bench/truth/tpch.questions.json new file mode 100644 index 000000000..8067bf518 --- /dev/null +++ b/scripts/bench/truth/tpch.questions.json @@ -0,0 +1,30 @@ +{ + "corpus": "tpch", + "notes": "One question per single-value definition in tpch.mappings.json, keyed by the same id — so a wrong answer points straight at the definition it needed, and the script can say whether that definition had a confirmed mapping. Questions are worded the way somebody who has never seen the schema would ask them: no table names, no column names, no SQL. The gold SQL is not repeated here; it lives with the definition.", + "questions": [ + { "id": "line_count", "ask": "How many order lines are there in total?" }, + { "id": "sum_qty", "ask": "What is the total number of units sold across all order lines?" }, + { "id": "base_price", "ask": "What are our total gross sales, before any discount or tax?" }, + { "id": "disc_revenue", "ask": "What is our total revenue after discounts, before tax?" }, + { "id": "charge", "ask": "What is our total revenue including tax, after discounts?" }, + { "id": "avg_qty", "ask": "On average, how many units are on an order line?" }, + { "id": "avg_price", "ask": "What is the average amount of an order line before discount?" }, + { "id": "avg_disc", "ask": "What discount rate do we give on average?" }, + { "id": "discount_given", "ask": "How much money did we give away in discounts in total?" }, + { "id": "returned_revenue", "ask": "What is the discounted revenue of the lines that customers returned?" }, + { "id": "late_lines", "ask": "How many order lines reached the customer later than the date we committed to?" }, + { "id": "order_count", "ask": "How many orders do we have?" }, + { "id": "order_total", "ask": "What is the total booked value of all orders?" }, + { "id": "avg_order_value", "ask": "What is the average value of an order?" }, + { "id": "urgent_orders", "ask": "How many orders are urgent or high priority?" }, + { "id": "customer_count", "ask": "How many customers do we have?" }, + { "id": "customer_balance", "ask": "What is the total account balance across all our customers?" }, + { "id": "positive_balance", "ask": "Counting only the customers who are in credit, what is their total account balance?" }, + { "id": "inventory_value", "ask": "What is the total value of the stock our suppliers hold, valued at supply cost?" }, + { "id": "avail_qty", "ask": "How many units in total do our suppliers have available?" }, + { "id": "min_supply_cost", "ask": "What is the lowest supply cost any supplier quotes?" }, + { "id": "supplier_count", "ask": "How many suppliers do we have?" }, + { "id": "part_count", "ask": "How many parts are in the catalogue?" }, + { "id": "avg_retail_price", "ask": "What is the average list price of a part?" } + ] +} diff --git a/scripts/bench/truth/wide.mappings.json b/scripts/bench/truth/wide.mappings.json new file mode 100644 index 000000000..80da5349d --- /dev/null +++ b/scripts/bench/truth/wide.mappings.json @@ -0,0 +1,282 @@ +{ + "corpus": "wide", + "db_schema": "dw", + "notes": "Unlike tpch, these definitions are NOT derivable from the schema. Every one carries at least one business convention the columns do not state: amounts are in cents, test orders are excluded, a status whitelist decides what counts as an order, refunds come off the top. `naive` records what a reader who only has the schema would plausibly write, and the gap between `gold` and `naive` is the whole point of the corpus — both run, both look reasonable, and only one is the number the business means.", + "conventions": [ + "Money columns (amt_*) are in CENTS. Every money definition divides by 100.", + "is_test = 1 marks internal test orders and is excluded from every business figure.", + "An order counts once it is paid: ord_st in (2,3,4). 1 = created but never paid, 5 = closed unpaid, 6 = fully refunded.", + "pay_st: 0 unpaid, 1 paid, 2 partially refunded, 3 fully refunded. Paid means pay_st in (1,2).", + "One row is one line of an order; an order is ord_id and usually has two lines.", + "Net figures subtract amt_rfnd. Gross figures do not." + ], + "metrics": [ + { + "id": "gmv", + "label": "GMV", + "central": true, + "gold": "SELECT round(sum(amt_pay) FILTER (WHERE pay_st IN (1,2) AND is_test = 0) / 100.0, 2) FROM dw.dwd_ord_dtl", + "naive": "SELECT sum(amt_pay) FROM dw.dwd_ord_dtl", + "note": "Cents, test orders, and unpaid rows — three conventions in one definition. The naive answer is off by a factor of a hundred and includes orders nobody paid for." + }, + { + "id": "net_sales", + "label": "Net sales after refunds", + "gold": "SELECT round(sum(amt_pay - amt_rfnd) FILTER (WHERE pay_st IN (1,2) AND is_test = 0) / 100.0, 2) FROM dw.dwd_ord_dtl", + "naive": "SELECT round(sum(amt_pay) / 100.0, 2) FROM dw.dwd_ord_dtl", + "near": "gmv" + }, + { + "id": "gross_merch", + "label": "Gross merchandise value before discount", + "gold": "SELECT round(sum(amt_total) FILTER (WHERE pay_st IN (1,2) AND is_test = 0) / 100.0, 2) FROM dw.dwd_ord_dtl", + "naive": "SELECT round(sum(amt_total) / 100.0, 2) FROM dw.dwd_ord_dtl", + "near": "gmv", + "note": "amt_total is before discount and coupon; amt_pay is what was actually charged. Both exist, both are money, one is the answer." + }, + { + "id": "valid_orders", + "label": "Paid orders", + "gold": "SELECT count(DISTINCT ord_id) FILTER (WHERE ord_st IN (2,3,4) AND is_test = 0) FROM dw.dwd_ord_dtl", + "naive": "SELECT count(*) FROM dw.dwd_ord_dtl", + "note": "A row is a line, not an order, and half the statuses never became a sale." + }, + { + "id": "valid_lines", + "label": "Paid order lines", + "gold": "SELECT count(*) FILTER (WHERE ord_st IN (2,3,4) AND is_test = 0) FROM dw.dwd_ord_dtl", + "naive": "SELECT count(*) FROM dw.dwd_ord_dtl", + "near": "valid_orders" + }, + { + "id": "buyers", + "label": "Buyers who paid", + "gold": "SELECT count(DISTINCT buyer_id) FILTER (WHERE pay_st IN (1,2) AND is_test = 0) FROM dw.dwd_ord_dtl", + "naive": "SELECT count(buyer_id) FROM dw.dwd_ord_dtl", + "note": "Every buyer in this corpus paid at least once, so the filter changes nothing here — the trap is forgetting DISTINCT, which turns 5000 buyers into 60000 rows." + }, + { + "id": "new_buyers", + "label": "First-time buyers", + "gold": "SELECT count(DISTINCT buyer_id) FILTER (WHERE is_1st = 1 AND pay_st IN (1,2) AND is_test = 0) FROM dw.dwd_ord_dtl", + "naive": "SELECT count(DISTINCT buyer_id) FILTER (WHERE is_1st = 1) FROM dw.dwd_ord_dtl", + "near": "buyers" + }, + { + "id": "aov", + "label": "Average order value", + "gold": "SELECT round((sum(amt_pay - amt_rfnd) FILTER (WHERE pay_st IN (1,2) AND is_test = 0) / 100.0) / count(DISTINCT ord_id) FILTER (WHERE ord_st IN (2,3,4) AND is_test = 0), 2) FROM dw.dwd_ord_dtl", + "naive": "SELECT round(avg(amt_pay) / 100.0, 2) FROM dw.dwd_ord_dtl", + "note": "Per order, not per line — the naive average is per row and lands about half the truth." + }, + { + "id": "units", + "label": "Units sold net of returns", + "gold": "SELECT sum(qty - qty_rfnd) FILTER (WHERE pay_st IN (1,2) AND is_test = 0) FROM dw.dwd_ord_dtl", + "naive": "SELECT sum(qty) FROM dw.dwd_ord_dtl" + }, + { + "id": "refund_amt", + "label": "Refunded amount", + "gold": "SELECT round(sum(amt_rfnd) FILTER (WHERE is_test = 0) / 100.0, 2) FROM dw.dwd_ord_dtl", + "naive": "SELECT sum(amt_rfnd) FROM dw.dwd_ord_dtl" + }, + { + "id": "refund_rate", + "label": "Refund rate over paid orders, percent", + "gold": "SELECT round(100.0 * count(DISTINCT ord_id) FILTER (WHERE pay_st IN (2,3) AND is_test = 0) / count(DISTINCT ord_id) FILTER (WHERE pay_st IN (1,2,3) AND is_test = 0), 2) FROM dw.dwd_ord_dtl", + "naive": "SELECT round(100.0 * sum(flg_r) / count(*), 2) FROM dw.dwd_ord_dtl" + }, + { + "id": "gross_profit", + "label": "Gross profit", + "gold": "SELECT round(sum(amt_pay - amt_rfnd - amt_cost) FILTER (WHERE pay_st IN (1,2) AND is_test = 0) / 100.0, 2) FROM dw.dwd_ord_dtl", + "naive": "SELECT round(sum(amt_pay - amt_cost) / 100.0, 2) FROM dw.dwd_ord_dtl" + }, + { + "id": "gross_margin", + "label": "Gross margin percent", + "gold": "SELECT round(100.0 * sum(amt_pay - amt_rfnd - amt_cost) FILTER (WHERE pay_st IN (1,2) AND is_test = 0) / sum(amt_pay - amt_rfnd) FILTER (WHERE pay_st IN (1,2) AND is_test = 0), 2) FROM dw.dwd_ord_dtl" + }, + { + "id": "freight", + "label": "Freight collected", + "gold": "SELECT round(sum(amt_frght) FILTER (WHERE pay_st IN (1,2) AND is_test = 0) / 100.0, 2) FROM dw.dwd_ord_dtl", + "naive": "SELECT sum(amt_frght) FROM dw.dwd_ord_dtl" + }, + { + "id": "discount_total", + "label": "Total discount given", + "gold": "SELECT round(sum(amt_disc + amt_cpn) FILTER (WHERE pay_st IN (1,2) AND is_test = 0) / 100.0, 2) FROM dw.dwd_ord_dtl", + "naive": "SELECT round(sum(amt_disc) / 100.0, 2) FROM dw.dwd_ord_dtl", + "note": "Two columns make up a discount: the promotion and the coupon. Summing one is a plausible, wrong answer." + }, + { + "id": "app_gmv", + "label": "GMV from the app channel", + "gold": "SELECT round(sum(amt_pay) FILTER (WHERE chnl = 1 AND pay_st IN (1,2) AND is_test = 0) / 100.0, 2) FROM dw.dwd_ord_dtl", + "naive": "SELECT round(sum(amt_pay) FILTER (WHERE chnl = 1) / 100.0, 2) FROM dw.dwd_ord_dtl", + "near": "gmv" + }, + { + "id": "avg_unit_price", + "label": "Average price per unit sold", + "gold": "SELECT round((sum(amt_pay - amt_rfnd) FILTER (WHERE pay_st IN (1,2) AND is_test = 0) / 100.0) / sum(qty - qty_rfnd) FILTER (WHERE pay_st IN (1,2) AND is_test = 0), 2) FROM dw.dwd_ord_dtl" + }, + { + "id": "test_orders", + "label": "Internal test orders", + "gold": "SELECT count(DISTINCT ord_id) FILTER (WHERE is_test = 1) FROM dw.dwd_ord_dtl", + "note": "The one question where is_test is the subject rather than a filter. It exists so the corpus is not only about excluding it." + } + ], + "dimensions": [ + { + "column": "dw.dwd_ord_dtl.shop_nm", + "from": "shop" + }, + { + "column": "dw.dwd_ord_dtl.cat_nm", + "from": "leaf category" + }, + { + "column": "dw.dwd_ord_dtl.cat_nm_l1", + "from": "top category" + }, + { + "column": "dw.dwd_ord_dtl.brand_nm", + "from": "brand" + }, + { + "column": "dw.dwd_ord_dtl.prov", + "from": "province" + }, + { + "column": "dw.dwd_ord_dtl.city", + "from": "city" + }, + { + "column": "dw.dwd_ord_dtl.chnl", + "from": "channel code" + }, + { + "column": "dw.dwd_ord_dtl.buyer_lvl", + "from": "buyer tier" + }, + { + "column": "dw.dwd_ord_dtl.src", + "from": "traffic source" + }, + { + "column": "dw.dwd_ord_dtl.dev", + "from": "device" + }, + { + "column": "dw.dwd_ord_dtl.stat_dt", + "from": "the time axis" + }, + { + "column": "dw.dwd_ord_dtl.ord_st", + "from": "order status" + }, + { + "column": "dw.dwd_ord_dtl.pay_st", + "from": "payment status" + }, + { + "column": "dw.dwd_ord_dtl.pay_mtd", + "from": "payment method" + }, + { + "column": "dw.dwd_ord_dtl.dt_crt", + "from": "order time — the other time axis besides stat_dt" + }, + { + "column": "dw.dim_shop.shop_nm", + "from": "shop, read from the dimension table instead of the flattened column" + } + ], + "keys": [ + { + "column": "dw.dwd_ord_dtl.shop_id", + "references": "dw.dim_shop.shop_id" + } + ], + "traps": [ + { + "column": "dw.dwd_ord_dtl.buyer_id", + "as": "metric", + "why": "integer key; sums cleanly and means nothing" + }, + { + "column": "dw.dwd_ord_dtl.seller_id", + "as": "metric", + "why": "integer key" + }, + { + "column": "dw.dwd_ord_dtl.shop_id", + "as": "metric", + "why": "integer key" + }, + { + "column": "dw.dwd_ord_dtl.item_id", + "as": "metric", + "why": "integer key" + }, + { + "column": "dw.dwd_ord_dtl.sku_id", + "as": "metric", + "why": "integer key" + }, + { + "column": "dw.dwd_ord_dtl.cat_id", + "as": "metric", + "why": "integer key" + }, + { + "column": "dw.dwd_ord_dtl.addr_id", + "as": "metric", + "why": "integer key" + }, + { + "column": "dw.dwd_ord_dtl.pay_id", + "as": "metric", + "why": "integer key" + }, + { + "column": "dw.dwd_ord_dtl.ord_id", + "as": "metric", + "why": "integer key; count(distinct) is a metric, sum is not" + }, + { + "column": "dw.dwd_ord_dtl.ver", + "as": "both", + "why": "always 1; a metric on it counts rows by accident, a dimension on it has one group" + }, + { + "column": "dw.dwd_ord_dtl.price_old", + "as": "both", + "why": "abandoned column, all NULL since the migration — a metric on it is silently empty" + }, + { + "column": "dw.dwd_ord_dtl.amt_pt", + "as": "metric", + "why": "loyalty points, not money; summing it into a revenue figure mixes units" + }, + { + "column": "dw.dwd_ord_dtl.rmk", + "as": "dimension", + "why": "free text typed by support; one group per row" + }, + { + "column": "dw.dwd_ord_dtl.ext", + "as": "dimension", + "why": "a JSON blob as text" + }, + { + "column": "dw.dwd_ord_dtl.etl_dt", + "as": "both", + "why": "when the ETL ran, not when anything happened — a time axis built on it is meaningless" + } + ] +} diff --git a/scripts/bench/truth/wide.questions.json b/scripts/bench/truth/wide.questions.json new file mode 100644 index 000000000..c7ae40a6b --- /dev/null +++ b/scripts/bench/truth/wide.questions.json @@ -0,0 +1,24 @@ +{ + "corpus": "wide", + "notes": "中文提问，因为这份语料本身是中文业务场景（店铺名、省份、类目都是中文），顺带量一次中文问数。问法是业务的说法，不含表名列名——一个不看 schema 的人会这么问。每一题对应 wide.mappings.json 里同 id 的口径，所以答错时脚本说得出它需要的那条口径有没有确认映射。", + "questions": [ + { "id": "gmv", "ask": "我们的 GMV 一共是多少？" }, + { "id": "net_sales", "ask": "扣掉退款之后的净销售额是多少？" }, + { "id": "gross_merch", "ask": "优惠和券之前的商品交易总额是多少？" }, + { "id": "valid_orders", "ask": "一共有多少笔有效订单？" }, + { "id": "valid_lines", "ask": "有效订单里一共有多少个商品行？" }, + { "id": "buyers", "ask": "有多少个买家真正付款下过单？" }, + { "id": "new_buyers", "ask": "有多少个首单买家？" }, + { "id": "aov", "ask": "客单价是多少？" }, + { "id": "units", "ask": "扣掉退货之后一共卖出去多少件商品？" }, + { "id": "refund_amt", "ask": "退款总金额是多少？" }, + { "id": "refund_rate", "ask": "退款率是多少？" }, + { "id": "gross_profit", "ask": "毛利是多少？" }, + { "id": "gross_margin", "ask": "毛利率是多少？" }, + { "id": "freight", "ask": "运费一共收了多少钱？" }, + { "id": "discount_total", "ask": "一共让出去多少优惠？" }, + { "id": "app_gmv", "ask": "APP 渠道的 GMV 是多少？" }, + { "id": "avg_unit_price", "ask": "平均每件商品卖多少钱？" }, + { "id": "test_orders", "ask": "有多少笔是内部测试订单？" } + ] +}