From 7c180d466cd35c5352c546f6d65e7c92de508702 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Fri, 18 Sep 2026 08:41:03 -0500 Subject: [PATCH] fix(mem_wal): build the transient FTS index with positions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A phrase query over a column with no FTS index returned nothing from the fresh tier while a plain match on the same rows succeeded. The base answers that query — `plan_phrase_query`'s no-index arm plans a flat scan, where token positions are implicit — but the transient index the fresh tier builds for an unindexed column took `InvertedIndexParams:: default()`, whose `with_position` is `false`, and `search_phrase_tokens` returns no hit for a multi-token phrase without positions. Every phrase hit in un-compacted rows was dropped, silently. When no persisted index covers the column, the transient index now carries positions. The persisted-index case still inherits the index's analyzer and positional settings, which is the contract that keeps the active rows agreeing with base and SSTable rows. The transient index lives for one query over the visible prefix, so the extra storage is bounded by that prefix. Co-Authored-By: Claude Fable 5.1 --- .../src/dataset/mem_wal/scanner/fts_search.rs | 102 +++++++++++++++++- 1 file changed, 100 insertions(+), 2 deletions(-) diff --git a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs index 8092d1b0329..5e9e6eb3bbf 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs @@ -343,12 +343,16 @@ fn transient_fts_index_store( // defaults here would have the active rows disagree with base and SSTable // rows about what matches — silently, by returning fewer rows. Defaults are // right only when no persisted index covers the column, where there is no - // contract to match. + // contract to match — except for positions. The base answers a phrase over + // an unindexed column from a flat scan, where positions are implicit, and a + // transient index built without them returns no phrase hit for rows the + // base finds. The index lives for one query over the visible prefix, so the + // extra storage is bounded by that prefix. for (column, field_id) in field_ids { let params = index_params .get(column) .cloned() - .unwrap_or_default() + .unwrap_or_else(|| InvertedIndexParams::default().with_position(true)) .document_granularity(document_granularity); store.add_fts_with_params( format!("__transient_fts_{column}"), @@ -2935,6 +2939,100 @@ mod tests { assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 0); } + /// A column with no FTS index is served by a transient index over the + /// visible prefix. The base answers a phrase there from a flat scan, so the + /// transient index has to carry positions or every phrase hit in fresh rows + /// is silently lost while a plain match on the same rows succeeds. + #[tokio::test] + async fn phrase_over_an_unindexed_column_reaches_the_active_memtable() { + use lance_index::scalar::inverted::query::PhraseQuery; + + let schema = fts_schema(); + let batch_store = Arc::new(BatchStore::with_capacity(16)); + let mut indexes = IndexStore::new(); + indexes.enable_pk_index(&[("id".to_string(), 0)]); + // Deliberately no FTS index on `text`. + let active_batch = make_batch( + &schema, + &[1, 2, 3], + &["alpha prose here", "prose alpha reversed", "nothing"], + ); + let (_, row_offset, batch_position) = batch_store.append(active_batch.clone()).unwrap(); + indexes + .insert_with_batch_position(&active_batch, row_offset, Some(batch_position)) + .unwrap(); + let indexes = Arc::new(indexes); + + let tmp = tempfile::tempdir().unwrap(); + let base_uri = format!("{}/base", tmp.path().to_str().unwrap()); + let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![]) + .with_in_memory_memtables( + uuid::Uuid::new_v4(), + InMemoryMemTables { + active: InMemoryMemTableRef { + batch_store, + index_store: indexes, + schema: schema.clone(), + generation: 1, + }, + frozen: vec![], + }, + ); + let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema); + let ctx = datafusion::prelude::SessionContext::new(); + let ids_for = |query: FullTextSearchQuery| { + let planner = &planner; + let ctx = &ctx; + async move { + let plan = planner + .plan_search(query, Some(10), Some(&["id".to_string()])) + .await + .expect("plans"); + let batches: Vec = plan + .execute(0, ctx.task_ctx()) + .unwrap() + .try_collect() + .await + .unwrap(); + let mut ids: Vec = batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + ids.sort_unstable(); + ids + } + }; + + let matched = ids_for(FullTextSearchQuery::new_query(IndexFtsQuery::Match( + MatchQuery::new("alpha".to_string()).with_column(Some("text".to_string())), + ))) + .await; + assert_eq!( + matched, + vec![1, 2], + "precondition: the transient index serves a match" + ); + + let phrase = ids_for(FullTextSearchQuery::new_query(IndexFtsQuery::Phrase( + PhraseQuery::new("alpha prose".to_string()).with_column(Some("text".to_string())), + ))) + .await; + assert_eq!( + phrase, + vec![1], + "the phrase matches row 1 only; row 2 carries the terms reversed" + ); + } + /// A multi-match reaches the active memtable across every queried column, /// and a row matching in more than one is returned once rather than per /// column.