From 69c3af39ac3660c91b1767fcc82016e164378526 Mon Sep 17 00:00:00 2001 From: doc Date: Thu, 21 May 2026 16:48:53 -0400 Subject: [PATCH] fix(retrieve): disable length normalization that suppressed long-doc recall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stage_length_normalization divided each result's fused score by (1 + log2(content_len/500)).max(1.0), penalizing long memories up to ~4x (a 4KB note /4). Cosine similarity is already length-invariant, so this double-penalized length and buried detailed runbooks/inventories under short, less-relevant entries: the #1 vector hit (0.658) was demoted to ~0.16 and dropped from the top-k. Disable it — length must not suppress recall in a memory store. Co-Authored-By: Claude Opus 4.7 (1M context) --- omem-server/src/retrieve/pipeline.rs | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/omem-server/src/retrieve/pipeline.rs b/omem-server/src/retrieve/pipeline.rs index 459ad31..adc3091 100644 --- a/omem-server/src/retrieve/pipeline.rs +++ b/omem-server/src/retrieve/pipeline.rs @@ -514,20 +514,15 @@ impl RetrievalPipeline { (entries, stage) } - fn stage_length_normalization(mut entries: Vec) -> (Vec, StageTrace) { + fn stage_length_normalization(entries: Vec) -> (Vec, StageTrace) { let stage_start = Instant::now(); let input_count = entries.len(); - for entry in &mut entries { - let len_ratio = entry.memory.content.len() as f32 / 500.0; - let log_val = if len_ratio > 0.0 { - len_ratio.log2() - } else { - 0.0 - }; - let denominator = (1.0 + log_val).max(1.0); - entry.rrf_score /= denominator; - } + // Length normalization DISABLED. Cosine vector similarity is already + // length-invariant, so dividing the fused score by (1 + log2(len/500)) + // double-penalized long memories — up to ~4x for a 4KB note — burying + // detailed runbooks/inventories under short, less-relevant entries. + // This is a recall store: document length must not suppress recall. let score_range = fusion_score_range(&entries); @@ -1075,14 +1070,17 @@ mod tests { let (long_result, _) = RetrievalPipeline::stage_length_normalization(long_entries); let long_score = long_result[0].rrf_score; + // Length normalization is disabled: cosine similarity is already + // length-invariant, so a long memory keeps the same fused score as a + // short one — no length penalty. assert!( - short_score > long_score, - "short ({short_score}) should score higher than long ({long_score})" + (short_score - long_score).abs() < f32::EPSILON, + "length must not change the score: short={short_score} long={long_score}" ); assert!( - (short_score - 1.0).abs() < f32::EPSILON, - "short content should not be penalized: got {short_score}" + (long_score - 1.0).abs() < f32::EPSILON, + "long content must not be penalized by length: got {long_score}" ); }