diff --git a/crates/khive-mcp/src/coordinator.rs b/crates/khive-mcp/src/coordinator.rs index 57febafb7..46c9d1f03 100644 --- a/crates/khive-mcp/src/coordinator.rs +++ b/crates/khive-mcp/src/coordinator.rs @@ -67,8 +67,16 @@ pub struct BackendSearchResult { pub backend_id: BackendId, pub entity_hits: Vec, pub note_hits: Vec, - /// Populated when this backend errored during the fan-out. + /// Whether this backend selected the vector arm for this search. + pub vector_selected: bool, + /// Populated when this backend errored during the fan-out. A whole-backend + /// failure (e.g. the text arm, or a fatal error before either arm ran) — + /// this backend contributed no hits at all. pub error: Option, + /// Populated when only the vector arm failed and the text arm still ran: + /// `entity_hits` still carries the text arm's results, and `error` above + /// stays `None`. + pub vector_error: Option, } /// Merged fan-out search result. @@ -175,6 +183,10 @@ pub(crate) mod tests { pub search_called: std::sync::atomic::AtomicBool, pub single_backend: bool, pub failed_backend: Option, + /// A backend whose vector arm alone fails — its text arm still runs + /// and contributes hits, so it must not appear in `failed_backend`'s + /// whole-backend-error reporting. + pub vector_failed_backend: Option, /// When `true`, `fan_out_search` returns zero hits regardless of /// substrate — used to construct the "complete-empty" (healthy, no /// match) and "degraded-empty" (backend failed, no survivor) @@ -195,6 +207,7 @@ pub(crate) mod tests { search_called: std::sync::atomic::AtomicBool::new(false), single_backend: false, failed_backend: None, + vector_failed_backend: None, empty_hits: false, last_search_request: std::sync::Mutex::new(None), last_limit: std::sync::atomic::AtomicU32::new(0), @@ -211,6 +224,7 @@ pub(crate) mod tests { search_called: std::sync::atomic::AtomicBool::new(false), single_backend: false, failed_backend: None, + vector_failed_backend: None, empty_hits: true, last_search_request: std::sync::Mutex::new(None), last_limit: std::sync::atomic::AtomicU32::new(0), @@ -224,6 +238,7 @@ pub(crate) mod tests { search_called: std::sync::atomic::AtomicBool::new(false), single_backend: false, failed_backend: Some(BackendId::parse(failed_backend).expect("valid backend id")), + vector_failed_backend: None, empty_hits: false, last_search_request: std::sync::Mutex::new(None), last_limit: std::sync::atomic::AtomicU32::new(0), @@ -239,6 +254,7 @@ pub(crate) mod tests { search_called: std::sync::atomic::AtomicBool::new(false), single_backend: false, failed_backend: Some(BackendId::parse(failed_backend).expect("valid backend id")), + vector_failed_backend: None, empty_hits: true, last_search_request: std::sync::Mutex::new(None), last_limit: std::sync::atomic::AtomicU32::new(0), @@ -246,12 +262,33 @@ pub(crate) mod tests { }) } + /// A backend's vector arm alone failed — its text arm still ran and + /// contributed hits, so this must read as `status="complete"` with + /// `arm_participation.text.status="ran"`, not as a whole-backend + /// failure. + pub fn vector_degraded_multi_backend(vector_failed_backend: &str) -> Arc { + Arc::new(Self { + link_called: std::sync::atomic::AtomicBool::new(false), + search_called: std::sync::atomic::AtomicBool::new(false), + single_backend: false, + failed_backend: None, + vector_failed_backend: Some( + BackendId::parse(vector_failed_backend).expect("valid backend id"), + ), + empty_hits: false, + last_search_request: std::sync::Mutex::new(None), + last_limit: std::sync::atomic::AtomicU32::new(0), + last_extra_visible: std::sync::Mutex::new(Vec::new()), + }) + } + pub fn single_backend_instance() -> Arc { Arc::new(Self { link_called: std::sync::atomic::AtomicBool::new(false), search_called: std::sync::atomic::AtomicBool::new(false), single_backend: true, failed_backend: None, + vector_failed_backend: None, empty_hits: false, last_search_request: std::sync::Mutex::new(None), last_limit: std::sync::atomic::AtomicU32::new(0), @@ -323,17 +360,47 @@ pub(crate) mod tests { } else { vec![] }, - per_backend: self - .failed_backend - .iter() - .cloned() - .map(|backend_id| BackendSearchResult { - backend_id, - entity_hits: vec![], - note_hits: vec![], - error: Some("injected search failure".to_string()), - }) - .collect(), + per_backend: std::iter::once(BackendSearchResult { + backend_id: BackendId::main(), + entity_hits: vec![], + note_hits: vec![], + vector_selected: true, + error: None, + vector_error: None, + }) + .chain( + self.failed_backend + .iter() + .cloned() + .map(|backend_id| BackendSearchResult { + backend_id, + entity_hits: vec![], + note_hits: vec![], + vector_selected: true, + error: Some("injected search failure".to_string()), + vector_error: None, + }), + ) + .chain( + self.vector_failed_backend + .iter() + .cloned() + .map(|backend_id| BackendSearchResult { + backend_id, + entity_hits: vec![SearchHit { + entity_id: id, + score: Default::default(), + source: SearchSource::Text, + title: Some("entity result".to_string()), + snippet: None, + }], + note_hits: vec![], + vector_selected: true, + error: None, + vector_error: Some("injected vector-arm failure".to_string()), + }), + ) + .collect(), partial: self.failed_backend.is_some(), entity_kinds: std::collections::HashMap::from([(id, "concept".to_string())]), note_kinds: std::collections::HashMap::from([(id, "observation".to_string())]), @@ -618,13 +685,17 @@ pub(crate) mod tests { #[serial_test::serial(config_ledger)] async fn degraded_search_advisory_survives_single_batch_chain_and_presentation() { let cases = [ - (r#"search(kind="note", query="x")"#, None), - (r#"[search(kind="entity", query="x"), stats()]"#, None), - (r#"search(kind="entity", query="x") | stats()"#, None), - (r#"search(kind="entity", query="x")"#, Some("human")), + (r#"search(kind="note", query="x")"#, None, true), + ( + r#"[search(kind="entity", query="x"), stats()]"#, + None, + false, + ), + (r#"search(kind="entity", query="x") | stats()"#, None, false), + (r#"search(kind="entity", query="x")"#, Some("human"), false), ]; - for (ops, presentation) in cases { + for (ops, presentation, is_note) in cases { let (registry, _runtime) = make_registry(); let coord = MockCoordinator::degraded_multi_backend("archive"); let server = KhiveMcpServer::from_registry_with_meta(registry, "local", "test-cfg") @@ -652,6 +723,15 @@ pub(crate) mod tests { ); assert_eq!(search["partial"], json!(true)); assert_eq!(search["missing_backends"], json!(["archive"])); + let expected_text_candidates = usize::from(!is_note); + assert_eq!( + search["arm_participation"], + json!({ + "text": {"status": "error", "candidate_count": expected_text_candidates}, + "vector": {"status": "error", "candidate_count": 1} + }), + "selected arms must remain typed on partial-with-hit responses" + ); assert_eq!( search["backend_errors"], json!({ @@ -693,11 +773,79 @@ pub(crate) mod tests { assert_eq!(search["ok"], json!(true), "unexpected response: {search}"); assert_eq!(search["status"], json!("complete")); assert_eq!(search["result"], json!([])); + assert_eq!( + search["arm_participation"], + json!({ + "text": {"status": "ran", "candidate_count": 0}, + "vector": {"status": "ran", "candidate_count": 0} + }) + ); assert!(search.get("partial").is_none()); assert!(search.get("missing_backends").is_none()); assert!(search.get("backend_errors").is_none()); } + /// A vector-arm-only failure (the text arm still ran and contributed a + /// hit) must read as a healthy `status="complete"` response, never a + /// whole-backend failure: no `partial`/`missing_backends`/`backend_errors`, + /// and `arm_participation` alone carries the vector arm's error while the + /// text arm still reports `"ran"`. + #[tokio::test] + #[serial_test::serial(config_ledger)] + async fn search_vector_arm_failure_reports_complete_status_with_arm_participation_error() { + let (registry, _runtime) = make_registry(); + let coord = MockCoordinator::vector_degraded_multi_backend("archive"); + let server = KhiveMcpServer::from_registry_with_meta(registry, "local", "test-cfg") + .with_coordinator(Arc::clone(&coord) as Arc); + + let raw = server + .dispatch_request_local(RequestParams { + ops: r#"search(kind="entity", query="LoRA")"#.to_string(), + presentation: None, + presentation_per_op: None, + save_to: None, + format: None, + format_per_op: None, + request_id: None, + }) + .await + .expect("a vector-arm-only failure is still a successful dispatch"); + let response: Value = serde_json::from_str(&raw).expect("JSON response"); + let search = &response["results"][0]; + assert_eq!(search["ok"], json!(true), "unexpected response: {search}"); + assert_eq!( + search["status"], + json!("complete"), + "unexpected response: {search}" + ); + assert!( + !search["result"].as_array().unwrap().is_empty(), + "text arm's hit must survive: {search}" + ); + assert!( + search.get("partial").is_none(), + "a vector-arm-only failure must not read as partial: {search}" + ); + assert!( + search.get("missing_backends").is_none(), + "a backend that returned text hits is not missing: {search}" + ); + assert!( + search.get("backend_errors").is_none(), + "unexpected response: {search}" + ); + assert_eq!( + search["arm_participation"]["text"]["status"], + json!("ran"), + "unexpected response: {search}" + ); + assert_eq!( + search["arm_participation"]["vector"]["status"], + json!("error"), + "unexpected response: {search}" + ); + } + /// ADR-130 §1 completeness contract, degraded-empty case: a backend /// failed and nothing survived — the operation must fail outright with /// `error.kind: "search_incomplete"`, never a successful empty result. @@ -731,6 +879,13 @@ pub(crate) mod tests { assert_eq!(search["error"]["kind"], json!("search_incomplete")); assert_eq!(search["error"]["retryable"], json!(false)); assert_eq!(search["error"]["missing_backends"], json!(["archive"])); + assert_eq!( + search["error"]["arm_participation"], + json!({ + "text": {"status": "error", "candidate_count": 0}, + "vector": {"status": "error", "candidate_count": 0} + }) + ); assert_eq!( search["error"]["backend_errors"], json!({ @@ -986,6 +1141,15 @@ pub(crate) mod tests { Some(expected_source), "{kind} hit must expose its retrieval source; got: {hit}" ); + let expected_text_candidates = usize::from(kind == "entity"); + assert_eq!( + entry["arm_participation"], + json!({ + "text": {"status": "ran", "candidate_count": expected_text_candidates}, + "vector": {"status": "ran", "candidate_count": 1} + }), + "{kind} search must count final candidates by source membership" + ); assert!(entry.get("partial").is_none()); assert!(entry.get("missing_backends").is_none()); } diff --git a/crates/khive-mcp/src/server.rs b/crates/khive-mcp/src/server.rs index 1c35623c7..885210d62 100644 --- a/crates/khive-mcp/src/server.rs +++ b/crates/khive-mcp/src/server.rs @@ -85,6 +85,60 @@ impl SearchStatus { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SearchArmStatus { + Ran, + Skipped, + Error, +} + +impl SearchArmStatus { + fn as_str(self) -> &'static str { + match self { + Self::Ran => "ran", + Self::Skipped => "skipped", + Self::Error => "error", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct SearchArmEvidence { + status: SearchArmStatus, + candidate_count: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct SearchArmParticipation { + text: SearchArmEvidence, + vector: SearchArmEvidence, +} + +impl SearchArmParticipation { + fn complete(vector_selected: bool) -> Self { + Self { + text: SearchArmEvidence { + status: SearchArmStatus::Ran, + candidate_count: 0, + }, + vector: SearchArmEvidence { + status: if vector_selected { + SearchArmStatus::Ran + } else { + SearchArmStatus::Skipped + }, + candidate_count: 0, + }, + } + } + + fn observe_result(&mut self, result: &Value) { + let (text, vector) = search_arm_candidate_counts(result); + self.text.candidate_count = text; + self.vector.candidate_count = vector; + } +} + #[derive(Clone, Debug, PartialEq, Eq)] struct BackendErrorDiagnostic { message: String, @@ -96,6 +150,7 @@ struct BackendErrorDiagnostic { #[derive(Debug, Default)] struct SearchDegradation { status: Option, + arm_participation: Option, missing_backends: Vec, backend_errors: BTreeMap, backend_errors_omitted: usize, @@ -106,16 +161,72 @@ impl SearchDegradation { /// registry dispatch, or (in principle) a coordinator fan-out where /// every selected backend succeeded — `from_result` is used for the /// latter instead, since it also has to compute `missing_backends`. - fn complete() -> Self { + fn complete(result: &Value, vector_selected: bool) -> Self { + let (_, vector_candidates) = search_arm_candidate_counts(result); + let mut arm_participation = + SearchArmParticipation::complete(vector_selected || vector_candidates > 0); + arm_participation.observe_result(result); Self { status: Some(SearchStatus::Complete), + arm_participation: Some(arm_participation), missing_backends: Vec::new(), backend_errors: BTreeMap::new(), backend_errors_omitted: 0, } } - fn from_result(result: &CoordSearchResult) -> Self { + fn from_result(result: &CoordSearchResult, final_result: &Value) -> Self { + let vector_selected = result + .per_backend + .iter() + .any(|backend| backend.vector_selected || backend.vector_error.is_some()) + || result.entity_hits.iter().any(|hit| { + matches!( + hit.source, + khive_runtime::SearchSource::Vector | khive_runtime::SearchSource::Both + ) + }) + || result.note_hits.iter().any(|hit| { + matches!( + hit.source, + khive_runtime::SearchSource::Vector | khive_runtime::SearchSource::Both + ) + }); + // A whole-backend `error` means that backend's dispatch task failed + // before either arm could be attributed (auth, timeout, join failure, + // or a failed text leg — the text leg fails loud inside + // `hybrid_search_outcome`, so a text-arm failure always surfaces + // here). `vector_error` is populated only when the text leg + // completed and the vector leg alone failed, so it never doubles as + // a text-arm signal. + let text_failed = result + .per_backend + .iter() + .any(|backend| backend.error.is_some()); + let vector_failed = result.per_backend.iter().any(|backend| { + backend.vector_error.is_some() || (backend.vector_selected && backend.error.is_some()) + }); + let mut arm_participation = SearchArmParticipation { + text: SearchArmEvidence { + status: if text_failed { + SearchArmStatus::Error + } else { + SearchArmStatus::Ran + }, + candidate_count: 0, + }, + vector: SearchArmEvidence { + status: if !vector_selected { + SearchArmStatus::Skipped + } else if vector_failed { + SearchArmStatus::Error + } else { + SearchArmStatus::Ran + }, + candidate_count: 0, + }, + }; + arm_participation.observe_result(final_result); let failed_backend_count = result .per_backend .iter() @@ -149,6 +260,7 @@ impl SearchDegradation { candidate.insert(backend, diagnostic); let candidate_degradation = Self { status: Some(SearchStatus::Partial), + arm_participation: Some(arm_participation), missing_backends: candidate.keys().cloned().collect(), backend_errors_omitted: failed_backend_count.saturating_sub(candidate.len()), backend_errors: candidate.clone(), @@ -191,6 +303,7 @@ impl SearchDegradation { } Self { status: Some(status), + arm_participation: Some(arm_participation), missing_backends, backend_errors, backend_errors_omitted, @@ -202,6 +315,37 @@ impl SearchDegradation { } } +fn search_arm_candidate_counts(result: &Value) -> (usize, usize) { + result + .as_array() + .into_iter() + .flatten() + .fold((0, 0), |(text, vector), hit| { + let source = hit.get("source").and_then(Value::as_str); + match source { + Some(s) if s == khive_runtime::SearchSource::Text.as_str() => (text + 1, vector), + Some(s) if s == khive_runtime::SearchSource::Vector.as_str() => (text, vector + 1), + Some(s) if s == khive_runtime::SearchSource::Both.as_str() => { + (text + 1, vector + 1) + } + _ => (text, vector), + } + }) +} + +fn search_arm_participation_value(participation: SearchArmParticipation) -> Value { + json!({ + "text": { + "status": participation.text.status.as_str(), + "candidate_count": participation.text.candidate_count, + }, + "vector": { + "status": participation.vector.status.as_str(), + "candidate_count": participation.vector.candidate_count, + }, + }) +} + fn bounded_backend_error_message(message: &str) -> String { let bounded_input: String = message .chars() @@ -281,6 +425,9 @@ fn search_diagnostic_value(degradation: &SearchDegradation) -> Value { "missing_backends": degradation.missing_backends, "backend_errors": backend_errors_value(°radation.backend_errors), }); + if let Some(participation) = degradation.arm_participation { + value["arm_participation"] = search_arm_participation_value(participation); + } if degradation.backend_errors_omitted > 0 { value["backend_errors_truncated"] = Value::Bool(true); value["backend_errors_omitted"] = json!(degradation.backend_errors_omitted); @@ -313,11 +460,16 @@ impl OpSuccess { /// (excluding `help=true`, which returns a schema rather than a result /// array) carries `status="complete"` (ADR-130 §1); every other verb keeps /// the untagged `OpSuccess::complete` — no `status` field on its envelope. -fn op_success_from_registry_result(tool: &str, is_help: bool, result: Value) -> OpSuccess { +fn op_success_from_registry_result( + tool: &str, + is_help: bool, + result: Value, + vector_selected: bool, +) -> OpSuccess { if tool == "search" && !is_help { OpSuccess { + degradation: SearchDegradation::complete(&result, vector_selected), result, - degradation: SearchDegradation::complete(), } } else { OpSuccess::complete(result) @@ -1803,7 +1955,12 @@ impl KhiveMcpServer { result, self.schedule_ticker_last_tick_micros.as_ref(), ); - let success = op_success_from_registry_result(&tool, is_help, result); + let vector_selected = self + .runtime + .as_ref() + .is_some_and(|runtime| runtime.vector_arm_selected()); + let success = + op_success_from_registry_result(&tool, is_help, result, vector_selected); chain_ok_envelope_or_depth_error(tool, success) } Err(error) => Err(DispatchFailure::from_runtime(&tool, error)), @@ -1912,6 +2069,10 @@ impl KhiveMcpServer { let coordinator: Option> = self.coordinator.clone(); let schedule_ticker_last_tick_micros = self.schedule_ticker_last_tick_micros.clone(); + let vector_selected = self + .runtime + .as_ref() + .is_some_and(|runtime| runtime.vector_arm_selected()); // ADR-096 Fork 1: a per-request identity overrides the default // namespace for both the coordinator intercept and the registry // dispatch below, so the two can't drift out of sync per op. @@ -1933,6 +2094,7 @@ impl KhiveMcpServer { let schedule_ticker_last_tick_micros = schedule_ticker_last_tick_micros.clone(); let op_identity = identity_owned.clone(); + let op_vector_selected = vector_selected; let op_mode = mode_for_op(i); let task_tool = op.tool.clone(); BatchTask { @@ -2044,7 +2206,12 @@ impl KhiveMcpServer { schedule_ticker_last_tick_micros.as_ref(), ); let success = - op_success_from_registry_result(&tool, is_help, result); + op_success_from_registry_result( + &tool, + is_help, + result, + op_vector_selected, + ); present_ok_envelope_or_depth_error( tool, success, @@ -2271,8 +2438,6 @@ async fn dispatch_via_coordinator_inner( .fan_out_search(&request, &namespace, &extra_visible) .await; khive_storage::ensure_request_read_active("search")?; - let degradation = SearchDegradation::from_result(&coord_result); - // Preserve the coordinator search response's compatibility // fields, and add the KG single-backend handler's canonical // row fields for shape parity (MIN-1): `kind` (duplicates @@ -2332,6 +2497,8 @@ async fn dispatch_via_coordinator_inner( .collect(); serde_json::to_value(items).unwrap_or_else(|_| json!([])) }; + let degradation = + SearchDegradation::from_result(&coord_result, &result_val); Ok(InterceptedDispatchResult::new(result_val, degradation)) }, @@ -2505,12 +2672,14 @@ fn ok_envelope(tool: String, success: OpSuccess) -> Value { } = success; let SearchDegradation { status, + arm_participation, missing_backends, backend_errors, backend_errors_omitted, } = degradation; let is_partial = status == Some(SearchStatus::Partial); let extra_fields = usize::from(status.is_some()) + + usize::from(arm_participation.is_some()) + if is_partial { 3 + usize::from(backend_errors_omitted > 0) * 2 } else { @@ -2528,6 +2697,12 @@ fn ok_envelope(tool: String, success: OpSuccess) -> Value { Value::String(status.as_str().to_string()), ); } + if let Some(participation) = arm_participation { + map.insert( + "arm_participation".to_string(), + search_arm_participation_value(participation), + ); + } // Legacy `partial`/`missing_backends` alias, compatibility-release only // (ADR-130 §Compatibility) — omitted for `status="complete"`. if is_partial { @@ -4039,12 +4214,13 @@ fn frame_budget_omission(entry: &Value, registry: &VerbRegistry) -> Value { json!(khive_runtime::daemon::MAX_FRAME_BYTES), ), ]); - // ADR-130 defines `status`/`partial`/`missing_backends`/`backend_errors*` - // only on a successful search entry. Once `ok` flips to false here - // they no longer belong at the top level; fold any that were present - // into `error.search` instead of dropping the diagnostic outright. + // ADR-130 defines `status`/`arm_participation`/`partial`/`missing_backends`/ + // `backend_errors*` only on a successful search entry. Once `ok` flips to + // false here they no longer belong at the top level; fold any that were + // present into `error.search` instead of dropping the diagnostic outright. let search_fields: serde_json::Map = [ "status", + "arm_participation", "partial", "missing_backends", "backend_errors", @@ -5782,10 +5958,12 @@ mod tests { .expect("valid backend id"), entity_hits: Vec::new(), note_hits: Vec::new(), + vector_selected: true, error: Some(format!( "backend failure {index}: {}", "\0\"\\".repeat(MAX_BACKEND_ERROR_MESSAGE_CHARS) )), + vector_error: None, }) .collect(); if reverse { @@ -5804,8 +5982,8 @@ mod tests { } } - let forward = SearchDegradation::from_result(°raded_result(false)); - let reversed = SearchDegradation::from_result(°raded_result(true)); + let forward = SearchDegradation::from_result(°raded_result(false), &json!([])); + let reversed = SearchDegradation::from_result(°raded_result(true), &json!([])); assert!(!forward.backend_errors.is_empty()); assert!(forward.backend_errors.len() <= MAX_BACKEND_ERROR_ENTRIES); @@ -5850,6 +6028,44 @@ mod tests { ); } + /// A populated `vector_error` is itself proof the vector arm was + /// selected and failed. `vector_selected` on the backend can be a + /// registry miss (stale/absent metadata) — it must never hide a + /// recorded vector-arm failure. + #[test] + #[serial_test::serial(config_ledger)] + fn vector_error_reports_arm_failure_even_when_vector_selected_is_false() { + let result = CoordSearchResult { + entity_hits: Vec::new(), + note_hits: Vec::new(), + per_backend: vec![crate::coordinator::BackendSearchResult { + backend_id: khive_runtime::BackendId::parse("main").expect("valid backend id"), + entity_hits: Vec::new(), + note_hits: Vec::new(), + vector_selected: false, + error: None, + vector_error: Some("injected vector-arm failure".to_string()), + }], + partial: false, + entity_kinds: std::collections::HashMap::new(), + note_kinds: std::collections::HashMap::new(), + entity_created_at: std::collections::HashMap::new(), + note_created_at: std::collections::HashMap::new(), + note_names: std::collections::HashMap::new(), + }; + + let degradation = SearchDegradation::from_result(&result, &json!([])); + let arm_participation = degradation + .arm_participation + .expect("arm participation must be computed"); + assert_eq!( + arm_participation.vector.status, + SearchArmStatus::Error, + "a recorded vector_error must report the vector arm as failed \ + regardless of the backend's vector_selected flag" + ); + } + #[test] #[serial_test::serial(config_ledger)] fn backend_id_credentials_are_absent_from_wire_and_warning() { @@ -5862,7 +6078,9 @@ mod tests { .expect("valid backend id"), entity_hits: Vec::new(), note_hits: Vec::new(), + vector_selected: true, error: Some("storage unavailable".to_string()), + vector_error: None, }], partial: true, entity_kinds: std::collections::HashMap::new(), @@ -5878,7 +6096,7 @@ mod tests { .without_time() .finish(); let degradation = tracing::subscriber::with_default(subscriber, || { - SearchDegradation::from_result(&result) + SearchDegradation::from_result(&result, &json!([])) }); let wire = search_diagnostic_value(°radation).to_string(); let logs = captured.contents(); @@ -5906,15 +6124,27 @@ mod tests { "tool": "search", "result": "oversized", "status": "complete", + "arm_participation": { + "text": {"status": "ran", "candidate_count": 0}, + "vector": {"status": "skipped", "candidate_count": 0} + }, }), ®istry, ); assert_eq!(omitted["ok"], json!(false)); assert!(omitted.get("status").is_none()); + assert!(omitted.get("arm_participation").is_none()); assert!(omitted.get("partial").is_none()); assert!(omitted.get("result").is_none()); assert_eq!(omitted["error"]["search"]["status"], json!("complete")); + assert_eq!( + omitted["error"]["search"]["arm_participation"], + json!({ + "text": {"status": "ran", "candidate_count": 0}, + "vector": {"status": "skipped", "candidate_count": 0} + }) + ); assert_eq!( omitted["error"]["code"], json!("response_frame_budget_exceeded") @@ -5932,6 +6162,10 @@ mod tests { "kind": "search_incomplete", "message": "no-match was not established because selected backends failed", "retryable": false, + "arm_participation": { + "text": {"status": "error", "candidate_count": 0}, + "vector": {"status": "error", "candidate_count": 0} + }, "missing_backends": ["archive"], "backend_errors": { "archive": { @@ -7243,6 +7477,16 @@ mod tests { result: json!([{"id": "11111111-1111-1111-1111-111111111111"}]), degradation: SearchDegradation { status: Some(SearchStatus::Partial), + arm_participation: Some(SearchArmParticipation { + text: SearchArmEvidence { + status: SearchArmStatus::Error, + candidate_count: 1, + }, + vector: SearchArmEvidence { + status: SearchArmStatus::Error, + candidate_count: 0, + }, + }), missing_backends: vec!["archive".to_string()], backend_errors: BTreeMap::from([( "archive".to_string(), @@ -8049,7 +8293,7 @@ mod tests { let resp = server .dispatch_request_local(RequestParams { - ops: r#"search(kind="entity", query="nothing here")"#.to_string(), + ops: r#"search(kind="entity", query="a deliberately long keyword dense query whose terms cannot all match any entity in this empty corpus")"#.to_string(), presentation: None, presentation_per_op: None, save_to: None, @@ -8064,6 +8308,14 @@ mod tests { assert_eq!(search["ok"], json!(true), "unexpected response: {search}"); assert_eq!(search["status"], json!("complete")); assert_eq!(search["result"], json!([])); + assert_eq!( + search["arm_participation"], + json!({ + "text": {"status": "ran", "candidate_count": 0}, + "vector": {"status": "skipped", "candidate_count": 0} + }), + "a dense zero-hit query must prove that text ran without a match" + ); assert!(search.get("partial").is_none()); // Chain (`|`), not a parallel batch: `search` must observe the @@ -8092,6 +8344,14 @@ mod tests { ); assert_eq!(search["ok"], json!(true), "unexpected response: {search}"); assert_eq!(search["status"], json!("complete")); + assert_eq!( + search["arm_participation"], + json!({ + "text": {"status": "ran", "candidate_count": 1}, + "vector": {"status": "skipped", "candidate_count": 0} + }), + "an exact-name presence check must expose its text-arm evidence" + ); assert!( search["result"] .as_array() diff --git a/crates/khive-runtime/src/lib.rs b/crates/khive-runtime/src/lib.rs index 68cfab040..39d500e2a 100644 --- a/crates/khive-runtime/src/lib.rs +++ b/crates/khive-runtime/src/lib.rs @@ -118,7 +118,7 @@ pub use operations::{ pub use operations::{ base_entity_endpoint_rules, base_entity_rule_allows, endpoint_matches, hex_prefix_to_uuid_pattern, merge_entry_metadata, uuid_prefix_bounds, EdgeEndpointKind, - EntityCreateSpec, LinkSpec, NoteSearchHit, QueryResult, Resolved, + EntityCreateSpec, LinkSpec, NoteSearchHit, NoteSearchOutcome, QueryResult, Resolved, }; pub use pack::{ resolve_explicit_namespace, ChannelIngestCapability, DispatchHook, HandlerDef, @@ -139,7 +139,7 @@ pub use reference_resolution::{resolve_reference, ReferenceCandidate, ReferenceR pub use reference_ring::{ReferenceRing, RingEntry}; pub use registry::{ObjectiveRegistry, RegisteredObjective}; pub use resource::{cpu_delta_us, process_resource_usage, ProcessResourceUsage}; -pub use retrieval::{SearchHit, SearchSource}; +pub use retrieval::{HybridSearchOutcome, SearchHit, SearchSource}; pub use runtime::{ assert_captured_db_anchor_consistent, assert_db_anchor_consistent, expand_tilde, parse_pack_list, resolve_db_anchor, resolve_project_actor_id, runtime_config_from_khive_config, diff --git a/crates/khive-runtime/src/operations.rs b/crates/khive-runtime/src/operations.rs index 6cb12b82f..e64af50d2 100644 --- a/crates/khive-runtime/src/operations.rs +++ b/crates/khive-runtime/src/operations.rs @@ -290,6 +290,15 @@ pub struct NoteSearchHit { pub snippet: Option, } +/// Result of [`KhiveRuntime::search_notes_outcome`]: the fused hits — text +/// hits alone when the vector arm failed — plus the vector arm's error, if +/// any. Mirrors [`crate::HybridSearchOutcome`] for the note substrate. +#[derive(Clone, Debug)] +pub struct NoteSearchOutcome { + pub hits: Vec, + pub vector_error: Option, +} + /// Re-insert hyphens at canonical UUID positions (8-4-4-4-12) into a /// hyphen-free hex prefix, so a `LIKE '%'` scan against the /// hyphenated `id` column matches correctly. Prefixes that already @@ -4111,6 +4120,68 @@ impl KhiveRuntime { tags_any: &[String], properties_filter: Option<&serde_json::Value>, ) -> RuntimeResult> { + let (hits, _vector_error) = self + .search_notes_inner( + token, + query_text, + query_vector, + limit, + note_kind, + include_superseded, + tags_any, + properties_filter, + false, + ) + .await?; + Ok(hits) + } + + /// Coordinator fan-out variant of [`Self::search_notes`]: the text arm + /// still fails loud, but a vector-arm failure after a successful text leg + /// is captured instead of discarding the text hits — mirrors + /// [`Self::hybrid_search_outcome`]'s contract for the entity substrate. + /// Reserved for `SubstrateCoordinator::fan_out_search_with_visibility`; + /// every other caller keeps the fail-loud [`Self::search_notes`] contract. + #[allow(clippy::too_many_arguments)] + pub async fn search_notes_outcome( + &self, + token: &NamespaceToken, + query_text: &str, + limit: u32, + note_kind: Option<&str>, + include_superseded: bool, + tags_any: &[String], + properties_filter: Option<&serde_json::Value>, + ) -> RuntimeResult { + let (hits, vector_error) = self + .search_notes_inner( + token, + query_text, + None, + limit, + note_kind, + include_superseded, + tags_any, + properties_filter, + true, + ) + .await?; + Ok(NoteSearchOutcome { hits, vector_error }) + } + + #[allow(clippy::too_many_arguments)] + async fn search_notes_inner( + &self, + token: &NamespaceToken, + query_text: &str, + query_vector: Option>, + limit: u32, + note_kind: Option<&str>, + include_superseded: bool, + tags_any: &[String], + properties_filter: Option<&serde_json::Value>, + tolerate_vector_error: bool, + ) -> RuntimeResult<(Vec, Option)> { const RRF_K: usize = 60; let candidates = limit.saturating_mul(4).max(limit); let visible_ns: Vec = token @@ -4176,15 +4247,25 @@ impl KhiveRuntime { )?; // Vector search filtered to notes. + let mut vector_error: Option = None; let vector_hits = if query_vector.is_some() || self.config().embedding_model.is_some() { - self.vector_search( - token, - query_vector, - Some(query_text), - candidates, - Some(SubstrateKind::Note), - ) - .await? + match self + .vector_search( + token, + query_vector, + Some(query_text), + candidates, + Some(SubstrateKind::Note), + ) + .await + { + Ok(hits) => hits, + Err(e) if tolerate_vector_error => { + vector_error = Some(e.to_string()); + Vec::new() + } + Err(e) => return Err(e), + } } else { vec![] }; @@ -4199,7 +4280,7 @@ impl KhiveRuntime { let candidate_ids: Vec = fused.iter().map(|hit| hit.entity_id).collect(); if candidate_ids.is_empty() { - return Ok(vec![]); + return Ok((vec![], vector_error)); } // Fetch each candidate note individually to get salience and apply @@ -4294,7 +4375,7 @@ impl KhiveRuntime { hits.sort_by(|a, b| b.score.cmp(&a.score).then(a.note_id.cmp(&b.note_id))); hits.truncate(limit as usize); - Ok(hits) + Ok((hits, vector_error)) } /// Resolve a short UUID prefix (8+ hex chars) to a full UUID. diff --git a/crates/khive-runtime/src/retrieval.rs b/crates/khive-runtime/src/retrieval.rs index 392b3e283..8ff28b06d 100644 --- a/crates/khive-runtime/src/retrieval.rs +++ b/crates/khive-runtime/src/retrieval.rs @@ -43,6 +43,15 @@ pub struct SearchHit { pub snippet: Option, } +/// Result of [`KhiveRuntime::hybrid_search_outcome`]: the fused hits — text +/// hits alone when the vector arm failed — plus the vector arm's error, if +/// any. +#[derive(Clone, Debug)] +pub struct HybridSearchOutcome { + pub hits: Vec, + pub vector_error: Option, +} + /// Which retrieval path(s) contributed to a hit. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum SearchSource { @@ -644,18 +653,21 @@ impl KhiveRuntime { tags_any: &[String], properties_filter: Option<&serde_json::Value>, ) -> RuntimeResult> { - self.hybrid_search_inner( - token, - query_text, - query_vector, - limit, - entity_kind, - entity_type, - tags_any, - properties_filter, - None, - ) - .await + let (hits, _vector_error) = self + .hybrid_search_inner( + token, + query_text, + query_vector, + limit, + entity_kind, + entity_type, + tags_any, + properties_filter, + None, + false, + ) + .await?; + Ok(hits) } /// `vector_similarity_floor` is a cosine-similarity value in `[-1.0, @@ -675,18 +687,58 @@ impl KhiveRuntime { properties_filter: Option<&serde_json::Value>, vector_similarity_floor: f64, ) -> RuntimeResult> { - self.hybrid_search_inner( - token, - query_text, - query_vector, - limit, - entity_kind, - entity_type, - tags_any, - properties_filter, - Some(vector_similarity_floor), - ) - .await + let (hits, _vector_error) = self + .hybrid_search_inner( + token, + query_text, + query_vector, + limit, + entity_kind, + entity_type, + tags_any, + properties_filter, + Some(vector_similarity_floor), + false, + ) + .await?; + Ok(hits) + } + + /// Coordinator fan-out variant of [`Self::hybrid_search`]: the text arm + /// still fails loud (propagated through `hybrid_search_inner`'s + /// `tolerate_vector_error=false` semantics for that leg), but a + /// vector-arm failure after a successful text leg is captured instead of + /// discarding the text hits. Reserved for + /// `SubstrateCoordinator::fan_out_search_with_visibility` — every other + /// caller keeps the fail-loud [`Self::hybrid_search`] contract, so a + /// single backend's vector-store outage does not misreport that + /// backend's text arm as failed too. + #[allow(clippy::too_many_arguments)] + pub async fn hybrid_search_outcome( + &self, + token: &NamespaceToken, + query_text: &str, + limit: u32, + entity_kind: Option<&str>, + entity_type: Option<&str>, + tags_any: &[String], + properties_filter: Option<&serde_json::Value>, + ) -> RuntimeResult { + let (hits, vector_error) = self + .hybrid_search_inner( + token, + query_text, + None, + limit, + entity_kind, + entity_type, + tags_any, + properties_filter, + None, + true, + ) + .await?; + Ok(HybridSearchOutcome { hits, vector_error }) } #[allow(clippy::too_many_arguments)] @@ -701,7 +753,8 @@ impl KhiveRuntime { tags_any: &[String], properties_filter: Option<&serde_json::Value>, vector_similarity_floor: Option, - ) -> RuntimeResult> { + tolerate_vector_error: bool, + ) -> RuntimeResult<(Vec, Option)> { let candidates = limit.saturating_mul(CANDIDATE_MULTIPLIER).max(limit); let visible_ns: Vec = token @@ -735,15 +788,25 @@ impl KhiveRuntime { query_text, )?; + let mut vector_error: Option = None; let mut vector_hits = if query_vector.is_some() || self.config().embedding_model.is_some() { - self.vector_search( - token, - query_vector, - Some(query_text), - candidates, - Some(SubstrateKind::Entity), - ) - .await? + match self + .vector_search( + token, + query_vector, + Some(query_text), + candidates, + Some(SubstrateKind::Entity), + ) + .await + { + Ok(hits) => hits, + Err(e) if tolerate_vector_error => { + vector_error = Some(e.to_string()); + Vec::new() + } + Err(e) => return Err(e), + } } else { Vec::new() }; @@ -813,7 +876,7 @@ impl KhiveRuntime { } fused.truncate(limit as usize); - Ok(fused) + Ok((fused, vector_error)) } /// Exact KNN over the full namespace's vector store. @@ -1388,10 +1451,138 @@ fn rrf_fuse( #[cfg(test)] mod tests { use super::*; + use std::sync::Arc; + use crate::runtime::{KhiveRuntime, NamespaceToken, RuntimeConfig}; use khive_storage::types::{TextSearchHit, VectorSearchHit}; use khive_types::namespace::Namespace; - use lattice_embed::EmbeddingModel; + use lattice_embed::{EmbedError, EmbeddingModel}; + + /// An `EmbeddingService` that always fails — used to drive a real + /// vector-arm failure (as opposed to an `Unconfigured` short-circuit) + /// through `embed_query_for_token` without loading actual model weights. + struct FailingEmbeddingService; + + #[async_trait::async_trait] + impl EmbeddingService for FailingEmbeddingService { + async fn embed( + &self, + _texts: &[String], + _model: EmbeddingModel, + ) -> Result>, EmbedError> { + Err(EmbedError::ModelInitialization( + "injected vector-arm failure".to_string(), + )) + } + + fn supports_model(&self, _model: EmbeddingModel) -> bool { + true + } + + fn name(&self) -> &'static str { + "hybrid-search-test-failing-embedding" + } + } + + struct FailingEmbedderProvider { + name: String, + dimensions: usize, + } + + #[async_trait::async_trait] + impl EmbedderProvider for FailingEmbedderProvider { + fn name(&self) -> &str { + &self.name + } + + fn dimensions(&self) -> usize { + self.dimensions + } + + async fn build(&self) -> RuntimeResult> { + Ok(Arc::new(FailingEmbeddingService)) + } + } + + /// Swap the runtime's registered embedder for the always-failing one, + /// re-keyed under the same model name so `hybrid_search`'s vector leg + /// (which resolves the embedder by the runtime's configured model name) + /// picks it up. `EmbedderRegistry::register` overwrites by name and + /// resets the provider's build cache, so this takes effect on the next + /// embed call without needing a fresh runtime. + fn break_vector_arm(runtime: &KhiveRuntime) { + let model = EmbeddingModel::AllMiniLmL6V2; + runtime.register_embedder(FailingEmbedderProvider { + name: model.to_string(), + dimensions: model.dimensions(), + }); + } + + /// An `EmbeddingService` that always succeeds with a fixed vector — used + /// to exercise a genuinely healthy vector leg without loading real model + /// weights. + struct ConstantEmbeddingService { + dimensions: usize, + } + + #[async_trait::async_trait] + impl EmbeddingService for ConstantEmbeddingService { + async fn embed( + &self, + texts: &[String], + _model: EmbeddingModel, + ) -> Result>, EmbedError> { + Ok(texts.iter().map(|_| vec![1.0; self.dimensions]).collect()) + } + + fn supports_model(&self, _model: EmbeddingModel) -> bool { + true + } + + fn name(&self) -> &'static str { + "hybrid-search-test-constant-embedding" + } + } + + struct ConstantEmbedderProvider { + name: String, + dimensions: usize, + } + + #[async_trait::async_trait] + impl EmbedderProvider for ConstantEmbedderProvider { + fn name(&self) -> &str { + &self.name + } + + fn dimensions(&self) -> usize { + self.dimensions + } + + async fn build(&self) -> RuntimeResult> { + Ok(Arc::new(ConstantEmbeddingService { + dimensions: self.dimensions, + })) + } + } + + /// A runtime configured with a healthy (constant, non-failing) embedder — + /// the vector leg genuinely runs and succeeds. + fn runtime_with_constant_embeddings() -> KhiveRuntime { + let model = EmbeddingModel::AllMiniLmL6V2; + let runtime = KhiveRuntime::new(RuntimeConfig { + db_path: None, + embedding_model: Some(model), + packs: vec!["kg".to_string()], + ..RuntimeConfig::no_embeddings() + }) + .expect("in-memory runtime"); + runtime.register_embedder(ConstantEmbedderProvider { + name: model.to_string(), + dimensions: model.dimensions(), + }); + runtime + } #[test] fn bounded_embedding_input_reserves_prefix_and_preserves_utf8() { @@ -1673,6 +1864,116 @@ mod tests { assert_eq!(embeddings[0].len(), model.dimensions()); } + // ---- hybrid_search_outcome: vector-arm failure must not discard text hits ---- + + /// Baseline (red without the fix): the fail-loud `hybrid_search` must + /// still propagate a vector-arm failure as a whole-call error, even + /// though the text leg found a match — proving `break_vector_arm` + /// genuinely exercises the vector leg and that `hybrid_search_outcome`'s + /// tolerance below is a real behavioral difference, not a no-op. + #[tokio::test] + async fn hybrid_search_still_fails_loud_on_vector_arm_error() { + let rt = runtime_with_constant_embeddings(); + let tok = NamespaceToken::local(); + rt.create_entity( + &tok, + "concept", + None, + "FlashAttention", + Some("IO-aware exact attention using tiling"), + None, + vec![], + ) + .await + .unwrap(); + break_vector_arm(&rt); + + let result = rt + .hybrid_search(&tok, "FlashAttention", None, 10, None, None, &[], None) + .await; + + assert!( + result.is_err(), + "the fail-loud entry point must still propagate a vector-arm failure, got {result:?}" + ); + } + + /// Green: the coordinator-only outcome variant preserves the text hit and + /// reports the vector failure separately instead of discarding the whole + /// call. + #[tokio::test] + async fn hybrid_search_outcome_preserves_text_hits_on_vector_arm_error() { + let rt = runtime_with_constant_embeddings(); + let tok = NamespaceToken::local(); + rt.create_entity( + &tok, + "concept", + None, + "FlashAttention", + Some("IO-aware exact attention using tiling"), + None, + vec![], + ) + .await + .unwrap(); + break_vector_arm(&rt); + + let outcome = rt + .hybrid_search_outcome(&tok, "FlashAttention", 10, None, None, &[], None) + .await + .expect("text leg must still succeed"); + + assert!( + !outcome.hits.is_empty(), + "text arm's hit must survive a vector-arm failure" + ); + assert!( + outcome.hits[0] + .title + .as_deref() + .unwrap_or_default() + .contains("FlashAttention"), + "surviving hit must be the text match" + ); + let vector_error = outcome + .vector_error + .expect("vector arm failure must be reported"); + assert!( + vector_error.contains("injected vector-arm failure"), + "vector_error must carry the underlying cause, got {vector_error:?}" + ); + } + + /// A healthy vector arm still leaves `vector_error` `None` — the outcome + /// variant does not manufacture a failure where there is none. + #[tokio::test] + async fn hybrid_search_outcome_has_no_vector_error_when_vector_arm_healthy() { + let rt = runtime_with_constant_embeddings(); + let tok = NamespaceToken::local(); + rt.create_entity( + &tok, + "concept", + None, + "FlashAttention", + Some("IO-aware exact attention using tiling"), + None, + vec![], + ) + .await + .unwrap(); + + let outcome = rt + .hybrid_search_outcome(&tok, "FlashAttention", 10, None, None, &[], None) + .await + .expect("hybrid search must succeed"); + + assert!(!outcome.hits.is_empty(), "should find the entity"); + assert!( + outcome.vector_error.is_none(), + "a healthy vector arm must not report an error" + ); + } + // ---- hybrid_search enrichment ---- #[tokio::test] diff --git a/crates/khive-runtime/src/runtime.rs b/crates/khive-runtime/src/runtime.rs index fc8a1ed02..aab66ff18 100644 --- a/crates/khive-runtime/src/runtime.rs +++ b/crates/khive-runtime/src/runtime.rs @@ -529,6 +529,14 @@ impl KhiveRuntime { &self.config } + /// Whether this runtime selects the vector arm for a hybrid search — + /// true exactly when a default embedding model is configured. Single + /// source of truth for the policy every fan-out and single-backend + /// dispatch path uses to report `arm_participation`/`vector_selected`. + pub fn vector_arm_selected(&self) -> bool { + self.config.embedding_model.is_some() + } + /// Return the immutable ADR-118 fresh-tail serving policy captured when /// this runtime was constructed. pub fn ann_fresh_tail_enabled(&self) -> bool { diff --git a/crates/kkernel/src/coordinator/dispatch.rs b/crates/kkernel/src/coordinator/dispatch.rs index 83d143caa..2c54088a9 100644 --- a/crates/kkernel/src/coordinator/dispatch.rs +++ b/crates/kkernel/src/coordinator/dispatch.rs @@ -77,13 +77,18 @@ pub(super) fn bounded_backend_cause_for_log(message: &str) -> String { /// Result of a single backend's entity-search contribution to a fan-out. /// /// `hits` may be empty when the backend returned no results. -/// `error` carries the backend-specific failure message on error. +/// `error` carries a whole-backend failure message (text arm, or a fatal +/// error before either arm ran); a backend that reported one is treated as +/// having contributed no hits at all. `vector_error` instead carries a +/// vector-arm-only failure: the text arm still ran and `hits` still carries +/// its results, so this backend is NOT `error`-failed. #[derive(Debug)] pub struct BackendSearchResult { pub backend_id: BackendId, pub hits: Vec, pub note_hits: Vec, pub error: Option, + pub vector_error: Option, } /// A located edge endpoint: which backend owns it, and its substrate kind. @@ -614,6 +619,7 @@ impl SubstrateCoordinator { hits: vec![], note_hits: vec![], error: Some(e.to_string()), + vector_error: None, }; return (vec![], vec![], vec![backend_result]); } @@ -638,10 +644,9 @@ impl SubstrateCoordinator { unreachable!("a pending future never resolves"); } runtime - .search_notes( + .search_notes_outcome( &token, request.query(), - None, search_limit, request.kind_filter(), include_superseded, @@ -654,8 +659,9 @@ impl SubstrateCoordinator { khive_storage::scope_request_read_deadline_at(request_deadline, search_fut); tokio::pin!(search_fut); match tokio::time::timeout_at(request_deadline.async_at(), &mut search_fut).await { - Ok(Ok(note_hits)) => { - let filtered_note_hits: Vec = note_hits + Ok(Ok(outcome)) => { + let filtered_note_hits: Vec = outcome + .hits .iter() .filter(|hit| { request @@ -668,8 +674,9 @@ impl SubstrateCoordinator { let backend_result = BackendSearchResult { backend_id: backend_id.clone(), hits: vec![], - note_hits, + note_hits: outcome.hits, error: None, + vector_error: outcome.vector_error, }; return (vec![], filtered_note_hits, vec![backend_result]); } @@ -679,6 +686,7 @@ impl SubstrateCoordinator { hits: vec![], note_hits: vec![], error: Some(e.to_string()), + vector_error: None, }; return (vec![], vec![], vec![backend_result]); } @@ -698,6 +706,7 @@ impl SubstrateCoordinator { hits: vec![], note_hits: vec![], error: Some(format!("backend search timed out after {timeout_ms}ms")), + vector_error: None, }; return (vec![], vec![], vec![backend_result]); } @@ -709,10 +718,9 @@ impl SubstrateCoordinator { unreachable!("a pending future never resolves"); } runtime - .hybrid_search( + .hybrid_search_outcome( &token, request.query(), - None, search_limit, request.kind_filter(), request.entity_type(), @@ -725,8 +733,9 @@ impl SubstrateCoordinator { khive_storage::scope_request_read_deadline_at(request_deadline, search_fut); tokio::pin!(search_fut); match tokio::time::timeout_at(request_deadline.async_at(), &mut search_fut).await { - Ok(Ok(hits)) => { - let filtered_hits: Vec = hits + Ok(Ok(outcome)) => { + let filtered_hits: Vec = outcome + .hits .iter() .filter(|hit| { request @@ -738,9 +747,10 @@ impl SubstrateCoordinator { .collect(); let backend_result = BackendSearchResult { backend_id: backend_id.clone(), - hits, + hits: outcome.hits, note_hits: vec![], error: None, + vector_error: outcome.vector_error, }; return (filtered_hits, vec![], vec![backend_result]); } @@ -750,6 +760,7 @@ impl SubstrateCoordinator { hits: vec![], note_hits: vec![], error: Some(e.to_string()), + vector_error: None, }; return (vec![], vec![], vec![backend_result]); } @@ -769,6 +780,7 @@ impl SubstrateCoordinator { hits: vec![], note_hits: vec![], error: Some(format!("backend search timed out after {timeout_ms}ms")), + vector_error: None, }; return (vec![], vec![], vec![backend_result]); } @@ -855,14 +867,15 @@ impl SubstrateCoordinator { "injected failure".to_string(), )), None::>, + None, ); } if search_notes { if let Some(hits) = note_override { - return (backend_id, Ok(vec![]), Some(hits)); + return (backend_id, Ok(vec![]), Some(hits), None); } } else if let Some(hits) = entity_override { - return (backend_id, Ok(hits), None); + return (backend_id, Ok(hits), None, None); } let token = match runtime.authorize_with_visibility(ns, extra_visible_task) { Ok(t) => t, @@ -871,15 +884,14 @@ impl SubstrateCoordinator { error = %bounded_backend_cause_for_log(&e.to_string()), "fan_out_search: authorization denied for namespace" ); - return (backend_id, Err(e), None); + return (backend_id, Err(e), None, None); } }; if search_notes { let result = runtime - .search_notes( + .search_notes_outcome( &token, &q, - None, sl, kf.as_deref(), include_superseded, @@ -893,15 +905,19 @@ impl SubstrateCoordinator { // would remove candidates the RRF merge needs to fairly // rank a hit that only places #2+ on any single backend. // `rrf_merge_note_hits` applies `limit` once, after merge. - Ok(note_hits) => (backend_id, Ok(vec![]), Some(note_hits)), - Err(e) => (backend_id, Err(e), None), + Ok(outcome) => ( + backend_id, + Ok(vec![]), + Some(outcome.hits), + outcome.vector_error, + ), + Err(e) => (backend_id, Err(e), None, None), } } else { let result = runtime - .hybrid_search( + .hybrid_search_outcome( &token, &q, - None, sl, kf.as_deref(), et.as_deref(), @@ -912,8 +928,8 @@ impl SubstrateCoordinator { match result { // See the note-substrate arm above: no per-backend // truncation before RRF merge (MAJ-4). - Ok(hits) => (backend_id, Ok(hits), None), - Err(e) => (backend_id, Err(e), None), + Ok(outcome) => (backend_id, Ok(outcome.hits), None, outcome.vector_error), + Err(e) => (backend_id, Err(e), None, None), } } }; @@ -954,7 +970,7 @@ impl SubstrateCoordinator { } }; match joined { - Ok(Ok(((backend_id, Ok(hits), note_hits_opt), completed_at))) + Ok(Ok(((backend_id, Ok(hits), note_hits_opt, vector_error), completed_at))) if completed_at <= request_deadline.async_at() => { let note_hits = note_hits_opt.unwrap_or_default(); @@ -969,9 +985,10 @@ impl SubstrateCoordinator { hits, note_hits, error: None, + vector_error, }); } - Ok(Ok(((backend_id, Err(e), _), completed_at))) + Ok(Ok(((backend_id, Err(e), _, _), completed_at))) if completed_at <= request_deadline.async_at() => { per_backend.push(BackendSearchResult { @@ -979,6 +996,7 @@ impl SubstrateCoordinator { hits: vec![], note_hits: vec![], error: Some(e.to_string()), + vector_error: None, }); } Ok(Err(join_err)) => { @@ -995,6 +1013,7 @@ impl SubstrateCoordinator { hits: vec![], note_hits: vec![], error: Some(error.to_string()), + vector_error: None, }); } Ok(Ok((_late_result, _completed_at))) => { @@ -1008,6 +1027,7 @@ impl SubstrateCoordinator { hits: vec![], note_hits: vec![], error: Some(format!("backend search timed out after {timeout_ms}ms")), + vector_error: None, }); } Err(_elapsed) => { @@ -1021,6 +1041,7 @@ impl SubstrateCoordinator { hits: vec![], note_hits: vec![], error: Some(format!("backend search timed out after {timeout_ms}ms")), + vector_error: None, }); } } diff --git a/crates/kkernel/src/coordinator/service.rs b/crates/kkernel/src/coordinator/service.rs index a47530562..bd0997983 100644 --- a/crates/kkernel/src/coordinator/service.rs +++ b/crates/kkernel/src/coordinator/service.rs @@ -163,11 +163,20 @@ impl CoordinatorService for SubstrateCoordinatorService { let coord_per_backend: Vec = per_backend .into_iter() - .map(|r| CoordBackendResult { - backend_id: r.backend_id, - entity_hits: r.hits, - note_hits: r.note_hits, - error: r.error, + .map(|r| { + let vector_selected = self + .inner + .registry() + .get(&r.backend_id) + .is_some_and(|entry| entry.runtime.vector_arm_selected()); + CoordBackendResult { + backend_id: r.backend_id, + entity_hits: r.hits, + note_hits: r.note_hits, + vector_selected, + error: r.error, + vector_error: r.vector_error, + } }) .collect(); diff --git a/crates/kkernel/src/coordinator/tests.rs b/crates/kkernel/src/coordinator/tests.rs index 39c9de25a..d468e02b6 100644 --- a/crates/kkernel/src/coordinator/tests.rs +++ b/crates/kkernel/src/coordinator/tests.rs @@ -83,6 +83,133 @@ fn memory_runtime_denied_with(cause: String) -> Arc { ) } +/// An `EmbeddingService` that always succeeds with a fixed vector — used to +/// stand up a runtime whose vector arm can later be broken independently of +/// entity creation (which also embeds). +struct ConstantEmbeddingService { + dimensions: usize, +} + +#[async_trait::async_trait] +impl lattice_embed::EmbeddingService for ConstantEmbeddingService { + async fn embed( + &self, + texts: &[String], + _model: lattice_embed::EmbeddingModel, + ) -> Result>, lattice_embed::EmbedError> { + Ok(texts.iter().map(|_| vec![1.0; self.dimensions]).collect()) + } + + fn supports_model(&self, _model: lattice_embed::EmbeddingModel) -> bool { + true + } + + fn name(&self) -> &'static str { + "coordinator-test-constant-embedding" + } +} + +struct ConstantEmbedderProvider { + name: String, + dimensions: usize, +} + +#[async_trait::async_trait] +impl khive_runtime::EmbedderProvider for ConstantEmbedderProvider { + fn name(&self) -> &str { + &self.name + } + + fn dimensions(&self) -> usize { + self.dimensions + } + + async fn build( + &self, + ) -> khive_runtime::RuntimeResult> { + Ok(Arc::new(ConstantEmbeddingService { + dimensions: self.dimensions, + })) + } +} + +/// An `EmbeddingService` that always fails — drives a real vector-arm +/// failure (not an `Unconfigured` short-circuit) through the coordinator's +/// fan-out. +struct FailingEmbeddingService; + +#[async_trait::async_trait] +impl lattice_embed::EmbeddingService for FailingEmbeddingService { + async fn embed( + &self, + _texts: &[String], + _model: lattice_embed::EmbeddingModel, + ) -> Result>, lattice_embed::EmbedError> { + Err(lattice_embed::EmbedError::ModelInitialization( + "injected vector-arm failure".to_string(), + )) + } + + fn supports_model(&self, _model: lattice_embed::EmbeddingModel) -> bool { + true + } + + fn name(&self) -> &'static str { + "coordinator-test-failing-embedding" + } +} + +struct FailingEmbedderProvider { + name: String, + dimensions: usize, +} + +#[async_trait::async_trait] +impl khive_runtime::EmbedderProvider for FailingEmbedderProvider { + fn name(&self) -> &str { + &self.name + } + + fn dimensions(&self) -> usize { + self.dimensions + } + + async fn build( + &self, + ) -> khive_runtime::RuntimeResult> { + Ok(Arc::new(FailingEmbeddingService)) + } +} + +/// A runtime configured with a healthy (constant) embedder — entity creation +/// and search both work until [`break_vector_arm`] swaps the provider out. +fn memory_runtime_with_constant_embeddings() -> Arc { + let model = lattice_embed::EmbeddingModel::AllMiniLmL6V2; + let runtime = KhiveRuntime::new(khive_runtime::RuntimeConfig { + db_path: None, + embedding_model: Some(model), + packs: vec!["kg".to_string()], + ..khive_runtime::RuntimeConfig::no_embeddings() + }) + .expect("in-memory runtime"); + runtime.register_embedder(ConstantEmbedderProvider { + name: model.to_string(), + dimensions: model.dimensions(), + }); + Arc::new(runtime) +} + +/// Swap the runtime's registered embedder for the always-failing one, keyed +/// under the same model name so the vector leg picks it up on the next +/// embed call — `EmbedderRegistry::register` overwrites by name. +fn break_vector_arm(runtime: &KhiveRuntime) { + let model = lattice_embed::EmbeddingModel::AllMiniLmL6V2; + runtime.register_embedder(FailingEmbedderProvider { + name: model.to_string(), + dimensions: model.dimensions(), + }); +} + fn search_hit(entity_id: Uuid, source: SearchSource) -> SearchHit { SearchHit { entity_id, @@ -432,6 +559,245 @@ async fn fan_out_search_single_backend_returns_hits() { assert!(per_backend[0].error.is_none(), "no error"); } +/// A vector-arm failure after a successful text leg must not discard the +/// text hit or mark the backend as whole-backend-failed: `hits` still +/// carries the text match, `error` stays `None`, and `vector_error` alone +/// reports the vector-arm cause. Single-backend early-return path. +#[tokio::test] +#[serial_test::serial(config_ledger)] +async fn fan_out_search_single_backend_preserves_text_hits_on_vector_arm_error() { + let runtime = memory_runtime_with_constant_embeddings(); + let coord = SubstrateCoordinator::single(Arc::clone(&runtime)); + let ns = Namespace::local(); + + let token = runtime.authorize(ns.clone()).unwrap(); + runtime + .create_entity( + &token, + "concept", + None, + "FlashAttention", + Some("IO-aware exact attention"), + None, + vec![], + ) + .await + .expect("create entity"); + break_vector_arm(&runtime); + + let request = validated_kg_search(serde_json::json!({ + "kind": "entity", + "query": "FlashAttention", + "limit": 10, + })); + let (hits, _note_hits, per_backend) = coord.fan_out_search(&request, &ns).await; + + assert!( + !hits.is_empty(), + "text arm's hit must survive a vector-arm failure" + ); + assert_eq!(per_backend.len(), 1, "single backend report"); + assert!( + per_backend[0].error.is_none(), + "vector-arm-only failure must not read as a whole-backend error: {:?}", + per_backend[0].error + ); + let vector_error = per_backend[0] + .vector_error + .as_deref() + .expect("vector arm failure must be reported"); + assert!( + vector_error.contains("injected vector-arm failure"), + "vector_error must carry the underlying cause, got {vector_error:?}" + ); +} + +/// Positive control: proves the note vector leg genuinely runs and fails when +/// exercised through the fail-loud `search_notes` entry point, establishing +/// that `break_vector_arm` is a real behavioral trigger for the note +/// substrate too — the note-substrate twin of +/// `hybrid_search_still_fails_loud_on_vector_arm_error` for entities. +#[tokio::test] +#[serial_test::serial(config_ledger)] +async fn search_notes_still_fails_loud_on_vector_arm_error() { + let runtime = memory_runtime_with_constant_embeddings(); + let ns = Namespace::local(); + let token = runtime.authorize(ns).unwrap(); + runtime + .create_note( + &token, + "observation", + Some("FlashAttentionNote"), + "IO-aware exact attention observation", + None, + None, + vec![], + ) + .await + .expect("create note"); + break_vector_arm(&runtime); + + let result = runtime + .search_notes( + &token, + "FlashAttentionNote", + None, + 10, + None, + false, + &[], + None, + ) + .await; + + assert!( + result.is_err(), + "the fail-loud search_notes entry point must still propagate a vector-arm failure, got {result:?}" + ); +} + +/// Note-substrate twin of +/// `fan_out_search_single_backend_preserves_text_hits_on_vector_arm_error`: a +/// vector-arm failure after a successful note text leg must not discard the +/// note's text hit or mark the backend as whole-backend-failed. +#[tokio::test] +#[serial_test::serial(config_ledger)] +async fn fan_out_search_single_backend_preserves_note_text_hits_on_vector_arm_error() { + let runtime = memory_runtime_with_constant_embeddings(); + let coord = SubstrateCoordinator::single(Arc::clone(&runtime)); + let ns = Namespace::local(); + + let token = runtime.authorize(ns.clone()).unwrap(); + runtime + .create_note( + &token, + "observation", + Some("FlashAttentionNote"), + "IO-aware exact attention observation", + None, + None, + vec![], + ) + .await + .expect("create note"); + break_vector_arm(&runtime); + + let request = validated_kg_search(serde_json::json!({ + "kind": "note", + "query": "FlashAttentionNote", + "limit": 10, + })); + let (_hits, note_hits, per_backend) = coord.fan_out_search(&request, &ns).await; + + assert!( + !note_hits.is_empty(), + "text arm's note hit must survive a vector-arm failure" + ); + assert_eq!(per_backend.len(), 1, "single backend report"); + assert!( + per_backend[0].error.is_none(), + "vector-arm-only failure must not read as a whole-backend error: {:?}", + per_backend[0].error + ); + let vector_error = per_backend[0] + .vector_error + .as_deref() + .expect("vector arm failure must be reported"); + assert!( + vector_error.contains("injected vector-arm failure"), + "vector_error must carry the underlying cause, got {vector_error:?}" + ); +} + +/// Same guarantee as the single-backend test above, but for the spawned +/// multi-backend fan-out path: a healthy sibling backend must not be +/// affected by another backend's vector-arm-only failure. +#[tokio::test] +#[serial_test::serial(config_ledger)] +async fn fan_out_search_multi_backend_vector_arm_failure_isolated_to_its_backend() { + let mut registry = BackendRegistry::new(); + let rt_broken = memory_runtime_with_constant_embeddings(); + let rt_healthy = memory_runtime(); + registry.register(backend_id("broken"), Arc::clone(&rt_broken)); + registry.register(backend_id("healthy"), Arc::clone(&rt_healthy)); + let coord = SubstrateCoordinator::new(registry); + let ns = Namespace::local(); + + let tok_broken = rt_broken.authorize(ns.clone()).unwrap(); + rt_broken + .create_entity( + &tok_broken, + "concept", + None, + "LoRA", + Some("Low-rank adaptation"), + None, + vec![], + ) + .await + .expect("create on broken backend"); + break_vector_arm(&rt_broken); + + let tok_healthy = rt_healthy.authorize(ns.clone()).unwrap(); + rt_healthy + .create_entity( + &tok_healthy, + "concept", + None, + "QLoRA", + Some("Quantised LoRA"), + None, + vec![], + ) + .await + .expect("create on healthy backend"); + + let request = validated_kg_search(serde_json::json!({ + "kind": "entity", + "query": "LoRA", + "limit": 10, + })); + let (merged_hits, _note_hits, per_backend) = coord.fan_out_search(&request, &ns).await; + + assert_eq!(per_backend.len(), 2, "both backends in report"); + assert!( + !merged_hits.is_empty(), + "merged results must still include the broken backend's text hit" + ); + + let broken_report = per_backend + .iter() + .find(|r| r.backend_id.as_str() == "broken") + .expect("broken backend reported"); + assert!( + broken_report.error.is_none(), + "broken backend's text arm succeeded — must not be a whole-backend error: {:?}", + broken_report.error + ); + assert!( + !broken_report.hits.is_empty(), + "broken backend's text hit must survive" + ); + assert!( + broken_report + .vector_error + .as_deref() + .unwrap_or_default() + .contains("injected vector-arm failure"), + "broken backend must report its vector_error, got {:?}", + broken_report.vector_error + ); + + let healthy_report = per_backend + .iter() + .find(|r| r.backend_id.as_str() == "healthy") + .expect("healthy backend reported"); + assert!( + healthy_report.error.is_none() && healthy_report.vector_error.is_none(), + "healthy sibling must be unaffected: {healthy_report:?}" + ); +} + #[tokio::test] #[serial_test::serial(config_ledger)] async fn fan_out_search_single_backend_applies_source_filter_before_limit() { @@ -3140,6 +3506,80 @@ async fn t7b_multi_backend_search_kind_filter_excludes_off_kind() { } } +/// `SubstrateCoordinatorService::fan_out_search`'s per-backend mapping +/// (`service.rs`'s `vector_error: r.vector_error` join line) must carry a +/// real vector-arm failure all the way to the rendered JSON envelope: +/// `arm_participation.vector.status == "error"`, +/// `arm_participation.text.status == "ran"`, and the text arm's +/// `candidate_count` equal to the number of text-sourced hits. Two backends +/// are required so the coordinator path is actually taken instead of falling +/// through to the single-backend registry dispatch (`is_single_backend()` +/// short-circuit in `khive-mcp/src/server.rs`). +#[tokio::test] +async fn coordinator_service_search_reports_vector_arm_error_in_json_envelope() { + let rt_broken = memory_runtime_with_constant_embeddings(); + let rt_healthy = memory_runtime(); + let ns = Namespace::local(); + + let token = rt_broken.authorize(ns).unwrap(); + rt_broken + .create_entity( + &token, + "concept", + None, + "FlashAttention", + Some("IO-aware exact attention"), + None, + vec![], + ) + .await + .expect("create entity"); + break_vector_arm(&rt_broken); + + let server = two_backend_server(Arc::clone(&rt_broken), Arc::clone(&rt_healthy)); + + let result_str = server + .dispatch_request_local(khive_mcp::tools::request::RequestParams { + ops: r#"search(kind="concept", query="FlashAttention")"#.to_string(), + presentation: None, + presentation_per_op: None, + save_to: None, + format: None, + format_per_op: None, + request_id: None, + }) + .await + .expect("dispatch"); + + let response: serde_json::Value = + serde_json::from_str(&result_str).expect("parse response JSON"); + let op = &response["results"][0]; + assert_eq!( + op["ok"].as_bool(), + Some(true), + "search op must succeed: {op}" + ); + + let hits = op["result"].as_array().expect("result must be array"); + let text_hit_count = hits.iter().filter(|hit| hit["source"] == "text").count(); + + assert_eq!( + op["arm_participation"]["vector"]["status"], + serde_json::json!("error"), + "vector arm must report error, got: {op}" + ); + assert_eq!( + op["arm_participation"]["text"]["status"], + serde_json::json!("ran"), + "text arm must report ran, got: {op}" + ); + assert_eq!( + op["arm_participation"]["text"]["candidate_count"].as_u64(), + Some(text_hit_count as u64), + "text candidate_count must equal the number of text-sourced hits, got: {op}" + ); +} + /// T7c: `min_score` floor filters out low-scoring hits. /// /// Seeds one entity, searches with an impossibly high min_score (1.0), and asserts diff --git a/docs/adr/ADR-130-search-response-completeness-and-ranking-evidence.md b/docs/adr/ADR-130-search-response-completeness-and-ranking-evidence.md index ffeafba69..a7efb60ad 100644 --- a/docs/adr/ADR-130-search-response-completeness-and-ranking-evidence.md +++ b/docs/adr/ADR-130-search-response-completeness-and-ranking-evidence.md @@ -763,6 +763,94 @@ v0.8.0 contract. Emitting `timeout` while keeping `retryable` unconditionally `false` is not a valid partial adoption, because it publishes a cause the reader can act on while denying the action the cause licenses. +## Amendment 3 (2026-08-30): per-arm participation evidence + +The KG search envelope MUST expose `arm_participation` on every successful +search and inside every `search_incomplete` error. It is an object with exactly +the keys `text` and `vector`; each value carries: + +- `status`, from the closed vocabulary `ran | skipped | error`; and +- `candidate_count`, a non-negative integer. + +The statuses describe selection and completion, not result presence: + +- `ran` means the arm was selected and every backend on which it was selected + completed that search. +- `skipped` means the arm was not selected on any backend. Today this occurs + for `vector` when no configured embedding model can produce a query vector. +- `error` means at least one backend on which the arm was selected failed + before the server could establish that arm's complete contribution. Other + backends may still have contributed candidates, so `error` does not require + `candidate_count` to be zero. + +`candidate_count` counts final canonical response candidates whose `source` +includes that arm, after every server-side predicate, source/score filter, and +the caller's result limit. A `source="both"` hit increments both counts. The +count therefore describes evidence in the response, not the backend's raw +pre-fusion candidate pool. It is bounded by the public search result limit and +does not alter ranking, fusion, eligibility, or truncation. + +In particular, `status="ran", candidate_count=0` is a clean zero-contribution +outcome and is distinct from both `skipped` and `error`. This is the fact needed +for long, keyword-dense FTS queries: an all-vector response can now prove that +the text arm completed with no surviving match instead of leaving completion +implicit. + +Arm evidence deliberately does not duplicate an error message or invent a +second cause taxonomy. On partial and `search_incomplete` responses, the +bounded per-backend causes remain in `backend_errors`, typed with the `kind` +vocabulary of the serving release: the single constant `backend_error` in +v0.8.0, and Amendment 2's closed `timeout | backend_error` from v0.9.0 per the +Compatibility section above. `arm_participation` states which +selected arms failed to complete; `backend_errors` states why and where the +backend search failed. Both fields MUST survive presentation and frame-budget +omission at their normal envelope location. + +This amendment does not make vector nearest neighbours an identity lookup. A +caller checking whether an entity name already exists MUST issue the bare +canonical name and require the matching row itself to carry +`source="text" | "both"`. It MUST NOT infer absence from an all-vector result, +or from text-arm `status="error" | "skipped"`. A text arm that ran with zero +final candidates establishes only that the lexical query produced no surviving +match under the requested filters. + +This change is additive in v0.8.0. Tolerant readers ignore the new object; +strict readers must add the two fixed arm keys and the closed status vocabulary. +Verification MUST cover an exact-name text hit, a 60-plus-character +keyword-dense text zero-hit, vector-only and `both` candidate counting, +complete-empty, partial-with-hit, degraded-empty, and frame-budget preservation. + +## Amendment 4 (2026-09-03): arm status is per-arm, not per-backend + +Amendment 3's status definitions read as backend-granular — `ran` as "every +backend on which [the arm] was selected completed that search", `error` as "a +backend on which the arm was selected failed" — and cannot unambiguously +express the coordinated multi-backend case where one backend's text leg +completes while that same backend's vector leg fails. This amendment restates +both statuses at arm granularity, superseding Amendment 3's wording (not its +intent) for `ran` and `error`: + +- `ran` means the arm completed on every backend on which it was selected. +- `error` means the arm itself failed on at least one backend on which it was + selected, whether or not that backend's other arm completed. A backend + whose text leg completed and whose vector leg alone failed makes `vector` + read `error` while `text` reads `ran`, on the same response. + +`skipped` is unchanged from Amendment 3. The backend-level completeness +fields — `missing_backends` and `backend_errors` — are also unchanged: they +still report whole-backend failures only. A backend whose only failure was +one arm does not appear in either field, and the response's top-level +`status` stays `"complete"`. + +Separately, the sentence in Amendment 3 stating "Both fields [`arm_participation` +and `backend_errors`] MUST survive presentation and frame-budget omission at +their normal envelope location" is corrected: frame-budget omission relocates +both fields from their normal top-level location to `error.search` on the +omitted envelope (the operation's `ok` flips to `false` once its result is +discarded, so the two fields move under the synthesized `error` object rather +than staying beside a `result` that no longer exists). They still survive +omission; they do not survive at the _same_ location. + ## References - the two follow-up items this record's fixes were split from diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index bee6b2ddb..2fc9a2d53 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -471,6 +471,51 @@ operation is `ok: false` with `error.kind="search_incomplete"`; the structured error carries the same diagnostics. `backend_errors_truncated` plus `backend_errors_omitted` explicitly report causes omitted by safety bounds. +Every successful KG search also carries `arm_participation` beside `result`; +`search_incomplete` carries the same object inside `error`: + +```json +{ + "arm_participation": { + "text": { "status": "ran", "candidate_count": 0 }, + "vector": { "status": "ran", "candidate_count": 8 } + } +} +``` + +Each arm status is `ran`, `skipped`, or `error`. `ran` with zero candidates +means that arm completed but contributed no final hit; `skipped` means it was +not selected (for example, vector search without a configured embedding model); +and `error` means that arm itself failed on at least one backend where it was +selected — including a backend whose _other_ arm completed normally. A backend +whose text leg completes and whose vector leg alone fails is therefore not +"missing": it still contributed usable text hits, so it does not appear in +`missing_backends` or `backend_errors`, and the response keeps +`status: "complete"`. `arm_participation.vector.status` is the only place that +failure surfaces on such a response — `text` still reports `"ran"`. A response +is `"partial"` (with `missing_backends`/`backend_errors`) only when a whole +backend's search failed outright — auth, timeout, or a failed text leg — not +merely one of its arms. On such degraded responses, the bounded reason stays in +`backend_errors`; its `kind` is the single constant `backend_error` in v0.8.0, +and the two-value `timeout | backend_error` vocabulary of ADR-130 Amendment 2 +ships in v0.9.0. Arm entries do not duplicate those messages. `candidate_count` +counts final hits +whose `source` includes the arm, after server filters and the result limit, so a +`both` hit increments both counts and each count is bounded by `limit`. + +This per-arm tolerance applies to the coordinated multi-backend search path, +for both entity and note substrates. A single-backend deployment (the +default — no coordinator installed) has no per-backend fan-out to isolate a +failing arm from: a vector-arm failure there fails the whole search call, and +the response carries no `arm_participation` at all. + +For an entity-name presence check, issue the short bare canonical name and +require the matching row itself to report `source: "text"` or `"both"`. An +all-vector response, or text-arm status `error`/`skipped`, is not evidence that +the name is absent. Long keyword-dense queries may legitimately report text +`ran` with `candidate_count: 0` because the lexical expression is selective; +the explicit arm evidence makes that different from a silent skip or failure. + Response shape (`kind="entity"` rows, `presentation="verbose"`): ```json