From 65d2568694cbb9565dfc436180588d2137e8de94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Tue, 15 Sep 2026 17:26:55 +0200 Subject: [PATCH 1/3] feat(compaction): render the summary as a hand-off carrying user messages verbatim After compaction the prompt opened with a bare "Conversation summary:" user message; the original request was gone and nothing told the model a previous instance had written the text. The summary is now rendered at prompt-build time as a hand-off message that frames it as such and embeds the user's earlier messages verbatim (tool results and older summaries excluded, newest-first within a budget). Stored messages and the UI divider are unchanged. --- crates/agent_core/src/runtime.rs | 73 ++++---- crates/agent_core/src/runtime/handoff.rs | 159 ++++++++++++++++++ crates/agent_core/src/runtime/tests.rs | 63 +++++++ crates/code_assistant_core/src/agent/tests.rs | 4 +- 4 files changed, 264 insertions(+), 35 deletions(-) create mode 100644 crates/agent_core/src/runtime/handoff.rs diff --git a/crates/agent_core/src/runtime.rs b/crates/agent_core/src/runtime.rs index 088f77a8..021a06a9 100644 --- a/crates/agent_core/src/runtime.rs +++ b/crates/agent_core/src/runtime.rs @@ -1,6 +1,7 @@ //! The agent loop. Application behavior plugs in through the hook traits in //! [`crate::hooks`]; application state travels type-erased in `extensions`. +mod handoff; #[cfg(test)] mod tests; mod tool_execution; @@ -848,15 +849,6 @@ impl AgentRuntime { Ok((response, request_id)) } - fn format_compaction_summary_for_prompt(summary: &str) -> String { - let trimmed = summary.trim(); - if trimmed.is_empty() { - "Conversation summary: (empty)".to_string() - } else { - format!("Conversation summary:\n{trimmed}") - } - } - fn extract_compaction_summary_text(blocks: &[ContentBlock]) -> String { let mut collected = Vec::new(); for block in blocks { @@ -888,23 +880,44 @@ impl AgentRuntime { messages } + /// The messages the next request is built from: everything from the last + /// compaction summary onwards, with the summary rendered as the hand-off + /// message that also carries the user's earlier messages verbatim. fn prompt_messages(&self) -> Vec { let path = self.conversation.path(); + let nodes = self.conversation.nodes(); let start = path .iter() .rposition(|id| { - self.conversation - .nodes() + nodes .get(id) .is_some_and(|node| node.message.is_compaction_summary) }) .unwrap_or(0); - path[start..] + let mut messages: Vec = path[start..] .iter() .filter(|id| !self.prompt_projection.omitted_nodes.contains(id)) - .filter_map(|id| self.conversation.nodes().get(id)) + .filter_map(|id| nodes.get(id)) .map(|node| node.message.clone()) - .collect() + .collect(); + if let Some(summary) = messages + .first_mut() + .filter(|message| message.is_compaction_summary) + { + let user_messages = handoff::user_message_texts( + path[..start] + .iter() + .filter_map(|id| nodes.get(id)) + .map(|node| &node.message), + ); + let summary_text = match &summary.content { + MessageContent::Text(text) => text.as_str(), + MessageContent::Structured(_) => "", + }; + summary.content = + MessageContent::Text(handoff::render_handoff(&user_messages, summary_text)); + } + messages } fn context_usage_ratio(&mut self) -> Result> { @@ -1294,28 +1307,22 @@ impl AgentRuntime { } for message in &mut messages { - match &mut message.content { - MessageContent::Structured(blocks) => { - for block in blocks { - if let ContentBlock::ToolResult { - tool_use_id, - content, - is_error, - .. - } = block - && let Some((output, error)) = outputs.get(tool_use_id) - { - *content = output.clone(); - if *error { - *is_error = Some(true); - } + if let MessageContent::Structured(blocks) = &mut message.content { + for block in blocks { + if let ContentBlock::ToolResult { + tool_use_id, + content, + is_error, + .. + } = block + && let Some((output, error)) = outputs.get(tool_use_id) + { + *content = output.clone(); + if *error { + *is_error = Some(true); } } } - MessageContent::Text(text) if message.is_compaction_summary => { - *text = Self::format_compaction_summary_for_prompt(text); - } - _ => {} } } messages diff --git a/crates/agent_core/src/runtime/handoff.rs b/crates/agent_core/src/runtime/handoff.rs new file mode 100644 index 00000000..056ec908 --- /dev/null +++ b/crates/agent_core/src/runtime/handoff.rs @@ -0,0 +1,159 @@ +//! The hand-off message a compacted conversation resumes from. +//! +//! After compaction the prompt no longer contains the exchanges before the +//! summary. The summary is rendered as a single user message that frames it +//! as a hand-off from a previous instance and carries the user's own +//! messages verbatim, so what was asked for survives the compaction. +use llm::{ContentBlock, Message, MessageContent, MessageRole}; + +/// Rough budget for the verbatim user messages, in characters (about 20k +/// tokens). The newest messages take precedence. +const USER_MESSAGES_CHAR_BUDGET: usize = 80_000; + +const PREAMBLE: &str = "Another instance of this assistant was working in this session and \ +reached the context limit. It wrote the hand-off below. The user's messages are \ +reproduced verbatim so nothing about what they asked for is lost. Build on the \ +work already done instead of repeating it; the workspace reflects everything the \ +previous instance did."; + +/// The verbatim text of the real user messages among `messages`. Tool-result +/// messages and earlier compaction summaries share the user role but are not +/// user messages; images are dropped. +pub(super) fn user_message_texts<'a>(messages: impl Iterator) -> Vec { + messages + .filter(|message| message.role == MessageRole::User && !message.is_compaction_summary) + .filter_map(|message| match &message.content { + MessageContent::Text(text) => Some(text.clone()), + MessageContent::Structured(blocks) => { + let text = blocks + .iter() + .filter_map(|block| match block { + ContentBlock::Text { text, .. } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("\n"); + (!text.trim().is_empty()).then_some(text) + } + }) + .collect() +} + +/// Renders the hand-off message from the user's messages and the summary the +/// previous instance wrote. +pub(super) fn render_handoff(user_messages: &[String], summary: &str) -> String { + render_handoff_within(user_messages, summary, USER_MESSAGES_CHAR_BUDGET) +} + +fn render_handoff_within(user_messages: &[String], summary: &str, budget: usize) -> String { + let mut remaining = budget; + let mut kept = 0; + for message in user_messages.iter().rev() { + if message.len() > remaining { + break; + } + remaining -= message.len(); + kept += 1; + } + let omitted = user_messages.len() - kept; + + let mut out = String::new(); + out.push_str("\n"); + out.push_str(PREAMBLE); + out.push_str("\n\n\n"); + if omitted > 0 { + out.push_str(&format!("({omitted} earlier messages omitted)\n")); + } + for (index, message) in user_messages.iter().enumerate().skip(omitted) { + out.push_str(&format!( + "\n{}\n\n", + index + 1, + message.trim() + )); + } + out.push_str("\n\n\n"); + let summary = summary.trim(); + out.push_str(if summary.is_empty() { + "(no summary available)" + } else { + summary + }); + out.push_str("\n\n"); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn user_message_texts_skips_tool_results_and_summaries() { + let messages = [ + Message::new_user("Explain the compaction feature"), + Message::new_assistant("Looking."), + Message::new_user_content(vec![ContentBlock::ToolResult { + tool_use_id: "t1".into(), + content: llm::ToolResultContent::text("file contents"), + is_error: None, + start_time: None, + end_time: None, + }]), + Message { + content: MessageContent::Text("old summary".into()), + is_compaction_summary: true, + ..Default::default() + }, + Message::new_user_content(vec![ + ContentBlock::new_text("Here is a screenshot"), + ContentBlock::Image { + media_type: "image/png".into(), + data: "aaaa".into(), + start_time: None, + end_time: None, + }, + ]), + ]; + + assert_eq!( + user_message_texts(messages.iter()), + vec![ + "Explain the compaction feature".to_string(), + "Here is a screenshot".to_string() + ] + ); + } + + #[test] + fn handoff_carries_user_messages_verbatim_before_the_summary() { + let rendered = render_handoff( + &["First ask".to_string(), "Second ask".to_string()], + "Did A, B remains", + ); + + let messages_at = rendered.find("").unwrap(); + let summary_at = rendered.find("").unwrap(); + assert!(rendered.starts_with("\n")); + assert!(rendered.ends_with("\n")); + assert!(messages_at < summary_at); + assert!(rendered.contains("\nFirst ask\n")); + assert!(rendered.contains("\nSecond ask\n")); + assert!(rendered.contains("\nDid A, B remains\n")); + } + + #[test] + fn handoff_keeps_the_newest_user_messages_within_the_budget() { + let messages = ["x".repeat(30), "y".repeat(30), "z".repeat(30)]; + let rendered = render_handoff_within(&messages, "summary", 70); + + assert!(rendered.contains("(1 earlier messages omitted)")); + assert!(!rendered.contains(&"x".repeat(30))); + assert!(rendered.contains("")); + assert!(rendered.contains("")); + } + + #[test] + fn handoff_marks_a_missing_summary() { + let rendered = render_handoff(&[], " "); + assert!(rendered.contains("\n(no summary available)\n")); + } +} diff --git a/crates/agent_core/src/runtime/tests.rs b/crates/agent_core/src/runtime/tests.rs index 2aba3b67..425d9b25 100644 --- a/crates/agent_core/src/runtime/tests.rs +++ b/crates/agent_core/src/runtime/tests.rs @@ -406,3 +406,66 @@ fn checkpoint_recovery_keeps_canonical_messages_and_tool_evidence() { > 50 * 1024 ); } + +fn summary(text: &str) -> Message { + Message { + content: MessageContent::Text(text.into()), + is_compaction_summary: true, + ..Default::default() + } +} + +#[test] +fn prompt_after_compaction_opens_with_a_handoff_carrying_the_user_messages() { + let (mut runtime, _) = runtime(); + runtime + .append_message(Message::new_user("Explain how compaction works")) + .unwrap(); + runtime + .append_message(Message::new_assistant_content(vec![call("t1")])) + .unwrap(); + runtime + .append_message(Message::new_user_content(vec![result("t1")])) + .unwrap(); + runtime + .append_message(summary("Read runtime.rs; nothing edited")) + .unwrap(); + runtime + .append_message(Message::new_assistant("Continuing")) + .unwrap(); + + let prompt = runtime.render_tool_results_in_messages(); + + assert_eq!(prompt.len(), 2); + let handoff = text(&prompt[0]); + assert!(handoff.starts_with(""), "{handoff}"); + assert!(handoff.contains("Explain how compaction works")); + assert!( + !handoff.contains("evidence"), + "tool results are not user messages" + ); + assert!(handoff.contains("\nRead runtime.rs; nothing edited\n")); + assert_eq!(text(&prompt[1]), "Continuing"); +} + +#[test] +fn prompt_after_a_second_compaction_carries_user_messages_from_before_the_first() { + let (mut runtime, _) = runtime(); + runtime + .append_message(Message::new_user("First ask")) + .unwrap(); + runtime.append_message(summary("summary one")).unwrap(); + runtime + .append_message(Message::new_user("Second ask")) + .unwrap(); + runtime.append_message(summary("summary two")).unwrap(); + + let prompt = runtime.render_tool_results_in_messages(); + + assert_eq!(prompt.len(), 1); + let handoff = text(&prompt[0]); + assert!(handoff.contains("\nFirst ask\n")); + assert!(handoff.contains("\nSecond ask\n")); + assert!(!handoff.contains("summary one")); + assert!(handoff.contains("\nsummary two\n")); +} diff --git a/crates/code_assistant_core/src/agent/tests.rs b/crates/code_assistant_core/src/agent/tests.rs index b1dee6b2..3f5dfd1c 100644 --- a/crates/code_assistant_core/src/agent/tests.rs +++ b/crates/code_assistant_core/src/agent/tests.rs @@ -1057,8 +1057,8 @@ async fn test_context_compaction_uses_only_messages_after_previous_summary() -> }; assert!( - !request_contains(old_user_text), - "Compaction request should skip messages before the previous summary", + request_contains(old_user_text), + "The hand-off carries user messages from before the previous summary verbatim", ); assert!( !request_contains(old_assistant_text), From 52b70ee1ffce6cba605a3e0b4f782012ce538d88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Tue, 15 Sep 2026 17:30:09 +0200 Subject: [PATCH 2/3] feat(compaction): hand-off prompt, pending message after the hand-off, retry on tool-call answers - The compaction prompt asks for a hand-off to another instance instead of a summary: what the user asked for and expects, progress, verified facts with paths, next steps, open questions. Tool calls are forbidden. - Compaction now runs before a pending user message is appended, so a fresh request follows the hand-off instead of being folded into it. - The request keeps the tool definitions (dropping them would invalidate the cached prompt prefix). A response without text is asked once more with a reminder; a second failure fails the turn instead of storing an empty summary. Thinking blocks no longer leak into the summary. --- crates/agent_core/src/runtime.rs | 82 +++++--- .../resources/compaction_prompt.md | 17 +- crates/code_assistant_core/src/agent/tests.rs | 190 ++++++++++++++++++ 3 files changed, 250 insertions(+), 39 deletions(-) diff --git a/crates/agent_core/src/runtime.rs b/crates/agent_core/src/runtime.rs index 021a06a9..93b7f597 100644 --- a/crates/agent_core/src/runtime.rs +++ b/crates/agent_core/src/runtime.rs @@ -28,6 +28,11 @@ use tools_core::{ }; use tracing::{debug, trace, warn}; +/// Appended to the compaction prompt when the model answered it with tool +/// calls instead of text. +const HANDOFF_TOOL_CALL_REMINDER: &str = "Reminder: this is a compaction request. Do not call any \ +tools; reply with the hand-off text only."; + /// Everything an [`AgentRuntime`] is built from. pub struct AgentRuntimeComponents { pub llm_provider: Box, @@ -358,6 +363,13 @@ impl AgentRuntime { loop { self.cancellation.check()?; + // Compact before a pending user message is appended: the hand-off + // covers the history so far and the new request follows it. + if self.should_trigger_compaction()? { + self.perform_compaction().await?; + continue; + } + // Check for pending user message and add it to history at start of each iteration if let Some(pending_blocks) = self.get_and_clear_pending_message() { let text_summary = text_summary_from_blocks(&pending_blocks); @@ -372,11 +384,6 @@ impl AgentRuntime { .await?; } - if self.should_trigger_compaction()? { - self.perform_compaction().await?; - continue; - } - let messages = self.render_tool_results_in_messages(); // Pre-allocate the node_id for this assistant message. @@ -849,24 +856,43 @@ impl AgentRuntime { Ok((response, request_id)) } - fn extract_compaction_summary_text(blocks: &[ContentBlock]) -> String { - let mut collected = Vec::new(); - for block in blocks { - match block { - ContentBlock::Text { text, .. } => collected.push(text.as_str()), - ContentBlock::Thinking { thinking, .. } => { - collected.push(thinking.as_str()); - } - _ => {} - } - } + fn handoff_text(blocks: &[ContentBlock]) -> Option { + let text = blocks + .iter() + .filter_map(|block| match block { + ContentBlock::Text { text, .. } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("\n"); + let text = text.trim(); + (!text.is_empty()).then(|| text.to_string()) + } - let merged = collected.join("\n").trim().to_string(); - if merged.is_empty() { - "No summary was generated.".to_string() - } else { - merged + /// Asks the model for the hand-off text. The request keeps the tool + /// definitions so the cached prompt prefix stays valid; a model that + /// answers with tool calls instead of text is asked once more. + async fn request_handoff(&mut self) -> Result { + let prompt = self.hooks.compaction.compaction_prompt().to_string(); + let messages = self.render_tool_results_in_messages(); + for reminder in [None, Some(HANDOFF_TOOL_CALL_REMINDER)] { + let text = match reminder { + None => prompt.to_string(), + Some(reminder) => format!("{prompt}\n\n{reminder}"), + }; + let mut request = messages.clone(); + request.push(Message { + role: MessageRole::User, + content: MessageContent::Text(text), + ..Default::default() + }); + let (response, _) = self.get_non_streaming_response(request).await?; + if let Some(text) = Self::handoff_text(&response.content) { + return Ok(text); + } + warn!("Compaction response contained no text; asking once more"); } + anyhow::bail!("The model did not produce a hand-off text for compaction") } /// The active-path messages from the last compaction summary onwards. @@ -1142,26 +1168,16 @@ impl AgentRuntime { async fn perform_compaction(&mut self) -> Result<()> { debug!("Starting context compaction"); - let compaction_message = Message { - role: MessageRole::User, - content: MessageContent::Text(self.hooks.compaction.compaction_prompt().to_string()), - ..Default::default() - }; - - let mut messages = self.render_tool_results_in_messages(); - messages.push(compaction_message); self.send_ui(AgentUiEvent::ActivityChanged { activity: AgentActivity::WaitingForResponse, }) .await?; - let response_result = self.get_non_streaming_response(messages).await; + let summary_result = self.request_handoff().await; self.send_ui(AgentUiEvent::ActivityChanged { activity: AgentActivity::Running, }) .await?; - let (response, _) = response_result?; - - let summary_text = Self::extract_compaction_summary_text(&response.content); + let summary_text = summary_result?; // The compaction policy may contribute an addendum to the summary // message — e.g. reminding the model which skills it had loaded, since diff --git a/crates/code_assistant_core/resources/compaction_prompt.md b/crates/code_assistant_core/resources/compaction_prompt.md index f2213526..5145440e 100644 --- a/crates/code_assistant_core/resources/compaction_prompt.md +++ b/crates/code_assistant_core/resources/compaction_prompt.md @@ -1,8 +1,13 @@ -The conversation history is nearing the model's context window limit. Provide a thorough summary that allows resuming the task without the earlier messages. Include: -- The current objectives or tasks. -- Key actions taken so far and their outcomes. -- Important files, commands, or decisions that matter for continuing. -- Outstanding questions or follow-up work that still needs attention. -Respond with plain text only. +You are performing a context checkpoint compaction: the conversation is nearing the model's context window limit. Write a hand-off for another instance of this assistant that will resume the task without access to the messages above. + +Include: +- What the user asked for and what kind of response they expect (an answer, an explanation, a change to the workspace). +- Current progress: what has been done, what was decided and why. +- Verified facts the next instance would otherwise have to rediscover: relevant files with their paths and what they contain, commands run and their results, constraints and user preferences. +- What remains to be done, as concrete next steps. +- Open questions the user still has to answer. + +Be structured and specific. The user's own messages are handed over verbatim alongside this text, so do not repeat them. +Do not call any tools in this response. Respond with plain text only. diff --git a/crates/code_assistant_core/src/agent/tests.rs b/crates/code_assistant_core/src/agent/tests.rs index 3f5dfd1c..6018068a 100644 --- a/crates/code_assistant_core/src/agent/tests.rs +++ b/crates/code_assistant_core/src/agent/tests.rs @@ -2619,3 +2619,193 @@ async fn test_granted_session_asks_only_once_per_tool() -> Result<()> { Ok(()) } + +fn compaction_test_agent( + responses: Vec>, +) -> (Agent, MockLLMProvider) { + let mock_llm = MockLLMProvider::new(responses); + let mock_llm_ref = mock_llm.clone(); + let components = AgentComponents { + llm_provider: Box::new(mock_llm), + project_manager: Arc::new(MockProjectManager::new()), + command_executor: Arc::new(create_command_executor_mock()), + ui: Arc::new(MockUI::default()), + state_persistence: Box::new(NoOpStatePersistence), + permission_handler: None, + permissions: Default::default(), + tool_registry: crate::tools::test_registry(), + sub_agent_runner: None, + wakeups: None, + pty_sessions: None, + browser_sessions: None, + terminal_interrupts: None, + session_source: None, + hooks_factory: None, + }; + let session_config = SessionConfig { + init_path: Some(PathBuf::from("./test_path")), + initial_project: String::new(), + tool_syntax: ToolSyntax::Native, + use_diff_blocks: false, + sandbox_policy: SandboxPolicy::DangerFullAccess, + ..SessionConfig::default() + }; + let mut agent = Agent::new(components, session_config); + agent.disable_naming_reminders(); + agent.set_test_session_metadata( + "session-1".to_string(), + SessionModelConfig::new_for_tests("test-model".to_string()), + ); + agent.set_test_context_limit(100); + (agent, mock_llm_ref) +} + +fn text_response(text: &str) -> LLMResponse { + LLMResponse { + content: vec![ContentBlock::new_text(text)], + usage: Usage::zero(), + rate_limit_info: None, + } +} + +fn idle_response() -> LLMResponse { + LLMResponse { + content: Vec::new(), + usage: Usage::zero(), + rate_limit_info: None, + } +} + +fn over_threshold_assistant(text: &str) -> Message { + Message::new_assistant(text) + .with_request_id(1) + .with_usage(Usage { + input_tokens: 85, + output_tokens: 12, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }) +} + +fn message_text(message: &Message) -> String { + match &message.content { + MessageContent::Text(text) => text.clone(), + MessageContent::Structured(blocks) => blocks + .iter() + .filter_map(|block| match block { + ContentBlock::Text { text, .. } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("\n"), + } +} + +#[tokio::test] +async fn test_pending_user_message_lands_after_the_compaction_handoff() -> Result<()> { + let (mut agent, mock_llm) = compaction_test_agent(vec![ + Ok(idle_response()), + Ok(text_response("hand-off text")), + ]); + agent.append_message(Message::new_user("Original request"))?; + agent.append_message(over_threshold_assistant("Working on it"))?; + let pending = Arc::new(std::sync::Mutex::new(Some(vec![ContentBlock::new_text( + "Follow-up question", + )]))); + agent.set_pending_message_ref(pending); + + agent.run_single_iteration().await?; + + let history = agent.message_history_for_tests(); + let summary_at = history + .iter() + .position(|message| message.is_compaction_summary) + .expect("compaction summary in history"); + assert_eq!( + message_text(&history[summary_at + 1]), + "Follow-up question", + "the pending message follows the summary instead of being folded into it" + ); + + let requests = mock_llm.get_requests(); + assert_eq!(requests.len(), 2); + assert!( + !requests[0] + .messages + .iter() + .any(|message| message_text(message).contains("Follow-up question")), + "the compaction request covers only the history before the pending message" + ); + let follow_up = &requests[1].messages; + assert!(message_text(&follow_up[0]).starts_with("")); + assert_eq!(message_text(&follow_up[1]), "Follow-up question"); + Ok(()) +} + +#[tokio::test] +async fn test_compaction_retries_once_when_the_model_answers_with_tool_calls() -> Result<()> { + let tool_only = LLMResponse { + content: vec![ContentBlock::new_tool_use( + "t1", + "read_files", + serde_json::json!({"paths": ["x"]}), + )], + usage: Usage::zero(), + rate_limit_info: None, + }; + let (mut agent, mock_llm) = compaction_test_agent(vec![ + Ok(idle_response()), + Ok(text_response("hand-off text")), + Ok(tool_only), + ]); + agent.append_message(Message::new_user("Original request"))?; + agent.append_message(over_threshold_assistant("Working on it"))?; + + agent.run_single_iteration().await?; + + let requests = mock_llm.get_requests(); + assert_eq!(requests.len(), 3, "compaction, retry, follow-up"); + let retry_prompt = message_text(requests[1].messages.last().unwrap()); + assert!( + retry_prompt.contains("system-compaction") + && retry_prompt.contains("Reminder: this is a compaction request"), + "{retry_prompt}" + ); + let summary = agent + .message_history_for_tests() + .into_iter() + .find(|message| message.is_compaction_summary) + .expect("compaction summary in history"); + assert_eq!(message_text(&summary), "hand-off text"); + Ok(()) +} + +#[tokio::test] +async fn test_compaction_fails_when_the_model_never_answers_with_text() -> Result<()> { + let tool_only = || LLMResponse { + content: vec![ContentBlock::new_tool_use( + "t1", + "read_files", + serde_json::json!({"paths": ["x"]}), + )], + usage: Usage::zero(), + rate_limit_info: None, + }; + let (mut agent, _) = compaction_test_agent(vec![Ok(tool_only()), Ok(tool_only())]); + agent.append_message(Message::new_user("Original request"))?; + agent.append_message(over_threshold_assistant("Working on it"))?; + + let error = agent + .run_single_iteration() + .await + .expect_err("a compaction without a hand-off text fails the turn"); + assert!(error.to_string().contains("hand-off"), "{error}"); + assert!( + !agent + .message_history_for_tests() + .iter() + .any(|message| message.is_compaction_summary), + "no empty summary is stored" + ); + Ok(()) +} From 6f69ca7d248c14d2ed20e309ad1c17ff081a1bb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Tue, 15 Sep 2026 17:30:45 +0200 Subject: [PATCH 3/3] docs(compaction): describe the hand-off implementation instead of the original plan --- docs/context-compaction.md | 133 ++++++++++++++++++++++++------------- 1 file changed, 87 insertions(+), 46 deletions(-) diff --git a/docs/context-compaction.md b/docs/context-compaction.md index 9333ae9a..8eea8736 100644 --- a/docs/context-compaction.md +++ b/docs/context-compaction.md @@ -1,46 +1,87 @@ -# Context Compaction Implementation Plan - -This document outlines the phased approach for adding automatic context compaction to the agent loop. The goal is to proactively summarize long conversations when the active model’s context window nears capacity, keep the UI history intact, and continue prompting the LLM with only the most recent summarized state. - -## Phase 1 – Configuration & Data Model -- Require a `context_token_limit` field in every model entry in `models.json`. -- Update the configuration loader (`crates/llm/src/provider_config.rs`) and validation logic to deserialize, store, and surface this limit. -- Propagate the limit into `SessionModelConfig` (`crates/code_assistant/src/persistence.rs`) and ensure session creation (`crates/code_assistant/src/session/manager.rs`) records it. -- Extend `llm::Message` with an `is_compaction_summary` flag (serde-defaulted to `false`) so we can tag summary messages without adding new content block variants. -- **Tests:** extend existing configuration loading tests (or add new ones) to assert `context_token_limit` is required and correctly parsed; add coverage verifying the new content block round-trips through serialization. - -## Phase 2 – Agent Compaction Logic -- Add helpers in `crates/code_assistant/src/agent/runner.rs` to read the context limit, calculate the percent of the window consumed based on the latest assistant `Usage`, and define a compaction threshold (e.g., 80%). -- Before building an `LLMRequest` in `run_single_iteration`, detect when the threshold is exceeded. -- When triggered, inject a system-authored prompt requesting a detailed summary, send it to the LLM without streaming, and store the response as a user message tagged with `is_compaction_summary`. -- Adjust the message-preparation path (`render_tool_results_in_messages` and any related helpers) so the next LLM request only includes messages from the last compaction summary onward, while keeping the full `message_history` for persistence and UI. -- **Tests:** add unit coverage to assert the compaction branch fires when expected, the summary block is stored correctly, and filtering logic feeds only post-summary messages to the provider. - -## Phase 3 – Persistence & Reload -- Ensure `ChatSession` serialization (`crates/code_assistant/src/persistence.rs`) handles the new summary block without data loss. -- Verify session loading (`Agent::load_from_session_state`) and `SessionInstance::convert_messages_to_ui_data` (`crates/code_assistant/src/session/instance.rs`) keep summaries visible while still allowing the agent to trim the prompt correctly. -- **Tests:** add persistence round-trip tests (if absent) that include a compaction summary and confirm reload semantics remain consistent. - -## Phase 4 – UI Presentation -- Extend `DisplayFragment` with `CompactionDivider` in `crates/code_assistant/src/ui/streaming/mod.rs`. -- Update stream processors (`json_processor.rs`, `xml_processor.rs`, `caret_processor.rs`) to emit the divider fragment when converting messages tagged with `is_compaction_summary`. -- Enhance GPUI components: - - Add a collapsible divider block in `crates/code_assistant/src/ui/gpui/elements.rs` showing the “conversation compacted” banner and the summary text. - - Ensure `MessagesView` (`crates/code_assistant/src/ui/gpui/messages.rs`) handles the fragment, including expand/collapse state management. -- **Tests:** add GPUI/component tests (or logic tests where available) validating the divider renders, defaults to collapsed, and expands to reveal the summary. - -## Phase 5 – Validation & Follow-Up -- Run formatting (`cargo fmt`), linting (`cargo clippy` once re-enabled), and targeted test suites (`cargo test` with focus on updated modules). -- Add or update documentation references pointing to this file if needed. -- **Tests:** confirm the new automated tests pass and consider adding integration coverage that simulates a full compaction cycle end-to-end. - -## Outcome - -─ Worked for 1h 18m 34s ────────────────────────────────────────────────────────────────────────────────────────────────── - -• - Added automatic compaction support: when prior assistant usage crosses the configured context window threshold, we now inject a summary request, persist the result as a user message tagged with `is_compaction_summary`, and refresh the UI so a collapsible “conversation compacted” banner appears (see `crates/code_assistant/src/agent/runner.rs`, `crates/code_assistant/src/ui/gpui/elements.rs`, and `crates/code_assistant/src/ui/gpui/mod.rs`). - - Introduced context_token_limit to the model pipeline, loading it from the shared provider configuration whenever the agent needs it (crates/llm/src/provider_config.rs#L26, crates/code_assistant/src/agent/runner.rs#L1239) instead of persisting it per session. - - Updated streaming/terminal/ACP surfaces to understand the new compaction summary block so replay, terminal, and ACP conversions all remain consistent (see crates/llm/src/anthropic.rs#L170, crates/code_assistant/src/ui/streaming/json_processor.rs#L311, crates/code_assistant/src/ui/terminal/ui.rs#L420, crates/code_assistant/src/acp/ui.rs#L130). - - Documented the rollout as a phased plan in docs/context-compaction.md and added focused unit coverage (crates/code_assistant/src/agent/tests.rs#L780) validating summary insertion and UI refresh. - - Tests: cargo test test_context_compaction_inserts_summary --tests (passes). Full cargo test is blocked in this sandbox by OS permission checks (integration test test_tool_limit_with_realistic_anthropic_chunks). +# Context Compaction + +When the conversation nears the model's context window, the agent asks the +model for a hand-off and continues in a fresh context that starts from it. +The full history stays in the session and the UI; only the prompt sent to +the LLM is trimmed. + +## Trigger + +- `CompactionPolicy` (`crates/agent_core/src/hooks.rs`) decides when and + with which prompt. The domain implementation is `TokenRatioCompaction` + (`crates/code_assistant_core/src/plugins/compaction.rs`): it compacts once + the last assistant turn's usage (input + cache write + cache read + + output) reaches 80% of the model's `context_token_limit` from + `models.json`. An unknown limit disables compaction for the run. +- The check runs at the top of every loop iteration in + `AgentRuntime::run_until_complete` (`crates/agent_core/src/runtime.rs`), + **before** a pending user message is appended. A request that arrives + while the context is full therefore follows the hand-off instead of being + folded into it. + +## The compaction request + +`request_handoff` sends the current prompt plus one user message holding +the compaction prompt +(`crates/code_assistant_core/resources/compaction_prompt.md`). The prompt +frames the task as a hand-off to another instance: what the user asked for +and expects, progress and decisions, verified facts with file paths, next +steps, open questions. It forbids tool calls. + +The request keeps the system prompt and the tool definitions unchanged. +Dropping the tools would change the cached prompt prefix (tools → system → +messages) and make the whole request a cache miss; a `tool_choice` change +would still invalidate the messages block. If the model answers with tool +calls instead of text anyway, the request is repeated once with a reminder +appended; a second failure fails the turn rather than storing an empty +summary. Only text blocks count as the hand-off; thinking blocks are +ignored. + +## Storage + +The hand-off text is appended as a user message flagged +`is_compaction_summary`. The policy may append an addendum +(`post_compaction_summary_addendum`), e.g. a reminder of the skills that +were loaded before compaction dropped their tool results. The UI receives +a `DisplayFragment::CompactionDivider` with the summary text and renders a +collapsible banner; the divider does not include the addendum. + +## Prompt after compaction + +`prompt_messages` builds the request from the last summary node onwards +and rewrites the summary node into the hand-off message +(`crates/agent_core/src/runtime/handoff.rs`): + +``` + + + + +…verbatim… +… + + + +…hand-off text (plus addendum)… + + +``` + +The user messages are collected from the whole active path before the +summary, so a second compaction still carries the messages from before the +first. Tool-result messages and earlier summaries are skipped (they share +the user role); images are dropped. The newest messages are kept within a +character budget, older ones are counted as omitted. Everything is one +text message, which avoids consecutive user turns that some providers +reject. + +## Tests + +- `crates/agent_core/src/runtime/handoff.rs` — rendering and message + selection. +- `crates/agent_core/src/runtime/tests.rs` — prompt projection after one + and two compactions. +- `crates/code_assistant_core/src/agent/tests.rs` — end-to-end: summary + insertion, pending message ordering, retry on tool-call answers, + failure without text, skill reminder addendum.