diff --git a/app/src/ai/agent/api.rs b/app/src/ai/agent/api.rs index b6062f2d..2d57a5dc 100644 --- a/app/src/ai/agent/api.rs +++ b/app/src/ai/agent/api.rs @@ -134,6 +134,9 @@ pub struct RequestParams { /// Full Bedrock conversation history for direct Bedrock calls. /// When present, the Bedrock path uses this instead of extracting from task_context. pub bedrock_message_history: Vec, + /// Compacted summary from a prior summarization pass. Injected into the system + /// prompt so it benefits from system-level caching. + pub bedrock_compact_summary: Option, /// Populated by the Bedrock path after building the message list. /// Contains the full messages sent (old history + new input) so the controller /// can store them back into the conversation for the next request cycle. @@ -328,6 +331,7 @@ impl RequestParams { parent_agent_id: None, agent_name: None, bedrock_message_history: Vec::new(), + bedrock_compact_summary: None, bedrock_messages_sent: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), is_summarization: false, } diff --git a/app/src/ai/agent/api/impl.rs b/app/src/ai/agent/api/impl.rs index 2d598f28..6e50210f 100644 --- a/app/src/ai/agent/api/impl.rs +++ b/app/src/ai/agent/api/impl.rs @@ -153,6 +153,7 @@ pub async fn generate_multi_agent_output( model_id, root_task_id: params.root_task_id.clone(), bedrock_message_history: params.bedrock_message_history.clone(), + bedrock_compact_summary: params.bedrock_compact_summary.clone(), bedrock_messages_sent: params.bedrock_messages_sent.clone(), is_summarization: params.is_summarization, }; diff --git a/app/src/ai/agent/api/impl_tests.rs b/app/src/ai/agent/api/impl_tests.rs index 14693ac7..70b3bc97 100644 --- a/app/src/ai/agent/api/impl_tests.rs +++ b/app/src/ai/agent/api/impl_tests.rs @@ -41,6 +41,7 @@ fn request_params_with_ask_user_question_enabled(ask_user_question_enabled: bool parent_agent_id: None, agent_name: None, bedrock_message_history: Vec::new(), + bedrock_compact_summary: None, bedrock_messages_sent: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), is_summarization: false, } diff --git a/app/src/ai/agent/conversation.rs b/app/src/ai/agent/conversation.rs index a2b5d839..b1bcb11f 100644 --- a/app/src/ai/agent/conversation.rs +++ b/app/src/ai/agent/conversation.rs @@ -241,6 +241,11 @@ pub struct AIConversation { /// context window size, NOT a cumulative total. current_context_tokens: u32, + /// When set, contains the compacted summary of prior conversation history. + /// Injected into the system prompt (not the messages array) so it benefits + /// from system-level caching at the 1-hour TTL. + compact_summary: Option, + /// Guards against repeated auto-compact triggers within the same high-usage window. /// Set to true when auto-compact fires; reset when summarization completes. has_pending_auto_compact: bool, @@ -298,6 +303,7 @@ impl AIConversation { is_remote_child: false, last_event_sequence: None, bedrock_message_history: Vec::new(), + compact_summary: None, current_context_tokens: 0, has_pending_auto_compact: false, subagent_retry_count: 0, @@ -498,6 +504,7 @@ impl AIConversation { is_remote_child: false, last_event_sequence, bedrock_message_history, + compact_summary: None, current_context_tokens: 0, has_pending_auto_compact: false, subagent_retry_count: 0, @@ -525,6 +532,14 @@ impl AIConversation { self.bedrock_message_history.extend(messages); } + pub fn compact_summary(&self) -> Option<&str> { + self.compact_summary.as_deref() + } + + pub fn set_compact_summary(&mut self, summary: Option) { + self.compact_summary = summary; + } + /// Assigns fresh exchange IDs to all exchanges in this conversation. /// Used when forking conversations to avoid ID collisions with persisted blocks. pub fn reassign_exchange_ids(&mut self) { diff --git a/app/src/ai/bedrock/client.rs b/app/src/ai/bedrock/client.rs index 9756a974..7d76bbee 100644 --- a/app/src/ai/bedrock/client.rs +++ b/app/src/ai/bedrock/client.rs @@ -133,6 +133,7 @@ impl BedrockClient { needs_create_task: bool, messages: Vec, system_prompt: Option, + compact_summary: Option, tools: Vec, max_tokens: i32, temperature: Option, @@ -158,6 +159,7 @@ impl BedrockClient { let converted = build_converse_request( messages.clone(), system_prompt.clone(), + compact_summary, tools.clone(), max_tokens, temperature, diff --git a/app/src/ai/bedrock/convert.rs b/app/src/ai/bedrock/convert.rs index fa9ecbd3..e57d4766 100644 --- a/app/src/ai/bedrock/convert.rs +++ b/app/src/ai/bedrock/convert.rs @@ -59,7 +59,7 @@ pub enum ContentPart { }, } -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct ToolDefinition { pub name: String, pub description: String, @@ -69,6 +69,7 @@ pub struct ToolDefinition { pub fn build_converse_request( messages: Vec, system_prompt: Option, + compact_summary: Option, tools: Vec, max_tokens: i32, temperature: Option, @@ -76,7 +77,7 @@ pub fn build_converse_request( stop_sequences: Option>, ) -> ConvertedRequest { let bedrock_messages = convert_messages(messages); - let system = convert_system_prompt(system_prompt); + let system = convert_system_prompt(system_prompt, compact_summary); let inference_config = build_inference_config(max_tokens, temperature, top_p, stop_sequences); let tool_config = build_tool_config(tools); @@ -267,22 +268,35 @@ fn coalesce_consecutive_roles(messages: Vec) -> Vec) -> Vec { - match system_prompt { - Some(prompt) if !prompt.is_empty() => { - vec![ - SystemContentBlock::Text(prompt), - SystemContentBlock::CachePoint( - CachePointBlock::builder() - .r#type(CachePointType::Default) - .ttl(CacheTtl::OneHour) - .build() - .expect("valid cache point"), - ), - ] +fn convert_system_prompt( + system_prompt: Option, + compact_summary: Option, +) -> Vec { + let mut blocks = Vec::new(); + + if let Some(prompt) = system_prompt { + if !prompt.is_empty() { + blocks.push(SystemContentBlock::Text(prompt)); } - _ => vec![], } + + if let Some(summary) = compact_summary { + blocks.push(SystemContentBlock::Text(format!( + "\n{summary}\n" + ))); + } + + if !blocks.is_empty() { + blocks.push(SystemContentBlock::CachePoint( + CachePointBlock::builder() + .r#type(CachePointType::Default) + .ttl(CacheTtl::OneHour) + .build() + .expect("valid cache point"), + )); + } + + blocks } fn build_inference_config( diff --git a/app/src/ai/bedrock/convert_tests.rs b/app/src/ai/bedrock/convert_tests.rs index 26389a80..efb1ff6b 100644 --- a/app/src/ai/bedrock/convert_tests.rs +++ b/app/src/ai/bedrock/convert_tests.rs @@ -10,7 +10,7 @@ fn test_text_message_converts_to_single_block() { content: MessageContent::Text("Hello".to_string()), }]; - let result = build_converse_request(messages, None, vec![], 4096, None, None, None); + let result = build_converse_request(messages, None, None, vec![], 4096, None, None, None); assert_eq!(result.messages.len(), 1); assert_eq!(result.messages[0].role(), &ConversationRole::User); @@ -29,7 +29,7 @@ fn test_tool_use_produces_valid_json_input() { }, }]; - let result = build_converse_request(messages, None, vec![], 4096, None, None, None); + let result = build_converse_request(messages, None, None, vec![], 4096, None, None, None); assert_eq!(result.messages.len(), 1); assert_eq!(result.messages[0].role(), &ConversationRole::Assistant); @@ -53,7 +53,7 @@ fn test_tool_result_with_matching_id() { }, }]; - let result = build_converse_request(messages, None, vec![], 4096, None, None, None); + let result = build_converse_request(messages, None, None, vec![], 4096, None, None, None); assert_eq!(result.messages.len(), 1); match &result.messages[0].content()[0] { @@ -75,7 +75,7 @@ fn test_tool_result_error_status() { }, }]; - let result = build_converse_request(messages, None, vec![], 4096, None, None, None); + let result = build_converse_request(messages, None, None, vec![], 4096, None, None, None); match &result.messages[0].content()[0] { ContentBlock::ToolResult(block) => { @@ -101,7 +101,7 @@ fn test_consecutive_same_role_messages_coalesced() { }, ]; - let result = build_converse_request(messages, None, vec![], 4096, None, None, None); + let result = build_converse_request(messages, None, None, vec![], 4096, None, None, None); assert_eq!(result.messages.len(), 1); assert_eq!(result.messages[0].content().len(), 2); @@ -126,7 +126,7 @@ fn test_alternating_roles_not_coalesced() { }, ]; - let result = build_converse_request(messages, None, vec![], 4096, None, None, None); + let result = build_converse_request(messages, None, None, vec![], 4096, None, None, None); assert_eq!(result.messages.len(), 3); assert_eq!(result.messages[0].role(), &ConversationRole::User); @@ -144,6 +144,7 @@ fn test_system_prompt_separated_from_messages() { let result = build_converse_request( messages, Some("You are a helpful assistant.".to_string()), + None, vec![], 4096, None, @@ -159,16 +160,16 @@ fn test_system_prompt_separated_from_messages() { #[test] fn test_empty_system_prompt_produces_empty_vec() { let result = - build_converse_request(vec![], Some("".to_string()), vec![], 4096, None, None, None); + build_converse_request(vec![], Some("".to_string()), None, vec![], 4096, None, None, None); assert!(result.system.is_empty()); - let result2 = build_converse_request(vec![], None, vec![], 4096, None, None, None); + let result2 = build_converse_request(vec![], None, None, vec![], 4096, None, None, None); assert!(result2.system.is_empty()); } #[test] fn test_empty_tools_produce_none_config() { - let result = build_converse_request(vec![], None, vec![], 4096, None, None, None); + let result = build_converse_request(vec![], None, None, vec![], 4096, None, None, None); assert!(result.tool_config.is_none()); } @@ -186,7 +187,7 @@ fn test_tool_definitions_produce_tool_config() { }), }]; - let result = build_converse_request(vec![], None, tools, 4096, None, None, None); + let result = build_converse_request(vec![], None, None, tools, 4096, None, None, None); assert!(result.tool_config.is_some()); let config = result.tool_config.unwrap(); @@ -195,7 +196,7 @@ fn test_tool_definitions_produce_tool_config() { #[test] fn test_inference_config_max_tokens_only() { - let result = build_converse_request(vec![], None, vec![], 8192, None, None, None); + let result = build_converse_request(vec![], None, None, vec![], 8192, None, None, None); assert_eq!(result.inference_config.max_tokens(), Some(8192)); assert_eq!(result.inference_config.temperature(), None); assert_eq!(result.inference_config.top_p(), None); @@ -207,6 +208,7 @@ fn test_inference_config_all_params() { let result = build_converse_request( vec![], None, + None, vec![], 4096, Some(0.7), @@ -233,7 +235,7 @@ fn test_multipart_content_produces_multiple_blocks() { ]), }]; - let result = build_converse_request(messages, None, vec![], 4096, None, None, None); + let result = build_converse_request(messages, None, None, vec![], 4096, None, None, None); assert_eq!(result.messages[0].content().len(), 2); assert!(matches!( @@ -271,7 +273,7 @@ fn test_tool_result_after_tool_use_coalesced_into_user_message() { }, ]; - let result = build_converse_request(messages, None, vec![], 4096, None, None, None); + let result = build_converse_request(messages, None, None, vec![], 4096, None, None, None); assert_eq!(result.messages.len(), 3); assert_eq!(result.messages[0].role(), &ConversationRole::User); diff --git a/app/src/ai/bedrock/e2e_tests.rs b/app/src/ai/bedrock/e2e_tests.rs index edbd150d..880fd3de 100644 --- a/app/src/ai/bedrock/e2e_tests.rs +++ b/app/src/ai/bedrock/e2e_tests.rs @@ -550,6 +550,7 @@ impl AgentSimulation { needs_create_task, self.conversation.clone(), Some(self.system_prompt.clone()), + None, self.tools.clone(), 8192, None, @@ -557,6 +558,7 @@ impl AgentSimulation { None, None, Arc::new(Mutex::new(Vec::new())), + false, ) .await .expect("converse_stream should succeed"); @@ -1137,6 +1139,7 @@ async fn test_reasoning_model_produces_substantial_output() { true, messages, system, + None, agent_tools(), 8192, None, @@ -1144,6 +1147,7 @@ async fn test_reasoning_model_produces_substantial_output() { None, None, Arc::new(Mutex::new(Vec::new())), + false, ) .await .expect("converse_stream should succeed"); @@ -1260,6 +1264,7 @@ async fn test_event_sequence_matches_controller_expectations() { true, messages, None, + None, vec![], 100, None, @@ -1267,6 +1272,7 @@ async fn test_event_sequence_matches_controller_expectations() { None, None, Arc::new(Mutex::new(Vec::new())), + false, ) .await .expect("should connect"); @@ -1375,6 +1381,7 @@ async fn test_followup_turn_does_not_send_create_task() { false, messages, None, + None, vec![], 100, None, @@ -1382,6 +1389,7 @@ async fn test_followup_turn_does_not_send_create_task() { None, None, Arc::new(Mutex::new(Vec::new())), + false, ) .await .expect("should connect"); @@ -1459,6 +1467,7 @@ async fn run_slash_command_test( true, messages, system_prompt, + None, tools, 4096, None, @@ -1466,6 +1475,7 @@ async fn run_slash_command_test( None, None, Arc::new(Mutex::new(Vec::new())), + false, ) .await .expect("stream should connect"); @@ -1687,6 +1697,7 @@ async fn test_slash_resume_conversation() { true, messages, None, + None, vec![], 256, None, @@ -1694,6 +1705,7 @@ async fn test_slash_resume_conversation() { None, None, Arc::new(Mutex::new(Vec::new())), + false, ) .await .expect("stream should connect"); @@ -1799,6 +1811,7 @@ async fn test_empty_messages_safety_check() { true, messages, None, + None, vec![], 100, None, @@ -1806,6 +1819,7 @@ async fn test_empty_messages_safety_check() { None, None, Arc::new(Mutex::new(Vec::new())), + false, ) .await .expect("safety fallback message should work"); @@ -1960,6 +1974,7 @@ async fn test_full_proto_round_trip_with_tool_history() { false, messages, system_prompt, + None, tools, 1024, None, @@ -1967,6 +1982,7 @@ async fn test_full_proto_round_trip_with_tool_history() { None, None, Arc::new(Mutex::new(Vec::new())), + false, ) .await; diff --git a/app/src/ai/bedrock/integration_tests.rs b/app/src/ai/bedrock/integration_tests.rs index da0cbe77..486df1ec 100644 --- a/app/src/ai/bedrock/integration_tests.rs +++ b/app/src/ai/bedrock/integration_tests.rs @@ -59,6 +59,7 @@ async fn collect_stream_output( true, messages, system_prompt, + None, tools, 8192, None, @@ -66,6 +67,7 @@ async fn collect_stream_output( None, None, Arc::new(Mutex::new(Vec::new())), + false, ) .await .expect("converse_stream should succeed"); diff --git a/app/src/ai/bedrock/translator.rs b/app/src/ai/bedrock/translator.rs index f3b729fb..331f2366 100644 --- a/app/src/ai/bedrock/translator.rs +++ b/app/src/ai/bedrock/translator.rs @@ -13,6 +13,7 @@ pub struct TranslatorRequest { pub model_id: String, pub root_task_id: Option, pub bedrock_message_history: Vec, + pub bedrock_compact_summary: Option, pub bedrock_messages_sent: Arc>>, pub is_summarization: bool, } @@ -71,12 +72,14 @@ pub async fn execute( request_translator::sanitize_messages_for_bedrock(&mut messages); let system_prompt = request_translator::extract_system_prompt(request); + let compact_summary = params.bedrock_compact_summary; let tools = request_translator::extract_tools(request); log::info!( - "[bedrock] Sending {} messages, system_prompt={}, tools={}", + "[bedrock] Sending {} messages, system_prompt={}, compact_summary={}, tools={}", messages.len(), system_prompt.is_some(), + compact_summary.is_some(), tools.len() ); @@ -94,6 +97,7 @@ pub async fn execute( needs_create_task, messages.clone(), system_prompt, + compact_summary, tools, 64000, None, diff --git a/app/src/ai/blocklist/controller.rs b/app/src/ai/blocklist/controller.rs index 655c7835..13ec52d5 100644 --- a/app/src/ai/blocklist/controller.rs +++ b/app/src/ai/blocklist/controller.rs @@ -1963,6 +1963,7 @@ impl BlocklistAIController { parent_agent_id, agent_name, bedrock_history, + bedrock_compact_summary, ) = { let Some(conversation) = history_model .as_ref(ctx) @@ -1986,6 +1987,7 @@ impl BlocklistAIController { conversation.parent_agent_id().map(str::to_string), conversation.agent_name().map(str::to_string), conversation.bedrock_message_history().to_vec(), + conversation.compact_summary().map(str::to_string), ) }; @@ -2053,6 +2055,7 @@ impl BlocklistAIController { request_params.parent_agent_id = parent_agent_id; request_params.agent_name = agent_name; request_params.bedrock_message_history = bedrock_history; + request_params.bedrock_compact_summary = bedrock_compact_summary; request_params.is_summarization = request_input .all_inputs() .any(|input| matches!(input, AIAgentInput::SummarizeConversation { .. })); @@ -2415,37 +2418,14 @@ impl BlocklistAIController { }); if let Some(summary) = summary_text { - use crate::ai::bedrock::convert::{ - ConversationMessage, MessageContent, - MessageRole, - }; - let assistant_reply = "Understood. I have the context from our previous conversation. How can I help you next?"; - let user_msg = format!( - "Here is a summary of our conversation so far:\n\n{summary}" - ); - let compacted = vec![ - ConversationMessage { - role: MessageRole::User, - content: MessageContent::Text(user_msg.clone()), - }, - ConversationMessage { - role: MessageRole::Assistant, - content: MessageContent::Text( - assistant_reply.to_string() - ), - }, - ]; log::info!( - "[bedrock] Compacted conversation history from {} messages to {} (summary)", - new_history.len(), - compacted.len() + "[bedrock] Compacted conversation history from {} messages to system-level summary", + new_history.len() ); - *conversation.bedrock_message_history_mut() = - compacted; + conversation.set_compact_summary(Some(summary.clone())); + *conversation.bedrock_message_history_mut() = Vec::new(); - // Estimate new context size from the compacted content. - // ~4 chars per token is a reasonable approximation. - let estimated_tokens = ((user_msg.len() + assistant_reply.len()) / 4) as u32; + let estimated_tokens = (summary.len() / 4) as u32; let max_context = crate::ai::bedrock::response_translator::context_window_for_model("claude-opus-4-6-20250514[1m]"); let new_usage = estimated_tokens as f32 / max_context as f32; conversation.set_context_window_usage(new_usage); diff --git a/app/src/ai/prompt_builder/mod.rs b/app/src/ai/prompt_builder/mod.rs index 01108fe5..f97219b7 100644 --- a/app/src/ai/prompt_builder/mod.rs +++ b/app/src/ai/prompt_builder/mod.rs @@ -23,7 +23,7 @@ mod tools; pub use context::PromptContext; pub use mode::Mode; pub use prompts::provider::Provider; -pub use tools::ToolSet; +pub use tools::tools_for_mode; use crate::ai::bedrock::convert::ToolDefinition;