public/galaxy
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Complete agent monitoring and Galaxy Control integration

Browse Source 
- expose command-monitor conversations and preserve visible agent transcripts
- add bounded polling and a dedicated shell interrupt tool
- improve direct-provider images, skills, tool history, and usage handling
- package and brand Galaxy Control across releases, installers, persistence, and docs
This commit is contained in:
rkw6086
2026-07-29 15:04:58 -05:00
parent 100f1eff1c
commit dbfa8bcd48
172 changed files with 6357 additions and 3825 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/src/ai/openai/convert.rs
+72 -17
View File
@@ -1,3 +1,5 @@
use base64::engine::general_purpose;
use base64::Engine as _;
use serde_json::{json, Value as JsonValue};
use crate::ai::provider::types::{
@@ -70,6 +72,11 @@ enum ConvertedMessages {
Multiple(Vec<JsonValue>),
}
enum UserContentPart {
Text(String),
Image { data: Vec<u8>, mime_type: String },
}
fn convert_message(msg: ConversationMessage) -> ConvertedMessages {
match msg.role {
MessageRole::User => convert_user_message(msg.content),
@@ -107,24 +114,22 @@ fn convert_user_message(content: MessageContent) -> ConvertedMessages {
}
MessageContent::MultiPart(parts) => {
let mut messages = Vec::new();
let mut text_parts: Vec<String> = Vec::new();
let mut user_content_parts = Vec::new();
for part in parts {
match part {
ContentPart::Text(text) => text_parts.push(text),
ContentPart::Text(text) => {
user_content_parts.push(UserContentPart::Text(text));
}
ContentPart::Image { data, mime_type } => {
user_content_parts.push(UserContentPart::Image { data, mime_type });
}
ContentPart::ToolResult {
tool_use_id,
content,
is_error,
} => {
// Flush any accumulated text as a user message first
if !text_parts.is_empty() {
messages.push(json!({
"role": "user",
"content": text_parts.join("\n"),
}));
text_parts.clear();
}
flush_user_content(&mut messages, &mut user_content_parts);
let result_content = if is_error {
format!("[ERROR] {content}")
} else {
@@ -137,17 +142,14 @@ fn convert_user_message(content: MessageContent) -> ConvertedMessages {
}));
}
ContentPart::ToolUse { .. } => {
text_parts.push("[unexpected tool_use in user message]".to_string());
user_content_parts.push(UserContentPart::Text(
"[unexpected tool_use in user message]".to_string(),
));
}
}
}
if !text_parts.is_empty() {
messages.push(json!({
"role": "user",
"content": text_parts.join("\n"),
}));
}
flush_user_content(&mut messages, &mut user_content_parts);
if messages.len() == 1 {
ConvertedMessages::Single(messages.into_iter().next().unwrap())
@@ -214,6 +216,12 @@ fn convert_assistant_message(content: MessageContent) -> ConvertedMessages {
}));
}
ContentPart::ToolResult { .. } => {}
ContentPart::Image { .. } => {
if !text_content.is_empty() {
text_content.push('\n');
}
text_content.push_str("[unexpected image in assistant message]");
}
}
}
@@ -232,6 +240,53 @@ fn convert_assistant_message(content: MessageContent) -> ConvertedMessages {
}
}
fn flush_user_content(messages: &mut Vec<JsonValue>, content_parts: &mut Vec<UserContentPart>) {
if content_parts.is_empty() {
return;
}
let has_image = content_parts
.iter()
.any(|part| matches!(part, UserContentPart::Image { .. }));
let content = if has_image {
JsonValue::Array(
std::mem::take(content_parts)
.into_iter()
.map(|part| match part {
UserContentPart::Text(text) => json!({
"type": "text",
"text": text,
}),
UserContentPart::Image { data, mime_type } => {
let data = general_purpose::STANDARD.encode(data);
json!({
"type": "image_url",
"image_url": {
"url": format!("data:{mime_type};base64,{data}"),
},
})
}
})
.collect(),
)
} else {
JsonValue::String(
std::mem::take(content_parts)
.into_iter()
.map(|part| match part {
UserContentPart::Text(text) => text,
UserContentPart::Image { .. } => unreachable!(),
})
.collect::<Vec<_>>()
.join("\n"),
)
};
messages.push(json!({
"role": "user",
"content": content,
}));
}
fn convert_tool_definition(tool: ToolDefinition) -> JsonValue {
json!({
"type": "function",
app/src/ai/openai/convert_tests.rs
+35
View File
@@ -23,6 +23,41 @@ fn test_simple_text_message_conversion() {
assert_eq!(request["stream"], true);
}
#[test]
fn test_multimodal_user_message_uses_openai_image_url_content() {
let messages = vec![ConversationMessage {
role: MessageRole::User,
content: MessageContent::MultiPart(vec![
ContentPart::Text("Describe this image".to_string()),
ContentPart::Image {
data: vec![1, 2, 3, 4],
mime_type: "image/png".to_string(),
},
]),
}];
let request = build_openai_request(messages, None, vec![], 1024, None, "test-model");
let content = request["messages"][0]["content"]
.as_array()
.expect("expected multimodal content array");
assert_eq!(
content,
&vec![
json!({
"type": "text",
"text": "Describe this image",
}),
json!({
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,AQIDBA==",
},
}),
]
);
}
#[test]
fn test_system_prompt_placement() {
let messages = vec![ConversationMessage {
app/src/ai/openai/mod.rs
+4
View File
@@ -11,3 +11,7 @@ mod convert_tests;
#[cfg(test)]
#[path = "request_translator_tests.rs"]
mod request_translator_tests;
#[cfg(test)]
#[path = "response_translator_tests.rs"]
mod response_translator_tests;
app/src/ai/openai/request_translator_tests.rs
+26
View File
@@ -197,3 +197,29 @@ fn test_ensure_ends_with_user_message_empty_messages() {
// Empty messages should stay empty
assert!(messages.is_empty());
}
#[test]
fn sanitizer_preserves_image_parts() {
let image_bytes = b"\x89PNG\r\n\x1a\nsanitizer".to_vec();
let mut messages = vec![ConversationMessage {
role: MessageRole::User,
content: MessageContent::MultiPart(vec![
ContentPart::Text("Describe this".to_string()),
ContentPart::Image {
data: image_bytes.clone(),
mime_type: "image/png".to_string(),
},
]),
}];
sanitize_messages_for_openai(&mut messages);
let MessageContent::MultiPart(parts) = &messages[0].content else {
panic!("expected multimodal message");
};
assert!(matches!(
&parts[1],
ContentPart::Image { data, mime_type }
if data == &image_bytes && mime_type == "image/png"
));
}
app/src/ai/openai/response_translator.rs
+142 -55
View File
@@ -10,7 +10,7 @@ use warp_multi_agent_api::{self as api, ClientAction, ResponseEvent};
use crate::ai::agent::api::Event;
use crate::ai::bedrock::response_translator::{
build_create_task, build_stream_init, context_window_for_model,
build_create_task, build_stream_init, context_window_for_model, recall_from_history,
};
use crate::ai::provider::types::{ContentPart, ConversationMessage, MessageContent, MessageRole};
use crate::server::server_api::AIApiError;
@@ -23,18 +23,41 @@ struct ToolCallAccumulator {
arguments: String,
}
pub fn openai_stream_to_response_events(
byte_stream: impl Stream<Item = Result<Bytes, reqwest::Error>> + Send + 'static,
task_id: String,
needs_create_task: bool,
user_query: Option<String>,
messages_sent: Arc<Mutex<Vec<ConversationMessage>>>,
pub struct OpenAIStreamContext {
pub task_id: String,
pub needs_create_task: bool,
pub user_query: Option<String>,
pub messages_sent: Arc<Mutex<Vec<ConversationMessage>>>,
pub model_id: String,
pub max_context_tokens: Option<u32>,
pub tool_result_archive: Vec<ConversationMessage>,
}
struct StreamUsage {
input_tokens: i32,
output_tokens: i32,
cache_read_tokens: i32,
cache_write_tokens: i32,
cost_in_cents: f32,
model_id: String,
max_context_tokens: Option<u32>,
_tool_result_archive: Vec<ConversationMessage>,
}
pub fn openai_stream_to_response_events(
byte_stream: impl Stream<Item = Result<Bytes, reqwest::Error>> + Send + 'static,
context: OpenAIStreamContext,
) -> BoxStream<'static, Event> {
use futures::StreamExt;
let OpenAIStreamContext {
task_id,
needs_create_task,
user_query,
messages_sent,
model_id,
max_context_tokens,
tool_result_archive,
} = context;
let request_id = Uuid::new_v4().to_string();
let conversation_id = Uuid::new_v4().to_string();
@@ -220,21 +243,80 @@ pub fn openai_stream_to_response_events(
if !full_text.is_empty() {
assistant_parts.push(ContentPart::Text(full_text.clone()));
}
let mut synthetic_tool_results: Vec<ContentPart> = Vec::new();
for tc in &tool_calls {
if tc.id.is_empty() || tc.name.is_empty() {
continue;
}
let event = build_tool_call_message(&task_id, &tc.id, &tc.name, &tc.arguments);
yield Ok(event);
let input: JsonValue = serde_json::from_str(&tc.arguments).unwrap_or(serde_json::json!({}));
assistant_parts.push(ContentPart::ToolUse {
tool_use_id: tc.id.clone(),
name: tc.name.clone(),
input,
input: input.clone(),
});
if tc.name == "recall_tool_history" {
log::info!("[openai] Handling recall_tool_history locally");
let search_query = input
.get("search_query")
.and_then(|value| value.as_str())
.unwrap_or("");
let tool_name_filter = input
.get("tool_name")
.and_then(|value| value.as_str())
.unwrap_or("");
let tool_use_id = input
.get("tool_use_id")
.and_then(|value| value.as_str())
.unwrap_or("");
let offset = input
.get("offset_from_end")
.and_then(|value| value.as_u64())
.unwrap_or(0) as usize;
let recall_result = match messages_sent.lock() {
Ok(sent) => recall_from_history(
&sent,
&tool_result_archive,
search_query,
tool_name_filter,
tool_use_id,
offset,
),
Err(_) => "Error: could not access conversation history.".to_string(),
};
synthetic_tool_results.push(ContentPart::ToolResult {
tool_use_id: tc.id.clone(),
content: recall_result,
is_error: false,
});
continue;
}
if !is_known_tool(&tc.name) {
log::warn!("[openai] Model called unknown tool: {}", tc.name);
let error_text = format!(
"Error: '{}' is not a valid tool. Please use one of the available tools.",
tc.name
);
synthetic_tool_results.push(ContentPart::ToolResult {
tool_use_id: tc.id.clone(),
content: error_text.clone(),
is_error: true,
});
let error_msg_id = Uuid::new_v4().to_string();
let error_display = format!("Failed tool call: `{}`\n\n{error_text}", tc.name);
yield Ok(build_add_agent_output_message(
&task_id,
&error_msg_id,
&error_display,
));
continue;
}
let event = build_tool_call_message(&task_id, &tc.id, &tc.name, &tc.arguments);
yield Ok(event);
}
// Store the complete assistant message in messages_sent
@@ -260,29 +342,33 @@ pub fn openai_stream_to_response_events(
if let Ok(mut sent) = messages_sent.lock() {
sent.push(assistant_msg);
}
}
// Emit hallucinated tool error results (tools the model called that aren't known)
for tc in &tool_calls {
if tc.id.is_empty() || tc.name.is_empty() {
continue;
}
if !is_known_tool(&tc.name) {
log::warn!("[openai] Model called unknown tool: {}", tc.name);
let error_result = ConversationMessage {
role: MessageRole::User,
content: MessageContent::ToolResult {
tool_use_id: tc.id.clone(),
content: format!(
"Error: '{}' is not a valid tool. Please use one of the available tools.",
tc.name
),
is_error: true,
},
};
if let Ok(mut sent) = messages_sent.lock() {
sent.push(error_result);
// Inline tools and rejected tool calls need immediate results so
// the next request never contains an unpaired tool use.
if !synthetic_tool_results.is_empty() {
let result_msg = if synthetic_tool_results.len() == 1 {
match synthetic_tool_results.remove(0) {
ContentPart::ToolResult {
tool_use_id,
content,
is_error,
} => ConversationMessage {
role: MessageRole::User,
content: MessageContent::ToolResult {
tool_use_id,
content,
is_error,
},
},
_ => unreachable!(),
}
} else {
ConversationMessage {
role: MessageRole::User,
content: MessageContent::MultiPart(synthetic_tool_results),
}
};
sent.push(result_msg);
}
}
}
@@ -302,13 +388,15 @@ pub fn openai_stream_to_response_events(
);
let finished_event = build_stream_finished(
stop_reason,
input_tokens,
output_tokens,
cache_read_tokens,
cache_write_tokens,
cost,
&model_id,
max_context_tokens,
StreamUsage {
input_tokens,
output_tokens,
cache_read_tokens,
cache_write_tokens,
cost_in_cents: cost,
model_id: model_id.clone(),
max_context_tokens,
},
);
yield Ok(finished_event);
@@ -445,16 +533,16 @@ fn build_tool_call_message(
)
}
fn build_stream_finished(
reason: stream_finished::Reason,
input_tokens: i32,
output_tokens: i32,
cache_read_tokens: i32,
cache_write_tokens: i32,
cost_in_cents: f32,
model_id: &str,
max_context_tokens: Option<u32>,
) -> ResponseEvent {
fn build_stream_finished(reason: stream_finished::Reason, usage: StreamUsage) -> ResponseEvent {
let StreamUsage {
input_tokens,
output_tokens,
cache_read_tokens,
cache_write_tokens,
cost_in_cents,
model_id,
max_context_tokens,
} = usage;
let total_tokens =
(input_tokens + output_tokens + cache_read_tokens + cache_write_tokens) as u32;
@@ -481,7 +569,7 @@ fn build_stream_finished(
}];
let max_context_tokens =
max_context_tokens.unwrap_or_else(|| context_window_for_model(model_id));
max_context_tokens.unwrap_or_else(|| context_window_for_model(&model_id));
// Context usage should reflect the full input including cached tokens
let effective_input = input_tokens + cache_read_tokens + cache_write_tokens;
let context_usage = if max_context_tokens > 0 {
@@ -571,6 +659,7 @@ const KNOWN_TOOLS: &[&str] = &[
"file_glob",
"search_codebase",
"write_to_long_running_shell_command",
"interrupt_shell_command",
"read_shell_command_output",
"transfer_shell_command_control_to_user",
"read_mcp_resource",
@@ -585,14 +674,12 @@ const KNOWN_TOOLS: &[&str] = &[
"create_documents",
"edit_documents",
"start_agent",
"send_message_to_agent",
"ask_user_question",
"suggest_next_prompt",
"read_skill",
"fetch_conversation",
"recall_tool_history",
];
fn is_known_tool(name: &str) -> bool {
pub(super) fn is_known_tool(name: &str) -> bool {
KNOWN_TOOLS.contains(&name) || name.starts_with("mcp__")
}
app/src/ai/openai/response_translator_tests.rs
+141
View File
@@ -0,0 +1,141 @@
use std::sync::{Arc, Mutex};
use bytes::Bytes;
use futures::{stream, StreamExt};
use serde_json::json;
use warp_multi_agent_api as api;
use super::response_translator::{
is_known_tool, openai_stream_to_response_events, OpenAIStreamContext,
};
use crate::ai::provider::types::{ConversationMessage, MessageContent, MessageRole};
async fn run_recall_tool_call() -> (Vec<api::ResponseEvent>, Vec<ConversationMessage>) {
let arguments = json!({"tool_use_id": "previous-tool-use"}).to_string();
let chunk = json!({
"choices": [{
"delta": {
"tool_calls": [{
"index": 0,
"id": "recall-tool-use",
"function": {
"name": "recall_tool_history",
"arguments": arguments,
},
}],
},
"finish_reason": "tool_calls",
}],
});
let sse = format!("data: {chunk}\n\ndata: [DONE]\n\n");
let byte_stream = stream::iter(vec![Ok::<Bytes, reqwest::Error>(Bytes::from(sse))]);
let messages_sent = Arc::new(Mutex::new(Vec::new()));
let archive = vec![
ConversationMessage {
role: MessageRole::Assistant,
content: MessageContent::ToolUse {
tool_use_id: "previous-tool-use".to_string(),
name: "run_shell_command".to_string(),
input: json!({"command": "cargo test"}),
},
},
ConversationMessage {
role: MessageRole::User,
content: MessageContent::ToolResult {
tool_use_id: "previous-tool-use".to_string(),
content: "all tests passed".to_string(),
is_error: false,
},
},
];
let events = openai_stream_to_response_events(
byte_stream,
OpenAIStreamContext {
task_id: "task-1".to_string(),
needs_create_task: false,
user_query: None,
messages_sent: messages_sent.clone(),
model_id: "test-model".to_string(),
max_context_tokens: Some(100_000),
tool_result_archive: archive,
},
)
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<Result<Vec<_>, _>>()
.expect("stream should succeed");
let history = messages_sent
.lock()
.expect("history lock should not be poisoned")
.clone();
(events, history)
}
fn has_tool_call(event: &api::ResponseEvent) -> bool {
let Some(api::response_event::Type::ClientActions(client_actions)) = &event.r#type else {
return false;
};
client_actions.actions.iter().any(|action| {
let Some(api::client_action::Action::AddMessagesToTask(add_messages)) = &action.action
else {
return false;
};
add_messages
.messages
.iter()
.any(|message| matches!(&message.message, Some(api::message::Message::ToolCall(_))))
})
}
#[tokio::test]
async fn recall_tool_history_uses_archive_and_stores_paired_result() {
let (_, history) = run_recall_tool_call().await;
assert_eq!(history.len(), 2);
let MessageContent::ToolUse {
tool_use_id,
name,
input,
} = &history[0].content
else {
panic!("expected assistant tool use");
};
assert_eq!(history[0].role, MessageRole::Assistant);
assert_eq!(tool_use_id, "recall-tool-use");
assert_eq!(name, "recall_tool_history");
assert_eq!(input, &json!({"tool_use_id": "previous-tool-use"}));
let MessageContent::ToolResult {
tool_use_id,
content,
is_error,
} = &history[1].content
else {
panic!("expected paired user tool result");
};
assert_eq!(history[1].role, MessageRole::User);
assert_eq!(tool_use_id, "recall-tool-use");
assert!(!is_error);
assert!(content.contains("Tool: run_shell_command"));
assert!(content.contains("Tool Use ID: previous-tool-use"));
assert!(content.contains("all tests passed"));
}
#[tokio::test]
async fn recall_tool_history_does_not_emit_a_client_tool_call() {
let (events, _) = run_recall_tool_call().await;
assert!(!events.iter().any(has_tool_call));
}
#[test]
fn direct_provider_known_tools_exclude_hosted_only_tools() {
assert!(!is_known_tool("send_message_to_agent"));
assert!(!is_known_tool("suggest_next_prompt"));
assert!(is_known_tool("recall_tool_history"));
assert!(is_known_tool("interrupt_shell_command"));
}
app/src/ai/openai/translator.rs
+10 -8
View File
@@ -5,7 +5,7 @@ use warp_multi_agent_api as api;
use super::client::{OpenAIClient, OpenAIClientConfig, OpenAIError};
use super::convert::build_openai_request;
use super::request_translator::sanitize_messages_for_openai;
use super::response_translator::openai_stream_to_response_events;
use super::response_translator::{openai_stream_to_response_events, OpenAIStreamContext};
use crate::ai::agent::api::ResponseStream;
use crate::ai::bedrock::request_translator;
use crate::ai::provider::types::{ConversationMessage, MessageContent, MessageRole};
@@ -149,13 +149,15 @@ pub async fn execute(
let stream = openai_stream_to_response_events(
byte_stream,
task_id,
needs_create_task,
user_query_text,
params.messages_sent.clone(),
model_id,
params.config.max_input_tokens,
params.tool_result_archive,
OpenAIStreamContext {
task_id,
needs_create_task,
user_query: user_query_text,
messages_sent: params.messages_sent.clone(),
model_id,
max_context_tokens: params.config.max_input_tokens,
tool_result_archive: params.tool_result_archive,
},
);
Ok(stream)