public/galaxy
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

first pass of merging in warp (doesn't build)

Browse Source
This commit is contained in:
Ryan Ward
2026-07-01 16:08:58 -05:00
parent 2f64909469
commit 4770ac06b5
3662 changed files with 414574 additions and 89772 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/src/ai/agent/api/convert_conversation.rs
+208 -62
View File
@@ -4,12 +4,28 @@
//! If some UI state is stored in the client, it needs to also be represented in the proto tasks somehow so it can be restored.
//! Some conversions may be lossy if it's not important to recover that UI state.
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use ai::agent::action_result::{
AskUserQuestionAnswerItem, AskUserQuestionResult, FetchConversationResult, ReadSkillResult,
RequestComputerUseResult, SendMessageToAgentResult, StartAgentResult, StartAgentVersion,
UseComputerResult,
};
use ai::skills::{ParsedSkill, SkillPathOrigin};
use chrono::{DateTime, Local, TimeZone};
use persistence::model::AgentConversationData;
use galaxy_core::command::ExitCode;
use warp_multi_agent_api as api;
use warp_multi_agent_api::ask_user_question_result::answer_item::Answer as AskUserQuestionAnswer;
use crate::ai::agent::api::convert_from::{
convert_user_query_mode, ConversionParams, ConvertAPIMessageToClientOutputMessage,
MaybeAIAgentOutputMessage,
};
use crate::ai::agent::conversation::update_todo_list_from_todo_op;
use crate::ai::agent::conversation::{AIConversation, AIConversationId};
use crate::ai::agent::conversation::{
update_todo_list_from_todo_op, AIConversation, AIConversationId, ServerAIConversationMetadata,
};
use crate::ai::agent::task::TaskId;
use crate::ai::agent::todos::AIAgentTodoList;
use crate::ai::agent::{
@@ -24,7 +40,7 @@ use crate::ai::agent::{
RequestFileEditsResult, SearchCodebaseFailureReason, SearchCodebaseResult, ServerOutputId,
Shared, ShellCommandCompletedTrigger, ShellCommandError, SuggestNewConversationResult,
SuggestPromptResult, TransferShellCommandControlToUserResult, UpdatedFileContext,
UploadArtifactResult, WriteToLongRunningShellCommandResult,
UploadArtifactResult, UserQueryMode, WriteToLongRunningShellCommandResult,
};
use crate::ai::block_context::BlockContext;
use crate::ai::document::ai_document_model::{AIDocumentId, AIDocumentVersion};
@@ -32,22 +48,6 @@ use crate::ai::llms::LLMId;
use crate::ai_assistant::execution_context::{WarpAiExecutionContext, WarpAiOsContext};
use crate::terminal::model::block::BlockId;
use crate::terminal::model::terminal_model::BlockIndex;
use ai::agent::action_result::{
AskUserQuestionAnswerItem, AskUserQuestionResult, FetchConversationResult, ReadSkillResult,
RequestComputerUseResult, SendMessageToAgentResult, StartAgentResult, StartAgentVersion,
UseComputerResult,
};
use ai::skills::ParsedSkill;
use chrono::{DateTime, Local, TimeZone};
use galaxy_core::command::ExitCode;
use persistence::model::AgentConversationData;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use warp_multi_agent_api as api;
use warp_multi_agent_api::ask_user_question_result::answer_item::Answer as AskUserQuestionAnswer;
use crate::ai::agent::conversation::ServerAIConversationMetadata;
use crate::ai::agent::UserQueryMode;
/// How to restore a conversation from the cloud.
pub enum RestorationMode {
@@ -81,12 +81,14 @@ pub fn convert_conversation_data_to_ai_conversation(
artifacts_json: None,
parent_agent_id: None,
agent_name: None,
orchestration_harness_type: None,
parent_conversation_id: None,
is_remote_child: false,
root_task_is_optimistic: None,
run_id: None,
autoexecute_override: None,
last_event_sequence: None,
progressive_summary: None,
messages_summarized_up_to: 0,
pinned: false,
},
RestorationMode::Continue => AgentConversationData {
server_conversation_token: Some(
@@ -98,16 +100,16 @@ pub fn convert_conversation_data_to_ai_conversation(
artifacts_json: serde_json::to_string(&metadata.artifacts).ok(),
parent_agent_id: None,
agent_name: None,
orchestration_harness_type: None,
parent_conversation_id: None,
// TODO: Populate run_id from server metadata once it is exposed
// in ServerAIConversationMetadata. For cloud conversations that
// were spawned via the server API, the run_id is created at task
// dispatch time; adding it here would avoid a round-trip to StreamInit.
run_id: None,
is_remote_child: false,
root_task_is_optimistic: None,
run_id: metadata
.ambient_agent_task_id
.map(|task_id| task_id.to_string()),
autoexecute_override: None,
last_event_sequence: None,
progressive_summary: None,
messages_summarized_up_to: 0,
pinned: false,
},
};
@@ -239,7 +241,8 @@ pub(crate) fn convert_input_context(context: Option<&api::InputContext>) -> Arc<
};
// Convert binary data to base64
use base64::{engine::general_purpose, Engine};
use base64::engine::general_purpose;
use base64::Engine;
let data = general_purpose::STANDARD.encode(&image.data);
result.push(AIAgentContext::Image(ImageContext {
@@ -436,7 +439,10 @@ impl ConvertToExchanges for &api::Task {
api::message::system_query::Type::ResumeConversation(_)
| api::message::system_query::Type::GeneratePassiveSuggestions(_)
// TODO: Implement this for real. ZB adding this to bump proto version for unrelated API changes.
| api::message::system_query::Type::SummarizeConversation(_)=> false,
| api::message::system_query::Type::SummarizeConversation(_)
// HandoffRehydration is injected by the server for agent-only
// context; the client must never render it as user input.
| api::message::system_query::Type::HandoffRehydration(_) => false,
}
}
api::message::Message::ToolCallResult(tool_call_result) => {
@@ -462,7 +468,10 @@ impl ConvertToExchanges for &api::Task {
}
api::message::Message::InvokeSkill(invoke_skill) => {
if let Some(api_skill) = invoke_skill.skill.clone() {
if let Ok(parsed_skill) = ParsedSkill::try_from(api_skill) {
if let Ok(parsed_skill) = ParsedSkill::try_from_api_with_origin(
api_skill,
&SkillPathOrigin::RestoredDisplayOnly,
) {
let user_query = invoke_skill
.user_query
.clone()
@@ -513,7 +522,8 @@ impl ConvertToExchanges for &api::Task {
| api::message::Message::DebugOutput(_)
| api::message::Message::ArtifactEvent(_)
| api::message::Message::MessagesReceivedFromAgents(_)
| api::message::Message::ModelUsed(_) => false,
| api::message::Message::ModelUsed(_)
| api::message::Message::OrchestrationConfigSnapshot(_) => false,
};
if !added_message_as_exchange_input {
@@ -524,6 +534,7 @@ impl ConvertToExchanges for &api::Task {
// TODO(alokedesai): Support persistence for the code review state.
active_code_review: None,
task_id: &TaskId::new(api_message.task_id.clone()),
skill_path_origin: &SkillPathOrigin::Unavailable,
})
{
current_outputs.push(output_msg);
@@ -581,6 +592,14 @@ pub(crate) fn convert_tool_call_result_to_input(
command: result.command.clone(),
output: finished.output.clone(),
exit_code: ExitCode::from(finished.exit_code),
start_ts: finished
.start_ts
.as_ref()
.map(|ts| proto_timestamp_to_local_datetime(ts.seconds, ts.nanos)),
completed_ts: finished
.finish_ts
.as_ref()
.map(|ts| proto_timestamp_to_local_datetime(ts.seconds, ts.nanos)),
}
}
Some(api::run_shell_command_result::Result::LongRunningCommandSnapshot(
@@ -627,6 +646,8 @@ pub(crate) fn convert_tool_call_result_to_input(
block_id: finished.command_id.clone().into(),
output: finished.output.clone(),
exit_code: ExitCode::from(finished.exit_code),
start_ts: finished.start_ts.as_ref().map(|ts| proto_timestamp_to_local_datetime(ts.seconds, ts.nanos)),
completed_ts: finished.finish_ts.as_ref().map(|ts| proto_timestamp_to_local_datetime(ts.seconds, ts.nanos)),
},
Some(api::write_to_long_running_shell_command_result::Result::Error(api::ShellCommandError{
r#type: Some(api::shell_command_error::Type::CommandNotFound(()))
@@ -1217,6 +1238,14 @@ pub(crate) fn convert_tool_call_result_to_input(
block_id: finished.command_id.clone().into(),
output: finished.output.clone(),
exit_code: ExitCode::from(finished.exit_code),
start_ts: finished
.start_ts
.as_ref()
.map(|ts| proto_timestamp_to_local_datetime(ts.seconds, ts.nanos)),
completed_ts: finished
.finish_ts
.as_ref()
.map(|ts| proto_timestamp_to_local_datetime(ts.seconds, ts.nanos)),
}
}
Some(
@@ -1267,6 +1296,8 @@ pub(crate) fn convert_tool_call_result_to_input(
block_id: finished.command_id.clone().into(),
output: finished.output.clone(),
exit_code: ExitCode::from(finished.exit_code),
start_ts: finished.start_ts.as_ref().map(|ts| proto_timestamp_to_local_datetime(ts.seconds, ts.nanos)),
completed_ts: finished.finish_ts.as_ref().map(|ts| proto_timestamp_to_local_datetime(ts.seconds, ts.nanos)),
},
Some(api::transfer_shell_command_control_to_user_result::Result::Error(
api::ShellCommandError {
@@ -1315,34 +1346,52 @@ pub(crate) fn convert_tool_call_result_to_input(
})
}
Some(ToolCallResultType::UseComputer(result)) => {
let use_computer_result = match &result.result {
Some(api::use_computer_result::Result::Success(success)) => {
let screenshot = success.screenshot.as_ref().map(|s| {
// The original dimensions are not preserved through the API, so we use
// the current dimensions for both.
computer_use::Screenshot {
width: s.width as usize,
height: s.height as usize,
original_width: s.width as usize,
original_height: s.height as usize,
data: s.data.clone(),
mime_type: s.mime_type.clone().into(),
}
});
let cursor_position = success
.cursor_position
.as_ref()
.map(|c| computer_use::Vector2I::new(c.x, c.y));
UseComputerResult::Success(computer_use::ActionResult {
screenshot,
cursor_position,
})
}
Some(api::use_computer_result::Result::Error(error)) => {
UseComputerResult::Error(error.message.clone())
}
None => UseComputerResult::Cancelled,
};
let use_computer_result =
match &result.result {
Some(api::use_computer_result::Result::Success(success)) => {
let screenshot = success.screenshot.as_ref().map(|s| {
// The original dimensions are not preserved through the API, so we use
// the current dimensions for both.
computer_use::Screenshot {
width: s.width as usize,
height: s.height as usize,
original_width: s.width as usize,
original_height: s.height as usize,
data: s.data.clone(),
mime_type: s.mime_type.clone().into(),
}
});
let cursor_position = success
.cursor_position
.as_ref()
.map(|c| computer_use::Vector2I::new(c.x, c.y));
let windows = success
.windows
.iter()
.map(convert_api_window_info)
.collect();
// A present captured-window message indicates a window screenshot was taken.
// The window id is an opaque string on the wire; on macOS it is a CGWindowID,
// so parse it back to a u32, defaulting to 0 when it is not parseable.
let captured_window = success.captured_window.as_ref().map(|c| {
computer_use::CapturedWindow {
window_id: c.window_id.parse().unwrap_or(0),
width_px: c.width_px,
height_px: c.height_px,
}
});
UseComputerResult::Success(computer_use::ActionResult {
screenshot,
cursor_position,
windows,
captured_window,
})
}
Some(api::use_computer_result::Result::Error(error)) => {
UseComputerResult::Error(error.message.clone())
}
None => UseComputerResult::Cancelled,
};
Some(AIAgentInput::ActionResult {
result: AIAgentActionResult {
@@ -1361,6 +1410,7 @@ pub(crate) fn convert_tool_call_result_to_input(
api::request_computer_use_result::Approved {
screen_dimensions: Some(screen_dimensions),
initial_screenshot: Some(initial_screenshot),
windows,
..
},
Some(platform),
@@ -1374,6 +1424,7 @@ pub(crate) fn convert_tool_call_result_to_input(
mime_type: initial_screenshot.mime_type.clone().into(),
},
platform,
windows: windows.iter().map(convert_api_window_info).collect(),
},
_ => RequestComputerUseResult::Error(
"Missing screen dimensions, initial screenshot, or valid platform"
@@ -1544,6 +1595,76 @@ pub(crate) fn convert_tool_call_result_to_input(
context,
})
}
Some(ToolCallResultType::RunAgentsResult(result)) => {
use ai::agent::action_result::{
RunAgentsAgentOutcome, RunAgentsAgentOutcomeKind, RunAgentsLaunchedExecutionMode,
RunAgentsResult,
};
let run_agents_result = match &result.outcome {
Some(api::run_agents_result::Outcome::Launched(launched)) => {
let execution_mode = match &launched.resolved_execution_mode {
Some(api::run_agents_result::launched::ResolvedExecutionMode::Remote(
remote,
)) => RunAgentsLaunchedExecutionMode::Remote {
environment_id: remote.environment_id.clone(),
worker_host: remote.worker_host.clone(),
computer_use_enabled: remote.computer_use_enabled,
},
Some(api::run_agents_result::launched::ResolvedExecutionMode::Local(_))
| None => RunAgentsLaunchedExecutionMode::Local,
};
let agents = launched
.agents
.iter()
.map(|outcome| RunAgentsAgentOutcome {
name: outcome.name.clone(),
kind: match &outcome.result {
Some(api::run_agents_result::agent_outcome::Result::Launched(
launched_agent,
)) => RunAgentsAgentOutcomeKind::Launched {
agent_id: launched_agent.agent_id.clone(),
},
Some(api::run_agents_result::agent_outcome::Result::Failed(
failed,
)) => RunAgentsAgentOutcomeKind::Failed {
error: failed.error.clone(),
},
None => RunAgentsAgentOutcomeKind::Failed {
error: String::new(),
},
},
})
.collect();
RunAgentsResult::Launched {
model_id: launched.resolved_model_id.clone(),
harness_type:
crate::ai::agent::api::convert_from::convert_run_agents_harness(
launched.resolved_harness.as_ref(),
)
.unwrap_or_default(),
execution_mode,
agents,
}
}
Some(api::run_agents_result::Outcome::Denied(denied)) => RunAgentsResult::Denied {
reason: denied.reason.clone(),
},
Some(api::run_agents_result::Outcome::Failure(failure)) => {
RunAgentsResult::Failure {
error: failure.error.clone(),
}
}
None => RunAgentsResult::Cancelled,
};
Some(AIAgentInput::ActionResult {
result: AIAgentActionResult {
id: tool_call_id.into(),
task_id: task_id.clone(),
result: AIAgentActionResultType::RunAgents(run_agents_result),
},
context,
})
}
// Deprecated/unused result types or absent result.
Some(ToolCallResultType::SuggestCreatePlan(..))
| Some(ToolCallResultType::SuggestPlan(..))
@@ -1551,6 +1672,7 @@ pub(crate) fn convert_tool_call_result_to_input(
log::warn!("No result present for tool call ID: {tool_call_id}");
None
}
Some(ToolCallResultType::WaitForEvents(_)) => None,
}
}
@@ -1678,8 +1800,14 @@ fn create_cancelled_result_for_tool_call(
ToolType::SendMessageToAgent(_) => {
AIAgentActionResultType::SendMessageToAgent(SendMessageToAgentResult::Cancelled)
}
ToolType::RunAgents(_) => {
AIAgentActionResultType::RunAgents(ai::agent::action_result::RunAgentsResult::Cancelled)
}
// These tools are deprecated.
ToolType::SuggestCreatePlan(_) | ToolType::SuggestPlan(_) => return None,
ToolType::WaitForEvents(_) => {
return None;
}
};
Some(AIAgentInput::ActionResult {
@@ -1777,6 +1905,10 @@ fn create_exchange_from_messages(
model_id: model.model_id.clone().into(),
display_name: model.model_display_name.clone(),
is_fallback: model.is_fallback,
prompt_cache_expires_at: model
.prompt_cache_expires_at
.as_ref()
.map(|ts| proto_timestamp_to_local_datetime(ts.seconds, ts.nanos)),
}),
request_cost: None,
};
@@ -1879,7 +2011,8 @@ where
| api::message::Message::DebugOutput(_)
| api::message::Message::ArtifactEvent(_)
| api::message::Message::InvokeSkill(_)
| api::message::Message::ModelUsed(_) => {
| api::message::Message::ModelUsed(_)
| api::message::Message::OrchestrationConfigSnapshot(_) => {
message.timestamp.as_ref().map(|timestamp| {
proto_timestamp_to_local_datetime(timestamp.seconds, timestamp.nanos)
})
@@ -1985,7 +2118,7 @@ fn convert_passive_suggestion_result_to_input(
context,
})
}
fn proto_timestamp_to_local_datetime(seconds: i64, nanos: i32) -> DateTime<Local> {
pub(crate) fn proto_timestamp_to_local_datetime(seconds: i64, nanos: i32) -> DateTime<Local> {
let nanos = if nanos < 0 { 0 } else { nanos as u32 };
Local
@@ -2014,6 +2147,19 @@ fn convert_api_platform(platform: i32) -> Option<computer_use::Platform> {
}
}
/// Reconstructs the internal computer_use window record from the API `WindowInfo` message.
fn convert_api_window_info(window: &api::WindowInfo) -> computer_use::WindowInfo {
computer_use::WindowInfo {
// The window id arrives as an opaque string; on macOS it is a CGWindowID (u32). Default to
// 0 when it is not parseable.
window_id: window.window_id.parse().unwrap_or(0),
pid: window.pid,
app_name: window.app_name.clone(),
title: window.title.clone(),
layer: window.layer,
}
}
#[cfg(test)]
#[path = "convert_conversation_tests.rs"]
mod tests;