public/galaxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first pass of merging in warp (doesn't build)

Browse Source
This commit is contained in:
Ryan Ward
2026-07-01 16:08:58 -05:00
parent 2f64909469
commit 4770ac06b5
3662 changed files with 414574 additions and 89772 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ai/src/agent/action/convert.rs
+61 -43
View File
@@ -1,26 +1,21 @@
use std::{path::PathBuf, time::Duration};
use std::path::PathBuf;
use std::time::Duration;
use galaxy_core::features::FeatureFlag;
use itertools::Itertools as _;
use uuid::Uuid;
use warp_multi_agent_api as api;
use crate::{
agent::{
action::{
AIAgentActionType, AIAgentPtyWriteMode, CommentSide, FileEdit, InsertReviewComment,
InsertedCommentLine, InsertedCommentLocation, ReadFilesRequest, ReadSkillRequest,
SearchCodebaseRequest, ShellCommandDelay, SuggestPromptRequest, UploadArtifactRequest,
UseComputerRequest,
},
action_result::{AnyFileContent, FileContext},
convert::ToolToAIAgentActionError,
FileLocations,
},
diff_validation::{ParsedDiff, V4AHunk},
document::AIDocumentId,
skills::SkillReference,
use crate::agent::action::{
AIAgentActionType, AIAgentPtyWriteMode, CommentSide, FileEdit, InsertReviewComment,
InsertedCommentLine, InsertedCommentLocation, ReadFilesRequest, SearchCodebaseRequest,
ShellCommandDelay, SuggestPromptRequest, UploadArtifactRequest, UseComputerRequest,
};
use crate::agent::action_result::{AnyFileContent, FileContext};
use crate::agent::convert::ToolToAIAgentActionError;
use crate::agent::FileLocations;
use crate::diff_validation::{ParsedDiff, V4AHunk};
use crate::document::AIDocumentId;
impl From<api::message::tool_call::RunShellCommand> for AIAgentActionType {
fn from(value: api::message::tool_call::RunShellCommand) -> Self {
@@ -430,10 +425,11 @@ impl TryFrom<api::message::tool_call::UseComputer> for AIAgentActionType {
.actions
.into_iter()
.map(|action| {
let target = convert_computer_use_target(action.target);
let Some(action_type) = action.r#type else {
return Err(ToolToAIAgentActionError::MissingComputerUseActionType);
};
match action_type {
let action = match action_type {
use_computer::action::Type::MouseMove(mouse_move) => {
Ok(computer_use::Action::MouseMove {
to: coordinates_to_vec(mouse_move.to.as_ref())?,
@@ -481,7 +477,9 @@ impl TryFrom<api::message::tool_call::UseComputer> for AIAgentActionType {
let key = convert_key(key_up.key)?;
Ok(computer_use::Action::KeyUp { key })
}
}
}?;
log_window_target_coord(&action, target);
Ok(computer_use::TargetedAction { action, target })
})
.try_collect()?;
let screenshot_params = value
@@ -505,19 +503,6 @@ impl From<api::message::tool_call::RequestComputerUse> for AIAgentActionType {
}
}
impl TryFrom<api::message::tool_call::ReadSkill> for AIAgentActionType {
type Error = ToolToAIAgentActionError;
fn try_from(value: api::message::tool_call::ReadSkill) -> Result<Self, Self::Error> {
match value.skill_reference {
Some(reference) => Ok(AIAgentActionType::ReadSkill(ReadSkillRequest {
skill: SkillReference::from(reference),
})),
None => Err(ToolToAIAgentActionError::MissingSkillReference),
}
}
}
impl From<api::message::tool_call::FetchConversation> for AIAgentActionType {
fn from(value: api::message::tool_call::FetchConversation) -> Self {
AIAgentActionType::FetchConversation {
@@ -526,18 +511,6 @@ impl From<api::message::tool_call::FetchConversation> for AIAgentActionType {
}
}
impl From<api::message::tool_call::read_skill::SkillReference> for SkillReference {
fn from(value: api::message::tool_call::read_skill::SkillReference) -> Self {
use warp_multi_agent_api::message::tool_call::read_skill::SkillReference as ApiSkillReference;
match value {
ApiSkillReference::SkillPath(skill_path) => {
SkillReference::Path(PathBuf::from(skill_path))
}
ApiSkillReference::BundledSkillId(id) => SkillReference::BundledSkillId(id),
}
}
}
/// Converts API ScreenshotParams to the internal computer_use type.
fn convert_screenshot_params(
params: api::message::tool_call::ScreenshotParams,
@@ -556,6 +529,51 @@ fn convert_screenshot_params(
max_long_edge_px: (params.max_long_edge_px > 0).then_some(params.max_long_edge_px as usize),
max_total_px: (params.max_total_px > 0).then_some(params.max_total_px as usize),
region,
target: convert_computer_use_target(params.target),
}
}
/// Converts an optional API `ComputerUseTarget` into the internal computer_use target. An absent
/// or `Screen` target maps to the legacy whole-screen behavior.
fn convert_computer_use_target(
target: Option<api::message::tool_call::ComputerUseTarget>,
) -> computer_use::Target {
use api::message::tool_call::computer_use_target::Target as ApiTarget;
match target.and_then(|t| t.target) {
// The proto window id is an opaque string; on macOS it is a CGWindowID, so parse it back
// to a u32. An unparseable id is treated as no valid window target and falls back to the
// legacy whole-screen behavior rather than panicking.
Some(ApiTarget::Window(window)) => match window.window_id.parse::<u32>() {
Ok(window_id) => computer_use::Target::Window {
window_id,
pid: window.pid,
},
Err(_) => computer_use::Target::Screen,
},
Some(ApiTarget::Screen(_)) | None => computer_use::Target::Screen,
}
}
/// Logs the raw server-provided coordinates for a window-targeted computer-use action, so the
/// agent-driven coordinate conversion can be compared against where the click actually lands.
/// Gated on COMPUTER_USE_DEBUG and routed through `log` so it surfaces in the app's log file.
fn log_window_target_coord(action: &computer_use::Action, target: computer_use::Target) {
if std::env::var_os("COMPUTER_USE_DEBUG").is_none() {
return;
}
let computer_use::Target::Window { window_id, pid } = target else {
return;
};
let coord = match action {
computer_use::Action::MouseMove { to } => Some(("mouse_move", to.x(), to.y())),
computer_use::Action::MouseDown { at, .. } => Some(("mouse_down", at.x(), at.y())),
computer_use::Action::MouseWheel { at, .. } => Some(("mouse_wheel", at.x(), at.y())),
_ => None,
};
if let Some((kind, x, y)) = coord {
log::info!(
"[computer_use] server->client {kind} window#={window_id} pid={pid} raw_coord=({x},{y})"
);
}
}
crates/ai/src/agent/action/mod.rs
+129 -25
View File
@@ -1,34 +1,32 @@
mod convert;
use std::{fmt::Display, ops::Range, path::PathBuf, time::Duration};
use std::fmt::Display;
use std::ops::Range;
use std::path::PathBuf;
use std::time::Duration;
use galaxy_terminal::model::BlockId;
use itertools::Itertools as _;
use serde::{Deserialize, Serialize};
use strum_macros::EnumDiscriminants;
use uuid::Uuid;
use crate::{
agent::{
action_result::{
AIAgentActionResultType, AskUserQuestionResult, CallMCPToolResult,
CreateDocumentsResult, EditDocumentsResult, FetchConversationResult, FileGlobResult,
FileGlobV2Result, GrepResult, InsertReviewCommentsResult, ReadDocumentsResult,
ReadFilesResult, ReadMCPResourceResult, ReadShellCommandOutputResult, ReadSkillResult,
RequestCommandOutputResult, RequestComputerUseResult, RequestFileEditsResult,
SearchCodebaseResult, SendMessageToAgentResult, StartAgentResult, StartAgentVersion,
SuggestNewConversationResult, SuggestPromptResult,
TransferShellCommandControlToUserResult, UploadArtifactResult, UseComputerResult,
WriteToLongRunningShellCommandResult,
},
AIAgentCitation, FileLocations,
},
diff_validation::ParsedDiff,
document::AIDocumentId,
skills::SkillReference,
};
pub use warp_multi_agent_api::LifecycleEventType;
use crate::agent::action_result::{
AIAgentActionResultType, AskUserQuestionResult, CallMCPToolResult, CreateDocumentsResult,
EditDocumentsResult, FetchConversationResult, FileGlobResult, FileGlobV2Result, GrepResult,
InsertReviewCommentsResult, ReadDocumentsResult, ReadFilesResult, ReadMCPResourceResult,
ReadShellCommandOutputResult, ReadSkillResult, RequestCommandOutputResult,
RequestComputerUseResult, RequestFileEditsResult, RunAgentsResult, SearchCodebaseResult,
SendMessageToAgentResult, StartAgentResult, StartAgentVersion, SuggestNewConversationResult,
SuggestPromptResult, TransferShellCommandControlToUserResult, UploadArtifactResult,
UseComputerResult, WaitForEventsResult, WriteToLongRunningShellCommandResult,
};
use crate::agent::{AIAgentCitation, FileLocations};
use crate::diff_validation::ParsedDiff;
use crate::document::AIDocumentId;
use crate::skills::SkillReference;
#[derive(Debug, Clone, Eq, PartialEq, EnumDiscriminants)]
pub enum AIAgentActionType {
/// The AI requested the output for a given command to be retrieved as context in responding to
@@ -48,7 +46,7 @@ pub enum AIAgentActionType {
/// result instead.
wait_until_completion: bool,
/// [`Some(true)`] iff the LLM thinks that the `command` might invoke pager.
/// [`Some(true)`] iff the LLM thinks that the `command` might invoke a pager.
uses_pager: Option<bool>,
/// The AI's rationale for requesting a command.
@@ -144,7 +142,9 @@ pub enum AIAgentActionType {
FetchConversation {
conversation_id: String,
},
// TODO(QUALITY-788): Delete legacy start_agent/start_agent_v2 action support once
// old preview orchestration history no longer needs parse/display/result compatibility.
// Linear issue: QUALITY-788.
StartAgent {
version: StartAgentVersion,
name: String,
@@ -167,6 +167,68 @@ pub enum AIAgentActionType {
AskUserQuestion {
questions: Vec<AskUserQuestionItem>,
},
/// AI requested batched orchestration of one-or-more child agents that
/// share run-wide configuration (model, harness, execution mode).
/// The full per-child prompt is computed at dispatch time as
/// `base_prompt + "\n\n" + agent_run_configs[i].prompt` (or just
/// `base_prompt` when the per-agent `prompt` is empty).
RunAgents(RunAgentsRequest),
/// Synthesized from a server-emitted Message::ToolCall::WaitForEvents;
/// dispatched by WaitForEventsExecutor.
WaitForEvents {
/// tool_call_id of the unresolved WaitForEvents call; used to
/// match inbound resume signals.
tool_call_id: String,
/// 0 means "unset" (prost flat-scalar convention); the executor
/// falls back to a default.
idle_timeout_seconds: i32,
},
}
/// Run-wide + per-agent configuration for a `RunAgents` tool call.
///
/// Mirrors the proto `RunAgents` message. Server-resolved fields
/// (`model_id`, `harness_type`, `execution_mode`'s remote details) are
/// folded in by the server's final tool-call re-emission once the
/// payload is complete; the client renders the full layout from a
/// fully-resolved instance only.
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct RunAgentsRequest {
pub summary: String,
pub base_prompt: String,
pub skills: Vec<SkillReference>,
pub model_id: String,
pub harness_type: String,
pub execution_mode: RunAgentsExecutionMode,
pub agent_run_configs: Vec<RunAgentsAgentRunConfig>,
pub plan_id: String,
/// Resolved client-side at dispatch time; not serialized to the wire.
pub harness_auth_secret_name: Option<String>,
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum RunAgentsExecutionMode {
Local,
Remote {
environment_id: String,
worker_host: String,
computer_use_enabled: bool,
},
}
impl RunAgentsExecutionMode {
pub fn is_remote(&self) -> bool {
matches!(self, Self::Remote { .. })
}
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct RunAgentsAgentRunConfig {
pub name: String,
pub prompt: String,
pub title: String,
}
#[derive(Debug, Clone, Eq, PartialEq)]
@@ -175,6 +237,11 @@ pub enum StartAgentExecutionMode {
/// `None` selects the legacy embedded local child-agent flow.
/// `Some(...)` selects a third-party CLI harness to launch locally.
harness_type: Option<String>,
/// `None` inherits the parent agent's preferred LLM (legacy behavior).
/// `Some(_)` overrides the child's preferred LLM with the supplied
/// model id (used by the orchestrate confirmation card so the user's
/// model selection is honored on local launches).
model_id: Option<String>,
},
Remote {
environment_id: String,
@@ -184,18 +251,27 @@ pub enum StartAgentExecutionMode {
worker_host: String,
harness_type: String,
title: String,
/// Name of a managed secret to forward as the authentication
/// credential for the remote child when running a non-Oz harness.
/// `None` means no client-side secret was selected — the remote
/// environment falls back to its own ambient credentials.
auth_secret_name: Option<String>,
},
}
impl StartAgentExecutionMode {
/// Constructs a local execution mode using the legacy v1 default harness.
pub fn local_with_defaults() -> Self {
Self::Local { harness_type: None }
Self::Local {
harness_type: None,
model_id: None,
}
}
/// Constructs a local execution mode for a specific third-party harness.
pub fn local_harness(harness_type: String) -> Self {
Self::Local {
harness_type: Some(harness_type),
model_id: None,
}
}
/// Constructs a remote execution mode using the legacy v1 defaults for
@@ -209,6 +285,7 @@ impl StartAgentExecutionMode {
worker_host: String::new(),
harness_type: String::new(),
title: String::new(),
auth_secret_name: None,
}
}
}
@@ -317,6 +394,10 @@ impl AIAgentActionType {
Self::AskUserQuestion { .. } => {
AIAgentActionResultType::AskUserQuestion(AskUserQuestionResult::Cancelled)
}
Self::RunAgents(_) => AIAgentActionResultType::RunAgents(RunAgentsResult::Cancelled),
Self::WaitForEvents { .. } => {
AIAgentActionResultType::WaitForEvents(WaitForEventsResult::Cancelled)
}
}
}
@@ -362,6 +443,10 @@ impl AIAgentActionType {
Self::AskUserQuestion { questions } => {
format!("Ask user {} question(s)", questions.len())
}
Self::RunAgents(req) => {
format!("Orchestrate {} agent(s)", req.agent_run_configs.len())
}
Self::WaitForEvents { .. } => "Wait for events".to_string(),
}
}
}
@@ -534,6 +619,24 @@ impl Display for AIAgentActionType {
AIAgentActionType::AskUserQuestion { questions } => {
write!(f, "AskUserQuestion: {} question(s)", questions.len())
}
AIAgentActionType::RunAgents(req) => {
let names = req
.agent_run_configs
.iter()
.map(|c| c.name.as_str())
.collect::<Vec<_>>()
.join(", ");
write!(f, "Orchestrate: summary='{}' agents=[{names}]", req.summary,)
}
AIAgentActionType::WaitForEvents {
tool_call_id,
idle_timeout_seconds,
} => {
write!(
f,
"WaitForEvents: tool_call_id={tool_call_id} idle_timeout_seconds={idle_timeout_seconds}"
)
}
}
}
}
@@ -665,7 +768,8 @@ pub struct CreateDocumentsRequest {
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct UseComputerRequest {
pub action_summary: String,
pub actions: Vec<computer_use::Action>,
/// Each action carries the surface (screen or a specific window) it targets.
pub actions: Vec<computer_use::TargetedAction>,
/// If set, a screenshot will be captured after the actions are executed.
pub screenshot_params: Option<computer_use::ScreenshotParams>,
}