public/galaxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run agent shell commands as independent background jobs

Browse Source
This commit is contained in:
Ryan Ward
2026-10-28 17:47:42 +00:00
parent 51782350f1
commit 3cce041a6d
52 changed files with 4536 additions and 681 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ai/src/agent/action_result/convert.rs
+39 -4
View File
@@ -6,7 +6,6 @@ use warp_multi_agent_api::ask_user_question_result::answer_item::{
use warp_multi_agent_api::{self as api};
use super::*;
use crate::agent::action_result::ShellCommandError;
use crate::agent::convert::ConvertToAPITypeError;
fn local_datetime_to_timestamp(timestamp: DateTime<Local>) -> prost_types::Timestamp {
@@ -21,6 +20,27 @@ impl TryFrom<RequestCommandOutputResult> for api::request::input::tool_call_resu
fn try_from(result: RequestCommandOutputResult) -> Result<Self, Self::Error> {
match result {
RequestCommandOutputResult::BackgroundJobStarted { job_id, command } => Ok(
api::request::input::tool_call_result::Result::RunShellCommand(
#[allow(deprecated)]
api::RunShellCommandResult {
command: command.clone(),
output: Default::default(),
exit_code: Default::default(),
result: Some(
api::run_shell_command_result::Result::LongRunningCommandSnapshot(
api::LongRunningShellCommandSnapshot {
command_id: format!("background-job:{job_id}"),
output: background_job_started_content(&job_id, &command),
cursor: String::new(),
is_alt_screen_active: false,
is_preempted: false,
},
),
),
},
),
),
RequestCommandOutputResult::Completed {
command,
block_id,
@@ -144,6 +164,21 @@ impl TryFrom<WriteToLongRunningShellCommandResult>
fn try_from(result: WriteToLongRunningShellCommandResult) -> Result<Self, Self::Error> {
match result {
WriteToLongRunningShellCommandResult::BackgroundJobCancellationRequested { job_id } => Ok(
api::request::input::tool_call_result::Result::WriteToLongRunningShellCommand(
api::WriteToLongRunningShellCommandResult {
result: Some(api::write_to_long_running_shell_command_result::Result::LongRunningCommandSnapshot(
api::LongRunningShellCommandSnapshot {
command_id: format!("background-job:{job_id}"),
output: format!("Cancellation requested for background job {job_id}. Completion will arrive separately."),
cursor: String::new(),
is_alt_screen_active: false,
is_preempted: false,
}
)),
},
)
),
WriteToLongRunningShellCommandResult::Snapshot { block_id, grid_contents, cursor, is_alt_screen_active, is_preempted } => Ok(
api::request::input::tool_call_result::Result::WriteToLongRunningShellCommand(
api::WriteToLongRunningShellCommandResult {
@@ -191,7 +226,7 @@ impl TryFrom<WriteToLongRunningShellCommandResult>
),
)
}
WriteToLongRunningShellCommandResult::Error(ShellCommandError::BlockNotFound) => {
WriteToLongRunningShellCommandResult::Error(_) => {
Ok(api::request::input::tool_call_result::Result::WriteToLongRunningShellCommand(
api::WriteToLongRunningShellCommandResult {
result: Some(
@@ -881,7 +916,7 @@ impl TryFrom<ReadShellCommandOutputResult> for api::request::input::tool_call_re
},
),
),
ReadShellCommandOutputResult::Error(ShellCommandError::BlockNotFound) => {
ReadShellCommandOutputResult::Error(_) => {
Ok(api::request::input::tool_call_result::Result::ReadShellCommandOutput(
api::ReadShellCommandOutputResult {
command: "".to_owned(),
@@ -965,7 +1000,7 @@ impl TryFrom<TransferShellCommandControlToUserResult>
},
),
),
TransferShellCommandControlToUserResult::Error(ShellCommandError::BlockNotFound) => {
TransferShellCommandControlToUserResult::Error(_) => {
Ok(api::request::input::tool_call_result::Result::TransferShellCommandControlToUser(
api::TransferShellCommandControlToUserResult {
result: Some(
crates/ai/src/agent/action_result/mod.rs
+35 -3
View File
@@ -126,6 +126,9 @@ impl AIAgentActionResultType {
| AIAgentActionResultType::RequestCommandOutput(
RequestCommandOutputResult::LongRunningCommandSnapshot { command, .. },
)
| AIAgentActionResultType::RequestCommandOutput(
RequestCommandOutputResult::BackgroundJobStarted { command, .. },
)
| AIAgentActionResultType::ReadShellCommandOutput(
ReadShellCommandOutputResult::CommandFinished { command, .. },
)
@@ -143,6 +146,9 @@ impl AIAgentActionResultType {
pub fn model_content(&self) -> String {
match self {
Self::RequestCommandOutput(result) => match result {
RequestCommandOutputResult::BackgroundJobStarted { job_id, command } => {
background_job_started_content(job_id, command)
}
RequestCommandOutputResult::Completed {
command,
output,
@@ -188,6 +194,7 @@ impl AIAgentActionResultType {
output, exit_code, ..
} => command_result_content(None, output, exit_code.value()),
WriteToLongRunningShellCommandResult::Cancelled
| WriteToLongRunningShellCommandResult::BackgroundJobCancellationRequested { .. }
| WriteToLongRunningShellCommandResult::Error(_) => result.to_string(),
},
Self::ReadFiles(result) => match result {
@@ -359,7 +366,7 @@ fn shell_snapshot_content(
.map(|is_preempted| format!("\nPreempted: {is_preempted}"))
.unwrap_or_default();
format!(
"{command}Command is still running.\nCommand ID: {block_id}\nCurrent output:\n{grid_contents}\nCursor: {cursor}\nAlt screen active: {is_alt_screen_active}{preempted}\nThe next assistant output MUST be a tool call: continue monitoring with `read_shell_command_output` using command_id `{block_id}` and a short wait. If the user's explicit stop condition is met, call `interrupt_shell_command` immediately. Do not end this turn with prose or report completion while the command is still running."
"{command}Command is still running.\nCommand ID: {block_id}\nCurrent output:\n{grid_contents}\nCursor: {cursor}\nAlt screen active: {is_alt_screen_active}{preempted}\nThis is a legacy terminal snapshot, not a new completion. Interactive terminal tools and command monitoring are retired. Do not poll or send process input; new commands use independent noninteractive background jobs."
)
}
@@ -432,6 +439,9 @@ impl Display for AIAgentActionResultType {
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum RequestCommandOutputResult {
/// The command was accepted by the independent noninteractive job runner.
/// Completion arrives later as a separate conversation event.
BackgroundJobStarted { job_id: String, command: String },
Completed {
block_id: BlockId,
command: String,
@@ -460,7 +470,7 @@ impl RequestCommandOutputResult {
pub fn is_successful(&self) -> bool {
match self {
Self::Completed { exit_code, .. } => exit_code.was_successful(),
Self::LongRunningCommandSnapshot { .. } => true,
Self::LongRunningCommandSnapshot { .. } | Self::BackgroundJobStarted { .. } => true,
Self::CancelledBeforeExecution
| Self::ExecutionError { .. }
| Self::Denylisted { .. } => false,
@@ -471,7 +481,9 @@ impl RequestCommandOutputResult {
match self {
Self::Completed { exit_code, .. } => !exit_code.was_successful(),
Self::ExecutionError { .. } | Self::Denylisted { .. } => true,
Self::CancelledBeforeExecution | Self::LongRunningCommandSnapshot { .. } => false,
Self::CancelledBeforeExecution
| Self::LongRunningCommandSnapshot { .. }
| Self::BackgroundJobStarted { .. } => false,
}
}
@@ -479,6 +491,7 @@ impl RequestCommandOutputResult {
pub fn loop_description(&self) -> String {
match self {
Self::Completed { command, .. }
| Self::BackgroundJobStarted { command, .. }
| Self::LongRunningCommandSnapshot { command, .. }
| Self::ExecutionError { command, .. }
| Self::Denylisted { command } => command.clone(),
@@ -490,6 +503,9 @@ impl RequestCommandOutputResult {
impl Display for RequestCommandOutputResult {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
RequestCommandOutputResult::BackgroundJobStarted { job_id, command } => {
write!(f, "Background job {job_id} started: {command}")
}
RequestCommandOutputResult::Completed {
command,
output,
@@ -520,13 +536,25 @@ impl Display for RequestCommandOutputResult {
}
}
fn background_job_started_content(job_id: &str, command: &str) -> String {
serde_json::json!({
"background_job_started": { "job_id": job_id, "command": command },
"message": "The command is running independently in the background. This is a launch acknowledgment, not a successful exit. Do not monitor, poll, or interact with it. A completion message will provide the command, logs, and exit status. Wait for that message before starting dependent work; the conversation remains available for further user instructions."
}).to_string()
}
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
pub enum ShellCommandError {
BlockNotFound,
NonInteractiveOnly,
BackgroundJobError(String),
}
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
pub enum WriteToLongRunningShellCommandResult {
BackgroundJobCancellationRequested {
job_id: String,
},
Snapshot {
block_id: BlockId,
grid_contents: String,
@@ -548,6 +576,7 @@ pub enum WriteToLongRunningShellCommandResult {
impl Display for WriteToLongRunningShellCommandResult {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::BackgroundJobCancellationRequested { job_id } => write!(f, "Cancellation requested for background job {job_id}. Its final status and logs will arrive separately."),
Self::Snapshot { .. } => {
write!(f, "Sent snapshot of long-running shell command to agent")
}
@@ -1054,6 +1083,9 @@ impl AIAgentActionResultType {
pub fn is_successful(&self) -> bool {
match self {
Self::RequestCommandOutput(r) => r.is_successful(),
Self::WriteToLongRunningShellCommand(
WriteToLongRunningShellCommandResult::BackgroundJobCancellationRequested { .. },
) => true,
Self::RequestFileEdits(RequestFileEditsResult::Success { .. })
| Self::ReadFiles(ReadFilesResult::Success { .. })
| Self::UploadArtifact(UploadArtifactResult::Success { .. })
crates/command/src/async.rs
+13
View File
@@ -115,6 +115,19 @@ impl Command {
}
}
/// Sets Windows process creation flags while preserving the wrapper's default flags.
#[cfg(windows)]
pub fn creation_flags(&mut self, flags: u32) -> &mut Self {
use async_process::windows::CommandExt;
#[cfg(not(feature = "test-util"))]
let flags = flags
| windows::Win32::System::Threading::CREATE_NO_WINDOW.0
| windows::Win32::System::Threading::CREATE_BREAKAWAY_FROM_JOB.0;
self.inner.creation_flags(flags);
self
}
/// Adds a single argument to pass to the program.
///
/// # Examples
crates/galaxy_agent_core/src/provider_run.rs
+27 -8
View File
@@ -1320,19 +1320,20 @@ impl ProviderRun {
})?);
}
}
let yielded_to_children = batch.calls.iter().any(|pending| {
is_local_agent_launch(&pending.call)
&& pending
.state
.result()
.is_some_and(|result| result.status == ToolResultStatus::Success)
let yielded_to_external_work = batch.calls.iter().any(|pending| {
pending.state.result().is_some_and(|result| {
result.status == ToolResultStatus::Success
&& (is_local_agent_launch(&pending.call)
|| is_background_job_launch(&pending.call, result))
})
});
self.transcript.push(tool_result_message(results));
self.task_list = task_list;
self.epoch = next_epoch;
// Launch acknowledgement is a yield, not an invitation to issue more tools.
// The client persists child membership and supplies a new input once the group finishes.
self.state = if yielded_to_children {
// The client owns the independent child/job lifecycle and supplies a new input once
// that work finishes. Every result in this batch is committed before yielding.
self.state = if yielded_to_external_work {
ProviderRunState::Done {
completion: ProviderRunCompletion {
stop_reason: StopReason::Completed,
@@ -1736,6 +1737,24 @@ fn is_local_agent_launch(call: &ToolCall) -> bool {
matches!(call.name.as_str(), "run_agents" | "start_agent")
}
fn is_background_job_launch(call: &ToolCall, result: &ToolResult) -> bool {
if call.name != "run_shell_command" {
return false;
}
let Ok(content) = serde_json::from_str::<serde_json::Value>(&result.content) else {
return false;
};
content.get("background_job_started").is_some_and(|job| {
job.get("job_id")
.and_then(serde_json::Value::as_str)
.is_some_and(|id| !id.is_empty())
&& job
.get("command")
.and_then(serde_json::Value::as_str)
.is_some()
})
}
fn validate_model_turn(turn: &CompletedModelTurn) -> Result<(), ProviderRunProtocolError> {
for part in &turn.assistant_content {
match part {
crates/galaxy_agent_core/src/provider_run_tests.rs
+90
View File
@@ -1850,3 +1850,93 @@ fn failed_child_launch_allows_a_corrective_model_turn() {
run.commit_tool_batch(&batch.work_id).unwrap();
assert!(matches!(run.state(), ProviderRunState::ReadyToCallModel));
}
#[test]
fn background_job_launch_yields_after_committing_the_complete_tool_batch() {
let mut run = run();
let batch = accept_tool_turn(
&mut run,
tool_turn(
vec![
tool_call("build", "run_shell_command"),
tool_call("inspect", "read_files"),
],
&["run_shell_command", "read_files"],
),
);
assert!(
batch
.calls
.iter()
.all(|pending| matches!(pending.state, PendingToolCallState::Proposed))
);
run.complete_tool(
&batch.work_id,
successful_result(
"build",
&json!({
"background_job_started": { "job_id": "job-1", "command": "cargo build" },
})
.to_string(),
),
)
.unwrap();
assert!(run.commit_tool_batch(&batch.work_id).is_err());
run.complete_tool(
&batch.work_id,
successful_result("inspect", "file contents"),
)
.unwrap();
run.commit_tool_batch(&batch.work_id).unwrap();
assert!(matches!(run.state(), ProviderRunState::Done { .. }));
assert_serialization_round_trip(&run);
let MessageContent::MultiPart(parts) = &run.transcript().last().unwrap().content else {
panic!("the batch should remain in provider history");
};
assert_eq!(
parts
.iter()
.filter(|part| matches!(part, ContentPart::ToolResult { .. }))
.count(),
2
);
}
#[test]
fn failed_or_legacy_shell_results_allow_the_normal_corrective_turn() {
for (status, content) in [
(
ToolResultStatus::Error,
json!({"background_job_started": {"job_id": "failed", "command": "build"}}).to_string(),
),
(
ToolResultStatus::Success,
"legacy terminal snapshot".to_owned(),
),
(
ToolResultStatus::Success,
json!({"output": "background_job_started", "exit_code": 0}).to_string(),
),
] {
let mut run = run();
let batch = accept_tool_turn(
&mut run,
tool_turn(
vec![tool_call("shell", "run_shell_command")],
&["run_shell_command"],
),
);
run.complete_tool(
&batch.work_id,
ToolResult {
call_id: "shell".to_owned(),
content,
status,
images: Vec::new(),
},
)
.unwrap();
run.commit_tool_batch(&batch.work_id).unwrap();
assert!(matches!(run.state(), ProviderRunState::ReadyToCallModel));
}
}
crates/remote_server/proto/remote_server.proto
+38
View File
@@ -49,6 +49,7 @@ message HostScopedRequest {
GitGenerateCommitMessageRequest git_generate_commit_message = 18;
GitGetCommittedBranchFilesRequest git_get_committed_branch_files = 19;
RipgrepSearchRequest ripgrep_search = 21;
BackgroundJobRequest background_job = 22;
}
}
@@ -125,6 +126,7 @@ message ServerMessage {
GitHubRepositoryInfoPush github_repository_info_push = 36;
RipgrepSearchResponse ripgrep_search_response = 37;
RemoteAgentContextSnapshot remote_agent_context_snapshot = 33;
BackgroundJobResponse background_job_response = 38;
}
}
@@ -177,6 +179,42 @@ message Abort {
message InitializeResponse {
string server_version = 1;
string host_id = 2;
bool supports_background_jobs = 3;
}
// Agent commands run independently of the requesting connection. Repeated starts with the
// same conversation/action identity return the existing job rather than running it again.
message BackgroundJobRequest {
enum Operation {
START = 0;
STATUS = 1;
CANCEL = 2;
}
Operation operation = 1;
string job_id = 2;
string conversation_id = 3;
string action_id = 4;
string command = 5;
string working_directory = 6;
map<string, string> environment = 7;
uint64 session_id = 8;
uint64 log_offset = 9;
// Distinguish an absent shell snapshot from a known, explicitly empty environment.
bool has_environment = 10;
}
message BackgroundJobResponse {
string job_json = 1;
bytes log_chunk = 2;
uint64 next_log_offset = 3;
uint64 total_log_bytes = 4;
bool logs_available = 5;
string error = 6;
// Bounded completion evidence survives unlinking the daemon's temporary log file.
bytes completion_log_tail = 7;
uint64 completion_log_bytes = 8;
bool completion_log_tail_available = 9;
bool job_not_found = 10;
}
// Sent by the client after the SSH session has been bootstrapped.
crates/remote_server/src/client/mod.rs
+11 -1
View File
@@ -201,6 +201,7 @@ pub struct RemoteServerClient {
/// `send_request` after inserting into `pending_requests` to avoid hanging
/// on a dead connection.
disconnected: Arc<AtomicBool>,
background_jobs_supported: AtomicBool,
/// Dedicated channel for `RequestFailed` telemetry events. Separate from
/// the lifecycle `event_tx` so that holding this sender does not keep the
@@ -313,6 +314,7 @@ impl RemoteServerClient {
outbound_tx,
pending_requests,
disconnected,
background_jobs_supported: AtomicBool::new(false),
failure_tx,
host_response_tx,
},
@@ -336,6 +338,10 @@ impl RemoteServerClient {
self.disconnected.load(Ordering::Acquire)
}
pub fn supports_background_jobs(&self) -> bool {
self.background_jobs_supported.load(Ordering::Acquire)
}
/// Sends an `Initialize` request and awaits the `InitializeResponse`.
pub async fn initialize(
&self,
@@ -357,7 +363,11 @@ impl RemoteServerClient {
let response = self.send_request_internal(request_id, msg).await?;
match response.message {
Some(server_message::Message::InitializeResponse(resp)) => Ok(resp),
Some(server_message::Message::InitializeResponse(resp)) => {
self.background_jobs_supported
.store(resp.supports_background_jobs, Ordering::Release);
Ok(resp)
}
other => {
safe_error!(
safe: ("Remote server unexpected response for Initialize"),
crates/remote_server/src/client_tests.rs
+9
View File
@@ -214,8 +214,10 @@ async fn initialize_round_trip() {
server_message::Message::InitializeResponse(InitializeResponse {
server_version: "test-0.1.0".to_string(),
host_id: "test-host-id".to_string(),
supports_background_jobs: true,
})
});
assert!(!client.supports_background_jobs());
let resp = client
.initialize(
@@ -229,6 +231,8 @@ async fn initialize_round_trip() {
)
.await
.unwrap();
assert!(resp.supports_background_jobs);
assert!(client.supports_background_jobs());
assert_eq!(resp.server_version, "test-0.1.0");
assert_eq!(resp.host_id, "test-host-id");
}
@@ -243,6 +247,7 @@ async fn initialize_sends_empty_auth_token_when_none() {
server_message::Message::InitializeResponse(InitializeResponse {
server_version: "test-0.1.0".to_string(),
host_id: "test-host-id".to_string(),
supports_background_jobs: true,
})
});
@@ -270,6 +275,7 @@ async fn initialize_sends_auth_token_when_provided() {
server_message::Message::InitializeResponse(InitializeResponse {
server_version: "test-0.1.0".to_string(),
host_id: "test-host-id".to_string(),
supports_background_jobs: true,
})
});
@@ -381,6 +387,7 @@ async fn is_disconnected_starts_false() {
server_message::Message::InitializeResponse(InitializeResponse {
server_version: "test-0.1.0".to_string(),
host_id: "test-host-id".to_string(),
supports_background_jobs: true,
})
});
@@ -427,6 +434,7 @@ async fn concurrent_in_flight_requests() {
server_message::Message::InitializeResponse(InitializeResponse {
server_version: "test-0.1.0".to_string(),
host_id: "test-host-id".to_string(),
supports_background_jobs: true,
})
});
let client = std::sync::Arc::new(client);
@@ -472,6 +480,7 @@ async fn mock_server_with_error_handling(
InitializeResponse {
server_version: "test-0.1.0".to_string(),
host_id: "test-host-id".to_string(),
supports_background_jobs: true,
},
)),
};
crates/remote_server/src/host_response_tests.rs
+1
View File
@@ -167,6 +167,7 @@ fn every_host_scoped_request_has_a_response_disposition() {
M::GitGenerateCommitMessage(_) => "manager::generate_commit_message",
M::GitGetCommittedBranchFiles(_) => "manager::get_committed_branch_files",
M::RipgrepSearch(_) => "manager::start_ripgrep_search",
M::BackgroundJob(_) => "manager::HostRequestHandle::background_job",
}
}
crates/remote_server/src/manager.rs
+17
View File
@@ -888,6 +888,23 @@ pub struct HostRequestHandle {
}
impl HostRequestHandle {
pub async fn background_job(
&self,
request: crate::proto::BackgroundJobRequest,
) -> Result<crate::proto::BackgroundJobResponse, HostRequestError> {
let message = self
.send(crate::proto::host_scoped_request::Message::BackgroundJob(
request,
))
.await?;
match message.message {
Some(crate::proto::server_message::Message::BackgroundJobResponse(response)) => {
Ok(response)
}
_ => Err(HostRequestError::UnexpectedResponse),
}
}
/// Sends a host-scoped request and awaits the raw `ServerMessage`.
///
/// Bounces to the main thread to call `send_host_request`, then
crates/remote_server/src/protocol_tests.rs
+45
View File
@@ -7,6 +7,50 @@ use crate::proto::{
RemoteAgentContextSnapshot, RemoteContextFileProto, RemoteSkillProto, ServerMessage,
};
#[tokio::test]
async fn background_job_requests_preserve_identity_and_byte_log_cursors() {
let request = ClientMessage::host_scoped(
"background-status".to_owned(),
crate::proto::host_scoped_request::Message::BackgroundJob(
crate::proto::BackgroundJobRequest {
operation: crate::proto::background_job_request::Operation::Status as i32,
job_id: "2ce65749-03d4-4482-910f-b9224575e9bd".to_owned(),
conversation_id: "43131875-abf9-4f98-b20a-b607d1b99125".to_owned(),
action_id: "job-action".to_owned(),
log_offset: 128 * 1024 + 3,
has_environment: true,
..Default::default()
},
),
);
let mut buffer = Vec::new();
write_message(&mut buffer, &request).await.unwrap();
let decoded: ClientMessage = read_message(&mut buffer.as_slice()).await.unwrap();
assert_eq!(decoded, request);
let response = ServerMessage {
request_id: request.request_id,
message: Some(server_message::Message::BackgroundJobResponse(
crate::proto::BackgroundJobResponse {
job_json: "{}".to_owned(),
log_chunk: vec![0xf0, 0x9f, 0x92, 0xa1, 0, 0xff],
next_log_offset: 128 * 1024 + 9,
total_log_bytes: 1024 * 1024,
logs_available: true,
error: String::new(),
completion_log_tail: "retained completion output".as_bytes().to_vec(),
completion_log_bytes: 1024 * 1024 + 7,
completion_log_tail_available: true,
job_not_found: false,
},
)),
};
let mut buffer = Vec::new();
write_message(&mut buffer, &response).await.unwrap();
let decoded: ServerMessage = read_message(&mut buffer.as_slice()).await.unwrap();
assert_eq!(decoded, response);
}
#[tokio::test]
async fn round_trip_client_message() {
let msg = ClientMessage::session_scoped(
@@ -108,6 +152,7 @@ async fn round_trip_server_message() {
InitializeResponse {
server_version: "0.1.0".to_string(),
host_id: "test-host".to_string(),
supports_background_jobs: true,
},
)),
};