Prevent idle event waits from stalling user follow-ups

This commit is contained in:
Ryan Ward
2026-11-06 21:25:17 +00:00
parent 38f1dcdef6
commit 3fa7f2f00c
4 changed files with 256 additions and 8 deletions
+37 -3
View File
@@ -1092,6 +1092,7 @@ enum ProviderCommandResult {
fn convert_provider_tool_batch(
action_context: &ProviderActionContext,
batch: &PendingToolBatch,
has_expected_orchestration_event: bool,
) -> (Vec<(AIAgentAction, bool)>, Vec<ToolResult>) {
let mut actions = Vec::new();
let mut invalid_results = Vec::new();
@@ -1100,7 +1101,18 @@ fn convert_provider_tool_batch(
.iter()
.filter(|pending| pending.state.result().is_none())
{
match action_context.action_from_tool_call(&pending.call) {
let action = action_context.action_from_tool_call(&pending.call).and_then(|action| {
if matches!(action.action, AIAgentActionType::WaitForEvents { .. }) {
if batch.calls.len() != 1 {
return Err("wait_for_events must be the only tool in its batch. Finish the other tool results first. Background commands deliver completion automatically; do not wait for them with this tool.".to_string());
}
if !has_expected_orchestration_event {
return Err("No child-agent or orchestration notification source is pending. Handle the latest user request now: inspect the requested logs or answer from available evidence. Background commands deliver completion automatically. This tool cannot schedule a follow-up, watch a log file, or wait for a future incident.".to_string());
}
}
Ok(action)
});
match action {
Ok(action) => actions.push((
action,
matches!(pending.state, PendingToolCallState::RecoveryPending),
@@ -8748,6 +8760,25 @@ impl BlocklistAIController {
batch: PendingToolBatch,
ctx: &mut ModelContext<Self>,
) {
// A generic wait has no way to observe files, incidents, or command output.
// Require a real orchestration source before parking a durable generation.
// Hosted tasks retain owner-side registration for out-of-band children/messages.
let has_expected_orchestration_event = batch.calls.len() == 1
&& batch.calls[0].call.name == "wait_for_events"
&& {
let history = BlocklistAIHistoryModel::as_ref(ctx);
let events = OrchestrationEventService::as_ref(ctx);
#[cfg(not(target_family = "wasm"))]
let has_pending_events = events.has_pending_events(conversation_id);
#[cfg(target_family = "wasm")]
let has_pending_events = false;
history
.conversation(&conversation_id)
.is_some_and(|conversation| conversation.task_id().is_some())
|| !active_descendant_conversation_ids(history, conversation_id).is_empty()
|| events.has_waiting_local_groups(conversation_id, ctx)
|| has_pending_events
};
let Some(run) = self
.active_provider_runs
.get_mut(&conversation_id)
@@ -8765,8 +8796,11 @@ impl BlocklistAIController {
.map(|title| (pending.call.id.clone(), title))
})
.collect::<HashMap<_, _>>();
let (mut converted_actions, invalid_results) =
convert_provider_tool_batch(&run.action_context, &batch);
let (mut converted_actions, invalid_results) = convert_provider_tool_batch(
&run.action_context,
&batch,
has_expected_orchestration_event,
);
// Seed each worker from the current durable transcript, including discoveries
// made in this run that have not reached the persisted conversation yet.
let history = BlocklistAIHistoryModel::as_ref(ctx);
+176 -2
View File
@@ -1455,7 +1455,8 @@ fn malformed_provider_tool_inputs_become_correlated_errors_without_dropping_vali
panic!("expected provider tool batch");
};
let (actions, errors) = super::convert_provider_tool_batch(&snapshot.action_context, &batch);
let (actions, errors) =
super::convert_provider_tool_batch(&snapshot.action_context, &batch, false);
assert_eq!(actions.len(), 1);
assert_eq!(actions[0].0.id.to_string(), "good-grep");
@@ -1492,7 +1493,8 @@ fn malformed_provider_tool_error_can_be_committed_and_run_continues() {
let Some(ProviderRunStep::DispatchTools(batch)) = snapshot.run.next_step().unwrap() else {
panic!("expected provider tool batch");
};
let (actions, errors) = super::convert_provider_tool_batch(&snapshot.action_context, &batch);
let (actions, errors) =
super::convert_provider_tool_batch(&snapshot.action_context, &batch, false);
assert!(actions.is_empty());
snapshot
@@ -4595,3 +4597,175 @@ fn restored_provider_checkpoint_waits_for_explicit_resume() {
});
});
}
fn dispatch_follow_up_tool_batch(
snapshot: &mut super::ActiveProviderRunSnapshot,
calls: Vec<ToolCall>,
) -> galaxy_agent_core::PendingToolBatch {
let Some(ProviderRunStep::CallModel(call)) = snapshot.run.next_step().unwrap() else {
panic!("expected model call");
};
let advertised_tools = calls.iter().map(|call| call.name.clone()).collect();
snapshot
.run
.accept_model_turn(
&call.work_id,
CompletedModelTurn {
assistant_content: Vec::new(),
tool_calls: calls,
usage: Usage::default(),
stop_reason: StopReason::Completed,
advertised_tools,
},
)
.unwrap();
let Some(ProviderRunStep::DispatchTools(batch)) = snapshot.run.next_step().unwrap() else {
panic!("expected tool batch");
};
batch
}
#[test]
fn wait_without_an_event_source_returns_a_correlated_error_and_allows_another_model_turn() {
let mut snapshot = provider_snapshot(AIConversationId::new());
let batch = dispatch_follow_up_tool_batch(
&mut snapshot,
vec![ToolCall {
id: "unsupported-wait".to_owned(),
name: "wait_for_events".to_owned(),
arguments: serde_json::json!({}),
}],
);
let (actions, errors) =
super::convert_provider_tool_batch(&snapshot.action_context, &batch, false);
assert_eq!(actions.len(), 0);
assert_eq!(errors.len(), 1);
assert_eq!(errors[0].call_id, "unsupported-wait");
assert_eq!(errors[0].status, galaxy_agent_core::ToolResultStatus::Error);
assert!(errors[0]
.content
.contains("No child-agent or orchestration notification source"));
assert!(errors[0].content.contains("cannot schedule a follow-up"));
snapshot
.run
.complete_tool(&batch.work_id, errors[0].clone())
.unwrap();
snapshot.run.commit_tool_batch(&batch.work_id).unwrap();
let MessageContent::MultiPart(parts) = &snapshot.run.transcript().last().unwrap().content
else {
panic!("expected committed tool result");
};
assert!(matches!(&parts[0], ContentPart::ToolResult {
tool_use_id, is_error: true, ..
} if tool_use_id == "unsupported-wait"));
assert!(matches!(
snapshot.run.next_step().unwrap(),
Some(ProviderRunStep::CallModel(_))
));
}
#[test]
fn sole_wait_with_an_expected_orchestration_notification_remains_executable() {
let mut snapshot = provider_snapshot(AIConversationId::new());
let batch = dispatch_follow_up_tool_batch(
&mut snapshot,
vec![ToolCall {
id: "child-wait".to_owned(),
name: "wait_for_events".to_owned(),
arguments: serde_json::json!({"idle_timeout_seconds": 45}),
}],
);
let (actions, errors) =
super::convert_provider_tool_batch(&snapshot.action_context, &batch, true);
assert_eq!(errors.len(), 0);
assert_eq!(actions.len(), 1);
assert_eq!(actions[0].0.id.to_string(), "child-wait");
assert!(
matches!(&actions[0].0.action, AIAgentActionType::WaitForEvents {
tool_call_id, idle_timeout_seconds: 45,
} if tool_call_id == "child-wait")
);
assert!(
matches!(snapshot.run.state(), ProviderRunState::AwaitingTools { batch } if !batch.is_complete())
);
}
#[test]
fn mixed_shell_launch_and_wait_preserves_the_launch_and_yields_after_every_result_commits() {
let mut snapshot = provider_snapshot(AIConversationId::new());
let batch = dispatch_follow_up_tool_batch(
&mut snapshot,
vec![
ToolCall {
id: "job-launch".to_owned(),
name: "run_shell_command".to_owned(),
arguments: serde_json::json!({"command": "echo ready"}),
},
ToolCall {
id: "mixed-wait".to_owned(),
name: "wait_for_events".to_owned(),
arguments: serde_json::json!({}),
},
],
);
let (actions, errors) =
super::convert_provider_tool_batch(&snapshot.action_context, &batch, true);
assert_eq!(actions.len(), 1);
assert_eq!(actions[0].0.id.to_string(), "job-launch");
assert!(
matches!(&actions[0].0.action, AIAgentActionType::RequestCommandOutput { command, .. } if command == "echo ready")
);
assert_eq!(errors.len(), 1);
assert_eq!(errors[0].call_id, "mixed-wait");
assert_eq!(errors[0].status, galaxy_agent_core::ToolResultStatus::Error);
assert!(errors[0].content.contains("only tool in its batch"));
snapshot
.run
.complete_tool(&batch.work_id, errors[0].clone())
.unwrap();
let error = snapshot.run.commit_tool_batch(&batch.work_id).unwrap_err();
assert!(error.to_string().contains("job-launch"));
assert!(matches!(
snapshot.run.state(),
ProviderRunState::AwaitingTools { .. }
));
snapshot
.run
.start_tool(&batch.work_id, "job-launch")
.unwrap();
snapshot
.run
.complete_tool(
&batch.work_id,
galaxy_agent_core::ToolResult {
call_id: "job-launch".to_owned(),
content: serde_json::json!({
"background_job_started": {"job_id": "job-1", "command": "echo ready"}
})
.to_string(),
status: galaxy_agent_core::ToolResultStatus::Success,
images: Vec::new(),
},
)
.unwrap();
snapshot.run.commit_tool_batch(&batch.work_id).unwrap();
let MessageContent::MultiPart(parts) = &snapshot.run.transcript().last().unwrap().content
else {
panic!("expected both committed tool results");
};
assert_eq!(parts.len(), 2);
assert!(matches!(&parts[0], ContentPart::ToolResult {
tool_use_id, is_error: false, ..
} if tool_use_id == "job-launch"));
assert!(matches!(&parts[1], ContentPart::ToolResult {
tool_use_id, is_error: true, ..
} if tool_use_id == "mixed-wait"));
assert!(
matches!(snapshot.run.next_step().unwrap(), Some(ProviderRunStep::Done(
ProviderRunOutcome::Completed(completion)
)) if completion.stop_reason == StopReason::Completed)
);
}
+3 -3
View File
@@ -1718,11 +1718,11 @@ pub(crate) fn tool_name_is_supported(name: &str, supported: &HashSet<api::ToolTy
/// Shared by direct-provider prompts and compatibility/background request prompts.
pub(crate) const CODING_DELEGATION_GUIDANCE: &str = "## Coding Delegation\nFor coding tasks, actively look for bounded implementation or review work to offload to available subagents early. Prefer delegation for substantial independent changes or a focused review of completed work; give the child the known files, just enough context, an explicit implementation or research role, the expected deliverable, and a validation check. The parent owns integration and reviews the result, returning corrections to the same child when needed. Do not duplicate delegated implementation while it runs. Keep trivial edits and tightly dependent steps local when delegation would add delay. Research and plan-mode assignments remain read-only. Delegate only when delegation tools are advertised; child agents are leaf workers and must not spawn other agents.\n\n## Dueling Models Review\nAfter a substantial coherent set of authorized code changes, the parent must request an independent read-only subagent review before reporting completion. Skip this workflow for trivial edits. Always select an available reviewer model; provider or model diversity is a preference, never a reason to skip review. Prefer a capable reviewer with a different model and a different provider from the implementer, using only the advertised model catalog. If no suitable different-provider option exists, select the next best capable different model; if only one model is available, use a separate reviewer conversation and disclose that limitation. Do not invent model IDs or provider diversity. Give the reviewer the requirements, changed files and diff scope, implementation model ID, completed checks, and known uncertainties. Ask for concrete correctness, regression, security, or missing-validation findings with file references and evidence; distinguish blocking findings from optional suggestions. The reviewer must inspect the current code and return findings or explicit LGTM, never rubber-stamp a summary. The parent evaluates findings, sends actionable corrections to the same implementer, and sends the revised diff and validation evidence back to the same reviewer. Continue repair and re-review until the reviewer returns LGTM for the current changes. Changes after approval need review again. Do not force stylistic churn, hide disagreements, or claim LGTM without an actual reviewer response. If verification is unavailable or a disagreement requires user input, report the concrete blocker and remaining findings honestly. The parent coordinates this loop; leaf implementers report readiness for parent review and never spawn reviewers themselves.\n\n";
pub(crate) const PROGRESS_COMMUNICATION_GUIDANCE: &str = "## Progress Updates\nFor an active implementation task, work quietly: the normal response to a tool result is the next necessary tool call, with no assistant text. Tool panes show activity. Do not greet, acknowledge instructions, announce intentions, restate the request or plan, recap each completed step, or reassure the user that work continues. This applies equally to the parent and subagents, follow-up instructions, tool results, command polls, and restored or compacted context. These all continue the same task; they do not start a new introduction.\nDuring work, send prose only to ask a necessary user decision, report a concrete blocker that requires user action, or flag a newly discovered urgent consequence the user must act on now. Otherwise save findings for the final result. Before speaking, compare the intended factual content with prior assistant messages and the conversation summary. If it adds no new actionable information, omit it entirely; paraphrasing a previous update still counts as repetition. Track what has already been communicated using that history, without extra tracking tools or files.\nExamples: after a read/search result, call the next tool without saying 'Got it' or 'I will inspect the files'; after a successful edit, run the relevant check without saying 'The change is in place'; while waiting for a command or subagent, use its wait tool without saying 'Still working'; after approval or a follow-up correction, perform the authorized work without another acknowledgement. Do not stop with a promise, an update, or a repeated question when work can continue. Answer explicit user questions, including status requests, briefly and directly, then continue. When finished, report the final outcome once.\n\n";
pub(crate) const PROGRESS_COMMUNICATION_GUIDANCE: &str = "## Progress Updates\nFor an active implementation task, work quietly: the normal response to a tool result is the next necessary tool call, with no assistant text. Tool panes show activity. Do not greet, acknowledge instructions, announce intentions, restate the request or plan, recap each completed step, or reassure the user that work continues. This applies equally to the parent and subagents, follow-up instructions, tool results, command polls, and restored or compacted context. These all continue the same task; they do not start a new introduction.\nDuring work, send prose only to ask a necessary user decision, report a concrete blocker that requires user action, or flag a newly discovered urgent consequence the user must act on now. Otherwise save findings for the final result. Before speaking, compare the intended factual content with prior assistant messages and the conversation summary. If it adds no new actionable information, omit it entirely; paraphrasing a previous update still counts as repetition. Track what has already been communicated using that history, without extra tracking tools or files.\nExamples: after a read/search result, call the next tool without saying 'Got it' or 'I will inspect the files'; after a successful edit, run the relevant check without saying 'The change is in place'; while waiting for an expected child-agent update, use wait_for_events without saying 'Still working'; background-command completion is delivered automatically; after approval or a follow-up correction, perform the authorized work without another acknowledgement. Do not stop with a promise, an update, or a repeated question when work can continue. Answer explicit user questions, including status requests, briefly and directly, then continue. When finished, report the final outcome once.\n\n";
pub(crate) const IMPLEMENTATION_PROGRESS_GUIDANCE: &str = "## Incremental Implementation\nFor an authorized implementation task, reuse the parent handoff's paths, findings, constraints, and acceptance criteria. Search until you have just enough context to make the requested change. Read the immediate implementation and a relevant example or test as needed, then make the smallest coherent change. The parent can review and request follow-up changes; do not try to anticipate every possible review concern before editing, and still validate your own work. Validate that slice before expanding the investigation. Do not inventory every related subsystem before editing. Further discovery must resolve a concrete uncertainty that affects the next change. When a local code change is reversible and a focused build or test can settle an uncertainty, make a small, evidence-based attempt and use the result to refine it instead of searching for complete certainty. Do not guess APIs or edit files you have not inspected. Read known files directly and batch independent focused reads; avoid repeating discovery already supplied by the parent. Prefer the nearest relevant check before a full build. Investigate first when a mistake could destroy data, affect production, expose credentials, or create irreversible external effects. For review-only work, inspect the changed code and its immediate dependencies, then report actionable findings; expand the search only to test a specific suspected defect. For delegated implementation work, include known paths and symbols, established findings, changes to preserve, a concrete first change, and its acceptance check; distinguish known facts from unanswered questions. Do not invent missing discoveries. Research-only, review, and planning requests remain read-only unless the user authorizes changes.\n\n";
pub(crate) const SHELL_EXECUTION_GUIDANCE: &str = "## Background Shell Jobs\nEvery agent-requested shell command runs as an independent, noninteractive background job with closed stdin and no terminal. A launch acknowledgment is not an exit result. Never poll or monitor jobs, send process input, open an interactive shell, or use terminal takeover. Galaxy automatically sends the command, logs, and exit status when a job succeeds, fails, or is canceled. Wait for that event before dependent work; independent jobs may be launched together. Do not restart a command merely because it is still running.\nUse noninteractive flags, disable pagers, and keep the job's main process in the foreground: do not append '&', use nohup/disown, or detach child processes. If interaction cannot be avoided, give the exact command to the user to run in their own terminal instead of executing it. Commands typed by users retain their normal interactive terminal behavior. Each job starts in the supplied working directory and inherits the session environment; cd/export changes inside a job do not persist to later jobs. Use explicit paths or a cd within that job when needed. Galaxy stores stdout/stderr in temporary logs; do not hide all output in another file. The user can view retained logs and terminate jobs through /background-jobs. Use interrupt_shell_command only to cancel this conversation's specific background job when requested or when an explicit stop condition is reached.\n\n";
pub(crate) const SHELL_EXECUTION_GUIDANCE: &str = "## Background Shell Jobs\nEvery agent-requested shell command runs as an independent, noninteractive background job with closed stdin and no terminal. A launch acknowledgment is not an exit result. Never poll jobs, send process input, open an interactive shell, or use terminal takeover. Background jobs continue independently; Galaxy automatically sends the command, logs, and exit status when a job succeeds, fails, or is canceled. Wait for that completion delivery before work that depends on the final outcome; independent work may continue. Do not use wait_for_events for background commands or restart a command merely because it is still running. If the user explicitly asks for current status or a log review, inspect the available evidence and answer now. When read_files is available and permitted, read the retained log path for the requested snapshot; do not defer that inspection until command exit or park the user request in a wait.\nUse noninteractive flags, disable pagers, and keep the job's main process in the foreground: do not append '&', use nohup/disown, or detach child processes. If interaction cannot be avoided, give the exact command to the user to run in their own terminal instead of executing it. Commands typed by users retain their normal interactive terminal behavior. Each job starts in the supplied working directory and inherits the session environment; cd/export changes inside a job do not persist to later jobs. Use explicit paths or a cd within that job when needed. Galaxy stores stdout/stderr in temporary logs; do not hide all output in another file. The user can view retained logs and terminate jobs through /background-jobs. Use interrupt_shell_command only to cancel this conversation's specific background job when requested or when an explicit stop condition is reached.\n\n";
fn computer_target_schema() -> serde_json::Value {
serde_json::json!({
@@ -2087,7 +2087,7 @@ pub fn default_tool_definitions() -> Vec<ToolDefinition> {
},
ToolDefinition {
name: "wait_for_events".to_string(),
description: "Yield after starting child agents or other asynchronous work. Use this when you are waiting for child-agent updates instead of repeating the same investigation yourself.".to_string(),
description: "Yield only for an expected update from an already-started child agent or orchestration event source. Call this tool alone after other tool results have committed. Do not use this for background-shell completion, reading or monitoring log files, reminders, or waiting for another user message. Handle a fresh user question or status/log-review request now using available evidence and permitted inspection tools; never park that request in a wait.".to_string(),
input_schema: serde_json::json!({
"type": "object",
"properties": {
@@ -697,3 +697,43 @@ fn compatibility_background_acknowledgment_does_not_invent_monitoring() {
};
assert_eq!(super::long_running_command_content(&snapshot), content);
}
#[test]
fn status_and_log_followups_are_not_parked_behind_event_waits() {
let request = api::Request {
settings: Some(api::request::Settings {
supported_tools: vec![
api::ToolType::RunShellCommand.into(),
api::ToolType::ReadFiles.into(),
api::ToolType::Subagent.into(),
],
..Default::default()
}),
..Default::default()
};
let prompt = extract_system_prompt(&request, &[]).expect("system prompt");
for instruction in [
"Background jobs continue independently",
"Do not use wait_for_events for background commands",
"inspect the available evidence and answer now",
"read the retained log path for the requested snapshot",
"do not defer that inspection until command exit",
] {
assert!(
prompt.contains(instruction),
"missing instruction: {instruction}"
);
}
assert!(!super::PROGRESS_COMMUNICATION_GUIDANCE
.contains("command or subagent, use its wait tool"));
let tools = extract_tools(&request);
let wait = tools
.iter()
.find(|tool| tool.name == "wait_for_events")
.expect("child wait tool");
assert!(wait.description.contains("expected update from an already-started child agent or orchestration event source"));
assert!(wait.description.contains("never park that request in a wait"));
assert!(!wait.description.contains("other asynchronous work"));
assert!(tools.iter().any(|tool| tool.name == "read_files"));
}