Fix orchestration and provider reliability

This commit is contained in:
Ryan Ward
2026-08-20 15:13:19 -05:00
parent c585961149
commit 1336f00dfb
17 changed files with 418 additions and 58 deletions
+1
View File
@@ -24,6 +24,7 @@ Environment variables:
- `WS_SERVER_URL` - WebSocket endpoint (default: `ws://localhost:8080/graphql/v2`) - `WS_SERVER_URL` - WebSocket endpoint (default: `ws://localhost:8080/graphql/v2`)
### Testing ### Testing
- During interactive bug-fix verification, make the requested code changes first and run only `cargo run --bin galaxy-oss` for the user to verify. Leave the app running for the user; do not stop it or treat the command timeout as a failure. Do not run `cargo fmt`, `cargo check`, `cargo clippy`, presubmit, or other validation commands until the user confirms the fix.
- `cargo nextest run --no-fail-fast --workspace --exclude command-signatures-v2` - Run tests with nextest - `cargo nextest run --no-fail-fast --workspace --exclude command-signatures-v2` - Run tests with nextest
- `cargo nextest run -p galaxy_completer --features v2` - Run completer tests with v2 features - `cargo nextest run -p galaxy_completer --features v2` - Run completer tests with v2 features
- `cargo test --doc` - Run doc tests - `cargo test --doc` - Run doc tests
+1
View File
@@ -460,6 +460,7 @@ embed-resource = "3.0"
# Note that we support channel-specific enables for these features # Note that we support channel-specific enables for these features
[features] [features]
remote_logs_local_file = []
tui = ["galaxyui_core/tui"] tui = ["galaxyui_core/tui"]
ai_resume_button = [] ai_resume_button = []
autoupdate = [] autoupdate = []
@@ -18,6 +18,7 @@ use crate::ai::agent_conversations_model::entry::AgentConversationEntryId;
use crate::ai::agent_conversations_model::{ use crate::ai::agent_conversations_model::{
AgentConversationNavigationSubject, AgentConversationsModel, AgentConversationNavigationSubject, AgentConversationsModel,
}; };
use crate::ai::blocklist::orchestration_topology::descendant_conversation_ids_in_spawn_order;
use crate::ai::blocklist::BlocklistAIHistoryModel; use crate::ai::blocklist::BlocklistAIHistoryModel;
use crate::terminal::view::TerminalAction; use crate::terminal::view::TerminalAction;
use crate::ui_components::blended_colors; use crate::ui_components::blended_colors;
@@ -54,6 +55,57 @@ pub(crate) fn conversation_id_for_agent_id(
}) })
} }
/// Resolve an agent identifier within the active orchestration tree first.
/// Agent ids are server-facing identifiers and can be reused by separate
/// orchestrators, so the global index is only a fallback for legacy output.
pub(crate) fn conversation_id_for_agent_id_in_orchestration(
agent_id: &str,
orchestrator_id: AIConversationId,
app: &AppContext,
) -> Option<AIConversationId> {
let canonical_id = canonical_agent_id(agent_id);
let history = BlocklistAIHistoryModel::as_ref(app);
let matches = |conversation: &AIConversation| {
conversation
.orchestration_agent_id()
.as_deref()
.is_some_and(|id| id == canonical_id)
|| conversation.run_id().is_some_and(|id| id == canonical_id)
|| conversation
.server_conversation_token()
.is_some_and(|token| token.as_str() == canonical_id)
};
std::iter::once(orchestrator_id)
.chain(descendant_conversation_ids_in_spawn_order(
history,
orchestrator_id,
))
.find(|conversation_id| {
history
.conversation(conversation_id)
.is_some_and(|conversation| matches(conversation))
})
.or_else(|| conversation_id_for_agent_id(canonical_id, app))
}
/// Resolve an agent id relative to the conversation currently shown by a
/// terminal view. Inline orchestration cards must use this instead of the
/// global agent-id index because separate orchestrators can have overlapping
/// server-facing child identifiers.
pub(crate) fn conversation_id_for_agent_id_in_terminal_view(
agent_id: &str,
terminal_view_id: EntityId,
app: &AppContext,
) -> Option<AIConversationId> {
let history = BlocklistAIHistoryModel::as_ref(app);
let active_conversation = history.active_conversation(terminal_view_id)?;
let orchestrator_id = active_conversation
.parent_conversation_id()
.unwrap_or_else(|| active_conversation.id());
conversation_id_for_agent_id_in_orchestration(agent_id, orchestrator_id, app)
}
/// True if the conversation is open in some other visible pane. Hidden /// True if the conversation is open in some other visible pane. Hidden
/// child-agent panes are excluded so unopened children don't look /// child-agent panes are excluded so unopened children don't look
/// "already open". /// "already open".
@@ -114,8 +166,13 @@ pub(crate) fn dispatch_focus_or_open_child_agent_pane(
let self_pane_group_id = let self_pane_group_id =
pane_group_id_containing_terminal_view(self_terminal_view_id, app); pane_group_id_containing_terminal_view(self_terminal_view_id, app);
if Some(owner_pane_group_id) == self_pane_group_id { if Some(owner_pane_group_id) == self_pane_group_id {
// Same pane group: swap to the child pane in place.
ctx.dispatch_typed_action(TerminalAction::RevealChildAgent { conversation_id }); ctx.dispatch_typed_action(TerminalAction::RevealChildAgent { conversation_id });
} else { } else {
// Different pane group: focus the exact canonical owner.
// The conversation id was resolved from the source
// orchestration tree, so this cannot select a same-named
// child belonging to another orchestrator.
ctx.dispatch_typed_action(WorkspaceAction::FocusTerminalViewInWorkspace { ctx.dispatch_typed_action(WorkspaceAction::FocusTerminalViewInWorkspace {
terminal_view_id: owner_view_id, terminal_view_id: owner_view_id,
}); });
+2 -1
View File
@@ -4756,11 +4756,12 @@ impl AIBlock {
) { ) {
ctx.subscribe_to_model(action_model, |me, action_model, event, ctx| { ctx.subscribe_to_model(action_model, |me, action_model, event, ctx| {
let action_id = event.action_id(); let action_id = event.action_id();
let is_finished_action = matches!(event, BlocklistAIActionEvent::FinishedAction { .. });
if event if event
.conversation_id() .conversation_id()
.is_some_and(|conversation_id| conversation_id != me.client_ids.conversation_id) .is_some_and(|conversation_id| conversation_id != me.client_ids.conversation_id)
|| me.is_finished() || (me.is_finished() && !is_finished_action)
|| !me.requested_action_ids.contains(action_id) || !me.requested_action_ids.contains(action_id)
{ {
// Technically, this subscription should be unregistered after `is_finished` is // Technically, this subscription should be unregistered after `is_finished` is
+4 -1
View File
@@ -209,7 +209,10 @@ impl BlocklistAIStatusBar {
.active_exchange_model .active_exchange_model
.as_ref() .as_ref()
.is_some_and(|model| model.conversation_id(ctx) == Some(*conversation_id)); .is_some_and(|model| model.conversation_id(ctx) == Some(*conversation_id));
if is_active_conversation && !new_status.is_in_progress() { if is_active_conversation
&& !new_status.is_in_progress()
&& !new_status.is_transient_error()
{
me.stop_warping_timer(); me.stop_warping_timer();
} }
ctx.notify(); ctx.notify();
@@ -337,6 +337,14 @@ pub fn render_warping_indicator<V: View>(
let mut should_render_waiting_icon = false; let mut should_render_waiting_icon = false;
let mut non_shimmering_text = None; let mut non_shimmering_text = None;
if let Some(status_message) = props
.model
.conversation(app)
.and_then(|conversation| conversation.status_error_message())
.filter(|message| message.starts_with("Retrying LLM request"))
{
non_shimmering_text = Some(format!("{status_message}"));
}
let message = if let Some(summarization_type) = summarization_type { let message = if let Some(summarization_type) = summarization_type {
// Choose the appropriate message based on summarization type // Choose the appropriate message based on summarization type
let base_message = match summarization_type { let base_message = match summarization_type {
@@ -22,8 +22,8 @@ use crate::ai::agent::{
use crate::ai::blocklist::action_model::AIActionStatus; use crate::ai::blocklist::action_model::AIActionStatus;
use crate::ai::blocklist::agent_view::orchestration_avatar::OrchestrationAvatar; use crate::ai::blocklist::agent_view::orchestration_avatar::OrchestrationAvatar;
use crate::ai::blocklist::agent_view::orchestration_conversation_links::{ use crate::ai::blocklist::agent_view::orchestration_conversation_links::{
conversation_id_for_agent_id, conversation_navigation_card_with_icon, conversation_id_for_agent_id, conversation_id_for_agent_id_in_orchestration,
dispatch_focus_or_open_child_agent_pane, conversation_navigation_card_with_icon, dispatch_focus_or_open_child_agent_pane,
}; };
use crate::ai::blocklist::block::model::AIBlockModelHelper; use crate::ai::blocklist::block::model::AIBlockModelHelper;
use crate::ai::blocklist::block::{ use crate::ai::blocklist::block::{
@@ -106,6 +106,28 @@ fn participant_for_agent_id(
OrchestrationParticipant::unknown_child() OrchestrationParticipant::unknown_child()
} }
fn participant_for_agent_id_in_orchestration(
agent_id: &str,
orchestrator_agent_id: Option<&str>,
orchestrator_conversation_id: AIConversationId,
app: &AppContext,
) -> OrchestrationParticipant {
if let Some(conversation_id) =
conversation_id_for_agent_id_in_orchestration(agent_id, orchestrator_conversation_id, app)
{
if let Some(conversation) =
BlocklistAIHistoryModel::as_ref(app).conversation(&conversation_id)
{
return participant_for_conversation(
conversation,
orchestrator_agent_id,
Some(agent_id),
);
}
}
participant_for_agent_id(agent_id, orchestrator_agent_id, app)
}
fn participant_for_conversation( fn participant_for_conversation(
conversation: &AIConversation, conversation: &AIConversation,
orchestrator_agent_id: Option<&str>, orchestrator_agent_id: Option<&str>,
@@ -346,15 +368,32 @@ pub(super) fn render_messages_received_from_agents(
.model .model
.conversation(app) .conversation(app)
.and_then(|conversation| orchestrator_agent_id_for_conversation(conversation, app)); .and_then(|conversation| orchestrator_agent_id_for_conversation(conversation, app));
let orchestrator_conversation_id = props.model.conversation(app).map(|conversation| {
conversation
.parent_conversation_id()
.unwrap_or_else(|| conversation.id())
});
let Some(orchestrator_conversation_id) = orchestrator_conversation_id else {
return Empty::new().finish();
};
let mut column = Flex::column().with_cross_axis_alignment(CrossAxisAlignment::Stretch); let mut column = Flex::column().with_cross_axis_alignment(CrossAxisAlignment::Stretch);
for (index, msg) in messages.iter().enumerate() { for (index, msg) in messages.iter().enumerate() {
let sender = let sender = participant_for_agent_id_in_orchestration(
participant_for_agent_id(&msg.sender_agent_id, orchestrator_agent_id.as_deref(), app); &msg.sender_agent_id,
orchestrator_agent_id.as_deref(),
orchestrator_conversation_id,
app,
);
let recipients = msg let recipients = msg
.addresses .addresses
.iter() .iter()
.map(|agent_id| { .map(|agent_id| {
participant_for_agent_id(agent_id, orchestrator_agent_id.as_deref(), app) participant_for_agent_id_in_orchestration(
agent_id,
orchestrator_agent_id.as_deref(),
orchestrator_conversation_id,
app,
)
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let row_message_id = received_message_collapsible_id(&msg.message_id); let row_message_id = received_message_collapsible_id(&msg.message_id);
@@ -580,7 +619,19 @@ pub(super) fn render_start_agent(
); );
return Empty::new().finish(); return Empty::new().finish();
}; };
let child_conversation_card_data = child_conversation_card_data_for_result(result, app); let orchestrator_conversation_id = props.model.conversation(app).map(|conversation| {
conversation
.parent_conversation_id()
.unwrap_or_else(|| conversation.id())
});
let child_conversation_card_data =
orchestrator_conversation_id.and_then(|orchestrator_id| {
child_conversation_card_data_for_result_in_orchestration(
result,
orchestrator_id,
app,
)
});
let (label_fragments, status_icon) = match result { let (label_fragments, status_icon) = match result {
StartAgentResult::Success { .. } => ( StartAgentResult::Success { .. } => (
vec![ vec![
@@ -841,6 +892,31 @@ fn child_conversation_card_data_for_result(
} }
} }
fn child_conversation_card_data_for_result_in_orchestration(
result: &StartAgentResult,
orchestrator_id: AIConversationId,
app: &AppContext,
) -> Option<ChildConversationCardData> {
match result {
StartAgentResult::Success { agent_id, .. } => {
let conversation_id =
conversation_id_for_agent_id_in_orchestration(agent_id, orchestrator_id, app)?;
let conversation =
BlocklistAIHistoryModel::as_ref(app).conversation(&conversation_id)?;
let agent_name = conversation.agent_name().unwrap_or("Agent").to_string();
let status = conversation.status().clone();
let title = available_conversation_title_for_id(conversation_id, app)?;
Some(ChildConversationCardData {
conversation_id,
agent_name,
title,
status,
})
}
StartAgentResult::Error { .. } | StartAgentResult::Cancelled { .. } => None,
}
}
fn available_conversation_title_for_id( fn available_conversation_title_for_id(
conversation_id: AIConversationId, conversation_id: AIConversationId,
app: &AppContext, app: &AppContext,
+51 -1
View File
@@ -6189,6 +6189,35 @@ impl BlocklistAIController {
} }
#[cfg(not(target_family = "wasm"))] #[cfg(not(target_family = "wasm"))]
if let Some(lifecycle) = lifecycle.as_ref() { if let Some(lifecycle) = lifecycle.as_ref() {
if lifecycle.phase == ProviderLlmLifecyclePhase::Requested {
BlocklistAIHistoryModel::handle(ctx).update(ctx, |history, ctx| {
history.update_conversation_status(
self.terminal_surface_id,
conversation_id,
ConversationStatus::InProgress,
ctx,
);
});
}
if lifecycle.phase == ProviderLlmLifecyclePhase::RetryScheduled {
let retry_message = format!(
"Retrying LLM request ({}/3): {}",
lifecycle.retry_attempt,
lifecycle
.error
.as_deref()
.unwrap_or("temporary provider error")
);
BlocklistAIHistoryModel::handle(ctx).update(ctx, |history, ctx| {
history.update_conversation_status_with_error(
self.terminal_surface_id,
conversation_id,
ConversationStatus::TransientError,
Some(RenderableAIError::other(retry_message, false)),
ctx,
);
});
}
remote_logging::log_model_event( remote_logging::log_model_event(
ctx, ctx,
provider_llm_lifecycle_remote_log_record( provider_llm_lifecycle_remote_log_record(
@@ -6199,7 +6228,28 @@ impl BlocklistAIController {
); );
} }
#[cfg(target_family = "wasm")] #[cfg(target_family = "wasm")]
let _ = lifecycle; if let Some(lifecycle) = lifecycle.as_ref() {
if lifecycle.phase == ProviderLlmLifecyclePhase::Requested {
BlocklistAIHistoryModel::handle(ctx).update(ctx, |history, ctx| {
history.update_conversation_status(
self.terminal_surface_id,
conversation_id,
ConversationStatus::InProgress,
ctx,
);
});
}
if lifecycle.phase == ProviderLlmLifecyclePhase::RetryScheduled {
BlocklistAIHistoryModel::handle(ctx).update(ctx, |history, ctx| {
history.update_conversation_status(
self.terminal_surface_id,
conversation_id,
ConversationStatus::TransientError,
ctx,
);
});
}
}
let _ = acknowledgement.send(result); let _ = acknowledgement.send(result);
} }
ProviderDriveMessage::Checkpoint { ProviderDriveMessage::Checkpoint {
@@ -30,7 +30,7 @@ use crate::ai::blocklist::action_model::{
RunAgentsExecutorEvent, RunAgentsSpawningSnapshot, RunAgentsExecutorEvent, RunAgentsSpawningSnapshot,
}; };
use crate::ai::blocklist::agent_view::orchestration_conversation_links::{ use crate::ai::blocklist::agent_view::orchestration_conversation_links::{
conversation_id_for_agent_id, conversation_navigation_card_with_icon, conversation_id_for_agent_id_in_terminal_view, conversation_navigation_card_with_icon,
dispatch_focus_or_open_child_agent_pane, dispatch_focus_or_open_child_agent_pane,
}; };
use crate::ai::blocklist::agent_view::orchestration_pill_bar::render_static_agent_pill; use crate::ai::blocklist::agent_view::orchestration_pill_bar::render_static_agent_pill;
@@ -1784,7 +1784,7 @@ fn render_run_agents_child_row(
let outcome_conversation_id = outcome.and_then(|outcome| match &outcome.kind { let outcome_conversation_id = outcome.and_then(|outcome| match &outcome.kind {
RunAgentsAgentOutcomeKind::Launched { agent_id } RunAgentsAgentOutcomeKind::Launched { agent_id }
| RunAgentsAgentOutcomeKind::Completed { agent_id, .. } => { | RunAgentsAgentOutcomeKind::Completed { agent_id, .. } => {
conversation_id_for_agent_id(agent_id, app) conversation_id_for_agent_id_in_terminal_view(agent_id, terminal_view_id, app)
} }
RunAgentsAgentOutcomeKind::Failed { .. } => None, RunAgentsAgentOutcomeKind::Failed { .. } => None,
}); });
+19 -19
View File
@@ -107,11 +107,11 @@ fn run_shell_command_readonly() -> ToolDefinition {
fn read_files() -> ToolDefinition { fn read_files() -> ToolDefinition {
ToolDefinition { ToolDefinition {
name: "read_files".to_string(), name: "read_files".to_string(),
description: "Read the contents of one or more files. Pass ALL file paths you need in a single call for efficiency. Returns file contents with path headers. Binary files are detected and skipped. Use absolute paths.".to_string(), description: "Read the contents of up to 6 focused files per call. If you need more, make another call. Returns file contents with path headers. Binary files are detected and skipped. Use absolute paths.".to_string(),
input_schema: serde_json::json!({ input_schema: serde_json::json!({
"type": "object", "type": "object",
"properties": { "properties": {
"files": { "type": "array", "items": { "type": "string" }, "description": "Absolute file paths to read" } "files": { "type": "array", "maxItems": 6, "items": { "type": "string" }, "description": "Absolute file paths to read (maximum 6 per call)" }
}, },
"required": ["files"] "required": ["files"]
}), }),
@@ -121,7 +121,7 @@ fn read_files() -> ToolDefinition {
fn apply_file_diffs() -> ToolDefinition { fn apply_file_diffs() -> ToolDefinition {
ToolDefinition { ToolDefinition {
name: "apply_file_diffs".to_string(), name: "apply_file_diffs".to_string(),
description: "Apply search/replace edits to files. Creates files if they don't exist (use empty search string). The search string must uniquely match one location in the file. Include enough surrounding context for uniqueness. For new files, use search=\"\" and put full content in replace.".to_string(), description: "Apply search/replace edits to exactly one file per call. For multiple files, make separate calls. Creates files if they don't exist (use empty search string). The search string must uniquely match one location in the file. Include enough surrounding context for uniqueness. For new files, use search=\"\" and put full content in replace.".to_string(),
input_schema: serde_json::json!({ input_schema: serde_json::json!({
"type": "object", "type": "object",
"properties": { "properties": {
@@ -136,11 +136,11 @@ fn apply_file_diffs() -> ToolDefinition {
fn grep() -> ToolDefinition { fn grep() -> ToolDefinition {
ToolDefinition { ToolDefinition {
name: "grep".to_string(), name: "grep".to_string(),
description: "Search for regex patterns in files. Uses git grep in git repos (respects .gitignore) or ripgrep otherwise. Returns file paths and matching line numbers. Use read_files afterward to see context around matches. Pass ALL patterns you need in one call.".to_string(), description: "Search for up to 3 focused regex patterns per call. Uses git grep in git repos (respects .gitignore) or ripgrep otherwise. Returns file paths and matching line numbers. Use read_files afterward to see context around matches.".to_string(),
input_schema: serde_json::json!({ input_schema: serde_json::json!({
"type": "object", "type": "object",
"properties": { "properties": {
"queries": { "type": "array", "items": { "type": "string" }, "description": "Regex patterns to search for" }, "queries": { "type": "array", "maxItems": 3, "items": { "type": "string" }, "description": "Regex patterns to search for (maximum 3 per call)" },
"path": { "type": "string", "description": "Directory to scope the search to" } "path": { "type": "string", "description": "Directory to scope the search to" }
}, },
"required": ["queries"] "required": ["queries"]
@@ -151,11 +151,11 @@ fn grep() -> ToolDefinition {
fn file_glob() -> ToolDefinition { fn file_glob() -> ToolDefinition {
ToolDefinition { ToolDefinition {
name: "file_glob".to_string(), name: "file_glob".to_string(),
description: "Find files matching glob patterns. Uses git ls-files in git repos. Returns absolute file paths of matches. Common patterns: '**/*.rs', 'src/**/*.ts', '**/Cargo.toml'. Pass ALL patterns in one call.".to_string(), description: "Find files matching up to 3 focused glob patterns per call. Uses git ls-files in git repos. Returns absolute file paths of matches. Make another call for more patterns.".to_string(),
input_schema: serde_json::json!({ input_schema: serde_json::json!({
"type": "object", "type": "object",
"properties": { "properties": {
"patterns": { "type": "array", "items": { "type": "string" }, "description": "Glob patterns to match files" }, "patterns": { "type": "array", "maxItems": 3, "items": { "type": "string" }, "description": "Glob patterns to match files (maximum 3 per call)" },
"path": { "type": "string", "description": "Directory to search from" } "path": { "type": "string", "description": "Directory to search from" }
}, },
"required": ["patterns"] "required": ["patterns"]
@@ -241,12 +241,12 @@ fn read_mcp_resource() -> ToolDefinition {
fn read_documents() -> ToolDefinition { fn read_documents() -> ToolDefinition {
ToolDefinition { ToolDefinition {
name: "read_plan".to_string(), name: "read_plan".to_string(),
description: "Read the contents of one or more Galaxy plan documents by their IDs. Plans are rich-text documents that appear in the Plans folder of Galaxy Drive." description: "Read the contents of up to 6 Galaxy plan documents per call. Make another call for more. Plans are rich-text documents that appear in the Plans folder of Galaxy Drive."
.to_string(), .to_string(),
input_schema: serde_json::json!({ input_schema: serde_json::json!({
"type": "object", "type": "object",
"properties": { "properties": {
"document_ids": { "type": "array", "items": { "type": "string" }, "description": "Plan document IDs to read" } "document_ids": { "type": "array", "maxItems": 6, "items": { "type": "string" }, "description": "Plan document IDs to read (maximum 6 per call)" }
}, },
"required": ["document_ids"] "required": ["document_ids"]
}), }),
@@ -256,12 +256,12 @@ fn read_documents() -> ToolDefinition {
fn create_documents() -> ToolDefinition { fn create_documents() -> ToolDefinition {
ToolDefinition { ToolDefinition {
name: "create_plan".to_string(), name: "create_plan".to_string(),
description: "Create a new plan document in Galaxy Drive's Plans folder. Plans are rich-text documents for tracking tasks, architecture decisions, and project notes." description: "Create exactly one plan document per call. Use separate calls for separate documents. Plans are rich-text documents for tracking tasks, architecture decisions, and project notes."
.to_string(), .to_string(),
input_schema: serde_json::json!({ input_schema: serde_json::json!({
"type": "object", "type": "object",
"properties": { "properties": {
"documents": { "type": "array", "items": { "type": "object", "properties": { "title": { "type": "string" }, "content": { "type": "string" } }, "required": ["title", "content"] }, "description": "Plan documents to create" } "documents": { "type": "array", "maxItems": 1, "items": { "type": "object", "properties": { "title": { "type": "string" }, "content": { "type": "string" } }, "required": ["title", "content"] }, "description": "One plan document to create" }
}, },
"required": ["documents"] "required": ["documents"]
}), }),
@@ -271,12 +271,12 @@ fn create_documents() -> ToolDefinition {
fn edit_documents() -> ToolDefinition { fn edit_documents() -> ToolDefinition {
ToolDefinition { ToolDefinition {
name: "edit_plan".to_string(), name: "edit_plan".to_string(),
description: "Edit an existing plan document in Galaxy Drive using search/replace diffs." description: "Edit exactly one plan document per call with incremental search/replace diffs. Use separate calls for separate documents."
.to_string(), .to_string(),
input_schema: serde_json::json!({ input_schema: serde_json::json!({
"type": "object", "type": "object",
"properties": { "properties": {
"diffs": { "type": "array", "items": { "type": "object", "properties": { "document_id": { "type": "string" }, "search": { "type": "string" }, "replace": { "type": "string" } }, "required": ["document_id", "search", "replace"] }, "description": "Edits to apply to plan documents" } "diffs": { "type": "array", "maxItems": 1, "items": { "type": "object", "properties": { "document_id": { "type": "string" }, "search": { "type": "string" }, "replace": { "type": "string" } }, "required": ["document_id", "search", "replace"] }, "description": "One incremental edit to apply to a plan document" }
}, },
"required": ["diffs"] "required": ["diffs"]
}), }),
@@ -364,11 +364,11 @@ fn fetch_conversation() -> ToolDefinition {
fn read_notebook() -> ToolDefinition { fn read_notebook() -> ToolDefinition {
ToolDefinition { ToolDefinition {
name: "read_notebook".to_string(), name: "read_notebook".to_string(),
description: "Read the contents of one or more Galaxy Drive notebooks by their IDs. Notebooks are user-created rich-text documents stored in Galaxy Drive.".to_string(), description: "Read the contents of up to 6 Galaxy Drive notebooks per call. Make another call for more. Notebooks are user-created rich-text documents stored in Galaxy Drive.".to_string(),
input_schema: serde_json::json!({ input_schema: serde_json::json!({
"type": "object", "type": "object",
"properties": { "properties": {
"document_ids": { "type": "array", "items": { "type": "string" }, "description": "Notebook IDs to read" } "document_ids": { "type": "array", "maxItems": 6, "items": { "type": "string" }, "description": "Notebook IDs to read (maximum 6 per call)" }
}, },
"required": ["document_ids"] "required": ["document_ids"]
}), }),
@@ -378,11 +378,11 @@ fn read_notebook() -> ToolDefinition {
fn create_notebook() -> ToolDefinition { fn create_notebook() -> ToolDefinition {
ToolDefinition { ToolDefinition {
name: "create_notebook".to_string(), name: "create_notebook".to_string(),
description: "Create a new notebook in Galaxy Drive. Notebooks are rich-text documents for general notes, documentation, and reference material.".to_string(), description: "Create exactly one notebook per call. Use separate calls for separate notebooks. Notebooks are rich-text documents for general notes, documentation, and reference material.".to_string(),
input_schema: serde_json::json!({ input_schema: serde_json::json!({
"type": "object", "type": "object",
"properties": { "properties": {
"documents": { "type": "array", "items": { "type": "object", "properties": { "title": { "type": "string" }, "content": { "type": "string" } }, "required": ["title", "content"] }, "description": "Notebooks to create" } "documents": { "type": "array", "maxItems": 1, "items": { "type": "object", "properties": { "title": { "type": "string" }, "content": { "type": "string" } }, "required": ["title", "content"] }, "description": "One notebook to create" }
}, },
"required": ["documents"] "required": ["documents"]
}), }),
@@ -392,12 +392,12 @@ fn create_notebook() -> ToolDefinition {
fn edit_notebook() -> ToolDefinition { fn edit_notebook() -> ToolDefinition {
ToolDefinition { ToolDefinition {
name: "edit_notebook".to_string(), name: "edit_notebook".to_string(),
description: "Edit an existing Galaxy Drive notebook using search/replace diffs." description: "Edit exactly one Galaxy Drive notebook per call with incremental search/replace diffs. Use separate calls for separate notebooks."
.to_string(), .to_string(),
input_schema: serde_json::json!({ input_schema: serde_json::json!({
"type": "object", "type": "object",
"properties": { "properties": {
"diffs": { "type": "array", "items": { "type": "object", "properties": { "document_id": { "type": "string" }, "search": { "type": "string" }, "replace": { "type": "string" } }, "required": ["document_id", "search", "replace"] }, "description": "Edits to apply to notebooks" } "diffs": { "type": "array", "maxItems": 1, "items": { "type": "object", "properties": { "document_id": { "type": "string" }, "search": { "type": "string" }, "replace": { "type": "string" } }, "required": ["document_id", "search", "replace"] }, "description": "One incremental edit to apply to a notebook" }
}, },
"required": ["diffs"] "required": ["diffs"]
}), }),
+67
View File
@@ -5,6 +5,13 @@
//! provider/model IDs, lifecycle states, counts, timings, and sanitized errors. //! provider/model IDs, lifecycle states, counts, timings, and sanitized errors.
use std::time::Duration; use std::time::Duration;
#[cfg(all(feature = "remote_logs_local_file", not(target_family = "wasm")))]
use std::{
fs::OpenOptions,
io::Write,
path::PathBuf,
sync::{Mutex, OnceLock},
};
use chrono::Utc; use chrono::Utc;
use galaxy_core::channel::ChannelState; use galaxy_core::channel::ChannelState;
@@ -58,6 +65,9 @@ struct RemoteLogPayload {
context: Value, context: Value,
} }
#[cfg(all(feature = "remote_logs_local_file", not(target_family = "wasm")))]
static LOCAL_LOG_FILE: OnceLock<Mutex<Option<PathBuf>>> = OnceLock::new();
impl RemoteLogConfig { impl RemoteLogConfig {
fn from_settings(settings: &AISettings) -> Option<Self> { fn from_settings(settings: &AISettings) -> Option<Self> {
if !*settings.remote_logging_enabled.value() { if !*settings.remote_logging_enabled.value() {
@@ -92,6 +102,9 @@ where
context: enrich_context(record.context), context: enrich_context(record.context),
}; };
#[cfg(all(feature = "remote_logs_local_file", not(target_family = "wasm")))]
write_local_log(&payload);
let _ = ctx.spawn( let _ = ctx.spawn(
async move { send_remote_log(config, payload).await }, async move { send_remote_log(config, payload).await },
|_, result, _| { |_, result, _| {
@@ -102,6 +115,60 @@ where
); );
} }
#[cfg(all(feature = "remote_logs_local_file", not(target_family = "wasm")))]
fn write_local_log(payload: &RemoteLogPayload) {
let state = LOCAL_LOG_FILE.get_or_init(|| Mutex::new(None));
let mut path = match state.lock() {
Ok(path) => path,
Err(error) => {
log::error!("[remote-logging] local log lock is poisoned: {error}");
return;
}
};
if path.is_none() {
let directory = std::env::temp_dir().join("galaxy-remote-logs");
if let Err(error) = std::fs::create_dir_all(&directory) {
log::error!(
"[remote-logging] failed to create local log directory {}: {error}",
directory.display()
);
return;
}
let file_path = directory.join(format!("galaxy-remote-{}.jsonl", std::process::id()));
*path = Some(file_path.clone());
log::warn!(
"[remote-logging] local diagnostic log enabled: {}",
file_path.display()
);
}
let Some(file_path) = path.as_ref() else {
return;
};
let line = match serde_json::to_string(payload) {
Ok(line) => line,
Err(error) => {
log::error!("[remote-logging] failed to serialize local log record: {error}");
return;
}
};
match OpenOptions::new().create(true).append(true).open(file_path) {
Ok(mut file) => {
if let Err(error) = writeln!(file, "{line}") {
log::error!(
"[remote-logging] failed to write local log {}: {error}",
file_path.display()
);
}
}
Err(error) => log::error!(
"[remote-logging] failed to open local log {}: {error}",
file_path.display()
),
}
}
pub(crate) fn sanitize_error(error: impl std::fmt::Display) -> String { pub(crate) fn sanitize_error(error: impl std::fmt::Display) -> String {
let compact = error let compact = error
.to_string() .to_string()
@@ -23,6 +23,11 @@ pub(crate) const BASE_PROVIDER_PROFILE: &str = "base";
pub(crate) const CLI_MONITOR_PROVIDER_PROFILE: &str = "cli-monitor"; pub(crate) const CLI_MONITOR_PROVIDER_PROFILE: &str = "cli-monitor";
const PROVIDER_MODEL_START_TIMEOUT: Duration = Duration::from_secs(120); const PROVIDER_MODEL_START_TIMEOUT: Duration = Duration::from_secs(120);
const PROVIDER_MODEL_EVENT_IDLE_TIMEOUT: Duration = Duration::from_secs(300); const PROVIDER_MODEL_EVENT_IDLE_TIMEOUT: Duration = Duration::from_secs(300);
const PROVIDER_MODEL_RETRY_DELAYS: [Duration; 3] = [
Duration::from_secs(1),
Duration::from_secs(3),
Duration::from_secs(5),
];
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub(crate) enum ProviderRunProjection { pub(crate) enum ProviderRunProjection {
@@ -856,6 +861,7 @@ impl ProviderRunCoordinator {
where where
F: FnMut(ProviderRunProjection) -> BoxFuture<'static, Result<(), String>>, F: FnMut(ProviderRunProjection) -> BoxFuture<'static, Result<(), String>>,
{ {
let error_message = error.message.clone();
let disposition = self let disposition = self
.run .run
.register_model_failure(&call.work_id, error.clone())?; .register_model_failure(&call.work_id, error.clone())?;
@@ -890,7 +896,25 @@ impl ProviderRunCoordinator {
project, project,
) )
.await?; .await?;
let delay = PROVIDER_MODEL_RETRY_DELAYS
.get(retry_attempt.saturating_sub(1) as usize)
.copied()
.unwrap_or_default();
log::warn!(
"rig model call failed; retrying attempt {retry_attempt} after {}s: {}",
delay.as_secs(),
error_message
);
if !delay.is_zero() {
Timer::after(delay).await;
}
return Ok(());
} }
log::error!(
"rig model call failed permanently after {} retries: {}",
call.retry_attempt,
error_message
);
Ok(()) Ok(())
} }
+13
View File
@@ -80,6 +80,11 @@ pub(crate) async fn prepare_provider_run(
let skill_path_origin = params.session_context.skill_path_origin(); let skill_path_origin = params.session_context.skill_path_origin();
let max_context_tokens = params.context_window_limit; let max_context_tokens = params.context_window_limit;
let mut cli_params = params.clone(); let mut cli_params = params.clone();
let cli_model_is_placeholder = params.cli_agent_model.as_str().trim().is_empty()
|| params
.cli_agent_model
.as_str()
.eq_ignore_ascii_case("placeholder");
let cli_provider_config = match cli_provider_config { let cli_provider_config = match cli_provider_config {
crate::ai::provider::ProviderConfig::None => { crate::ai::provider::ProviderConfig::None => {
// The CLI model can be absent from a model-specific provider routing table even when // The CLI model can be absent from a model-specific provider routing table even when
@@ -87,6 +92,14 @@ pub(crate) async fn prepare_provider_run(
cli_params.model = params.model.clone(); cli_params.model = params.model.clone();
base_provider_config.clone() base_provider_config.clone()
} }
provider_config if cli_model_is_placeholder => {
// A placeholder CLI model is used while preferences are still
// loading. Never send it to a provider: fall back to the working
// base model so command monitoring cannot terminate the run with
// a provider-side invalid-model error.
cli_params.model = params.model.clone();
base_provider_config.clone()
}
provider_config => { provider_config => {
cli_params.model = params.cli_agent_model.clone(); cli_params.model = params.cli_agent_model.clone();
provider_config provider_config
+81 -24
View File
@@ -1,4 +1,4 @@
use std::collections::HashMap; use std::collections::{HashMap, HashSet};
use std::time::Duration; use std::time::Duration;
use ai::diff_validation::ParsedDiff; use ai::diff_validation::ParsedDiff;
@@ -18,6 +18,10 @@ use crate::ai::agent::{
}; };
use crate::ai::document::ai_document_model::AIDocumentId; use crate::ai::document::ai_document_model::AIDocumentId;
const MAX_BATCH_READ_ITEMS: usize = 6;
const MAX_BATCH_SEARCH_ITEMS: usize = 3;
const MAX_RUN_AGENTS: usize = 8;
pub(crate) fn action_from_tool_call( pub(crate) fn action_from_tool_call(
task_id: &str, task_id: &str,
call: &ToolCall, call: &ToolCall,
@@ -47,7 +51,7 @@ pub(crate) fn action_from_tool_call(
citations: Vec::new(), citations: Vec::new(),
}, },
"read_files" => AIAgentActionType::ReadFiles(ReadFilesRequest { "read_files" => AIAgentActionType::ReadFiles(ReadFilesRequest {
locations: required_array(input, "files")? locations: limited_required_array(input, "files", MAX_BATCH_READ_ITEMS)?
.iter() .iter()
.enumerate() .enumerate()
.map(|(index, file)| file_location(file, index)) .map(|(index, file)| file_location(file, index))
@@ -58,11 +62,11 @@ pub(crate) fn action_from_tool_call(
title: Some(required_string(input, "summary")?), title: Some(required_string(input, "summary")?),
}, },
"grep" => AIAgentActionType::Grep { "grep" => AIAgentActionType::Grep {
queries: required_strings(input, "queries")?, queries: limited_required_strings(input, "queries", MAX_BATCH_SEARCH_ITEMS)?,
path: optional_string(input, "path")?.unwrap_or_default(), path: optional_string(input, "path")?.unwrap_or_default(),
}, },
"file_glob" => AIAgentActionType::FileGlob { "file_glob" => AIAgentActionType::FileGlob {
patterns: required_strings(input, "patterns")?, patterns: limited_required_strings(input, "patterns", MAX_BATCH_SEARCH_ITEMS)?,
path: optional_string(input, "path")?.filter(|path| !path.is_empty()), path: optional_string(input, "path")?.filter(|path| !path.is_empty()),
}, },
"search_codebase" => AIAgentActionType::SearchCodebase(SearchCodebaseRequest { "search_codebase" => AIAgentActionType::SearchCodebase(SearchCodebaseRequest {
@@ -110,19 +114,23 @@ pub(crate) fn action_from_tool_call(
}, },
"read_plan" | "read_documents" | "read_notebook" => { "read_plan" | "read_documents" | "read_notebook" => {
AIAgentActionType::ReadDocuments(ReadDocumentsRequest { AIAgentActionType::ReadDocuments(ReadDocumentsRequest {
document_ids: required_strings(input, "document_ids")? document_ids: limited_required_strings(
.into_iter() input,
.map(|id| { "document_ids",
AIDocumentId::try_from(id.clone()).map_err(|_| { MAX_BATCH_READ_ITEMS,
format!("invalid document_ids entry: {id:?} is not a document ID") )?
}) .into_iter()
.map(|id| {
AIDocumentId::try_from(id.clone()).map_err(|_| {
format!("invalid document_ids entry: {id:?} is not a document ID")
}) })
.collect::<Result<_, _>>()?, })
.collect::<Result<_, _>>()?,
}) })
} }
"create_plan" | "create_documents" | "create_notebook" => { "create_plan" | "create_documents" | "create_notebook" => {
AIAgentActionType::CreateDocuments(CreateDocumentsRequest { AIAgentActionType::CreateDocuments(CreateDocumentsRequest {
documents: required_array(input, "documents")? documents: limited_required_array(input, "documents", 1)?
.iter() .iter()
.enumerate() .enumerate()
.map(|(index, document)| { .map(|(index, document)| {
@@ -137,7 +145,7 @@ pub(crate) fn action_from_tool_call(
} }
"edit_plan" | "edit_documents" | "edit_notebook" => { "edit_plan" | "edit_documents" | "edit_notebook" => {
AIAgentActionType::EditDocuments(EditDocumentsRequest { AIAgentActionType::EditDocuments(EditDocumentsRequest {
diffs: required_array(input, "diffs")? diffs: limited_required_array(input, "diffs", 1)?
.iter() .iter()
.enumerate() .enumerate()
.map(|(index, diff)| { .map(|(index, diff)| {
@@ -160,18 +168,22 @@ pub(crate) fn action_from_tool_call(
model_id: optional_string(input, "model_id")?.unwrap_or_default(), model_id: optional_string(input, "model_id")?.unwrap_or_default(),
harness_type: optional_string(input, "harness_type")?.unwrap_or_default(), harness_type: optional_string(input, "harness_type")?.unwrap_or_default(),
execution_mode: run_agents_execution_mode(input)?, execution_mode: run_agents_execution_mode(input)?,
agent_run_configs: nonempty_required_array(input, "agent_run_configs")? agent_run_configs: limited_nonempty_required_array(
.iter() input,
.enumerate() "agent_run_configs",
.map(|(index, config)| { MAX_RUN_AGENTS,
require_object(config, &format!("agent_run_configs[{index}]"))?; )?
Ok(RunAgentsAgentRunConfig { .iter()
name: required_nonempty_string(config, "name")?, .enumerate()
prompt: required_nonempty_string(config, "prompt")?, .map(|(index, config)| {
title: optional_string(config, "title")?.unwrap_or_default(), require_object(config, &format!("agent_run_configs[{index}]"))?;
}) Ok(RunAgentsAgentRunConfig {
name: required_nonempty_string(config, "name")?,
prompt: required_nonempty_string(config, "prompt")?,
title: optional_string(config, "title")?.unwrap_or_default(),
}) })
.collect::<Result<_, String>>()?, })
.collect::<Result<_, String>>()?,
plan_id: optional_string(input, "plan_id")?.unwrap_or_default(), plan_id: optional_string(input, "plan_id")?.unwrap_or_default(),
harness_auth_secret_name: None, harness_auth_secret_name: None,
}), }),
@@ -317,6 +329,21 @@ fn required_array<'a>(
.ok_or_else(|| format!("invalid field {key:?}: expected an array")) .ok_or_else(|| format!("invalid field {key:?}: expected an array"))
} }
fn limited_required_array<'a>(
input: &'a serde_json::Value,
key: &str,
maximum: usize,
) -> Result<&'a Vec<serde_json::Value>, String> {
let values = required_array(input, key)?;
if values.len() > maximum {
return Err(format!(
"invalid field {key:?}: expected no more than {maximum} items, got {}",
values.len()
));
}
Ok(values)
}
fn nonempty_required_array<'a>( fn nonempty_required_array<'a>(
input: &'a serde_json::Value, input: &'a serde_json::Value,
key: &str, key: &str,
@@ -328,10 +355,30 @@ fn nonempty_required_array<'a>(
Ok(values) Ok(values)
} }
fn limited_nonempty_required_array<'a>(
input: &'a serde_json::Value,
key: &str,
maximum: usize,
) -> Result<&'a Vec<serde_json::Value>, String> {
let values = limited_required_array(input, key, maximum)?;
if values.is_empty() {
return Err(format!("invalid field {key:?}: expected at least one item"));
}
Ok(values)
}
fn required_strings(input: &serde_json::Value, key: &str) -> Result<Vec<String>, String> { fn required_strings(input: &serde_json::Value, key: &str) -> Result<Vec<String>, String> {
strings_from_array(required_array(input, key)?, key) strings_from_array(required_array(input, key)?, key)
} }
fn limited_required_strings(
input: &serde_json::Value,
key: &str,
maximum: usize,
) -> Result<Vec<String>, String> {
strings_from_array(limited_required_array(input, key, maximum)?, key)
}
fn optional_strings(input: &serde_json::Value, key: &str) -> Result<Option<Vec<String>>, String> { fn optional_strings(input: &serde_json::Value, key: &str) -> Result<Option<Vec<String>>, String> {
input input
.get(key) .get(key)
@@ -556,6 +603,16 @@ fn file_edits(input: &serde_json::Value) -> Result<Vec<FileEdit>, String> {
"invalid file edits: expected at least one diff, new file, or deleted file".to_string(), "invalid file edits: expected at least one diff, new file, or deleted file".to_string(),
); );
} }
let distinct_files = edits
.iter()
.filter_map(FileEdit::file)
.collect::<HashSet<_>>();
if distinct_files.len() > 1 {
return Err(format!(
"apply_file_diffs accepts one file per call; received {} distinct files. Apply each file in a separate call.",
distinct_files.len()
));
}
Ok(edits) Ok(edits)
} }
+3 -1
View File
@@ -98,7 +98,9 @@ impl Default for ProviderRunLimits {
fn default() -> Self { fn default() -> Self {
Self { Self {
max_model_turns: 100, max_model_turns: 100,
max_model_retries_per_turn: 2, // Three retries means the initial call plus three delayed retries;
// the fourth failure is surfaced to the user.
max_model_retries_per_turn: 3,
} }
} }
} }
@@ -930,7 +930,7 @@ fn restored_state_validation_rejects_retry_counter_and_terminal_corruption() {
); );
let excessive_retries = mutate_run_json(&run(), |value| { let excessive_retries = mutate_run_json(&run(), |value| {
value["model_retries"] = json!(3); value["model_retries"] = json!(4);
}); });
assert!(restored_state_error(&excessive_retries).contains("model-retry counter")); assert!(restored_state_error(&excessive_retries).contains("model-retry counter"));
+2 -2
View File
@@ -513,13 +513,13 @@ fn init_internal(
let mut base_logger = env_logger::builder(); let mut base_logger = env_logger::builder();
base_logger.filter_level(LevelFilter::Info); base_logger.filter_level(LevelFilter::Warn);
// Only include `WARN` or higher logs for wgpu. By default, wgpu outputs logs at the `INFO` // Only include `WARN` or higher logs for wgpu. By default, wgpu outputs logs at the `INFO`
// level multiple times _per_ frame. See https://github.com/gfx-rs/wgpu/issues/3206. // level multiple times _per_ frame. See https://github.com/gfx-rs/wgpu/issues/3206.
// Naga is overly noisy at `DEBUG`, so increase to `INFO`. // Naga is overly noisy at `DEBUG`, so increase to `INFO`.
base_logger base_logger
.filter(Some("naga"), LevelFilter::Info) .filter(Some("naga"), LevelFilter::Warn)
.filter(Some("wgpu_core"), LevelFilter::Warn) .filter(Some("wgpu_core"), LevelFilter::Warn)
// Since we always pair an insertion with a deletion to avoid duplicate, // Since we always pair an insertion with a deletion to avoid duplicate,
// tantivy will log a lot of warnings for deleting a non-existing doc. // tantivy will log a lot of warnings for deleting a non-existing doc.