From a2a26331991c12caeb45bf625618cfb31af90791 Mon Sep 17 00:00:00 2001 From: Ryan Ward Date: Wed, 30 Sep 2026 09:55:24 -0500 Subject: [PATCH] Improve Galaxy permissions, computer-use routing, and agent UX --- AGENTS.md | 2 +- app/src/ai/blocklist/action_model.rs | 15 ++- .../ai/blocklist/action_model/risk_review.rs | 124 ++++++++++++++++++ app/src/ai/blocklist/block.rs | 1 + .../ai/blocklist/block/view_impl/output.rs | 97 +++++++++++--- app/src/ai/blocklist/controller.rs | 108 +++++++++++++-- app/src/ai/blocklist/controller_tests.rs | 3 + .../inline_action/requested_command.rs | 17 ++- app/src/ai/blocklist/permissions.rs | 10 +- app/src/ai/blocklist/permissions_tests.rs | 16 +-- app/src/ai/execution_profiles/editor/mod.rs | 35 +---- .../execution_profiles/editor/ui_helpers.rs | 8 +- .../ai/execution_profiles/profiles_tests.rs | 14 +- app/src/ai/llms.rs | 31 +++-- app/src/ai/llms_tests.rs | 30 +++++ app/src/ai/provider/request_translator.rs | 4 +- app/src/ai/runtime/mod.rs | 2 +- .../ai/runtime/provider_run_coordinator.rs | 78 ++++++----- .../runtime/provider_run_coordinator_tests.rs | 50 +++++++ app/src/ai/runtime/rig.rs | 29 ++++ app/src/ai/runtime/rig_request.rs | 2 +- app/src/ai/runtime/rig_tests.rs | 29 ++++ app/src/settings/onboarding.rs | 8 +- app/src/settings_view/ai_page.rs | 32 +---- .../settings_view/execution_profile_view.rs | 15 ++- app/src/terminal/view.rs | 5 + app/src/workspaces/gql_convert.rs | 2 +- .../src/ai_execution_profile.rs | 12 +- crates/galaxy_agent_core/src/provider_run.rs | 12 ++ .../src/provider_run_tests.rs | 12 ++ 30 files changed, 623 insertions(+), 180 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 50a8aab7..f14a9d73 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -218,7 +218,7 @@ Key invariants: ### Galaxy Decides Permissions -- `GalaxyDecides` is an opt-in execution-profile permission for Galaxy-owned files, commands, PTY input, MCP, computer use, and agent coordination. External CLI agents retain their own permission flows. Existing profile defaults remain unchanged. +- `GalaxyDecides` is an execution-profile permission for Galaxy-owned files, commands, PTY input, MCP, computer use, and agent coordination. External CLI agents retain their own permission flows. Legacy `AgentDecides` profile values deserialize as `GalaxyDecides`; new profiles use it for file reads, file edits, and MCP. - `app/src/ai/blocklist/action_model/risk_review.rs` assesses the exact pending action with the active terminal-use model in a separate, tool-free request, falling back to the base model when the terminal-use model is unconfigured. Risk 0–3 permits automatic execution; 4–5 requires user approval. Existing denylist/protected-path/organization restrictions still require confirmation. - Invalid, unavailable, oversized, or timed-out assessments never grant execution. Assessment calls are bounded to 30 seconds; approval requests expire after 60 seconds and return a correlated denial telling the agent to choose another approach. - Reviews are scoped to conversation, action contents, and provider execution identity. Cancellation, replacement, and conversation cleanup invalidate them and cancel the reviewer request. Never treat a late callback or expired approval click as authorization. diff --git a/app/src/ai/blocklist/action_model.rs b/app/src/ai/blocklist/action_model.rs index 7b8d9f28..b7748b6b 100644 --- a/app/src/ai/blocklist/action_model.rs +++ b/app/src/ai/blocklist/action_model.rs @@ -15,7 +15,6 @@ mod execute; mod preprocess; mod risk_review; - use std::collections::{HashMap, HashSet, VecDeque}; use std::path::PathBuf; use std::sync::Arc; @@ -43,6 +42,7 @@ use galaxyui::{AppContext, Entity, EntityId, ModelContext, ModelHandle, Singleto use itertools::Itertools; use parking_lot::FairMutex; use preprocess::{PendingPreprocessedActions, PreprocessId}; +pub(crate) use risk_review::{ReviewDisplay, ReviewDisplayState}; use self::execute::ask_user_question::AskUserQuestionExecutor; use self::execute::search_codebase::SearchCodebaseExecutor; @@ -784,6 +784,7 @@ pub struct BlocklistAIActionModel { denied_permissions: HashSet<(AIConversationId, AIAgentActionId)>, pending_permissions: HashSet<(AIConversationId, AIAgentActionId)>, action_reviews: HashMap<(AIConversationId, AIAgentActionId), risk_review::ActionReview>, + review_outcomes: HashMap<(AIConversationId, AIAgentActionId), risk_review::ReviewOutcome>, permission_denial_reasons: HashMap<(AIConversationId, AIAgentActionId), String>, /// Actions parked because their executor-specific UI or state was not ready yet. @@ -911,6 +912,7 @@ impl BlocklistAIActionModel { denied_permissions: Default::default(), pending_permissions: Default::default(), action_reviews: Default::default(), + review_outcomes: Default::default(), permission_denial_reasons: Default::default(), not_ready_actions: Default::default(), deferred_provider_preprocessing: Default::default(), @@ -2214,6 +2216,15 @@ impl BlocklistAIActionModel { pending_action: &AIAgentAction, ctx: &mut ModelContext, ) { + if self + .review_reason(conversation_id, &pending_action.id) + .is_some() + { + self.review_outcomes.insert( + (conversation_id, pending_action.id.clone()), + risk_review::ReviewOutcome::denied_by_user(pending_action), + ); + } self.denied_permissions .insert((conversation_id, pending_action.id.clone())); #[cfg(not(target_family = "wasm"))] @@ -2363,6 +2374,8 @@ impl BlocklistAIActionModel { .retain(|(conv_id, _)| *conv_id != conversation_id); self.action_reviews .retain(|(conv_id, _), _| *conv_id != conversation_id); + self.review_outcomes + .retain(|(conv_id, _), _| *conv_id != conversation_id); self.permission_denial_reasons .retain(|(conv_id, _), _| *conv_id != conversation_id); self.pending_actions.remove(&conversation_id); diff --git a/app/src/ai/blocklist/action_model/risk_review.rs b/app/src/ai/blocklist/action_model/risk_review.rs index c3fc8572..64acee90 100644 --- a/app/src/ai/blocklist/action_model/risk_review.rs +++ b/app/src/ai/blocklist/action_model/risk_review.rs @@ -84,6 +84,57 @@ enum ReviewDecision { Ask { reason: String, deadline: Instant }, } +#[derive(Clone, Copy)] +pub(crate) enum ReviewDisplayState { + Assessing, + AwaitingApproval, + Approved, + Denied, +} + +pub(crate) struct ReviewDisplay { + pub(crate) state: ReviewDisplayState, + pub(crate) summary: String, + pub(crate) detail: String, +} + +pub(super) struct ReviewOutcome { + summary: String, + detail: String, + approved: bool, +} + +impl ReviewOutcome { + pub(super) fn denied_by_user(action: &AIAgentAction) -> Self { + Self { + summary: review_summary(action), + detail: "Denied by you.".to_string(), + approved: false, + } + } +} + +fn review_summary(action: &AIAgentAction) -> String { + let description = match &action.action { + AIAgentActionType::RequestCommandOutput { command, .. } => { + format!("Run: {command}") + } + _ => action.action.user_friendly_name(), + }; + let summary = description.split_whitespace().collect::>().join(" "); + shorten_for_review(&summary, 160) +} + +fn shorten_for_review(text: &str, limit: usize) -> String { + let mut characters = text.chars(); + let shortened = characters.by_ref().take(limit).collect::(); + if characters.next().is_some() { + format!("{shortened}…") + } else { + shortened + } +} + impl BlocklistAIActionModel { fn review_environment(&self, ctx: &AppContext) -> ReviewEnvironment { let session = self.active_session.as_ref(ctx); @@ -267,6 +318,7 @@ impl BlocklistAIActionModel { && self.review_requires_human(action, conversation_id, ctx)) }) { self.action_reviews.remove(&key); + self.review_outcomes.remove(&key); } if let Some(ActionReview { decision: ReviewDecision::Ask { deadline, .. }, @@ -279,6 +331,20 @@ impl BlocklistAIActionModel { } } if user_initiated || !self.uses_galaxy_decides(action, ctx) { + if user_initiated { + if let Some(review) = self.action_reviews.get(&key) { + if let ReviewDecision::Ask { reason, .. } = &review.decision { + self.review_outcomes.insert( + key.clone(), + ReviewOutcome { + summary: review_summary(action), + detail: format!("Approved by you. {reason}"), + approved: true, + }, + ); + } + } + } self.action_reviews.remove(&key); return Some(false); } @@ -289,6 +355,7 @@ impl BlocklistAIActionModel { }; } let token = uuid::Uuid::new_v4(); + self.review_outcomes.remove(&key); let (cancel, control) = turn_control(); let execution_ref = self.provider_tool_execution_ref(conversation_id, &action.id); self.action_reviews.insert( @@ -408,6 +475,18 @@ impl BlocklistAIActionModel { .as_ref() .is_ok_and(|assessment| assessment.allows_automatic_execution(policy_requires_human)) { + let assessment = result.as_ref().unwrap(); + self.review_outcomes.insert( + key.clone(), + ReviewOutcome { + summary: review_summary(&action), + detail: format!( + "Approved by Galaxy · risk {}/5. {}", + assessment.risk, assessment.reason + ), + approved: true, + }, + ); self.action_reviews.get_mut(&key).unwrap().decision = ReviewDecision::Allow; self.try_to_execute_available_actions(conversation_id, ctx); return; @@ -486,6 +565,43 @@ impl BlocklistAIActionModel { } } + pub(crate) fn review_display( + &self, + conversation_id: AIConversationId, + action_id: &AIAgentActionId, + ) -> Option { + let key = (conversation_id, action_id.clone()); + if let Some(review) = self.action_reviews.get(&key) { + let summary = review_summary(&review.action); + return Some(match &review.decision { + ReviewDecision::Assessing => ReviewDisplay { + state: ReviewDisplayState::Assessing, + summary, + detail: "Galaxy is assessing the permission risk…".to_string(), + }, + ReviewDecision::Ask { reason, .. } => ReviewDisplay { + state: ReviewDisplayState::AwaitingApproval, + summary, + detail: shorten_for_review(reason, 320), + }, + ReviewDecision::Allow => ReviewDisplay { + state: ReviewDisplayState::Approved, + summary, + detail: shorten_for_review(&self.review_outcomes.get(&key)?.detail, 320), + }, + }); + } + self.review_outcomes.get(&key).map(|outcome| ReviewDisplay { + state: if outcome.approved { + ReviewDisplayState::Approved + } else { + ReviewDisplayState::Denied + }, + summary: outcome.summary.clone(), + detail: shorten_for_review(&outcome.detail, 320), + }) + } + fn reject_reviewed_action( &mut self, action: &AIAgentAction, @@ -507,6 +623,14 @@ impl BlocklistAIActionModel { return; }; let action = queue.remove(index).unwrap(); + self.review_outcomes.insert( + key.clone(), + ReviewOutcome { + summary: review_summary(&action), + detail: "Denied automatically after 60 seconds without a response.".to_string(), + approved: false, + }, + ); self.pending_permissions.remove(&key); #[cfg(not(target_family = "wasm"))] log_tool_event( diff --git a/app/src/ai/blocklist/block.rs b/app/src/ai/blocklist/block.rs index 6eddbba5..f89b7438 100644 --- a/app/src/ai/blocklist/block.rs +++ b/app/src/ai/blocklist/block.rs @@ -4918,6 +4918,7 @@ impl AIBlock { } BlocklistAIActionEvent::ActionBlockedOnUserConfirmation { .. } => { ctx.emit(AIBlockEvent::ActionBlockedOnUserConfirmation); + ctx.notify(); } BlocklistAIActionEvent::FinishedAction { action_id, .. } => { me.abort_auto_expand_requested_command_timer(); diff --git a/app/src/ai/blocklist/block/view_impl/output.rs b/app/src/ai/blocklist/block/view_impl/output.rs index 87665423..72a1acf1 100644 --- a/app/src/ai/blocklist/block/view_impl/output.rs +++ b/app/src/ai/blocklist/block/view_impl/output.rs @@ -70,7 +70,7 @@ use crate::ai::agent::{ }; use crate::ai::agent_conversations_model::AgentConversationsModel; use crate::ai::ambient_agents::AmbientAgentTaskId; -use crate::ai::blocklist::action_model::AIActionStatus; +use crate::ai::blocklist::action_model::{AIActionStatus, ReviewDisplay, ReviewDisplayState}; use crate::ai::blocklist::block::model::{AIBlockModel, AIBlockModelHelper, AIBlockOutputStatus}; use crate::ai::blocklist::block::view_impl::common::{ MaybeShimmeringText, BLOCKED_ACTION_MESSAGE_FOR_GREP_OR_FILE_GLOB, @@ -319,23 +319,12 @@ pub(super) fn render(props: Props, app: &AppContext) -> Box { for output_message in output.messages.iter() { if let AIAgentOutputMessageType::Action(action) = &output_message.message { - if let Some(reason) = props + if let Some(review) = props .action_model .as_ref(app) - .review_reason(props.conversation_id, &action.id) + .review_display(props.conversation_id, &action.id) { - output_items.add_child( - Text::new( - reason, - appearance.ai_font_family(), - appearance.monospace_font_size(), - ) - .with_color(blended_colors::text_main( - appearance.theme(), - appearance.theme().surface_1(), - )) - .finish(), - ); + output_items.add_child(render_review_status(review, app)); } } match &output_message.message { @@ -1331,6 +1320,55 @@ fn render_streaming_tool_call( render_tool_pane_shell(Clipped::new(header).finish(), false, false, false, app) } +fn render_review_status(review: ReviewDisplay, app: &AppContext) -> Box { + let appearance = Appearance::as_ref(app); + let icon = match review.state { + ReviewDisplayState::Assessing => Icon::Loading.to_galaxyui_icon( + galaxy_core::ui::theme::Fill::Solid(internal_colors::neutral_6(appearance.theme())), + ), + ReviewDisplayState::AwaitingApproval => icons::yellow_stop_icon(appearance), + ReviewDisplayState::Approved => inline_action_icons::green_check_icon(appearance), + ReviewDisplayState::Denied => inline_action_icons::red_x_icon(appearance), + }; + let title = match review.state { + ReviewDisplayState::Assessing => "Checking permission", + ReviewDisplayState::AwaitingApproval => "Approval required · 60 seconds", + ReviewDisplayState::Approved => "Approved", + ReviewDisplayState::Denied => "Denied", + }; + let header = HeaderConfig::new(title, app) + .with_icon(icon) + .with_corner_radius_override(CornerRadius::with_all(Radius::Pixels(8.))) + .render(app); + let detail = format!("{}\n{}", review.summary, review.detail); + let body = Container::new( + Text::new( + detail, + appearance.ai_font_family(), + appearance.monospace_font_size(), + ) + .with_color(blended_colors::text_main( + appearance.theme(), + appearance.theme().surface_1(), + )) + .finish(), + ) + .with_margin_left(16.) + .with_margin_right(16.) + .with_margin_bottom(12.) + .finish(); + render_tool_pane_shell( + Flex::column() + .with_child(Clipped::new(header).finish()) + .with_child(body) + .finish(), + matches!(review.state, ReviewDisplayState::AwaitingApproval), + false, + false, + app, + ) +} + fn render_runtime_activity( output_message: &AIAgentOutputMessage, activity: &RuntimeActivity, @@ -3084,9 +3122,32 @@ fn render_use_computer( app: &AppContext, ) -> Box { let appearance = Appearance::handle(app).as_ref(app); - - let mut renderable_action = RenderableAction::new(&request.action_summary, app) - .with_icon(action_icon(action_id, props.action_model, props.model, app).finish()); + let status = props + .action_model + .as_ref(app) + .get_action_status(props.conversation_id, action_id); + let mut renderable_action = RenderableAction::new(&request.action_summary, app); + if status.as_ref().is_some_and(|status| status.is_blocked()) { + let buttons = props + .action_buttons + .get(action_id) + .expect("Button states must exist for each requested action."); + renderable_action = renderable_action + .with_header(blocked_action_header( + action_id.clone(), + "OK if I use computer control for this action?", + buttons.run_button.clone(), + buttons.cancel_button.clone(), + props.action_model, + props.model, + app, + )) + .with_highlighted_border() + .with_background_color(appearance.theme().background().into_solid()); + } else { + renderable_action = renderable_action + .with_icon(action_icon(action_id, props.action_model, props.model, app).finish()); + } // Add a "View screenshot" button if the action result contains a screenshot. let has_screenshot = props diff --git a/app/src/ai/blocklist/controller.rs b/app/src/ai/blocklist/controller.rs index fb4d679f..e5eb52f4 100644 --- a/app/src/ai/blocklist/controller.rs +++ b/app/src/ai/blocklist/controller.rs @@ -87,6 +87,7 @@ use crate::ai::runtime::{ ProviderRunCoordinator, ProviderRunProfile, ProviderRunProjection, ProviderRunResponseProjector, ProviderToolExecutionRef, ProviderToolLifecycleOutcome, RuntimeResponseConfig, BASE_PROVIDER_PROFILE, CLI_MONITOR_PROVIDER_PROFILE, + COMPUTER_USE_PROVIDER_PROFILE, }; use crate::ai::AIRequestUsageModel; use crate::cloud_object::model::persistence::CloudModel; @@ -1206,6 +1207,7 @@ struct ActiveProviderRunCheckpoint { run: ProviderRun, base_request: TurnRequest, cli_monitor_request: Option, + computer_use_request: Option, response_config: RuntimeResponseConfig, action_context: ProviderActionContext, persistence_offset: usize, @@ -1229,10 +1231,15 @@ impl ActiveProviderRunCheckpoint { .coordinator .profile_request(CLI_MONITOR_PROVIDER_PROFILE) .cloned(); + let computer_use_request = run + .coordinator + .profile_request(COMPUTER_USE_PROVIDER_PROFILE) + .cloned(); Ok(Self { run: run.coordinator.run().clone(), base_request, cli_monitor_request, + computer_use_request, response_config: run.response_config.clone(), action_context: run.action_context.clone(), persistence_offset: run.persistence_offset, @@ -1255,6 +1262,8 @@ struct ActiveProviderRunSnapshot { run: ProviderRun, base_request: TurnRequest, cli_monitor_request: Option, + #[serde(default)] + computer_use_request: Option, response_config: RuntimeResponseConfig, action_context: ProviderActionContext, projection_target: ProviderProjectionTarget, @@ -1303,6 +1312,7 @@ impl ActiveProviderRunSnapshot { run: checkpoint.run, base_request: checkpoint.base_request, cli_monitor_request: checkpoint.cli_monitor_request, + computer_use_request: checkpoint.computer_use_request, response_config: checkpoint.response_config, action_context: checkpoint.action_context, projection_target: slot.projection_target.clone(), @@ -1372,6 +1382,13 @@ impl ActiveProviderRunSnapshot { "provider run uses the CLI profile without a persisted request".to_string(), ); } + COMPUTER_USE_PROVIDER_PROFILE if self.computer_use_request.is_some() => {} + COMPUTER_USE_PROVIDER_PROFILE => { + return Err( + "provider run uses the computer-use profile without a persisted request" + .to_string(), + ); + } profile => { return Err(format!( "provider run uses unknown request profile '{profile}'" @@ -2212,18 +2229,18 @@ fn provider_run_terminal_remote_log_record( enum ProviderDriveMessage { Projection { - lifecycle: Option, + lifecycle: Box>, latest_usage: Option, tool_call_progress: ProviderToolCallProgressUpdate, events: Vec, acknowledgement: oneshot::Sender>, }, Checkpoint { - checkpoint: ActiveProviderRunCheckpoint, + checkpoint: Box, acknowledgement: oneshot::Sender>, }, Blocked { - run: ActiveProviderRun, + run: Box, result: Result, }, } @@ -5881,6 +5898,10 @@ impl BlocklistAIController { .cli_monitor_request .as_ref() .map(|request| ResponseStream::resolve_provider_config(request.model.as_str(), ctx)); + let computer_use_provider_config = snapshot + .computer_use_request + .as_ref() + .map(|request| ResponseStream::resolve_provider_config(request.model.as_str(), ctx)); let _ = ctx.spawn( async move { let base_runtime = @@ -5910,6 +5931,23 @@ impl BlocklistAIController { .with_max_context_tokens(cli_context_limit), ); } + if let Some(computer_use_request) = snapshot.computer_use_request.as_ref() { + let provider_config = computer_use_provider_config.ok_or_else(|| { + anyhow!("restored computer-use request is missing its provider config") + })?; + let context_limit = provider_context_window_tokens( + &provider_config, + computer_use_request.model.as_str(), + base_context_limit, + ); + let runtime = + provider_runtime_for_request(provider_config, computer_use_request).await?; + profiles.insert( + COMPUTER_USE_PROVIDER_PROFILE.to_string(), + ProviderRunProfile::new(runtime, computer_use_request.clone()) + .with_max_context_tokens(context_limit), + ); + } Ok::<_, anyhow::Error>(PreparedRestoredProviderRun { snapshot, profiles, @@ -6071,6 +6109,7 @@ impl BlocklistAIController { run: provider_run, base_request: _, cli_monitor_request: _, + computer_use_request: _, response_config, action_context, projection_target, @@ -6533,11 +6572,16 @@ impl BlocklistAIController { credit_multiplier: model.usage_metadata.credit_multiplier, }) .collect(); + let computer_use_provider_config = ResponseStream::resolve_provider_config( + request_params.computer_use_model.as_str(), + ctx, + ); let _ = ctx.spawn( async move { prepare_provider_run( base_provider_config, cli_provider_config, + computer_use_provider_config, request_params, orchestration_models, ) @@ -6587,6 +6631,7 @@ impl BlocklistAIController { let PreparedProviderRun { base_profile, cli_monitor_profile, + computer_use_profile, tool_result_archive, messages_sent, persistence_offset, @@ -6612,6 +6657,12 @@ impl BlocklistAIController { return; } } + if let Some(profile) = computer_use_profile { + if let Err(error) = coordinator.insert_profile(COMPUTER_USE_PROVIDER_PROFILE, profile) { + self.fail_provider_startup(conversation_id, stream_id, error.to_string(), ctx); + return; + } + } let cancellation_reason = self .active_provider_runs .get(&conversation_id) @@ -7481,7 +7532,7 @@ impl BlocklistAIController { let (acknowledgement, receiver) = oneshot::channel(); projection_sender .send(ProviderDriveMessage::Projection { - lifecycle, + lifecycle: Box::new(lifecycle), latest_usage, tool_call_progress, events, @@ -7503,7 +7554,7 @@ impl BlocklistAIController { let (acknowledgement, receiver) = oneshot::channel(); checkpoint_sender .send(ProviderDriveMessage::Checkpoint { - checkpoint, + checkpoint: Box::new(checkpoint), acknowledgement, }) .await @@ -7519,7 +7570,10 @@ impl BlocklistAIController { .await .map_err(|error| error.to_string()); let _ = sender - .send(ProviderDriveMessage::Blocked { run, result }) + .send(ProviderDriveMessage::Blocked { + run: Box::new(run), + result, + }) .await; }, |_, _, _| {}, @@ -7598,7 +7652,7 @@ impl BlocklistAIController { } }; let mut should_refresh_status = progress_changed; - if let Some(lifecycle) = lifecycle.as_ref() { + if let Some(lifecycle) = lifecycle.as_ref().as_ref() { should_refresh_status |= { let slot = self .active_provider_runs @@ -7648,7 +7702,7 @@ impl BlocklistAIController { ); }); } - let model_turn_requested = lifecycle.as_ref().is_some_and(|lifecycle| { + let model_turn_requested = lifecycle.as_ref().as_ref().is_some_and(|lifecycle| { matches!(lifecycle.phase, ProviderLlmLifecyclePhase::Requested) }); let projection_succeeded = result.is_ok(); @@ -7681,7 +7735,7 @@ impl BlocklistAIController { } => { let result = match self.active_provider_runs.get_mut(&conversation_id) { Some(slot) if checkpoint.run.id() == &slot.run_id => { - slot.checkpoint = Some(checkpoint); + slot.checkpoint = Some(*checkpoint); self.persist_active_provider_run(conversation_id, ctx) } Some(_) => Err("provider checkpoint run identity did not match".to_string()), @@ -7690,7 +7744,7 @@ impl BlocklistAIController { let _ = acknowledgement.send(result); } ProviderDriveMessage::Blocked { run, result } => { - self.handle_provider_run_blocked(conversation_id, run, result, ctx); + self.handle_provider_run_blocked(conversation_id, *run, result, ctx); } } } @@ -8263,10 +8317,44 @@ impl BlocklistAIController { ) { return; } + // Screenshot results belong to the next model turn. Select its runtime only after + // the complete tool batch is committed, so a restart sees the same decision. + let next_tool_profile = match run.coordinator.run().state() { + ProviderRunState::AwaitingTools { batch } => { + let used_computer = batch.calls.iter().any(|pending| { + matches!( + pending.call.name.as_str(), + "request_computer_use" | "use_computer" + ) + }); + used_computer.then_some(COMPUTER_USE_PROVIDER_PROFILE) + } + _ => None, + }; let mut should_resume = false; let should_drive = match run.coordinator.apply_tool_lifecycle(execution_ref, event) { Ok(ProviderToolLifecycleOutcome::Pending) => false, Ok(ProviderToolLifecycleOutcome::BatchCommitted) => { + if let Some(profile) = next_tool_profile.filter(|_| { + run.coordinator + .profile_request(COMPUTER_USE_PROVIDER_PROFILE) + .is_some() + }) { + if let Err(error) = run.coordinator.run_mut().set_ready_profile(profile) { + self.fail_active_provider_run(conversation_id, error.to_string(), ctx); + return; + } + } else if run.coordinator.run().profile().as_str() == COMPUTER_USE_PROVIDER_PROFILE + { + if let Err(error) = run + .coordinator + .run_mut() + .set_ready_profile(BASE_PROVIDER_PROFILE) + { + self.fail_active_provider_run(conversation_id, error.to_string(), ctx); + return; + } + } let work_id = execution_ref.work_id(); should_resume = record_provider_batch_signal( &mut slot.committed_provider_batch, diff --git a/app/src/ai/blocklist/controller_tests.rs b/app/src/ai/blocklist/controller_tests.rs index f92652be..097e5005 100644 --- a/app/src/ai/blocklist/controller_tests.rs +++ b/app/src/ai/blocklist/controller_tests.rs @@ -260,6 +260,7 @@ fn provider_snapshot(conversation_id: AIConversationId) -> super::ActiveProvider ), base_request: TurnRequest::new("provider-model", messages), cli_monitor_request: None, + computer_use_request: None, response_config: crate::ai::runtime::RuntimeResponseConfig { task_id: task_id.to_string(), conversation_id: conversation_id.to_string(), @@ -1042,6 +1043,7 @@ fn cancelling_provider_startup_keeps_slot_and_durable_cancellation() { run: snapshot.run.clone(), base_request: snapshot.base_request.clone(), cli_monitor_request: snapshot.cli_monitor_request.clone(), + computer_use_request: snapshot.computer_use_request.clone(), response_config: snapshot.response_config.clone(), action_context: snapshot.action_context.clone(), persistence_offset: snapshot.persistence_offset, @@ -1157,6 +1159,7 @@ fn same_conversation_follow_up_waits_for_cancelled_provider_generation_cleanup() run: old_snapshot.run.clone(), base_request: old_snapshot.base_request.clone(), cli_monitor_request: old_snapshot.cli_monitor_request.clone(), + computer_use_request: old_snapshot.computer_use_request.clone(), response_config: old_snapshot.response_config.clone(), action_context: old_snapshot.action_context.clone(), persistence_offset: old_snapshot.persistence_offset, diff --git a/app/src/ai/blocklist/inline_action/requested_command.rs b/app/src/ai/blocklist/inline_action/requested_command.rs index 537e6af0..aabdf43e 100644 --- a/app/src/ai/blocklist/inline_action/requested_command.rs +++ b/app/src/ai/blocklist/inline_action/requested_command.rs @@ -1250,13 +1250,20 @@ impl RequestedCommandView { self.block_model.as_ref().conversation(app).is_some_and( |conversation| conversation.status().is_error(), ); - - if is_errored { - AGENT_ERRORED_COMMAND_MESSAGE.into() + let (status, short_status) = if is_errored { + (AGENT_ERRORED_COMMAND_MESSAGE, "Agent needs help") } else if *is_blocked { - AGENT_NEEDS_INPUT_MESSAGE.into() + (AGENT_NEEDS_INPUT_MESSAGE, "Needs input") } else { - MONITORING_COMMAND_MESSAGE.into() + (MONITORING_COMMAND_MESSAGE, "Agent monitoring") + }; + let command = self.get_header_title_text(); + if command.trim().is_empty() { + status.into() + } else { + font_override = Some(appearance.monospace_font_family()); + badge = Some(short_status.to_string()); + command.into() } } LongRunningCommandControlState::User { reason } => { diff --git a/app/src/ai/blocklist/permissions.rs b/app/src/ai/blocklist/permissions.rs index 9063863e..8e639ebf 100644 --- a/app/src/ai/blocklist/permissions.rs +++ b/app/src/ai/blocklist/permissions.rs @@ -705,7 +705,7 @@ impl BlocklistAIPermissions { } match self.get_read_files_setting(ctx, terminal_view_id) { - ActionPermission::AgentDecides | ActionPermission::Unknown => { + ActionPermission::Unknown => { // For now, we always read files. We don't ask the user for permission. FileReadPermission::Allowed(FileReadPermissionAllowedReason::AgentDecided) } @@ -838,9 +838,7 @@ impl BlocklistAIPermissions { .is_some_and(|uid| self.get_mcp_denylist(ctx, terminal_view_id).contains(&uid)); match self.get_mcp_permissions_setting(ctx, terminal_view_id) { - ActionPermission::AgentDecides | ActionPermission::Unknown => { - allowlisted && !denylisted - } + ActionPermission::Unknown => allowlisted && !denylisted, ActionPermission::AlwaysAllow => !denylisted, ActionPermission::AlwaysAsk | ActionPermission::GalaxyDecides => { allowlisted && !denylisted @@ -855,7 +853,7 @@ impl BlocklistAIPermissions { ctx: &AppContext, ) -> FileWritePermission { match self.get_apply_code_diffs_setting(ctx, terminal_view_id) { - ActionPermission::AgentDecides | ActionPermission::Unknown => { + ActionPermission::Unknown => { FileWritePermission::Denied(FileWritePermissionDeniedReason::AgentDecided) } ActionPermission::AlwaysAllow => FileWritePermission::Allowed( @@ -915,7 +913,7 @@ impl BlocklistAIPermissions { } match self.get_execute_commands_setting(ctx, terminal_view_id) { - ActionPermission::AgentDecides | ActionPermission::Unknown => { + ActionPermission::Unknown => { if FeatureFlag::AgentDecidesCommandExecution.is_enabled() && is_risky == Some(false) { return CommandExecutionPermission::Allowed( diff --git a/app/src/ai/blocklist/permissions_tests.rs b/app/src/ai/blocklist/permissions_tests.rs index 594bc216..318d9f2e 100644 --- a/app/src/ai/blocklist/permissions_tests.rs +++ b/app/src/ai/blocklist/permissions_tests.rs @@ -337,11 +337,11 @@ fn test_can_write_files() { .. } = initialize_permissions_test(&mut app); - // Test AgentDecides setting + // Test GalaxyDecides setting before the independent review gate. profile_model.update(&mut app, |model, ctx| { model.set_apply_code_diffs( *model.active_profile(Some(terminal_view_id), ctx).id(), - &ActionPermission::AgentDecides, + &ActionPermission::GalaxyDecides, ctx, ); }); @@ -352,9 +352,9 @@ fn test_can_write_files() { assert!( matches!( result, - FileWritePermission::Denied(FileWritePermissionDeniedReason::AgentDecided) + FileWritePermission::Denied(FileWritePermissionDeniedReason::AlwaysAskEnabled) ), - "not allowed because AgentDecides right now just means ask" + "GalaxyDecides requires the independent action review" ); }); @@ -1223,7 +1223,7 @@ fn test_can_use_mcp_server_always_ask_denylist_overrides_allowlist() { } #[test] -fn test_can_use_mcp_server_agent_decides() { +fn test_can_use_mcp_server_galaxy_decides() { App::test((), |mut app| async move { let PermissionsTestState { convo_id, @@ -1239,7 +1239,7 @@ fn test_can_use_mcp_server_agent_decides() { profile_model.update(&mut app, |model, ctx| { model.set_mcp_permissions( *model.active_profile(Some(terminal_view_id), ctx).id(), - &ActionPermission::AgentDecides, + &ActionPermission::GalaxyDecides, ctx, ); model.add_to_mcp_allowlist( @@ -1269,7 +1269,7 @@ fn test_can_use_mcp_server_agent_decides() { } #[test] -fn test_can_use_mcp_server_agent_decides_denylist_overrides_allowlist() { +fn test_can_use_mcp_server_galaxy_decides_denylist_overrides_allowlist() { App::test((), |mut app| async move { let PermissionsTestState { convo_id, @@ -1284,7 +1284,7 @@ fn test_can_use_mcp_server_agent_decides_denylist_overrides_allowlist() { profile_model.update(&mut app, |model, ctx| { model.set_mcp_permissions( *model.active_profile(Some(terminal_view_id), ctx).id(), - &ActionPermission::AgentDecides, + &ActionPermission::GalaxyDecides, ctx, ); model.add_to_mcp_allowlist( diff --git a/app/src/ai/execution_profiles/editor/mod.rs b/app/src/ai/execution_profiles/editor/mod.rs index 758a15fa..3f31bbc6 100644 --- a/app/src/ai/execution_profiles/editor/mod.rs +++ b/app/src/ai/execution_profiles/editor/mod.rs @@ -284,12 +284,6 @@ impl ExecutionProfileEditorView { let mut dropdown = Dropdown::new(ctx); dropdown.set_items( vec![ - DropdownItem::new( - "Agent decides", - ExecutionProfileEditorViewAction::SetApplyCodeDiffs { - permission: ActionPermission::AgentDecides, - }, - ), DropdownItem::new( "Always allow", ExecutionProfileEditorViewAction::SetApplyCodeDiffs { @@ -318,12 +312,6 @@ impl ExecutionProfileEditorView { let mut dropdown = Dropdown::new(ctx); dropdown.set_items( vec![ - DropdownItem::new( - "Agent decides", - ExecutionProfileEditorViewAction::SetReadFiles { - permission: ActionPermission::AgentDecides, - }, - ), DropdownItem::new( "Always allow", ExecutionProfileEditorViewAction::SetReadFiles { @@ -352,12 +340,6 @@ impl ExecutionProfileEditorView { let mut dropdown = Dropdown::new(ctx); dropdown.set_items( vec![ - DropdownItem::new( - "Agent decides", - ExecutionProfileEditorViewAction::SetExecuteCommands { - permission: ActionPermission::AgentDecides, - }, - ), DropdownItem::new( "Always allow", ExecutionProfileEditorViewAction::SetExecuteCommands { @@ -420,12 +402,6 @@ impl ExecutionProfileEditorView { let mut dropdown = Dropdown::new(ctx); dropdown.set_items( vec![ - DropdownItem::new( - "Agent decides", - ExecutionProfileEditorViewAction::SetCallMcpServers { - permission: ActionPermission::AgentDecides, - }, - ), DropdownItem::new( "Always allow", ExecutionProfileEditorViewAction::SetCallMcpServers { @@ -824,7 +800,7 @@ impl ExecutionProfileEditorView { Self::refresh_filterable_model_dropdown( &me.computer_use_model_dropdown, current_permissions.computer_use_model.clone(), - |prefs, _| prefs.get_computer_use_llm_choices().collect_vec(), + |prefs, app| prefs.get_computer_use_llm_choices(app).collect_vec(), |id| ExecutionProfileEditorViewAction::SetComputerUseModel { id }, |prefs| prefs.get_default_computer_use_model().id.clone(), &me.upgrade_footer_mouse_state, @@ -1003,7 +979,7 @@ impl ExecutionProfileEditorView { Self::refresh_filterable_model_dropdown( &self.computer_use_model_dropdown, current_permissions.computer_use_model.clone(), - |prefs, _| prefs.get_computer_use_llm_choices().collect_vec(), + |prefs, app| prefs.get_computer_use_llm_choices(app).collect_vec(), |id| ExecutionProfileEditorViewAction::SetComputerUseModel { id }, |prefs| prefs.get_default_computer_use_model().id.clone(), &self.upgrade_footer_mouse_state, @@ -1091,10 +1067,9 @@ impl ExecutionProfileEditorView { } let active = match current_permission { - ActionPermission::AgentDecides | ActionPermission::Unknown => 0, - ActionPermission::AlwaysAllow => 1, - ActionPermission::AlwaysAsk => 2, - ActionPermission::GalaxyDecides => 3, + ActionPermission::AlwaysAllow => 0, + ActionPermission::AlwaysAsk | ActionPermission::Unknown => 1, + ActionPermission::GalaxyDecides => 2, }; menu.set_selected_by_index(active, ctx); diff --git a/app/src/ai/execution_profiles/editor/ui_helpers.rs b/app/src/ai/execution_profiles/editor/ui_helpers.rs index 5fc90d27..72f7dacb 100644 --- a/app/src/ai/execution_profiles/editor/ui_helpers.rs +++ b/app/src/ai/execution_profiles/editor/ui_helpers.rs @@ -474,9 +474,7 @@ pub fn render_permissions_section( ), ]); - if profile_data.read_files == ActionPermission::AlwaysAsk - || profile_data.read_files == ActionPermission::AgentDecides - { + if profile_data.read_files == ActionPermission::AlwaysAsk { column.add_child(render_directory_allowlist_section( view, profile_data, @@ -514,7 +512,7 @@ pub fn render_permissions_section( app, )); } - ActionPermission::AgentDecides | ActionPermission::Unknown => { + ActionPermission::Unknown => { column.add_children([ render_command_allowlist_section(view, profile_data, appearance, app), render_command_denylist_section(view, profile_data, appearance, app), @@ -600,7 +598,7 @@ pub fn render_permissions_section( appearance, )); } - ActionPermission::AgentDecides | ActionPermission::Unknown => { + ActionPermission::Unknown => { column.add_children([ render_mcp_allowlist_section(view, profile_data, app, appearance), render_mcp_denylist_section(view, profile_data, app, appearance), diff --git a/app/src/ai/execution_profiles/profiles_tests.rs b/app/src/ai/execution_profiles/profiles_tests.rs index e49a3245..1cab78d7 100644 --- a/app/src/ai/execution_profiles/profiles_tests.rs +++ b/app/src/ai/execution_profiles/profiles_tests.rs @@ -108,12 +108,12 @@ fn edits_persist_default_profile_locally_when_logged_out() { let default_profile_id = profile_model.read(&app, |model, _ctx| model.default_profile_id()); // Sanity-check the precondition: the baseline `apply_code_diffs` - // on a fresh default profile is the enum default (`AgentDecides`). + // on a fresh default profile is `GalaxyDecides`. profile_model.read(&app, |model, ctx| { assert!( matches!( model.default_profile(ctx).data().apply_code_diffs, - ActionPermission::AgentDecides + ActionPermission::GalaxyDecides ), "unexpected baseline apply_code_diffs" ); @@ -402,3 +402,13 @@ fn filters_non_owned_non_default_profile_from_list() { }); }) } + +#[test] +fn legacy_agent_decides_profiles_use_galaxy_decides() { + let permission: ActionPermission = serde_json::from_str("\"AgentDecides\"").unwrap(); + assert_eq!(permission, ActionPermission::GalaxyDecides); + assert_eq!( + serde_json::to_string(&permission).unwrap(), + "\"GalaxyDecides\"" + ); +} diff --git a/app/src/ai/llms.rs b/app/src/ai/llms.rs index 606c3d16..76b30e59 100644 --- a/app/src/ai/llms.rs +++ b/app/src/ai/llms.rs @@ -2076,9 +2076,15 @@ impl LLMPreferences { .unwrap_or(&self.models_by_feature.agent_mode) } - /// Returns the set of LLMs available for computer use agent. - pub fn get_computer_use_llm_choices(&self) -> impl Iterator { - self.get_computer_use_available().choices.iter() + /// Returns computer-use models plus configured models that can inspect screenshots. + pub fn get_computer_use_llm_choices(&self, app: &AppContext) -> impl Iterator { + let available = self.get_computer_use_available(); + available.choices.iter().chain( + self.get_base_llm_choices_for_agent_mode(app) + .filter(move |info| { + info.vision_supported && available.info_for_id(&info.id).is_none() + }), + ) } /// Returns the `LLMInfo` for the computer use agent model. @@ -2089,13 +2095,15 @@ impl LLMPreferences { ) -> &'a LLMInfo { let profile = AIExecutionProfilesModel::as_ref(app).active_profile(terminal_view_id, app); - let available = self.get_computer_use_available(); profile .data() .computer_use_model .clone() - .and_then(|id| available.info_for_id(&id)) - .unwrap_or_else(|| available.default_llm_info()) + .and_then(|id| { + self.get_computer_use_llm_choices(app) + .find(|info| info.id == id) + }) + .unwrap_or_else(|| self.get_default_computer_use_model()) } /// Returns the default computer use model as a fallback. @@ -2600,8 +2608,15 @@ impl LLMPreferences { } if let Some(preferred_llm_id) = &profile.data().computer_use_model { if self - .get_computer_use_available() - .usable_info_for_id(preferred_llm_id, ctx) + .get_computer_use_llm_choices(ctx) + .find(|info| info.id == *preferred_llm_id) + .filter(|info| { + let has_byok_key = + is_using_api_key_for_provider(&info.provider, ctx); + info.disable_reason.as_ref().is_none_or(|reason| { + !reason.should_clear_preference(has_byok_key) + }) + }) .is_none() { profiles.set_computer_use_model(profile_id, None, ctx); diff --git a/app/src/ai/llms_tests.rs b/app/src/ai/llms_tests.rs index b7059f4e..523bc2d6 100644 --- a/app/src/ai/llms_tests.rs +++ b/app/src/ai/llms_tests.rs @@ -236,6 +236,36 @@ fn empty_preferences() -> LLMPreferences { } } +#[test] +fn computer_use_picker_includes_vision_models_without_duplicate_entries() { + App::test((), |mut app| async move { + initialize_settings_for_tests(&mut app); + let mut preferences = empty_preferences(); + let auto = default_computer_use_llms().choices[0].clone(); + let mut visual = auto.clone(); + visual.id = "visual-model".into(); + visual.display_name = "Visual model".to_owned(); + visual.vision_supported = true; + let mut text_only = auto.clone(); + text_only.id = "text-only-model".into(); + text_only.display_name = "Text-only model".to_owned(); + preferences.models_by_feature.agent_mode.choices = vec![visual.clone(), text_only]; + preferences.models_by_feature.computer_use = Some(AvailableLLMs { + default_id: "computer-use-agent-auto".into(), + choices: vec![auto, visual], + preferred_codex_model_id: None, + }); + + app.read(|ctx| { + let ids = preferences + .get_computer_use_llm_choices(ctx) + .map(|info| info.id.as_str()) + .collect::>(); + assert_eq!(ids, ["computer-use-agent-auto", "visual-model"]); + }); + }); +} + #[test] fn bedrock_and_openai_provider_models_coexist_in_picker() { App::test((), |mut app| async move { diff --git a/app/src/ai/provider/request_translator.rs b/app/src/ai/provider/request_translator.rs index f463f10f..92054d81 100644 --- a/app/src/ai/provider/request_translator.rs +++ b/app/src/ai/provider/request_translator.rs @@ -1722,11 +1722,11 @@ pub(crate) fn tool_name_is_supported(name: &str, supported: &HashSet Vec { vec![ diff --git a/app/src/ai/runtime/mod.rs b/app/src/ai/runtime/mod.rs index 39fec4b4..42e0048a 100644 --- a/app/src/ai/runtime/mod.rs +++ b/app/src/ai/runtime/mod.rs @@ -12,7 +12,7 @@ pub(crate) use event_translator::{ pub(crate) use provider_run_coordinator::{ ProviderRunBlock, ProviderRunCoordinator, ProviderRunProfile, ProviderRunProjection, ProviderToolExecutionRef, ProviderToolLifecycleOutcome, BASE_PROVIDER_PROFILE, - CLI_MONITOR_PROVIDER_PROFILE, + CLI_MONITOR_PROVIDER_PROFILE, COMPUTER_USE_PROVIDER_PROFILE, }; pub(crate) use rig::{ prepare_provider_run, provider_context_window_tokens, provider_runtime_for_request, diff --git a/app/src/ai/runtime/provider_run_coordinator.rs b/app/src/ai/runtime/provider_run_coordinator.rs index 66eac805..6943fe4b 100644 --- a/app/src/ai/runtime/provider_run_coordinator.rs +++ b/app/src/ai/runtime/provider_run_coordinator.rs @@ -23,6 +23,7 @@ use crate::ai::model_output_logging; pub(crate) const BASE_PROVIDER_PROFILE: &str = "base"; pub(crate) const CLI_MONITOR_PROVIDER_PROFILE: &str = "cli-monitor"; +pub(crate) const COMPUTER_USE_PROVIDER_PROFILE: &str = "computer-use"; const PROVIDER_MODEL_START_TIMEOUT: Duration = Duration::from_secs(120); const PROVIDER_MODEL_EVENT_IDLE_TIMEOUT: Duration = Duration::from_secs(90); const PROVIDER_MODEL_RETRY_DELAYS: [Duration; 3] = [ @@ -752,18 +753,6 @@ impl ProviderRunCoordinator { return Ok(()); } buffer.text.push_str(&text); - if !self - .project_or_fail_acknowledged( - ProviderRunProjection::ModelEvent { - work_id: call.work_id.clone(), - event: AgentEvent::TextDelta { text }, - }, - project, - ) - .await? - { - return Ok(()); - } } AgentEvent::ReasoningDelta { text } => { if !self @@ -775,18 +764,6 @@ impl ProviderRunCoordinator { return Ok(()); } buffer.reasoning.push_str(&text); - if !self - .project_or_fail_acknowledged( - ProviderRunProjection::ModelEvent { - work_id: call.work_id.clone(), - event: AgentEvent::ReasoningDelta { text }, - }, - project, - ) - .await? - { - return Ok(()); - } } AgentEvent::ReasoningCompleted { text, signature } if text.is_empty() && signature.is_none() => @@ -806,18 +783,6 @@ impl ProviderRunCoordinator { } buffer.reasoning.clone_from(&text); buffer.reasoning_signature.clone_from(&signature); - if !self - .project_or_fail_acknowledged( - ProviderRunProjection::ModelEvent { - work_id: call.work_id.clone(), - event: AgentEvent::ReasoningCompleted { text, signature }, - }, - project, - ) - .await? - { - return Ok(()); - } } AgentEvent::Tool { event: ToolEvent::Proposed { call: tool_call }, @@ -864,6 +829,47 @@ impl ProviderRunCoordinator { { return Ok(()); } + // A model can speak before its tool call arrives. Wait until the turn + // finishes so routine tool-turn narration never reaches the conversation. + // Keep it in the canonical turn for provider continuity. + if buffer.tool_calls.is_empty() + && !matches!( + &reason, + StopReason::Cancelled | StopReason::ContextWindowExceeded + ) + { + if !buffer.reasoning.is_empty() + && !self + .project_or_fail_acknowledged( + ProviderRunProjection::ModelEvent { + work_id: call.work_id.clone(), + event: AgentEvent::ReasoningCompleted { + text: buffer.reasoning.clone(), + signature: buffer.reasoning_signature.clone(), + }, + }, + project, + ) + .await? + { + return Ok(()); + } + if !buffer.text.is_empty() + && !self + .project_or_fail_acknowledged( + ProviderRunProjection::ModelEvent { + work_id: call.work_id.clone(), + event: AgentEvent::TextDelta { + text: buffer.text.clone(), + }, + }, + project, + ) + .await? + { + return Ok(()); + } + } if !self .project_or_fail_acknowledged( ProviderRunProjection::ModelTurnFinished { diff --git a/app/src/ai/runtime/provider_run_coordinator_tests.rs b/app/src/ai/runtime/provider_run_coordinator_tests.rs index d7eb2e17..020752f6 100644 --- a/app/src/ai/runtime/provider_run_coordinator_tests.rs +++ b/app/src/ai/runtime/provider_run_coordinator_tests.rs @@ -301,6 +301,56 @@ fn answer_turn() -> ScriptedTurn { ]) } +#[tokio::test] +async fn tool_turn_narration_stays_in_history_but_out_of_visible_output() { + let runtime = Arc::new(ScriptedRuntime::new(vec![tool_turn()])); + let mut coordinator = coordinator(runtime); + let mut projections = Vec::new(); + let (_sender, control) = turn_control(); + + let block = coordinator + .drive_until_blocked(control, collect_projection(&mut projections)) + .await + .unwrap(); + + assert!(matches!(block, ProviderRunBlock::Tools(_))); + assert!(coordinator + .run() + .transcript() + .iter() + .any(|message| format!("{:?}", message.content).contains("I will inspect it."))); + assert!(!projections.iter().any(|projection| matches!( + projection, + ProviderRunProjection::ModelEvent { + event: AgentEvent::TextDelta { .. } + | AgentEvent::ReasoningDelta { .. } + | AgentEvent::ReasoningCompleted { .. }, + .. + } + ))); +} + +#[tokio::test] +async fn completed_answer_is_still_projected() { + let runtime = Arc::new(ScriptedRuntime::new(vec![answer_turn()])); + let mut coordinator = coordinator(runtime); + let mut projections = Vec::new(); + let (_sender, control) = turn_control(); + + coordinator + .drive_until_blocked(control, collect_projection(&mut projections)) + .await + .unwrap(); + + assert!(projections.iter().any(|projection| matches!( + projection, + ProviderRunProjection::ModelEvent { + event: AgentEvent::TextDelta { text }, + .. + } if text == "The issue is fixed." + ))); +} + fn coordinator(runtime: Arc) -> ProviderRunCoordinator { ProviderRunCoordinator::from_request( "run-1", diff --git a/app/src/ai/runtime/rig.rs b/app/src/ai/runtime/rig.rs index e875e7a4..dd7fc7a7 100644 --- a/app/src/ai/runtime/rig.rs +++ b/app/src/ai/runtime/rig.rs @@ -31,6 +31,7 @@ use crate::settings::OpenAIProviderKind; pub(crate) struct PreparedProviderRun { pub(crate) base_profile: ProviderRunProfile, pub(crate) cli_monitor_profile: Option, + pub(crate) computer_use_profile: Option, pub(crate) tool_result_archive: Vec, pub(crate) messages_sent: Arc>>, pub(crate) persistence_offset: usize, @@ -123,6 +124,7 @@ impl ProviderActionContext { pub(crate) async fn prepare_provider_run( base_provider_config: crate::ai::provider::ProviderConfig, cli_provider_config: crate::ai::provider::ProviderConfig, + computer_use_provider_config: crate::ai::provider::ProviderConfig, mut params: RequestParams, orchestration_models: Vec, ) -> anyhow::Result { @@ -130,6 +132,32 @@ pub(crate) async fn prepare_provider_run( crate::ai::agent::api::prepare_direct_provider_params(&mut params); let skill_path_origin = params.session_context.skill_path_origin(); let mut cli_params = params.clone(); + let computer_use_model = params.computer_use_model.as_str(); + let computer_use_profile = if params.computer_use_enabled + && !computer_use_model.is_empty() + && computer_use_model != "computer-use-agent-auto" + && computer_use_model != params.model.as_str() + && !matches!( + &computer_use_provider_config, + crate::ai::provider::ProviderConfig::None + ) { + let mut computer_params = params.clone(); + computer_params.model = params.computer_use_model.clone(); + let (runtime, prepared) = prepare_provider_profile( + computer_use_provider_config, + computer_params, + supported_tools.clone(), + supported_cli_agent_tools.clone(), + None, + ) + .await?; + Some( + ProviderRunProfile::new(runtime, prepared.request) + .with_max_context_tokens(prepared.max_context_tokens), + ) + } else { + None + }; let cli_model_is_placeholder = params.cli_agent_model.as_str().trim().is_empty() || params .cli_agent_model @@ -210,6 +238,7 @@ pub(crate) async fn prepare_provider_run( base_profile: ProviderRunProfile::new(base_runtime, request) .with_max_context_tokens(prepared_context_tokens), cli_monitor_profile, + computer_use_profile, tool_result_archive, messages_sent, persistence_offset, diff --git a/app/src/ai/runtime/rig_request.rs b/app/src/ai/runtime/rig_request.rs index bca4f5ef..9265faf4 100644 --- a/app/src/ai/runtime/rig_request.rs +++ b/app/src/ai/runtime/rig_request.rs @@ -824,7 +824,7 @@ fn build_system_prompt( "You are Galaxy, a local-first software-engineering and terminal agent. Complete the user's task through inspection, implementation, and proportionate validation. Galaxy owns tool permissions and execution; use only the tools advertised in this request and treat every result as authoritative evidence.\n\n", ); prompt.push_str( - "## Communication Style\nSpeak naturally, warmly, and directly, like a thoughtful collaborator working alongside the user. Default to short responses and expand only when complexity or the user's request warrants it. Acknowledge a correction once when useful; do not repeatedly agree that the user is right, praise them, or add generic reassurance. For nontrivial work, briefly tell the user what you are checking once before the first tool call of a new user request. Tool results and compaction checkpoints continue the same request: resume the work without another greeting, acknowledgement, or restatement of the plan. Supply the short purpose title requested by each tool schema. Between dependent tool calls, add a concise update only when a result materially changes what you learned or what you will do next; ground it in specifics instead of generic activity narration. Do not narrate every routine read, repeat the plan, or end a turn with only a progress update when useful work can continue. In the final response, lead with the outcome and keep the handoff compact.\n\n", + "## Communication Style\nSpeak naturally, warmly, and directly, like a thoughtful collaborator working alongside the user. Work quietly by default while using tools; tool panes already show the activity. Do not send an introductory acknowledgement, plan, or generic progress message. Send prose during work only for a question needing the user's input, a consequential blocker, or a material finding the user needs to act on now. Tool results and compaction checkpoints continue the same request: resume work without another greeting or restatement. Supply the short purpose title requested by each tool schema. Do not narrate routine reads, repeat the plan, or end a turn with only a progress update when useful work can continue. In the final response, lead with the outcome and keep the handoff compact.\n\n", ); prompt.push_str( "## Execution Contract\nBefore another read or search, identify the missing fact it will resolve. Reuse findings already present in the conversation. Once enough evidence exists for the requested change, implement it and validate; additional inspection should resolve a specific uncertainty. Batch independent reads within the tool limits and wait for results before dependent work. Continue until the user's requested outcome is complete and validated. Do not stop at an intermediate analysis, plan, status update, or promise of future work, and do not ask the user to say \"continue\". After each tool result, choose and perform the next necessary step. Stop only when the request is fulfilled or a concrete blocker requires user input; identify that blocker explicitly.\n\n", diff --git a/app/src/ai/runtime/rig_tests.rs b/app/src/ai/runtime/rig_tests.rs index 611de65c..e685ff1c 100644 --- a/app/src/ai/runtime/rig_tests.rs +++ b/app/src/ai/runtime/rig_tests.rs @@ -31,6 +31,7 @@ async fn missing_cli_provider_route_falls_back_to_base_provider_profile() { let prepared = prepare_provider_run( ProviderConfig::OpenAI(openai_config("base-provider-model")), ProviderConfig::None, + ProviderConfig::None, params, Vec::new(), ) @@ -58,6 +59,34 @@ async fn missing_cli_provider_route_falls_back_to_base_provider_profile() { .any(|tool| tool.name == "read_shell_command_output")); } +#[tokio::test] +async fn selected_computer_use_model_gets_its_own_profile() { + let mut params = RequestParams::new_for_test(); + params.model = LLMId::from("base-selection"); + params.computer_use_model = LLMId::from("computer-selection"); + params.computer_use_enabled = true; + + let prepared = prepare_provider_run( + ProviderConfig::OpenAI(openai_config("base-provider-model")), + ProviderConfig::None, + ProviderConfig::OpenAI(openai_config("computer-provider-model")), + params, + Vec::new(), + ) + .await + .unwrap(); + + assert_eq!( + prepared + .computer_use_profile + .expect("selected computer-use model should have a profile") + .request + .model + .as_str(), + "computer-provider-model" + ); +} + #[test] fn subscription_context_budget_uses_catalog_limit_and_clamps_stale_overrides() { let mut config = openai_config("subscription-model"); diff --git a/app/src/settings/onboarding.rs b/app/src/settings/onboarding.rs index 226561cc..bf1191b3 100644 --- a/app/src/settings/onboarding.rs +++ b/app/src/settings/onboarding.rs @@ -219,13 +219,13 @@ fn action_permissions_for_onboarding_autonomy( write_to_pty: WriteToPtyPermission::AlwaysAllow, }, // Partial autonomy: reads are always allowed, applying code diffs - // always asks, and the agent decides on command / MCP execution - // (asking only for sensitive actions). + // always asks, and Galaxy reviews command / MCP execution + // (asking for approval on higher-risk actions). AgentAutonomy::Partial => OnboardingAutonomyPermissions { apply_code_diffs: ActionPermission::AlwaysAsk, read_files: ActionPermission::AlwaysAllow, - execute_commands: ActionPermission::AgentDecides, - mcp_permissions: ActionPermission::AgentDecides, + execute_commands: ActionPermission::GalaxyDecides, + mcp_permissions: ActionPermission::GalaxyDecides, write_to_pty: WriteToPtyPermission::AlwaysAsk, }, AgentAutonomy::None => OnboardingAutonomyPermissions { diff --git a/app/src/settings_view/ai_page.rs b/app/src/settings_view/ai_page.rs index 1cad9312..5ad7db40 100644 --- a/app/src/settings_view/ai_page.rs +++ b/app/src/settings_view/ai_page.rs @@ -1415,10 +1415,6 @@ impl AISettingsPageView { dropdown.set_items( vec![ - DropdownItem::new( - "Agent decides", - AISettingsPageAction::SetApplyCodeDiffs(ActionPermission::AgentDecides), - ), DropdownItem::new( "Always allow", AISettingsPageAction::SetApplyCodeDiffs(ActionPermission::AlwaysAllow), @@ -1449,10 +1445,6 @@ impl AISettingsPageView { dropdown.set_menu_width(AI_SETTINGS_DROPDOWN_WIDTH, ctx); dropdown.set_items( vec![ - DropdownItem::new( - "Agent decides", - AISettingsPageAction::SetReadFiles(ActionPermission::AgentDecides), - ), DropdownItem::new( "Always allow", AISettingsPageAction::SetReadFiles(ActionPermission::AlwaysAllow), @@ -1483,10 +1475,6 @@ impl AISettingsPageView { dropdown.set_menu_width(AI_SETTINGS_DROPDOWN_WIDTH, ctx); dropdown.set_items( vec![ - DropdownItem::new( - "Agent decides", - AISettingsPageAction::SetExecuteCommands(ActionPermission::AgentDecides), - ), DropdownItem::new( "Always allow", AISettingsPageAction::SetExecuteCommands(ActionPermission::AlwaysAllow), @@ -1551,10 +1539,6 @@ impl AISettingsPageView { dropdown.set_menu_width(AI_SETTINGS_DROPDOWN_WIDTH, ctx); dropdown.set_items( vec![ - DropdownItem::new( - "Agent decides", - AISettingsPageAction::SetMCPPermissions(ActionPermission::AgentDecides), - ), DropdownItem::new( "Always allow", AISettingsPageAction::SetMCPPermissions(ActionPermission::AlwaysAllow), @@ -2609,10 +2593,9 @@ impl AISettingsPageView { } let active = match current_permission { - ActionPermission::AgentDecides | ActionPermission::Unknown => 0, - ActionPermission::AlwaysAllow => 1, - ActionPermission::AlwaysAsk => 2, - ActionPermission::GalaxyDecides => 3, + ActionPermission::AlwaysAllow => 0, + ActionPermission::AlwaysAsk | ActionPermission::Unknown => 1, + ActionPermission::GalaxyDecides => 2, }; menu.set_selected_by_index(active, ctx); @@ -4748,9 +4731,7 @@ impl AgentsWidget { app, )); - if execute_commands_setting == ActionPermission::AlwaysAsk - || execute_commands_setting == ActionPermission::AgentDecides - { + if execute_commands_setting == ActionPermission::AlwaysAsk { let command_allowlist = BlocklistAIPermissions::as_ref(app).get_execute_commands_allowlist(app, None); execute_commands_flex.add_child( @@ -5256,9 +5237,7 @@ impl AgentsWidget { ); column.add_child(permission_setting); - if current_mcp_setting == ActionPermission::AlwaysAsk - || current_mcp_setting == ActionPermission::AgentDecides - { + if current_mcp_setting == ActionPermission::AlwaysAsk { let allowlist = self.render_mcp_list( "MCP allowlist", "Allow the Galaxy Agent to call these MCP servers.", @@ -5275,7 +5254,6 @@ impl AgentsWidget { if current_mcp_setting == ActionPermission::AlwaysAllow || current_mcp_setting == ActionPermission::GalaxyDecides - || current_mcp_setting == ActionPermission::AgentDecides { let denylist = self.render_mcp_list( "MCP denylist", diff --git a/app/src/settings_view/execution_profile_view.rs b/app/src/settings_view/execution_profile_view.rs index b3527f7b..d91c900f 100644 --- a/app/src/settings_view/execution_profile_view.rs +++ b/app/src/settings_view/execution_profile_view.rs @@ -122,7 +122,11 @@ impl View for ExecutionProfileView { let computer_use_model = profile .computer_use_model .as_ref() - .and_then(|id| llm_preferences.get_llm_info(id)) + .and_then(|id| { + llm_preferences + .get_computer_use_llm_choices(app) + .find(|info| info.id == *id) + }) .map(|info| info.display_name.clone()) .unwrap_or_else(|| "Auto".to_string()); @@ -224,9 +228,7 @@ impl View for ExecutionProfileView { ), )); - if profile.read_files == ActionPermission::AlwaysAsk - || profile.read_files == ActionPermission::AgentDecides - { + if profile.read_files == ActionPermission::AlwaysAsk { permissions_column.add_child(render_directory_allowlist( &profile, appearance, @@ -259,7 +261,7 @@ impl View for ExecutionProfileView { is_any_ai_enabled, )); } - ActionPermission::AgentDecides | ActionPermission::Unknown => { + ActionPermission::Unknown => { permissions_column.add_child(render_command_allowlist( &profile, appearance, @@ -341,7 +343,7 @@ impl View for ExecutionProfileView { is_any_ai_enabled, )); } - ActionPermission::AgentDecides | ActionPermission::Unknown => { + ActionPermission::Unknown => { permissions_column.add_child(render_mcp_allowlist( &profile, appearance, @@ -697,7 +699,6 @@ fn render_action_permission_line_with_icon( is_ai_enabled: bool, ) -> Box { let permission_text = match permission { - ActionPermission::AgentDecides => "Agent decides", ActionPermission::AlwaysAllow => "Always allow", ActionPermission::AlwaysAsk => "Always ask", ActionPermission::GalaxyDecides => "Galaxy Decides", diff --git a/app/src/terminal/view.rs b/app/src/terminal/view.rs index e9d12bc6..3ccce325 100644 --- a/app/src/terminal/view.rs +++ b/app/src/terminal/view.rs @@ -20439,6 +20439,11 @@ impl TerminalView { if is_restored { return; } + self.model + .lock() + .block_list_mut() + .mark_rich_content_dirty(block.id()); + ctx.notify(); self.focus_ai_block_if_self_focused(&block, ctx); self.maybe_send_agent_mode_desktop_notification(&conversation_id, ctx); } diff --git a/app/src/workspaces/gql_convert.rs b/app/src/workspaces/gql_convert.rs index 0210eb78..ec509fab 100644 --- a/app/src/workspaces/gql_convert.rs +++ b/app/src/workspaces/gql_convert.rs @@ -661,7 +661,7 @@ fn convert_gql_ai_autonomy_value_to_action_permission( gql_ai_autonomy_value: GqlAiAutonomyValue, ) -> Option { match gql_ai_autonomy_value { - GqlAiAutonomyValue::AgentDecides => Some(ActionPermission::AgentDecides), + GqlAiAutonomyValue::AgentDecides => Some(ActionPermission::GalaxyDecides), GqlAiAutonomyValue::AlwaysAllow => Some(ActionPermission::AlwaysAllow), GqlAiAutonomyValue::AlwaysAsk => Some(ActionPermission::AlwaysAsk), GqlAiAutonomyValue::RespectUserSetting => None, diff --git a/crates/cloud_object_models/src/ai_execution_profile.rs b/crates/cloud_object_models/src/ai_execution_profile.rs index 0fb4933d..6352a953 100644 --- a/crates/cloud_object_models/src/ai_execution_profile.rs +++ b/crates/cloud_object_models/src/ai_execution_profile.rs @@ -17,10 +17,10 @@ pub const PROFILE_NAME_MAX_LENGTH: usize = 50; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum ActionPermission { - AgentDecides, AlwaysAllow, AlwaysAsk, /// Independently assess each action before allowing execution. + #[serde(alias = "AgentDecides")] GalaxyDecides, // This is intended to catch deserialization errors whenever we add new variants to this enum. Say we // want to add a "Never" variant. Without this catch-all, old clients wouldn't be able to deserialize @@ -35,9 +35,7 @@ impl ActionPermission { ActionPermission::GalaxyDecides => { "Galaxy reviews each action with your terminal-use model. Risk 0–3 runs automatically; risk 4–5 asks you, and is rejected after 60 seconds without a response." } - ActionPermission::AgentDecides | ActionPermission::Unknown => { - "The Agent chooses the safest path: acting on its own when confident, and asking for approval when uncertain." - } + ActionPermission::Unknown => "Unknown setting.", ActionPermission::AlwaysAllow => { "Give the Agent full autonomy — no manual approval ever required." } @@ -419,11 +417,11 @@ impl Default for AIExecutionProfile { Self { name: Default::default(), is_default_profile: false, - apply_code_diffs: ActionPermission::AgentDecides, - read_files: ActionPermission::AgentDecides, + apply_code_diffs: ActionPermission::GalaxyDecides, + read_files: ActionPermission::GalaxyDecides, execute_commands: ActionPermission::AlwaysAsk, write_to_pty: WriteToPtyPermission::AlwaysAsk, - mcp_permissions: ActionPermission::AgentDecides, + mcp_permissions: ActionPermission::GalaxyDecides, ask_user_question: AskUserQuestionPermission::AlwaysAsk, run_agents: RunAgentsPermission::AlwaysAsk, command_denylist: DEFAULT_COMMAND_EXECUTION_DENYLIST.clone(), diff --git a/crates/galaxy_agent_core/src/provider_run.rs b/crates/galaxy_agent_core/src/provider_run.rs index c38f3423..7a8c2533 100644 --- a/crates/galaxy_agent_core/src/provider_run.rs +++ b/crates/galaxy_agent_core/src/provider_run.rs @@ -867,6 +867,18 @@ impl ProviderRun { matches!(self.state, ProviderRunState::ReadyToCallModel).then(|| self.current_work_id()) } + /// Selects the runtime for the next turn after a tool batch has been committed. + pub fn set_ready_profile( + &mut self, + profile: impl Into, + ) -> Result<(), ProviderRunProtocolError> { + if !matches!(self.state, ProviderRunState::ReadyToCallModel) { + return Err(self.unexpected_state(ProviderRunPhase::ReadyToCallModel)); + } + self.profile = profile.into(); + Ok(()) + } + fn model_turn_limit_reached(&self) -> bool { self.limits .max_model_turns diff --git a/crates/galaxy_agent_core/src/provider_run_tests.rs b/crates/galaxy_agent_core/src/provider_run_tests.rs index 7a445708..9267f3e9 100644 --- a/crates/galaxy_agent_core/src/provider_run_tests.rs +++ b/crates/galaxy_agent_core/src/provider_run_tests.rs @@ -18,6 +18,18 @@ fn run() -> ProviderRun { run_with_limits(ProviderRunLimits::default()) } +#[test] +fn ready_profile_can_change_without_replaying_or_adding_history() { + let mut run = run(); + let original_transcript = run.transcript().to_vec(); + run.set_ready_profile("computer-use").unwrap(); + + assert_eq!(run.profile().as_str(), "computer-use"); + assert_eq!(run.transcript(), original_transcript); + assert_eq!(next_model_call(&mut run).profile.as_str(), "computer-use"); + assert!(run.set_ready_profile("base").is_err()); +} + fn next_model_call(run: &mut ProviderRun) -> ProviderModelCall { let Some(ProviderRunStep::CallModel(call)) = run.next_step().unwrap() else { panic!("expected model call");