Fix LiteLLM context budgets and direct computer use

Keep efficient context windows separate from expanded maxima throughout discovery, display and provider budgets. Expose correlated computer-use requests and actions with existing permissions, preserve screenshots as images, and retain them across checkpoints, compaction and explicit history recall.

Validation: cargo build on Linux passed. Added focused regression coverage; test execution and native GUI verification remain outside this CLI build-only workflow.

app/src/ai/runtime/event_translator_tests.rs

@@ -104,6 +104,7 @@ fn restored_task_progress_replay_does_not_duplicate_the_persisted_checklist() {
                 arguments: serde_json::to_value(&snapshot).unwrap(),
             },
             state: galaxy_agent_core::PendingToolCallState::Resolved { result: galaxy_agent_core::ToolResult {
+                images: Vec::new(),
                 call_id: "progress".to_owned(), status: galaxy_agent_core::ToolResultStatus::Success,
                 content: snapshot.to_tool_result_content(),
             }},

app/src/ai/runtime/mod.rs

@@ -4,6 +4,7 @@ mod provider_run_coordinator;
 mod rig;
 mod rig_request;
 mod rig_tool;
+pub(crate) use rig_tool::action_from_tool_call;
 mod task_progress;
 
 pub(crate) use child_context::add_parent_context;

app/src/ai/runtime/provider_run_coordinator_tests.rs

@@ -461,6 +461,7 @@ async fn coordinator_awaiting_tools(
 fn tool_result(call_id: &str, content: &str, status: ToolResultStatus) -> ToolEvent {
     ToolEvent::Completed {
         result: ToolResult {
+            images: Vec::new(),
             call_id: call_id.to_string(),
             content: content.to_string(),
             status,
@@ -514,6 +515,7 @@ async fn one_run_drives_model_tool_and_followup_turns_with_atomic_history() {
         .complete_tool(
             &batch.work_id,
             ToolResult {
+                images: Vec::new(),
                 call_id: "read-1".to_string(),
                 content: "manifest contents".to_string(),
                 status: ToolResultStatus::Success,

app/src/ai/runtime/rig.rs

@@ -133,20 +133,25 @@ pub(crate) async fn prepare_provider_run(
     let skill_path_origin = params.session_context.skill_path_origin();
     let mut cli_params = params.clone();
     let computer_use_model = params.computer_use_model.as_str();
-    let computer_use_profile = if params.computer_use_enabled
-        && !computer_use_model.is_empty()
-        && computer_use_model != "computer-use-agent-auto"
-        && computer_use_model != params.model.as_str()
-        && !matches!(
-            &computer_use_provider_config,
-            crate::ai::provider::ProviderConfig::None
-        ) {
+    let computer_use_profile = if supported_tools.contains(&ToolType::RequestComputerUse) {
         let mut computer_params = params.clone();
-        computer_params.model = params.computer_use_model.clone();
+        let computer_provider = if computer_use_model.is_empty()
+            || computer_use_model == "computer-use-agent-auto"
+            || computer_use_model == params.model.as_str()
+            || matches!(&computer_use_provider_config, crate::ai::provider::ProviderConfig::None)
+        {
+            base_provider_config.clone()
+        } else {
+            computer_params.model = params.computer_use_model.clone();
+            computer_use_provider_config
+        };
+        let mut computer_tools = supported_tools.clone();
+        computer_tools.retain(|tool| *tool != ToolType::RequestComputerUse);
+        computer_tools.push(ToolType::UseComputer);
         let (runtime, prepared) = prepare_provider_profile(
-            computer_use_provider_config,
+            computer_provider,
             computer_params,
-            supported_tools.clone(),
+            computer_tools,
             supported_cli_agent_tools.clone(),
             None,
         )

app/src/ai/runtime/rig_request.rs

@@ -389,17 +389,7 @@ fn input_messages(
 ) -> Vec<ConversationMessage> {
     let mut messages = Vec::new();
     if !tool_results.is_empty() {
-        let mut parts = tool_results
-            .into_iter()
-            .map(|result| {
-                let is_error = result.is_error();
-                ContentPart::ToolResult {
-                    tool_use_id: result.call_id,
-                    content: result.content,
-                    is_error,
-                }
-            })
-            .collect::<Vec<_>>();
+        let mut parts = ToolResult::into_batch_content_parts(tool_results);
         let content = if parts.len() == 1 {
             let ContentPart::ToolResult {
                 tool_use_id,
@@ -1043,6 +1033,12 @@ fn build_system_prompt(
     {
         prompt.push_str(CODING_DELEGATION_GUIDANCE);
     }
+    if tools.iter().any(|tool| tool.name == "request_computer_use") {
+        prompt.push_str("## Desktop Computer Use\nUse request_computer_use for tasks that require seeing or interacting with the user's desktop. This controls the machine running Galaxy, including when the terminal is connected to another host. Request the specific authorized task and wait for the screenshot. A successful request switches to the configured computer-use model; rejected or failed requests do not grant control.\n\n");
+    }
+    if tools.iter().any(|tool| tool.name == "use_computer") {
+        prompt.push_str("## Desktop Computer Use\nInspect the latest returned screenshot before acting. Use use_computer with small action batches, then inspect the new screenshot. Use original physical pixel coordinates: multiply displayed image x by original_width/width and y by original_height/height. Every button/key press requires both down and up actions. For window targeting, select an exact window_id/pid from the latest windows list; never invent a window. Stop on errors or denied permission and report the actual cause. Galaxy's Full Disk Access is separate from Screen Recording and Accessibility permissions. When desktop work is complete and terminal/file work remains, call the appropriate normal tool and Galaxy resumes the base model. Only finish the response when the whole user's task is complete.\n\n");
+    }
     prompt.push_str("## Available Tools\n");
     if tools.is_empty() {
         prompt.push_str("No tools are available. Do not invent tool calls.\n");

app/src/ai/runtime/rig_request_tests.rs

@@ -533,6 +533,7 @@ fn lrc_snapshot_follow_up_uses_the_cli_monitor_prompt_and_tools() {
         ),
     };
     let snapshot_tool_result = ToolResult {
+        images: Vec::new(),
         call_id: "run-call".to_string(),
         content: result.result.model_content(),
         status: ToolResultStatus::Success,
@@ -842,6 +843,7 @@ fn normalized_tool_outcomes_are_the_only_action_results_sent_to_rig() {
     let tool_results = statuses
         .iter()
         .map(|(call_id, status, _)| ToolResult {
+            images: Vec::new(),
             call_id: (*call_id).to_string(),
             content: format!("normalized-{call_id}"),
             status: *status,

app/src/ai/runtime/rig_tests.rs

@@ -87,6 +87,27 @@ async fn selected_computer_use_model_gets_its_own_profile() {
     );
 }
 
+#[tokio::test]
+async fn automatic_and_same_model_computer_use_keep_a_dedicated_profile() {
+    for selected in ["computer-use-agent-auto", "base-selection", "unroutable-model"] {
+        let mut params = RequestParams::new_for_test();
+        params.model = LLMId::from("base-selection");
+        params.computer_use_model = LLMId::from(selected);
+        params.computer_use_enabled = true;
+        let prepared = prepare_provider_run(
+            ProviderConfig::OpenAI(openai_config("base-provider-model")),
+            ProviderConfig::None, ProviderConfig::None, params, Vec::new(),
+        ).await.unwrap();
+        assert!(prepared.base_profile.request.tools.iter().any(|tool| tool.name == "request_computer_use"));
+        assert!(!prepared.base_profile.request.tools.iter().any(|tool| tool.name == "use_computer"));
+        let computer = prepared.computer_use_profile.unwrap();
+        assert_eq!(computer.request.model.as_str(), "base-provider-model");
+        assert!(computer.request.tools.iter().any(|tool| tool.name == "use_computer"));
+        assert!(!computer.request.tools.iter().any(|tool| tool.name == "request_computer_use"));
+        assert!(computer.request.system_prompt.unwrap().contains("Inspect the latest returned screenshot before acting"));
+    }
+}
+
 #[test]
 fn subscription_context_budget_uses_catalog_limit_and_clamps_stale_overrides() {
     let mut config = openai_config("subscription-model");

app/src/ai/runtime/rig_tool.rs

@@ -55,6 +55,23 @@ pub(crate) fn action_from_tool_call(
                 rationale: optional_string(input, "rationale")?,
                 citations: Vec::new(),
             },
+            "request_computer_use" => AIAgentActionType::RequestComputerUse(
+                crate::ai::agent::RequestComputerUseRequest {
+                    task_summary: required_nonempty_string(input, "task_summary")?,
+                    screenshot_params: Some(computer_screenshot_params(input)?),
+                },
+            ),
+            "use_computer" => AIAgentActionType::UseComputer(
+                crate::ai::agent::UseComputerRequest {
+                    action_summary: required_nonempty_string(input, "action_summary")?,
+                    actions: computer_actions(input)?,
+                    screenshot_params: if optional_boolean(input, "take_screenshot")?.unwrap_or(true) {
+                        Some(computer_screenshot_params(input)?)
+                    } else {
+                        None
+                    },
+                },
+            ),
             "read_files" => AIAgentActionType::ReadFiles(ReadFilesRequest {
                 locations: limited_required_array(input, "files", MAX_BATCH_READ_ITEMS)?
                     .iter()
@@ -472,6 +489,113 @@ fn optional_nonnegative_i32(input: &serde_json::Value, key: &str) -> Result<Opti
         .transpose()
 }
 
+fn computer_i32(input: &serde_json::Value, key: &str) -> Result<i32, String> {
+    input.get(key).and_then(serde_json::Value::as_i64)
+        .and_then(|value| i32::try_from(value).ok())
+        .ok_or_else(|| format!("invalid field {key:?}: expected a 32-bit integer"))
+}
+
+fn computer_target(input: &serde_json::Value) -> Result<computer_use::Target, String> {
+    let Some(target) = input.get("target") else { return Ok(computer_use::Target::Screen); };
+    require_object(target, "target")?;
+    match required_string(target, "type")?.as_str() {
+        "screen" => Ok(computer_use::Target::Screen),
+        "window" => {
+            let window_id = optional_bounded_u64(target, "window_id", u64::from(u32::MAX))?
+                .filter(|id| *id > 0).ok_or("window target requires a non-zero window_id")? as u32;
+            let pid = computer_i32(target, "pid")?;
+            if pid <= 0 { return Err("window target requires a positive pid".to_string()); }
+            if !crate::features::FeatureFlag::BackgroundComputerUse.is_enabled()
+                || !computer_use::background_supported()
+            {
+                return Err("Background window targeting is unavailable; use the screen target.".to_string());
+            }
+            Ok(computer_use::Target::Window { window_id, pid })
+        }
+        _ => Err("target.type must be screen or window".to_string()),
+    }
+}
+
+fn computer_screenshot_params(input: &serde_json::Value) -> Result<computer_use::ScreenshotParams, String> {
+    let empty = serde_json::json!({});
+    let params = input.get("screenshot_params").unwrap_or(&empty);
+    require_object(params, "screenshot_params")?;
+    let positive = |key, maximum, default| -> Result<usize, String> {
+        let value = optional_bounded_u64(params, key, maximum)?.unwrap_or(default);
+        if value == 0 { return Err(format!("{key} must be positive")); }
+        Ok(value as usize)
+    };
+    Ok(computer_use::ScreenshotParams {
+        max_long_edge_px: Some(positive("max_long_edge_px", 4096, 1568)?),
+        max_total_px: Some(positive("max_total_px", 4_194_304, 1_150_000)?),
+        region: None,
+        target: computer_target(params)?,
+    })
+}
+
+fn computer_actions(input: &serde_json::Value) -> Result<Vec<computer_use::TargetedAction>, String> {
+    use computer_use::{Action, Key, MouseButton, ScrollDirection, ScrollDistance, TargetedAction, Vector2I};
+    let mut wait_seconds = 0.0;
+    limited_required_array(input, "actions", 20)?.iter().map(|input| {
+        require_object(input, "action")?;
+        let position = || -> Result<Vector2I, String> {
+            Ok(Vector2I::new(computer_i32(input, "x")?, computer_i32(input, "y")?))
+        };
+        let button = || -> Result<MouseButton, String> {
+            match required_string(input, "button")?.as_str() {
+                "left" => Ok(MouseButton::Left), "right" => Ok(MouseButton::Right),
+                "middle" => Ok(MouseButton::Middle), "back" => Ok(MouseButton::Back),
+                "forward" => Ok(MouseButton::Forward),
+                _ => Err("button must be left, right, middle, back, or forward".to_string()),
+            }
+        };
+        let key = || -> Result<Key, String> {
+            match (input.get("key"), input.get("keycode")) {
+                (Some(_), None) => {
+                    let text = required_string(input, "key")?;
+                    let mut chars = text.chars();
+                    let key = chars.next().ok_or("key must contain exactly one character")?;
+                    if chars.next().is_some() { return Err("key must contain exactly one character".to_string()); }
+                    Ok(Key::Char(key))
+                }
+                (None, Some(_)) => Ok(Key::Keycode(computer_i32(input, "keycode")?)),
+                _ => Err("provide exactly one of key or keycode".to_string()),
+            }
+        };
+        let action = match required_string(input, "type")?.as_str() {
+            "mouse_move" => Action::MouseMove { to: position()? },
+            "mouse_down" => Action::MouseDown { button: button()?, at: position()? },
+            "mouse_up" => Action::MouseUp { button: button()? },
+            "mouse_wheel" => Action::MouseWheel {
+                at: position()?,
+                direction: match required_string(input, "direction")?.as_str() {
+                    "up" => ScrollDirection::Up, "down" => ScrollDirection::Down,
+                    "left" => ScrollDirection::Left, "right" => ScrollDirection::Right,
+                    _ => return Err("direction must be up, down, left, or right".to_string()),
+                },
+                distance: ScrollDistance::Pixels(optional_nonnegative_i32(input, "pixels")?.ok_or("mouse_wheel requires pixels")?),
+            },
+            "type_text" => {
+                let text = required_string(input, "text")?;
+                if text.len() > 100_000 { return Err("text must be no longer than 100000 bytes".to_string()); }
+                Action::TypeText { text }
+            }
+            "key_down" => Action::KeyDown { key: key()? },
+            "key_up" => Action::KeyUp { key: key()? },
+            "wait" => {
+                let seconds = input.get("seconds").and_then(serde_json::Value::as_f64)
+                    .filter(|seconds| seconds.is_finite() && *seconds >= 0.0 && *seconds <= 10.0)
+                    .ok_or("seconds must be between 0 and 10")?;
+                wait_seconds += seconds;
+                if wait_seconds > 10.0 { return Err("a computer-use batch may wait at most 10 seconds".to_string()); }
+                Action::Wait(Duration::from_secs_f64(seconds))
+            }
+            _ => return Err("unknown computer action type".to_string()),
+        };
+        Ok(TargetedAction { action, target: computer_target(input)? })
+    }).collect()
+}
+
 fn skill_references(
     input: &serde_json::Value,
     skill_path_origin: &SkillPathOrigin,

app/src/ai/runtime/rig_tool_tests.rs

@@ -16,6 +16,55 @@ fn call(name: &str, arguments: serde_json::Value) -> ToolCall {
     }
 }
 
+#[test]
+fn computer_use_request_always_captures_a_bounded_initial_screenshot() {
+    let action = action_from_tool_call("task", &call("request_computer_use", serde_json::json!({
+        "task_summary": "Inspect the browser",
+    })), &SkillPathOrigin::Local, &HashMap::new()).unwrap();
+    let AIAgentActionType::RequestComputerUse(request) = action.action else { panic!("expected computer-use request"); };
+    let screenshot = request.screenshot_params.unwrap();
+    assert_eq!(screenshot.max_long_edge_px, Some(1568));
+    assert_eq!(screenshot.max_total_px, Some(1_150_000));
+    assert_eq!(screenshot.target, computer_use::Target::Screen);
+}
+
+#[test]
+fn computer_actions_preserve_coordinates_keys_and_screenshot_choices() {
+    let action = action_from_tool_call("task", &call("use_computer", serde_json::json!({
+        "action_summary": "Select the field and type",
+        "actions": [
+            { "type": "mouse_down", "button": "left", "x": 420, "y": 310 },
+            { "type": "mouse_up", "button": "left" },
+            { "type": "type_text", "text": "hello" },
+            { "type": "key_down", "keycode": 36 },
+            { "type": "key_up", "keycode": 36 },
+        ],
+        "take_screenshot": false,
+    })), &SkillPathOrigin::Local, &HashMap::new()).unwrap();
+    let AIAgentActionType::UseComputer(request) = action.action else { panic!("expected computer actions"); };
+    assert_eq!(request.actions.len(), 5);
+    assert_eq!(request.actions[0].action, computer_use::Action::MouseDown {
+        button: computer_use::MouseButton::Left, at: computer_use::Vector2I::new(420, 310),
+    });
+    assert_eq!(request.actions[3].action, computer_use::Action::KeyDown { key: computer_use::Key::Keycode(36) });
+    assert_eq!(request.screenshot_params, None);
+}
+
+#[test]
+fn computer_actions_reject_invalid_targets_keys_and_unbounded_waiting() {
+    for actions in [
+        serde_json::json!([{ "type": "key_down", "key": "Return" }]),
+        serde_json::json!([{ "type": "key_up", "key": "a", "keycode": 0 }]),
+        serde_json::json!([{ "type": "mouse_move", "x": 1, "y": 2, "target": { "type": "window", "window_id": 0, "pid": 12 } }]),
+        serde_json::json!([{ "type": "wait", "seconds": 6 }, { "type": "wait", "seconds": 6 }]),
+        serde_json::json!(vec![serde_json::json!({ "type": "wait", "seconds": 0 }); 21]),
+    ] {
+        assert!(action_from_tool_call("task", &call("use_computer", serde_json::json!({
+            "action_summary": "Invalid request", "actions": actions,
+        })), &SkillPathOrigin::Local, &HashMap::new()).is_err());
+    }
+}
+
 #[test]
 fn shell_calls_become_domain_actions_without_a_proto_round_trip() {
     let action = action_from_tool_call(

app/src/ai/runtime/task_progress_tests.rs

@@ -106,6 +106,7 @@ fn rejected_plan_creation_cannot_seed_visible_or_durable_tasks() {
                 arguments: serde_json::json!({"documents": [{"content": "## Tasks\n- [ ] Inspect\n- [ ] Verify"}]}),
             },
             state: PendingToolCallState::Resolved { result: galaxy_agent_core::ToolResult {
+                images: Vec::new(),
                 call_id: "plan".to_owned(),
                 content: "Not advertised".to_owned(),
                 status: galaxy_agent_core::ToolResultStatus::Error,