Fix macOS computer use and restored command replay

This commit is contained in:
Ryan Ward
2026-10-27 21:04:03 +00:00
parent 7ce0236825
commit 1647791a9e
10 changed files with 192 additions and 45 deletions
@@ -95,15 +95,19 @@ impl RequestComputerUseExecutor {
let background_enabled = FeatureFlag::BackgroundComputerUse.is_enabled();
ActionExecution::Async {
execute_future: Box::pin(async move {
let result = actor
.perform_actions(
&[],
computer_use::Options {
screenshot_params,
background_enabled,
},
)
.await;
let result = async {
computer_use::request_screen_capture_access()?;
actor
.perform_actions(
&[],
computer_use::Options {
screenshot_params,
background_enabled,
},
)
.await
}
.await;
(result, platform)
}),
on_complete: Box::new(|action_result, _ctx| match action_result {
@@ -44,8 +44,14 @@ impl UseComputerExecutor {
// Gate per-window targeting behind the client feature flag. When off, the actor forces the
// legacy full-screen path so results are identical to the pre-existing implementation.
let background_enabled = FeatureFlag::BackgroundComputerUse.is_enabled();
// Actor initialization reads the macOS keyboard layout through Carbon APIs that require
// the main thread. Construct it here, just like RequestComputerUseExecutor, before
// moving the Send actor into the background action future.
#[cfg(target_os = "macos")]
let mut actor = computer_use::create_actor();
ActionExecution::new_async(
async move {
#[cfg(not(target_os = "macos"))]
let mut actor = computer_use::create_actor();
match actor
.perform_actions(
+14 -1
View File
@@ -301,6 +301,9 @@ pub struct Block {
render_delay_complete: Arc<AtomicBool>,
was_long_running: AtomicBool,
bootstrap_stage: BootstrapStage,
/// Snapshot replay must not emit live command lifecycle events, even when using
/// the normal block appearance after bootstrapping.
is_replaying_snapshot: bool,
show_bootstrap_block: bool,
show_in_band_command_blocks: bool,
@@ -509,6 +512,9 @@ pub struct PromptInfo {
impl From<&Block> for BlockType {
fn from(block: &Block) -> Self {
if block.is_replaying_snapshot {
return BlockType::Restored;
}
if block.is_for_in_band_command {
return BlockType::InBandCommand;
}
@@ -980,6 +986,7 @@ impl Block {
background_executor,
event_proxy,
bootstrap_stage,
is_replaying_snapshot: false,
show_bootstrap_block: show_warp_bootstrap_input,
show_in_band_command_blocks,
show_memory_stats,
@@ -1945,6 +1952,10 @@ impl Block {
self.bootstrap_stage
}
pub(super) fn set_replaying_snapshot(&mut self, is_replaying_snapshot: bool) {
self.is_replaying_snapshot = is_replaying_snapshot;
}
pub(super) fn should_emit_visible_bootstrap_block_event(&self) -> bool {
self.bootstrap_stage == BootstrapStage::ScriptExecution
&& !self.visible_bootstrap_block_event_sent
@@ -2986,7 +2997,9 @@ impl Block {
self.header_grid.preexec(data.clone());
let is_for_in_band_command = command_executor::is_in_band_command(data.command.as_str());
if self.bootstrap_stage() == BootstrapStage::PostBootstrapPrecmd {
if self.bootstrap_stage() == BootstrapStage::PostBootstrapPrecmd
&& !self.is_replaying_snapshot
{
self.event_proxy
.send_terminal_event(Event::AfterBlockStarted {
block_id: self.id.clone(),
+6
View File
@@ -2971,6 +2971,9 @@ impl BlockList {
Some(prompt_metadata),
block.is_local,
);
// Restoring a snapshot rebuilds terminal state; it does not execute its command.
// Keep this separate from bootstrap_stage, which also controls block appearance.
self.active_block_mut().set_replaying_snapshot(true);
if let Some(shell_host) = &block.shell_host {
self.active_block_mut().set_shell_host(shell_host.clone());
}
@@ -2981,6 +2984,7 @@ impl BlockList {
None => {
// The start_ts is set iff the block was started. So if it's not set, we're done.
self.update_active_block_height();
self.active_block_mut().set_replaying_snapshot(false);
return;
}
Some(start_ts) => start_ts,
@@ -3035,6 +3039,7 @@ impl BlockList {
None => {
// The completed_ts is set iff the block was `finish`ed. So if it's not set, we're done.
self.update_active_block_height();
self.active_block_mut().set_replaying_snapshot(false);
return;
}
Some(completed_ts) => completed_ts,
@@ -3061,6 +3066,7 @@ impl BlockList {
self.active_block_mut().set_prompt_snapshot(prompt_snapshot);
}
}
self.active_block_mut().set_replaying_snapshot(false);
}
/// Marks the end of the active block and creates the next block.
+68
View File
@@ -702,6 +702,74 @@ pub fn test_restore_completed_blocks() {
));
}
#[test]
fn restored_agent_command_snapshots_do_not_emit_live_command_events() {
for shared_session in [false, true] {
let (events_tx, events_rx) = async_channel::unbounded();
let mut block_list = new_bootstrapped_block_list(
None,
None,
ChannelEventListener::builder_for_test()
.with_terminal_events_tx(events_tx)
.build(),
);
while events_rx.try_recv().is_ok() {}
let conversation_id = AIConversationId::new();
let action_id = AIAgentActionId::from("saved-command".to_owned());
let metadata = SerializedAIMetadata::from(AgentInteractionMetadata::new_hidden(
action_id.clone(),
conversation_id,
));
let mut snapshot = SerializedBlock::new_for_test(b"echo saved".to_vec(), b"saved".to_vec());
snapshot.ai_metadata = Some(serde_json::to_string(&metadata).unwrap());
snapshot.agent_view_visibility =
Some(AgentViewVisibility::new_from_conversation(conversation_id).into());
if shared_session {
block_list.load_shared_session_scrollback(&[snapshot.clone()]);
} else {
block_list.insert_restored_block(&snapshot);
}
let restored = block_list.block_with_id(&snapshot.id).unwrap();
assert_eq!(restored.requested_command_action_id(), Some(&action_id));
assert_eq!(restored.ai_conversation_id(), Some(conversation_id));
assert_eq!(restored.is_restored(), false);
assert_eq!(restored.completed_ts(), snapshot.completed_ts.as_ref());
let mut restored_completion_count = 0;
while let Ok(event) = events_rx.try_recv() {
match event {
Event::AfterBlockStarted { .. } => {
panic!("snapshot replay must not start a live command")
}
Event::BlockCompleted(completed) if completed.block_id == snapshot.id => {
assert!(matches!(completed.block_type, BlockType::Restored));
restored_completion_count += 1;
}
_ => {}
}
}
assert_eq!(restored_completion_count, 1);
// A subsequent actual command must still drive the normal lifecycle.
insert_block(&mut block_list, "echo live", "live");
let mut live_started = false;
let mut live_completed = false;
while let Ok(event) = events_rx.try_recv() {
match event {
Event::AfterBlockStarted { .. } => live_started = true,
Event::BlockCompleted(completed) => {
live_completed |= matches!(completed.block_type, BlockType::User(_));
}
_ => {}
}
}
assert!(live_started);
assert!(live_completed);
}
}
#[test]
pub fn test_restore_blocks_with_local_status() {
let (events_tx, events_rx) = async_channel::unbounded();