#[cfg_attr(macos, path = "mac/mod.rs")] #[cfg_attr(linux, path = "linux/mod.rs")] #[cfg_attr(windows, path = "windows/mod.rs")] #[cfg(not(noop))] mod imp; mod noop; #[cfg(any(macos, linux, windows))] mod screenshot_utils; use std::borrow::Cow; use async_trait::async_trait; // Clippy doesn't like us pulling in a file as two different modules, // so we add this alias instead of using another cfg_attr on the imp // module definition. #[cfg(noop)] use noop as imp; pub use pathfinder_geometry::vector::Vector2I; use serde::{Deserialize, Serialize}; use serde_with::{DurationSecondsWithFrac, serde_as}; /// The platform that computer use is running on. #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum Platform { Mac, Windows, LinuxX11, LinuxWayland, } pub fn is_supported_on_current_platform() -> bool { if cfg!(feature = "test-util") { noop::is_supported_on_current_platform() } else { imp::is_supported_on_current_platform() } } /// Returns an actor that can perform actions on the computer. pub fn create_actor() -> Box { if cfg!(feature = "test-util") { Box::new(noop::Actor::new()) } else { Box::new(imp::Actor::new()) } } /// Returns whether background, per-window control (driving a specific window without raising it /// or moving the cursor) is available on this client and OS. When false, callers should target /// the whole screen / frontmost application. pub fn background_supported() -> bool { if cfg!(feature = "test-util") { noop::background_supported() } else { imp::background_supported() } } /// Enumerates the on-screen windows, returning their metadata so a caller can pick one to /// target. Returns an empty list on platforms where window enumeration is unsupported. pub fn enumerate_windows() -> Vec { #[cfg(macos)] { imp::enumerate_windows() } #[cfg(not(macos))] { Vec::new() } } /// Experimental: lists on-screen windows as a formatted diagnostic string. macOS only. /// /// Unlike [`enumerate_windows`], which returns slim [`WindowInfo`] records for window selection /// and wire serialization, this function returns richer data including window bounds, formatted /// as a human-readable table for CLI debugging. The two use separate types intentionally: /// [`WindowInfo`] is kept wire-safe and bounds-free; the diagnostic output carries bounds that /// are not part of the API representation. #[cfg(macos)] pub fn experimental_list_windows() -> Result { Ok(imp::list_windows()) } /// Experimental: lists on-screen windows. Unsupported on this platform. #[cfg(not(macos))] pub fn experimental_list_windows() -> Result { Err("Window listing is only supported on macOS.".to_string()) } /// The surface that a computer-use action or screenshot targets. /// /// `Screen` reproduces the legacy behavior of acting on the whole screen / frontmost /// application. `Window` drives a specific background window of a specific process without /// raising it or moving the global cursor. #[derive(Debug, Clone, Copy, Default, Eq, PartialEq, Serialize, Deserialize)] pub enum Target { /// Target the whole screen / frontmost application (legacy behavior). #[default] Screen, /// Target a specific background window of a specific process. Window { /// The platform window id (a `CGWindowID` on macOS). Must be a concrete, non-zero id /// selected from the enumerated window list. `0` is the "unknown" sentinel and is /// rejected by the actor, since coordinate remapping and window capture both require a /// known window. window_id: u32, /// The pid of the process that owns the window. pid: i32, }, } /// An action paired with the surface it targets. /// /// The target is carried per-action so a single batch can, in principle, drive more than one /// window. An absent / `Screen` target reproduces the legacy whole-screen behavior. #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub struct TargetedAction { pub action: Action, #[serde(default)] pub target: Target, } impl TargetedAction { /// Builds a screen-targeted action (legacy behavior). pub fn screen(action: Action) -> Self { Self { action, target: Target::Screen, } } } /// Metadata about an on-screen window, so a caller can select a window to target. /// Mirrors the fields of the `WindowInfo` API message. #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub struct WindowInfo { /// The platform window id (a `CGWindowID` on macOS). pub window_id: u32, /// The pid of the process that owns the window. pub pid: i32, /// The owning application's name (e.g. "Arc", "Notes"). pub app_name: String, /// The window title, if available. pub title: String, /// The window layer (0 is a normal application window). pub layer: i32, } /// Metadata describing a captured window screenshot. #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub struct CapturedWindow { /// The platform window id that was captured. pub window_id: u32, /// The width of the native captured image, in pixels. pub width_px: i32, /// The height of the native captured image, in pixels. pub height_px: i32, } #[async_trait] pub trait Actor: Send + Sync + 'static { /// Returns the platform that this actor is running on, if known. fn platform(&self) -> Option; async fn perform_actions( &mut self, actions: &[TargetedAction], options: Options, ) -> Result; } /// A key that can be pressed or released. #[derive(Debug, Clone, Eq, Hash, PartialEq, Serialize, Deserialize)] pub enum Key { /// A platform-specific keycode. On macOS and Windows, this is a virtual keycode. /// On Linux, this is an X11 keysym. Keycode(i32), /// A character key (e.g., 'a', '+'). On Windows, `Key::Char` only supports characters in /// the Basic Multilingual Plane (BMP, `U+0000`–`U+FFFF`). Supplementary-plane characters /// (emoji, some CJK extension blocks, etc.) will return an error; use `TypeText` instead for /// those. Char(char), } /// The actions that an actor can perform on the computer. #[serde_as] #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub enum Action { Wait(#[serde_as(as = "DurationSecondsWithFrac")] std::time::Duration), MouseDown { button: MouseButton, #[serde(with = "Vector2IDef")] at: Vector2I, }, MouseUp { button: MouseButton, }, MouseMove { #[serde(with = "Vector2IDef")] to: Vector2I, }, MouseWheel { #[serde(with = "Vector2IDef")] at: Vector2I, direction: ScrollDirection, distance: ScrollDistance, }, TypeText { text: String, }, KeyDown { key: Key, }, KeyUp { key: Key, }, } /// The direction of a scroll action. #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)] pub enum ScrollDirection { Up, Down, Left, Right, } /// The distance of a scroll action. #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)] pub enum ScrollDistance { /// Scroll by a number of pixels. Pixels(i32), /// Scroll by a number of discrete "clicks" (wheel notches). Clicks(i32), } /// A rectangular region defined by top-left and bottom-right corners. /// Coordinates are physical pixels relative to the selected screenshot target. #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)] pub struct ScreenshotRegion { #[serde(with = "Vector2IDef")] pub top_left: Vector2I, #[serde(with = "Vector2IDef")] pub bottom_right: Vector2I, } impl ScreenshotRegion { /// Validates that the region has valid coordinates for screenshot capture. /// /// Returns an error if: /// - `top_left` has negative coordinates /// - `bottom_right` is not strictly greater than `top_left` in both dimensions pub fn validate(&self) -> Result<(), String> { if self.top_left.x() < 0 || self.top_left.y() < 0 { return Err(format!( "Screenshot region top_left must be non-negative, got ({}, {})", self.top_left.x(), self.top_left.y() )); } if self.bottom_right.x() <= self.top_left.x() { return Err(format!( "Screenshot region must have positive width (bottom_right.x {} must be > top_left.x {})", self.bottom_right.x(), self.top_left.x() )); } if self.bottom_right.y() <= self.top_left.y() { return Err(format!( "Screenshot region must have positive height (bottom_right.y {} must be > top_left.y {})", self.bottom_right.y(), self.top_left.y() )); } Ok(()) } } /// Parameters for taking a screenshot after actions. /// If provided, a screenshot will be taken; if `None`, no screenshot is taken. #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)] pub struct ScreenshotParams { /// The maximum length of the long edge of the screenshot in pixels. pub max_long_edge_px: Option, /// The maximum total number of pixels in the screenshot. pub max_total_px: Option, /// Optional sub-region of `target` to capture, in target-relative physical pixels. /// If `None`, captures the full target. #[serde(default)] pub region: Option, /// The surface to capture. `Screen` captures the main display (legacy); `Window` captures /// a specific window's image. #[serde(default)] pub target: Target, } pub struct Options { /// If set, a screenshot will be captured after the actions are executed. /// The parameters specify what constraints, if any, to apply to the screenshot. pub screenshot_params: Option, /// Whether background, per-window computer use is enabled. When false, actors must behave /// exactly like the legacy full-screen path: any window target is ignored, only the main /// display is captured, and no window list or captured-window metadata is returned. pub background_enabled: bool, } /// The buttons of a mouse. #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub enum MouseButton { Left, Right, Middle, /// Mouse button 3 (Back). Back, /// Mouse button 4 (Forward). Forward, } /// The result of performing an action. #[derive(Debug, Clone, Eq, PartialEq)] pub struct ActionResult { pub screenshot: Option, pub cursor_position: Option, /// The on-screen windows, refreshed after the actions run, so the caller always has a fresh /// list to target next. Empty on platforms without window enumeration. pub windows: Vec, /// Metadata about the captured window, populated only when a window target was /// screenshotted, so window-local coordinates map onto the screenshot image. pub captured_window: Option, } impl ActionResult { /// Builds a result that carries no window list or captured-window metadata (used by /// platforms and code paths that do not support per-window targeting). pub fn legacy(screenshot: Option, cursor_position: Option) -> Self { Self { screenshot, cursor_position, windows: Vec::new(), captured_window: None, } } } /// A simple representation of a screenshot. #[derive(Clone, Eq, PartialEq)] pub struct Screenshot { /// The width of the screenshot image data in pixels. pub width: usize, /// The height of the screenshot image data in pixels. pub height: usize, /// The original width of the screenshot before any downscaling was applied. pub original_width: usize, /// The original height of the screenshot before any downscaling was applied. pub original_height: usize, // TODO(AGENT-2283): consider making this a type that is cheap to clone // (e.g.: `Arc<[u8]>`) pub data: Vec, pub mime_type: Cow<'static, str>, } impl std::fmt::Debug for Screenshot { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Screenshot") .field("width", &self.width) .field("height", &self.height) .field("original_width", &self.original_width) .field("original_height", &self.original_height) .field("num_data_bytes", &self.data.len()) .finish() } } /// Remote derive helper for `Vector2I` from `pathfinder_geometry`. #[derive(Serialize, Deserialize)] #[serde(remote = "Vector2I")] struct Vector2IDef { #[serde(getter = "get_vector2i_x")] x: i32, #[serde(getter = "get_vector2i_y")] y: i32, } fn get_vector2i_x(v: &Vector2I) -> i32 { v.x() } fn get_vector2i_y(v: &Vector2I) -> i32 { v.y() } impl From for Vector2I { fn from(def: Vector2IDef) -> Self { Vector2I::new(def.x, def.y) } }