first pass of merging in warp (doesn't build)
This commit is contained in:
Ryan Ward
@@ -25,29 +25,36 @@ tokio = { workspace = true, features = ["rt", "macros"] }
|
||||
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
command.workspace = true
|
||||
core-foundation.workspace = true
|
||||
dispatch2 = "0.3.0"
|
||||
image.workspace = true
|
||||
instant.workspace = true
|
||||
libc.workspace = true
|
||||
objc2.workspace = true
|
||||
objc2-app-kit.workspace = true
|
||||
# `NSEvent`/`NSGraphicsContext`/`objc2-core-graphics` are needed to build window-targeted mouse
|
||||
# events via `NSEvent::mouseEventWithType_...` and bridge to `CGEvent` for PID-targeted posting.
|
||||
objc2-app-kit = { workspace = true, features = [
|
||||
"NSEvent",
|
||||
"NSGraphicsContext",
|
||||
"objc2-core-graphics",
|
||||
] }
|
||||
objc2-core-foundation.workspace = true
|
||||
objc2-core-graphics.workspace = true
|
||||
# The `libc` feature is required for `CGEvent::post_to_pid`, which delivers events to a
|
||||
# specific process without moving the global cursor.
|
||||
objc2-core-graphics = { workspace = true, features = ["libc"] }
|
||||
tempfile.workspace = true
|
||||
galaxyui.workspace = true
|
||||
galaxyui_core.workspace = true
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
[target.'cfg(any(target_os = "linux", target_os = "freebsd"))'.dependencies]
|
||||
ashpd.workspace = true
|
||||
futures.workspace = true
|
||||
image.workspace = true
|
||||
url.workspace = true
|
||||
galaxyui.workspace = true
|
||||
galaxyui_core.workspace = true
|
||||
x11rb = { workspace = true, features = ["xtest"] }
|
||||
zbus.workspace = true
|
||||
|
||||
[target.'cfg(target_os = "windows")'.dependencies]
|
||||
image.workspace = true
|
||||
galaxyui.workspace = true
|
||||
galaxyui_core.workspace = true
|
||||
windows = { workspace = true, features = [
|
||||
"Win32_Foundation",
|
||||
"Win32_Graphics_Gdi",
|
||||
|
||||
@@ -3,7 +3,7 @@ use cfg_aliases::cfg_aliases;
|
||||
fn main() {
|
||||
cfg_aliases! {
|
||||
macos: { target_os = "macos" },
|
||||
linux: { target_os = "linux" },
|
||||
linux: { any(target_os = "linux", target_os = "freebsd") },
|
||||
noop: { not(any(macos, linux, windows)) },
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,17 +4,44 @@ use std::path::PathBuf;
|
||||
|
||||
use clap::{Parser, Subcommand, ValueEnum};
|
||||
use computer_use::{
|
||||
Action, Key, MouseButton, Options, ScreenshotParams, ScreenshotRegion, Vector2I,
|
||||
Action, Key, MouseButton, Options, ScreenshotParams, ScreenshotRegion, Target, TargetedAction,
|
||||
Vector2I,
|
||||
};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "use_computer")]
|
||||
#[command(about = "Manually test computer use actions")]
|
||||
struct Cli {
|
||||
/// Experimental (macOS only): target a specific background window/process instead of the
|
||||
/// screen. Deliver events directly to this process ID (and `--window-id`, if given) without
|
||||
/// moving the real cursor or raising the window.
|
||||
#[arg(long, global = true)]
|
||||
pid: Option<i32>,
|
||||
|
||||
/// Experimental (macOS only): the CGWindowID of the window to target. Required when `--pid`
|
||||
/// is given. Use the `windows` subcommand to list window ids.
|
||||
#[arg(long, global = true)]
|
||||
window_id: Option<u32>,
|
||||
|
||||
#[command(subcommand)]
|
||||
command: Command,
|
||||
}
|
||||
|
||||
impl Cli {
|
||||
/// Resolves the per-action / screenshot target from the CLI flags. A `--pid` selects a
|
||||
/// background window target; otherwise the legacy whole-screen target is used. Callers must
|
||||
/// validate that `--window-id` is present whenever `--pid` is given (see `main`), so the
|
||||
/// ambiguous `0` sentinel is never sent to the actor.
|
||||
fn target(&self) -> Target {
|
||||
match (self.pid, self.window_id) {
|
||||
(Some(pid), Some(window_id)) => Target::Window { window_id, pid },
|
||||
// `--pid` without `--window-id` is rejected up front in `main`; fall back to the
|
||||
// screen target here so a missing id can never become a `0`-id window target.
|
||||
(Some(_), None) | (None, _) => Target::Screen,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Command {
|
||||
/// Perform a mouse click (mouse down + mouse up) at a position.
|
||||
@@ -46,6 +73,9 @@ enum Command {
|
||||
/// The key to press. Can be a single character (e.g., "a") or a keycode (e.g., "0x24" for Return on macOS).
|
||||
key: String,
|
||||
},
|
||||
/// Experimental (macOS only): list on-screen windows with their window number, owner PID,
|
||||
/// owner name, layer, and bounds, to help identify the right target PID/window.
|
||||
Windows,
|
||||
}
|
||||
|
||||
#[derive(Clone, ValueEnum)]
|
||||
@@ -94,6 +124,29 @@ fn parse_region(s: &str) -> Result<(i32, i32, i32, i32), String> {
|
||||
async fn main() {
|
||||
let cli = Cli::parse();
|
||||
|
||||
// Window listing does not go through the actor's action model; handle it up front.
|
||||
if let Command::Windows = cli.command {
|
||||
match computer_use::experimental_list_windows() {
|
||||
Ok(text) => print!("{text}"),
|
||||
Err(e) => {
|
||||
eprintln!("Error: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// A window target needs a concrete window id; require it alongside `--pid` so the ambiguous
|
||||
// `0` sentinel is never sent to the actor.
|
||||
if cli.pid.is_some() && cli.window_id.is_none() {
|
||||
eprintln!(
|
||||
"--window-id is required when --pid is given. Use the `windows` subcommand to list \
|
||||
window ids."
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
let target = cli.target();
|
||||
let mut actor = computer_use::create_actor();
|
||||
|
||||
let (actions, screenshot_params, output_path) = match cli.command {
|
||||
@@ -124,6 +177,7 @@ async fn main() {
|
||||
max_long_edge_px: None,
|
||||
max_total_px: None,
|
||||
region,
|
||||
target,
|
||||
}),
|
||||
Some(output),
|
||||
)
|
||||
@@ -154,9 +208,21 @@ async fn main() {
|
||||
None,
|
||||
)
|
||||
}
|
||||
// Handled before the actor is created, above.
|
||||
Command::Windows => unreachable!(),
|
||||
};
|
||||
|
||||
let options = Options { screenshot_params };
|
||||
// Pair every action with the resolved target before handing off to the actor.
|
||||
let actions: Vec<TargetedAction> = actions
|
||||
.into_iter()
|
||||
.map(|action| TargetedAction { action, target })
|
||||
.collect();
|
||||
// The CLI is a developer tool for exercising window targeting, so background per-window
|
||||
// control is always enabled here.
|
||||
let options = Options {
|
||||
screenshot_params,
|
||||
background_enabled: true,
|
||||
};
|
||||
|
||||
match actor.perform_actions(&actions, options).await {
|
||||
Ok(result) => {
|
||||
|
||||
@@ -7,15 +7,14 @@ mod noop;
|
||||
#[cfg(any(macos, linux, windows))]
|
||||
mod screenshot_utils;
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use async_trait::async_trait;
|
||||
// Clippy doesn't like us pulling in a file as two different modules,
|
||||
// so we add this alias instead of using another cfg_attr on the imp
|
||||
// module definition.
|
||||
#[cfg(noop)]
|
||||
use noop as imp;
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use async_trait::async_trait;
|
||||
pub use pathfinder_geometry::vector::Vector2I;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_with::{DurationSecondsWithFrac, serde_as};
|
||||
@@ -46,6 +45,117 @@ pub fn create_actor() -> Box<dyn Actor> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether background, per-window control (driving a specific window without raising it
|
||||
/// or moving the cursor) is available on this client and OS. When false, callers should target
|
||||
/// the whole screen / frontmost application.
|
||||
pub fn background_supported() -> bool {
|
||||
if cfg!(feature = "test-util") {
|
||||
noop::background_supported()
|
||||
} else {
|
||||
imp::background_supported()
|
||||
}
|
||||
}
|
||||
|
||||
/// Enumerates the on-screen windows, returning their metadata so a caller can pick one to
|
||||
/// target. Returns an empty list on platforms where window enumeration is unsupported.
|
||||
pub fn enumerate_windows() -> Vec<WindowInfo> {
|
||||
#[cfg(macos)]
|
||||
{
|
||||
imp::enumerate_windows()
|
||||
}
|
||||
#[cfg(not(macos))]
|
||||
{
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Experimental: lists on-screen windows as a formatted diagnostic string. macOS only.
|
||||
///
|
||||
/// Unlike [`enumerate_windows`], which returns slim [`WindowInfo`] records for window selection
|
||||
/// and wire serialization, this function returns richer data including window bounds, formatted
|
||||
/// as a human-readable table for CLI debugging. The two use separate types intentionally:
|
||||
/// [`WindowInfo`] is kept wire-safe and bounds-free; the diagnostic output carries bounds that
|
||||
/// are not part of the API representation.
|
||||
#[cfg(macos)]
|
||||
pub fn experimental_list_windows() -> Result<String, String> {
|
||||
Ok(imp::list_windows())
|
||||
}
|
||||
|
||||
/// Experimental: lists on-screen windows. Unsupported on this platform.
|
||||
#[cfg(not(macos))]
|
||||
pub fn experimental_list_windows() -> Result<String, String> {
|
||||
Err("Window listing is only supported on macOS.".to_string())
|
||||
}
|
||||
|
||||
/// The surface that a computer-use action or screenshot targets.
|
||||
///
|
||||
/// `Screen` reproduces the legacy behavior of acting on the whole screen / frontmost
|
||||
/// application. `Window` drives a specific background window of a specific process without
|
||||
/// raising it or moving the global cursor.
|
||||
#[derive(Debug, Clone, Copy, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||
pub enum Target {
|
||||
/// Target the whole screen / frontmost application (legacy behavior).
|
||||
#[default]
|
||||
Screen,
|
||||
/// Target a specific background window of a specific process.
|
||||
Window {
|
||||
/// The platform window id (a `CGWindowID` on macOS). Must be a concrete, non-zero id
|
||||
/// selected from the enumerated window list. `0` is the "unknown" sentinel and is
|
||||
/// rejected by the actor, since coordinate remapping and window capture both require a
|
||||
/// known window.
|
||||
window_id: u32,
|
||||
/// The pid of the process that owns the window.
|
||||
pid: i32,
|
||||
},
|
||||
}
|
||||
|
||||
/// An action paired with the surface it targets.
|
||||
///
|
||||
/// The target is carried per-action so a single batch can, in principle, drive more than one
|
||||
/// window. An absent / `Screen` target reproduces the legacy whole-screen behavior.
|
||||
#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TargetedAction {
|
||||
pub action: Action,
|
||||
#[serde(default)]
|
||||
pub target: Target,
|
||||
}
|
||||
|
||||
impl TargetedAction {
|
||||
/// Builds a screen-targeted action (legacy behavior).
|
||||
pub fn screen(action: Action) -> Self {
|
||||
Self {
|
||||
action,
|
||||
target: Target::Screen,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Metadata about an on-screen window, so a caller can select a window to target.
|
||||
/// Mirrors the fields of the `WindowInfo` API message.
|
||||
#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
|
||||
pub struct WindowInfo {
|
||||
/// The platform window id (a `CGWindowID` on macOS).
|
||||
pub window_id: u32,
|
||||
/// The pid of the process that owns the window.
|
||||
pub pid: i32,
|
||||
/// The owning application's name (e.g. "Arc", "Notes").
|
||||
pub app_name: String,
|
||||
/// The window title, if available.
|
||||
pub title: String,
|
||||
/// The window layer (0 is a normal application window).
|
||||
pub layer: i32,
|
||||
}
|
||||
/// Metadata describing a captured window screenshot.
|
||||
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
|
||||
pub struct CapturedWindow {
|
||||
/// The platform window id that was captured.
|
||||
pub window_id: u32,
|
||||
/// The width of the native captured image, in pixels.
|
||||
pub width_px: i32,
|
||||
/// The height of the native captured image, in pixels.
|
||||
pub height_px: i32,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait Actor: Send + Sync + 'static {
|
||||
/// Returns the platform that this actor is running on, if known.
|
||||
@@ -53,7 +163,7 @@ pub trait Actor: Send + Sync + 'static {
|
||||
|
||||
async fn perform_actions(
|
||||
&mut self,
|
||||
actions: &[Action],
|
||||
actions: &[TargetedAction],
|
||||
options: Options,
|
||||
) -> Result<ActionResult, String>;
|
||||
}
|
||||
@@ -124,7 +234,7 @@ pub enum ScrollDistance {
|
||||
}
|
||||
|
||||
/// A rectangular region defined by top-left and bottom-right corners.
|
||||
/// Coordinates are in physical screen pixels (same coordinate space as mouse actions).
|
||||
/// Coordinates are physical pixels relative to the selected screenshot target.
|
||||
#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ScreenshotRegion {
|
||||
#[serde(with = "Vector2IDef")]
|
||||
@@ -173,15 +283,24 @@ pub struct ScreenshotParams {
|
||||
pub max_long_edge_px: Option<usize>,
|
||||
/// The maximum total number of pixels in the screenshot.
|
||||
pub max_total_px: Option<usize>,
|
||||
/// Optional region to capture. If `None`, captures the full display.
|
||||
/// Optional sub-region of `target` to capture, in target-relative physical pixels.
|
||||
/// If `None`, captures the full target.
|
||||
#[serde(default)]
|
||||
pub region: Option<ScreenshotRegion>,
|
||||
/// The surface to capture. `Screen` captures the main display (legacy); `Window` captures
|
||||
/// a specific window's image.
|
||||
#[serde(default)]
|
||||
pub target: Target,
|
||||
}
|
||||
|
||||
pub struct Options {
|
||||
/// If set, a screenshot will be captured after the actions are executed.
|
||||
/// The parameters specify what constraints, if any, to apply to the screenshot.
|
||||
pub screenshot_params: Option<ScreenshotParams>,
|
||||
/// Whether background, per-window computer use is enabled. When false, actors must behave
|
||||
/// exactly like the legacy full-screen path: any window target is ignored, only the main
|
||||
/// display is captured, and no window list or captured-window metadata is returned.
|
||||
pub background_enabled: bool,
|
||||
}
|
||||
|
||||
/// The buttons of a mouse.
|
||||
@@ -201,6 +320,25 @@ pub enum MouseButton {
|
||||
pub struct ActionResult {
|
||||
pub screenshot: Option<Screenshot>,
|
||||
pub cursor_position: Option<Vector2I>,
|
||||
/// The on-screen windows, refreshed after the actions run, so the caller always has a fresh
|
||||
/// list to target next. Empty on platforms without window enumeration.
|
||||
pub windows: Vec<WindowInfo>,
|
||||
/// Metadata about the captured window, populated only when a window target was
|
||||
/// screenshotted, so window-local coordinates map onto the screenshot image.
|
||||
pub captured_window: Option<CapturedWindow>,
|
||||
}
|
||||
|
||||
impl ActionResult {
|
||||
/// Builds a result that carries no window list or captured-window metadata (used by
|
||||
/// platforms and code paths that do not support per-window targeting).
|
||||
pub fn legacy(screenshot: Option<Screenshot>, cursor_position: Option<Vector2I>) -> Self {
|
||||
Self {
|
||||
screenshot,
|
||||
cursor_position,
|
||||
windows: Vec::new(),
|
||||
captured_window: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A simple representation of a screenshot.
|
||||
|
||||
@@ -4,7 +4,7 @@ mod x11;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::{Action, ActionResult, Options};
|
||||
use crate::{ActionResult, Options, TargetedAction};
|
||||
|
||||
/// Returns true if a Wayland environment is available.
|
||||
fn is_wayland_available() -> bool {
|
||||
@@ -24,6 +24,12 @@ pub fn is_supported_on_current_platform() -> bool {
|
||||
is_wayland_available() || is_x11_available()
|
||||
}
|
||||
|
||||
/// Reports whether background, per-window control is available. The Linux input stack drives the
|
||||
/// whole screen / frontmost application, so per-window background control is unsupported.
|
||||
pub fn background_supported() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub struct Actor {
|
||||
inner: ActorInner,
|
||||
}
|
||||
@@ -77,7 +83,7 @@ impl super::Actor for Actor {
|
||||
|
||||
async fn perform_actions(
|
||||
&mut self,
|
||||
actions: &[Action],
|
||||
actions: &[TargetedAction],
|
||||
options: Options,
|
||||
) -> Result<ActionResult, String> {
|
||||
match &mut self.inner {
|
||||
|
||||
@@ -11,14 +11,13 @@ mod screenshot;
|
||||
mod session;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use galaxyui::r#async::Timer;
|
||||
use pathfinder_geometry::vector::Vector2I;
|
||||
|
||||
use crate::{Action, ActionResult, Options};
|
||||
|
||||
use keyboard::Keyboard;
|
||||
use mouse::Mouse;
|
||||
use pathfinder_geometry::vector::Vector2I;
|
||||
use session::PortalSession;
|
||||
use galaxyui_core::r#async::Timer;
|
||||
|
||||
use crate::{Action, ActionResult, Options, TargetedAction};
|
||||
|
||||
/// An actor that performs computer use actions on Wayland via XDG portals.
|
||||
pub struct Actor {
|
||||
@@ -58,7 +57,7 @@ impl crate::Actor for Actor {
|
||||
|
||||
async fn perform_actions(
|
||||
&mut self,
|
||||
actions: &[Action],
|
||||
actions: &[TargetedAction],
|
||||
options: Options,
|
||||
) -> Result<ActionResult, String> {
|
||||
// Ensure we have an active session before processing actions.
|
||||
@@ -67,7 +66,10 @@ impl crate::Actor for Actor {
|
||||
|
||||
let mut last_mouse_position: Option<Vector2I> = None;
|
||||
|
||||
for action in actions {
|
||||
for targeted in actions {
|
||||
// Per-window targeting is not supported on Wayland; act on the screen / focused
|
||||
// surface regardless of the requested target.
|
||||
let action: &Action = &targeted.action;
|
||||
// Re-acquire session reference each iteration (borrow checker workaround).
|
||||
let session = self
|
||||
.session
|
||||
@@ -153,9 +155,6 @@ impl crate::Actor for Actor {
|
||||
self.mouse.last_position()
|
||||
};
|
||||
|
||||
Ok(ActionResult {
|
||||
screenshot,
|
||||
cursor_position,
|
||||
})
|
||||
Ok(ActionResult::legacy(screenshot, cursor_position))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,12 +7,13 @@ mod screenshot;
|
||||
use async_trait::async_trait;
|
||||
use galaxyui::r#async::Timer;
|
||||
use pathfinder_geometry::vector::Vector2I;
|
||||
use galaxyui_core::r#async::Timer;
|
||||
use x11rb::connection::Connection;
|
||||
use x11rb::protocol::xproto::{self, ConnectionExt as _};
|
||||
use x11rb::protocol::xtest::ConnectionExt as _;
|
||||
use x11rb::rust_connection::RustConnection;
|
||||
|
||||
use crate::{Action, ActionResult, Options};
|
||||
use crate::{Action, ActionResult, Options, TargetedAction};
|
||||
|
||||
/// An actor that performs computer use actions on X11.
|
||||
pub struct Actor {
|
||||
@@ -71,14 +72,17 @@ impl crate::Actor for Actor {
|
||||
|
||||
async fn perform_actions(
|
||||
&mut self,
|
||||
actions: &[Action],
|
||||
actions: &[TargetedAction],
|
||||
options: Options,
|
||||
) -> Result<ActionResult, String> {
|
||||
let mut mouse = mouse::Mouse::new(&self.conn, self.root_window());
|
||||
let mut keyboard = keyboard::Keyboard::new(&self.conn, &self.keyboard_mapping);
|
||||
let mut last_mouse_position: Option<Vector2I> = None;
|
||||
|
||||
for action in actions {
|
||||
for targeted in actions {
|
||||
// Per-window targeting is not supported on X11; act on the screen regardless of the
|
||||
// requested target.
|
||||
let action: &Action = &targeted.action;
|
||||
match action {
|
||||
Action::Wait(duration) => {
|
||||
Timer::after(*duration).await;
|
||||
@@ -135,9 +139,6 @@ impl crate::Actor for Actor {
|
||||
Some(mouse.current_position()?)
|
||||
};
|
||||
|
||||
Ok(ActionResult {
|
||||
screenshot,
|
||||
cursor_position,
|
||||
})
|
||||
Ok(ActionResult::legacy(screenshot, cursor_position))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,332 @@
|
||||
//! Background window activation without private SkyLight APIs.
|
||||
//!
|
||||
//! Background input (delivering clicks/keys to a window that is not frontmost, without moving
|
||||
//! the real cursor) requires the target window to believe it is in the AppKit-active input
|
||||
//! state. We achieve that the way a real first click on an inactive window would, but without
|
||||
//! the visual side effect of bringing the app to the front:
|
||||
//!
|
||||
//! 1. Install per-process [`CGEvent`] event taps (`CGEventTapCreateForPid`) on both the
|
||||
//! previously-frontmost app and the target app. The taps run on a dedicated run-loop thread
|
||||
//! and drop the focus-change messages that macOS would otherwise deliver to the previous app
|
||||
//! to switch the user's frontmost application.
|
||||
//! 2. Send an `appKitDefined` activation primer event (an `NSEvent` of type `AppKitDefined`,
|
||||
//! subtype `ApplicationActivated`) directly to the target process via `CGEventPostToPid`.
|
||||
//! 3. Send a single "primer" left click to the exact center of the window. While the window is
|
||||
//! inactive, macOS routes this first click through its activation flow instead of firing a
|
||||
//! UI action, so it activates the window without otherwise affecting the app.
|
||||
//!
|
||||
//! State is tracked in a process-global registry keyed by `(pid, window)` rather than on the
|
||||
//! per-action [`super::Actor`] (which is recreated for every computer-use turn). This lets a
|
||||
//! single computer-use session activate a window once and reuse that activation across later
|
||||
//! turns without re-sending the disruptive center click, and lets concurrent computer-use
|
||||
//! sessions targeting different windows coexist while serializing each activation handshake
|
||||
//! through the registry lock.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::ffi::c_void;
|
||||
use std::ptr::NonNull;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex, OnceLock, mpsc};
|
||||
use std::thread::{self, JoinHandle};
|
||||
use std::time::Duration;
|
||||
|
||||
use objc2_app_kit::{NSEvent, NSEventModifierFlags, NSEventSubtype, NSEventType};
|
||||
use objc2_core_foundation::{CFMachPort, CFRunLoop, CGPoint, kCFRunLoopDefaultMode};
|
||||
use objc2_core_graphics::{
|
||||
CGEvent, CGEventTapOptions, CGEventTapPlacement, CGEventTapProxy, CGEventType, CGMouseButton,
|
||||
};
|
||||
|
||||
use super::window::{self, WindowInfo};
|
||||
|
||||
/// Delay after the `appKitDefined` activation primer, giving AppKit time to process it before
|
||||
/// the center primer click arrives.
|
||||
const APPKIT_PRIMER_DELAY: Duration = Duration::from_millis(20);
|
||||
/// Delay between the center primer's mouse-down and mouse-up.
|
||||
const PRIMER_CLICK_HOLD: Duration = Duration::from_millis(30);
|
||||
/// Delay after the center primer click settles.
|
||||
const PRIMER_CLICK_SETTLE: Duration = Duration::from_millis(20);
|
||||
/// How long each run-loop service iteration blocks before re-checking the stop flag.
|
||||
const RUN_LOOP_SERVICE_INTERVAL: f64 = 0.1;
|
||||
|
||||
/// Ensures the window described by `info` (owned by `target_pid`) is in the AppKit-active input
|
||||
/// state so background events are accepted, without raising it or moving the cursor.
|
||||
///
|
||||
/// The first call for a given `(pid, window)` installs focus-suppression taps and sends the
|
||||
/// activation primers (including the center primer click). Subsequent calls for the same window
|
||||
/// are a no-op, so the disruptive center click is never re-sent across turns. If the target app
|
||||
/// is already frontmost, no activation is needed and this is a no-op.
|
||||
pub fn ensure_activated(target_pid: libc::pid_t, info: &WindowInfo) {
|
||||
let window_number = info.number;
|
||||
if window_number <= 0 {
|
||||
return;
|
||||
}
|
||||
let key = (target_pid, window_number);
|
||||
|
||||
let mut registry = registry().lock().unwrap();
|
||||
if registry.contains_key(&key) {
|
||||
// Already activated this window; do not re-activate (a second center click would land
|
||||
// as a real click rather than being absorbed by the activation flow).
|
||||
return;
|
||||
}
|
||||
|
||||
// The window that currently owns input focus, so we can both protect it from deactivation
|
||||
// and detect when the target app is already frontmost.
|
||||
let previous = window::frontmost_window();
|
||||
if previous.map(|(pid, _)| pid) == Some(target_pid) {
|
||||
// The target app is already frontmost: events route to it normally and no focus would
|
||||
// be stolen, so there is nothing to activate or suppress.
|
||||
return;
|
||||
}
|
||||
|
||||
let suppress = Arc::new(AtomicBool::new(true));
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
|
||||
// Install focus-suppression taps before sending the activation click, so the focus-change
|
||||
// messages it triggers are intercepted. Skip the taps if there is no distinct previous app.
|
||||
let thread = match previous {
|
||||
Some((previous_pid, _)) if previous_pid != target_pid => {
|
||||
spawn_tap_thread(previous_pid, target_pid, suppress.clone(), stop.clone())
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let has_taps = thread.is_some();
|
||||
|
||||
// Activate: AppKit activation primer, then the center-of-window primer click.
|
||||
post_appkit_activation(
|
||||
target_pid,
|
||||
window_number,
|
||||
NSEventSubtype::ApplicationActivated.0,
|
||||
);
|
||||
thread::sleep(APPKIT_PRIMER_DELAY);
|
||||
post_center_primer(target_pid, info);
|
||||
|
||||
registry.insert(
|
||||
key,
|
||||
ActiveSession {
|
||||
suppress,
|
||||
stop,
|
||||
thread,
|
||||
has_taps,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// The process-global registry of activated windows, keyed by `(pid, window_number)`.
|
||||
fn registry() -> &'static Mutex<HashMap<(libc::pid_t, i64), ActiveSession>> {
|
||||
static REGISTRY: OnceLock<Mutex<HashMap<(libc::pid_t, i64), ActiveSession>>> = OnceLock::new();
|
||||
REGISTRY.get_or_init(|| Mutex::new(HashMap::new()))
|
||||
}
|
||||
|
||||
/// Tracks an activated window: the focus-suppression tap thread (if any) and the flags used to
|
||||
/// drive and tear it down.
|
||||
struct ActiveSession {
|
||||
/// While set, the tap callback drops focus-change messages headed to the previous app.
|
||||
suppress: Arc<AtomicBool>,
|
||||
/// Signals the tap thread's run loop to exit.
|
||||
stop: Arc<AtomicBool>,
|
||||
/// The run-loop thread servicing the taps, joined on teardown.
|
||||
thread: Option<JoinHandle<()>>,
|
||||
has_taps: bool,
|
||||
}
|
||||
|
||||
impl Drop for ActiveSession {
|
||||
fn drop(&mut self) {
|
||||
if self.has_taps {
|
||||
self.suppress.store(false, Ordering::SeqCst);
|
||||
self.stop.store(true, Ordering::SeqCst);
|
||||
if let Some(thread) = self.thread.take() {
|
||||
let _ = thread.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-tap data handed to the C event-tap callback via its `user_info` pointer.
|
||||
struct TapContext {
|
||||
/// Shared with [`ActiveSession`]; gates focus-message suppression.
|
||||
suppress: Arc<AtomicBool>,
|
||||
/// Whether this tap is attached to the previously-frontmost app (whose focus messages we
|
||||
/// drop) versus the target app (which we always let through).
|
||||
is_previous: bool,
|
||||
}
|
||||
|
||||
/// Spawns the run-loop thread that installs and services the focus-suppression taps for
|
||||
/// `previous_pid` and `target_pid`. Returns the thread handle once both taps are installed, or
|
||||
/// `None` if tap creation failed (e.g. missing permission).
|
||||
fn spawn_tap_thread(
|
||||
previous_pid: libc::pid_t,
|
||||
target_pid: libc::pid_t,
|
||||
suppress: Arc<AtomicBool>,
|
||||
stop: Arc<AtomicBool>,
|
||||
) -> Option<JoinHandle<()>> {
|
||||
let (ready_tx, ready_rx) = mpsc::channel();
|
||||
let handle = thread::Builder::new()
|
||||
.name("cu-bg-activation".to_string())
|
||||
.spawn(move || run_tap_loop(previous_pid, target_pid, suppress, stop, ready_tx))
|
||||
.ok()?;
|
||||
|
||||
match ready_rx.recv() {
|
||||
Ok(true) => Some(handle),
|
||||
_ => {
|
||||
// Tap installation failed (or the thread died); reclaim it and report no taps.
|
||||
let _ = handle.join();
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Body of the tap thread: creates the taps on this thread's run loop, signals readiness, then
|
||||
/// services the run loop until asked to stop, tearing the taps down on exit.
|
||||
fn run_tap_loop(
|
||||
previous_pid: libc::pid_t,
|
||||
target_pid: libc::pid_t,
|
||||
suppress: Arc<AtomicBool>,
|
||||
stop: Arc<AtomicBool>,
|
||||
ready_tx: mpsc::Sender<bool>,
|
||||
) {
|
||||
let Some(run_loop) = CFRunLoop::current() else {
|
||||
let _ = ready_tx.send(false);
|
||||
return;
|
||||
};
|
||||
let mode = unsafe { kCFRunLoopDefaultMode };
|
||||
|
||||
// Keep the taps, their run-loop sources, and the heap-allocated contexts alive for as long
|
||||
// as the run loop runs. The contexts are reclaimed (and dropped) after the loop exits.
|
||||
let mut taps = Vec::new();
|
||||
let mut sources = Vec::new();
|
||||
let mut contexts: Vec<*mut TapContext> = Vec::new();
|
||||
|
||||
for (pid, is_previous) in [(previous_pid, true), (target_pid, false)] {
|
||||
let context = Box::into_raw(Box::new(TapContext {
|
||||
suppress: suppress.clone(),
|
||||
is_previous,
|
||||
}));
|
||||
// SAFETY: `tap_callback` matches the `CGEventTapCallBack` ABI and `context` is a valid,
|
||||
// owned `TapContext` pointer that outlives the tap (it is dropped only after the run
|
||||
// loop stops, below).
|
||||
let tap = unsafe {
|
||||
CGEvent::tap_create_for_pid(
|
||||
pid,
|
||||
CGEventTapPlacement::HeadInsertEventTap,
|
||||
CGEventTapOptions::Default,
|
||||
u64::MAX,
|
||||
Some(tap_callback),
|
||||
context as *mut c_void,
|
||||
)
|
||||
};
|
||||
let Some(tap) = tap else {
|
||||
// Reclaim the context this tap would have owned.
|
||||
drop(unsafe { Box::from_raw(context) });
|
||||
continue;
|
||||
};
|
||||
let Some(source) = CFMachPort::new_run_loop_source(None, Some(&tap), 0) else {
|
||||
tap.invalidate();
|
||||
drop(unsafe { Box::from_raw(context) });
|
||||
continue;
|
||||
};
|
||||
run_loop.add_source(Some(&source), mode);
|
||||
CGEvent::tap_enable(&tap, true);
|
||||
taps.push(tap);
|
||||
sources.push(source);
|
||||
contexts.push(context);
|
||||
}
|
||||
|
||||
let installed = !taps.is_empty();
|
||||
let _ = ready_tx.send(installed);
|
||||
if !installed {
|
||||
return;
|
||||
}
|
||||
|
||||
// Service the taps until teardown. `run_in_mode` blocks up to the interval (sleeping when
|
||||
// idle), so re-checking the flag this way costs only a brief teardown latency.
|
||||
while !stop.load(Ordering::SeqCst) {
|
||||
CFRunLoop::run_in_mode(mode, RUN_LOOP_SERVICE_INTERVAL, false);
|
||||
}
|
||||
|
||||
for tap in &taps {
|
||||
tap.invalidate();
|
||||
}
|
||||
drop(sources);
|
||||
drop(taps);
|
||||
for context in contexts {
|
||||
drop(unsafe { Box::from_raw(context) });
|
||||
}
|
||||
}
|
||||
|
||||
/// The C event-tap callback. Returns the event to pass it through, or null to drop it.
|
||||
///
|
||||
/// Focus-change messages do not have a stable public `CGEventType` across macOS versions, so
|
||||
/// they are identified by their raw values (13, 19, 20). When suppression is active we drop
|
||||
/// those headed to the previous app (keeping it from being deactivated, i.e. keeping it the
|
||||
/// user's frontmost app) while letting the target app's activation through.
|
||||
unsafe extern "C-unwind" fn tap_callback(
|
||||
_proxy: CGEventTapProxy,
|
||||
event_type: CGEventType,
|
||||
event: NonNull<CGEvent>,
|
||||
user_info: *mut c_void,
|
||||
) -> *mut CGEvent {
|
||||
let event_ptr = event.as_ptr();
|
||||
let Some(context) = (unsafe { (user_info as *const TapContext).as_ref() }) else {
|
||||
return event_ptr;
|
||||
};
|
||||
let is_focus_message = matches!(event_type.0, 13 | 19 | 20);
|
||||
if is_focus_message && context.is_previous && context.suppress.load(Ordering::SeqCst) {
|
||||
return std::ptr::null_mut();
|
||||
}
|
||||
event_ptr
|
||||
}
|
||||
|
||||
/// Sends an `appKitDefined` application-activation event to the target process. `subtype`
|
||||
/// selects activate ([`NSEventSubtype::ApplicationActivated`]) vs. deactivate
|
||||
/// ([`NSEventSubtype::ApplicationDeactivated`]).
|
||||
fn post_appkit_activation(target_pid: libc::pid_t, window_number: i64, subtype: i16) {
|
||||
let Some(event) = NSEvent::otherEventWithType_location_modifierFlags_timestamp_windowNumber_context_subtype_data1_data2(
|
||||
NSEventType::AppKitDefined,
|
||||
CGPoint { x: 0.0, y: 0.0 },
|
||||
NSEventModifierFlags::empty(),
|
||||
0.0,
|
||||
window_number as isize,
|
||||
None,
|
||||
subtype,
|
||||
0,
|
||||
0,
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
let Some(cg_event) = event.CGEvent() else {
|
||||
return;
|
||||
};
|
||||
// Associate the event with the target window so AppKit applies the activation to it.
|
||||
super::mouse::set_window_addressing_fields(&cg_event, window_number);
|
||||
CGEvent::post_to_pid(target_pid, Some(&cg_event));
|
||||
}
|
||||
|
||||
/// Sends a single left click to the exact center of the window. On an inactive window this is
|
||||
/// absorbed by the activation flow rather than firing a UI action; the center avoids the
|
||||
/// title-bar traffic-light controls (which respond even when inactive).
|
||||
fn post_center_primer(target_pid: libc::pid_t, info: &WindowInfo) {
|
||||
let center = CGPoint {
|
||||
x: info.x + info.width / 2.0,
|
||||
y: info.y + info.height / 2.0,
|
||||
};
|
||||
super::mouse::post_window_mouse_event(
|
||||
target_pid,
|
||||
info,
|
||||
CGEventType::LeftMouseDown,
|
||||
CGMouseButton::Left,
|
||||
center,
|
||||
1,
|
||||
1.0,
|
||||
);
|
||||
thread::sleep(PRIMER_CLICK_HOLD);
|
||||
super::mouse::post_window_mouse_event(
|
||||
target_pid,
|
||||
info,
|
||||
CGEventType::LeftMouseUp,
|
||||
CGMouseButton::Left,
|
||||
center,
|
||||
1,
|
||||
0.0,
|
||||
);
|
||||
thread::sleep(PRIMER_CLICK_SETTLE);
|
||||
}
|
||||
@@ -1,37 +1,86 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use objc2_core_graphics::{
|
||||
CGEvent, CGEventFlags, CGEventSource, CGEventSourceStateID, CGEventTapLocation, CGKeyCode,
|
||||
};
|
||||
use objc2_core_graphics::{CGEvent, CGEventFlags, CGEventSource, CGEventSourceStateID, CGKeyCode};
|
||||
|
||||
use super::keycode_cache;
|
||||
use crate::Key;
|
||||
use super::post::PostTarget;
|
||||
use super::{activation, keycode_cache, window};
|
||||
use crate::{Key, Target};
|
||||
|
||||
/// Manages keyboard state and posts keyboard events to the system.
|
||||
pub struct Keyboard {
|
||||
/// Cache of character-to-keycode mappings for the current keyboard layout.
|
||||
cache: HashMap<char, CGKeyCode>,
|
||||
/// Where synthesized events are delivered.
|
||||
post_target: PostTarget,
|
||||
/// The window id and pid of the current window target, used to activate the window before
|
||||
/// keyboard events are posted. `None` when targeting the HID tap (screen/frontmost behavior).
|
||||
///
|
||||
/// Mouse events activate the target window lazily from the event location; keyboard events
|
||||
/// carry no coordinates, so activation must be triggered explicitly here instead.
|
||||
window_context: Option<(u32, libc::pid_t)>,
|
||||
/// The currently-held modifier flags, accumulated from modifier key-down/up events.
|
||||
///
|
||||
/// Synthetic modifier key events posted via `CGEventPostToPid` do not update the session's
|
||||
/// modifier state, so we track it ourselves and stamp it onto every key event. Without this,
|
||||
/// a shortcut sent as discrete events (e.g. Command-down, n-down, n-up, Command-up) arrives as
|
||||
/// a plain "n" and is treated as text rather than as Cmd+N.
|
||||
current_flags: CGEventFlags,
|
||||
}
|
||||
|
||||
impl Keyboard {
|
||||
pub fn new() -> Self {
|
||||
pub fn new(target: PostTarget) -> Self {
|
||||
Self {
|
||||
cache: keycode_cache::build_cache(),
|
||||
post_target: target,
|
||||
window_context: None,
|
||||
current_flags: CGEventFlags::empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets where subsequent synthesized key events are delivered. Called per-action so typing
|
||||
/// can be routed to a specific background process.
|
||||
pub fn set_target(&mut self, target: Target) {
|
||||
self.post_target = match target {
|
||||
Target::Screen => PostTarget::HidTap,
|
||||
Target::Window { pid, .. } => PostTarget::Pid(pid as libc::pid_t),
|
||||
};
|
||||
self.window_context = match target {
|
||||
Target::Window { window_id, pid } => Some((window_id, pid as libc::pid_t)),
|
||||
Target::Screen => None,
|
||||
};
|
||||
}
|
||||
|
||||
/// Sends a key down event for the given key.
|
||||
pub fn key_down(&self, key: &Key) -> Result<(), String> {
|
||||
post_key_down(self.resolve_keycode(key)?)
|
||||
///
|
||||
/// If the key is a modifier, its mask is folded into the held-modifier flags first so the
|
||||
/// key-down itself carries the now-active modifier; the flags are then stamped on the event.
|
||||
pub fn key_down(&mut self, key: &Key) -> Result<(), String> {
|
||||
self.ensure_window_activated();
|
||||
let keycode = self.resolve_keycode(key)?;
|
||||
if let Some(mask) = modifier_mask(keycode) {
|
||||
self.current_flags |= mask;
|
||||
}
|
||||
post_key_event(keycode, true, self.current_flags, self.post_target)
|
||||
}
|
||||
|
||||
/// Sends a key up event for the given key.
|
||||
pub fn key_up(&self, key: &Key) -> Result<(), String> {
|
||||
post_key_up(self.resolve_keycode(key)?)
|
||||
///
|
||||
/// If the key is a modifier, its mask is removed from the held-modifier flags so the key-up
|
||||
/// reflects the modifier being released; the updated flags are then stamped on the event.
|
||||
///
|
||||
/// Activation is not triggered here: `KeyUp` always follows a `KeyDown` that already
|
||||
/// activated the window, and a lone `KeyUp` with no prior `KeyDown` is a no-op regardless.
|
||||
pub fn key_up(&mut self, key: &Key) -> Result<(), String> {
|
||||
let keycode = self.resolve_keycode(key)?;
|
||||
if let Some(mask) = modifier_mask(keycode) {
|
||||
self.current_flags &= !mask;
|
||||
}
|
||||
post_key_event(keycode, false, self.current_flags, self.post_target)
|
||||
}
|
||||
|
||||
/// Simulates typing text by sending Quartz events.
|
||||
pub fn type_text(&self, text: &str) -> Result<(), String> {
|
||||
self.ensure_window_activated();
|
||||
let source = CGEventSource::new(CGEventSourceStateID::CombinedSessionState);
|
||||
|
||||
// Send one character at a time for better compatibility with various applications.
|
||||
@@ -41,12 +90,27 @@ impl Keyboard {
|
||||
//
|
||||
// TODO(vorporeal): when sending an ASCII character, send it using virtual key codes
|
||||
// for better compatibility.
|
||||
type_unicode_char(ch, source.as_deref())?;
|
||||
type_unicode_char(ch, source.as_deref(), self.post_target)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Ensures the target window is activated before keyboard events are posted.
|
||||
///
|
||||
/// Mouse events activate the target window lazily from the event location (see
|
||||
/// [`super::mouse::Mouse`]); keyboard events carry no coordinates, so activation must be
|
||||
/// triggered here instead. The call is idempotent per `(pid, window)` pair, so calling it
|
||||
/// before each keyboard event is cheap and never re-sends the activation primer.
|
||||
fn ensure_window_activated(&self) {
|
||||
let Some((window_id, pid)) = self.window_context else {
|
||||
return;
|
||||
};
|
||||
if let Some(info) = window::window_by_id(window_id) {
|
||||
activation::ensure_activated(pid, &info);
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves a Key to a CGKeyCode.
|
||||
///
|
||||
/// The key can be:
|
||||
@@ -69,26 +133,51 @@ impl Keyboard {
|
||||
}
|
||||
}
|
||||
|
||||
/// Posts a key down event for the given virtual keycode.
|
||||
fn post_key_down(keycode: CGKeyCode) -> Result<(), String> {
|
||||
let source = CGEventSource::new(CGEventSourceStateID::CombinedSessionState);
|
||||
let event = CGEvent::new_keyboard_event(source.as_deref(), keycode, true)
|
||||
.ok_or_else(|| format!("Failed to create key down event for keycode {}", keycode))?;
|
||||
CGEvent::post(CGEventTapLocation::HIDEventTap, Some(&event));
|
||||
Ok(())
|
||||
/// Maps a modifier virtual keycode to its `CGEventFlags` mask, handling both the left and right
|
||||
/// variants. Returns `None` for non-modifier keys.
|
||||
fn modifier_mask(keycode: CGKeyCode) -> Option<CGEventFlags> {
|
||||
Some(match keycode {
|
||||
// Command: left 0x37 (55), right 0x36 (54).
|
||||
54 | 55 => CGEventFlags::MaskCommand,
|
||||
// Shift: left 0x38 (56), right 0x3C (60).
|
||||
56 | 60 => CGEventFlags::MaskShift,
|
||||
// Control: left 0x3B (59), right 0x3E (62).
|
||||
59 | 62 => CGEventFlags::MaskControl,
|
||||
// Option/Alt: left 0x3A (58), right 0x3D (61).
|
||||
58 | 61 => CGEventFlags::MaskAlternate,
|
||||
// Fn: 0x3F (63).
|
||||
63 => CGEventFlags::MaskSecondaryFn,
|
||||
// Caps Lock: 0x39 (57).
|
||||
57 => CGEventFlags::MaskAlphaShift,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Posts a key up event for the given virtual keycode.
|
||||
fn post_key_up(keycode: CGKeyCode) -> Result<(), String> {
|
||||
/// Posts a key event (down or up) for the given virtual keycode, stamping the currently-held
|
||||
/// modifier flags so shortcuts route through the app's key-equivalent handling.
|
||||
fn post_key_event(
|
||||
keycode: CGKeyCode,
|
||||
is_down: bool,
|
||||
flags: CGEventFlags,
|
||||
target: PostTarget,
|
||||
) -> Result<(), String> {
|
||||
let source = CGEventSource::new(CGEventSourceStateID::CombinedSessionState);
|
||||
let event = CGEvent::new_keyboard_event(source.as_deref(), keycode, false)
|
||||
.ok_or_else(|| format!("Failed to create key up event for keycode {}", keycode))?;
|
||||
CGEvent::post(CGEventTapLocation::HIDEventTap, Some(&event));
|
||||
let event =
|
||||
CGEvent::new_keyboard_event(source.as_deref(), keycode, is_down).ok_or_else(|| {
|
||||
let direction = if is_down { "down" } else { "up" };
|
||||
format!("Failed to create key {direction} event for keycode {keycode}")
|
||||
})?;
|
||||
CGEvent::set_flags(Some(&event), flags);
|
||||
target.post(&event);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Generates a Quartz event signifying the typing of a single Unicode character.
|
||||
fn type_unicode_char(ch: char, source: Option<&CGEventSource>) -> Result<(), String> {
|
||||
fn type_unicode_char(
|
||||
ch: char,
|
||||
source: Option<&CGEventSource>,
|
||||
target: PostTarget,
|
||||
) -> Result<(), String> {
|
||||
let mut buf = [0u16; 2];
|
||||
let encoded = ch.encode_utf16(&mut buf);
|
||||
|
||||
@@ -110,13 +199,13 @@ fn type_unicode_char(ch: char, source: Option<&CGEventSource>) -> Result<(), Str
|
||||
CGEvent::set_flags(Some(&key_down), CGEventFlags::empty());
|
||||
|
||||
// Post the key down event.
|
||||
CGEvent::post(CGEventTapLocation::HIDEventTap, Some(&key_down));
|
||||
target.post(&key_down);
|
||||
|
||||
// Create and post a corresponding key up event.
|
||||
let key_up = CGEvent::new_keyboard_event(source, 0, false)
|
||||
.ok_or("Failed to create key up event for TypeText.")?;
|
||||
CGEvent::set_flags(Some(&key_up), CGEventFlags::empty());
|
||||
CGEvent::post(CGEventTapLocation::HIDEventTap, Some(&key_up));
|
||||
target.post(&key_up);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -4,28 +4,26 @@
|
||||
//! The translation depends on the current keyboard layout.
|
||||
//!
|
||||
//! The Carbon APIs used for translation (`TISCopyCurrentKeyboardInputSource`,
|
||||
//! `UCKeyTranslate`) are not thread-safe, so we build the cache on the main thread
|
||||
//! using GCD dispatch.
|
||||
//! `UCKeyTranslate`) are not thread-safe, so all cache builds are serialized through a
|
||||
//! process-wide mutex. This matters because concurrent computer-use agents can build the cache
|
||||
//! from different worker threads at the same time.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::ptr::NonNull;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use core_foundation::base::{CFType, CFTypeRef, TCFType};
|
||||
use core_foundation::data::CFData;
|
||||
use dispatch2::run_on_main;
|
||||
use objc2_core_foundation::{CFData, CFRetained, CFString, CFType};
|
||||
use objc2_core_graphics::CGKeyCode;
|
||||
|
||||
// Carbon Text Input Services types and functions.
|
||||
#[allow(non_camel_case_types)]
|
||||
type TISInputSourceRef = CFTypeRef;
|
||||
|
||||
#[link(name = "Carbon", kind = "framework")]
|
||||
unsafe extern "C" {
|
||||
fn TISCopyCurrentKeyboardInputSource() -> TISInputSourceRef;
|
||||
fn TISCopyCurrentKeyboardLayoutInputSource() -> TISInputSourceRef;
|
||||
fn TISGetInputSourceProperty(source: TISInputSourceRef, key: CFTypeRef) -> CFTypeRef;
|
||||
fn TISCopyCurrentKeyboardInputSource() -> Option<NonNull<CFType>>;
|
||||
fn TISCopyCurrentKeyboardLayoutInputSource() -> Option<NonNull<CFType>>;
|
||||
fn TISGetInputSourceProperty(source: &CFType, key: &CFString) -> *mut CFType;
|
||||
|
||||
// Property key for getting the keyboard layout data.
|
||||
static kTISPropertyUnicodeKeyLayoutData: CFTypeRef;
|
||||
static kTISPropertyUnicodeKeyLayoutData: &'static CFString;
|
||||
|
||||
fn LMGetKbdType() -> u8;
|
||||
}
|
||||
@@ -62,14 +60,20 @@ const SHIFT_MODIFIER: u32 = 1 << 1;
|
||||
|
||||
/// Builds a character-to-keycode cache for the current keyboard layout.
|
||||
///
|
||||
/// This function dispatches to the main thread to call Carbon APIs safely.
|
||||
/// The Carbon translation APIs are not thread-safe, so the build is serialized through a
|
||||
/// process-wide mutex. The work runs on the calling thread rather than being dispatched to the
|
||||
/// main thread: in the headless `agent run` CLI the main thread never services the GCD main
|
||||
/// queue, so a synchronous main-queue dispatch would deadlock.
|
||||
/// TODO(QUALITY-271): Store the modifier keys as well.
|
||||
pub fn build_cache() -> HashMap<char, CGKeyCode> {
|
||||
run_on_main(|_| build_cache_on_main_thread())
|
||||
static BUILD_LOCK: Mutex<()> = Mutex::new(());
|
||||
let _guard = BUILD_LOCK.lock().unwrap();
|
||||
build_cache_locked()
|
||||
}
|
||||
|
||||
/// Builds the cache on the main thread where Carbon APIs are safe to call.
|
||||
fn build_cache_on_main_thread() -> HashMap<char, CGKeyCode> {
|
||||
/// Builds the cache while the process-wide build lock is held, serializing access to the
|
||||
/// non-thread-safe Carbon APIs.
|
||||
fn build_cache_locked() -> HashMap<char, CGKeyCode> {
|
||||
let mut cache = HashMap::new();
|
||||
|
||||
// Get the keyboard layout data.
|
||||
@@ -79,7 +83,7 @@ fn build_cache_on_main_thread() -> HashMap<char, CGKeyCode> {
|
||||
return cache;
|
||||
};
|
||||
|
||||
let layout_ptr = layout_data.as_ptr() as *const UCKeyboardLayout;
|
||||
let layout_ptr = layout_data.byte_ptr() as *const UCKeyboardLayout;
|
||||
let keyboard_type = unsafe { LMGetKbdType() } as u32;
|
||||
|
||||
// Iterate through all possible keycodes (0-127) and build the mapping.
|
||||
@@ -103,35 +107,36 @@ fn build_cache_on_main_thread() -> HashMap<char, CGKeyCode> {
|
||||
}
|
||||
|
||||
/// Gets the keyboard layout data from the current input source.
|
||||
unsafe fn get_keyboard_layout_data() -> Option<CFData> {
|
||||
unsafe fn get_keyboard_layout_data() -> Option<CFRetained<CFData>> {
|
||||
// TISCopy* functions follow CF "Copy" semantics - caller owns the reference.
|
||||
// Wrap in CFType so they're released when dropped.
|
||||
let source = unsafe { CFType::wrap_under_create_rule(TISCopyCurrentKeyboardInputSource()) };
|
||||
let mut layout_data = unsafe {
|
||||
TISGetInputSourceProperty(source.as_CFTypeRef(), kTISPropertyUnicodeKeyLayoutData)
|
||||
// Wrap in CFRetained so they're released when dropped.
|
||||
let source = unsafe {
|
||||
CFRetained::from_raw(
|
||||
TISCopyCurrentKeyboardInputSource().expect("Attempted to create a NULL object."),
|
||||
)
|
||||
};
|
||||
let mut layout_data =
|
||||
unsafe { TISGetInputSourceProperty(&source, kTISPropertyUnicodeKeyLayoutData) };
|
||||
|
||||
// Some keyboard layouts (e.g., Japanese, Chinese) don't have layout data on the
|
||||
// regular input source. Try the keyboard layout input source instead.
|
||||
let _layout_source;
|
||||
if layout_data.is_null() {
|
||||
// Keep this alive until we're done with layout_data.
|
||||
_layout_source =
|
||||
unsafe { CFType::wrap_under_create_rule(TISCopyCurrentKeyboardLayoutInputSource()) };
|
||||
layout_data = unsafe {
|
||||
TISGetInputSourceProperty(
|
||||
_layout_source.as_CFTypeRef(),
|
||||
kTISPropertyUnicodeKeyLayoutData,
|
||||
_layout_source = unsafe {
|
||||
CFRetained::from_raw(
|
||||
TISCopyCurrentKeyboardLayoutInputSource()
|
||||
.expect("Attempted to create a NULL object."),
|
||||
)
|
||||
};
|
||||
layout_data =
|
||||
unsafe { TISGetInputSourceProperty(&_layout_source, kTISPropertyUnicodeKeyLayoutData) };
|
||||
}
|
||||
|
||||
if layout_data.is_null() {
|
||||
return None;
|
||||
}
|
||||
let layout_data = NonNull::new(layout_data)?;
|
||||
|
||||
// The returned CFData is not retained, so we need to retain it.
|
||||
Some(unsafe { CFData::wrap_under_get_rule(layout_data as _) })
|
||||
Some(unsafe { CFRetained::retain(layout_data.cast::<CFData>()) })
|
||||
}
|
||||
|
||||
/// Translates a keycode to a character using UCKeyTranslate.
|
||||
|
||||
@@ -1,18 +1,112 @@
|
||||
mod activation;
|
||||
mod keyboard;
|
||||
mod keycode_cache;
|
||||
mod mouse;
|
||||
mod post;
|
||||
mod screenshot;
|
||||
mod util;
|
||||
mod window;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use galaxyui::r#async::Timer;
|
||||
use pathfinder_geometry::vector::Vector2I;
|
||||
use post::PostTarget;
|
||||
use util::{display_scale_factor_for_window, main_display_scale_factor};
|
||||
use galaxyui_core::r#async::Timer;
|
||||
|
||||
use crate::{Action, ActionResult, Options};
|
||||
use crate::{Action, ActionResult, Options, Target, TargetedAction};
|
||||
|
||||
pub fn is_supported_on_current_platform() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Reports whether background, per-window control is available. On macOS the background input
|
||||
/// stack (focus-without-raise + window-targeted posting) is present, so this is always true.
|
||||
pub fn background_supported() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Enumerates the on-screen windows as crate-level [`crate::WindowInfo`] records.
|
||||
pub fn enumerate_windows() -> Vec<crate::WindowInfo> {
|
||||
window::enumerate_windows()
|
||||
}
|
||||
|
||||
/// Maps a computer-use [`Target`] to the lower-level [`PostTarget`] used for event delivery.
|
||||
fn post_target_for(target: Target) -> PostTarget {
|
||||
match target {
|
||||
Target::Screen => PostTarget::HidTap,
|
||||
Target::Window { pid, .. } => PostTarget::Pid(pid as libc::pid_t),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a copy of `action` with its coordinates remapped for the given target.
|
||||
///
|
||||
/// For a `Window` target the incoming coordinates are window-local pixels in the captured window
|
||||
/// screenshot's space; they are translated to global points using the backing scale of the
|
||||
/// display containing the window, then encoded for the existing screen-pixel mouse pipeline.
|
||||
/// `Screen` targets retain the legacy global-pixel behavior.
|
||||
fn remap_action_for_target(action: &Action, target: Target) -> Result<Action, String> {
|
||||
let Target::Window { window_id, .. } = target else {
|
||||
return Ok(action.clone());
|
||||
};
|
||||
let remap = |p: Vector2I| -> Result<Vector2I, String> {
|
||||
let info = window::window_by_id(window_id)
|
||||
.ok_or_else(|| format!("Failed to resolve target window {window_id}."))?;
|
||||
let pixels_per_point =
|
||||
display_scale_factor_for_window(info.x, info.y, info.width, info.height).ok_or_else(
|
||||
|| {
|
||||
format!(
|
||||
"Target window {window_id} is not fully contained on one display with a known scale factor."
|
||||
)
|
||||
},
|
||||
)?;
|
||||
let screen_scale = main_display_scale_factor();
|
||||
let global_point_x = info.x + f64::from(p.x()) / pixels_per_point;
|
||||
let global_point_y = info.y + f64::from(p.y()) / pixels_per_point;
|
||||
let global = Vector2I::new(
|
||||
(global_point_x * screen_scale).round() as i32,
|
||||
(global_point_y * screen_scale).round() as i32,
|
||||
);
|
||||
Ok(global)
|
||||
};
|
||||
Ok(match action {
|
||||
Action::MouseMove { to } => Action::MouseMove { to: remap(*to)? },
|
||||
Action::MouseDown { button, at } => Action::MouseDown {
|
||||
button: button.clone(),
|
||||
at: remap(*at)?,
|
||||
},
|
||||
Action::MouseWheel {
|
||||
at,
|
||||
direction,
|
||||
distance,
|
||||
} => Action::MouseWheel {
|
||||
at: remap(*at)?,
|
||||
direction: *direction,
|
||||
distance: *distance,
|
||||
},
|
||||
other => other.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Experimental: lists on-screen windows (number, owner PID/name, layer, bounds) for
|
||||
/// diagnosing PID/window targeting.
|
||||
pub fn list_windows() -> String {
|
||||
let mut out = String::from("window# owner_pid layer bounds(x,y,w,h) owner_name\n");
|
||||
for w in window::list_windows() {
|
||||
out.push_str(&format!(
|
||||
"{:<7} {:<9} {:<5} ({:.0},{:.0},{:.0},{:.0}) {}\n",
|
||||
w.number,
|
||||
w.owner_pid,
|
||||
w.layer,
|
||||
w.x,
|
||||
w.y,
|
||||
w.width,
|
||||
w.height,
|
||||
w.owner_name.as_deref().unwrap_or("<unknown>"),
|
||||
));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub struct Actor {
|
||||
keyboard: keyboard::Keyboard,
|
||||
mouse: mouse::Mouse,
|
||||
@@ -20,9 +114,11 @@ pub struct Actor {
|
||||
|
||||
impl Actor {
|
||||
pub fn new() -> Self {
|
||||
// The post target now defaults to the HID event tap (legacy screen/frontmost behavior)
|
||||
// and is overridden per-action when an action targets a specific window.
|
||||
Self {
|
||||
keyboard: keyboard::Keyboard::new(),
|
||||
mouse: mouse::Mouse::new(),
|
||||
keyboard: keyboard::Keyboard::new(PostTarget::HidTap),
|
||||
mouse: mouse::Mouse::new(PostTarget::HidTap),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -35,11 +131,42 @@ impl super::Actor for Actor {
|
||||
|
||||
async fn perform_actions(
|
||||
&mut self,
|
||||
actions: &[Action],
|
||||
actions: &[TargetedAction],
|
||||
options: Options,
|
||||
) -> Result<ActionResult, String> {
|
||||
for action in actions {
|
||||
match action {
|
||||
// When background computer use is disabled, force the legacy full-screen path: ignore any
|
||||
// window target, deliver events through the HID tap, and treat coordinates as global
|
||||
// pixels. This keeps behavior byte-identical to the pre-existing implementation.
|
||||
let background = options.background_enabled;
|
||||
for targeted in actions {
|
||||
let target = if background {
|
||||
targeted.target
|
||||
} else {
|
||||
Target::Screen
|
||||
};
|
||||
|
||||
// A window target must carry a concrete window id. `0` is the "unknown" sentinel
|
||||
// produced by the CLI default and by unparseable wire ids; reject it here rather than
|
||||
// failing later in window resolution with an opaque message, since a well-behaved
|
||||
// caller always echoes a real window id selected from the enumerated window list.
|
||||
if let Target::Window { window_id: 0, .. } = target {
|
||||
return Err(
|
||||
"A window target requires a non-zero window id. Select a window from the \
|
||||
enumerated window list."
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
// Route this action to its target: the HID tap for screen actions, or directly to the
|
||||
// owning process for a window action (without raising it or moving the cursor).
|
||||
let post_target = post_target_for(target);
|
||||
self.mouse.set_target(post_target);
|
||||
self.keyboard.set_target(target);
|
||||
|
||||
// For a window target, translate window-local coordinates through the containing
|
||||
// display's point mapping.
|
||||
let action = remap_action_for_target(&targeted.action, target)?;
|
||||
match &action {
|
||||
Action::Wait(duration) => {
|
||||
Timer::after(*duration).await;
|
||||
}
|
||||
@@ -69,15 +196,30 @@ impl super::Actor for Actor {
|
||||
}
|
||||
}
|
||||
|
||||
let screenshot = if let Some(params) = options.screenshot_params {
|
||||
Some(screenshot::take(params)?)
|
||||
} else {
|
||||
None
|
||||
let (screenshot, captured_window) = match options.screenshot_params {
|
||||
Some(mut params) => {
|
||||
// With background computer use disabled, never capture a specific window: force the
|
||||
// legacy main-display capture, which returns no captured-window metadata.
|
||||
if !background {
|
||||
params.target = Target::Screen;
|
||||
}
|
||||
let (screenshot, captured) = screenshot::take(params)?;
|
||||
(Some(screenshot), captured)
|
||||
}
|
||||
None => (None, None),
|
||||
};
|
||||
|
||||
Ok(ActionResult {
|
||||
screenshot,
|
||||
cursor_position: Some(self.mouse.current_position()?),
|
||||
// Refresh the window list so the caller has up-to-date targets to choose from. When
|
||||
// background computer use is disabled, omit it so the result matches the legacy shape.
|
||||
windows: if background {
|
||||
window::enumerate_windows()
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
captured_window,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,17 +1,19 @@
|
||||
use instant::Instant;
|
||||
use std::time::Duration;
|
||||
|
||||
use galaxyui::r#async::Timer;
|
||||
use instant::Instant;
|
||||
use objc2::rc::Retained;
|
||||
use objc2_core_foundation::CGPoint;
|
||||
use objc2_core_graphics::{
|
||||
CGEvent, CGEventSource, CGEventSourceStateID, CGEventTapLocation, CGEventType, CGMouseButton,
|
||||
CGEvent, CGEventField, CGEventSource, CGEventSourceStateID, CGEventType, CGMouseButton,
|
||||
CGScrollEventUnit,
|
||||
};
|
||||
use pathfinder_geometry::vector::Vector2I;
|
||||
use galaxyui_core::r#async::Timer;
|
||||
|
||||
use crate::{MouseButton, ScrollDirection, ScrollDistance};
|
||||
|
||||
use super::post::PostTarget;
|
||||
use super::util::main_display_scale_factor;
|
||||
use super::window;
|
||||
use crate::{MouseButton, ScrollDirection, ScrollDistance};
|
||||
|
||||
const POSITION_POLL_INTERVAL: Duration = Duration::from_micros(500);
|
||||
const POSITION_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
@@ -38,15 +40,31 @@ pub fn from_cgpoint(point: CGPoint) -> Vector2I {
|
||||
/// Manages mouse state and posts mouse events to the system.
|
||||
pub struct Mouse {
|
||||
held_buttons: HeldButtons,
|
||||
/// Where synthesized events are delivered.
|
||||
target: PostTarget,
|
||||
/// The most recently requested cursor position, in CGEvent point coordinates.
|
||||
///
|
||||
/// When delivering events directly to a PID, `CGEventPostToPid` does not move the real
|
||||
/// cursor, so the global cursor position cannot be used to locate clicks. We track the
|
||||
/// intended position here and use it as the location for button and move events.
|
||||
virtual_position: CGPoint,
|
||||
}
|
||||
|
||||
impl Mouse {
|
||||
pub fn new() -> Self {
|
||||
pub fn new(target: PostTarget) -> Self {
|
||||
Self {
|
||||
held_buttons: HeldButtons::default(),
|
||||
target,
|
||||
virtual_position: CGPoint { x: 0.0, y: 0.0 },
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets where subsequent synthesized events are delivered. Called per-action so a batch can
|
||||
/// drive the HID tap for some actions and a specific process for others.
|
||||
pub fn set_target(&mut self, target: PostTarget) {
|
||||
self.target = target;
|
||||
}
|
||||
|
||||
pub async fn move_to(&mut self, target: Vector2I) -> Result<(), String> {
|
||||
let (event_type, cg_button) = if let Some(held) = self.held_buttons.primary_down() {
|
||||
(mouse_dragged_event_type(&held), (&held).into())
|
||||
@@ -54,23 +72,44 @@ impl Mouse {
|
||||
(CGEventType::MouseMoved, CGMouseButton::Left)
|
||||
};
|
||||
|
||||
self.post_event(event_type, to_cgpoint(target), cg_button)?;
|
||||
self.wait_for_position(target).await
|
||||
let point = to_cgpoint(target);
|
||||
self.virtual_position = point;
|
||||
// A drag is part of an active click, so it carries the click state; a plain move does
|
||||
// not.
|
||||
let click_state = if self.held_buttons.primary_down().is_some() {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
self.post_event(event_type, point, cg_button, click_state)?;
|
||||
|
||||
// `CGEventPostToPid` does not move the real cursor, so polling the global cursor
|
||||
// position would always time out. Only wait when injecting through the HID tap.
|
||||
if self.target.is_pid_targeted() {
|
||||
Ok(())
|
||||
} else {
|
||||
self.wait_for_position(target).await
|
||||
}
|
||||
}
|
||||
|
||||
pub fn button_down(&mut self, button: &MouseButton) -> Result<(), String> {
|
||||
let point = self.current_position_cgpoint()?;
|
||||
let point = self.event_location()?;
|
||||
self.held_buttons.set_down(button, true);
|
||||
self.post_event(mouse_down_event_type(button), point, button.into())
|
||||
self.post_event(mouse_down_event_type(button), point, button.into(), 1)
|
||||
}
|
||||
|
||||
pub fn button_up(&mut self, button: &MouseButton) -> Result<(), String> {
|
||||
let point = self.current_position_cgpoint()?;
|
||||
let point = self.event_location()?;
|
||||
self.held_buttons.set_down(button, false);
|
||||
self.post_event(mouse_up_event_type(button), point, button.into())
|
||||
self.post_event(mouse_up_event_type(button), point, button.into(), 1)
|
||||
}
|
||||
|
||||
pub fn current_position(&mut self) -> Result<Vector2I, String> {
|
||||
// In PID-targeted mode the real cursor is never moved, so report the tracked virtual
|
||||
// position instead of the (unrelated) global cursor location.
|
||||
if self.target.is_pid_targeted() {
|
||||
return Ok(from_cgpoint(self.virtual_position));
|
||||
}
|
||||
let cg_point = self.current_position_cgpoint()?;
|
||||
Ok(from_cgpoint(cg_point))
|
||||
}
|
||||
@@ -118,13 +157,25 @@ impl Mouse {
|
||||
)
|
||||
})?;
|
||||
|
||||
CGEvent::post(CGEventTapLocation::HIDEventTap, Some(&event));
|
||||
self.target.post(&event);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// Private implementation details.
|
||||
impl Mouse {
|
||||
/// Returns the location to use for a button event.
|
||||
///
|
||||
/// In HID mode this is the real cursor position; in PID-targeted mode the real cursor is
|
||||
/// never moved, so the tracked virtual position is used instead.
|
||||
fn event_location(&mut self) -> Result<CGPoint, String> {
|
||||
if self.target.is_pid_targeted() {
|
||||
Ok(self.virtual_position)
|
||||
} else {
|
||||
self.current_position_cgpoint()
|
||||
}
|
||||
}
|
||||
|
||||
/// Waits for the mouse to reach the target position, polling until it arrives
|
||||
/// or times out.
|
||||
async fn wait_for_position(&mut self, target: Vector2I) -> Result<(), String> {
|
||||
@@ -162,28 +213,204 @@ impl Mouse {
|
||||
Ok(pos)
|
||||
}
|
||||
|
||||
/// Posts a mouse event.
|
||||
///
|
||||
/// `click_state` is the click count (1 for a single click, 2 for a double click, etc.) and
|
||||
/// should be 0 for non-button events like plain moves. Many applications ignore synthetic
|
||||
/// clicks that lack a non-zero click state, so it is set for button-down, button-up, and
|
||||
/// drag events.
|
||||
fn post_event(
|
||||
&mut self,
|
||||
event_type: CGEventType,
|
||||
point: CGPoint,
|
||||
button: CGMouseButton,
|
||||
click_state: i64,
|
||||
) -> Result<(), String> {
|
||||
let source = CGEventSource::new(CGEventSourceStateID::CombinedSessionState);
|
||||
// For a PID target with an owned window under the point, deliver a window-targeted event
|
||||
// directly to the owning process via `CGEventPostToPid`, without raising the window or
|
||||
// moving the cursor. Falls back to a plain CGEvent via the configured target when there
|
||||
// is no PID target or no owned window under the point.
|
||||
if let Some(pid) = self.target.pid()
|
||||
&& let Some(info) = window::window_at(pid, point.x, point.y)
|
||||
{
|
||||
let is_down = matches!(
|
||||
event_type,
|
||||
CGEventType::LeftMouseDown
|
||||
| CGEventType::RightMouseDown
|
||||
| CGEventType::OtherMouseDown
|
||||
);
|
||||
let is_move = matches!(event_type, CGEventType::MouseMoved);
|
||||
// Activate the target window in the background on a hover or a press (not on drags),
|
||||
// so the pre-click mouse-moved and the press both land on an active window. This is
|
||||
// idempotent per window and does not raise the window or steal the user's frontmost
|
||||
// focus.
|
||||
if is_down || is_move {
|
||||
super::activation::ensure_activated(pid, &info);
|
||||
}
|
||||
post_window_mouse_event(
|
||||
pid,
|
||||
&info,
|
||||
event_type,
|
||||
button,
|
||||
point,
|
||||
click_state,
|
||||
event_pressure(event_type),
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let event = CGEvent::new_mouse_event(source.as_deref(), event_type, point, button)
|
||||
.ok_or_else(|| {
|
||||
format!(
|
||||
"Failed to create mouse event (type={:?}, position=({}, {}), button={:?}). \
|
||||
The cause is unknown.",
|
||||
event_type, point.x, point.y, button
|
||||
)
|
||||
})?;
|
||||
|
||||
CGEvent::post(CGEventTapLocation::HIDEventTap, Some(&event));
|
||||
// Fallback: no PID target, or no owned window under the point. Post a plain event via the
|
||||
// configured target (HID tap for screen targets, CGEventPostToPid for a PID target).
|
||||
let event = build_plain_mouse_event(event_type, point, button, click_state)?;
|
||||
self.target.post(&event);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds and posts a mouse event targeted at `info` (the window under `global_point`) directly
|
||||
/// to the owning process via `CGEventPostToPid`.
|
||||
///
|
||||
/// On the `postToPid` path the WindowServer does not run its normal hit-testing, so the event
|
||||
/// must carry the fields AppKit reads to route and interpret it: the click state, the pressure,
|
||||
/// the target pid, the window-under-pointer number, the private window-addressing fields, and
|
||||
/// the window-local location.
|
||||
pub(super) fn post_window_mouse_event(
|
||||
pid: libc::pid_t,
|
||||
info: &window::WindowInfo,
|
||||
event_type: CGEventType,
|
||||
button: CGMouseButton,
|
||||
global_point: CGPoint,
|
||||
click_state: i64,
|
||||
pressure: f64,
|
||||
) {
|
||||
let source = CGEventSource::new(CGEventSourceStateID::CombinedSessionState);
|
||||
let Some(event) = CGEvent::new_mouse_event(source.as_deref(), event_type, global_point, button)
|
||||
else {
|
||||
log::warn!("Failed to create window-targeted mouse event (type={event_type:?}).");
|
||||
return;
|
||||
};
|
||||
|
||||
// `CGEventSetWindowLocation` wants window-local coordinates with a top-left origin (the
|
||||
// global screen point translated by the window origin).
|
||||
let window_local = CGPoint {
|
||||
x: global_point.x - info.x,
|
||||
y: global_point.y - info.y,
|
||||
};
|
||||
|
||||
if click_state > 0 {
|
||||
CGEvent::set_integer_value_field(
|
||||
Some(&event),
|
||||
CGEventField::MouseEventClickState,
|
||||
click_state,
|
||||
);
|
||||
}
|
||||
CGEvent::set_double_value_field(Some(&event), CGEventField::MouseEventPressure, pressure);
|
||||
CGEvent::set_integer_value_field(
|
||||
Some(&event),
|
||||
CGEventField::EventTargetUnixProcessID,
|
||||
pid as i64,
|
||||
);
|
||||
CGEvent::set_integer_value_field(
|
||||
Some(&event),
|
||||
CGEventField::MouseEventWindowUnderMousePointer,
|
||||
info.number,
|
||||
);
|
||||
CGEvent::set_integer_value_field(
|
||||
Some(&event),
|
||||
CGEventField::MouseEventWindowUnderMousePointerThatCanHandleThisEvent,
|
||||
info.number,
|
||||
);
|
||||
set_window_addressing_fields(&event, info.number);
|
||||
set_window_location(&event, window_local);
|
||||
|
||||
CGEvent::post_to_pid(pid, Some(&event));
|
||||
}
|
||||
|
||||
/// Stamps the private window-addressing fields the WindowServer uses to route an event to a
|
||||
/// specific window on the `postToPid` path: field 51 carries the target window number, and field
|
||||
/// 58 flags that the window number is valid.
|
||||
pub(super) fn set_window_addressing_fields(event: &CGEvent, window_number: i64) {
|
||||
CGEvent::set_integer_value_field(Some(event), CGEventField(51), window_number);
|
||||
CGEvent::set_integer_value_field(Some(event), CGEventField(58), 1);
|
||||
}
|
||||
|
||||
/// Returns the pressure value for a mouse event: full pressure while a button is held (a press
|
||||
/// or drag) and zero otherwise (a move or release), mirroring what a real device reports.
|
||||
fn event_pressure(event_type: CGEventType) -> f64 {
|
||||
match event_type {
|
||||
CGEventType::LeftMouseDown
|
||||
| CGEventType::RightMouseDown
|
||||
| CGEventType::OtherMouseDown
|
||||
| CGEventType::LeftMouseDragged
|
||||
| CGEventType::RightMouseDragged
|
||||
| CGEventType::OtherMouseDragged => 1.0,
|
||||
_ => 0.0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds a plain CGEvent mouse event at the global `point`, stamping the click state. Used by the
|
||||
/// non-window fallback delivery path.
|
||||
fn build_plain_mouse_event(
|
||||
event_type: CGEventType,
|
||||
point: CGPoint,
|
||||
button: CGMouseButton,
|
||||
click_state: i64,
|
||||
) -> Result<Retained<CGEvent>, String> {
|
||||
let source = CGEventSource::new(CGEventSourceStateID::CombinedSessionState);
|
||||
let event = CGEvent::new_mouse_event(source.as_deref(), event_type, point, button).ok_or_else(
|
||||
|| {
|
||||
format!(
|
||||
"Failed to create mouse event (type={event_type:?}, position=({}, {}), \
|
||||
button={button:?}). The cause is unknown.",
|
||||
point.x, point.y
|
||||
)
|
||||
},
|
||||
)?;
|
||||
if click_state > 0 {
|
||||
CGEvent::set_integer_value_field(
|
||||
Some(&event),
|
||||
CGEventField::MouseEventClickState,
|
||||
click_state,
|
||||
);
|
||||
}
|
||||
Ok(event.into())
|
||||
}
|
||||
|
||||
/// Sets the window-local location on a `CGEvent` via the private `CGEventSetWindowLocation`.
|
||||
///
|
||||
/// There is no public setter for this field, which AppKit reads on the `postToPid` delivery
|
||||
/// path. The symbol is resolved once at runtime. `location` is window-local, top-left origin.
|
||||
fn set_window_location(event: &CGEvent, location: CGPoint) {
|
||||
use std::ffi::c_void;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
type SetWindowLocationFn = unsafe extern "C" fn(*mut c_void, CGPoint);
|
||||
// The macOS value of `RTLD_DEFAULT`, used to search all loaded images for the symbol.
|
||||
const RTLD_DEFAULT: *mut c_void = -2isize as *mut c_void;
|
||||
|
||||
static RESOLVED: OnceLock<Option<SetWindowLocationFn>> = OnceLock::new();
|
||||
let resolved = RESOLVED.get_or_init(|| unsafe {
|
||||
let sym = libc::dlsym(RTLD_DEFAULT, c"CGEventSetWindowLocation".as_ptr());
|
||||
if sym.is_null() {
|
||||
None
|
||||
} else {
|
||||
Some(std::mem::transmute::<*mut c_void, SetWindowLocationFn>(sym))
|
||||
}
|
||||
});
|
||||
|
||||
match resolved {
|
||||
Some(set_window_location) => {
|
||||
let event_ptr = event as *const CGEvent as *mut c_void;
|
||||
unsafe { set_window_location(event_ptr, location) };
|
||||
}
|
||||
None => {
|
||||
log::warn!(
|
||||
"CGEventSetWindowLocation could not be resolved; background clicks may not land."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// Button state tracking
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
use objc2_core_graphics::{CGEvent, CGEventTapLocation};
|
||||
|
||||
/// Describes where synthesized Quartz events are delivered.
|
||||
///
|
||||
/// This selects between the legacy whole-screen delivery and background, per-window delivery.
|
||||
/// `HidTap` reproduces the historical behavior of injecting events as if they came from real
|
||||
/// hardware, while `Pid` delivers directly to a process for background, non-interfering control.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum PostTarget {
|
||||
/// Inject at the HID event tap, exactly as real hardware would. This moves the real
|
||||
/// cursor and the event is routed to whichever application is frontmost.
|
||||
HidTap,
|
||||
/// Deliver the event directly to a specific process by PID via `CGEventPostToPid`. This
|
||||
/// does not move the global cursor and does not require the target to be frontmost, at
|
||||
/// the cost of reduced reliability (especially for mouse events).
|
||||
Pid(libc::pid_t),
|
||||
}
|
||||
|
||||
impl PostTarget {
|
||||
/// Returns true when events are delivered directly to a process rather than the HID tap.
|
||||
pub fn is_pid_targeted(self) -> bool {
|
||||
matches!(self, PostTarget::Pid(_))
|
||||
}
|
||||
|
||||
/// Returns the target PID, if events are delivered directly to a process.
|
||||
pub fn pid(self) -> Option<libc::pid_t> {
|
||||
match self {
|
||||
PostTarget::Pid(pid) => Some(pid),
|
||||
PostTarget::HidTap => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Posts the given event according to this target.
|
||||
pub fn post(self, event: &CGEvent) {
|
||||
match self {
|
||||
PostTarget::HidTap => CGEvent::post(CGEventTapLocation::HIDEventTap, Some(event)),
|
||||
PostTarget::Pid(pid) => CGEvent::post_to_pid(pid, Some(event)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,25 @@
|
||||
use command::blocking::Command;
|
||||
use image::GenericImageView;
|
||||
|
||||
use super::util::main_display_scale_factor;
|
||||
use crate::ScreenshotParams;
|
||||
use crate::{CapturedWindow, ScreenshotParams, Target};
|
||||
|
||||
/// Captures a screenshot of the main display using the built-in macOS
|
||||
/// `screencapture` CLI.
|
||||
pub fn take(params: ScreenshotParams) -> Result<crate::Screenshot, String> {
|
||||
/// Captures a screenshot according to `params`, using the built-in macOS `screencapture` CLI.
|
||||
///
|
||||
/// When the params target a window, that specific window is captured (without raising it) and the
|
||||
/// returned [`CapturedWindow`] describes the image so window-local coordinates can be mapped onto
|
||||
/// it. Otherwise the main display is captured and the second tuple element is `None`.
|
||||
pub fn take(
|
||||
params: ScreenshotParams,
|
||||
) -> Result<(crate::Screenshot, Option<CapturedWindow>), String> {
|
||||
match params.target {
|
||||
Target::Window { window_id, .. } => take_window(window_id, params),
|
||||
Target::Screen => Ok((take_screen(params)?, None)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Captures the main display, optionally restricted to a region (legacy behavior).
|
||||
fn take_screen(params: ScreenshotParams) -> Result<crate::Screenshot, String> {
|
||||
let output_dir = tempfile::tempdir()
|
||||
.map_err(|e| format!("Failed to create temporary directory for screenshot: {e}"))?;
|
||||
let output_path = output_dir.path().join("screenshot.png");
|
||||
@@ -34,15 +48,85 @@ pub fn take(params: ScreenshotParams) -> Result<crate::Screenshot, String> {
|
||||
.output()
|
||||
.map_err(|e| format!("Failed to run screencapture: {e}"))?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let detail = if stderr.trim().is_empty() {
|
||||
format!("exit code {}", output.status)
|
||||
} else {
|
||||
format!("exit code {}: {}", output.status, stderr.trim())
|
||||
};
|
||||
return Err(format!("screencapture failed with {detail}"));
|
||||
}
|
||||
check_status(&output)?;
|
||||
|
||||
crate::screenshot_utils::load_and_process_screenshot(&output_path, params)
|
||||
}
|
||||
|
||||
/// Captures a single window by its `CGWindowID` without raising it, returning the processed image
|
||||
/// plus metadata describing the captured pixels.
|
||||
fn take_window(
|
||||
window_id: u32,
|
||||
params: ScreenshotParams,
|
||||
) -> Result<(crate::Screenshot, Option<CapturedWindow>), String> {
|
||||
let output_dir = tempfile::tempdir()
|
||||
.map_err(|e| format!("Failed to create temporary directory for screenshot: {e}"))?;
|
||||
let output_path = output_dir.path().join("window.png");
|
||||
|
||||
let output = Command::new("/usr/sbin/screencapture")
|
||||
.args([
|
||||
"-x", // Do not play sounds.
|
||||
"-tpng", // Capture to PNG format.
|
||||
"-o", // Omit the window's drop shadow.
|
||||
])
|
||||
// -l <windowid> captures only the window with the given id, even when it is not frontmost.
|
||||
.arg("-l")
|
||||
.arg(window_id.to_string())
|
||||
.arg(&output_path)
|
||||
.output()
|
||||
.map_err(|e| format!("Failed to run screencapture: {e}"))?;
|
||||
|
||||
check_status(&output)?;
|
||||
|
||||
let image = image::ImageReader::open(&output_path)
|
||||
.map_err(|e| format!("Failed to open screenshot file: {e}"))?
|
||||
.decode()
|
||||
.map_err(|e| format!("Failed to decode screenshot: {e}"))?;
|
||||
let (full_width_px, full_height_px) = image.dimensions();
|
||||
let image = if let Some(region) = params.region {
|
||||
region.validate()?;
|
||||
if region.bottom_right.x() as u32 > full_width_px
|
||||
|| region.bottom_right.y() as u32 > full_height_px
|
||||
{
|
||||
return Err(format!(
|
||||
"Screenshot region ({}, {}) to ({}, {}) is outside window {window_id} dimensions {full_width_px}x{full_height_px}.",
|
||||
region.top_left.x(),
|
||||
region.top_left.y(),
|
||||
region.bottom_right.x(),
|
||||
region.bottom_right.y(),
|
||||
));
|
||||
}
|
||||
image.crop_imm(
|
||||
region.top_left.x() as u32,
|
||||
region.top_left.y() as u32,
|
||||
(region.bottom_right.x() - region.top_left.x()) as u32,
|
||||
(region.bottom_right.y() - region.top_left.y()) as u32,
|
||||
)
|
||||
} else {
|
||||
image
|
||||
};
|
||||
let screenshot = crate::screenshot_utils::process_screenshot(image, params)?;
|
||||
|
||||
// The captured metadata refers to the native (pre-downscale) capture, so window-local pixel
|
||||
// coordinates sent by the agent map directly onto the captured window image.
|
||||
let captured = CapturedWindow {
|
||||
window_id,
|
||||
width_px: screenshot.original_width as i32,
|
||||
height_px: screenshot.original_height as i32,
|
||||
};
|
||||
Ok((screenshot, Some(captured)))
|
||||
}
|
||||
|
||||
/// Returns an error describing a failed `screencapture` invocation.
|
||||
fn check_status(output: &std::process::Output) -> Result<(), String> {
|
||||
if output.status.success() {
|
||||
return Ok(());
|
||||
}
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let detail = if stderr.trim().is_empty() {
|
||||
format!("exit code {}", output.status)
|
||||
} else {
|
||||
format!("exit code {}: {}", output.status, stderr.trim())
|
||||
};
|
||||
Err(format!("screencapture failed with {detail}"))
|
||||
}
|
||||
|
||||
@@ -2,13 +2,67 @@
|
||||
///
|
||||
/// This is used to convert between pixel coordinates (as returned by screenshot tools)
|
||||
/// and point coordinates (as used by CGEvent and screencapture).
|
||||
///
|
||||
/// This intentionally avoids `NSScreen::mainScreen`, which must run on the main thread and is
|
||||
/// reached via a synchronous dispatch to the main queue. In the headless `agent run` CLI the
|
||||
/// main thread never services that queue, so such a dispatch deadlocks. The backing scale factor
|
||||
/// is instead derived purely from thread-safe Core Graphics calls as the ratio of the main
|
||||
/// display's current mode pixel width to its point width.
|
||||
pub fn main_display_scale_factor() -> f64 {
|
||||
use dispatch2::run_on_main;
|
||||
use objc2_app_kit::NSScreen;
|
||||
use objc2_core_graphics::{CGDisplayCopyDisplayMode, CGDisplayMode, CGMainDisplayID};
|
||||
|
||||
run_on_main(|mtm| {
|
||||
NSScreen::mainScreen(mtm)
|
||||
.map(|screen| screen.backingScaleFactor())
|
||||
.unwrap_or(1.0)
|
||||
})
|
||||
let Some(mode) = CGDisplayCopyDisplayMode(CGMainDisplayID()) else {
|
||||
return 1.0;
|
||||
};
|
||||
let width_points = CGDisplayMode::width(Some(&mode));
|
||||
if width_points == 0 {
|
||||
return 1.0;
|
||||
}
|
||||
CGDisplayMode::pixel_width(Some(&mode)) as f64 / width_points as f64
|
||||
}
|
||||
|
||||
/// Returns the backing scale factor of the display that fully contains a window.
|
||||
///
|
||||
/// A window spanning displays with different backing scale factors does not have one valid
|
||||
/// target-wide pixel-to-point conversion and is therefore intentionally unsupported.
|
||||
pub fn display_scale_factor_for_window(x: f64, y: f64, width: f64, height: f64) -> Option<f64> {
|
||||
use objc2_core_graphics::{
|
||||
CGDirectDisplayID, CGDisplayBounds, CGDisplayCopyDisplayMode, CGDisplayMode, CGError,
|
||||
CGGetActiveDisplayList,
|
||||
};
|
||||
|
||||
const MAX_ACTIVE_DISPLAYS: u32 = 32;
|
||||
let mut displays: [CGDirectDisplayID; MAX_ACTIVE_DISPLAYS as usize] =
|
||||
[0; MAX_ACTIVE_DISPLAYS as usize];
|
||||
let mut display_count = 0;
|
||||
if unsafe {
|
||||
CGGetActiveDisplayList(
|
||||
MAX_ACTIVE_DISPLAYS,
|
||||
displays.as_mut_ptr(),
|
||||
&mut display_count,
|
||||
)
|
||||
} != CGError::Success
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
for id in displays.into_iter().take(display_count as usize) {
|
||||
let bounds = CGDisplayBounds(id);
|
||||
let contains_window = x >= bounds.origin.x
|
||||
&& y >= bounds.origin.y
|
||||
&& x + width <= bounds.origin.x + bounds.size.width
|
||||
&& y + height <= bounds.origin.y + bounds.size.height;
|
||||
if !contains_window {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mode = CGDisplayCopyDisplayMode(id)?;
|
||||
let width_points = CGDisplayMode::width(Some(&mode));
|
||||
if width_points == 0 {
|
||||
return None;
|
||||
}
|
||||
return Some(CGDisplayMode::pixel_width(Some(&mode)) as f64 / width_points as f64);
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
//! Experimental helper for locating the on-screen window under a point for a given process.
|
||||
//!
|
||||
//! PID-targeted synthetic mouse events (`CGEventPostToPid`) bypass the WindowServer's
|
||||
//! hit-testing, so they arrive at the target process without an associated window. To let
|
||||
//! AppKit route them, we reconstruct the target window (its number and bounds) so the event
|
||||
//! can be built as a window-targeted `NSEvent` with window-local coordinates.
|
||||
|
||||
use objc2_core_foundation::{CFArray, CFDictionary, CFNumber, CFRetained, CFString, CFType};
|
||||
use objc2_core_graphics::{
|
||||
CGWindowListCopyWindowInfo, CGWindowListOption, kCGNullWindowID, kCGWindowBounds,
|
||||
kCGWindowLayer, kCGWindowName, kCGWindowNumber, kCGWindowOwnerName, kCGWindowOwnerPID,
|
||||
};
|
||||
type WindowDictionary = CFDictionary<CFString, CFType>;
|
||||
type BoundsDictionary = CFDictionary<CFString, CFNumber>;
|
||||
|
||||
/// Describes an on-screen window: its window number and bounds in global screen points
|
||||
/// (top-left origin), matching `kCGWindowBounds` and `CGEvent` location coordinates.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct WindowInfo {
|
||||
pub number: i64,
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
pub width: f64,
|
||||
pub height: f64,
|
||||
}
|
||||
|
||||
impl WindowInfo {
|
||||
fn contains(&self, x: f64, y: f64) -> bool {
|
||||
x >= self.x && y >= self.y && x < self.x + self.width && y < self.y + self.height
|
||||
}
|
||||
}
|
||||
|
||||
/// Finds the on-screen window owned by `pid` that contains the given point.
|
||||
///
|
||||
/// The point is in global screen points with a top-left origin. When no owned window's bounds
|
||||
/// contain the point, this falls back to the frontmost normal window owned by `pid`.
|
||||
pub fn window_at(pid: libc::pid_t, x: f64, y: f64) -> Option<WindowInfo> {
|
||||
// The returned list is ordered front-to-back.
|
||||
let info = window_list()?;
|
||||
|
||||
// Read the window-info keys once; accessing the framework statics is unsafe.
|
||||
let owner_pid_key = unsafe { kCGWindowOwnerPID };
|
||||
let layer_key = unsafe { kCGWindowLayer };
|
||||
let number_key = unsafe { kCGWindowNumber };
|
||||
let bounds_key = unsafe { kCGWindowBounds };
|
||||
|
||||
let mut fallback: Option<WindowInfo> = None;
|
||||
for dict in info.iter() {
|
||||
// Only consider windows owned by the target process.
|
||||
if dict_i64(&dict, owner_pid_key) != Some(pid as i64) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Only consider normal (layer 0) windows; menus and similar live on other layers.
|
||||
if dict_i64(&dict, layer_key) != Some(0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let (Some(number), Some((bx, by, bw, bh))) = (
|
||||
dict_i64(&dict, number_key),
|
||||
dict_bounds(&dict, bounds_key).and_then(|b| read_bounds(&b)),
|
||||
) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let window = WindowInfo {
|
||||
number,
|
||||
x: bx,
|
||||
y: by,
|
||||
width: bw,
|
||||
height: bh,
|
||||
};
|
||||
|
||||
// Remember the frontmost owned window in case nothing contains the point.
|
||||
if fallback.is_none() {
|
||||
fallback = Some(window);
|
||||
}
|
||||
|
||||
if window.contains(x, y) {
|
||||
return Some(window);
|
||||
}
|
||||
}
|
||||
|
||||
fallback
|
||||
}
|
||||
|
||||
/// Finds the on-screen window with the given `window_id`, returning its number and bounds in
|
||||
/// global screen points (top-left origin). Used to resolve a `Target::Window` to concrete
|
||||
/// geometry for window-local coordinate remapping and window-scoped screenshot scaling.
|
||||
pub fn window_by_id(window_id: u32) -> Option<WindowInfo> {
|
||||
let info = window_list()?;
|
||||
|
||||
let number_key = unsafe { kCGWindowNumber };
|
||||
let bounds_key = unsafe { kCGWindowBounds };
|
||||
|
||||
for dict in info.iter() {
|
||||
if dict_i64(&dict, number_key) != Some(window_id as i64) {
|
||||
continue;
|
||||
}
|
||||
let (bx, by, bw, bh) = dict_bounds(&dict, bounds_key).and_then(|b| read_bounds(&b))?;
|
||||
return Some(WindowInfo {
|
||||
number: window_id as i64,
|
||||
x: bx,
|
||||
y: by,
|
||||
width: bw,
|
||||
height: bh,
|
||||
});
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Returns the `(owner_pid, window_number)` of the frontmost on-screen normal window, i.e. the
|
||||
/// window that currently has input focus. Used to deactivate the previous window when moving
|
||||
/// focus to a target without raising it.
|
||||
pub fn frontmost_window() -> Option<(libc::pid_t, i64)> {
|
||||
let info = window_list()?;
|
||||
|
||||
let owner_pid_key = unsafe { kCGWindowOwnerPID };
|
||||
let layer_key = unsafe { kCGWindowLayer };
|
||||
let number_key = unsafe { kCGWindowNumber };
|
||||
|
||||
// The list is front-to-back; the first normal (layer 0) window is the focused one.
|
||||
for dict in info.iter() {
|
||||
if dict_i64(&dict, layer_key) != Some(0) {
|
||||
continue;
|
||||
}
|
||||
if let (Some(pid), Some(number)) =
|
||||
(dict_i64(&dict, owner_pid_key), dict_i64(&dict, number_key))
|
||||
{
|
||||
return Some((pid as libc::pid_t, number));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// A description of an on-screen window, for diagnostics and enumeration.
|
||||
pub struct WindowDescription {
|
||||
pub number: i64,
|
||||
pub owner_pid: i64,
|
||||
pub owner_name: Option<String>,
|
||||
pub title: Option<String>,
|
||||
pub layer: i64,
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
pub width: f64,
|
||||
pub height: f64,
|
||||
}
|
||||
|
||||
/// Enumerates on-screen windows as crate-level [`crate::WindowInfo`] records, so the agent can
|
||||
/// pick a window to target. Ordered front-to-back, excluding desktop elements.
|
||||
pub fn enumerate_windows() -> Vec<crate::WindowInfo> {
|
||||
list_windows()
|
||||
.into_iter()
|
||||
.map(|w| crate::WindowInfo {
|
||||
window_id: w.number.max(0) as u32,
|
||||
pid: w.owner_pid as i32,
|
||||
app_name: w.owner_name.unwrap_or_default(),
|
||||
title: w.title.unwrap_or_default(),
|
||||
layer: w.layer as i32,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Lists on-screen windows (excluding desktop elements), front-to-back, for diagnostics.
|
||||
pub fn list_windows() -> Vec<WindowDescription> {
|
||||
let Some(info) = window_list() else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let owner_pid_key = unsafe { kCGWindowOwnerPID };
|
||||
let owner_name_key = unsafe { kCGWindowOwnerName };
|
||||
let name_key = unsafe { kCGWindowName };
|
||||
let layer_key = unsafe { kCGWindowLayer };
|
||||
let number_key = unsafe { kCGWindowNumber };
|
||||
let bounds_key = unsafe { kCGWindowBounds };
|
||||
|
||||
let mut windows = Vec::new();
|
||||
for dict in info.iter() {
|
||||
let (Some(number), Some(owner_pid)) =
|
||||
(dict_i64(&dict, number_key), dict_i64(&dict, owner_pid_key))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let (bx, by, bw, bh) = dict_bounds(&dict, bounds_key)
|
||||
.and_then(|b| read_bounds(&b))
|
||||
.unwrap_or((0.0, 0.0, 0.0, 0.0));
|
||||
|
||||
windows.push(WindowDescription {
|
||||
number,
|
||||
owner_pid,
|
||||
owner_name: dict_string(&dict, owner_name_key),
|
||||
// The window title requires the Screen Recording permission to be readable; it is
|
||||
// often empty otherwise.
|
||||
title: dict_string(&dict, name_key),
|
||||
layer: dict_i64(&dict, layer_key).unwrap_or(0),
|
||||
x: bx,
|
||||
y: by,
|
||||
width: bw,
|
||||
height: bh,
|
||||
});
|
||||
}
|
||||
windows
|
||||
}
|
||||
|
||||
/// Returns the on-screen window list with its documented key and value types.
|
||||
fn window_list() -> Option<CFRetained<CFArray<WindowDictionary>>> {
|
||||
let option =
|
||||
CGWindowListOption::OptionOnScreenOnly | CGWindowListOption::ExcludeDesktopElements;
|
||||
let info = CGWindowListCopyWindowInfo(option, kCGNullWindowID)?;
|
||||
|
||||
// SAFETY: Core Graphics documents the result as an array of dictionaries with CFString
|
||||
// keys and heterogeneous CFType values.
|
||||
Some(unsafe { CFRetained::cast_unchecked(info) })
|
||||
}
|
||||
|
||||
/// Reads a string value from a window dictionary.
|
||||
fn dict_string(dict: &WindowDictionary, key: &CFString) -> Option<String> {
|
||||
Some(dict.get(key)?.downcast::<CFString>().ok()?.to_string())
|
||||
}
|
||||
|
||||
/// Reads an integer value from a window dictionary.
|
||||
fn dict_i64(dict: &WindowDictionary, key: &CFString) -> Option<i64> {
|
||||
dict.get(key)?.downcast::<CFNumber>().ok()?.as_i64()
|
||||
}
|
||||
|
||||
/// Reads a bounds dictionary from a window dictionary.
|
||||
fn dict_bounds(dict: &WindowDictionary, key: &CFString) -> Option<CFRetained<BoundsDictionary>> {
|
||||
let bounds = dict.get(key)?.downcast::<CFDictionary>().ok()?;
|
||||
|
||||
// SAFETY: Core Graphics documents kCGWindowBounds as a dictionary with CFString keys and
|
||||
// CFNumber values.
|
||||
Some(unsafe { CFRetained::cast_unchecked(bounds) })
|
||||
}
|
||||
|
||||
/// Reads the `X`, `Y`, `Width`, `Height` numbers from a `kCGWindowBounds` dictionary.
|
||||
fn read_bounds(bounds: &BoundsDictionary) -> Option<(f64, f64, f64, f64)> {
|
||||
let get = |name: &'static str| -> Option<f64> {
|
||||
let key = CFString::from_static_str(name);
|
||||
bounds.get(&key)?.as_f64()
|
||||
};
|
||||
|
||||
Some((get("X")?, get("Y")?, get("Width")?, get("Height")?))
|
||||
}
|
||||
@@ -6,6 +6,12 @@ pub fn is_supported_on_current_platform() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Reports whether background, per-window control is available. The noop backend performs no
|
||||
/// real actions, so per-window background control is unsupported.
|
||||
pub fn background_supported() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub struct Actor;
|
||||
|
||||
impl Actor {
|
||||
@@ -22,12 +28,9 @@ impl super::Actor for Actor {
|
||||
|
||||
async fn perform_actions(
|
||||
&mut self,
|
||||
_actions: &[super::Action],
|
||||
_actions: &[super::TargetedAction],
|
||||
_options: super::Options,
|
||||
) -> Result<ActionResult, String> {
|
||||
Ok(ActionResult {
|
||||
screenshot: None,
|
||||
cursor_position: None,
|
||||
})
|
||||
Ok(ActionResult::legacy(None, None))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,12 +7,12 @@ mod mouse;
|
||||
mod screenshot;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use galaxyui::r#async::Timer;
|
||||
use galaxyui_core::r#async::Timer;
|
||||
use windows::Win32::System::StationsAndDesktops::{
|
||||
CloseDesktop, DESKTOP_ACCESS_FLAGS, DESKTOP_CONTROL_FLAGS, HDESK, OpenInputDesktop,
|
||||
};
|
||||
|
||||
use crate::{Action, ActionResult, Options};
|
||||
use crate::{Action, ActionResult, Options, TargetedAction};
|
||||
|
||||
/// Returns whether computer_use can drive input on this machine right now.
|
||||
///
|
||||
@@ -24,6 +24,12 @@ pub fn is_supported_on_current_platform() -> bool {
|
||||
probe_input_desktop_available()
|
||||
}
|
||||
|
||||
/// Reports whether background, per-window control is available. The Windows input stack drives the
|
||||
/// screen / foreground window, so per-window background control is unsupported.
|
||||
pub fn background_supported() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Shared probe used by both [`is_supported_on_current_platform`] and [`Actor::new`] so the
|
||||
/// "can we drive input right now?" logic lives in one place. This still runs the probe on each
|
||||
/// call (it's a cheap `OpenInputDesktop` / `CloseDesktop` round-trip) — we don't cache it because
|
||||
@@ -103,7 +109,7 @@ impl super::Actor for Actor {
|
||||
|
||||
async fn perform_actions(
|
||||
&mut self,
|
||||
actions: &[Action],
|
||||
actions: &[TargetedAction],
|
||||
options: Options,
|
||||
) -> Result<ActionResult, String> {
|
||||
// Probe at the top of every call so transient loss of the input desktop (workstation
|
||||
@@ -115,7 +121,10 @@ impl super::Actor for Actor {
|
||||
let keyboard = &mut self.keyboard;
|
||||
let mouse = &mut self.mouse;
|
||||
|
||||
for action in actions {
|
||||
for targeted in actions {
|
||||
// Per-window targeting is not supported on Windows; act on the screen / foreground
|
||||
// window regardless of the requested target.
|
||||
let action: &Action = &targeted.action;
|
||||
match action {
|
||||
Action::Wait(duration) => {
|
||||
Timer::after(*duration).await;
|
||||
@@ -152,9 +161,9 @@ impl super::Actor for Actor {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(ActionResult {
|
||||
Ok(ActionResult::legacy(
|
||||
screenshot,
|
||||
cursor_position: Some(mouse.current_position()?),
|
||||
})
|
||||
Some(mouse.current_position()?),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user