Initial public release of Warp.
Repo-Sync-Origin: warpdotdev/warp-internal@12af1d983b
This commit is contained in:
David Stern
@@ -0,0 +1,38 @@
|
||||
//! Shared X11 keysym utilities for Linux keyboard input.
|
||||
//!
|
||||
//! Both X11 and Wayland (via the RemoteDesktop portal) use X11 keysyms
|
||||
//! for keyboard input, so this module provides common conversion functions.
|
||||
|
||||
use std::ops::RangeInclusive;
|
||||
|
||||
/// Keysym range for uppercase ASCII letters (A-Z).
|
||||
pub const UPPERCASE_KEYSYMS: RangeInclusive<u32> = 0x41..=0x5A;
|
||||
|
||||
/// X11 keysym for left shift key.
|
||||
pub const XK_SHIFT_L: u32 = 0xFFE1;
|
||||
|
||||
/// Converts a Unicode character to an X11 keysym.
|
||||
pub fn char_to_keysym(ch: char) -> u32 {
|
||||
let code = ch as u32;
|
||||
|
||||
// ASCII characters map directly to keysyms for the printable range.
|
||||
if (0x20..=0x7E).contains(&code) {
|
||||
return code;
|
||||
}
|
||||
|
||||
// Latin-1 supplement (0x80-0xFF) also maps directly.
|
||||
if (0xA0..=0xFF).contains(&code) {
|
||||
return code;
|
||||
}
|
||||
|
||||
// For other Unicode characters, X11 uses the Unicode value + 0x01000000.
|
||||
0x01000000 | code
|
||||
}
|
||||
|
||||
/// Returns true if the keysym requires shift to be pressed.
|
||||
///
|
||||
/// This is a simple heuristic based on uppercase letters. Callers with access
|
||||
/// to keyboard mapping data may want to use more sophisticated logic.
|
||||
pub fn keysym_needs_shift(keysym: u32) -> bool {
|
||||
UPPERCASE_KEYSYMS.contains(&keysym)
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
mod keysym;
|
||||
mod wayland;
|
||||
mod x11;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::{Action, ActionResult, Options};
|
||||
|
||||
/// Returns true if a Wayland environment is available.
|
||||
fn is_wayland_available() -> bool {
|
||||
std::env::var("WAYLAND_DISPLAY")
|
||||
.map(|v| !v.is_empty())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Returns true if an X11 environment is available.
|
||||
fn is_x11_available() -> bool {
|
||||
std::env::var("DISPLAY")
|
||||
.map(|v| !v.is_empty())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
pub fn is_supported_on_current_platform() -> bool {
|
||||
is_wayland_available() || is_x11_available()
|
||||
}
|
||||
|
||||
pub struct Actor {
|
||||
inner: ActorInner,
|
||||
}
|
||||
|
||||
enum ActorInner {
|
||||
/// Wayland environment (uses XDG portals for input and screenshots).
|
||||
Wayland(Box<wayland::Actor>),
|
||||
/// X11 environment (uses XTEST for input).
|
||||
X11(Box<x11::Actor>),
|
||||
/// No supported display server available.
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
impl Actor {
|
||||
pub fn new() -> Self {
|
||||
let inner = if is_wayland_available() {
|
||||
// On Wayland, use native XDG portals for input and screenshots.
|
||||
match wayland::Actor::new() {
|
||||
Ok(actor) => ActorInner::Wayland(Box::new(actor)),
|
||||
Err(e) => {
|
||||
log::error!("Failed to create Wayland actor: {e}");
|
||||
ActorInner::Unsupported
|
||||
}
|
||||
}
|
||||
} else if is_x11_available() {
|
||||
// Pure X11 environment.
|
||||
match x11::Actor::new() {
|
||||
Ok(actor) => ActorInner::X11(Box::new(actor)),
|
||||
Err(e) => {
|
||||
log::error!("Failed to create X11 actor: {e}");
|
||||
ActorInner::Unsupported
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ActorInner::Unsupported
|
||||
};
|
||||
|
||||
Self { inner }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl super::Actor for Actor {
|
||||
fn platform(&self) -> Option<super::Platform> {
|
||||
match &self.inner {
|
||||
ActorInner::Wayland(actor) => actor.platform(),
|
||||
ActorInner::X11(actor) => actor.platform(),
|
||||
ActorInner::Unsupported => None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn perform_actions(
|
||||
&mut self,
|
||||
actions: &[Action],
|
||||
options: Options,
|
||||
) -> Result<ActionResult, String> {
|
||||
match &mut self.inner {
|
||||
ActorInner::Wayland(actor) => actor.perform_actions(actions, options).await,
|
||||
ActorInner::X11(actor) => actor.perform_actions(actions, options).await,
|
||||
ActorInner::Unsupported => Err(
|
||||
"Computer use is not available: No supported display server detected. \
|
||||
X11 or Wayland is required."
|
||||
.to_string(),
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
//! Keyboard input handling for Wayland via the RemoteDesktop portal.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use ashpd::desktop::Session;
|
||||
use ashpd::desktop::remote_desktop::{KeyState, RemoteDesktop};
|
||||
|
||||
use super::super::keysym::{XK_SHIFT_L, char_to_keysym, keysym_needs_shift};
|
||||
use crate::Key;
|
||||
|
||||
/// Keyboard state for tracking auto-shifted keys.
|
||||
pub struct Keyboard {
|
||||
/// Keysyms currently held that required auto-shift.
|
||||
/// When non-empty, shift is being held by us.
|
||||
auto_shift_keys: HashSet<i32>,
|
||||
}
|
||||
|
||||
impl Keyboard {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
auto_shift_keys: HashSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Types a string of text by sending keysym events.
|
||||
pub async fn type_text<'a>(
|
||||
&mut self,
|
||||
remote_desktop: &RemoteDesktop<'a>,
|
||||
session: &Session<'a, RemoteDesktop<'a>>,
|
||||
text: &str,
|
||||
) -> Result<(), String> {
|
||||
for ch in text.chars() {
|
||||
self.type_char(remote_desktop, session, ch).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sends a key down event for the given key.
|
||||
///
|
||||
/// For `Key::Char`, this will automatically press shift if needed.
|
||||
pub async fn key_down<'a>(
|
||||
&mut self,
|
||||
remote_desktop: &RemoteDesktop<'a>,
|
||||
session: &Session<'a, RemoteDesktop<'a>>,
|
||||
key: &Key,
|
||||
) -> Result<(), String> {
|
||||
let (keysym, needs_shift) = self.resolve_key(key);
|
||||
|
||||
if needs_shift {
|
||||
// Press shift only if this is the first auto-shifted key.
|
||||
if self.auto_shift_keys.is_empty() {
|
||||
self.press_shift(remote_desktop, session).await?;
|
||||
}
|
||||
self.auto_shift_keys.insert(keysym);
|
||||
}
|
||||
|
||||
remote_desktop
|
||||
.notify_keyboard_keysym(session, keysym, KeyState::Pressed)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to send key down: {e}"))
|
||||
}
|
||||
|
||||
/// Sends a key up event for the given key.
|
||||
///
|
||||
/// For `Key::Char`, this will automatically release shift if it was auto-pressed
|
||||
/// and this is the last auto-shifted key being released.
|
||||
pub async fn key_up<'a>(
|
||||
&mut self,
|
||||
remote_desktop: &RemoteDesktop<'a>,
|
||||
session: &Session<'a, RemoteDesktop<'a>>,
|
||||
key: &Key,
|
||||
) -> Result<(), String> {
|
||||
let (keysym, _) = self.resolve_key(key);
|
||||
|
||||
remote_desktop
|
||||
.notify_keyboard_keysym(session, keysym, KeyState::Released)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to send key up: {e}"))?;
|
||||
|
||||
// Release shift only if this key was auto-shifted and it's the last one.
|
||||
if self.auto_shift_keys.remove(&keysym) && self.auto_shift_keys.is_empty() {
|
||||
self.release_shift(remote_desktop, session).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn type_char<'a>(
|
||||
&mut self,
|
||||
remote_desktop: &RemoteDesktop<'a>,
|
||||
session: &Session<'a, RemoteDesktop<'a>>,
|
||||
ch: char,
|
||||
) -> Result<(), String> {
|
||||
let keysym = char_to_keysym(ch) as i32;
|
||||
let needs_shift = keysym_needs_shift(keysym as u32);
|
||||
|
||||
// Press shift if needed, then press and release the key, then release shift.
|
||||
if needs_shift {
|
||||
self.press_shift(remote_desktop, session).await?;
|
||||
}
|
||||
|
||||
remote_desktop
|
||||
.notify_keyboard_keysym(session, keysym, KeyState::Pressed)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to send key press: {e}"))?;
|
||||
|
||||
remote_desktop
|
||||
.notify_keyboard_keysym(session, keysym, KeyState::Released)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to send key release: {e}"))?;
|
||||
|
||||
if needs_shift {
|
||||
self.release_shift(remote_desktop, session).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resolves a `Key` to a keysym and shift requirement.
|
||||
fn resolve_key(&self, key: &Key) -> (i32, bool) {
|
||||
match key {
|
||||
Key::Keycode(keysym) => {
|
||||
// Key::Keycode uses X11 keysyms, which we can use directly.
|
||||
(*keysym, false)
|
||||
}
|
||||
Key::Char(ch) => {
|
||||
let keysym = char_to_keysym(*ch) as i32;
|
||||
let needs_shift = keysym_needs_shift(keysym as u32);
|
||||
(keysym, needs_shift)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn press_shift<'a>(
|
||||
&self,
|
||||
remote_desktop: &RemoteDesktop<'a>,
|
||||
session: &Session<'a, RemoteDesktop<'a>>,
|
||||
) -> Result<(), String> {
|
||||
remote_desktop
|
||||
.notify_keyboard_keysym(session, XK_SHIFT_L as i32, KeyState::Pressed)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to press shift: {e}"))
|
||||
}
|
||||
|
||||
async fn release_shift<'a>(
|
||||
&self,
|
||||
remote_desktop: &RemoteDesktop<'a>,
|
||||
session: &Session<'a, RemoteDesktop<'a>>,
|
||||
) -> Result<(), String> {
|
||||
remote_desktop
|
||||
.notify_keyboard_keysym(session, XK_SHIFT_L as i32, KeyState::Released)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to release shift: {e}"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
//! Wayland implementation of computer use actions.
|
||||
//!
|
||||
//! This module handles the Wayland environment using native XDG portals:
|
||||
//! - RemoteDesktop portal for input injection (keyboard, mouse)
|
||||
//! - ScreenCast portal for absolute pointer positioning (stream IDs)
|
||||
//! - Screenshot portal for taking screenshots
|
||||
|
||||
mod keyboard;
|
||||
mod mouse;
|
||||
mod screenshot;
|
||||
mod session;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use pathfinder_geometry::vector::Vector2I;
|
||||
use warpui::r#async::Timer;
|
||||
|
||||
use crate::{Action, ActionResult, Options};
|
||||
|
||||
use keyboard::Keyboard;
|
||||
use mouse::Mouse;
|
||||
use session::PortalSession;
|
||||
|
||||
/// An actor that performs computer use actions on Wayland via XDG portals.
|
||||
pub struct Actor {
|
||||
/// The portal session, created lazily on first use.
|
||||
session: Option<PortalSession<'static>>,
|
||||
/// Keyboard state for tracking shift and other modifiers.
|
||||
keyboard: Keyboard,
|
||||
/// Mouse state for tracking position.
|
||||
mouse: Mouse,
|
||||
}
|
||||
|
||||
impl Actor {
|
||||
pub fn new() -> Result<Self, String> {
|
||||
Ok(Self {
|
||||
session: None,
|
||||
keyboard: Keyboard::new(),
|
||||
mouse: Mouse::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Ensures a portal session is available, creating one if needed.
|
||||
async fn ensure_session(&mut self) -> Result<(), String> {
|
||||
if self.session.is_none() {
|
||||
self.session = Some(PortalSession::new().await?);
|
||||
// Wait for the permission dialog to fully dismiss before returning.
|
||||
Timer::after(std::time::Duration::from_millis(500)).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl crate::Actor for Actor {
|
||||
fn platform(&self) -> Option<crate::Platform> {
|
||||
Some(crate::Platform::LinuxWayland)
|
||||
}
|
||||
|
||||
async fn perform_actions(
|
||||
&mut self,
|
||||
actions: &[Action],
|
||||
options: Options,
|
||||
) -> Result<ActionResult, String> {
|
||||
// Ensure we have an active session before processing actions.
|
||||
// This may show a permission dialog on first use.
|
||||
self.ensure_session().await?;
|
||||
|
||||
let mut last_mouse_position: Option<Vector2I> = None;
|
||||
|
||||
for action in actions {
|
||||
// Re-acquire session reference each iteration (borrow checker workaround).
|
||||
let session = self
|
||||
.session
|
||||
.as_ref()
|
||||
.expect("session must exist after ensure_session");
|
||||
let remote_desktop = session.remote_desktop();
|
||||
let portal_session = session.session();
|
||||
let stream_id = session.stream_id();
|
||||
|
||||
match action {
|
||||
Action::Wait(duration) => {
|
||||
Timer::after(*duration).await;
|
||||
}
|
||||
Action::MouseDown { button, at } => {
|
||||
session.require_pointer()?;
|
||||
self.mouse
|
||||
.move_to(remote_desktop, portal_session, stream_id, *at)
|
||||
.await?;
|
||||
self.mouse
|
||||
.button_down(remote_desktop, portal_session, button)
|
||||
.await?;
|
||||
last_mouse_position = Some(*at);
|
||||
}
|
||||
Action::MouseUp { button } => {
|
||||
session.require_pointer()?;
|
||||
self.mouse
|
||||
.button_up(remote_desktop, portal_session, button)
|
||||
.await?;
|
||||
}
|
||||
Action::MouseMove { to } => {
|
||||
session.require_pointer()?;
|
||||
self.mouse
|
||||
.move_to(remote_desktop, portal_session, stream_id, *to)
|
||||
.await?;
|
||||
last_mouse_position = Some(*to);
|
||||
}
|
||||
Action::MouseWheel {
|
||||
at,
|
||||
direction,
|
||||
distance,
|
||||
} => {
|
||||
session.require_pointer()?;
|
||||
self.mouse
|
||||
.move_to(remote_desktop, portal_session, stream_id, *at)
|
||||
.await?;
|
||||
self.mouse
|
||||
.scroll(remote_desktop, portal_session, direction, distance)
|
||||
.await?;
|
||||
last_mouse_position = Some(*at);
|
||||
}
|
||||
Action::TypeText { text } => {
|
||||
session.require_keyboard()?;
|
||||
self.keyboard
|
||||
.type_text(remote_desktop, portal_session, text)
|
||||
.await?;
|
||||
}
|
||||
Action::KeyDown { key } => {
|
||||
session.require_keyboard()?;
|
||||
self.keyboard
|
||||
.key_down(remote_desktop, portal_session, key)
|
||||
.await?;
|
||||
}
|
||||
Action::KeyUp { key } => {
|
||||
session.require_keyboard()?;
|
||||
self.keyboard
|
||||
.key_up(remote_desktop, portal_session, key)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Take screenshot if requested.
|
||||
let screenshot = if let Some(params) = options.screenshot_params {
|
||||
Some(screenshot::take(params).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Get the final cursor position.
|
||||
let cursor_position = if let Some(pos) = last_mouse_position {
|
||||
Some(pos)
|
||||
} else {
|
||||
self.mouse.last_position()
|
||||
};
|
||||
|
||||
Ok(ActionResult {
|
||||
screenshot,
|
||||
cursor_position,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
//! Mouse input handling for Wayland via the RemoteDesktop portal.
|
||||
|
||||
use ashpd::desktop::Session;
|
||||
use ashpd::desktop::remote_desktop::{Axis, KeyState, RemoteDesktop};
|
||||
use pathfinder_geometry::vector::Vector2I;
|
||||
|
||||
use crate::{MouseButton, ScrollDirection, ScrollDistance};
|
||||
|
||||
/// Linux evdev button codes.
|
||||
const BTN_LEFT: i32 = 0x110;
|
||||
const BTN_RIGHT: i32 = 0x111;
|
||||
const BTN_MIDDLE: i32 = 0x112;
|
||||
const BTN_SIDE: i32 = 0x113; // Back.
|
||||
const BTN_EXTRA: i32 = 0x114; // Forward.
|
||||
|
||||
/// Mouse state for the Wayland portal.
|
||||
pub struct Mouse {
|
||||
/// The last known mouse position (for returning in ActionResult).
|
||||
last_position: Option<Vector2I>,
|
||||
}
|
||||
|
||||
impl Mouse {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
last_position: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Moves the mouse to an absolute position.
|
||||
pub async fn move_to<'a>(
|
||||
&mut self,
|
||||
remote_desktop: &RemoteDesktop<'a>,
|
||||
session: &Session<'a, RemoteDesktop<'a>>,
|
||||
stream_id: u32,
|
||||
target: Vector2I,
|
||||
) -> Result<(), String> {
|
||||
remote_desktop
|
||||
.notify_pointer_motion_absolute(
|
||||
session,
|
||||
stream_id,
|
||||
target.x() as f64,
|
||||
target.y() as f64,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to move mouse: {e}"))?;
|
||||
|
||||
self.last_position = Some(target);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Presses a mouse button down.
|
||||
pub async fn button_down<'a>(
|
||||
&self,
|
||||
remote_desktop: &RemoteDesktop<'a>,
|
||||
session: &Session<'a, RemoteDesktop<'a>>,
|
||||
button: &MouseButton,
|
||||
) -> Result<(), String> {
|
||||
let evdev_button = mouse_button_to_evdev(button);
|
||||
|
||||
remote_desktop
|
||||
.notify_pointer_button(session, evdev_button, KeyState::Pressed)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to press mouse button: {e}"))
|
||||
}
|
||||
|
||||
/// Releases a mouse button.
|
||||
pub async fn button_up<'a>(
|
||||
&self,
|
||||
remote_desktop: &RemoteDesktop<'a>,
|
||||
session: &Session<'a, RemoteDesktop<'a>>,
|
||||
button: &MouseButton,
|
||||
) -> Result<(), String> {
|
||||
let evdev_button = mouse_button_to_evdev(button);
|
||||
|
||||
remote_desktop
|
||||
.notify_pointer_button(session, evdev_button, KeyState::Released)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to release mouse button: {e}"))
|
||||
}
|
||||
|
||||
/// Performs a scroll action.
|
||||
pub async fn scroll<'a>(
|
||||
&self,
|
||||
remote_desktop: &RemoteDesktop<'a>,
|
||||
session: &Session<'a, RemoteDesktop<'a>>,
|
||||
direction: &ScrollDirection,
|
||||
distance: &ScrollDistance,
|
||||
) -> Result<(), String> {
|
||||
match distance {
|
||||
ScrollDistance::Clicks(clicks) => {
|
||||
// Use discrete scrolling for click-based scrolling.
|
||||
let (axis, steps) = match direction {
|
||||
ScrollDirection::Up => (Axis::Vertical, -*clicks),
|
||||
ScrollDirection::Down => (Axis::Vertical, *clicks),
|
||||
ScrollDirection::Left => (Axis::Horizontal, -*clicks),
|
||||
ScrollDirection::Right => (Axis::Horizontal, *clicks),
|
||||
};
|
||||
|
||||
remote_desktop
|
||||
.notify_pointer_axis_discrete(session, axis, steps)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to scroll: {e}"))
|
||||
}
|
||||
ScrollDistance::Pixels(pixels) => {
|
||||
// Use smooth scrolling for pixel-based scrolling.
|
||||
let (dx, dy) = match direction {
|
||||
ScrollDirection::Up => (0.0, -(*pixels as f64)),
|
||||
ScrollDirection::Down => (0.0, *pixels as f64),
|
||||
ScrollDirection::Left => (-(*pixels as f64), 0.0),
|
||||
ScrollDirection::Right => (*pixels as f64, 0.0),
|
||||
};
|
||||
|
||||
remote_desktop
|
||||
.notify_pointer_axis(session, dx, dy, true)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to scroll: {e}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the last known mouse position.
|
||||
pub fn last_position(&self) -> Option<Vector2I> {
|
||||
self.last_position
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts a MouseButton to a Linux evdev button code.
|
||||
fn mouse_button_to_evdev(button: &MouseButton) -> i32 {
|
||||
match button {
|
||||
MouseButton::Left => BTN_LEFT,
|
||||
MouseButton::Right => BTN_RIGHT,
|
||||
MouseButton::Middle => BTN_MIDDLE,
|
||||
MouseButton::Back => BTN_SIDE,
|
||||
MouseButton::Forward => BTN_EXTRA,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
//! Screenshot capture for Wayland using the XDG Desktop Portal.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use futures::StreamExt as _;
|
||||
use zbus::zvariant;
|
||||
|
||||
use crate::{Screenshot, ScreenshotParams};
|
||||
|
||||
/// A D-Bus proxy for the Screenshot portal.
|
||||
#[zbus::proxy(
|
||||
interface = "org.freedesktop.portal.Screenshot",
|
||||
default_service = "org.freedesktop.portal.Desktop",
|
||||
default_path = "/org/freedesktop/portal/desktop"
|
||||
)]
|
||||
trait ScreenshotPortal {
|
||||
/// Takes a screenshot.
|
||||
///
|
||||
/// Returns an object path for a Request object that will receive the response.
|
||||
fn screenshot(
|
||||
&self,
|
||||
parent_window: &str,
|
||||
options: HashMap<&str, zvariant::Value<'_>>,
|
||||
) -> zbus::fdo::Result<zvariant::OwnedObjectPath>;
|
||||
}
|
||||
|
||||
/// A D-Bus proxy for portal Request objects.
|
||||
#[zbus::proxy(
|
||||
interface = "org.freedesktop.portal.Request",
|
||||
default_service = "org.freedesktop.portal.Desktop"
|
||||
)]
|
||||
trait PortalRequest {
|
||||
/// Signal emitted when the request completes.
|
||||
#[zbus(signal)]
|
||||
fn response(
|
||||
&self,
|
||||
response: u32,
|
||||
results: HashMap<String, zvariant::OwnedValue>,
|
||||
) -> zbus::fdo::Result<()>;
|
||||
}
|
||||
|
||||
/// Takes a screenshot using the XDG Desktop Portal.
|
||||
pub async fn take(params: ScreenshotParams) -> Result<Screenshot, String> {
|
||||
let connection = zbus::Connection::session()
|
||||
.await
|
||||
.map_err(|e| format!("Failed to connect to D-Bus session bus: {e}"))?;
|
||||
|
||||
let screenshot_proxy = ScreenshotPortalProxy::new(&connection)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to create screenshot portal proxy: {e}"))?;
|
||||
|
||||
// Request a non-interactive screenshot.
|
||||
let mut options: HashMap<&str, zvariant::Value> = HashMap::new();
|
||||
options.insert("interactive", zvariant::Value::Bool(false));
|
||||
|
||||
let request_path = screenshot_proxy
|
||||
.screenshot("", options)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to request screenshot: {e}"))?;
|
||||
|
||||
// Wait for the response signal.
|
||||
let request_proxy = PortalRequestProxy::builder(&connection)
|
||||
.path(request_path)
|
||||
.map_err(|e| format!("Failed to build request proxy: {e}"))?
|
||||
.build()
|
||||
.await
|
||||
.map_err(|e| format!("Failed to create request proxy: {e}"))?;
|
||||
|
||||
let mut response_stream = request_proxy
|
||||
.receive_response()
|
||||
.await
|
||||
.map_err(|e| format!("Failed to subscribe to response signal: {e}"))?;
|
||||
|
||||
// Wait for the response.
|
||||
let response = response_stream
|
||||
.next()
|
||||
.await
|
||||
.ok_or("Screenshot request was cancelled or timed out")?;
|
||||
|
||||
let args = response
|
||||
.args()
|
||||
.map_err(|e| format!("Failed to get response arguments: {e}"))?;
|
||||
|
||||
// Response code 0 means success, 1 means cancelled, 2 means other error.
|
||||
if args.response != 0 {
|
||||
return Err(format!(
|
||||
"Screenshot request failed with response code: {}",
|
||||
args.response
|
||||
));
|
||||
}
|
||||
|
||||
// Extract the URI from the results.
|
||||
let uri_value = args
|
||||
.results
|
||||
.get("uri")
|
||||
.ok_or("Screenshot response missing 'uri' field")?;
|
||||
|
||||
let uri: &str = uri_value
|
||||
.downcast_ref()
|
||||
.map_err(|e| format!("Failed to get URI from response: {e}"))?;
|
||||
|
||||
// Parse the file:// URI and read the file.
|
||||
let path = url::Url::parse(uri)
|
||||
.map_err(|e| format!("Failed to parse screenshot URI: {e}"))?
|
||||
.to_file_path()
|
||||
.map_err(|_| format!("Screenshot URI is not a valid file path: {uri}"))?;
|
||||
|
||||
// Load the image from the temporary file.
|
||||
let img = image::ImageReader::open(&path)
|
||||
.map_err(|e| format!("Failed to open screenshot file: {e}"))?
|
||||
.decode()
|
||||
.map_err(|e| format!("Failed to decode screenshot: {e}"))?;
|
||||
|
||||
// Clean up the temporary file created by the portal.
|
||||
let _ = std::fs::remove_file(&path);
|
||||
|
||||
// If capturing a region, crop the full-display image.
|
||||
// The XDG Portal doesn't support region capture natively.
|
||||
let img = if let Some(region) = params.region {
|
||||
region.validate()?;
|
||||
crate::screenshot_utils::crop_to_region(img, region.top_left, region.bottom_right)
|
||||
} else {
|
||||
img
|
||||
};
|
||||
|
||||
crate::screenshot_utils::process_screenshot(img, params)
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
//! Session management for the XDG RemoteDesktop and ScreenCast portals.
|
||||
//!
|
||||
//! This module handles creating and maintaining a portal session that enables
|
||||
//! input emulation via the RemoteDesktop portal and provides stream IDs for
|
||||
//! absolute pointer positioning via the ScreenCast portal.
|
||||
|
||||
use ashpd::desktop::PersistMode;
|
||||
use ashpd::desktop::remote_desktop::{DeviceType, RemoteDesktop};
|
||||
use ashpd::desktop::screencast::{CursorMode, Screencast, SourceType};
|
||||
use ashpd::enumflags2::BitFlags;
|
||||
|
||||
/// A portal session that provides input emulation capabilities.
|
||||
///
|
||||
/// This session combines RemoteDesktop (for input) and ScreenCast (for absolute
|
||||
/// positioning coordinates) into a single session with one permission dialog.
|
||||
pub struct PortalSession<'a> {
|
||||
remote_desktop: RemoteDesktop<'a>,
|
||||
session: ashpd::desktop::Session<'a, RemoteDesktop<'a>>,
|
||||
/// The PipeWire stream node ID for the primary monitor.
|
||||
/// Used for absolute pointer positioning.
|
||||
stream_id: u32,
|
||||
/// The device types that were granted by the user.
|
||||
granted_devices: BitFlags<DeviceType>,
|
||||
}
|
||||
|
||||
impl<'a> PortalSession<'a> {
|
||||
/// Creates and starts a new portal session.
|
||||
///
|
||||
/// This will show a permission dialog to the user (unless permissions are
|
||||
/// already persisted from a previous session).
|
||||
pub async fn new() -> Result<Self, String> {
|
||||
let remote_desktop = RemoteDesktop::new()
|
||||
.await
|
||||
.map_err(|e| format!("Failed to create RemoteDesktop proxy: {e}"))?;
|
||||
|
||||
let screencast = Screencast::new()
|
||||
.await
|
||||
.map_err(|e| format!("Failed to create Screencast proxy: {e}"))?;
|
||||
|
||||
// Create a RemoteDesktop session. This session is shared with ScreenCast.
|
||||
let session = remote_desktop
|
||||
.create_session()
|
||||
.await
|
||||
.map_err(|e| format!("Failed to create RemoteDesktop session: {e}"))?;
|
||||
|
||||
// Select input devices (keyboard and pointer).
|
||||
// Note: Some portals don't support persistence for remote desktop sessions,
|
||||
// so we use DoNot to avoid errors.
|
||||
remote_desktop
|
||||
.select_devices(
|
||||
&session,
|
||||
DeviceType::Keyboard | DeviceType::Pointer,
|
||||
None, // No restore token.
|
||||
PersistMode::DoNot,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to select devices: {e}"))?;
|
||||
|
||||
// Select screencast sources (monitors) to get stream IDs for absolute positioning.
|
||||
// We use CursorMode::Metadata since we only need the stream ID, not the video.
|
||||
screencast
|
||||
.select_sources(
|
||||
&session,
|
||||
CursorMode::Metadata,
|
||||
SourceType::Monitor.into(),
|
||||
true, // Allow multiple monitors.
|
||||
None, // No restore token.
|
||||
PersistMode::DoNot,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to select screencast sources: {e}"))?;
|
||||
|
||||
// Start the session. This shows the permission dialog to the user.
|
||||
let response = remote_desktop
|
||||
.start(&session, None)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to start session request: {e}"))?
|
||||
.response()
|
||||
.map_err(|e| format!("Session start failed: {e}"))?;
|
||||
|
||||
// Extract the stream ID from the response.
|
||||
let streams = response
|
||||
.streams()
|
||||
.ok_or("No streams returned from ScreenCast")?;
|
||||
|
||||
if streams.is_empty() {
|
||||
return Err("No monitors available for screen casting".to_string());
|
||||
}
|
||||
|
||||
// Use the first stream (primary monitor) for now.
|
||||
let stream_id = streams[0].pipe_wire_node_id();
|
||||
|
||||
// Get the devices that were actually granted by the user.
|
||||
let granted_devices = response.devices();
|
||||
|
||||
Ok(Self {
|
||||
remote_desktop,
|
||||
session,
|
||||
stream_id,
|
||||
granted_devices,
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns a reference to the RemoteDesktop proxy.
|
||||
pub fn remote_desktop(&self) -> &RemoteDesktop<'a> {
|
||||
&self.remote_desktop
|
||||
}
|
||||
|
||||
/// Returns a reference to the session.
|
||||
pub fn session(&self) -> &ashpd::desktop::Session<'a, RemoteDesktop<'a>> {
|
||||
&self.session
|
||||
}
|
||||
|
||||
/// Returns the stream ID for the primary monitor.
|
||||
///
|
||||
/// This ID is used for absolute pointer positioning via
|
||||
/// `notify_pointer_motion_absolute`.
|
||||
pub fn stream_id(&self) -> u32 {
|
||||
self.stream_id
|
||||
}
|
||||
|
||||
/// Validates that keyboard input permission was granted.
|
||||
///
|
||||
/// Returns an error with a clear message for the agent if permission was denied.
|
||||
pub fn require_keyboard(&self) -> Result<(), String> {
|
||||
if self.granted_devices.contains(DeviceType::Keyboard) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(
|
||||
"Keyboard input permission was not granted. \
|
||||
The user must allow keyboard input in the portal dialog to perform keyboard actions."
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates that pointer/mouse input permission was granted.
|
||||
///
|
||||
/// Returns an error with a clear message for the agent if permission was denied.
|
||||
pub fn require_pointer(&self) -> Result<(), String> {
|
||||
if self.granted_devices.contains(DeviceType::Pointer) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err("Mouse input permission was not granted. \
|
||||
The user must allow mouse input in the portal dialog to perform mouse actions."
|
||||
.to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
//! Keyboard input handling for X11 using XTEST.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use x11rb::connection::Connection;
|
||||
use x11rb::protocol::xproto;
|
||||
use x11rb::protocol::xtest::ConnectionExt as _;
|
||||
use x11rb::rust_connection::RustConnection;
|
||||
|
||||
use super::super::keysym::{UPPERCASE_KEYSYMS, XK_SHIFT_L, char_to_keysym};
|
||||
use crate::Key;
|
||||
|
||||
/// A resolved key with its keycode and shift requirement.
|
||||
struct ResolvedKey {
|
||||
keycode: u8,
|
||||
needs_shift: bool,
|
||||
}
|
||||
|
||||
/// Keyboard state tracking for an X11 connection.
|
||||
pub struct Keyboard<'a> {
|
||||
conn: &'a RustConnection,
|
||||
keyboard_mapping: &'a xproto::GetKeyboardMappingReply,
|
||||
/// Keycodes currently held that required auto-shift.
|
||||
/// When non-empty, shift is being held by us.
|
||||
auto_shift_keys: HashSet<u8>,
|
||||
}
|
||||
|
||||
impl<'a> Keyboard<'a> {
|
||||
pub fn new(
|
||||
conn: &'a RustConnection,
|
||||
keyboard_mapping: &'a xproto::GetKeyboardMappingReply,
|
||||
) -> Self {
|
||||
Self {
|
||||
conn,
|
||||
keyboard_mapping,
|
||||
auto_shift_keys: HashSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn type_text(&mut self, text: &str) -> Result<(), String> {
|
||||
// For each character, we need to find a keycode that produces it.
|
||||
// This is complex because X11 keycodes depend on the keyboard layout.
|
||||
for ch in text.chars() {
|
||||
self.type_char(ch)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sends a key down event for the given key.
|
||||
///
|
||||
/// For `Key::Char`, this will automatically press shift if needed.
|
||||
pub fn key_down(&mut self, key: &Key) -> Result<(), String> {
|
||||
let resolved = self.resolve_key(key)?;
|
||||
if resolved.needs_shift {
|
||||
// Press shift only if this is the first auto-shifted key.
|
||||
if self.auto_shift_keys.is_empty() {
|
||||
self.press_shift()?;
|
||||
}
|
||||
self.auto_shift_keys.insert(resolved.keycode);
|
||||
}
|
||||
self.key_press(resolved.keycode)
|
||||
}
|
||||
|
||||
/// Sends a key up event for the given key.
|
||||
///
|
||||
/// For `Key::Char`, this will automatically release shift if it was auto-pressed
|
||||
/// and this is the last auto-shifted key being released.
|
||||
pub fn key_up(&mut self, key: &Key) -> Result<(), String> {
|
||||
let resolved = self.resolve_key(key)?;
|
||||
self.key_release(resolved.keycode)?;
|
||||
// Release shift only if this key was auto-shifted and it's the last one.
|
||||
if self.auto_shift_keys.remove(&resolved.keycode) && self.auto_shift_keys.is_empty() {
|
||||
self.release_shift()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn type_char(&mut self, ch: char) -> Result<(), String> {
|
||||
let resolved = self.resolve_key_for_char(ch)?;
|
||||
|
||||
// Press shift if needed, then press and release the key, then release shift.
|
||||
if resolved.needs_shift {
|
||||
self.press_shift()?;
|
||||
}
|
||||
self.key_press(resolved.keycode)?;
|
||||
self.key_release(resolved.keycode)?;
|
||||
if resolved.needs_shift {
|
||||
self.release_shift()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resolves a `Key` to a keycode and shift requirement.
|
||||
fn resolve_key(&self, key: &Key) -> Result<ResolvedKey, String> {
|
||||
match key {
|
||||
Key::Keycode(keysym) => {
|
||||
if *keysym < 0 {
|
||||
return Err(format!("Invalid keysym: {keysym} (must be non-negative)"));
|
||||
}
|
||||
let keysym = *keysym as u32;
|
||||
let keycode = self.find_keycode_for_keysym(keysym)?;
|
||||
// For explicit keysyms, caller manages modifiers.
|
||||
Ok(ResolvedKey {
|
||||
keycode,
|
||||
needs_shift: false,
|
||||
})
|
||||
}
|
||||
Key::Char(ch) => self.resolve_key_for_char(*ch),
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves a character to a keycode and shift requirement.
|
||||
fn resolve_key_for_char(&self, ch: char) -> Result<ResolvedKey, String> {
|
||||
let keysym = char_to_keysym(ch);
|
||||
let keycode = self.find_keycode_for_keysym(keysym)?;
|
||||
let needs_shift = self.keysym_needs_shift(keysym, keycode);
|
||||
Ok(ResolvedKey {
|
||||
keycode,
|
||||
needs_shift,
|
||||
})
|
||||
}
|
||||
|
||||
/// Presses the shift key.
|
||||
fn press_shift(&mut self) -> Result<(), String> {
|
||||
let shift_keycode = self.find_keycode_for_keysym(XK_SHIFT_L)?;
|
||||
self.key_press(shift_keycode)
|
||||
}
|
||||
|
||||
/// Releases the shift key.
|
||||
fn release_shift(&mut self) -> Result<(), String> {
|
||||
let shift_keycode = self.find_keycode_for_keysym(XK_SHIFT_L)?;
|
||||
self.key_release(shift_keycode)
|
||||
}
|
||||
|
||||
fn key_press(&mut self, keycode: u8) -> Result<(), String> {
|
||||
self.conn
|
||||
.xtest_fake_input(
|
||||
xproto::KEY_PRESS_EVENT,
|
||||
keycode,
|
||||
x11rb::CURRENT_TIME,
|
||||
x11rb::NONE,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
.map_err(|e| format!("Failed to send key press: {e}"))?;
|
||||
|
||||
self.conn
|
||||
.flush()
|
||||
.map_err(|e| format!("Failed to flush X11 connection: {e}"))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn key_release(&mut self, keycode: u8) -> Result<(), String> {
|
||||
self.conn
|
||||
.xtest_fake_input(
|
||||
xproto::KEY_RELEASE_EVENT,
|
||||
keycode,
|
||||
x11rb::CURRENT_TIME,
|
||||
x11rb::NONE,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
.map_err(|e| format!("Failed to send key release: {e}"))?;
|
||||
|
||||
self.conn
|
||||
.flush()
|
||||
.map_err(|e| format!("Failed to flush X11 connection: {e}"))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn find_keycode_for_keysym(&self, keysym: u32) -> Result<u8, String> {
|
||||
let setup = self.conn.setup();
|
||||
let min_keycode = setup.min_keycode;
|
||||
let max_keycode = setup.max_keycode;
|
||||
|
||||
let keysyms_per_keycode = self.keyboard_mapping.keysyms_per_keycode as usize;
|
||||
|
||||
// Search for a keycode that produces the desired keysym.
|
||||
for keycode in min_keycode..=max_keycode {
|
||||
let offset = (keycode - min_keycode) as usize * keysyms_per_keycode;
|
||||
for i in 0..keysyms_per_keycode {
|
||||
if offset + i < self.keyboard_mapping.keysyms.len()
|
||||
&& self.keyboard_mapping.keysyms[offset + i] == keysym
|
||||
{
|
||||
return Ok(keycode);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try to find the unshifted version for uppercase letters.
|
||||
if UPPERCASE_KEYSYMS.contains(&keysym) {
|
||||
let lower = keysym + 0x20;
|
||||
return self.find_keycode_for_keysym(lower);
|
||||
}
|
||||
|
||||
Err(format!(
|
||||
"No keycode found for keysym 0x{:x} (char: {:?})",
|
||||
keysym,
|
||||
char::from_u32(keysym)
|
||||
))
|
||||
}
|
||||
|
||||
fn keysym_needs_shift(&self, keysym: u32, keycode: u8) -> bool {
|
||||
let setup = self.conn.setup();
|
||||
let min_keycode = setup.min_keycode;
|
||||
|
||||
let keysyms_per_keycode = self.keyboard_mapping.keysyms_per_keycode as usize;
|
||||
let offset = (keycode - min_keycode) as usize * keysyms_per_keycode;
|
||||
|
||||
// If the keysym is in position 0, no shift needed.
|
||||
// If it's in position 1, shift is needed.
|
||||
if offset < self.keyboard_mapping.keysyms.len()
|
||||
&& self.keyboard_mapping.keysyms[offset] == keysym
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if offset + 1 < self.keyboard_mapping.keysyms.len()
|
||||
&& self.keyboard_mapping.keysyms[offset + 1] == keysym
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// For uppercase letters, assume shift is needed.
|
||||
UPPERCASE_KEYSYMS.contains(&keysym)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
//! X11 implementation of computer use actions using the XTEST extension.
|
||||
|
||||
mod keyboard;
|
||||
mod mouse;
|
||||
mod screenshot;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use pathfinder_geometry::vector::Vector2I;
|
||||
use warpui::r#async::Timer;
|
||||
use x11rb::connection::Connection;
|
||||
use x11rb::protocol::xproto::{self, ConnectionExt as _};
|
||||
use x11rb::protocol::xtest::ConnectionExt as _;
|
||||
use x11rb::rust_connection::RustConnection;
|
||||
|
||||
use crate::{Action, ActionResult, Options};
|
||||
|
||||
/// An actor that performs computer use actions on X11.
|
||||
pub struct Actor {
|
||||
conn: RustConnection,
|
||||
screen_index: usize,
|
||||
/// Cached keyboard mapping for this connection. This avoids querying the server on every
|
||||
/// character typed.
|
||||
keyboard_mapping: xproto::GetKeyboardMappingReply,
|
||||
}
|
||||
|
||||
impl Actor {
|
||||
pub fn new() -> Result<Self, String> {
|
||||
let (conn, screen_index) =
|
||||
RustConnection::connect(None).map_err(|e| format!("Failed to connect to X11: {e}"))?;
|
||||
|
||||
// Verify XTEST extension is available. XTEST is part of the Xorg server and is
|
||||
// typically present by default. On Wayland or X servers without XTEST, this will
|
||||
// fail and computer use will not be available on X11.
|
||||
conn.xtest_get_version(2, 2)
|
||||
.map_err(|e| format!("XTEST extension not available: {e}"))?
|
||||
.reply()
|
||||
.map_err(|e| format!("XTEST extension query failed: {e}"))?;
|
||||
|
||||
// Pre-fetch and cache the keyboard mapping for this connection to avoid
|
||||
// round-trips for every character typed.
|
||||
let setup = conn.setup();
|
||||
let min_keycode = setup.min_keycode;
|
||||
let max_keycode = setup.max_keycode;
|
||||
let keyboard_mapping = conn
|
||||
.get_keyboard_mapping(min_keycode, max_keycode - min_keycode + 1)
|
||||
.map_err(|e| format!("Failed to get keyboard mapping: {e}"))?
|
||||
.reply()
|
||||
.map_err(|e| format!("Failed to get keyboard mapping reply: {e}"))?;
|
||||
|
||||
Ok(Self {
|
||||
conn,
|
||||
screen_index,
|
||||
keyboard_mapping,
|
||||
})
|
||||
}
|
||||
|
||||
fn root_window(&self) -> xproto::Window {
|
||||
self.conn.setup().roots[self.screen_index].root
|
||||
}
|
||||
|
||||
fn screen(&self) -> &xproto::Screen {
|
||||
&self.conn.setup().roots[self.screen_index]
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl crate::Actor for Actor {
|
||||
fn platform(&self) -> Option<crate::Platform> {
|
||||
Some(crate::Platform::LinuxX11)
|
||||
}
|
||||
|
||||
async fn perform_actions(
|
||||
&mut self,
|
||||
actions: &[Action],
|
||||
options: Options,
|
||||
) -> Result<ActionResult, String> {
|
||||
let mut mouse = mouse::Mouse::new(&self.conn, self.root_window());
|
||||
let mut keyboard = keyboard::Keyboard::new(&self.conn, &self.keyboard_mapping);
|
||||
let mut last_mouse_position: Option<Vector2I> = None;
|
||||
|
||||
for action in actions {
|
||||
match action {
|
||||
Action::Wait(duration) => {
|
||||
Timer::after(*duration).await;
|
||||
}
|
||||
Action::MouseDown { button, at } => {
|
||||
mouse.move_to(*at)?;
|
||||
mouse.focus_window_under_pointer()?;
|
||||
mouse.button_down(button)?;
|
||||
last_mouse_position = Some(*at);
|
||||
}
|
||||
Action::MouseUp { button } => {
|
||||
mouse.button_up(button)?;
|
||||
}
|
||||
Action::MouseMove { to } => {
|
||||
mouse.move_to(*to)?;
|
||||
last_mouse_position = Some(*to);
|
||||
}
|
||||
Action::MouseWheel {
|
||||
at,
|
||||
direction,
|
||||
distance,
|
||||
} => {
|
||||
mouse.move_to(*at)?;
|
||||
mouse.scroll(direction, distance)?;
|
||||
last_mouse_position = Some(*at);
|
||||
}
|
||||
Action::TypeText { text } => {
|
||||
keyboard.type_text(text)?;
|
||||
}
|
||||
Action::KeyDown { key } => {
|
||||
keyboard.key_down(key)?;
|
||||
}
|
||||
Action::KeyUp { key } => {
|
||||
keyboard.key_up(key)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let screenshot = if let Some(params) = options.screenshot_params {
|
||||
Some(screenshot::take(
|
||||
&self.conn,
|
||||
self.screen(),
|
||||
self.root_window(),
|
||||
params,
|
||||
)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Get the final mouse position.
|
||||
let cursor_position = if let Some(pos) = last_mouse_position {
|
||||
Some(pos)
|
||||
} else {
|
||||
Some(mouse.current_position()?)
|
||||
};
|
||||
|
||||
Ok(ActionResult {
|
||||
screenshot,
|
||||
cursor_position,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
//! Mouse input handling for X11 using XTEST.
|
||||
|
||||
use pathfinder_geometry::vector::Vector2I;
|
||||
use x11rb::connection::Connection;
|
||||
use x11rb::protocol::xproto::{self, ConnectionExt as _};
|
||||
use x11rb::protocol::xtest::ConnectionExt as _;
|
||||
use x11rb::rust_connection::RustConnection;
|
||||
|
||||
use crate::{MouseButton, ScrollDirection, ScrollDistance};
|
||||
|
||||
/// Mouse state tracking for an X11 connection.
|
||||
pub struct Mouse<'a> {
|
||||
conn: &'a RustConnection,
|
||||
root_window: xproto::Window,
|
||||
held_buttons: HeldButtons,
|
||||
}
|
||||
|
||||
impl<'a> Mouse<'a> {
|
||||
pub fn new(conn: &'a RustConnection, root_window: xproto::Window) -> Self {
|
||||
Self {
|
||||
conn,
|
||||
root_window,
|
||||
held_buttons: HeldButtons::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn move_to(&mut self, target: Vector2I) -> Result<(), String> {
|
||||
// Use WarpPointer to move the pointer. Unlike XTEST MotionNotify, this
|
||||
// reliably updates the server's pointer position, ensuring that subsequent
|
||||
// button events are delivered to the correct window.
|
||||
self.conn
|
||||
.warp_pointer(
|
||||
x11rb::NONE, // src_window (unconstrained)
|
||||
self.root_window, // dst_window (absolute coordinates)
|
||||
0, // src_x (unused)
|
||||
0, // src_y (unused)
|
||||
0, // src_width (unused)
|
||||
0, // src_height (unused)
|
||||
target.x() as i16,
|
||||
target.y() as i16,
|
||||
)
|
||||
.map_err(|e| format!("Failed to warp pointer: {e}"))?;
|
||||
|
||||
self.conn
|
||||
.flush()
|
||||
.map_err(|e| format!("Failed to flush X11 connection: {e}"))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn button_down(&mut self, button: &MouseButton) -> Result<(), String> {
|
||||
let x11_button = mouse_button_to_x11(button);
|
||||
self.held_buttons.set_down(button, true);
|
||||
|
||||
self.conn
|
||||
.xtest_fake_input(
|
||||
xproto::BUTTON_PRESS_EVENT,
|
||||
x11_button,
|
||||
x11rb::CURRENT_TIME,
|
||||
x11rb::NONE,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
.map_err(|e| format!("Failed to send button down: {e}"))?;
|
||||
|
||||
self.conn
|
||||
.flush()
|
||||
.map_err(|e| format!("Failed to flush X11 connection: {e}"))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn button_up(&mut self, button: &MouseButton) -> Result<(), String> {
|
||||
let x11_button = mouse_button_to_x11(button);
|
||||
self.held_buttons.set_down(button, false);
|
||||
|
||||
self.conn
|
||||
.xtest_fake_input(
|
||||
xproto::BUTTON_RELEASE_EVENT,
|
||||
x11_button,
|
||||
x11rb::CURRENT_TIME,
|
||||
x11rb::NONE,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
.map_err(|e| format!("Failed to send button up: {e}"))?;
|
||||
|
||||
self.conn
|
||||
.flush()
|
||||
.map_err(|e| format!("Failed to flush X11 connection: {e}"))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn scroll(
|
||||
&mut self,
|
||||
direction: &ScrollDirection,
|
||||
distance: &ScrollDistance,
|
||||
) -> Result<(), String> {
|
||||
// In X11, scroll is done via button presses. Buttons 4/5 are vertical scroll,
|
||||
// buttons 6/7 are horizontal scroll.
|
||||
let (button, count) = match (direction, distance) {
|
||||
(ScrollDirection::Up, ScrollDistance::Clicks(n)) => (4u8, *n),
|
||||
(ScrollDirection::Down, ScrollDistance::Clicks(n)) => (5u8, *n),
|
||||
(ScrollDirection::Left, ScrollDistance::Clicks(n)) => (6u8, *n),
|
||||
(ScrollDirection::Right, ScrollDistance::Clicks(n)) => (7u8, *n),
|
||||
// For pixel scrolling, approximate with clicks. X11 doesn't have native pixel scroll.
|
||||
(ScrollDirection::Up, ScrollDistance::Pixels(px)) => (4u8, (*px / 15).max(1)),
|
||||
(ScrollDirection::Down, ScrollDistance::Pixels(px)) => (5u8, (*px / 15).max(1)),
|
||||
(ScrollDirection::Left, ScrollDistance::Pixels(px)) => (6u8, (*px / 15).max(1)),
|
||||
(ScrollDirection::Right, ScrollDistance::Pixels(px)) => (7u8, (*px / 15).max(1)),
|
||||
};
|
||||
|
||||
for _ in 0..count.abs() {
|
||||
// Button press.
|
||||
self.conn
|
||||
.xtest_fake_input(
|
||||
xproto::BUTTON_PRESS_EVENT,
|
||||
button,
|
||||
x11rb::CURRENT_TIME,
|
||||
x11rb::NONE,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
.map_err(|e| format!("Failed to send scroll button press: {e}"))?;
|
||||
|
||||
// Button release.
|
||||
self.conn
|
||||
.xtest_fake_input(
|
||||
xproto::BUTTON_RELEASE_EVENT,
|
||||
button,
|
||||
x11rb::CURRENT_TIME,
|
||||
x11rb::NONE,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
.map_err(|e| format!("Failed to send scroll button release: {e}"))?;
|
||||
}
|
||||
|
||||
self.conn
|
||||
.flush()
|
||||
.map_err(|e| format!("Failed to flush X11 connection: {e}"))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sets input focus to the deepest window under the current pointer position.
|
||||
/// This simulates the click-to-focus behavior that a window manager would
|
||||
/// normally provide. Without a WM (e.g. in Xvfb), windows do not receive
|
||||
/// focus automatically, so keyboard and some button events may not be
|
||||
/// delivered correctly.
|
||||
pub fn focus_window_under_pointer(&self) -> Result<(), String> {
|
||||
let mut window = self.root_window;
|
||||
|
||||
// Walk down the window tree to find the deepest child under the pointer.
|
||||
loop {
|
||||
let reply = self
|
||||
.conn
|
||||
.query_pointer(window)
|
||||
.map_err(|e| format!("Failed to query pointer: {e}"))?
|
||||
.reply()
|
||||
.map_err(|e| format!("Failed to get pointer reply: {e}"))?;
|
||||
|
||||
if reply.child == x11rb::NONE {
|
||||
break;
|
||||
}
|
||||
window = reply.child;
|
||||
}
|
||||
|
||||
// Set input focus to the deepest window found.
|
||||
self.conn
|
||||
.set_input_focus(xproto::InputFocus::PARENT, window, x11rb::CURRENT_TIME)
|
||||
.map_err(|e| format!("Failed to set input focus: {e}"))?;
|
||||
|
||||
self.conn
|
||||
.flush()
|
||||
.map_err(|e| format!("Failed to flush X11 connection: {e}"))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn current_position(&mut self) -> Result<Vector2I, String> {
|
||||
let reply = self
|
||||
.conn
|
||||
.query_pointer(self.root_window)
|
||||
.map_err(|e| format!("Failed to query pointer: {e}"))?
|
||||
.reply()
|
||||
.map_err(|e| format!("Failed to get pointer reply: {e}"))?;
|
||||
|
||||
let pos = Vector2I::new(reply.root_x as i32, reply.root_y as i32);
|
||||
Ok(pos)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct HeldButtons {
|
||||
left: bool,
|
||||
right: bool,
|
||||
middle: bool,
|
||||
back: bool,
|
||||
forward: bool,
|
||||
}
|
||||
|
||||
impl HeldButtons {
|
||||
fn set_down(&mut self, button: &MouseButton, down: bool) {
|
||||
match button {
|
||||
MouseButton::Left => self.left = down,
|
||||
MouseButton::Right => self.right = down,
|
||||
MouseButton::Middle => self.middle = down,
|
||||
MouseButton::Back => self.back = down,
|
||||
MouseButton::Forward => self.forward = down,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn mouse_button_to_x11(button: &MouseButton) -> u8 {
|
||||
match button {
|
||||
MouseButton::Left => 1,
|
||||
MouseButton::Middle => 2,
|
||||
MouseButton::Right => 3,
|
||||
MouseButton::Back => 8,
|
||||
MouseButton::Forward => 9,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
//! Screenshot capture for X11.
|
||||
|
||||
use x11rb::protocol::xproto::{self, ConnectionExt as _, ImageFormat};
|
||||
use x11rb::rust_connection::RustConnection;
|
||||
|
||||
use crate::{Screenshot, ScreenshotParams};
|
||||
|
||||
/// Takes a screenshot of the root window or a region of it.
|
||||
pub fn take(
|
||||
conn: &RustConnection,
|
||||
screen: &xproto::Screen,
|
||||
root: xproto::Window,
|
||||
params: ScreenshotParams,
|
||||
) -> Result<Screenshot, String> {
|
||||
// Determine the capture region.
|
||||
let (x, y, width, height) = if let Some(region) = params.region {
|
||||
region.validate()?;
|
||||
let x = region.top_left.x() as i16;
|
||||
let y = region.top_left.y() as i16;
|
||||
let width = (region.bottom_right.x() - region.top_left.x()) as u16;
|
||||
let height = (region.bottom_right.y() - region.top_left.y()) as u16;
|
||||
(x, y, width, height)
|
||||
} else {
|
||||
(0, 0, screen.width_in_pixels, screen.height_in_pixels)
|
||||
};
|
||||
|
||||
// Get the image from the root window.
|
||||
// TODO: Consider compositing the cursor into the screenshot in the future.
|
||||
let image = conn
|
||||
.get_image(
|
||||
ImageFormat::Z_PIXMAP,
|
||||
root,
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
!0, // plane_mask: all planes
|
||||
)
|
||||
.map_err(|e| format!("Failed to request screenshot: {e}"))?
|
||||
.reply()
|
||||
.map_err(|e| format!("Failed to get screenshot reply: {e}"))?;
|
||||
|
||||
// Convert the X11 image data to an image::RgbImage.
|
||||
// X11 typically returns BGRA or BGR depending on depth.
|
||||
let depth = image.depth;
|
||||
let data = image.data;
|
||||
|
||||
let rgb_data = convert_x11_image_to_rgb(&data, width as usize, height as usize, depth)?;
|
||||
|
||||
let img = image::RgbImage::from_raw(width as u32, height as u32, rgb_data)
|
||||
.ok_or("Failed to create image from raw data")?;
|
||||
|
||||
let img = image::DynamicImage::ImageRgb8(img);
|
||||
|
||||
crate::screenshot_utils::process_screenshot(img, params)
|
||||
}
|
||||
|
||||
/// Converts X11 image data (typically BGRA or BGR) to RGB.
|
||||
fn convert_x11_image_to_rgb(
|
||||
data: &[u8],
|
||||
width: usize,
|
||||
height: usize,
|
||||
depth: u8,
|
||||
) -> Result<Vec<u8>, String> {
|
||||
let mut rgb = Vec::with_capacity(width * height * 3);
|
||||
|
||||
match depth {
|
||||
24 => {
|
||||
// 24-bit: BGR format, 3 bytes per pixel (but often padded to 4).
|
||||
// X11 often uses 32-bit alignment even for 24-bit depth.
|
||||
let bytes_per_pixel = if data.len() >= width * height * 4 {
|
||||
4
|
||||
} else {
|
||||
3
|
||||
};
|
||||
|
||||
for y in 0..height {
|
||||
for x in 0..width {
|
||||
let offset = (y * width + x) * bytes_per_pixel;
|
||||
if offset + 2 < data.len() {
|
||||
let b = data[offset];
|
||||
let g = data[offset + 1];
|
||||
let r = data[offset + 2];
|
||||
rgb.push(r);
|
||||
rgb.push(g);
|
||||
rgb.push(b);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
32 => {
|
||||
// 32-bit: BGRA format, 4 bytes per pixel.
|
||||
for y in 0..height {
|
||||
for x in 0..width {
|
||||
let offset = (y * width + x) * 4;
|
||||
if offset + 2 < data.len() {
|
||||
let b = data[offset];
|
||||
let g = data[offset + 1];
|
||||
let r = data[offset + 2];
|
||||
// Skip alpha at offset + 3.
|
||||
rgb.push(r);
|
||||
rgb.push(g);
|
||||
rgb.push(b);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
return Err(format!("Unsupported screen depth: {depth}"));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(rgb)
|
||||
}
|
||||
Reference in New Issue
Block a user