public/galaxy
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Initial public release of Warp.

Browse Source 
Repo-Sync-Origin: warpdotdev/warp-internal@12af1d983b
This commit is contained in:
David Stern
2026-04-28 08:43:33 -05:00
commit 0dbd3d567a
4982 changed files with 1431549 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/computer_use/src/windows/dpi.rs
+47
View File
@@ -0,0 +1,47 @@
//! Thread-local DPI-awareness helper for Windows computer_use actions.
//!
//! Several Win32 APIs this crate calls (`SetCursorPos`, `GetCursorPos`,
//! `GetSystemMetrics(SM_*VIRTUALSCREEN)`, `BitBlt`, …) return *logical* coordinates if the
//! calling thread is not DPI-aware, which causes mis-located clicks and scaled/cropped screenshots
//! on HiDPI monitors.
//!
//! Rather than relying on the host process manifest, we opt every computer_use operation into
//! per-monitor-v2 awareness for the duration of the call via [`DpiAwarenessGuard`].
use windows::Win32::UI::HiDpi::{
DPI_AWARENESS_CONTEXT, DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2, SetThreadDpiAwarenessContext,
};
/// RAII guard that requests per-monitor-v2 DPI awareness for the current thread and restores the
/// previous context when dropped.
///
/// Requires Windows 10 version 1703 or newer (when `PER_MONITOR_AWARE_V2` shipped — 1607 only
/// had V1). On older systems (or when the process awareness cannot be overridden)
/// `SetThreadDpiAwarenessContext` returns a null context; in that case this guard is a no-op.
pub(super) struct DpiAwarenessGuard {
previous: Option<DPI_AWARENESS_CONTEXT>,
}
impl DpiAwarenessGuard {
/// Enters per-monitor-v2 DPI awareness for the calling thread.
pub(super) fn enter_per_monitor_v2() -> Self {
// SAFETY: `SetThreadDpiAwarenessContext` has no preconditions and mutates only
// thread-local state.
let prev =
unsafe { SetThreadDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) };
let previous = if prev.0.is_null() { None } else { Some(prev) };
Self { previous }
}
}
impl Drop for DpiAwarenessGuard {
fn drop(&mut self) {
if let Some(prev) = self.previous {
// SAFETY: `prev` was returned by a prior successful call to
// `SetThreadDpiAwarenessContext` on this same thread.
unsafe {
let _ = SetThreadDpiAwarenessContext(prev);
}
}
}
}
crates/computer_use/src/windows/keyboard.rs
+545
View File
@@ -0,0 +1,545 @@
//! Keyboard input handling for Windows using SendInput.
use std::collections::HashMap;
use std::mem::size_of;
use std::ptr;
use windows::Win32::Foundation::GetLastError;
use windows::Win32::UI::Input::KeyboardAndMouse::{
GetKeyboardLayout, HKL, INPUT, INPUT_0, INPUT_KEYBOARD, KEYBD_EVENT_FLAGS, KEYBDINPUT,
KEYEVENTF_EXTENDEDKEY, KEYEVENTF_KEYUP, KEYEVENTF_SCANCODE, KEYEVENTF_UNICODE, MAPVK_VK_TO_VSC,
MapVirtualKeyExW, SendInput, VIRTUAL_KEY, VK_LSHIFT, VK_RSHIFT, VK_SHIFT, VkKeyScanExW,
};
use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowThreadProcessId};
use crate::Key;
/// How a logical [`Key`] was resolved for dispatch.
enum ResolvedKey {
/// Dispatch via virtual-key code / scan code. May auto-press `VK_SHIFT`.
Vk { vk: u16, needs_shift: bool },
/// Dispatch as a UTF-16 code unit via `KEYEVENTF_UNICODE`. Used as a fallback when the layout
/// would require ctrl/alt to produce the character (e.g., AltGr-accessed keys on European
/// layouts). Unicode input bypasses the keyboard layout entirely, so no modifier handling is
/// needed.
Unicode(u16),
}
/// Bookkeeping for a logical key we've sent a down event for. We store the *resolved* state at
/// `key_down` time so that `key_up` can release exactly what we pressed, even if the active
/// keyboard layout has since changed (e.g., the user hit the IME/layout-switch hotkey between down
/// and up).
enum PressedKey {
/// Key was dispatched via virtual-key code / scan code.
Vk {
/// Virtual-key code that was dispatched on `key_down`.
vk: u16,
/// Whether we auto-pressed `VK_SHIFT` for this key; the matching `key_up` is responsible
/// for releasing shift when the last auto-shifted entry goes away.
auto_shifted: bool,
},
/// Key was dispatched as a UTF-16 code unit via `KEYEVENTF_UNICODE`. Release sends the same
/// unit up. No shift bookkeeping because Unicode input bypasses the keyboard layout.
Unicode(u16),
}
/// Manages keyboard state and posts keyboard events to the system.
///
/// Callers must pair each `key_down` with a matching `key_up` for the same logical key before the
/// next `key_down` on that key. `pressed_keys` is keyed by the original logical `Key`, so repeated
/// `key_down` of the same key without an intervening `key_up` overwrites the earlier bookkeeping.
///
/// **Auto-shift contract**: only the *first* `Key::Char` that requires shift while no shift is
/// already held is recorded as the shift "owner" (`auto_shifted: true`). Subsequent shifted chars
/// pressed while shift remains held ride on that first press (`auto_shifted: false`). Releasing
/// the first char releases `VK_SHIFT`, leaving the later chars physically held without shift — the
/// OS will produce unshifted output for them on their eventual `key_up`. Callers that need
/// multiple shifted chars held simultaneously should use `Key::Keycode(VK_SHIFT.0)` directly.
pub struct Keyboard {
/// Logical keys currently pressed, keyed by the caller-supplied `Key`. Storing the resolved VK
/// and auto-shift flag here — rather than re-resolving in `key_up` — ensures we release the
/// exact key we pressed even if the active keyboard layout changes between the two calls.
pressed_keys: HashMap<Key, PressedKey>,
/// Set to `true` when a synthetic `VK_SHIFT` release dispatch failed, meaning shift may still
/// be held in the OS with no `pressed_keys` entry to release it. We retry the release at the
/// top of every subsequent `key_down` / `key_up` until it succeeds.
pending_shift_release: bool,
}
impl Default for Keyboard {
fn default() -> Self {
Self::new()
}
}
impl Keyboard {
pub fn new() -> Self {
Self {
pressed_keys: HashMap::new(),
pending_shift_release: false,
}
}
/// Retries a previously-failed synthetic shift release if one is outstanding. Called at the
/// top of every public mutating entrypoint so a transient `SendInput` failure can't leave
/// shift stuck across the rest of the session.
fn flush_pending_shift_release(&mut self, hkl: HKL) {
if self.pending_shift_release && send_vk(VK_SHIFT.0, true, hkl).is_ok() {
self.pending_shift_release = false;
}
}
/// Whether any currently-pressed logical key has `auto_shifted: true` — i.e., this
/// `Keyboard` is responsible for the `VK_SHIFT` currently held down. Short-circuits on the
/// first match, unlike a count-based check.
fn has_auto_shifted_press(&self) -> bool {
self.pressed_keys.values().any(|p| {
matches!(
p,
PressedKey::Vk {
auto_shifted: true,
..
}
)
})
}
/// Whether any currently-pressed logical key is an explicit shift keycode
/// (`Key::Keycode(VK_SHIFT | VK_LSHIFT | VK_RSHIFT)`). Used to avoid
/// synthesizing shift presses/releases on top of the caller's own shift state.
fn explicit_shift_held(&self) -> bool {
self.pressed_keys.values().any(|p| {
matches!(
p,
PressedKey::Vk { vk, auto_shifted: false } if is_shift_vk(*vk)
)
})
}
/// Sends a key down event for the given key.
///
/// For `Key::Char`, this will automatically press shift if needed. We skip the synthetic
/// shift press if the caller is already holding shift explicitly (via
/// `Key::Keycode(VK_SHIFT)`), and conversely `key_up` won't synthesize a shift release while
/// that explicit shift entry is still tracked. The shift press and the main VK press are
/// batched into a single `SendInput` call so other input can't be interleaved in the common
/// case. `SendInput` can still partially succeed (e.g., UIPI blocks the main-VK entry after
/// shift was already delivered); when that happens we best-effort release shift before
/// returning, and if that release itself fails we mark it pending so a subsequent call can
/// retry.
pub fn key_down(&mut self, key: &Key) -> Result<(), String> {
// Resolve the foreground window's keyboard layout once per public call so every
// `SendInput` entry we build against it (shift + main VK) sees a consistent snapshot and
// we avoid three redundant Win32 queries per INPUT.
let hkl = foreground_keyboard_layout();
self.flush_pending_shift_release(hkl);
let resolved = resolve_key(key, hkl)?;
match resolved {
ResolvedKey::Vk { vk, needs_shift } => self.key_down_vk(key, vk, needs_shift, hkl),
ResolvedKey::Unicode(unit) => {
// Unicode dispatch bypasses the keyboard layout, so no shift bookkeeping is
// required. A single down event is sufficient; `key_up` will send the matching
// up event using the unit we record here.
send_inputs(&[make_unicode_input(unit, false)])?;
self.pressed_keys
.insert(key.clone(), PressedKey::Unicode(unit));
Ok(())
}
}
}
/// Shared `Key::Keycode` / shift-auto `Key::Char` path for [`key_down`].
fn key_down_vk(
&mut self,
key: &Key,
vk: u16,
needs_shift: bool,
hkl: HKL,
) -> Result<(), String> {
// Only send a fresh shift press if no other pressed key is already holding shift, whether
// auto-shifted by us or explicitly pressed by the caller.
let shift_already_held = self.has_auto_shifted_press() || self.explicit_shift_held();
let pressed_shift_now = needs_shift && !shift_already_held;
let mut inputs: Vec<INPUT> = Vec::with_capacity(2);
if pressed_shift_now {
inputs.push(build_vk_input(VK_SHIFT.0, false, hkl));
}
inputs.push(build_vk_input(vk, false, hkl));
// Dispatch first; only record the press after `SendInput` succeeds so `pressed_keys`
// never reflects a press we didn't actually send.
let (sent, result) = send_inputs_tracked(&inputs);
if let Err(e) = result {
// Partial-send: only compensate for shift if it was actually dispatched. When
// `sent == 0` `SendInput` failed before queueing anything (e.g., UIPI block on the
// first event), so the shift `INPUT` never reached the OS and synthesizing a
// `VK_SHIFT` up here would spuriously release the real user's shift if they happen
// to be holding it. The shift entry is always `inputs[0]` when `pressed_shift_now`,
// so `sent >= 1` tells us it went through.
if pressed_shift_now && sent >= 1 && send_vk(VK_SHIFT.0, true, hkl).is_err() {
self.pending_shift_release = true;
}
return Err(e);
}
// `auto_shifted` records whether *we* actually pressed shift for this key, not whether
// the key needed shift. Otherwise if another source of shift (explicit `VK_SHIFT`
// keycode, earlier auto-shifted key) was already down and released before this key,
// `key_up` would synthesize a spurious `VK_SHIFT` release that the OS never asked for.
//
// If a caller violates the pair-each-down-with-an-up contract and issues two `key_down`s
// for the same logical key, preserve the `auto_shifted: true` bit so the matching
// `key_up` still releases the shift we pressed originally.
let already_auto_shifted = matches!(
self.pressed_keys.get(key),
Some(PressedKey::Vk {
auto_shifted: true,
..
})
);
self.pressed_keys.insert(
key.clone(),
PressedKey::Vk {
vk,
auto_shifted: pressed_shift_now || already_auto_shifted,
},
);
Ok(())
}
/// Sends a key up event for the given key.
///
/// Uses the VK recorded at `key_down` time (not a fresh resolution against the current
/// keyboard layout), so a mid-action layout switch still releases the key we originally
/// pressed. If no prior `key_down` is tracked, we fall back to resolving now for best-effort
/// delivery.
///
/// The shift release is attempted even when the primary key-up fails so that a single
/// `SendInput` failure does not leave shift stuck. If the shift release itself fails, we
/// mark it pending so the next `key_down` / `key_up` retries it.
pub fn key_up(&mut self, key: &Key) -> Result<(), String> {
let hkl = foreground_keyboard_layout();
self.flush_pending_shift_release(hkl);
let Some(pressed) = self.pressed_keys.remove(key) else {
// No recorded press; resolve against the current layout as a best-effort fallback so a
// stray `key_up` still reaches the OS.
return match resolve_key(key, hkl)? {
ResolvedKey::Vk { vk, .. } => send_vk(vk, true, hkl),
ResolvedKey::Unicode(unit) => send_inputs(&[make_unicode_input(unit, true)]),
};
};
match pressed {
PressedKey::Unicode(unit) => {
// Unicode dispatch has no shift bookkeeping; just send the matching up.
send_inputs(&[make_unicode_input(unit, true)])
}
PressedKey::Vk { vk, auto_shifted } => {
let primary = send_vk(vk, true, hkl);
// Attempt shift release regardless of whether the primary key-up succeeded, so a
// single `SendInput` failure can't leave shift stuck. Only release if this was
// an auto-shifted key, no other auto-shifted keys remain, and the caller isn't
// holding shift explicitly.
let should_release_shift =
auto_shifted && !self.has_auto_shifted_press() && !self.explicit_shift_held();
let shift_result = if should_release_shift {
match send_vk(VK_SHIFT.0, true, hkl) {
Ok(()) => Ok(()),
Err(e) => {
// Mark the release as pending so the next call retries. Without
// this, shift stays held in the OS with nothing left in
// `pressed_keys` to release it.
self.pending_shift_release = true;
Err(e)
}
}
} else {
Ok(())
};
// Report the primary failure first, falling back to the shift-release failure
// if the primary succeeded.
primary.and(shift_result)
}
}
}
/// Simulates typing text by sending Unicode keyboard events.
///
/// Using `KEYEVENTF_UNICODE` bypasses the keyboard layout and works with any character the
/// target application can accept as Unicode input. The entire string is batched into a single
/// `SendInput` call so the OS cannot interleave other input between characters.
///
/// Takes `&mut self` so it can `flush_pending_shift_release` like `key_down`/`key_up`,
/// otherwise a stuck auto-shift from a prior failed release would persist through the whole
/// typing call and on into any non-keyboard actions that follow.
pub fn type_text(&mut self, text: &str) -> Result<(), String> {
self.flush_pending_shift_release(foreground_keyboard_layout());
// Each UTF-16 code unit produces one down + one up INPUT. The UTF-8 byte length is a
// valid upper bound on the UTF-16 unit count (single-byte ASCII → 1 unit, 2-byte → 1
// unit, 3-byte BMP → 1 unit, 4-byte supplementary → 2 units), so `bytes * 2` never
// under-counts. This over-allocates ~3x for 3-byte UTF-8 strings (CJK, Cyrillic) but
// avoids an extra O(n) `chars().count()` pass just to size the buffer.
let mut inputs: Vec<INPUT> = Vec::with_capacity(text.len().saturating_mul(2));
for ch in text.chars() {
let mut buf = [0u16; 2];
let encoded = ch.encode_utf16(&mut buf);
// Emit a down/up pair per UTF-16 unit. `KEYEVENTF_UNICODE` delivered via
// `TranslateMessage` / `WM_CHAR` expects each surrogate as its own down/up pair;
// emitting all downs then all ups has been observed to drop half of the sequence in
// some targets.
for &unit in encoded.iter() {
inputs.push(make_unicode_input(unit, false));
inputs.push(make_unicode_input(unit, true));
}
}
send_inputs(&inputs)
}
}
/// Resolves a `Key` to either a virtual-key dispatch (with optional auto-shift) or a Unicode
/// code-unit dispatch.
fn resolve_key(key: &Key, hkl: HKL) -> Result<ResolvedKey, String> {
match key {
Key::Keycode(code) => {
let vk = u16::try_from(*code).map_err(|_| {
format!(
"Invalid virtual-key code {code}: must be in range 0..={}",
u16::MAX
)
})?;
// For explicit VKs, the caller manages modifiers.
Ok(ResolvedKey::Vk {
vk,
needs_shift: false,
})
}
Key::Char(ch) => resolve_char(*ch, hkl),
}
}
/// Resolves a character to either a VK (with optional shift) or a Unicode code-unit dispatch,
/// using the given keyboard layout handle (typically the foreground window's). This matches what
/// a real keystroke would look like to the target application when the user is running a
/// different input language / IME than Warp's thread.
///
/// Falls back to `ResolvedKey::Unicode` when the layout would require ctrl/alt to produce the
/// character (e.g., AltGr-accessed keys on several European layouts) so `Key::Char` remains
/// portable across layouts instead of erroring out.
fn resolve_char(ch: char, hkl: HKL) -> Result<ResolvedKey, String> {
// VkKeyScanExW only supports characters in the BMP (single UTF-16 unit). Supplementary-plane
// characters still work via the Unicode path (they'd need a surrogate pair there, which is
// what `type_text` handles); `Key::Char` is a single `char` so callers can't currently
// express a supplementary-plane key event here.
let mut buf = [0u16; 2];
let encoded = ch.encode_utf16(&mut buf);
if encoded.len() != 1 {
return Err(format!(
"Character '{ch}' is outside the Basic Multilingual Plane (BMP); use TypeText for emoji and other supplementary-plane characters"
));
}
let unit = encoded[0];
// SAFETY: `VkKeyScanExW` is a pure query and is safe to call from any thread; `hkl` is
// either a valid HKL or null (null falls back to the calling thread's layout).
let result = unsafe { VkKeyScanExW(unit, hkl) };
if result == -1 {
// No VK mapping at all in this layout; fall back to Unicode dispatch.
return Ok(ResolvedKey::Unicode(unit));
}
// Low byte is the VK code; high byte is the shift state.
// bit 0: shift, bit 1: ctrl, bit 2: alt.
let bytes = result.to_le_bytes();
let vk = bytes[0] as u16;
let shift_state = bytes[1];
let needs_shift = (shift_state & 0x01) != 0;
let needs_ctrl = (shift_state & 0x02) != 0;
let needs_alt = (shift_state & 0x04) != 0;
if needs_ctrl || needs_alt {
// Character requires ctrl and/or alt (e.g., AltGr on European layouts). Synthesizing
// those modifiers can also trigger unwanted shortcuts in the target app, so fall back
// to layout-bypassing Unicode dispatch instead.
return Ok(ResolvedKey::Unicode(unit));
}
Ok(ResolvedKey::Vk { vk, needs_shift })
}
/// Builds the `INPUT` record for a single key down or key up event on the given virtual-key
/// code, without dispatching it. See [`send_vk`] for the full description of the scan-code
/// translation. The caller supplies the target keyboard layout so shift + main-VK entries built
/// for the same public call can share a consistent snapshot.
fn build_vk_input(vk: u16, is_up: bool, hkl: HKL) -> INPUT {
// SAFETY: `MapVirtualKeyExW` has no preconditions; reads the given HKL (null = calling
// thread's layout) and returns 0 if no mapping exists.
let scan = unsafe { MapVirtualKeyExW(vk as u32, MAPVK_VK_TO_VSC, Some(hkl)) } as u16;
let mut flag_bits: u32 = 0;
let (w_vk, w_scan) = if scan != 0 {
flag_bits |= KEYEVENTF_SCANCODE.0;
if is_extended_vk(vk) {
flag_bits |= KEYEVENTF_EXTENDEDKEY.0;
}
(0u16, scan)
} else {
// No scan-code mapping for this VK; dispatch by virtual-key code.
(vk, 0u16)
};
if is_up {
flag_bits |= KEYEVENTF_KEYUP.0;
}
INPUT {
r#type: INPUT_KEYBOARD,
Anonymous: INPUT_0 {
ki: KEYBDINPUT {
wVk: VIRTUAL_KEY(w_vk),
wScan: w_scan,
dwFlags: KEYBD_EVENT_FLAGS(flag_bits),
time: 0,
dwExtraInfo: 0,
},
},
}
}
/// Sends a single key down or key up event for the given virtual-key code, resolved against the
/// given keyboard layout.
///
/// We translate the virtual-key code to a hardware scan code via `MapVirtualKeyExW` and dispatch
/// with `KEYEVENTF_SCANCODE` (plus `KEYEVENTF_EXTENDEDKEY` for keys that require the 0xE0
/// prefix). This reaches targets that filter synthesized VK-only events (games, some
/// remote-desktop clients). The OS still translates the scan code back into the corresponding
/// virtual-key code for standard window messages, so VK-reading consumers are unaffected. If no
/// scan-code mapping exists we fall back to VK-only dispatch.
fn send_vk(vk: u16, is_up: bool, hkl: HKL) -> Result<(), String> {
send_inputs(&[build_vk_input(vk, is_up, hkl)])
}
/// Returns true if `vk` is one of the shift virtual-key codes (generic / left / right).
fn is_shift_vk(vk: u16) -> bool {
vk == VK_SHIFT.0 || vk == VK_LSHIFT.0 || vk == VK_RSHIFT.0
}
/// Returns the keyboard layout (`HKL`) currently active on the foreground window's thread,
/// falling back to the calling thread's layout (HKL `0`) if there is no foreground window. Using
/// the foreground window's HKL makes `Key::Char` resolution match what a real keystroke would
/// produce for the target application, which matters in multilingual setups where Warp's thread
/// layout can differ from the app's.
fn foreground_keyboard_layout() -> HKL {
// SAFETY: `GetForegroundWindow` has no preconditions; returns null if no foreground window.
let hwnd = unsafe { GetForegroundWindow() };
if hwnd.0.is_null() {
// SAFETY: `GetKeyboardLayout(0)` returns the calling thread's layout.
return unsafe { GetKeyboardLayout(0) };
}
// SAFETY: `hwnd` is a valid window handle; we pass a null `lpdwProcessId`.
let thread_id = unsafe { GetWindowThreadProcessId(hwnd, Some(ptr::null_mut())) };
// SAFETY: `GetKeyboardLayout` has no preconditions; 0 means "calling thread's layout".
unsafe { GetKeyboardLayout(thread_id) }
}
/// Returns true if the given virtual-key code is an "extended" key (scan code prefixed with
/// 0xE0). `MapVirtualKeyW(MAPVK_VK_TO_VSC)` strips the 0xE0 prefix, so we set
/// `KEYEVENTF_EXTENDEDKEY` ourselves for these VKs.
fn is_extended_vk(vk: u16) -> bool {
// Values from <winuser.h>. See "About Keyboard Input" on MSDN for the canonical list of
// extended keys.
const VK_PRIOR: u16 = 0x21;
const VK_NEXT: u16 = 0x22;
const VK_END: u16 = 0x23;
const VK_HOME: u16 = 0x24;
const VK_LEFT: u16 = 0x25;
const VK_UP: u16 = 0x26;
const VK_RIGHT: u16 = 0x27;
const VK_DOWN: u16 = 0x28;
const VK_SNAPSHOT: u16 = 0x2C;
const VK_INSERT: u16 = 0x2D;
const VK_DELETE: u16 = 0x2E;
const VK_LWIN: u16 = 0x5B;
const VK_RWIN: u16 = 0x5C;
const VK_APPS: u16 = 0x5D;
const VK_DIVIDE: u16 = 0x6F;
const VK_NUMLOCK: u16 = 0x90;
const VK_RCONTROL: u16 = 0xA3;
const VK_RMENU: u16 = 0xA5;
matches!(
vk,
VK_PRIOR
| VK_NEXT
| VK_END
| VK_HOME
| VK_LEFT
| VK_UP
| VK_RIGHT
| VK_DOWN
| VK_SNAPSHOT
| VK_INSERT
| VK_DELETE
| VK_LWIN
| VK_RWIN
| VK_APPS
| VK_DIVIDE
| VK_NUMLOCK
| VK_RCONTROL
| VK_RMENU
)
}
fn make_unicode_input(unit: u16, is_up: bool) -> INPUT {
let flags = if is_up {
KEYBD_EVENT_FLAGS(KEYEVENTF_UNICODE.0 | KEYEVENTF_KEYUP.0)
} else {
KEYEVENTF_UNICODE
};
INPUT {
r#type: INPUT_KEYBOARD,
Anonymous: INPUT_0 {
ki: KEYBDINPUT {
wVk: VIRTUAL_KEY(0),
wScan: unit,
dwFlags: flags,
time: 0,
dwExtraInfo: 0,
},
},
}
}
/// Dispatches a batch of `INPUT` events via `SendInput`.
fn send_inputs(inputs: &[INPUT]) -> Result<(), String> {
send_inputs_tracked(inputs).1
}
/// Dispatches a batch of `INPUT` events via `SendInput`, returning the number of events the OS
/// actually queued alongside the pass/fail `Result`. Callers that need to take compensating
/// action keyed off partial delivery (e.g., "did the shift entry get through?") can branch on
/// the `sent` count; callers that only care about pass/fail can use [`send_inputs`] directly.
fn send_inputs_tracked(inputs: &[INPUT]) -> (u32, Result<(), String>) {
if inputs.is_empty() {
return (0, Ok(()));
}
// SAFETY: `inputs` is a valid slice of `INPUT` with the correct element size, and `SendInput`
// does not retain the pointer beyond the call.
let sent = unsafe { SendInput(inputs, size_of::<INPUT>() as i32) };
if sent as usize != inputs.len() {
// SAFETY: `GetLastError` has no preconditions; reads the calling thread's last-error.
let last_error = unsafe { GetLastError() }.0;
return (
sent,
Err(format!(
"SendInput dispatched only {sent}/{} keyboard events \
(GetLastError={last_error}, blocked by UIPI or other input?)",
inputs.len(),
)),
);
}
(sent, Ok(()))
}
crates/computer_use/src/windows/mod.rs
+160
View File
@@ -0,0 +1,160 @@
//! Windows implementation of computer use actions using the Win32 SendInput
//! API for input and GDI for screenshots.
mod dpi;
mod keyboard;
mod mouse;
mod screenshot;
use async_trait::async_trait;
use warpui::r#async::Timer;
use windows::Win32::System::StationsAndDesktops::{
CloseDesktop, DESKTOP_ACCESS_FLAGS, DESKTOP_CONTROL_FLAGS, HDESK, OpenInputDesktop,
};
use crate::{Action, ActionResult, Options};
/// Returns whether computer_use can drive input on this machine right now.
///
/// Reports `false` when there is no accessible input desktop (e.g., the process is running under
/// Session 0 as a Windows service, the workstation is locked, or the user has switched to a
/// different secure desktop). In those cases `SendInput` silently no-ops and GDI desktop capture
/// fails, so we'd rather fail fast here than surface the error mid-action.
pub fn is_supported_on_current_platform() -> bool {
probe_input_desktop_available()
}
/// Shared probe used by both [`is_supported_on_current_platform`] and [`Actor::new`] so the
/// "can we drive input right now?" logic lives in one place. This still runs the probe on each
/// call (it's a cheap `OpenInputDesktop` / `CloseDesktop` round-trip) — we don't cache it because
/// availability can change at runtime (workstation lock, secure desktop swap, Remote Desktop
/// reconnect).
fn probe_input_desktop_available() -> bool {
InputDesktop::acquire().is_some()
}
/// RAII wrapper for an `HDESK` returned by `OpenInputDesktop`. Guarantees the handle is closed
/// (or at least that a close attempt is made and logged on failure) even if the caller returns
/// early. Modeled after the GDI handle guards in `screenshot.rs`.
struct InputDesktop(HDESK);
impl InputDesktop {
fn acquire() -> Option<Self> {
// SAFETY: `OpenInputDesktop` has no preconditions. We pass `false` for inheritance and
// request no specific access (just probing for existence).
let handle =
unsafe { OpenInputDesktop(DESKTOP_CONTROL_FLAGS(0), false, DESKTOP_ACCESS_FLAGS(0)) };
handle.ok().map(Self)
}
}
impl Drop for InputDesktop {
fn drop(&mut self) {
// SAFETY: `self.0` is a valid HDESK returned by `OpenInputDesktop` and has not been
// closed yet.
unsafe {
if let Err(e) = CloseDesktop(self.0) {
log::warn!("CloseDesktop failed in InputDesktop::drop: {e}");
}
}
}
}
/// Actor holds Keyboard/Mouse state unconditionally — both are cheap to construct and have no
/// side effects — so a `perform_actions` call can recover as soon as an input desktop is
/// reachable again, even if `Actor::new` ran while the desktop was temporarily inaccessible
/// (workstation locked at startup, RDP disconnect, etc.). Supportability is decided per call by
/// [`probe_input_desktop_available`] rather than being cached in the actor's shape.
pub struct Actor {
keyboard: keyboard::Keyboard,
mouse: mouse::Mouse,
}
impl Actor {
pub fn new() -> Self {
Self {
keyboard: keyboard::Keyboard::new(),
mouse: mouse::Mouse::new(),
}
}
}
impl Default for Actor {
fn default() -> Self {
Self::new()
}
}
/// Error returned by `perform_actions` when the input desktop is inaccessible at call time
/// (workstation lock, secure desktop swap, RDP reconnect, Session 0 service, …).
const NO_INPUT_DESKTOP_ERROR: &str = "Computer use is not available: no accessible input desktop";
#[async_trait]
impl super::Actor for Actor {
fn platform(&self) -> Option<super::Platform> {
// Live probe so callers can use `platform().is_some()` as a current "can drive input"
// signal. Matches the Linux `Unsupported`-returns-None convention.
if probe_input_desktop_available() {
Some(super::Platform::Windows)
} else {
None
}
}
async fn perform_actions(
&mut self,
actions: &[Action],
options: Options,
) -> Result<ActionResult, String> {
// Probe at the top of every call so transient loss of the input desktop (workstation
// lock, secure desktop swap, RDP reconnect) surfaces as a descriptive error instead of
// letting `SendInput` silently no-op. Cheap `OpenInputDesktop`/`CloseDesktop` round-trip.
if !probe_input_desktop_available() {
return Err(NO_INPUT_DESKTOP_ERROR.to_string());
}
let keyboard = &mut self.keyboard;
let mouse = &mut self.mouse;
for action in actions {
match action {
Action::Wait(duration) => {
Timer::after(*duration).await;
}
Action::MouseDown { button, at } => {
mouse.move_to(*at)?;
mouse.button_down(button)?;
}
Action::MouseUp { button } => mouse.button_up(button)?,
Action::MouseMove { to } => mouse.move_to(*to)?,
Action::MouseWheel {
at,
direction,
distance,
} => {
mouse.move_to(*at)?;
mouse.scroll(direction, distance)?;
}
Action::TypeText { text } => {
keyboard.type_text(text)?;
}
Action::KeyDown { key } => {
keyboard.key_down(key)?;
}
Action::KeyUp { key } => {
keyboard.key_up(key)?;
}
}
}
let screenshot = if let Some(params) = options.screenshot_params {
Some(screenshot::take(params)?)
} else {
None
};
Ok(ActionResult {
screenshot,
cursor_position: Some(mouse.current_position()?),
})
}
}
crates/computer_use/src/windows/mouse.rs
+352
View File
@@ -0,0 +1,352 @@
//! Mouse input handling for Windows.
//!
//! Absolute positioning is done via `SetCursorPos` (physical pixel coordinates on DPI-aware
//! processes; logical coordinates otherwise), which avoids the normalized-coordinate math
//! required by `SendInput` with `MOUSEEVENTF_ABSOLUTE`. Button presses, releases, and wheel
//! scrolls go through `SendInput`.
use std::ffi::c_void;
use std::mem::size_of;
use pathfinder_geometry::vector::Vector2I;
use windows::Win32::Foundation::{GetLastError, POINT};
use windows::Win32::Graphics::Gdi::{MONITOR_DEFAULTTONEAREST, MonitorFromPoint};
use windows::Win32::UI::HiDpi::{GetDpiForMonitor, MDT_EFFECTIVE_DPI};
use windows::Win32::UI::Input::KeyboardAndMouse::{
INPUT, INPUT_0, INPUT_MOUSE, MOUSE_EVENT_FLAGS, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_HWHEEL,
MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP, MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP,
MOUSEEVENTF_MOVE, MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_VIRTUALDESK,
MOUSEEVENTF_WHEEL, MOUSEEVENTF_XDOWN, MOUSEEVENTF_XUP, MOUSEINPUT, SendInput,
};
use windows::Win32::UI::WindowsAndMessaging::{
GetCursorPos, GetSystemMetrics, SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN,
SM_YVIRTUALSCREEN, SPI_GETWHEELSCROLLCHARS, SPI_GETWHEELSCROLLLINES,
SYSTEM_PARAMETERS_INFO_ACTION, SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS, SetCursorPos,
SystemParametersInfoW,
};
use super::dpi::DpiAwarenessGuard;
use crate::{MouseButton, ScrollDirection, ScrollDistance};
/// One wheel "click" in `MOUSEEVENTF_WHEEL`/`MOUSEEVENTF_HWHEEL` units.
/// See the Win32 `WHEEL_DELTA` constant.
const WHEEL_DELTA: i32 = 120;
/// `XBUTTON1` / `XBUTTON2` values for `mouseData` when sending X-button events. These match the
/// Win32 header values and are not currently exposed through the `windows` crate's
/// `KeyboardAndMouse` module.
const XBUTTON1: u32 = 0x0001;
const XBUTTON2: u32 = 0x0002;
/// Nominal line height (in logical pixels at 100% scale) used as the baseline when translating
/// the user's `SPI_GETWHEELSCROLLLINES` setting into a pixel-per-click factor. The actual line
/// height we use is this value scaled by the cursor-monitor DPI over `USER_DEFAULT_SCREEN_DPI`,
/// so `ScrollDistance::Pixels` stays proportional to what the user sees on HiDPI displays (~20px
/// at 125% scale, ~24px at 150%) — including secondary monitors in mixed-DPI setups.
const NOMINAL_LINE_HEIGHT_PX: i32 = 16;
/// The "default" (1x) DPI value Windows reports; matches `USER_DEFAULT_SCREEN_DPI`.
const DEFAULT_DPI: u32 = 96;
/// Fallback used if `SPI_GETWHEELSCROLLLINES` is unavailable or returns a sentinel value (e.g.,
/// `WHEEL_PAGESCROLL`). Matches the documented Windows default of three lines per wheel click.
const DEFAULT_WHEEL_SCROLL_LINES: u32 = 3;
/// Upper bound applied to the user's `SPI_GETWHEELSCROLL{LINES,CHARS}` setting before it's
/// multiplied by `NOMINAL_LINE_HEIGHT_PX`. Without this clamp an unusually large configured value
/// (or a corrupt value written by a partial `SystemParametersInfoW` call) would produce a
/// huge pixels-per-click factor, forcing every small pixel scroll to round up to a single click.
const MAX_WHEEL_SCROLL_LINES: u32 = 100;
/// Manages mouse state and posts mouse events to the system.
pub struct Mouse;
impl Default for Mouse {
fn default() -> Self {
Self::new()
}
}
impl Mouse {
pub fn new() -> Self {
Self
}
pub fn move_to(&mut self, target: Vector2I) -> Result<(), String> {
// Ensure this thread is per-monitor-v2 DPI aware so `SetCursorPos` receives coordinates in
// physical pixels rather than being scaled.
let _dpi_guard = DpiAwarenessGuard::enter_per_monitor_v2();
// SAFETY: SetCursorPos accepts any i32 coordinates; it will clamp to the available display
// region. This has no preconditions.
unsafe { SetCursorPos(target.x(), target.y()) }.map_err(|e| {
format!(
"Failed to move cursor to ({}, {}): {e}",
target.x(),
target.y()
)
})?;
// Also emit a `SendInput` mouse-move so consumers of raw input (`WM_INPUT`) and low-level
// mouse hooks (`WH_MOUSE_LL`) — common in games, anti-cheat, and some remote-desktop
// clients — see the motion. `SetCursorPos` alone only posts `WM_MOUSEMOVE` to the window
// under the cursor. Best-effort: we ignore a `SendInput` failure here because the cursor
// is already at the target position from `SetCursorPos` above.
match normalized_virtual_desk_coords(target) {
Some((dx, dy)) => {
let _ = send_mouse_event_with_coords(
MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK,
0,
dx,
dy,
);
}
None => {
log::warn!(
"Skipping WM_INPUT-visible cursor move for ({}, {}): invalid virtual-screen \
metrics",
target.x(),
target.y(),
);
}
}
Ok(())
}
pub fn button_down(&mut self, button: &MouseButton) -> Result<(), String> {
let (flags, mouse_data) = button_down_event(button);
send_mouse_event(flags, mouse_data)
}
pub fn button_up(&mut self, button: &MouseButton) -> Result<(), String> {
let (flags, mouse_data) = button_up_event(button);
send_mouse_event(flags, mouse_data)
}
pub fn current_position(&mut self) -> Result<Vector2I, String> {
// Match the DPI awareness used by `move_to` so the reported position is in the same
// coordinate space as the coordinates we send.
let _dpi_guard = DpiAwarenessGuard::enter_per_monitor_v2();
let mut point = POINT { x: 0, y: 0 };
// SAFETY: `point` is a valid, writable `POINT`.
unsafe { GetCursorPos(&mut point) }
.map_err(|e| format!("Failed to get cursor position: {e}"))?;
Ok(Vector2I::new(point.x, point.y))
}
pub fn scroll(
&mut self,
direction: &ScrollDirection,
distance: &ScrollDistance,
) -> Result<(), String> {
// Match the DPI awareness used by `move_to` / `current_position` so
// `cursor_monitor_dpi` (invoked via `pixels_per_click` → `scaled_line_height_px`) resolves
// the cursor's monitor in physical pixels even when the host process is not manifest-
// declared per-monitor-v2 DPI aware.
let _dpi_guard = DpiAwarenessGuard::enter_per_monitor_v2();
// Windows expresses wheel amounts in multiples of WHEEL_DELTA (120 per "click"). Positive
// values scroll forward (up/right); negative values scroll backward (down/left).
// Both `Clicks` and `Pixels` are treated as unsigned magnitudes here; the direction is
// encoded separately in `ScrollDirection`, so we `saturating_abs()` either branch to avoid
// a negative distance canceling out `ScrollDirection` and scrolling the wrong way.
//
// Resolve axis flags and sign from `direction` in a single match so the
// vertical/horizontal decision lives in exactly one place.
let (flags, sign) = match direction {
ScrollDirection::Up => (MOUSEEVENTF_WHEEL, 1),
ScrollDirection::Down => (MOUSEEVENTF_WHEEL, -1),
// Horizontal wheel: positive = right, negative = left.
ScrollDirection::Right => (MOUSEEVENTF_HWHEEL, 1),
ScrollDirection::Left => (MOUSEEVENTF_HWHEEL, -1),
};
let is_horizontal = flags == MOUSEEVENTF_HWHEEL;
let magnitude: i32 = match distance {
ScrollDistance::Clicks(clicks) => clicks.saturating_abs().saturating_mul(WHEEL_DELTA),
ScrollDistance::Pixels(pixels) => {
// Derive pixels-per-click from the user's actual system setting
// (`SPI_GETWHEELSCROLLLINES` for vertical, `SPI_GETWHEELSCROLLCHARS` for
// horizontal) so we respect mouse / trackpad driver configuration instead of a
// hard-coded constant.
//
// `pixels` is treated as a magnitude because the direction is encoded separately
// in `ScrollDirection`. A zero-pixel request is a no-op; non-zero requests below
// `pixels_per_click` round up to a single click so the scroll is still observable.
let abs_pixels = pixels.saturating_abs();
if abs_pixels == 0 {
0
} else {
let per_click = pixels_per_click(is_horizontal);
let clicks = (abs_pixels / per_click).clamp(1, i32::MAX / WHEEL_DELTA);
clicks.saturating_mul(WHEEL_DELTA)
}
}
};
let signed_amount = magnitude.saturating_mul(sign);
// Skip zero-delta wheel events (e.g., `Clicks(0)` or `Pixels(0)`). Windows would still
// dispatch them as observable `WM_MOUSEWHEEL`s even though no scrolling happens.
if signed_amount == 0 {
return Ok(());
}
// `mouseData` is declared as a `u32` but `MOUSEEVENTF_WHEEL`/`HWHEEL` reinterpret the
// bits as a signed `i32` (positive scrolls up/right, negative scrolls down/left). `as u32`
// on an `i32` is the well-defined two's-complement reinterpretation we want here.
send_mouse_event(flags, signed_amount as u32)
}
}
/// Returns the number of pixels that correspond to one wheel "click" on the requested axis,
/// derived from the user's `SPI_GETWHEELSCROLL{LINES,CHARS}` setting. Horizontal wheel on Windows
/// is conventionally driven by `SPI_GETWHEELSCROLLCHARS`, not `SPI_GETWHEELSCROLLLINES`. Falls
/// back to the Windows default (3 lines/chars) if the setting is unavailable or set to the
/// `WHEEL_PAGESCROLL` sentinel.
fn pixels_per_click(is_horizontal: bool) -> i32 {
let spi: SYSTEM_PARAMETERS_INFO_ACTION = if is_horizontal {
SPI_GETWHEELSCROLLCHARS
} else {
SPI_GETWHEELSCROLLLINES
};
let mut units: u32 = DEFAULT_WHEEL_SCROLL_LINES;
// SAFETY: `units` is a valid writable u32 and we pass its size implicitly via the fixed-layout
// `SPI_GETWHEELSCROLL*` contract. The call does not retain the pointer beyond the call.
let result = unsafe {
SystemParametersInfoW(
spi,
0,
Some(&mut units as *mut u32 as *mut c_void),
SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS(0),
)
};
// If the call fails, or the user has configured page scrolling (WHEEL_PAGESCROLL == u32::MAX),
// fall back to the documented default.
if result.is_err() || units == 0 || units == u32::MAX {
units = DEFAULT_WHEEL_SCROLL_LINES;
}
// Clamp to `[1, MAX_WHEEL_SCROLL_LINES]` so this function's caller can always divide by the
// result without risking a divide-by-zero (even if a future refactor drops the `== 0` guard
// above) and so an unusually large setting can't produce a huge pixels-per-click factor.
let units = units.clamp(1, MAX_WHEEL_SCROLL_LINES);
(units as i32).saturating_mul(scaled_line_height_px())
}
/// Returns the nominal line height in *physical* pixels for the monitor the cursor is currently
/// on, so `ScrollDistance::Pixels` translations stay proportional to the user's display scaling
/// even on mixed-DPI multi-monitor setups (which `GetDpiForSystem` can't express).
fn scaled_line_height_px() -> i32 {
let dpi = cursor_monitor_dpi();
// `NOMINAL_LINE_HEIGHT_PX * dpi / 96`, saturating; integer math is sufficient at the
// precision we care about here.
let scaled = (NOMINAL_LINE_HEIGHT_PX as i64).saturating_mul(dpi as i64) / DEFAULT_DPI as i64;
// Re-clamp back into `i32` range and ensure at least 1 so callers can divide safely.
scaled.clamp(1, i32::MAX as i64) as i32
}
/// Returns the effective DPI of the monitor currently containing the cursor, falling back to
/// `DEFAULT_DPI` if any step of the query fails. Using the cursor's monitor (rather than the
/// primary) keeps `ScrollDistance::Pixels` proportional to the display the user is actually
/// scrolling on.
fn cursor_monitor_dpi() -> u32 {
let mut point = POINT { x: 0, y: 0 };
// SAFETY: `point` is a valid, writable `POINT`.
if unsafe { GetCursorPos(&mut point) }.is_err() {
return DEFAULT_DPI;
}
// SAFETY: `MonitorFromPoint` has no preconditions; `MONITOR_DEFAULTTONEAREST` guarantees a
// non-null handle when any monitor exists.
let hmonitor = unsafe { MonitorFromPoint(point, MONITOR_DEFAULTTONEAREST) };
if hmonitor.is_invalid() {
return DEFAULT_DPI;
}
let mut dpi_x: u32 = 0;
let mut dpi_y: u32 = 0;
// SAFETY: `hmonitor` is valid; `dpi_x`/`dpi_y` are writable u32s.
if unsafe { GetDpiForMonitor(hmonitor, MDT_EFFECTIVE_DPI, &mut dpi_x, &mut dpi_y) }.is_err() {
return DEFAULT_DPI;
}
// Guard against the (unexpected) 0 return so we never produce a 0-pixel line height.
if dpi_x == 0 { DEFAULT_DPI } else { dpi_x }
}
/// Translates a virtual-screen pixel coordinate into the `[0, 65535]` normalized absolute
/// coordinates `SendInput` expects when `MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK` is set.
/// Returns `None` if the virtual screen metrics are unusable.
fn normalized_virtual_desk_coords(target: Vector2I) -> Option<(i32, i32)> {
// SAFETY: `GetSystemMetrics` has no preconditions.
let virt_x = unsafe { GetSystemMetrics(SM_XVIRTUALSCREEN) };
let virt_y = unsafe { GetSystemMetrics(SM_YVIRTUALSCREEN) };
let virt_w = unsafe { GetSystemMetrics(SM_CXVIRTUALSCREEN) };
let virt_h = unsafe { GetSystemMetrics(SM_CYVIRTUALSCREEN) };
if virt_w <= 0 || virt_h <= 0 {
return None;
}
// Normalize into `[0, 65535]` across the virtual desktop. Use i64 to avoid overflow when the
// virtual screen is large.
let dx = (target.x() as i64 - virt_x as i64) * 65535 / virt_w as i64;
let dy = (target.y() as i64 - virt_y as i64) * 65535 / virt_h as i64;
Some((dx.clamp(0, 65535) as i32, dy.clamp(0, 65535) as i32))
}
/// Returns the `(flags, mouseData)` pair for a mouse button-down event.
fn button_down_event(button: &MouseButton) -> (MOUSE_EVENT_FLAGS, u32) {
match button {
MouseButton::Left => (MOUSEEVENTF_LEFTDOWN, 0),
MouseButton::Right => (MOUSEEVENTF_RIGHTDOWN, 0),
MouseButton::Middle => (MOUSEEVENTF_MIDDLEDOWN, 0),
MouseButton::Back => (MOUSEEVENTF_XDOWN, XBUTTON1),
MouseButton::Forward => (MOUSEEVENTF_XDOWN, XBUTTON2),
}
}
/// Returns the `(flags, mouseData)` pair for a mouse button-up event.
fn button_up_event(button: &MouseButton) -> (MOUSE_EVENT_FLAGS, u32) {
match button {
MouseButton::Left => (MOUSEEVENTF_LEFTUP, 0),
MouseButton::Right => (MOUSEEVENTF_RIGHTUP, 0),
MouseButton::Middle => (MOUSEEVENTF_MIDDLEUP, 0),
MouseButton::Back => (MOUSEEVENTF_XUP, XBUTTON1),
MouseButton::Forward => (MOUSEEVENTF_XUP, XBUTTON2),
}
}
/// Dispatches a single mouse event via `SendInput` with `dx` = `dy` = 0 (i.e., at the current
/// cursor position). Use [`send_mouse_event_with_coords`] for absolute-positioned events such as
/// `MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE`.
fn send_mouse_event(flags: MOUSE_EVENT_FLAGS, mouse_data: u32) -> Result<(), String> {
send_mouse_event_with_coords(flags, mouse_data, 0, 0)
}
/// Dispatches a single mouse event via `SendInput`. `dx`/`dy` are interpreted per Win32 docs:
/// absolute `[0, 65535]` normalized coordinates when `MOUSEEVENTF_ABSOLUTE` is set, otherwise
/// relative movement.
fn send_mouse_event_with_coords(
flags: MOUSE_EVENT_FLAGS,
mouse_data: u32,
dx: i32,
dy: i32,
) -> Result<(), String> {
let input = INPUT {
r#type: INPUT_MOUSE,
Anonymous: INPUT_0 {
mi: MOUSEINPUT {
dx,
dy,
mouseData: mouse_data,
dwFlags: flags,
time: 0,
dwExtraInfo: 0,
},
},
};
// SAFETY: `input` is a valid `INPUT` of mouse type, with the correct size passed to
// `SendInput`. The call does not retain any pointer beyond the call.
let sent = unsafe { SendInput(&[input], size_of::<INPUT>() as i32) };
if sent != 1 {
// SAFETY: `GetLastError` has no preconditions; reads the calling thread's last-error.
let last_error = unsafe { GetLastError() }.0;
return Err(format!(
"SendInput failed to dispatch mouse event (flags={:#x}, GetLastError={last_error})",
flags.0,
));
}
Ok(())
}
crates/computer_use/src/windows/screenshot.rs
+330
View File
@@ -0,0 +1,330 @@
//! Screenshot capture for Windows using GDI.
//!
//! Captures the full virtual screen (all monitors) or a sub-region of it by compositing the screen
//! contents into an offscreen DIB via `BitBlt`, then reading the pixel data out with `GetDIBits`.
//! The resulting BGRA data is converted to RGBA and handed off to the shared screenshot processing
//! pipeline.
use std::mem::size_of;
use image::{DynamicImage, RgbaImage};
use windows::Win32::Graphics::Gdi::{
BI_RGB, BITMAPINFO, BITMAPINFOHEADER, BitBlt, CreateCompatibleBitmap, CreateCompatibleDC,
DIB_RGB_COLORS, DeleteDC, DeleteObject, GetDC, GetDIBits, HBITMAP, HDC, HGDIOBJ, ReleaseDC,
SRCCOPY, SelectObject,
};
use windows::Win32::UI::WindowsAndMessaging::{
GetSystemMetrics, SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN,
};
use super::dpi::DpiAwarenessGuard;
use crate::{Screenshot, ScreenshotParams};
/// Captures a screenshot of the full virtual screen (or a region of it).
///
/// On multi-monitor setups the virtual screen spans every display and its origin may be at
/// negative coordinates (e.g., if a secondary monitor is positioned left of the primary).
/// `ScreenshotRegion::validate` currently requires non-negative region coordinates, so callers
/// cannot reach areas with negative virtual-screen coordinates via region captures; those areas
/// are still included in the full-screen capture.
///
/// TODO: relax the non-negative check in `ScreenshotRegion::validate`
/// (`crates/computer_use/src/lib.rs`) so the region path can reach monitors positioned above /
/// left of the primary. The Win32 side of this module already supports negative coordinates; the
/// restriction is shared across Mac / Linux / Windows, so this is a platform-neutral follow-up.
pub fn take(params: ScreenshotParams) -> Result<Screenshot, String> {
// Opt this thread into per-monitor-v2 DPI awareness so the virtual-screen metrics and `BitBlt`
// all operate in physical pixels, regardless of the host process manifest. Dropped at end of
// scope to restore prior context.
let _dpi_guard = DpiAwarenessGuard::enter_per_monitor_v2();
// SAFETY: GetSystemMetrics has no preconditions.
let virt_x = unsafe { GetSystemMetrics(SM_XVIRTUALSCREEN) };
let virt_y = unsafe { GetSystemMetrics(SM_YVIRTUALSCREEN) };
let virt_w = unsafe { GetSystemMetrics(SM_CXVIRTUALSCREEN) };
let virt_h = unsafe { GetSystemMetrics(SM_CYVIRTUALSCREEN) };
if virt_w <= 0 || virt_h <= 0 {
return Err(format!(
"Virtual screen has invalid dimensions ({virt_w}x{virt_h})"
));
}
let max_x = virt_x.saturating_add(virt_w);
let max_y = virt_y.saturating_add(virt_h);
// Determine the region to capture. Coordinates are in the same space as the virtual screen,
// matching `SetCursorPos` (pixel coordinates for DPI-aware processes, logical coordinates
// otherwise).
let (src_x, src_y, width, height) = if let Some(region) = params.region {
region.validate()?;
let w = region.bottom_right.x() - region.top_left.x();
let h = region.bottom_right.y() - region.top_left.y();
// Validate against both ends of the virtual screen. `ScreenshotRegion::validate` only
// enforces `top_left >= 0`, which can't catch the uncommon case where the virtual-screen
// origin itself is positive (e.g., primary monitor repositioned) — without the explicit
// `< virt_x/virt_y` check, `BitBlt` would silently sample pixels off the virtual screen.
if region.top_left.x() < virt_x
|| region.top_left.y() < virt_y
|| region.bottom_right.x() > max_x
|| region.bottom_right.y() > max_y
{
return Err(format!(
"Screenshot region ({}, {})-({}, {}) exceeds virtual screen bounds \
({virt_x}, {virt_y})-({max_x}, {max_y})",
region.top_left.x(),
region.top_left.y(),
region.bottom_right.x(),
region.bottom_right.y(),
));
}
(region.top_left.x(), region.top_left.y(), w, h)
} else {
(virt_x, virt_y, virt_w, virt_h)
};
let rgba = capture_rgba(src_x, src_y, width, height)?;
let img = RgbaImage::from_raw(width as u32, height as u32, rgba)
.ok_or_else(|| "Failed to construct image from GDI pixel data".to_string())?;
let img = DynamicImage::ImageRgba8(img);
crate::screenshot_utils::process_screenshot(img, params)
}
/// Captures the screen into a freshly allocated RGBA buffer.
///
/// The caller is responsible for providing a valid region on the virtual screen; this function
/// does not clip coordinates itself. Source coordinates are in virtual-screen space (the same
/// space as `SetCursorPos`).
fn capture_rgba(src_x: i32, src_y: i32, width: i32, height: i32) -> Result<Vec<u8>, String> {
/// `HGDI_ERROR` = `(HGDIOBJ)(LONG_PTR)-1`, returned by `SelectObject` on a type mismatch.
/// Named here because `HGDIOBJ::is_invalid()` only checks for NULL, and the `windows` crate
/// we're on doesn't expose an `HGDI_ERROR` constant we can compare against directly.
const HGDI_ERROR_SENTINEL: isize = -1;
// Use RAII guards so every GDI handle is released even on early returns.
let screen_dc = ScreenDc::acquire()?;
let mem_dc = MemoryDc::create_compatible(screen_dc.handle())?;
let bitmap = Bitmap::create_compatible(screen_dc.handle(), width, height)?;
// SAFETY: `mem_dc` and `bitmap` are valid GDI handles owned by the guards.
let prev_object = unsafe { SelectObject(mem_dc.handle(), bitmap.handle().into()) };
// `SelectObject` returns NULL on general failure or `HGDI_ERROR` on type mismatch; check both.
if prev_object.is_invalid() || prev_object.0 as isize == HGDI_ERROR_SENTINEL {
return Err("SelectObject failed for screenshot bitmap".to_string());
}
// RAII-restore the previously-selected object on drop so the `Bitmap` guard can safely
// `DeleteObject` it even if `BitBlt` / `GetDIBits` panic (per MSDN, `DeleteObject` fails on
// an HBITMAP still selected into a DC, which would leak both the bitmap and DC).
let _restore_select_guard = SelectObjectGuard {
dc: mem_dc.handle(),
prev_object,
};
// SAFETY: both DCs are valid; BitBlt reads the screen and writes into the compatible memory
// DC we just prepared.
unsafe {
BitBlt(
mem_dc.handle(),
0,
0,
width,
height,
Some(screen_dc.handle()),
src_x,
src_y,
SRCCOPY,
)
}
.map_err(|e| format!("BitBlt failed while capturing screen: {e}"))?;
let buffer = read_bitmap_bits(mem_dc.handle(), bitmap.handle(), width, height)?;
Ok(convert_bgra_to_rgba(buffer))
}
/// RAII guard that restores a previously-`SelectObject`'d GDI object into `dc` when dropped,
/// making the select/restore lifecycle panic-safe.
struct SelectObjectGuard {
dc: HDC,
prev_object: HGDIOBJ,
}
impl Drop for SelectObjectGuard {
fn drop(&mut self) {
// SAFETY: `dc` is the same DC the caller used with `SelectObject`; `prev_object` is the
// handle that `SelectObject` returned. Both are still valid at this point because the
// underlying DC / bitmap guards own them and haven't been dropped yet (Rust drops fields
// and locals in reverse declaration order; this guard is declared before the outer
// bitmap / DC guards go out of scope).
unsafe { SelectObject(self.dc, self.prev_object) };
}
}
/// Reads `width x height` pixels from `bitmap` as 32-bit top-down BGRA.
fn read_bitmap_bits(
mem_dc: HDC,
bitmap: HBITMAP,
width: i32,
height: i32,
) -> Result<Vec<u8>, String> {
// BITMAPINFO has a flexible-array of color entries at the end; for 32bpp BI_RGB we don't need
// any, and the single-element default is sufficient.
let mut info = BITMAPINFO {
bmiHeader: BITMAPINFOHEADER {
biSize: size_of::<BITMAPINFOHEADER>() as u32,
biWidth: width,
// Negative height requests a top-down DIB, so the first row in the buffer corresponds
// to the top of the image.
biHeight: -height,
biPlanes: 1,
biBitCount: 32,
biCompression: BI_RGB.0,
biSizeImage: 0,
biXPelsPerMeter: 0,
biYPelsPerMeter: 0,
biClrUsed: 0,
biClrImportant: 0,
},
bmiColors: Default::default(),
};
let byte_count = (width as usize)
.checked_mul(height as usize)
.and_then(|n| n.checked_mul(4))
.ok_or_else(|| format!("Screenshot dimensions {width}x{height} overflow buffer size"))?;
let mut buffer = vec![0u8; byte_count];
// SAFETY: `buffer` is large enough for the requested pixels; `info` is a valid BITMAPINFO
// describing the requested format. `GetDIBits` does not retain any of the pointers after it
// returns.
let scanlines = unsafe {
GetDIBits(
mem_dc,
bitmap,
0,
height as u32,
Some(buffer.as_mut_ptr() as *mut _),
&mut info,
DIB_RGB_COLORS,
)
};
// `GetDIBits` returns the number of scan lines actually copied. Anything less than the
// requested height means the buffer is only partially populated; treat that as a failure so we
// don't silently decode a truncated image.
if scanlines != height {
return Err(format!(
"GetDIBits copied {scanlines} of {height} scan lines for screenshot"
));
}
Ok(buffer)
}
/// Converts a tightly packed BGRA buffer (as produced by `GetDIBits` with `biBitCount = 32` and
/// `BI_RGB`) to RGBA in-place.
///
/// GDI does not populate the alpha channel for `BI_RGB`, so we force it to `0xFF` to produce a
/// fully opaque RGBA image.
fn convert_bgra_to_rgba(mut buffer: Vec<u8>) -> Vec<u8> {
for chunk in buffer.chunks_exact_mut(4) {
// Swap B and R channels so the 4-byte BGRA pixel becomes RGBA.
chunk.swap(0, 2);
chunk[3] = 0xFF;
}
buffer
}
// ---------------------------------------------------------------------------
// RAII guards for GDI handles
// ---------------------------------------------------------------------------
/// RAII wrapper for the screen device context.
///
/// `GetDC(NULL)` returns a DC whose coordinate space spans the entire virtual screen, so `BitBlt`
/// can source pixels from any monitor.
struct ScreenDc(HDC);
impl ScreenDc {
fn acquire() -> Result<Self, String> {
// SAFETY: `GetDC(None)` returns a DC for the virtual screen or a null handle on failure.
let hdc = unsafe { GetDC(None) };
if hdc.is_invalid() {
return Err("GetDC(NULL) returned a null handle".to_string());
}
Ok(Self(hdc))
}
fn handle(&self) -> HDC {
self.0
}
}
impl Drop for ScreenDc {
fn drop(&mut self) {
// SAFETY: `self.0` is a DC returned by `GetDC(None)` and has not been released yet.
let released = unsafe { ReleaseDC(None, self.0) };
if released == 0 {
// Not fatal (the process can still continue), but indicates a handle-lifetime
// regression worth investigating.
log::warn!("ReleaseDC returned 0 for the screen DC");
}
}
}
/// RAII wrapper for a memory device context created with `CreateCompatibleDC`.
struct MemoryDc(HDC);
impl MemoryDc {
fn create_compatible(screen: HDC) -> Result<Self, String> {
// SAFETY: `screen` is a valid DC returned from `GetDC`.
let hdc = unsafe { CreateCompatibleDC(Some(screen)) };
if hdc.is_invalid() {
return Err("CreateCompatibleDC failed".to_string());
}
Ok(Self(hdc))
}
fn handle(&self) -> HDC {
self.0
}
}
impl Drop for MemoryDc {
fn drop(&mut self) {
// SAFETY: `self.0` was created by `CreateCompatibleDC` and has not been deleted yet.
unsafe {
let _ = DeleteDC(self.0);
}
}
}
/// RAII wrapper for a GDI bitmap handle.
struct Bitmap(HBITMAP);
impl Bitmap {
fn create_compatible(screen: HDC, width: i32, height: i32) -> Result<Self, String> {
// SAFETY: `screen` is a valid DC; width and height are positive.
let hbitmap = unsafe { CreateCompatibleBitmap(screen, width, height) };
if hbitmap.is_invalid() {
return Err(format!(
"CreateCompatibleBitmap failed for {width}x{height} bitmap"
));
}
Ok(Self(hbitmap))
}
fn handle(&self) -> HBITMAP {
self.0
}
}
impl Drop for Bitmap {
fn drop(&mut self) {
// SAFETY: `self.0` was created by `CreateCompatibleBitmap` and has not been deleted yet.
// It must not be currently selected into a DC; callers restore the previous object before
// dropping.
let obj: HGDIOBJ = self.0.into();
unsafe {
let _ = DeleteObject(obj);
}
}
}