From b4c896a94b1ae4abead02ef3ed408d17ac3dcd61 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 6 Nov 2026 08:40:30 -0800 Subject: [PATCH 1/5] feat(drivers): agent-desktop on Linux, on an X11Driver base the reference driver shares Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/pyproject.toml | 5 + .../agentenv_protocol/computer/__init__.py | 5 +- .../src/agentenv_protocol/computer/_x11.py | 395 +++++++++++++++++ src/agent_env/drivers/README.md | 27 +- .../drivers/envs/agent-desktop/driver.py | 61 ++- .../drivers/envs/agent-desktop/env.toml | 9 + .../drivers/envs/agent-desktop/install.sh | 4 +- .../envs/agent-desktop/requirements.txt | 4 + .../drivers/envs/reference/driver.py | 414 +----------------- .../drivers/envs/reference/requirements.txt | 6 +- tst/unit/drivers/agent_desktop_driver_test.py | 60 ++- tst/unit/drivers/reference_driver_test.py | 1 + 12 files changed, 530 insertions(+), 461 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/_x11.py create mode 100644 src/agent_env/drivers/envs/agent-desktop/requirements.txt diff --git a/packages/agentenv-protocol/pyproject.toml b/packages/agentenv-protocol/pyproject.toml index 093b5a355..0bbc6c46c 100644 --- a/packages/agentenv-protocol/pyproject.toml +++ b/packages/agentenv-protocol/pyproject.toml @@ -56,6 +56,11 @@ macos = [ "pyobjc-framework-Quartz>=11; sys_platform == 'darwin'", "pyobjc-framework-ScreenCaptureKit>=11; sys_platform == 'darwin'", ] +# A Linux driver on X11Driver: mss captures the X display and XTEST sends its input, through python-xlib. +x11 = [ + "mss>=9,<11", + "python-xlib>=0.33,<0.34", +] dev = [ "a2a-sdk[http-server]>=0.3.26,<0.4", "python-multipart>=0.0.13", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py index 3eb088927..12d3350a2 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py @@ -98,16 +98,18 @@ ) from .driver import DEFAULT_IDENTITY, ComputerDriverEnvironment, DriverDeclaration, computer_driver from ._macos import MacOSDriver + from ._x11 import X11Driver from .recording import RecordedFrame, RecordedSegment, RecordHints, Recording, frames_in_thread # The names whose modules need the computer extra (mcp, av and Pillow), by module; MacOSDriver's, the macos extra too, -# on first use of the Mac. +# on first use of the Mac, and X11Driver's, the x11 extra. _ON_USE = { "driver": ("ComputerDriverEnvironment", "computer_driver", "DEFAULT_IDENTITY", "DriverDeclaration"), "card": ("COMPUTER", "COMPUTER_READY", "COMPUTER_DISPLAY", "COMPUTER_SPACES", "COMPUTER_POINTER", "COMPUTER_KEYBOARD", "COMPUTER_PAGE", "COMPUTER_EXEC", "COMPUTER_RECORDING"), "recording": ("Recording", "RecordHints", "RecordedFrame", "RecordedSegment", "frames_in_thread"), "_macos": ("MacOSDriver",), + "_x11": ("X11Driver",), } @@ -120,6 +122,7 @@ def __getattr__(name: str) -> Any: __all__ = [ "ComputerDriverEnvironment", "computer_driver", "DEFAULT_IDENTITY", "DriverDeclaration", "MacOSDriver", + "X11Driver", "ComputerAppEnvironment", "computer_app", "install", "linux", "install_step", "ready_check", "Linux", "Service", "Launch", "Ready", "Packages", "AptSource", "Download", "RunScript", "AppDeclaration", "AppOs", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/_x11.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/_x11.py new file mode 100644 index 000000000..5dd758671 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/_x11.py @@ -0,0 +1,395 @@ +"""computer/v1 on an X11 desktop on Linux, for the reference driver and for a driver whose software serves tools of its +own, as agent-desktop's does: mss captures the screen and XTEST, through python-xlib, sends the input. X calls run off +the event loop, one at a time, on a connection checked before each call and reopened when it drops, so the driver +outlives a restarted X server, and input that lets go of a button returns once the server has let go of it. It records +the display through FFmpeg's ``x11grab``, by PyAV. A driver subclasses ``X11Driver``, declares itself with +``computer_driver``, and serves its own tools beside these. Needs the ``x11`` extra: mss and python-xlib.""" +from __future__ import annotations + +import asyncio +import contextlib +import math +import os +import threading +import time +from collections import OrderedDict +from datetime import datetime, timezone +from typing import Any, AsyncIterator, Callable, Iterator + +import av +import mss +from Xlib import X, XK +from Xlib import display as xdisplay +from Xlib.error import ConnectionClosedError, DisplayConnectionError +from Xlib.ext import xtest + +from .driver import ComputerDriverEnvironment +from .errors import DeliveryFailed, DeviceUnavailable, Unsupported +from .operations import (Display, Image, ReadyCheck, capture, click, display, drag, key_down, key_up, pointer_down, + pointer_move, pointer_position, pointer_up, press, record, scroll, type_text) +from .platform import Platform +from .recording import RecordedFrame, RecordHints, Recording, frames_in_thread +from .settings import ComputerSettings + +BUTTONS = {"left": 1, "middle": 2, "right": 3, "back": 8, "forward": 9} +#: The pointer state's bit for each button that has one. +BUTTON_MASKS = {"left": X.Button1Mask, "middle": X.Button2Mask, "right": X.Button3Mask} +SCROLL = {"up": 4, "down": 5, "left": 6, "right": 7} +#: W3C key values and their X keysym names; any other single character is typed as its own keysym. +KEYSYMS = { + "Enter": "Return", "Tab": "Tab", "Escape": "Escape", "Backspace": "BackSpace", "Delete": "Delete", + "ArrowUp": "Up", "ArrowDown": "Down", "ArrowLeft": "Left", "ArrowRight": "Right", "Home": "Home", "End": "End", + "PageUp": "Prior", "PageDown": "Next", "Insert": "Insert", "Control": "Control_L", "Shift": "Shift_L", + "Alt": "Alt_L", "Meta": "Super_L", "Super": "Super_L", "CapsLock": "Caps_Lock", "ContextMenu": "Menu", + "PrintScreen": "Print", "Pause": "Pause", " ": "space", **{f"F{n}": f"F{n}" for n in range(1, 25)}, +} +TYPED = {"\n": "Return", "\t": "Tab"} +#: The pause between the presses of a double or triple click, in seconds. +CLICK_GAP_S = 0.04 +#: How long a button's release may take to reach the app, and how often the driver looks. +RELEASE_WAIT_S = 5.0 +RELEASE_POLL_S = 0.005 +#: How far apart a drag's pointer moves are, in pixels. +DRAG_STEP_PX = 20 +DROPPED = (ConnectionClosedError, DisplayConnectionError, OSError) + + +class X11Driver(ComputerDriverEnvironment): + """The computer/v1 operations, the pointer and keyboard groups among them, on an X11 desktop.""" + + buttons = ("left", "right", "middle", "back", "forward") + max_click_count = 3 + max_drag_points = 50 + recording = Recording(max_fps=30) + + def __init__(self, settings: ComputerSettings | None = None, platform: Platform | None = None) -> None: + super().__init__(settings, platform) + self._x: _Connection | None = None + self._x_lock = threading.Lock() + #: The keycodes each key held by @key_down pressed, which @key_up releases. + self._held: dict[str, list[int]] = {} + + @display + async def screen(self) -> Display: + def read(c): + s = c.d.screen() + return Display(s.width_in_pixels, s.height_in_pixels, 1.0) + return await self._read(read) + + @capture + async def grab(self) -> Image: + return await asyncio.to_thread(_grab) + + @record + async def stream(self, hints: RecordHints) -> AsyncIterator[RecordedFrame]: + """The display through FFmpeg's ``x11grab``. Its frames carry the wall clock, so each one's time is when it + was captured.""" + size = await self.screen() + options = {"framerate": str(hints.fps), "video_size": f"{size.width}x{size.height}", + "draw_mouse": "1" if hints.cursor else "0"} + + def frames() -> Iterator[RecordedFrame]: + with av.open(os.environ.get("DISPLAY", ":0"), format="x11grab", options=options) as source: + for frame in source.decode(video=0): + yield RecordedFrame(frame, datetime.fromtimestamp(frame.time, timezone.utc)) + + async for frame in frames_in_thread(frames): + yield frame + + async def ready_checks(self) -> list[ReadyCheck]: + try: + has = await self._read(lambda c: c.d.has_extension("XTEST")) + except DeviceUnavailable as e: + return [ReadyCheck("xtest", False, e.message)] + return [ReadyCheck("xtest", bool(has), "XTEST available" if has else "XTEST missing")] + + @click + async def click_at(self, x, y, button, count, modifiers) -> None: + syms = [_keysym(k) for k in modifiers] + + def act(c): + held = c.hold(syms) + c.move(x, y) + for i in range(count): + c.button(button, True) + c.button(button, False) + if i + 1 < count: + c.d.sync() + time.sleep(CLICK_GAP_S) + c.released(button) + c.release(reversed(held)) + await self._send(act) + + @drag + async def drag_along(self, path, button, duration_ms, modifiers) -> None: + syms = [_keysym(k) for k in modifiers] + steps = _steps(path) + pause = duration_ms / 1000 / len(steps) + held: list[int] = [] + + def start(c): + held.extend(c.hold(syms)) + c.move(*path[0]) + c.button(button, True) + + def finish(c): + c.button(button, False) + c.released(button) + c.release(reversed(held)) + + try: + await self._send(start) + for px, py in steps: + await self._send(lambda c, px=px, py=py: c.move(px, py)) + await asyncio.sleep(pause) + except BaseException: + with contextlib.suppress(Exception): + await self._send(finish) + raise + await self._send(finish) + + @scroll + async def scroll_at(self, x, y, direction, amount, modifiers) -> None: + syms = [_keysym(k) for k in modifiers] + + def act(c): + held = c.hold(syms) + c.move(x, y) + for _ in range(amount): + xtest.fake_input(c.d, X.ButtonPress, SCROLL[direction]) + xtest.fake_input(c.d, X.ButtonRelease, SCROLL[direction]) + c.release(reversed(held)) + await self._send(act) + + @pointer_move + async def move(self, x, y) -> None: + await self._send(lambda c: c.move(x, y)) + + @pointer_down + async def down(self, button) -> None: + await self._send(lambda c: c.button(button, True)) + + @pointer_up + async def up(self, button) -> None: + def act(c): + c.button(button, False) + c.released(button) + await self._send(act) + + @pointer_position + async def where(self) -> tuple[int, int]: + def read(c): + p = c.d.screen().root.query_pointer() + return p.root_x, p.root_y + return await self._read(read) + + @type_text + async def type_in(self, text) -> None: + syms = [XK.string_to_keysym(TYPED[ch]) if ch in TYPED else _keysym(ch) for ch in text] + await self._read(lambda c: c.check(syms)) + for sym in syms: + hold = self.settings.tap_hold_ms / 1000 + await self._send(lambda c, sym=sym: c.tap(sym, hold)) + await asyncio.sleep(self.settings.type_delay_ms / 1000) + + @press + async def press_keys(self, keys) -> None: + syms = [_keysym(k) for k in keys] + hold = self.settings.tap_hold_ms / 1000 + + def act(c): + pressed = c.hold(syms) + c.d.sync() + time.sleep(hold) + c.release(reversed(pressed)) + await self._send(act) + + @key_down + async def keys_down(self, keys) -> None: + syms = [_keysym(k) for k in keys] + + def act(c): + for key, pressed in zip(keys, c.hold_each(syms)): + self._held.setdefault(key, []).extend(pressed) + await self._send(act) + + @key_up + async def keys_up(self, keys) -> None: + def act(c): + for key in keys: + c.release(reversed(self._held.pop(key, None) or c.codes(_keysym(key))[-1:])) + await self._send(act) + + async def _read(self, fn: Callable[[_Connection], Any]) -> Any: + return await asyncio.to_thread(self._x_call, fn, False) + + async def _send(self, fn: Callable[[_Connection], Any]) -> Any: + return await asyncio.to_thread(self._x_call, fn, True) + + def _x_call(self, fn: Callable[[_Connection], Any], sends_input: bool) -> Any: + """``fn`` on a live connection. When the connection drops, a read is retried once on a new one; input is not + sent again, since some of it may have arrived.""" + with self._x_lock: + for attempt in (1, 2): + c = self._connected() + try: + result = fn(c) + c.d.sync() + return result + except DROPPED as e: + self._x = None + if sends_input: + raise DeliveryFailed(f"the X connection dropped while sending input: {type(e).__name__}") from None + if attempt == 2: + raise DeviceUnavailable(f"the X server is unavailable: {type(e).__name__}") from None + + def _connected(self) -> _Connection: + """The connection, opened or reopened as needed and checked with a round trip before anything is sent.""" + for attempt in (1, 2): + try: + if self._x is None: + self._x = _Connection() + self._held.clear() + self._x.d.sync() + return self._x + except DROPPED as e: + self._x = None + if attempt == 2: + raise DeviceUnavailable(f"the X server is unavailable: {type(e).__name__}") from None + + +class _Connection: + """One connection to the X server, with XTEST input on it. A character the keymap lacks is typed through a spare + keycode mapped to it, kept for that character and reused; once all are taken, the least recently used one that + no key holds down is mapped again.""" + + def __init__(self) -> None: + self.d = xdisplay.Display() + first = self.d.display.info.min_keycode + mapping = self.d.get_keyboard_mapping(first, self.d.display.info.max_keycode - first + 1) + self.width = len(mapping[0]) + self.spare = [first + i for i, syms in enumerate(mapping) if not any(syms)] + self.mapped: OrderedDict[int, int] = OrderedDict() + self.shift = self.d.keysym_to_keycode(XK.string_to_keysym("Shift_L")) + + def move(self, x, y) -> None: + xtest.fake_input(self.d, X.MotionNotify, x=int(x), y=int(y)) + + def button(self, button: str, down: bool) -> None: + xtest.fake_input(self.d, X.ButtonPress if down else X.ButtonRelease, BUTTONS[button]) + + def released(self, button: str) -> None: + """Wait, up to RELEASE_WAIT_S, until the server has let go of ``button``. A window manager's passive grab holds + a click on a window until it replays it to the app, while keys reach the app at once, so a key sent before then + overtakes the click, and so do the next click's moves.""" + deadline = time.monotonic() + RELEASE_WAIT_S + while self.d.screen().root.query_pointer().mask & BUTTON_MASKS.get(button, 0) and time.monotonic() < deadline: + time.sleep(RELEASE_POLL_S) + + def hold(self, keysyms: list[int]) -> list[int]: + """``hold_each``'s keycodes, in the order they were pressed.""" + return [code for pressed in self.hold_each(keysyms) for code in pressed] + + def hold_each(self, keysyms: list[int]) -> list[list[int]]: + """Press what the keysyms need that isn't down already, Shift first where a character needs it; for each + keysym, the keycodes this pressed, so the caller releases only those. Every keycode is found before the first + press, so keysyms that can't all be held at once are refused with nothing sent.""" + down = self._down() + planned: list[list[int]] = [] + for keysym in keysyms: + planned.append(self.codes(keysym, busy=down | {code for codes in planned for code in codes})) + pressed = [] + for codes in planned: + pressed.append([code for code in codes if code not in down]) + for code in pressed[-1]: + xtest.fake_input(self.d, X.KeyPress, code) + down.add(code) + return pressed + + def release(self, codes) -> None: + for code in codes: + xtest.fake_input(self.d, X.KeyRelease, code) + + def tap(self, keysym: int, hold_s: float) -> None: + """Type one character: its key pressed and released, with Shift around it when it needs Shift and nothing + holds Shift already.""" + down = self._down() + *modifiers, code = self.codes(keysym, busy=down) + pressed = [m for m in modifiers if m not in down] + for m in pressed: + xtest.fake_input(self.d, X.KeyPress, m) + xtest.fake_input(self.d, X.KeyPress, code) + self.d.sync() + time.sleep(hold_s) + xtest.fake_input(self.d, X.KeyRelease, code) + self.release(reversed(pressed)) + + def codes(self, keysym: int, busy: set[int] = frozenset()) -> list[int]: + """The keycodes that make ``keysym``: Shift and the key when it is the key's second level, else one key; a + keysym off the keymap is mapped to a spare keycode not in ``busy``.""" + codes = self._on_keymap(keysym) + if codes is not None: + return codes + code = self._free(busy) + self.d.change_keyboard_mapping(code, [(keysym,) * self.width]) + self.d.sync() + self.mapped[keysym] = code + return [code] + + def check(self, keysyms: list[int]) -> None: + """Refuse, before anything is sent, text with a character off the keymap when no spare keycode is free.""" + if any(self._on_keymap(k) is None for k in keysyms): + self._free(self._down(), take=False) + + def _free(self, busy: set[int], take: bool = True) -> int: + """A spare keycode not in ``busy``, the least recently used mapped one when none is unmapped.""" + if self.spare: + return self.spare.pop() if take else self.spare[-1] + for keysym, code in self.mapped.items(): + if code not in busy: + if take: + del self.mapped[keysym] + return code + raise Unsupported("no spare keycode is free for a character off the X keymap: the server has none, or keys " + "hold them all down") + + def _on_keymap(self, keysym: int) -> list[int] | None: + if keysym in self.mapped: + self.mapped.move_to_end(keysym) + return [self.mapped[keysym]] + code = self.d.keysym_to_keycode(keysym) + if code and self.d.keycode_to_keysym(code, 0) == keysym: + return [code] + if code and self.d.keycode_to_keysym(code, 1) == keysym: + return [self.shift, code] + return None + + def _down(self) -> set[int]: + return {8 * i + bit for i, byte in enumerate(self.d.query_keymap()) for bit in range(8) if byte >> bit & 1} + + +def _grab() -> Image: + try: + with mss.MSS() as m: + shot = m.grab(m.monitors[0]) + return Image(shot.width, shot.height, bytes(shot.bgra), "BGRA") + except Exception as e: + raise DeviceUnavailable(f"capture failed: {type(e).__name__}: {e}") from None + + +def _steps(path: list) -> list[tuple[float, float]]: + """The points a drag moves through after its start, about every DRAG_STEP_PX pixels.""" + steps = [] + for (x0, y0), (x1, y1) in zip(path, path[1:]): + n = max(2, int(math.hypot(x1 - x0, y1 - y0) / DRAG_STEP_PX)) + steps += [(x0 + (x1 - x0) * k / n, y0 + (y1 - y0) * k / n) for k in range(1, n + 1)] + return steps + + +def _keysym(key: str) -> int: + if key in KEYSYMS: + return XK.string_to_keysym(KEYSYMS[key]) + if len(key) == 1: + code = ord(key) + return code if code < 0x100 else 0x01000000 | code + raise Unsupported(f"this driver has no X keysym for {key!r}") diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index 23806bb06..356139f01 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -5,8 +5,9 @@ runs a bundle's `desktop` env with another driver for one run, and `--set envs/d the driver's own tools; the bundle stays as written, and what changes is written under ids ending in the settings, such as `desktop(desktop.driver cua-driver)`. -`envs/reference/` is the reference driver, for an X11 desktop on Linux (`amd64` and `arm64`): `mss` captures the -screen and XTEST, through `python-xlib`, sends the input, foreground only. It serves the seven `computer_*` tools, the +`envs/reference/` is the reference driver, for an X11 desktop on Linux (`amd64` and `arm64`): `X11Driver` +(`agentenv_protocol.computer._x11`, the protocol's `x11` extra) as it is, `mss` capturing the screen and XTEST, through +`python-xlib`, sending the input, foreground only. It serves the seven `computer_*` tools, the pointer and keyboard groups, and `computer-ready`, `computer-display`, `computer-spaces`, `computer-keyboard`, and `computer-exec` and `data/add` through the base class's Linux defaults, and `computer-recording`, from FFmpeg's `x11grab` through PyAV at up to 30 fps. Name it from another bundle as `reference`, the @@ -76,24 +77,26 @@ To move it to another cua-driver version: `computer-ready` names any tool or executable that doesn't match its pin. `envs/agent-desktop/` is Vercel's [agent-desktop](https://github.com/vercel-labs/agent-desktop) as a driver, for macOS -(`arm64`, composed onto a VM bundle such as `macos-26`). agent-desktop 1.2.0 reads apps' accessibility trees and acts -on what it finds; its release's tarball, checked by sha256, is unpacked into the driver's `bin/` by `install.sh`. -Agents get what its own MCP server, `agent-desktop mcp`, serves, as `mcp.json` pins it: its instructions and its 67 -`desktop_*` tools, each call forwarded to a server of its own, and its skill, its references and its guide for macOS +(`arm64`, composed onto a VM bundle such as `macos-26`) and for an X11 desktop on Linux (`amd64` and `arm64`). +agent-desktop 1.2.0 reads apps' accessibility trees, AT-SPI's on Linux, and acts on what it finds; its release's +tarball for the OS and arch, checked by sha256, is unpacked into the driver's `bin/` by `install.sh`. +Agents get what its own MCP server, `agent-desktop mcp`, serves, as `mcp.json` pins it, the same on both OSes: its +instructions and its 67 `desktop_*` tools, each call forwarded to a server of its own, and its skill, its references and its guide for the OS leading the computer's skill. The agent doesn't share the computer's files, so each screenshot a call saves to an `output_path`, in a batch too, comes back beside the answer (up to 8 MB in all, a line naming the rest), and the instructions add a line saying other files it writes stay on the computer. The `computer_*` tools, gate 3's probe, `computer-ready` and recording run on -`MacOSDriver` (`agentenv_protocol.computer._macos`): ScreenCaptureKit captures the display and Quartz events, posted -where the hardware's are, send the input, from the driver's own process. `[install.macos]` grants Accessibility, Post +`MacOSDriver` (`agentenv_protocol.computer._macos`) on macOS: ScreenCaptureKit captures the display and Quartz events, +posted where the hardware's are, send the input, from the driver's own process. On Linux they run on `X11Driver`, as +the reference driver's do, and agent-desktop gets the session's display and bus, where it finds AT-SPI. `[install.macos]` grants Accessibility, Post Event and Screen Recording to the driver's Python, its venv's own copy of the pinned one, and the LaunchAgent starts the driver on it, so agent-desktop, which the driver starts, has them too, and nothing else does. A tool can't pass agent-desktop's global flags, so the server's come from the driver's own setting, `headed`, which a deploy passes in `driver_settings`: off, agent-desktop's default, it acts through the accessibility API alone and refuses what needs physical input (double clicks, hovers, drags, the mouse commands); on, it allows those and prefers physical input, -moving the pointer and focus. `computer-ready` checks the two executables' sha256, that the server lists the tools -`mcp.json` pins, and the TCC permissions. To move it to another version, pin the tarball's URL and sha256 in `env.toml` -and the executables' in `driver.py`'s `EXECUTABLES`, and take `mcp.json` again from what that version's server says at -connect and lists, in a macOS computer. Name it as `agent-desktop`, the alias `aliases.toml` gives it. +moving the pointer and focus. `computer-ready` checks the two executables' sha256 for the OS and arch, that the +server lists the tools `mcp.json` pins, and the TCC permissions on macOS, XTEST on Linux. To move it to another version, +pin each tarball's URL and sha256 in `env.toml` and the executables' in `driver.py`'s `EXECUTABLES`, and take `mcp.json` +again from what that version's server says at connect and lists. Name it as `agent-desktop`, the alias `aliases.toml` gives it. `envs/macos-harness/` is browser-use's [macOS harness](https://github.com/browser-use/macos-harness) as a driver, for the same macOS computers. macos-harness 0.2.1, from PyPI into the driver's venv, reads an app as numbered text and acts diff --git a/src/agent_env/drivers/envs/agent-desktop/driver.py b/src/agent_env/drivers/envs/agent-desktop/driver.py index aa077df50..f5d417679 100644 --- a/src/agent_env/drivers/envs/agent-desktop/driver.py +++ b/src/agent_env/drivers/envs/agent-desktop/driver.py @@ -1,14 +1,14 @@ -"""Vercel's agent-desktop as a computer driver on macOS. agent-desktop reads apps' accessibility trees and acts on what -it finds; its own MCP server, ``agent-desktop mcp``, serves each of its commands as a tool, and agents get what that +"""Vercel's agent-desktop as a computer driver on macOS and on an X11 desktop on Linux. agent-desktop reads apps' +accessibility trees, AT-SPI's on Linux, and acts on what it finds; its own MCP server, ``agent-desktop mcp``, serves each of its commands as a tool, and agents get what that server serves at its pinned version (``mcp.json``): its instructions and its tools, each call forwarded to a server of its own and answered as the server answers it, with each screenshot the call saved to a file beside the answer, since the agent can't read the computer's files; agent-desktop's skill leads the computer's. The computer/v1 -operations are MacOSDriver's. This process's Python is granted Accessibility, Post Event and Screen Recording, and -agent-desktop, which it starts, has them as its own. A tool can't pass agent-desktop's global flags, so the server's +operations are MacOSDriver's on macOS and X11Driver's on Linux. On macOS this process's Python is granted +Accessibility, Post Event and Screen Recording, and agent-desktop, which it starts, has them as its own. A tool can't pass agent-desktop's global flags, so the server's come from the driver's ``headed`` setting: off, agent-desktop's default, it acts through the accessibility API alone and refuses what needs physical input (double clicks, hovers, drags, the mouse commands); on, it allows those and prefers -physical input, moving the pointer and focus. It needs ``agentenv-framework-protocol[computer,macos]`` and -agent-desktop's release where it runs.""" +physical input, moving the pointer and focus. It needs ``agentenv-framework-protocol[computer,macos]`` on macOS or +``[computer,x11]`` on Linux, and agent-desktop's release where it runs.""" from __future__ import annotations import asyncio @@ -18,6 +18,7 @@ import hashlib import json import os +import sys from pathlib import Path from typing import Any, AsyncIterator @@ -26,20 +27,36 @@ from mcp.types import CallToolResult, ImageContent, TextContent from agentenv_protocol import EnvironmentTool, ForwardedTool, server_instructions -from agentenv_protocol.computer import ComputerSettings, MacOSDriver, ReadyCheck, computer_driver +from agentenv_protocol.computer import ComputerSettings, ReadyCheck, computer_driver + +MACOS = sys.platform == "darwin" +if MACOS: + from agentenv_protocol.computer import MacOSDriver as OSDriver +else: + from agentenv_protocol.computer import X11Driver as OSDriver HERE = Path(__file__).resolve().parent -#: agent-desktop's executables, as its release unpacks into bin/, and the sha256 of each at the pinned version, 1.2.0. +#: agent-desktop's executables, as its release for each OS and arch unpacks into bin/, and the sha256 of each at the +#: pinned version, 1.2.0. BIN = HERE / "bin" -EXECUTABLES = {"agent-desktop": "dec75e0bba0d1f9b589bdb88188770ad1a199cde9e02cf0a5b94175491f738e9", - "agent-desktop-macos-helper": "4a795a40ba7fc93b726b0834342f0225779c30d83e28f1f219db479860c38a78"} +EXECUTABLES = { + "macos/arm64": {"agent-desktop": "dec75e0bba0d1f9b589bdb88188770ad1a199cde9e02cf0a5b94175491f738e9", + "agent-desktop-macos-helper": "4a795a40ba7fc93b726b0834342f0225779c30d83e28f1f219db479860c38a78"}, + "amd64": {"agent-desktop": "da52a1eb2152cdc21b756ab20e6208d242da8062c73cd20bc8a8ae2a0384392c", + "agent-desktop-linux-clipboard-owner": "3e96f92b805c0d1f9de309935d62f4189b8e11e1385ad022bb878487bb3942ec"}, + "arm64": {"agent-desktop": "915de59a8b3d22c9ad33023740bbcc5fe0b4fa830de9587bbf761237eafeb76f", + "agent-desktop-linux-clipboard-owner": "a6bfde6778dadcf9fd04ba94de17cdf205eb0b23b643c3f34073d264d2117cc3"}, +} EXECUTABLE = BIN / "agent-desktop" -#: What ``agent-desktop mcp`` says at connect and lists at the pinned version: its instructions and its tools. +#: What ``agent-desktop mcp`` says at connect and lists at the pinned version, the same on macOS and Linux: its +#: instructions and its tools. MANIFEST = HERE / "mcp.json" -#: What of the driver's environment agent-desktop gets: the session user's own, none of a deploy's values. -KEPT = ("HOME", "USER", "LOGNAME", "TMPDIR", "LANG") +#: What of the driver's environment agent-desktop gets: the session user's own, and on Linux its display and the +#: session bus AT-SPI is found on; none of a deploy's values. +KEPT = ("HOME", "USER", "LOGNAME", "TMPDIR", "LANG") + ( + () if MACOS else ("DISPLAY", "XAUTHORITY", "DBUS_SESSION_BUS_ADDRESS", "XDG_RUNTIME_DIR")) #: The PATH it runs, and launches apps, with. -PATH = "/usr/bin:/bin:/usr/sbin:/sbin" +PATH = "/usr/bin:/bin:/usr/sbin:/sbin" if MACOS else "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" #: How long one of its commands that reads its skills may take. SKILL_TIMEOUT_S = 60 #: The most bytes of saved screenshots one answer carries. @@ -50,8 +67,9 @@ "export, can't be read from where you run.") -@computer_driver(name="agent-desktop", supports=[{"os": {"type": "macos", "display": ["aqua"]}, "arch": ["arm64"]}]) -class AgentDesktop(MacOSDriver): +@computer_driver(name="agent-desktop", supports=[{"os": {"type": "macos", "display": ["aqua"]}, "arch": ["arm64"]}, + {"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) +class AgentDesktop(OSDriver): class Settings(ComputerSettings): headed: bool = False @@ -71,7 +89,7 @@ def agent_desktop_instructions(self) -> str: async def ready_checks(self) -> list[ReadyCheck]: """agent-desktop's executables and the tools its server lists, as pinned, checked until they pass once, and the - TCC permissions, every time.""" + OS's own checks, the TCC permissions on macOS and XTEST on Linux, every time.""" if self._pinned is None or not all(check.ok for check in self._pinned): self._pinned = await self._pinned_checks() return [*self._pinned, *await super().ready_checks()] @@ -113,8 +131,9 @@ async def _server(self) -> AsyncIterator[ClientSession]: yield session async def _pinned_checks(self) -> list[ReadyCheck]: - digests = await asyncio.to_thread(_digests) - differ = sorted(name for name, digest in digests.items() if digest != EXECUTABLES[name]) + pinned = EXECUTABLES[f"macos/{self.platform.arch}" if MACOS else self.platform.arch] + digests = await asyncio.to_thread(_digests, pinned) + differ = sorted(name for name, digest in digests.items() if digest != pinned[name]) checks = [ReadyCheck("agent-desktop", not differ, "the pinned executables" if not differ else f"these differ from the pinned agent-desktop 1.2.0: {', '.join(differ)}")] if differ: @@ -181,9 +200,9 @@ def _environment() -> dict[str, str]: return {**{name: os.environ[name] for name in KEPT if name in os.environ}, "PATH": PATH} -def _digests() -> dict[str, str | None]: +def _digests(names) -> dict[str, str | None]: return {name: hashlib.sha256((BIN / name).read_bytes()).hexdigest() if (BIN / name).is_file() else None - for name in EXECUTABLES} + for name in names} async def _agent_desktop(*words: str) -> dict: diff --git a/src/agent_env/drivers/envs/agent-desktop/env.toml b/src/agent_env/drivers/envs/agent-desktop/env.toml index 3cefe2a5e..1a5d66872 100644 --- a/src/agent_env/drivers/envs/agent-desktop/env.toml +++ b/src/agent_env/drivers/envs/agent-desktop/env.toml @@ -2,6 +2,15 @@ type = "computer_driver" module = "driver.py:AgentDesktop" port = 18765 +[install.linux] +python = "requirements.txt" +downloads = [ + { url = "https://github.com/vercel-labs/agent-desktop/releases/download/v1.2.0/agent-desktop-v1.2.0-x86_64-unknown-linux-gnu.tar.gz", sha256 = "591ae7f4a38479dc045df97e27447c0c8b530441730ab6b990e27c813162af44", to = "agent-desktop.tar.gz", arch = "amd64" }, + { url = "https://github.com/vercel-labs/agent-desktop/releases/download/v1.2.0/agent-desktop-v1.2.0-aarch64-unknown-linux-gnu.tar.gz", sha256 = "68e8ebbb4f80e44485d2f1cf972943a38ee43108e2e0ce56bdfa641fd1599f68", to = "agent-desktop.tar.gz", arch = "arm64" }, +] +run = [{ script = "install.sh", scope = "session" }] +services = [{ name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:AgentDesktop" }] + [install.macos] python = "requirements-macos.txt" downloads = [ diff --git a/src/agent_env/drivers/envs/agent-desktop/install.sh b/src/agent_env/drivers/envs/agent-desktop/install.sh index c4abe8b3b..0689e1728 100755 --- a/src/agent_env/drivers/envs/agent-desktop/install.sh +++ b/src/agent_env/drivers/envs/agent-desktop/install.sh @@ -1,8 +1,8 @@ #!/bin/sh # agent-desktop's release, from the tarball the install downloaded and checked, in bin/ beside the driver, as the session -# user, whom its helper must belong to, with the quarantine a download carries removed. +# user, whom its macOS helper must belong to, with the quarantine a download carries on macOS removed. set -eu mkdir -p bin tar -xzf agent-desktop.tar.gz -C bin -xattr -dr com.apple.quarantine bin || true +[ "$(uname)" != Darwin ] || xattr -dr com.apple.quarantine bin || true rm -f agent-desktop.tar.gz diff --git a/src/agent_env/drivers/envs/agent-desktop/requirements.txt b/src/agent_env/drivers/envs/agent-desktop/requirements.txt new file mode 100644 index 000000000..3a944ae7d --- /dev/null +++ b/src/agent_env/drivers/envs/agent-desktop/requirements.txt @@ -0,0 +1,4 @@ +# What a composed Linux computer installs into the driver's venv: the protocol with X11Driver, whose screen and input +# need its x11 extra, and the MCP client the forwarded tools reach agent-desktop's server with. A release pins it to the +# version it ships. agent-desktop itself is its release's executables, whose sha256 the driver checks. +agentenv-framework-protocol[computer,x11]>=0.1.470 diff --git a/src/agent_env/drivers/envs/reference/driver.py b/src/agent_env/drivers/envs/reference/driver.py index ff3b4965d..5b4b41f1b 100644 --- a/src/agent_env/drivers/envs/reference/driver.py +++ b/src/agent_env/drivers/envs/reference/driver.py @@ -1,418 +1,14 @@ -"""The reference computer driver, for an X11 desktop: mss captures the screen and XTEST, through python-xlib, sends the -input. X calls run off the event loop, one at a time, on a connection checked before each call and reopened when it -drops, so the driver outlives a restarted X server, and input that lets go of a button returns once the server has let -go of it. It records the display through FFmpeg's ``x11grab``, by PyAV. It -needs ``agentenv-framework-protocol[computer]``, ``mss`` and ``python-xlib`` where it runs.""" +"""The reference computer driver, for an X11 desktop: computer/v1 as ``X11Driver`` sends it, mss capturing the screen and +XTEST, through python-xlib, sending the input. It needs ``agentenv-framework-protocol[computer,x11]`` where it runs.""" from __future__ import annotations -import asyncio -import contextlib -import math -import os -import threading -import time -from collections import OrderedDict -from datetime import datetime, timezone -from typing import Any, AsyncIterator, Callable, Iterator - -import av -import mss -from Xlib import X, XK -from Xlib import display as xdisplay -from Xlib.error import ConnectionClosedError, DisplayConnectionError -from Xlib.ext import xtest - -from agentenv_protocol.computer import ( - ComputerDriverEnvironment, - DeliveryFailed, - DeviceUnavailable, - Display, - Image, - ReadyCheck, - RecordedFrame, - RecordHints, - Recording, - Unsupported, - capture, - click, - computer_driver, - display, - drag, - frames_in_thread, - key_down, - key_up, - pointer_down, - pointer_move, - pointer_position, - pointer_up, - press, - record, - scroll, - type_text, -) - -BUTTONS = {"left": 1, "middle": 2, "right": 3, "back": 8, "forward": 9} -#: The pointer state's bit for each button that has one. -BUTTON_MASKS = {"left": X.Button1Mask, "middle": X.Button2Mask, "right": X.Button3Mask} -SCROLL = {"up": 4, "down": 5, "left": 6, "right": 7} -#: W3C key values and their X keysym names; any other single character is typed as its own keysym. -KEYSYMS = { - "Enter": "Return", "Tab": "Tab", "Escape": "Escape", "Backspace": "BackSpace", "Delete": "Delete", - "ArrowUp": "Up", "ArrowDown": "Down", "ArrowLeft": "Left", "ArrowRight": "Right", "Home": "Home", "End": "End", - "PageUp": "Prior", "PageDown": "Next", "Insert": "Insert", "Control": "Control_L", "Shift": "Shift_L", - "Alt": "Alt_L", "Meta": "Super_L", "Super": "Super_L", "CapsLock": "Caps_Lock", "ContextMenu": "Menu", - "PrintScreen": "Print", "Pause": "Pause", " ": "space", **{f"F{n}": f"F{n}" for n in range(1, 25)}, -} -TYPED = {"\n": "Return", "\t": "Tab"} -#: The pause between the presses of a double or triple click, in seconds. -CLICK_GAP_S = 0.04 -#: How long a button's release may take to reach the app, and how often the driver looks. -RELEASE_WAIT_S = 5.0 -RELEASE_POLL_S = 0.005 -#: How far apart a drag's pointer moves are, in pixels. -DRAG_STEP_PX = 20 -DROPPED = (ConnectionClosedError, DisplayConnectionError, OSError) +from agentenv_protocol.computer import X11Driver, computer_driver @computer_driver(name="reference", supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) -class ReferenceDriver(ComputerDriverEnvironment): - buttons = ("left", "right", "middle", "back", "forward") - max_click_count = 3 - max_drag_points = 50 - recording = Recording(max_fps=30) - - def __init__(self) -> None: - super().__init__() - self._x: _Connection | None = None - self._x_lock = threading.Lock() - #: The keycodes each key held by @key_down pressed, which @key_up releases. - self._held: dict[str, list[int]] = {} - - @display - async def screen(self) -> Display: - def read(c): - s = c.d.screen() - return Display(s.width_in_pixels, s.height_in_pixels, 1.0) - return await self._read(read) - - @capture - async def grab(self) -> Image: - return await asyncio.to_thread(_grab) - - @record - async def stream(self, hints: RecordHints) -> AsyncIterator[RecordedFrame]: - """The display through FFmpeg's ``x11grab``. Its frames carry the wall clock, so each one's time is when it - was captured.""" - size = await self.screen() - options = {"framerate": str(hints.fps), "video_size": f"{size.width}x{size.height}", - "draw_mouse": "1" if hints.cursor else "0"} - - def frames() -> Iterator[RecordedFrame]: - with av.open(os.environ.get("DISPLAY", ":0"), format="x11grab", options=options) as source: - for frame in source.decode(video=0): - yield RecordedFrame(frame, datetime.fromtimestamp(frame.time, timezone.utc)) - - async for frame in frames_in_thread(frames): - yield frame - - async def ready_checks(self) -> list[ReadyCheck]: - try: - has = await self._read(lambda c: c.d.has_extension("XTEST")) - except DeviceUnavailable as e: - return [ReadyCheck("xtest", False, e.message)] - return [ReadyCheck("xtest", bool(has), "XTEST available" if has else "XTEST missing")] - - @click - async def click_at(self, x, y, button, count, modifiers) -> None: - syms = [_keysym(k) for k in modifiers] - - def act(c): - held = c.hold(syms) - c.move(x, y) - for i in range(count): - c.button(button, True) - c.button(button, False) - if i + 1 < count: - c.d.sync() - time.sleep(CLICK_GAP_S) - c.released(button) - c.release(reversed(held)) - await self._send(act) - - @drag - async def drag_along(self, path, button, duration_ms, modifiers) -> None: - syms = [_keysym(k) for k in modifiers] - steps = _steps(path) - pause = duration_ms / 1000 / len(steps) - held: list[int] = [] - - def start(c): - held.extend(c.hold(syms)) - c.move(*path[0]) - c.button(button, True) - - def finish(c): - c.button(button, False) - c.released(button) - c.release(reversed(held)) - - try: - await self._send(start) - for px, py in steps: - await self._send(lambda c, px=px, py=py: c.move(px, py)) - await asyncio.sleep(pause) - except BaseException: - with contextlib.suppress(Exception): - await self._send(finish) - raise - await self._send(finish) - - @scroll - async def scroll_at(self, x, y, direction, amount, modifiers) -> None: - syms = [_keysym(k) for k in modifiers] - - def act(c): - held = c.hold(syms) - c.move(x, y) - for _ in range(amount): - xtest.fake_input(c.d, X.ButtonPress, SCROLL[direction]) - xtest.fake_input(c.d, X.ButtonRelease, SCROLL[direction]) - c.release(reversed(held)) - await self._send(act) - - @pointer_move - async def move(self, x, y) -> None: - await self._send(lambda c: c.move(x, y)) - - @pointer_down - async def down(self, button) -> None: - await self._send(lambda c: c.button(button, True)) - - @pointer_up - async def up(self, button) -> None: - def act(c): - c.button(button, False) - c.released(button) - await self._send(act) - - @pointer_position - async def where(self) -> tuple[int, int]: - def read(c): - p = c.d.screen().root.query_pointer() - return p.root_x, p.root_y - return await self._read(read) - - @type_text - async def type_in(self, text) -> None: - syms = [XK.string_to_keysym(TYPED[ch]) if ch in TYPED else _keysym(ch) for ch in text] - await self._read(lambda c: c.check(syms)) - for sym in syms: - hold = self.settings.tap_hold_ms / 1000 - await self._send(lambda c, sym=sym: c.tap(sym, hold)) - await asyncio.sleep(self.settings.type_delay_ms / 1000) - - @press - async def press_keys(self, keys) -> None: - syms = [_keysym(k) for k in keys] - hold = self.settings.tap_hold_ms / 1000 - - def act(c): - pressed = c.hold(syms) - c.d.sync() - time.sleep(hold) - c.release(reversed(pressed)) - await self._send(act) - - @key_down - async def keys_down(self, keys) -> None: - syms = [_keysym(k) for k in keys] - - def act(c): - for key, pressed in zip(keys, c.hold_each(syms)): - self._held.setdefault(key, []).extend(pressed) - await self._send(act) - - @key_up - async def keys_up(self, keys) -> None: - def act(c): - for key in keys: - c.release(reversed(self._held.pop(key, None) or c.codes(_keysym(key))[-1:])) - await self._send(act) - - async def _read(self, fn: Callable[[_Connection], Any]) -> Any: - return await asyncio.to_thread(self._x_call, fn, False) - - async def _send(self, fn: Callable[[_Connection], Any]) -> Any: - return await asyncio.to_thread(self._x_call, fn, True) - - def _x_call(self, fn: Callable[[_Connection], Any], sends_input: bool) -> Any: - """``fn`` on a live connection. When the connection drops, a read is retried once on a new one; input is not - sent again, since some of it may have arrived.""" - with self._x_lock: - for attempt in (1, 2): - c = self._connected() - try: - result = fn(c) - c.d.sync() - return result - except DROPPED as e: - self._x = None - if sends_input: - raise DeliveryFailed(f"the X connection dropped while sending input: {type(e).__name__}") from None - if attempt == 2: - raise DeviceUnavailable(f"the X server is unavailable: {type(e).__name__}") from None - - def _connected(self) -> _Connection: - """The connection, opened or reopened as needed and checked with a round trip before anything is sent.""" - for attempt in (1, 2): - try: - if self._x is None: - self._x = _Connection() - self._held.clear() - self._x.d.sync() - return self._x - except DROPPED as e: - self._x = None - if attempt == 2: - raise DeviceUnavailable(f"the X server is unavailable: {type(e).__name__}") from None - - -class _Connection: - """One connection to the X server, with XTEST input on it. A character the keymap lacks is typed through a spare - keycode mapped to it, kept for that character and reused; once all are taken, the least recently used one that - no key holds down is mapped again.""" - - def __init__(self) -> None: - self.d = xdisplay.Display() - first = self.d.display.info.min_keycode - mapping = self.d.get_keyboard_mapping(first, self.d.display.info.max_keycode - first + 1) - self.width = len(mapping[0]) - self.spare = [first + i for i, syms in enumerate(mapping) if not any(syms)] - self.mapped: OrderedDict[int, int] = OrderedDict() - self.shift = self.d.keysym_to_keycode(XK.string_to_keysym("Shift_L")) - - def move(self, x, y) -> None: - xtest.fake_input(self.d, X.MotionNotify, x=int(x), y=int(y)) - - def button(self, button: str, down: bool) -> None: - xtest.fake_input(self.d, X.ButtonPress if down else X.ButtonRelease, BUTTONS[button]) - - def released(self, button: str) -> None: - """Wait, up to RELEASE_WAIT_S, until the server has let go of ``button``. A window manager's passive grab holds - a click on a window until it replays it to the app, while keys reach the app at once, so a key sent before then - overtakes the click, and so do the next click's moves.""" - deadline = time.monotonic() + RELEASE_WAIT_S - while self.d.screen().root.query_pointer().mask & BUTTON_MASKS.get(button, 0) and time.monotonic() < deadline: - time.sleep(RELEASE_POLL_S) - - def hold(self, keysyms: list[int]) -> list[int]: - """``hold_each``'s keycodes, in the order they were pressed.""" - return [code for pressed in self.hold_each(keysyms) for code in pressed] - - def hold_each(self, keysyms: list[int]) -> list[list[int]]: - """Press what the keysyms need that isn't down already, Shift first where a character needs it; for each - keysym, the keycodes this pressed, so the caller releases only those. Every keycode is found before the first - press, so keysyms that can't all be held at once are refused with nothing sent.""" - down = self._down() - planned: list[list[int]] = [] - for keysym in keysyms: - planned.append(self.codes(keysym, busy=down | {code for codes in planned for code in codes})) - pressed = [] - for codes in planned: - pressed.append([code for code in codes if code not in down]) - for code in pressed[-1]: - xtest.fake_input(self.d, X.KeyPress, code) - down.add(code) - return pressed - - def release(self, codes) -> None: - for code in codes: - xtest.fake_input(self.d, X.KeyRelease, code) - - def tap(self, keysym: int, hold_s: float) -> None: - """Type one character: its key pressed and released, with Shift around it when it needs Shift and nothing - holds Shift already.""" - down = self._down() - *modifiers, code = self.codes(keysym, busy=down) - pressed = [m for m in modifiers if m not in down] - for m in pressed: - xtest.fake_input(self.d, X.KeyPress, m) - xtest.fake_input(self.d, X.KeyPress, code) - self.d.sync() - time.sleep(hold_s) - xtest.fake_input(self.d, X.KeyRelease, code) - self.release(reversed(pressed)) - - def codes(self, keysym: int, busy: set[int] = frozenset()) -> list[int]: - """The keycodes that make ``keysym``: Shift and the key when it is the key's second level, else one key; a - keysym off the keymap is mapped to a spare keycode not in ``busy``.""" - codes = self._on_keymap(keysym) - if codes is not None: - return codes - code = self._free(busy) - self.d.change_keyboard_mapping(code, [(keysym,) * self.width]) - self.d.sync() - self.mapped[keysym] = code - return [code] - - def check(self, keysyms: list[int]) -> None: - """Refuse, before anything is sent, text with a character off the keymap when no spare keycode is free.""" - if any(self._on_keymap(k) is None for k in keysyms): - self._free(self._down(), take=False) - - def _free(self, busy: set[int], take: bool = True) -> int: - """A spare keycode not in ``busy``, the least recently used mapped one when none is unmapped.""" - if self.spare: - return self.spare.pop() if take else self.spare[-1] - for keysym, code in self.mapped.items(): - if code not in busy: - if take: - del self.mapped[keysym] - return code - raise Unsupported("no spare keycode is free for a character off the X keymap: the server has none, or keys " - "hold them all down") - - def _on_keymap(self, keysym: int) -> list[int] | None: - if keysym in self.mapped: - self.mapped.move_to_end(keysym) - return [self.mapped[keysym]] - code = self.d.keysym_to_keycode(keysym) - if code and self.d.keycode_to_keysym(code, 0) == keysym: - return [code] - if code and self.d.keycode_to_keysym(code, 1) == keysym: - return [self.shift, code] - return None - - def _down(self) -> set[int]: - return {8 * i + bit for i, byte in enumerate(self.d.query_keymap()) for bit in range(8) if byte >> bit & 1} - - -def _grab() -> Image: - try: - with mss.MSS() as m: - shot = m.grab(m.monitors[0]) - return Image(shot.width, shot.height, bytes(shot.bgra), "BGRA") - except Exception as e: - raise DeviceUnavailable(f"capture failed: {type(e).__name__}: {e}") from None - - -def _steps(path: list) -> list[tuple[float, float]]: - """The points a drag moves through after its start, about every DRAG_STEP_PX pixels.""" - steps = [] - for (x0, y0), (x1, y1) in zip(path, path[1:]): - n = max(2, int(math.hypot(x1 - x0, y1 - y0) / DRAG_STEP_PX)) - steps += [(x0 + (x1 - x0) * k / n, y0 + (y1 - y0) * k / n) for k in range(1, n + 1)] - return steps - - -def _keysym(key: str) -> int: - if key in KEYSYMS: - return XK.string_to_keysym(KEYSYMS[key]) - if len(key) == 1: - code = ord(key) - return code if code < 0x100 else 0x01000000 | code - raise Unsupported(f"this driver has no X keysym for {key!r}") +class ReferenceDriver(X11Driver): + pass if __name__ == "__main__": diff --git a/src/agent_env/drivers/envs/reference/requirements.txt b/src/agent_env/drivers/envs/reference/requirements.txt index c9bafe585..c7712b70c 100644 --- a/src/agent_env/drivers/envs/reference/requirements.txt +++ b/src/agent_env/drivers/envs/reference/requirements.txt @@ -1,5 +1,3 @@ # What a composed computer installs into the driver's venv. A release pins agentenv-framework-protocol to the version -# it ships, which serves the driver with agentenv_protocol.computer.serve. -agentenv-framework-protocol[computer]>=0.1.381 -mss>=9,<11 -python-xlib>=0.33,<0.34 +# it ships, which serves the driver with agentenv_protocol.computer.serve, and X11Driver with its x11 extra. +agentenv-framework-protocol[computer,x11]>=0.1.470 diff --git a/tst/unit/drivers/agent_desktop_driver_test.py b/tst/unit/drivers/agent_desktop_driver_test.py index df7730145..40112d196 100644 --- a/tst/unit/drivers/agent_desktop_driver_test.py +++ b/tst/unit/drivers/agent_desktop_driver_test.py @@ -1,7 +1,8 @@ -"""The agent-desktop driver against a fake agent-desktop: its instructions and tools are its server's as pinned, each call -reaches a server of its own, ``--headed`` when the driver's setting says, with none of the driver's environment, and -the screenshots it saves come back beside its answer; a server that fails answers an error; readiness checks the pinned executables and the tools the server lists; and its -skill is agent-desktop's own, with its references and its guide for macOS.""" +"""The agent-desktop driver against a fake agent-desktop, on macOS and on Linux: its instructions and tools are its +server's as pinned, each call reaches a server of its own, ``--headed`` when the driver's setting says, with none of the +driver's environment, and the screenshots it saves come back beside its answer; a server that fails answers an error; +readiness checks the OS's pinned executables, the tools the server lists and the OS's own checks; and its skill is +agent-desktop's own, with its references and its guide for the OS.""" from __future__ import annotations import base64 @@ -10,12 +11,13 @@ import json import sys import textwrap +import types from pathlib import Path import pytest import agent_env -from agentenv_protocol.computer import _macos +from agentenv_protocol.computer import Platform, ReadyCheck, _macos DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "agent-desktop" / "driver.py" FAKE = textwrap.dedent('''\ @@ -66,32 +68,64 @@ async def main(): ''') +PLATFORMS = {"macos": Platform(os="macos", version="26.5", arch="arm64", display="aqua"), + "linux": Platform(os="linux", distribution="ubuntu", version="24.04", arch="amd64", display="x11")} +HELPERS = {"macos": "agent-desktop-macos-helper", "linux": "agent-desktop-linux-clipboard-owner"} + + class Granted: def missing(self) -> list[str]: return [] -@pytest.fixture -def module(monkeypatch, tmp_path): +async def xtest(self) -> list[ReadyCheck]: + return [ReadyCheck("xtest", True, "XTEST available")] + + +@pytest.fixture(params=["macos", "linux"]) +def module(request, monkeypatch, tmp_path): monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) monkeypatch.setenv("HOME", str(tmp_path)) monkeypatch.setenv("AGENTENV_CONTROL_TOKEN", "a-deploys-secret") monkeypatch.setattr(_macos, "Quartz", Granted) + monkeypatch.setattr(sys, "platform", "darwin" if request.param == "macos" else "linux") + monkeypatch.setattr("agentenv_protocol.computer.driver.detect", lambda: PLATFORMS[request.param]) + if request.param == "linux": + _fake_x11(monkeypatch) spec = importlib.util.spec_from_file_location("agent_desktop_driver", DRIVER) loaded = importlib.util.module_from_spec(spec) spec.loader.exec_module(loaded) + if request.param == "linux": + monkeypatch.setattr(loaded.OSDriver, "ready_checks", xtest) (tmp_path / "fake").mkdir() bin_ = tmp_path / "bin" bin_.mkdir() (bin_ / "agent-desktop").write_text(f"#!{sys.executable}\n{FAKE}") (bin_ / "agent-desktop").chmod(0o755) - (bin_ / "agent-desktop-macos-helper").write_text("helper") + (bin_ / HELPERS[request.param]).write_text("helper") (tmp_path / "fake" / "tools.json").write_text(loaded.MANIFEST.read_text()) monkeypatch.setattr(loaded, "BIN", bin_) monkeypatch.setattr(loaded, "EXECUTABLE", bin_ / "agent-desktop") return loaded +def _fake_x11(monkeypatch) -> None: + """mss and python-xlib as inert stand-ins, enough for X11Driver's module to load, and that module loaded afresh.""" + def module(name: str, **attrs) -> types.ModuleType: + m = types.ModuleType(name) + m.__dict__.update(attrs) + monkeypatch.setitem(sys.modules, name, m) + return m + + X = module("Xlib.X", Button1Mask=1 << 8, Button2Mask=1 << 9, Button3Mask=1 << 10) + xtest_ = module("Xlib.ext.xtest") + module("Xlib", X=X, XK=module("Xlib.XK"), display=module("Xlib.display")) + module("Xlib.error", ConnectionClosedError=OSError, DisplayConnectionError=OSError) + module("Xlib.ext", xtest=xtest_) + module("mss") + monkeypatch.delitem(sys.modules, "agentenv_protocol.computer._x11", raising=False) + + def _calls(home: Path) -> list[dict]: return [json.loads(line) for line in (home / "fake" / "argv.jsonl").read_text().splitlines()] @@ -164,8 +198,9 @@ async def test_a_server_that_fails_before_it_answers_is_an_error_saying_the_comm async def test_readiness_checks_the_pinned_executables_then_the_tools_its_server_lists(module, tmp_path, monkeypatch): driver = module.AgentDesktop() (wrong, *_) = await driver.ready_checks() - monkeypatch.setattr(module, "EXECUTABLES", {name: hashlib.sha256((tmp_path / "bin" / name).read_bytes()).hexdigest() - for name in module.EXECUTABLES}) + key = "macos/arm64" if module.MACOS else "amd64" + monkeypatch.setitem(module.EXECUTABLES, key, {name: hashlib.sha256((tmp_path / "bin" / name).read_bytes()).hexdigest() + for name in module.EXECUTABLES[key]}) ready = await driver.ready_checks() manifest = json.loads(module.MANIFEST.read_text()) manifest["tools"][0]["inputSchema"] = {"type": "object", "properties": {}} @@ -173,9 +208,10 @@ async def test_readiness_checks_the_pinned_executables_then_the_tools_its_server driver._pinned = None (_, changed, _) = await driver.ready_checks() - assert not wrong.ok and "agent-desktop, agent-desktop-macos-helper" in wrong.detail + helper = "agent-desktop-macos-helper" if module.MACOS else "agent-desktop-linux-clipboard-owner" + assert not wrong.ok and f"agent-desktop, {helper}" in wrong.detail assert [(check.name, check.ok) for check in ready] == [("agent-desktop", True), ("agent-desktop tools", True), - ("macOS permissions", True)] + ("macOS permissions" if module.MACOS else "xtest", True)] assert not changed.ok and changed.detail == "these tools differ from mcp.json: desktop_snapshot" diff --git a/tst/unit/drivers/reference_driver_test.py b/tst/unit/drivers/reference_driver_test.py index 91c05bd2c..a71298a61 100644 --- a/tst/unit/drivers/reference_driver_test.py +++ b/tst/unit/drivers/reference_driver_test.py @@ -176,6 +176,7 @@ def module(name: str, **attrs) -> types.ModuleType: module("Xlib", X=X, XK=XK, display=display) module("mss", MSS=FakeMSS) module("av", open=FakeGrab.open) + monkeypatch.delitem(sys.modules, "agentenv_protocol.computer._x11", raising=False) monkeypatch.setattr("agentenv_protocol.computer.driver.detect", lambda: LINUX) monkeypatch.delenv("ENVIRONMENT_NAME", raising=False) monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) From 91b9164b9f04af013cf717ed20b18abd0f848c0b Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 6 Nov 2026 08:54:24 -0800 Subject: [PATCH 2/5] fix(drivers): pin the protocol the X11 drivers need at the workspace's version, which a checkout builds from Co-Authored-By: Claude Opus 5.5 --- src/agent_env/drivers/envs/agent-desktop/requirements.txt | 2 +- src/agent_env/drivers/envs/reference/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agent_env/drivers/envs/agent-desktop/requirements.txt b/src/agent_env/drivers/envs/agent-desktop/requirements.txt index 3a944ae7d..80deff123 100644 --- a/src/agent_env/drivers/envs/agent-desktop/requirements.txt +++ b/src/agent_env/drivers/envs/agent-desktop/requirements.txt @@ -1,4 +1,4 @@ # What a composed Linux computer installs into the driver's venv: the protocol with X11Driver, whose screen and input # need its x11 extra, and the MCP client the forwarded tools reach agent-desktop's server with. A release pins it to the # version it ships. agent-desktop itself is its release's executables, whose sha256 the driver checks. -agentenv-framework-protocol[computer,x11]>=0.1.470 +agentenv-framework-protocol[computer,x11]>=0.1.469 diff --git a/src/agent_env/drivers/envs/reference/requirements.txt b/src/agent_env/drivers/envs/reference/requirements.txt index c7712b70c..6f46c32d8 100644 --- a/src/agent_env/drivers/envs/reference/requirements.txt +++ b/src/agent_env/drivers/envs/reference/requirements.txt @@ -1,3 +1,3 @@ # What a composed computer installs into the driver's venv. A release pins agentenv-framework-protocol to the version # it ships, which serves the driver with agentenv_protocol.computer.serve, and X11Driver with its x11 extra. -agentenv-framework-protocol[computer,x11]>=0.1.470 +agentenv-framework-protocol[computer,x11]>=0.1.469 From 5bdfc767864b1ba2b60c26f27ac7c8db56b08ceb Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 6 Nov 2026 09:06:00 -0800 Subject: [PATCH 3/5] fix(drivers): agent-desktop's Linux install unpacks as root, which owns the driver's folder Co-Authored-By: Claude Opus 5.5 --- src/agent_env/drivers/envs/agent-desktop/env.toml | 2 +- src/agent_env/drivers/envs/agent-desktop/install.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/agent_env/drivers/envs/agent-desktop/env.toml b/src/agent_env/drivers/envs/agent-desktop/env.toml index 1a5d66872..9c71402cb 100644 --- a/src/agent_env/drivers/envs/agent-desktop/env.toml +++ b/src/agent_env/drivers/envs/agent-desktop/env.toml @@ -8,7 +8,7 @@ downloads = [ { url = "https://github.com/vercel-labs/agent-desktop/releases/download/v1.2.0/agent-desktop-v1.2.0-x86_64-unknown-linux-gnu.tar.gz", sha256 = "591ae7f4a38479dc045df97e27447c0c8b530441730ab6b990e27c813162af44", to = "agent-desktop.tar.gz", arch = "amd64" }, { url = "https://github.com/vercel-labs/agent-desktop/releases/download/v1.2.0/agent-desktop-v1.2.0-aarch64-unknown-linux-gnu.tar.gz", sha256 = "68e8ebbb4f80e44485d2f1cf972943a38ee43108e2e0ce56bdfa641fd1599f68", to = "agent-desktop.tar.gz", arch = "arm64" }, ] -run = [{ script = "install.sh", scope = "session" }] +run = [{ script = "install.sh" }] services = [{ name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:AgentDesktop" }] [install.macos] diff --git a/src/agent_env/drivers/envs/agent-desktop/install.sh b/src/agent_env/drivers/envs/agent-desktop/install.sh index 0689e1728..e53986ea5 100755 --- a/src/agent_env/drivers/envs/agent-desktop/install.sh +++ b/src/agent_env/drivers/envs/agent-desktop/install.sh @@ -1,6 +1,6 @@ #!/bin/sh -# agent-desktop's release, from the tarball the install downloaded and checked, in bin/ beside the driver, as the session -# user, whom its macOS helper must belong to, with the quarantine a download carries on macOS removed. +# agent-desktop's release, from the tarball the install downloaded and checked, in bin/ beside the driver: on macOS as the +# session user, whom its helper must belong to, with the quarantine a download carries removed, and on Linux as root. set -eu mkdir -p bin tar -xzf agent-desktop.tar.gz -C bin From ca5d44a7ef3837b16da988c0621e40df687234b3 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 6 Nov 2026 09:12:49 -0800 Subject: [PATCH 4/5] build: lock the protocol's x11 extra (mss, python-xlib) Co-Authored-By: Claude Opus 5.5 --- uv.lock | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index b78a6b215..135908701 100644 --- a/uv.lock +++ b/uv.lock @@ -321,6 +321,10 @@ macos = [ { name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin'" }, { name = "pyobjc-framework-screencapturekit", marker = "sys_platform == 'darwin'" }, ] +x11 = [ + { name = "mss" }, + { name = "python-xlib" }, +] [package.metadata] requires-dist = [ @@ -334,6 +338,7 @@ requires-dist = [ { name = "mcp", marker = "extra == 'app'", specifier = ">=1.25,<2" }, { name = "mcp", marker = "extra == 'computer'", specifier = ">=1.25,<2" }, { name = "mcp", marker = "extra == 'dev'", specifier = ">=1.25,<2" }, + { name = "mss", marker = "extra == 'x11'", specifier = ">=9,<11" }, { name = "pillow", marker = "extra == 'computer'", specifier = ">=10.4" }, { name = "pillow", marker = "extra == 'dev'", specifier = ">=10.4" }, { name = "pydantic", specifier = ">=2.11" }, @@ -343,6 +348,7 @@ requires-dist = [ { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, { name = "python-multipart", marker = "extra == 'agent'", specifier = ">=0.0.13" }, { name = "python-multipart", marker = "extra == 'dev'", specifier = ">=0.0.13" }, + { name = "python-xlib", marker = "extra == 'x11'", specifier = ">=0.33,<0.34" }, { name = "regex", marker = "extra == 'agent'", specifier = ">=2023.12.25" }, { name = "regex", marker = "extra == 'dev'", specifier = ">=2023.12.25" }, { name = "starlette", specifier = ">=0.37" }, @@ -353,7 +359,7 @@ requires-dist = [ { name = "websockets", marker = "extra == 'computer'", specifier = ">=13" }, { name = "websockets", marker = "extra == 'dev'", specifier = ">=13" }, ] -provides-extras = ["agent", "app", "computer", "macos", "dev"] +provides-extras = ["agent", "app", "computer", "macos", "x11", "dev"] [[package]] name = "aiofile" @@ -3927,6 +3933,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bf/f2/c53904c4bc3da828fb75678bf489ece808a277affa73f8a7a91dfb7c5851/msgpack-1.2.4-cp315-cp315t-win_arm64.whl", hash = "sha256:e38d854f1c60e94f46d998765201126338bde3fe1b029be0d849094bcbef72c2", size = 77995, upload-time = "2026-10-26T17:55:47.016Z" }, ] +[[package]] +name = "mss" +version = "10.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/5d/eee782a6d674f562c946ae6a026f4c595ea2b7b031f290bf9fbf60da09b5/mss-10.2.0.tar.gz", hash = "sha256:ab271860775545e62f29d7b11f82f279ac1048f5bbdd26cfad84830208dbd393", size = 200317, upload-time = "2026-04-23T10:44:57.305Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/c3/313e14f245c79b4c05bd0f3a84a4813aa26fa10f8993aebd91d04c5fad3f/mss-10.2.0-py3-none-any.whl", hash = "sha256:e79f428899280e7e64e38365b5bfed683851ebea807eeaeadaf06eb8e0d67197", size = 67106, upload-time = "2026-04-23T10:44:56.266Z" }, +] + [[package]] name = "multidict" version = "6.7.1" @@ -5943,6 +5958,18 @@ client = [ { name = "websocket-client" }, ] +[[package]] +name = "python-xlib" +version = "0.33" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/f5/8c0653e5bb54e0cbdfe27bf32d41f27bc4e12faa8742778c17f2a71be2c0/python-xlib-0.33.tar.gz", hash = "sha256:55af7906a2c75ce6cb280a584776080602444f75815a7aff4d287bb2d7018b32", size = 269068, upload-time = "2022-12-25T18:53:00.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/b8/ff33610932e0ee81ae7f1269c890f697d56ff74b9f5b2ee5d9b7fa2c5355/python_xlib-0.33-py2.py3-none-any.whl", hash = "sha256:c3534038d42e0df2f1392a1b30a15a4ff5fdc2b86cfa94f072bf11b10a164398", size = 182185, upload-time = "2022-12-25T18:52:58.662Z" }, +] + [[package]] name = "pytz" version = "2026.5" From b7631b2bff0826bc7304786d7f250c9ebd440904 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 6 Nov 2026 09:30:55 -0800 Subject: [PATCH 5/5] test(drivers): agent-desktop's server gets the Linux session's display and bus, and not on macOS Co-Authored-By: Claude Opus 5.5 --- tst/unit/drivers/agent_desktop_driver_test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tst/unit/drivers/agent_desktop_driver_test.py b/tst/unit/drivers/agent_desktop_driver_test.py index 40112d196..7a743a449 100644 --- a/tst/unit/drivers/agent_desktop_driver_test.py +++ b/tst/unit/drivers/agent_desktop_driver_test.py @@ -71,6 +71,8 @@ async def main(): PLATFORMS = {"macos": Platform(os="macos", version="26.5", arch="arm64", display="aqua"), "linux": Platform(os="linux", distribution="ubuntu", version="24.04", arch="amd64", display="x11")} HELPERS = {"macos": "agent-desktop-macos-helper", "linux": "agent-desktop-linux-clipboard-owner"} +#: What agent-desktop needs of a Linux session to reach the display and AT-SPI on its bus. +SESSION = ("DISPLAY", "XAUTHORITY", "DBUS_SESSION_BUS_ADDRESS", "XDG_RUNTIME_DIR") class Granted: @@ -87,6 +89,8 @@ def module(request, monkeypatch, tmp_path): monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) monkeypatch.setenv("HOME", str(tmp_path)) monkeypatch.setenv("AGENTENV_CONTROL_TOKEN", "a-deploys-secret") + for name in SESSION: + monkeypatch.setenv(name, f"the session's {name}") monkeypatch.setattr(_macos, "Quartz", Granted) monkeypatch.setattr(sys, "platform", "darwin" if request.param == "macos" else "linux") monkeypatch.setattr("agentenv_protocol.computer.driver.detect", lambda: PLATFORMS[request.param]) @@ -154,6 +158,7 @@ async def test_a_call_reaches_a_server_of_its_own_headed_when_the_setting_says_w (first, second) = _calls(tmp_path) assert first["argv"] == ["mcp"] and second["argv"] == ["mcp", "--headed"] assert "AGENTENV_CONTROL_TOKEN" not in first["env"] and "HOME" in first["env"] + assert set(SESSION) & set(first["env"]) == (set() if module.MACOS else set(SESSION)) @pytest.mark.asyncio