From b73452f7e905d9fe0b6e0e8a4e1f1dc2d52be9f5 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 02:39:56 -0500 Subject: [PATCH 1/5] fix(flow): verify complete typed values and halt on uncertain focus Signed-off-by: abrichr --- packages/flow/CHANGELOG.md | 2 + .../backends/win_agent/README.md | 7 + .../backends/win_agent/server.py | 37 +++- .../backends/windows_backend.py | 6 + .../flow/openadapt_flow/runtime/replayer.py | 84 ++++++--- packages/flow/tests/test_replayer.py | 172 +++++++++++++++++- packages/flow/tests/test_win_agent_server.py | 128 +++++++++++++ 7 files changed, 401 insertions(+), 35 deletions(-) diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md index 5aad2da667..61bd63f8b6 100644 --- a/packages/flow/CHANGELOG.md +++ b/packages/flow/CHANGELOG.md @@ -43,6 +43,8 @@ ### Fixes +- Typed input now needs the complete whitespace-squashed OCR value, including values shorter than four characters. Native and pixel-only runs halt when delivery cannot be confirmed instead of clicking and typing again. The Windows agent checks the foreground window before and after each character or clipboard paste; a change stops delivery and reports that input may be partial ([#1478](https://github.com/OpenAdaptAI/OpenAdapt/issues/1478)). + - A qualification case can run on `--backend windows`. `WindowsBackend` now answers the qualification environment observer from one `/context/identity` call: the foreground executable's name, the SHA-256 of that executable file, the live logon-session digest, and a digest of the machine id and exact OS build. Before, `qualify run-case` on Windows refused with "qualification backend cannot observe the complete environment". The digest is withheld when the file changed after the process started (an updater replacing a running program), and it describes the host executable for an app run by an interpreter, browser or shared runtime. Guarded input now also binds the digest and the environment, so a same-named program from another path cannot take the foreground between the check and the input. The agent's session digest now accepts a connected RDP or Citrix session, not only the console session, so the agent can run inside a remote user session. An agent without the new `context_environment_v1` fields still refuses ([#1696](https://github.com/OpenAdaptAI/OpenAdapt/issues/1696)). - A covered-band dismiss step takes its dialog box from the window the backend proved covers the band, before any pixel search, so a native message box that the accessibility tree doesn't label as a dialog can be dismissed. Without a proved window the step is refused as before. diff --git a/packages/flow/openadapt_flow/backends/win_agent/README.md b/packages/flow/openadapt_flow/backends/win_agent/README.md index 81bc208efd..d311445afd 100644 --- a/packages/flow/openadapt_flow/backends/win_agent/README.md +++ b/packages/flow/openadapt_flow/backends/win_agent/README.md @@ -25,6 +25,13 @@ Only the Python **standard library** is imported at load; `mss`/Pillow (for the screenshot) and `pyautogui`/`uiautomation` (used by the input handlers) import lazily. No Flask required in the guest. +Typing binds to the foreground window handle and process. The agent checks +that identity before and after each ASCII character or Unicode clipboard +paste. If it changes, delivery stops with `input_focus_changed`; the client +reports uncertain delivery because input may already be partial. When the +foreground cannot be read, the agent refuses before typing. Update the guest +agent to get these checks. + ## Security (TLS in transit + loopback default + bearer token) `/screenshot` returns a PNG of the live patient chart and the input routes drive diff --git a/packages/flow/openadapt_flow/backends/win_agent/server.py b/packages/flow/openadapt_flow/backends/win_agent/server.py index 08739a254d..6769ee2f26 100644 --- a/packages/flow/openadapt_flow/backends/win_agent/server.py +++ b/packages/flow/openadapt_flow/backends/win_agent/server.py @@ -447,8 +447,29 @@ def _perform_input(payload: dict[str, Any]) -> dict[str, Any]: pyautogui.FAILSAFE = False if text: + foreground = _foreground_window_identity() + if foreground is None: + raise AgentRequestError( + 503, + "input_focus_unavailable", + "cannot bind keyboard input to a foreground window", + ) + + def require_foreground() -> None: + if _foreground_window_identity() != foreground: + raise AgentRequestError( + 409, + "input_focus_changed", + "foreground window changed during typing; delivery may be partial", + ) + if text.isascii(): - pyautogui.write(text, interval=float(interval)) + for index, char in enumerate(text): + require_foreground() + pyautogui.write(char, interval=0, _pause=False) + require_foreground() + if index + 1 < len(text): + time.sleep(float(interval)) else: encoded = base64.b64encode(text.encode("utf-8")).decode("ascii") ps = ( @@ -463,7 +484,9 @@ def _perform_input(payload: dict[str, Any]) -> dict[str, Any]: if completed.returncode != 0: raise RuntimeError("Set-Clipboard failed") time.sleep(0.2) + require_foreground() pyautogui.hotkey("ctrl", "v") + require_foreground() return _delivery_receipt("physical_type_text", native=False) if action == "press": @@ -2198,8 +2221,8 @@ def _running_image_digest(pid: int, path: str) -> Optional[str]: return _image_file_digest(path) -def _foreground_process_id() -> Optional[int]: - """The process that owns the foreground window, without reading its title.""" +def _foreground_window_identity() -> Optional[tuple[int, int]]: + """Foreground window and owning process, without reading a window title.""" if os.name != "nt": return None @@ -2229,7 +2252,13 @@ def _foreground_process_id() -> Optional[int]: return None except Exception: # noqa: BLE001 - unavailable means unverifiable return None - return int(pid.value) + return int(hwnd), int(pid.value) + + +def _foreground_process_id() -> Optional[int]: + """The process that owns the foreground window, without reading its title.""" + identity = _foreground_window_identity() + return identity[1] if identity is not None else None def _foreground_application_identity() -> Optional[str]: diff --git a/packages/flow/openadapt_flow/backends/windows_backend.py b/packages/flow/openadapt_flow/backends/windows_backend.py index 62d473744e..45145d110b 100644 --- a/packages/flow/openadapt_flow/backends/windows_backend.py +++ b/packages/flow/openadapt_flow/backends/windows_backend.py @@ -498,6 +498,12 @@ def _post_typed_action(self, path: str, payload: dict) -> dict: "guarded Windows input refused " f"({code}): {message or 'identity binding changed'}" ) + if path in {"/input", "/input/guarded"} and code == "input_focus_changed": + raise ActionDeliveryUncertain( + operation=operation, + native=False, + cause_type="ForegroundChanged", + ) if response.status_code >= 500: raise ActionDeliveryUncertain( operation=operation, diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index f79e998b4c..b5bcc42fc8 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -48,6 +48,7 @@ import threading import time import uuid +from collections import Counter from copy import deepcopy from dataclasses import dataclass from datetime import date, datetime, timezone @@ -85,6 +86,7 @@ RemoteFrameContractBackend, RichPointerActionBackend, SelectOptionBackend, + StructuralBackend, StructuralElementNotActionable, StructuralResolutionRefused, SubmitBindingBackend, @@ -17536,14 +17538,12 @@ def _typed_input_landed( """Did the just-typed ``text`` visibly land? Exact structural readback is authoritative when the backend exposes - it. Otherwise, for an OCR-able value (>= - ``identity.MIN_PARAM_CHARS`` squashed chars), a contiguous squashed - run of the value (retried at 2x resolution) must be readable in the - field region. Pixel-only acceptance is restricted to steps declared - ``secret``: masked password dots may render no readable text. A dialog + it. Otherwise the complete whitespace-squashed value must match the + field's new OCR text (retried at 2x resolution), including short values. + Whole unchanged context lines may be excluded. Pixel-only acceptance + is restricted to steps declared ``secret``: masked password dots may render no readable text. A dialog painting over the region or an unreadable ordinary field must never - count as "input landed". Values too short for OCR to arbitrate retain - the legacy diff fallback. + count as "input landed". Returns: ``(landed, changed)`` — the verdict, and whether the field @@ -17606,21 +17606,45 @@ def _typed_input_landed( == identity_mod.squash(text).casefold() ): return True, True + if not changed: + return False, False needle = identity_mod.squash(text) - if len(needle) < identity_mod.MIN_PARAM_CHARS: - return changed, changed # too short for OCR to arbitrate - need = identity_mod.required_run(len(needle)) - after_hay = self._ocr_squashed(after_png, region) - if identity_mod.longest_run(needle, after_hay) >= need: + + def exact_ocr_value( + before: bytes, after: bytes, crop: Optional[Region] + ) -> bool: + observed = [ + identity_mod.squash(line.text) + for line in self.vision.ocr(after, region=crop) + if identity_mod.squash(line.text) + ] + if needle and "".join(observed) == needle: + return True + if allow_masked or not needle: + return False + # The readback window may include unchanged labels beside the + # field. Remove whole unchanged OCR lines, with multiplicity; + # every remaining character must belong to the complete value. + previous = Counter( + identity_mod.squash(line.text) + for line in self.vision.ocr(before, region=crop) + ) + added = [] + for value in observed: + if previous[value]: + previous[value] -= 1 + else: + added.append(value) + return "".join(added) == needle + + if exact_ocr_value(baseline_png, after_png, region): return True, changed if region is not None: upscaled = identity_mod.upscale_crop(after_png, region) - if upscaled is not None: - up_hay = self._ocr_squashed(upscaled, None) - if identity_mod.longest_run(needle, up_hay) >= need: + before_upscaled = identity_mod.upscale_crop(baseline_png, region) + if upscaled is not None and before_upscaled is not None: + if exact_ocr_value(before_upscaled, upscaled, None): return True, changed - if not changed: - return False, False # Pixels changed but the value is unreadable: masked rendering is # the only acceptable explanation, and masked rendering adds no # confidently readable alphanumeric text (dot glyphs OCR as @@ -17656,10 +17680,10 @@ def _verify_typed_input( ) -> Optional[str]: """Verify a TYPE action landed; one guarded refocus-and-retype retry. - The retry only fires when exact readback or pixels prove the first - attempt changed NOTHING in the field (keystrokes fell on a - non-rendering target): re-click - the field (when its point is known), select-all so a false-negative + A retry needs a browser surface, structural browser observations and + exact field readback proving no change. An opaque or native surface + cannot prove where the first input went and must halt. For a browser, + re-click the field (when its point is known), select-all so a false-negative first attempt is replaced rather than duplicated, retype. When the region DID change but the value cannot be read (a dialog over the field, an unexpected re-render), retyping is not safe — select-all @@ -17689,12 +17713,24 @@ def _verify_typed_input( "or instead of the input) — retyping is unsafe in this " "state; run aborted" ) - if not allow_retry: + can_retry = ( + allow_retry + and workflow.surface in (None, "web") + and isinstance(self.backend, StructuralBackend) + and bool(self.backend.url) + and baseline_field_value is not None + ) + if not can_retry: result.input_verified = False + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal return ( f"Typed input could not be verified for step '{step.id}' " - f"({step.intent}): the identity-bound browser field showed no " - "confirmed change; refusing an unleased refocus/retype retry" + f"({step.intent}): the target field showed no confirmed change; " + "refusing an unleased refocus/retype retry after possibly stray input" ) result.input_retried = True retry_resolution = resolution diff --git a/packages/flow/tests/test_replayer.py b/packages/flow/tests/test_replayer.py index a0ca49a7fb..cf15b68e2e 100644 --- a/packages/flow/tests/test_replayer.py +++ b/packages/flow/tests/test_replayer.py @@ -1552,7 +1552,7 @@ def test_typed_recovery_refuses_inputs_changed_during_verification(bundle, run_d result = report.results[0] assert report.success is False assert backend.actions == [("type", "hello")] - assert result.input_retried is True + assert result.input_retried is False assert result.input_verified is False assert "authorization no longer matches the current runtime inputs" in ( result.error or "" @@ -3813,6 +3813,7 @@ def test_visual_click_crop_does_not_narrow_following_type_verification(bundle, r Match(point=(210, 150), region=(20, 20, 12, 8), confidence=0.95) ] vision.pixels_changed_results = [True] + vision.ocr_lines = [OcrLine("MA")] backend = FakeBackend(text_value_supported=False) workflow = Workflow( name="wf", @@ -3918,7 +3919,7 @@ def test_type_verification_refocuses_and_retypes_once(bundle, run_dir): vision.template_results = [ Match(point=(110, 105), region=(100, 100, 50, 20), confidence=0.95) ] - backend = FakeBackend(type_accept_results=[False, True]) + backend = StructuralFakeBackend(type_accept_results=[False, True]) workflow = Workflow( name="wf", steps=[ @@ -3951,7 +3952,7 @@ def test_type_verification_failure_halts_run(bundle, run_dir): vision.template_results = [ Match(point=(110, 105), region=(100, 100, 50, 20), confidence=0.95) ] - backend = FakeBackend(type_accept_results=[False, False]) + backend = StructuralFakeBackend(type_accept_results=[False, False]) workflow = Workflow( name="wf", steps=[ @@ -4011,6 +4012,163 @@ def test_type_verification_ocr_reads_the_value(bundle, run_dir): assert report.results[1].input_retried is False +@pytest.mark.parametrize( + "value,observed", + [ + ("1234567890", "12345678"), + ("A", "Wrong"), + ("12", "17"), + ("123", "Unrelated"), + ("alpha", "alphaX"), + ("alpha", "Xalpha"), + ("hello world", "hello worl"), + ], +) +def test_typed_input_requires_the_exact_ocr_value(bundle, run_dir, value, observed): + vision = _type_vision() + vision.ocr_lines = [OcrLine(observed)] + backend = FakeBackend(text_value_supported=False) + report = Replayer(backend, vision=vision).run( + _type_workflow(), + params={"note": value}, + bundle_dir=bundle, + run_dir=run_dir, + ) + assert report.success is False + result = report.results[1] + assert result.input_verified is False + assert result.input_retried is False + assert backend.actions.count(("type", value)) == 1 + assert ("press", "ControlOrMeta+a") not in backend.actions + + +@pytest.mark.parametrize("value,observed", [("7", "7"), ("Hello world", "hello WORLD")]) +def test_exact_ocr_input_including_short_values_is_verified( + bundle, run_dir, value, observed +): + vision = _type_vision() + vision.ocr_lines = [OcrLine(observed)] + backend = FakeBackend(text_value_supported=False) + report = Replayer(backend, vision=vision).run( + _type_workflow(), + params={"note": value}, + bundle_dir=bundle, + run_dir=run_dir, + ) + assert report.success is True + assert report.results[1].input_verified is True + assert report.results[1].input_retried is False + + +@pytest.mark.parametrize("exact_readback", [False, True]) +def test_opaque_typing_never_retries_an_unconfirmed_delivery( + bundle, run_dir, exact_readback +): + vision = _type_vision() + vision.pixels_changed_results = [False, False] + backend = FakeBackend( + text_value_supported=exact_readback, type_accept_results=[False, True] + ) + report = Replayer(backend, vision=vision).run( + _type_workflow(), + params={"note": "hello world"}, + bundle_dir=bundle, + run_dir=run_dir, + ) + assert report.success is False + result = report.results[1] + assert result.input_verified is False + assert result.input_retried is False + assert backend.actions == [("click", 110, 105, False), ("type", "hello world")] + + +def test_unavailable_browser_context_does_not_authorize_a_typing_retry(bundle, run_dir): + vision = _type_vision() + backend = StructuralFakeBackend(url=None, type_accept_results=[False, True]) + report = Replayer(backend, vision=vision).run( + _type_workflow(), + params={"note": "hello world"}, + bundle_dir=bundle, + run_dir=run_dir, + ) + assert report.success is False + assert report.results[1].input_retried is False + assert backend.actions == [("click", 110, 105, False), ("type", "hello world")] + + +def test_an_opaque_surface_cannot_opt_into_browser_typing_recovery(bundle, run_dir): + from openadapt_flow.ir import StepResult + + backend = StructuralFakeBackend() + workflow = _type_workflow() + workflow.surface = "rdp" + result = StepResult(step_id="t1", intent="type note", ok=False) + error = Replayer(backend, vision=FakeVision())._verify_typed_input( + workflow.steps[1], + "hello world", + (110, 105), + backend.screenshot(), + result, + workflow=workflow, + params={"note": "hello world"}, + bundle_dir=bundle, + run_dir=run_dir, + resolution=None, + baseline_field_value="", + allow_retry=True, + ) + assert error is not None + assert result.input_verified is False + assert result.input_retried is False + assert backend.actions == [] + + +@pytest.mark.parametrize( + "observed,verified", + [ + (["Open field", "hello world"], True), + (["Open field", "hello", "world"], True), + (["Open field", "hello worl"], False), + (["Open field", "hello world", "extra"], False), + (["Open field", "extra hello world"], False), + ], +) +def test_ocr_context_never_excuses_a_partial_or_extra_value( + bundle, run_dir, observed, verified +): + vision = _type_vision() + after = [OcrLine(value) for value in observed] + before = [OcrLine("Open field")] + vision.ocr_results = [after, before, after, before] + backend = FakeBackend(text_value_supported=False) + report = Replayer(backend, vision=vision).run( + _type_workflow(), + params={"note": "hello world"}, + bundle_dir=bundle, + run_dir=run_dir, + ) + assert report.success is verified + assert report.results[1].input_verified is verified + assert report.results[1].input_retried is False + assert backend.actions.count(("type", "hello world")) == 1 + + +def test_unchanged_pixels_cannot_verify_an_existing_ocr_value(bundle, run_dir): + vision = _type_vision() + vision.ocr_lines = [OcrLine("hello world")] + vision.pixels_changed_results = [False] + backend = FakeBackend(text_value_supported=False, type_accept_results=[False]) + report = Replayer(backend, vision=vision).run( + _type_workflow(), + params={"note": "hello world"}, + bundle_dir=bundle, + run_dir=run_dir, + ) + assert report.success is False + assert report.results[1].input_verified is False + assert report.results[1].input_retried is False + + def test_type_dialog_over_field_halts_without_retyping(bundle, run_dir): """ADDED 2026-07-09 (adversarial review, P2a/P2b): a dialog rendering over the field region changes pixels — under the old diff-alone rule @@ -4138,8 +4296,8 @@ def test_type_masked_dots_reading_as_noise_still_accepts(bundle, run_dir, monkey def test_type_without_known_field_diffs_full_frame_and_cannot_refocus(bundle, run_dir): """A TYPE step not preceded by a click (keyboard-only focus moves) has - no field point: verification diffs the whole frame, and the retry - retypes without a refocus click.""" + no field point or structural browser readback: verification cannot + prove where input went, so it halts without a second delivery.""" vision = FakeVision() vision.pixels_changed_results = [False, False] backend = FakeBackend( @@ -4156,8 +4314,8 @@ def test_type_without_known_field_diffs_full_frame_and_cannot_refocus(bundle, ru ) assert report.success is False assert vision.pixels_changed_calls[0] is None # full-frame diff - # Retry typed again but never clicked or selected-all. - assert backend.actions == [("type", "North"), ("type", "North")] + assert report.results[0].input_retried is False + assert backend.actions == [("type", "North")] # -- structural postconditions (URL/title change, new tab) -------------------- diff --git a/packages/flow/tests/test_win_agent_server.py b/packages/flow/tests/test_win_agent_server.py index 0ecae8f447..8084144c3e 100644 --- a/packages/flow/tests/test_win_agent_server.py +++ b/packages/flow/tests/test_win_agent_server.py @@ -1815,3 +1815,131 @@ def test_locator_at_keeps_a_unique_automation_id(monkeypatch) -> None: _perform_uia("locator-at", {"x": 560, "y": 400})["locator"]["automation_id"] == "noteBox" ) + + +@pytest.fixture +def physical_keyboard(monkeypatch): + """A physical-input seam with no desktop, clipboard or real key events.""" + from openadapt_flow.backends.win_agent import server + + state = types.SimpleNamespace( + foreground=(101, 202), typed="", pastes=0, after_write=None, after_paste=None + ) + fake = types.ModuleType("pyautogui") + + def write(text, **_kwargs): + state.typed += text + if state.after_write is not None: + state.after_write() + + def hotkey(*keys): + assert keys == ("ctrl", "v") + state.pastes += 1 + if state.after_paste is not None: + state.after_paste() + + fake.write = write + fake.hotkey = hotkey + monkeypatch.setitem(sys.modules, "pyautogui", fake) + monkeypatch.setattr(server, "_foreground_window_identity", lambda: state.foreground) + monkeypatch.setattr(server.time, "sleep", lambda _delay: None) + return state + + +def test_physical_typing_preserves_the_complete_value_with_stable_focus( + physical_keyboard, +): + receipt = _perform_input( + {"action": "type_text", "text": "safe-text", "interval_s": 0} + ) + assert physical_keyboard.typed == "safe-text" + assert receipt["status"] == "delivered" + assert receipt["operation"] == "physical_type_text" + + +def test_physical_typing_refuses_an_unobservable_foreground(physical_keyboard): + physical_keyboard.foreground = None + with pytest.raises(AgentRequestError, match="foreground window"): + _perform_input({"action": "type_text", "text": "abc"}) + assert physical_keyboard.typed == "" + assert physical_keyboard.pastes == 0 + + +@pytest.mark.parametrize("changed", [(303, 202), (101, 404), None]) +def test_physical_typing_stops_after_focus_changes(physical_keyboard, changed): + physical_keyboard.after_write = lambda: setattr( + physical_keyboard, "foreground", changed + ) + with pytest.raises(AgentRequestError, match="delivery may be partial") as error: + _perform_input({"action": "type_text", "text": "abc"}) + assert error.value.code == "input_focus_changed" + assert physical_keyboard.typed == "a" + + +def test_physical_typing_refuses_focus_change_before_the_first_key( + physical_keyboard, monkeypatch +): + from openadapt_flow.backends.win_agent import server + + readings = iter([(101, 202), (303, 202)]) + monkeypatch.setattr(server, "_foreground_window_identity", lambda: next(readings)) + with pytest.raises(AgentRequestError, match="delivery may be partial"): + _perform_input({"action": "type_text", "text": "abc"}) + assert physical_keyboard.typed == "" + + +def test_unicode_paste_is_refused_if_focus_changes_during_clipboard_setup( + physical_keyboard, monkeypatch +): + from openadapt_flow.backends.win_agent import server + + def clipboard(*_args, **_kwargs): + physical_keyboard.foreground = (303, 202) + return types.SimpleNamespace(returncode=0) + + monkeypatch.setattr(server.subprocess, "run", clipboard) + with pytest.raises(AgentRequestError, match="delivery may be partial"): + _perform_input({"action": "type_text", "text": "caf\u00e9"}) + assert physical_keyboard.typed == "" + assert physical_keyboard.pastes == 0 + + +def test_unicode_paste_reports_uncertainty_if_focus_changes_after_delivery( + physical_keyboard, monkeypatch +): + from openadapt_flow.backends.win_agent import server + + monkeypatch.setattr( + server.subprocess, "run", lambda *_a, **_k: types.SimpleNamespace(returncode=0) + ) + physical_keyboard.after_paste = lambda: setattr( + physical_keyboard, "foreground", (303, 202) + ) + with pytest.raises(AgentRequestError, match="delivery may be partial"): + _perform_input({"action": "type_text", "text": "caf\u00e9"}) + assert physical_keyboard.pastes == 1 + + +def test_focus_change_over_http_is_uncertain_and_never_uses_legacy_delivery( + physical_keyboard, +): + from openadapt_flow.backend import ActionDeliveryUncertain + from openadapt_flow.backends.windows_backend import WindowsBackend + + physical_keyboard.after_write = lambda: setattr( + physical_keyboard, "foreground", (303, 202) + ) + legacy_calls = [] + agent = RunningAgent( + AgentConfig(host="127.0.0.1", port=0, allow_legacy_exec=True), + legacy_fn=_recording_legacy_fn(legacy_calls), + ) + try: + backend = WindowsBackend(agent.url, allow_legacy_exec=True) + with pytest.raises(ActionDeliveryUncertain) as error: + backend.type_text("abc") + assert error.value.operation == "physical_type_text" + assert physical_keyboard.typed == "a" + assert legacy_calls == [] + finally: + agent.close() From 4c207c5f7ce1e91d0e28887fc260bd885a0afb85 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 02:55:50 -0500 Subject: [PATCH 2/5] fix(flow): avoid delivery permits on typed-input verification halts Signed-off-by: abrichr --- .../flow/openadapt_flow/runtime/replayer.py | 20 ++++++++++++++----- packages/flow/tests/test_replayer.py | 19 +++++++++++++++++- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index b5bcc42fc8..aa45aa3f89 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -17538,10 +17538,12 @@ def _typed_input_landed( """Did the just-typed ``text`` visibly land? Exact structural readback is authoritative when the backend exposes - it. Otherwise the complete whitespace-squashed value must match the - field's new OCR text (retried at 2x resolution), including short values. + it. Otherwise the complete lowercased, whitespace-squashed value must + match the field's new OCR text (retried at 2x resolution), including + short values. Whole unchanged context lines may be excluded. Pixel-only acceptance - is restricted to steps declared ``secret``: masked password dots may render no readable text. A dialog + is restricted to steps declared ``secret``: masked password dots may + render no readable text. A dialog painting over the region or an unreadable ordinary field must never count as "input landed". @@ -17722,10 +17724,18 @@ def _verify_typed_input( ) if not can_retry: result.input_verified = False - refusal = self._delivery_authorization_refusal( - workflow, params, step, result + # This is a halt, not a new delivery edge: recheck authority + # without consuming a durable fence or an approval permit. + refusal = self._fresh_actuation_authorization_refusal( + workflow, params, step ) if refusal is not None: + result.safety_halt = True + result.failure_category = ( + "governed_refusal" + if self.governed_authorization is not None + else "safety_halt" + ) return refusal return ( f"Typed input could not be verified for step '{step.id}' " diff --git a/packages/flow/tests/test_replayer.py b/packages/flow/tests/test_replayer.py index cf15b68e2e..2ef74f1a71 100644 --- a/packages/flow/tests/test_replayer.py +++ b/packages/flow/tests/test_replayer.py @@ -4069,7 +4069,21 @@ def test_opaque_typing_never_retries_an_unconfirmed_delivery( backend = FakeBackend( text_value_supported=exact_readback, type_accept_results=[False, True] ) - report = Replayer(backend, vision=vision).run( + + class CountingDeliveryGuard: + before_calls = 0 + acknowledgments = 0 + + def before_delivery(self): + self.before_calls += 1 + + def acknowledge_delivery(self): + self.acknowledgments += 1 + + guard = CountingDeliveryGuard() + replayer = Replayer(backend, vision=vision) + replayer._durable_continuation_guard = guard + report = replayer.run( _type_workflow(), params={"note": "hello world"}, bundle_dir=bundle, @@ -4081,6 +4095,9 @@ def test_opaque_typing_never_retries_an_unconfirmed_delivery( assert result.input_retried is False assert backend.actions == [("click", 110, 105, False), ("type", "hello world")] + assert guard.before_calls == guard.acknowledgments == len(backend.actions) == 2 + assert replayer._active_delivery_acknowledgers == () + def test_unavailable_browser_context_does_not_authorize_a_typing_retry(bundle, run_dir): vision = _type_vision() From 9e82eb0124d2f3f680c92be3a298a429be6ff8e9 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 05:19:54 -0500 Subject: [PATCH 3/5] fix(flow): read the full width of unknown text fields Signed-off-by: abrichr --- .../flow/openadapt_flow/runtime/replayer.py | 18 +++++------ .../flow/tests/test_pixel_scale_replay.py | 28 ++++++++++++++++- packages/flow/tests/test_replayer.py | 31 +++++++++++++++++++ .../flow/tests/test_sanitized_artifact.py | 5 +++ 4 files changed, 70 insertions(+), 12 deletions(-) diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index aa45aa3f89..f2c82f8fd8 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -397,13 +397,11 @@ def _frame_viewport(frame_png: bytes) -> tuple[int, int]: # modal at 0.24, leaving a deliberate margin on both sides. PC_STRUCTURAL_TEMPLATE_THRESHOLD = 0.8 -# Typed-input verification: size of the "field region" diffed/OCRed around -# the focusing click point after a TYPE action. Generous on purpose — the -# typed text renders at the field's own left edge / first line, not at the -# click point — and clamped to the viewport. When no focusing click is -# known (keyboard-only focus moves, e.g. Tab between fields), the whole -# frame is used instead. -FIELD_REGION_SIZE = (640, 240) +# Typed-input verification: height of the visual readback band around the +# focusing click after TYPE. Without structural field bounds, read the full +# viewport width: a centered crop can hide the value's beginning or suffix. +# With no focusing click (e.g. Tab between fields), use the whole frame. +FIELD_REGION_HEIGHT = 240 FIELD_REGION_PAD = 16 # Typed-input verification, secret-only diff acceptance: when a SECRET typed @@ -17470,11 +17468,9 @@ def _field_region( return (x, y, right - x, bottom - y) if field_point is None: return None - w = min(FIELD_REGION_SIZE[0], vw) - h = min(FIELD_REGION_SIZE[1], vh) - x = min(max(0, field_point[0] - w // 2), max(0, vw - w)) + h = min(FIELD_REGION_HEIGHT, vh) y = min(max(0, field_point[1] - h // 2), max(0, vh - h)) - return (x, y, w, h) + return (0, y, vw, h) def _ocr_squashed(self, png: bytes, region: Optional[Region]) -> str: """Squashed OCR text of a (region of a) frame.""" diff --git a/packages/flow/tests/test_pixel_scale_replay.py b/packages/flow/tests/test_pixel_scale_replay.py index c55686a9d9..e701c0bbdc 100644 --- a/packages/flow/tests/test_pixel_scale_replay.py +++ b/packages/flow/tests/test_pixel_scale_replay.py @@ -90,6 +90,29 @@ def test_canvas_kiosk_at_150_percent_cleartype_off_books_the_right_record( def test_wrong_record_at_a_new_scale_still_halts_before_the_write( + tmp_path: Path, bundles: dict[str, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + kiosk = ScaledKiosk(SPECS["canvas_notes"](), RenderProfile(1.25, cleartype=True)) + verify_input = Replayer._verify_typed_input + + def switch_after_verified_input(self, *args, **kwargs): + error = verify_input(self, *args, **kwargs) + assert error is None, "the real input check must pass before the identity fault" + kiosk.select_patient(1) + return error + + # Exercise the next action's identity check independently of the TYPE + # check. A switch during readback also changes the field's context and + # must already halt there (covered separately below). + monkeypatch.setattr(Replayer, "_verify_typed_input", switch_after_verified_input) + report = _replay(tmp_path, bundles["canvas_notes"], kiosk) + assert not report.success + assert kiosk.state.saved == [] + failure = _first_failure(report) + assert failure.identity is not None and failure.identity.status == "mismatch" + + +def test_record_switch_during_typed_readback_halts_without_retry( tmp_path: Path, bundles: dict[str, Path] ) -> None: kiosk = ScaledKiosk(SPECS["canvas_notes"](), RenderProfile(1.25, cleartype=True)) @@ -101,7 +124,10 @@ def switch_record(k: ScaledKiosk, text: str) -> None: assert not report.success assert kiosk.state.saved == [] failure = _first_failure(report) - assert failure.identity is not None and failure.identity.status == "mismatch" + assert failure.input_verified is False + assert failure.input_retried is False + assert failure.safety_halt is True + assert kiosk.state.values["note"] == sweep.REPLAY["canvas_notes"]["note"] def test_unconfirmable_identity_at_a_new_scale_halts( diff --git a/packages/flow/tests/test_replayer.py b/packages/flow/tests/test_replayer.py index 2ef74f1a71..58c1af6907 100644 --- a/packages/flow/tests/test_replayer.py +++ b/packages/flow/tests/test_replayer.py @@ -3911,6 +3911,37 @@ def test_type_verification_prefers_exact_structural_field_region() -> None: ) +@pytest.mark.parametrize( + "rendered,verified", + [ + ([(20, "7")], True), + ([(20, "X"), (900, "7")], False), + ([(900, "7"), (1400, "X")], False), + ], +) +def test_visual_input_readback_cannot_hide_text_outside_a_centered_crop( + rendered, verified +): + before = make_png((1600, 800)) + after = make_png((1600, 800), color=(220, 220, 220)) + + class CroppedVision(FakeVision): + def ocr(self, png, *, region=None): + if png == before or region is None: + return [] + x, _y, width, _height = region + text = "".join(value for left, value in rendered if x <= left < x + width) + return [OcrLine(text)] if text else [] + + backend = FakeBackend(frame=after, viewport=(1600, 800), text_value_supported=False) + landed, changed = Replayer(backend, vision=CroppedVision())._typed_input_landed( + "7", (900, 500), before + ) + assert landed is verified + assert changed is True + assert backend.actions == [] + + def test_type_verification_refocuses_and_retypes_once(bundle, run_dir): """Focus theft: the first attempt lands nowhere visible; the retry re-clicks the field, selects all (replace, never append), retypes, and diff --git a/packages/flow/tests/test_sanitized_artifact.py b/packages/flow/tests/test_sanitized_artifact.py index e365a652e9..e4ab761b5e 100644 --- a/packages/flow/tests/test_sanitized_artifact.py +++ b/packages/flow/tests/test_sanitized_artifact.py @@ -129,6 +129,7 @@ def _recording(tmp_path: Path) -> Path: class _RuntimeBackend: def __init__(self): self.actions: list[tuple[str, str]] = [] + self.value = "" @property def viewport(self): @@ -139,6 +140,10 @@ def screenshot(self): def type_text(self, value): self.actions.append(("type", value)) + self.value += value + + def focused_text_value(self): + return self.value def click(self, x, y, *, double=False): raise AssertionError("type-only parameter test must not click") From 1b4d57da171e830a31e8ec34071e0a32d457b7df Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 03:55:26 -0800 Subject: [PATCH 4/5] fix(flow): keep masked-input change evidence local Signed-off-by: abrichr --- .../flow/openadapt_flow/runtime/replayer.py | 11 ++++++-- packages/flow/tests/test_replayer.py | 28 +++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index 8d515df195..f1c3180f4b 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -402,6 +402,7 @@ def _frame_viewport(frame_png: bytes) -> tuple[int, int]: # viewport width: a centered crop can hide the value's beginning or suffix. # With no focusing click (e.g. Tab between fields), use the whole frame. FIELD_REGION_HEIGHT = 240 +MASKED_FIELD_REGION_WIDTH = 640 FIELD_REGION_PAD = 16 # Typed-input verification, secret-only diff acceptance: when a SECRET typed @@ -17476,6 +17477,8 @@ def _field_region( self, field_point: Optional[Point], structural_region: Optional[Region] = None, + *, + masked: bool = False, ) -> Optional[Region]: """Region to observe for typed input, or None for the whole frame.""" vw, vh = self.backend.viewport @@ -17489,9 +17492,11 @@ def _field_region( return (x, y, right - x, bottom - y) if field_point is None: return None + w = min(MASKED_FIELD_REGION_WIDTH, vw) if masked else vw h = min(FIELD_REGION_HEIGHT, vh) + x = min(max(0, field_point[0] - w // 2), max(0, vw - w)) y = min(max(0, field_point[1] - h // 2), max(0, vh - h)) - return (0, y, vw, h) + return (x, y, w, h) def _ocr_squashed(self, png: bytes, region: Optional[Region]) -> str: """Squashed OCR text of a (region of a) frame.""" @@ -17570,7 +17575,9 @@ def _typed_input_landed( retyping is only safe when nothing changed). """ after_png = self._settle_seeded() - region = self._field_region(field_point, field_region) + # Masked input has no readable value: keep its pixel-only change + # evidence local, so an unrelated change elsewhere cannot confirm it. + region = self._field_region(field_point, field_region, masked=allow_masked) if baseline_field_value is not None: expected = baseline_field_value + text after_at_point = ( diff --git a/packages/flow/tests/test_replayer.py b/packages/flow/tests/test_replayer.py index 58c1af6907..3dbe487a98 100644 --- a/packages/flow/tests/test_replayer.py +++ b/packages/flow/tests/test_replayer.py @@ -3942,6 +3942,34 @@ def ocr(self, png, *, region=None): assert backend.actions == [] +@pytest.mark.parametrize("changed_x,expected", [(50, False), (900, True)]) +def test_masked_input_needs_a_change_near_the_target(changed_x, expected): + from PIL import ImageDraw + + from openadapt_flow.vision import pixels_changed + + before = make_png((1600, 800)) + image = Image.open(io.BytesIO(before)) + ImageDraw.Draw(image).rectangle( + (changed_x - 40, 450, changed_x + 40, 550), fill="black" + ) + stream = io.BytesIO() + image.save(stream, format="PNG") + after = stream.getvalue() + + class PixelVision(FakeVision): + def pixels_changed(self, before, after, *, region=None, **kwargs): + return pixels_changed(before, after, region=region) + + backend = FakeBackend(frame=after, viewport=(1600, 800), text_value_supported=False) + landed, changed = Replayer(backend, vision=PixelVision())._typed_input_landed( + "secret", (900, 500), before, allow_masked=True + ) + assert landed is expected + assert changed is expected + assert backend.actions == [] + + def test_type_verification_refocuses_and_retypes_once(bundle, run_dir): """Focus theft: the first attempt lands nowhere visible; the retry re-clicks the field, selects all (replace, never append), retypes, and From b96974a42b1a21093e126e7074d7011e003194c3 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 04:23:32 -0800 Subject: [PATCH 5/5] test(flow): update page-settle region stub for masked input Signed-off-by: abrichr --- packages/flow/tests/test_page_settle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/flow/tests/test_page_settle.py b/packages/flow/tests/test_page_settle.py index 3d4ac8879c..cd3c777c30 100644 --- a/packages/flow/tests/test_page_settle.py +++ b/packages/flow/tests/test_page_settle.py @@ -368,7 +368,7 @@ def _typed(label, text="Massachusetts"): r.backend = _SelectBackend(label) r.vision = _PixelVision() r._settle_seeded = lambda: b"after" - r._field_region = lambda point, region: (0, 0, 10, 10) + r._field_region = lambda point, region, *, masked=False: (0, 0, 10, 10) return r._typed_input_landed(text, (5, 5), b"before"), r.vision.ocr_calls