abrichr · Nov 4, 2026 · Nov 4, 2026 · Nov 4, 2026 · Nov 4, 2026 · Nov 4, 2026
@@ -43,6 +43,8 @@
 
 ### Fixes
 
+- Typed input now needs the complete whitespace-squashed OCR value, including values shorter than four characters. Native and pixel-only runs halt when delivery cannot be confirmed instead of clicking and typing again. The Windows agent checks the foreground window before and after each character or clipboard paste; a change stops delivery and reports that input may be partial ([#1478](https://github.com/OpenAdaptAI/OpenAdapt/issues/1478)).
+
 - A REST read names its own records path: `effects.kind: rest`, a reward recipe of kind `rest`, `RestRecordVerifier`, `RestCompensator` and the fault gate's in-memory copy require `records_path` (and `records_key`, `delete_path_template`) instead of defaulting to the demo app's `/api/db`; `run --effects-kind rest` takes `--effects-records-path`. Compile-time REST binding admission (`compile_recording(admit_api_bindings=True, admission_fixture=...)`) takes the app's read path, write path, server-assigned fields and idempotency key field from an `AdmissionFixture`, and proposes nothing when no write was captured and no write path is named. Effect mining and `scaffold-verifier` mine an idempotency key only from a field the caller names. Verifier identities and fault-gate check digests of existing configs are unchanged.
 
 - A FHIR read names its resource and field map: `effects.kind: fhir`, a reward recipe of kind `fhir` and `FhirEffectVerifier` require `resource_type` and `field_paths` instead of defaulting to one app's note-as-Observation layout (`DEFAULT_OBSERVATION_PATHS` stays importable for callers whose app stores notes that way). An app profile's `no_record_values` lists the record-field values its audit entries use for "no record" (OpenEMR's profile: `"0"`; profile version `openemr-frame-profile/2026-11-03.1`, so a config that pins the old profile digest is refused until it pins the new one); the frame and rollback no longer treat `0` as "no record" in every app. `generate ensure-contract` reads only the generic server-fact names from `facts.json`; the Medplum probe's translation moved to `benchmark/fhir_medplum/readpath/facts.py`.

@@ -25,6 +25,13 @@ Only the Python **standard library** is imported at load; `mss`/Pillow (for the
 screenshot) and `pyautogui`/`uiautomation` (used by the input handlers) import
 lazily. No Flask required in the guest.
 
+Typing binds to the foreground window handle and process. The agent checks
+that identity before and after each ASCII character or Unicode clipboard
+paste. If it changes, delivery stops with `input_focus_changed`; the client
+reports uncertain delivery because input may already be partial. When the
+foreground cannot be read, the agent refuses before typing. Update the guest
+agent to get these checks.
+
 ## Security (TLS in transit + loopback default + bearer token)
 
 `/screenshot` returns a PNG of the live patient chart and the input routes drive

@@ -449,8 +449,29 @@ def _perform_input(payload: dict[str, Any]) -> dict[str, Any]:
 
         pyautogui.FAILSAFE = False
         if text:
+            foreground = _foreground_window_identity()
+            if foreground is None:
+                raise AgentRequestError(
+                    503,
+                    "input_focus_unavailable",
+                    "cannot bind keyboard input to a foreground window",
+                )
+
+            def require_foreground() -> None:
+                if _foreground_window_identity() != foreground:
+                    raise AgentRequestError(
+                        409,
+                        "input_focus_changed",
+                        "foreground window changed during typing; delivery may be partial",
+                    )
+
             if text.isascii():
-                pyautogui.write(text, interval=float(interval))
+                for index, char in enumerate(text):
+                    require_foreground()
+                    pyautogui.write(char, interval=0, _pause=False)
+                    require_foreground()
+                    if index + 1 < len(text):
+                        time.sleep(float(interval))
             else:
                 encoded = base64.b64encode(text.encode("utf-8")).decode("ascii")
                 ps = (
@@ -465,7 +486,9 @@ def _perform_input(payload: dict[str, Any]) -> dict[str, Any]:
                 if completed.returncode != 0:
                     raise RuntimeError("Set-Clipboard failed")
                 time.sleep(0.2)
+                require_foreground()
                 pyautogui.hotkey("ctrl", "v")
+                require_foreground()
         return _delivery_receipt("physical_type_text", native=False)
 
     if action == "press":
@@ -2280,8 +2303,8 @@ def _running_image_digest(pid: int, path: str) -> Optional[str]:
     return _image_file_digest(path)
 
 
-def _foreground_process_id() -> Optional[int]:
-    """The process that owns the foreground window, without reading its title."""
+def _foreground_window_identity() -> Optional[tuple[int, int]]:
+    """Foreground window and owning process, without reading a window title."""
 
     if os.name != "nt":
         return None
@@ -2311,7 +2334,13 @@ def _foreground_process_id() -> Optional[int]:
             return None
     except Exception:  # noqa: BLE001 - unavailable means unverifiable
         return None
-    return int(pid.value)
+    return int(hwnd), int(pid.value)
+
+
+def _foreground_process_id() -> Optional[int]:
+    """The process that owns the foreground window, without reading its title."""
+    identity = _foreground_window_identity()
+    return identity[1] if identity is not None else None
 
 
 def _foreground_application_identity() -> Optional[str]:

@@ -498,6 +498,12 @@ def _post_typed_action(self, path: str, payload: dict) -> dict:
                     "guarded Windows input refused "
                     f"({code}): {message or 'identity binding changed'}"
                 )
+            if path in {"/input", "/input/guarded"} and code == "input_focus_changed":
+                raise ActionDeliveryUncertain(
+                    operation=operation,
+                    native=False,
+                    cause_type="ForegroundChanged",
+                )
             if response.status_code >= 500:
                 raise ActionDeliveryUncertain(
                     operation=operation,

@@ -48,6 +48,7 @@
 import threading
 import time
 import uuid
+from collections import Counter
 from copy import deepcopy
 from dataclasses import dataclass
 from datetime import date, datetime, timezone
@@ -85,6 +86,7 @@
     RemoteFrameContractBackend,
     RichPointerActionBackend,
     SelectOptionBackend,
+    StructuralBackend,
     StructuralElementNotActionable,
     StructuralResolutionRefused,
     SubmitBindingBackend,
@@ -395,13 +397,12 @@ def _frame_viewport(frame_png: bytes) -> tuple[int, int]:
 # modal at 0.24, leaving a deliberate margin on both sides.
 PC_STRUCTURAL_TEMPLATE_THRESHOLD = 0.8
 
-# Typed-input verification: size of the "field region" diffed/OCRed around
-# the focusing click point after a TYPE action. Generous on purpose — the
-# typed text renders at the field's own left edge / first line, not at the
-# click point — and clamped to the viewport. When no focusing click is
-# known (keyboard-only focus moves, e.g. Tab between fields), the whole
-# frame is used instead.
-FIELD_REGION_SIZE = (640, 240)
+# Typed-input verification: height of the visual readback band around the
+# focusing click after TYPE. Without structural field bounds, read the full
+# viewport width: a centered crop can hide the value's beginning or suffix.
+# With no focusing click (e.g. Tab between fields), use the whole frame.
+FIELD_REGION_HEIGHT = 240
+MASKED_FIELD_REGION_WIDTH = 640
 FIELD_REGION_PAD = 16
 
 # Typed-input verification, secret-only diff acceptance: when a SECRET typed
@@ -17476,6 +17477,8 @@ def _field_region(
         self,
         field_point: Optional[Point],
         structural_region: Optional[Region] = None,
+        *,
+        masked: bool = False,
     ) -> Optional[Region]:
         """Region to observe for typed input, or None for the whole frame."""
         vw, vh = self.backend.viewport
@@ -17489,8 +17492,8 @@ def _field_region(
                 return (x, y, right - x, bottom - y)
         if field_point is None:
             return None
-        w = min(FIELD_REGION_SIZE[0], vw)
-        h = min(FIELD_REGION_SIZE[1], vh)
+        w = min(MASKED_FIELD_REGION_WIDTH, vw) if masked else vw
+        h = min(FIELD_REGION_HEIGHT, vh)
         x = min(max(0, field_point[0] - w // 2), max(0, vw - w))
         y = min(max(0, field_point[1] - h // 2), max(0, vh - h))
         return (x, y, w, h)
@@ -17557,22 +17560,24 @@ def _typed_input_landed(
         """Did the just-typed ``text`` visibly land?
 
         Exact structural readback is authoritative when the backend exposes
-        it. Otherwise, for an OCR-able value (>=
-        ``identity.MIN_PARAM_CHARS`` squashed chars), a contiguous squashed
-        run of the value (retried at 2x resolution) must be readable in the
-        field region. Pixel-only acceptance is restricted to steps declared
-        ``secret``: masked password dots may render no readable text. A dialog
+        it. Otherwise the complete lowercased, whitespace-squashed value must
+        match the field's new OCR text (retried at 2x resolution), including
+        short values.
+        Whole unchanged context lines may be excluded. Pixel-only acceptance
+        is restricted to steps declared ``secret``: masked password dots may
+        render no readable text. A dialog
         painting over the region or an unreadable ordinary field must never
-        count as "input landed". Values too short for OCR to arbitrate retain
-        the legacy diff fallback.
+        count as "input landed".
 
         Returns:
             ``(landed, changed)`` — the verdict, and whether the field
             region's pixels changed at all (the caller's retry decision:
             retyping is only safe when nothing changed).
         """
         after_png = self._settle_seeded()
-        region = self._field_region(field_point, field_region)
+        # Masked input has no readable value: keep its pixel-only change
+        # evidence local, so an unrelated change elsewhere cannot confirm it.
+        region = self._field_region(field_point, field_region, masked=allow_masked)
         if baseline_field_value is not None:
             expected = baseline_field_value + text
             after_at_point = (
@@ -17627,21 +17632,45 @@ def _typed_input_landed(
                 == identity_mod.squash(text).casefold()
             ):
                 return True, True
+        if not changed:
+            return False, False
         needle = identity_mod.squash(text)
-        if len(needle) < identity_mod.MIN_PARAM_CHARS:
-            return changed, changed  # too short for OCR to arbitrate
-        need = identity_mod.required_run(len(needle))
-        after_hay = self._ocr_squashed(after_png, region)
-        if identity_mod.longest_run(needle, after_hay) >= need:
+
+        def exact_ocr_value(
+            before: bytes, after: bytes, crop: Optional[Region]
+        ) -> bool:
+            observed = [
+                identity_mod.squash(line.text)
+                for line in self.vision.ocr(after, region=crop)
+                if identity_mod.squash(line.text)
+            ]
+            if needle and "".join(observed) == needle:
+                return True
+            if allow_masked or not needle:
+                return False
+            # The readback window may include unchanged labels beside the
+            # field. Remove whole unchanged OCR lines, with multiplicity;
+            # every remaining character must belong to the complete value.
+            previous = Counter(
+                identity_mod.squash(line.text)
+                for line in self.vision.ocr(before, region=crop)
+            )
+            added = []
+            for value in observed:
+                if previous[value]:
+                    previous[value] -= 1
+                else:
+                    added.append(value)
+            return "".join(added) == needle
+
+        if exact_ocr_value(baseline_png, after_png, region):
             return True, changed
         if region is not None:
             upscaled = identity_mod.upscale_crop(after_png, region)
-            if upscaled is not None:
-                up_hay = self._ocr_squashed(upscaled, None)
-                if identity_mod.longest_run(needle, up_hay) >= need:
+            before_upscaled = identity_mod.upscale_crop(baseline_png, region)
+            if upscaled is not None and before_upscaled is not None:
+                if exact_ocr_value(before_upscaled, upscaled, None):
                     return True, changed
-        if not changed:
-            return False, False
         # Pixels changed but the value is unreadable: masked rendering is
         # the only acceptable explanation, and masked rendering adds no
         # confidently readable alphanumeric text (dot glyphs OCR as
@@ -17677,10 +17706,10 @@ def _verify_typed_input(
     ) -> Optional[str]:
         """Verify a TYPE action landed; one guarded refocus-and-retype retry.
 
-        The retry only fires when exact readback or pixels prove the first
-        attempt changed NOTHING in the field (keystrokes fell on a
-        non-rendering target): re-click
-        the field (when its point is known), select-all so a false-negative
+        A retry needs a browser surface, structural browser observations and
+        exact field readback proving no change. An opaque or native surface
+        cannot prove where the first input went and must halt. For a browser,
+        re-click the field (when its point is known), select-all so a false-negative
         first attempt is replaced rather than duplicated, retype. When the
         region DID change but the value cannot be read (a dialog over the
         field, an unexpected re-render), retyping is not safe — select-all
@@ -17710,12 +17739,32 @@ def _verify_typed_input(
                 "or instead of the input) — retyping is unsafe in this "
                 "state; run aborted"
             )
-        if not allow_retry:
+        can_retry = (
+            allow_retry
+            and workflow.surface in (None, "web")
+            and isinstance(self.backend, StructuralBackend)
+            and bool(self.backend.url)
+            and baseline_field_value is not None
+        )
+        if not can_retry:
             result.input_verified = False
+            # This is a halt, not a new delivery edge: recheck authority
+            # without consuming a durable fence or an approval permit.
+            refusal = self._fresh_actuation_authorization_refusal(
+                workflow, params, step
+            )
+            if refusal is not None:
+                result.safety_halt = True
+                result.failure_category = (
+                    "governed_refusal"
+                    if self.governed_authorization is not None
+                    else "safety_halt"
+                )
+                return refusal
             return (
                 f"Typed input could not be verified for step '{step.id}' "
-                f"({step.intent}): the identity-bound browser field showed no "
-                "confirmed change; refusing an unleased refocus/retype retry"
+                f"({step.intent}): the target field showed no confirmed change; "
+                "refusing an unleased refocus/retype retry after possibly stray input"
             )
         result.input_retried = True
         retry_resolution = resolution

@@ -368,7 +368,7 @@ def _typed(label, text="Massachusetts"):
     r.backend = _SelectBackend(label)
     r.vision = _PixelVision()
     r._settle_seeded = lambda: b"after"
-    r._field_region = lambda point, region: (0, 0, 10, 10)
+    r._field_region = lambda point, region, *, masked=False: (0, 0, 10, 10)
     return r._typed_input_landed(text, (5, 5), b"before"), r.vision.ocr_calls
 
 

@@ -90,6 +90,29 @@ def test_canvas_kiosk_at_150_percent_cleartype_off_books_the_right_record(
 
 
 def test_wrong_record_at_a_new_scale_still_halts_before_the_write(
+    tmp_path: Path, bundles: dict[str, Path], monkeypatch: pytest.MonkeyPatch
+) -> None:
+    kiosk = ScaledKiosk(SPECS["canvas_notes"](), RenderProfile(1.25, cleartype=True))
+    verify_input = Replayer._verify_typed_input
+
+    def switch_after_verified_input(self, *args, **kwargs):
+        error = verify_input(self, *args, **kwargs)
+        assert error is None, "the real input check must pass before the identity fault"
+        kiosk.select_patient(1)
+        return error
+
+    # Exercise the next action's identity check independently of the TYPE
+    # check. A switch during readback also changes the field's context and
+    # must already halt there (covered separately below).
+    monkeypatch.setattr(Replayer, "_verify_typed_input", switch_after_verified_input)
+    report = _replay(tmp_path, bundles["canvas_notes"], kiosk)
+    assert not report.success
+    assert kiosk.state.saved == []
+    failure = _first_failure(report)
+    assert failure.identity is not None and failure.identity.status == "mismatch"
+
+
+def test_record_switch_during_typed_readback_halts_without_retry(
     tmp_path: Path, bundles: dict[str, Path]
 ) -> None:
     kiosk = ScaledKiosk(SPECS["canvas_notes"](), RenderProfile(1.25, cleartype=True))
@@ -101,7 +124,10 @@ def switch_record(k: ScaledKiosk, text: str) -> None:
     assert not report.success
     assert kiosk.state.saved == []
     failure = _first_failure(report)
-    assert failure.identity is not None and failure.identity.status == "mismatch"
+    assert failure.input_verified is False
+    assert failure.input_retried is False
+    assert failure.safety_halt is True
+    assert kiosk.state.values["note"] == sweep.REPLAY["canvas_notes"]["note"]
 
 
 def test_unconfirmable_identity_at_a_new_scale_halts(