From 49954d48bf0aba24612552ee9b6539971b3fd53d Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 11:59:20 -0500 Subject: [PATCH 1/5] Covered band: hold the dismiss click to the proved dialog and a notice-only message; the cover must be what cut the band --- packages/flow/CHANGELOG.md | 2 + packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md | 17 +- .../openadapt_flow/backends/macos_backend.py | 7 +- .../flow/openadapt_flow/repair/self_repair.py | 235 +++++++++++++++++- packages/flow/tests/test_macos_sheets.py | 7 +- .../flow/tests/test_repair_covered_band.py | 234 ++++++++++++++++- 6 files changed, 489 insertions(+), 13 deletions(-) diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md index 99ec48b05..9fb7d1764 100644 --- a/packages/flow/CHANGELOG.md +++ b/packages/flow/CHANGELOG.md @@ -25,6 +25,8 @@ ### Fixes +- Self-repair on a covered identity band dismisses only the proved dialog: the covering window must span the band from where the read stops to its right edge at full height, the dismiss click must land in that same window, and a dialog whose text could discard, end, overwrite or commit something is refused ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). + - Self-repair can take a stop where the app's own dialog covers the next step's identity band: when the backend proves the covering window belongs to the window the last verified step was in, and the part of the band read still matches the record, the dialog is dismissed after the previous click and the step re-checks its identity in full. Another record behind the dialog stays an identity halt. The runtime lets a backend remember a verified record's window (`record_verified_window`); macOS answers `owned_window_over` from the window's proved sheet ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). - macOS: a run can now act in the pinned window's own AppKit sheet (an `NSAlert` attached to it), which is a separate window. `MacOSBackend` accepts it only when AX lists exactly one `AXSheet` on the window and exactly one window of the same process has its bounds inside it; the capture composites exactly that sheet over the window, and focus, pointer and click-target checks accept it only inside its bounds. Any other window is still refused ([#1711](https://github.com/OpenAdaptAI/OpenAdapt/issues/1711)). diff --git a/packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md b/packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md index 554802c0d..027cb99aa 100644 --- a/packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md +++ b/packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md @@ -156,15 +156,30 @@ when all of these hold: (`record_verified_window` at that step, then `owned_window_over`). On macOS that is the window's one proved sheet; on Windows an owned window of the same process, first over the band in the live z-order; +- that window is what cut the band: its rectangle spans the band from where + the text read stops to the band's right edge, at the band's full height + (the share of the band left uncovered is within 0.1 of the share of the + recorded text read). A popup that only touches the band, such as a tooltip + over a column's ellipsis, does not count; - the part of the band that was read still matches the recorded record: no contradicted, look-alike, glyph-ambiguous or unexplained name token, and at least half the band matched. Another patient's header behind the dialog fails this and stays an identity halt; - the previous step is a click that doesn't save, acted on a trusted rung. +The dismiss click is held to that dialog. It must land in the very window +`owned_window_over` reported (asked again at the click point), and the +dialog's text must read as a notice only: words such as discard, delete, +end, overwrite, save, submit or unsaved refuse it, and so does a dialog with +nothing readable in it. Answering "OK" to "Discard unsaved changes?" would +lose data. + Nothing after that is relaxed: the probe replays the halted step with its own identity check in full once the dialog is gone, then the record check and a -person decide as for any repair. +person decide as for any repair. That full check is what protects the case +where the dialog hides exactly where two records differ (a same-name patient +with another birth date): the read beside the dialog can match, and the +recheck refuses. ## One lifecycle on every backend diff --git a/packages/flow/openadapt_flow/backends/macos_backend.py b/packages/flow/openadapt_flow/backends/macos_backend.py index 5001dbb41..d22494046 100644 --- a/packages/flow/openadapt_flow/backends/macos_backend.py +++ b/packages/flow/openadapt_flow/backends/macos_backend.py @@ -948,7 +948,12 @@ def owned_window_over( sx, sy, sw, sh = rect x, y, w, h = (int(v) for v in region) if sx < x + w and x < sx + sw and sy < y + h and y < sy + sh: - return {"kind": "sheet", "window_id": sheet[0], "pid": window.pid} + return { + "kind": "sheet", + "window_id": sheet[0], + "pid": window.pid, + "region": [int(sx), int(sy), int(sw), int(sh)], + } return None def _attached_sheet( diff --git a/packages/flow/openadapt_flow/repair/self_repair.py b/packages/flow/openadapt_flow/repair/self_repair.py index 39812f57f..6a2c70936 100644 --- a/packages/flow/openadapt_flow/repair/self_repair.py +++ b/packages/flow/openadapt_flow/repair/self_repair.py @@ -722,6 +722,8 @@ def __init__( self._sleep = sleep self._monotonic = monotonic self._token = secrets.token_urlsafe(32) + #: The proved cover when this run's halt is a covered band (#1724). + self._covered: Optional[_CoveredHalt] = None # -- public entry point -------------------------------------------------- @@ -743,6 +745,7 @@ def run(self, bundle_dir: Path | str, run_dir: Path | str) -> SelfRepairResult: agent_dialect=self.agent.dialect, prior_bundle_digest=_digest_of(workflow), ) + self._covered = None session = self.target.open() try: report = self.target.execute(workflow, bundle_dir, session, run_dir) @@ -778,6 +781,7 @@ def run(self, bundle_dir: Path | str, run_dir: Path | str) -> SelfRepairResult: work, ) evidence.halted_run_dir = str(run_dir) + self._covered = covered found = covered if covered is not None else _repairable(report, workflow) if isinstance(found, str): evidence.steps.append( @@ -1175,6 +1179,13 @@ def _patch( raise SelfRepairRefused( "a bridge can only be added to a linear workflow for now" ) + if self._covered is not None and [op.kind for op in ops] != ["click"]: + # A covered band is repaired by dismissing the one proved dialog: + # a single click in it (checked in _new_click), nothing typed. + raise SelfRepairRefused( + "a covered identity band is repaired by one click that dismisses " + "the covering dialog, and nothing else" + ) if mode == "replace" and ops[-1].kind != _POINTER_KINDS[step.action]: raise SelfRepairRefused( f"{step.id} is a {_POINTER_KINDS[step.action]}; the bridge's last " @@ -1527,6 +1538,8 @@ def _new_click( f"the bridge clicks {label!r}, which reads as a save; a bridge may " "only lead to the recorded save" ) + if self._covered is not None: + self._dismisses_the_cover(session, point, screen) try: admission = admit_new_click( session.backend, @@ -1568,6 +1581,58 @@ def _new_click( ) return step, _crop_png(screen, region), label, admission, files + def _dismisses_the_cover( + self, session: AppSession, point: tuple[int, int], screen: bytes + ) -> None: + """Refuse a covered-band bridge click outside the proved dialog. + + The click must land in the very window ``owned_window_over`` reported + over the band (asked again at the point, against the same verified + window), and that dialog's text must read as a notice only + (:func:`_dialog_message_refusal`). + """ + + covered = self._covered + assert covered is not None + probe = getattr(session.backend, "owned_window_over", None) + try: + here = ( + probe( + (int(point[0]), int(point[1]), 1, 1), pinned_step_id=covered.pinned + ) + if callable(probe) + else None + ) + except Exception: # noqa: BLE001 - no proof is a refusal + here = None + reported = _window_key(covered.cover) + if ( + reported is None + or not isinstance(here, dict) + or _window_key(here) != reported + ): + raise SelfRepairRefused( + "the bridge click is not in the dialog that covers the identity " + "band; only that dialog may be dismissed" + ) + rect = _cover_region(covered.cover) + text = "" + if rect is not None: + x, y, w, h = rect + try: + lines = self.vision.ocr(screen) + except Exception: # noqa: BLE001 - unreadable is a refusal below + lines = [] + text = " ".join( + str(line.text) + for line in lines + if x <= line.region[0] + line.region[2] / 2 < x + w + and y <= line.region[1] + line.region[3] / 2 < y + h + ) + refusal = _dialog_message_refusal(text) + if refusal is not None: + raise SelfRepairRefused(refusal) + def _landmarks( self, landmarks_for: Callable[..., Any], @@ -2664,7 +2729,7 @@ def _covered_band_halt( backend: Any, *, recorded_band: Callable[[Step], Optional[str]] = lambda step: None, -) -> "Optional[tuple[Step, str, Any]]": +) -> "Optional[_CoveredHalt]": """An identity halt caused by the app's own window over the band (#1724). A dialog the app opens after a step (a session notice, an update prompt) @@ -2680,7 +2745,9 @@ def _covered_band_halt( - the backend proves that a window of the app itself covers the step's identity band on the halted screen (``owned_window_over``): a window of the same process, owned by the top-level window the backend captured - when the last identity-verified step passed (``pinned_step_id``); + when the last identity-verified step passed (``pinned_step_id``). Its + reported ``region`` must span the band from where the read stops to the + band's right edge, at full height (:func:`_cover_explains_cut`); - the identity was read from the screen (``context``), its band has no run parameter, and the text read is the recorded band word for word up to where the dialog cuts it (only the last word read may be cut short), @@ -2748,7 +2815,8 @@ def _covered_band_halt( or report.halt.state_id != step.id ): return None - region = tuple(int(v) for v in step.anchor.identifier_region) + rx, ry, rw, rh = (int(v) for v in step.anchor.identifier_region) + region = (rx, ry, rw, rh) if halted.resolution is not None: dx = int(halted.resolution.point[0]) - int(step.anchor.click_point[0]) dy = int(halted.resolution.point[1]) - int(step.anchor.click_point[1]) @@ -2769,9 +2837,15 @@ def _covered_band_halt( cover = probe(region, pinned_step_id=pinned) except Exception: # noqa: BLE001 - no proof means no reclassification return None - if not cover or not _uncovered_band_matches( - step, halted.identity.observed, recorded_band(step) - ): + if not isinstance(cover, dict) or not cover: + return None + try: + recorded = recorded_band(step) + except Exception: # noqa: BLE001 - an unreadable record is doubt + return None + if not _uncovered_band_matches( + step, halted.identity.observed, recorded + ) or not _cover_explains_cut(cover, region, halted.identity.observed, recorded): return None previous = workflow.steps[index - 1] acted = next((r for r in report.results if r.step_id == previous.id), None) @@ -2786,7 +2860,154 @@ def _covered_band_halt( or is_consequential(previous, workflow) ): return None - return previous, "after", acted + return _CoveredHalt((previous, "after", acted), cover=dict(cover), pinned=pinned) + + +def _window_key(window: Mapping[str, Any]) -> Optional[tuple[int, int]]: + """A reported window's ``(id, pid)``: ``hwnd`` on Windows, ``window_id`` on + macOS. None when either is missing or not a positive integer.""" + + ident = window.get("hwnd", window.get("window_id")) + pid = window.get("pid") + if any( + isinstance(v, bool) or not isinstance(v, int) or v <= 0 for v in (ident, pid) + ): + return None + return int(ident), int(pid) # type: ignore[arg-type] + + +class _CoveredHalt(tuple): + """``(previous step, "after", its result)``, with the proved cover. + + ``cover`` is what ``owned_window_over`` returned for the band and + ``pinned`` the step whose verified window it was checked against; the + dismiss click is held to that same window (:meth:`SelfRepair._new_click`). + """ + + cover: dict + pinned: str + + def __new__(cls, items: tuple, *, cover: dict, pinned: str) -> "_CoveredHalt": + found = super().__new__(cls, items) + found.cover = cover + found.pinned = pinned + return found + + +#: How far the share of the band left uncovered may be from the share of the +#: recorded text read: the dialog's left edge must be where the read stops. +#: On the lab's Patient Notes notice the two are 0.79 and 0.81. +_CUT_TOLERANCE = 0.1 + + +def _cover_region(cover: Mapping[str, Any]) -> Optional[tuple[int, int, int, int]]: + """The covering window's ``region`` as ``(x, y, w, h)``, or None.""" + + value = cover.get("region") + if not isinstance(value, (list, tuple)) or len(value) != 4: + return None + if any(isinstance(v, bool) or not isinstance(v, int) for v in value): + return None + x, y, w, h = value + if w <= 0 or h <= 0: + return None + return x, y, w, h + + +def _cover_explains_cut( + cover: Mapping[str, Any], + band: tuple[int, int, int, int], + observed: str, + recorded: Optional[str], +) -> bool: + """Whether the covering window is what cut the band where the read stops. + + The window must span the band from a point inside it to its right edge, + at the band's full height, and its left edge must sit where the text + read ends: the share of the band's width left uncovered is within + ``_CUT_TOLERANCE`` of the share of the recorded text read. A popup that + only touches the band (a tooltip over a column's ellipsis) fails. A + cover with no ``region`` is doubt. + """ + + from openadapt_flow.runtime.identity import ocr_canonical, squash + + rect = _cover_region(cover) + if rect is None or not (recorded or "").strip(): + return False + bx, by, bw, bh = band + cx, cy, cw, ch = rect + if not ( + bx < cx < bx + bw and cy <= by and cy + ch >= by + bh and cx + cw >= bx + bw + ): + return False + read = len(ocr_canonical(squash(observed or ""))) + whole = len(ocr_canonical(squash(str(recorded)))) + if whole <= 0 or read <= 0: + return False + return abs((cx - bx) / bw - read / whole) <= _CUT_TOLERANCE + + +#: Words in a dialog's message that make dismissing it more than an +#: acknowledgement: its buttons may discard, end, overwrite or commit. +_CONSEQUENTIAL_DIALOG_WORDS = frozenset( + { + "apply", + "approve", + "cancel", + "clear", + "close", + "commit", + "confirm", + "delete", + "discard", + "end", + "erase", + "logout", + "lose", + "lost", + "overwrite", + "post", + "purge", + "remove", + "replace", + "reset", + "revert", + "save", + "send", + "sign", + "submit", + "terminate", + "undo", + "unsaved", + "void", + } +) + + +def _dialog_message_refusal(text: str) -> Optional[str]: + """Why a covering dialog's text forbids dismissing it, or None. + + Only a dialog whose words carry no destructive or committing meaning may + be dismissed by a bridge: "Discard unsaved changes?" or "End the other + session?" answered with "OK" loses data. Unreadable text is a refusal. + """ + + from openadapt_flow.risk import is_write_shaped + + words = { + w for w in "".join(c if c.isalnum() else " " for c in text.casefold()).split() + } + if not words: + return "the covering dialog's message can't be read" + hits = sorted(words & _CONSEQUENTIAL_DIALOG_WORDS) + if hits or is_write_shaped(text): + return ( + "the covering dialog's message reads as more than a notice " + f"({', '.join(hits) or 'a save'}); dismissing it may discard, end or " + "commit something" + ) + return None #: The least share of the recorded band the uncovered part must match. diff --git a/packages/flow/tests/test_macos_sheets.py b/packages/flow/tests/test_macos_sheets.py index 563e5f25e..65bf4cc8c 100644 --- a/packages/flow/tests/test_macos_sheets.py +++ b/packages/flow/tests/test_macos_sheets.py @@ -203,7 +203,12 @@ def test_owned_window_over_needs_the_verified_window_and_an_overlap() -> None: # The sheet's screen rect (110, 60, 200, 150) is window pixels # ((110-10)*2, (60-20)*2, 400, 300) = (200, 80, 400, 300). hit = target.owned_window_over((300, 100, 50, 20), pinned_step_id="s1") - assert hit == {"kind": "sheet", "window_id": SHEET_ID, "pid": TARGET_WINDOW.pid} + assert hit == { + "kind": "sheet", + "window_id": SHEET_ID, + "pid": TARGET_WINDOW.pid, + "region": [200, 80, 400, 300], + } assert target.owned_window_over((0, 0, 50, 20), pinned_step_id="s1") is None assert target.owned_window_over((300, 100, 50, 20), pinned_step_id="s2") is None diff --git a/packages/flow/tests/test_repair_covered_band.py b/packages/flow/tests/test_repair_covered_band.py index 3b7f24efc..88b53c912 100644 --- a/packages/flow/tests/test_repair_covered_band.py +++ b/packages/flow/tests/test_repair_covered_band.py @@ -128,9 +128,20 @@ def _report( ) +def _cover_for(observed: str = READ_BESIDE_DIALOG, band=BAND) -> list[int]: + """A dialog whose left edge is where ``observed`` stops, over the band's + right part at full height (the shape the session notice has).""" + from openadapt_flow.runtime.identity import ocr_canonical, squash + + share = len(ocr_canonical(squash(observed))) / len(ocr_canonical(squash(RECORDED))) + left = band[0] + round(band[2] * share) + return [left, band[1] - 40, band[0] + band[2] - left + 60, band[3] + 120] + + class Backend: - def __init__(self, covered: Any = True) -> None: + def __init__(self, covered: Any = True, region: Any = "default") -> None: self.covered = covered + self.region = _cover_for() if region == "default" else region self.regions: list[tuple] = [] self.pinned: list[str] = [] @@ -139,7 +150,12 @@ def owned_window_over(self, region, *, pinned_step_id) -> Optional[dict]: self.pinned.append(pinned_step_id) if isinstance(self.covered, Exception): raise self.covered - return {"kind": "dialog", "title": "Session notice"} if self.covered else None + if not self.covered: + return None + cover = {"title": "Session notice", "hwnd": 7, "owner_hwnd": 3, "pid": 42} + if self.region is not None: + cover["region"] = list(self.region) + return cover def test_a_covered_band_is_repaired_after_the_previous_click() -> None: @@ -321,8 +337,220 @@ def test_a_hidden_birth_date_is_reclassified_only_for_the_probe_to_recheck() -> # same-name patient could be behind it. Reclassifying only routes the # stop to an ``after`` repair: the probe replays this step with its own # identity check on the uncovered band, which refuses another birth date. + observed = "Patient: Neil Sorenson (19" found = _halt( - _report(observed="Patient: Neil Sorenson (19"), _workflow(), Backend() + _report(observed=observed), _workflow(), Backend(region=_cover_for(observed)) ) assert found is not None and found[1] == "after" assert found[0].id == "search" # the note step itself is replayed unchanged + + +# -- the cover must be what cut the band (review of #1735) --------------------- + + +def test_the_found_halt_carries_the_proved_cover() -> None: + found = _halt(_report(), _workflow(), Backend()) + assert found.pinned == "search" + assert found.cover["hwnd"] == 7 and found.cover["region"] == _cover_for() + + +@pytest.mark.parametrize( + "region", + [ + None, # no rect reported: doubt + [0, 0, 0, 0], + "garbage", + # a tooltip over the band's right end, far right of where the read stops + [BAND[0] + BAND[2] - 8, BAND[1] - 2, 60, BAND[3] + 4], + # starts where the read stops but is shorter than the band + [_cover_for()[0], BAND[1] + 5, 200, BAND[3] - 10], + # starts where the read stops but ends before the band's right edge + [_cover_for()[0], BAND[1] - 10, 20, BAND[3] + 20], + # covers the band from its left edge: nothing could be read beside it + [BAND[0] - 5, BAND[1] - 10, 400, BAND[3] + 20], + # its left edge well left of where the read stops + [BAND[0] + 40, BAND[1] - 10, 400, BAND[3] + 20], + ], +) +def test_a_window_that_did_not_cut_the_band_is_not_the_cause(region) -> None: + assert _halt(_report(), _workflow(), Backend(region=region)) is None + + +def test_a_recorded_band_that_raises_is_doubt() -> None: + def broken(step): + raise OSError("crop unreadable") + + assert ( + _covered_band_halt(_report(), _workflow(), Backend(), recorded_band=broken) + is None + ) + + +# -- the dismiss click: only in the proved dialog, only on a notice ------------- + + +class _Line: + def __init__(self, text: str, region: tuple[int, int, int, int]) -> None: + self.text, self.region = text, region + + +def _guard( + *, at: Any, message: str, cover: Optional[dict] = None, lines_in: bool = True +): + """Run ``SelfRepair._dismisses_the_cover`` on a stand-in self.""" + from types import SimpleNamespace + + from openadapt_flow.repair.self_repair import SelfRepair, _CoveredHalt + + cover = cover or {"hwnd": 7, "pid": 42, "region": [380, 300, 270, 160]} + found = _CoveredHalt(("s", "after", None), cover=cover, pinned="search") + lines = [ + _Line("Session notice", (395, 318, 80, 14)), + _Line(message, (450, 374, 170, 14)), + _Line("OK", (590, 433, 20, 14)), + _Line("Patient: Neil Sorenson (1984-", (230, 320, 150, 14)), # outside + ] + if not lines_in: + lines = lines[-1:] + vision = SimpleNamespace(ocr=lambda png: lines) + queried: list[tuple] = [] + + class Here: + def owned_window_over(self, region, *, pinned_step_id): + queried.append((tuple(region), pinned_step_id)) + if isinstance(at, Exception): + raise at + return at + + me = SimpleNamespace(_covered=found, vision=vision) + session = SimpleNamespace(backend=Here()) + SelfRepair._dismisses_the_cover(me, session, (598, 440), b"png") + return queried + + +def test_a_notice_in_the_proved_dialog_may_be_dismissed() -> None: + queried = _guard( + at={"hwnd": 7, "pid": 42}, message="Your session expires in 5 minutes." + ) + assert queried == [((598, 440, 1, 1), "search")] + + +@pytest.mark.parametrize( + "at", + [ + None, # nothing owned at the point + {"hwnd": 8, "pid": 42}, # another of the app's windows + {"hwnd": 7, "pid": 43}, + {"title": "x"}, # no id + RuntimeError("agent"), + ], +) +def test_a_click_outside_the_proved_dialog_is_refused(at) -> None: + from openadapt_flow.repair.self_repair import SelfRepairRefused + + with pytest.raises(SelfRepairRefused, match="not in the dialog"): + _guard(at=at, message="Your session expires in 5 minutes.") + + +def test_a_cover_without_an_id_never_matches_a_click_without_one() -> None: + from openadapt_flow.repair.self_repair import SelfRepairRefused + + with pytest.raises(SelfRepairRefused, match="not in the dialog"): + _guard( + at={"title": "x"}, + message="Your session expires in 5 minutes.", + cover={"region": [380, 300, 270, 160]}, + ) + + +@pytest.mark.parametrize( + "message", + [ + "Discard unsaved changes?", + "End the other session and continue?", + "Overwrite the existing note?", + "Save changes before closing?", + "Delete this draft?", + "Submit the order now?", + "Your changes will be lost.", + ], +) +def test_a_dialog_whose_message_can_lose_or_commit_data_is_refused(message) -> None: + from openadapt_flow.repair.self_repair import SelfRepairRefused + + with pytest.raises(SelfRepairRefused, match="dialog's message"): + _guard(at={"hwnd": 7, "pid": 42}, message=message) + + +# -- what protects a hidden difference above the floor: the full recheck ------- + + +@pytest.mark.parametrize( + "after_dismiss", + [ + "Patient: Neil Sorenson (1991-07-02)", # same name, another birth date + "Patient: Neil Sorensen (1984-03-12)", # the surname differs past the cut + "Patient: Neil Sorenson (1984-", # the dialog is still there + ], +) +def test_the_probes_full_identity_check_refuses_what_the_dialog_hid( + after_dismiss, +) -> None: + # The note step's identity is checked again, unchanged, when the probe + # replays the repaired bundle. Whatever the dialog hid must match then. + from openadapt_flow.runtime.identity_template import verify_template_identity + + hidden = "Patient: Neil Sorenson (19" + assert _halt( + _report(observed=hidden), _workflow(), Backend(region=_cover_for(hidden)) + ) + step = _workflow().steps[1] + assert step.anchor.identity_template == TEMPLATE # not relaxed by the repair + check = verify_template_identity(TEMPLATE, after_dismiss) + assert check.status != "verified" + assert verify_template_identity(TEMPLATE, RECORDED).status == "verified" + + +def test_a_dialog_with_nothing_readable_in_it_is_refused() -> None: + from openadapt_flow.repair.self_repair import SelfRepairRefused + + with pytest.raises(SelfRepairRefused, match="can't be read"): + _guard(at={"hwnd": 7, "pid": 42}, message="", lines_in=False) + + +@pytest.mark.parametrize( + "kinds", [["type"], ["click", "click"], ["double_click"], ["click", "type"]] +) +def test_a_covered_band_bridge_is_exactly_one_click(kinds) -> None: + from types import SimpleNamespace + + from openadapt_flow.repair.self_repair import ( + SelfRepair, + SelfRepairRefused, + _CoveredHalt, + ) + + ops = iter(SimpleNamespace(kind=kind) for kind in kinds) + me = SimpleNamespace( + _covered=_CoveredHalt(("s", "after", None), cover={}, pinned="search"), + max_bridge=4, + _one_op=lambda native: next(ops), + ) + proposal = SimpleNamespace(bridge=lambda: [{}] * len(kinds)) + workflow = _workflow() + with pytest.raises(SelfRepairRefused, match="one click that dismisses"): + SelfRepair._patch( + me, + workflow, + None, + workflow.steps[0], + "after", + proposal, + None, + b"", + 1, + None, + None, + [False], + None, + ) From dbb09c02aad93be84435f1c4b5f435820e616204 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 12:03:51 -0500 Subject: [PATCH 2/5] Covered band: allow only notice words in a dialog that a bridge dismisses (an allowlist, no questions) --- packages/flow/CHANGELOG.md | 2 +- packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md | 12 +- .../flow/openadapt_flow/repair/self_repair.py | 105 +++++++++++------- .../flow/tests/test_repair_covered_band.py | 66 ++++++++++- 4 files changed, 136 insertions(+), 49 deletions(-) diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md index 9fb7d1764..33467f5c8 100644 --- a/packages/flow/CHANGELOG.md +++ b/packages/flow/CHANGELOG.md @@ -25,7 +25,7 @@ ### Fixes -- Self-repair on a covered identity band dismisses only the proved dialog: the covering window must span the band from where the read stops to its right edge at full height, the dismiss click must land in that same window, and a dialog whose text could discard, end, overwrite or commit something is refused ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). +- Self-repair on a covered identity band dismisses only the proved dialog: the covering window must span the band from where the read stops to its right edge at full height, the dismiss click must land in that same window, and only a dialog whose every word is a notice word (no question) may be dismissed ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). - Self-repair can take a stop where the app's own dialog covers the next step's identity band: when the backend proves the covering window belongs to the window the last verified step was in, and the part of the band read still matches the record, the dialog is dismissed after the previous click and the step re-checks its identity in full. Another record behind the dialog stays an identity halt. The runtime lets a backend remember a verified record's window (`record_verified_window`); macOS answers `owned_window_over` from the window's proved sheet ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). diff --git a/packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md b/packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md index 027cb99aa..e0d810c62 100644 --- a/packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md +++ b/packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md @@ -169,10 +169,14 @@ when all of these hold: The dismiss click is held to that dialog. It must land in the very window `owned_window_over` reported (asked again at the click point), and the -dialog's text must read as a notice only: words such as discard, delete, -end, overwrite, save, submit or unsaved refuse it, and so does a dialog with -nothing readable in it. Answering "OK" to "Discard unsaved changes?" would -lose data. +dialog's text must read as a notice only. Every word in it must be on a +short list of notice words (a session that expires, a version that is +available, "OK"), and it may not ask a question. Anything else refuses, +including a dialog in another language or with nothing readable in it. +Answering "OK" to "Discard unsaved changes?" would lose data. The bridge is +exactly that one click. A covered-band repair gets one attempt: a later +attempt reopens the app, whose dialog is a new window, so its click is +refused. Nothing after that is relaxed: the probe replays the halted step with its own identity check in full once the dialog is gone, then the record check and a diff --git a/packages/flow/openadapt_flow/repair/self_repair.py b/packages/flow/openadapt_flow/repair/self_repair.py index 6a2c70936..bcdeec990 100644 --- a/packages/flow/openadapt_flow/repair/self_repair.py +++ b/packages/flow/openadapt_flow/repair/self_repair.py @@ -2948,39 +2948,51 @@ def _cover_explains_cut( return abs((cx - bx) / bw - read / whole) <= _CUT_TOLERANCE -#: Words in a dialog's message that make dismissing it more than an -#: acknowledgement: its buttons may discard, end, overwrite or commit. -_CONSEQUENTIAL_DIALOG_WORDS = frozenset( +#: The only words a dismissable covering dialog may show (title, message +#: and button), casefolded: a notice that something expires, is available or +#: is informational, answered with "OK". Any other word, a question mark, or +#: no readable word refuses. An allowlist, so an unlisted "Log out of the +#: other session?", "Your changes will be deleted" or a dialog in another +#: language is refused rather than dismissed. +_NOTICE_DIALOG_WORDS = frozenset( { - "apply", - "approve", - "cancel", - "clear", - "close", - "commit", - "confirm", - "delete", - "discard", - "end", - "erase", - "logout", - "lose", - "lost", - "overwrite", - "post", - "purge", - "remove", - "replace", - "reset", - "revert", - "save", - "send", - "sign", - "submit", - "terminate", - "undo", - "unsaved", - "void", + "a", + "about", + "an", + "app", + "application", + "available", + "been", + "for", + "has", + "hour", + "hours", + "in", + "info", + "information", + "is", + "minute", + "minutes", + "new", + "notice", + "now", + "of", + "ok", + "reminder", + "second", + "seconds", + "session", + "soon", + "the", + "this", + "update", + "version", + "will", + "you", + "your", + "expire", + "expires", + "expiring", } ) @@ -2988,24 +3000,31 @@ def _cover_explains_cut( def _dialog_message_refusal(text: str) -> Optional[str]: """Why a covering dialog's text forbids dismissing it, or None. - Only a dialog whose words carry no destructive or committing meaning may - be dismissed by a bridge: "Discard unsaved changes?" or "End the other - session?" answered with "OK" loses data. Unreadable text is a refusal. + Only a notice may be dismissed by a bridge: every word must be in + ``_NOTICE_DIALOG_WORDS`` (numbers aside) and nothing may ask a question. + Answering "OK" to "Discard unsaved changes?" or "Log out of the other + session?" loses data, so anything not plainly a notice is refused, + including unreadable text. """ - from openadapt_flow.risk import is_write_shaped - - words = { + words = [ w for w in "".join(c if c.isalnum() else " " for c in text.casefold()).split() - } + ] if not words: return "the covering dialog's message can't be read" - hits = sorted(words & _CONSEQUENTIAL_DIALOG_WORDS) - if hits or is_write_shaped(text): + if "?" in text: + return ( + "the covering dialog asks a question; answering it may discard, end " + "or commit something" + ) + other = sorted( + {w for w in words if w not in _NOTICE_DIALOG_WORDS and not w.isdecimal()} + ) + if other: return ( "the covering dialog's message reads as more than a notice " - f"({', '.join(hits) or 'a save'}); dismissing it may discard, end or " - "commit something" + f"({', '.join(other[:6])}); dismissing it may discard, end or commit " + "something" ) return None diff --git a/packages/flow/tests/test_repair_covered_band.py b/packages/flow/tests/test_repair_covered_band.py index 88b53c912..e361bd2a4 100644 --- a/packages/flow/tests/test_repair_covered_band.py +++ b/packages/flow/tests/test_repair_covered_band.py @@ -473,12 +473,21 @@ def test_a_cover_without_an_id_never_matches_a_click_without_one() -> None: "Delete this draft?", "Submit the order now?", "Your changes will be lost.", + # what a list of forbidden words would miss (review of this PR) + "Are you sure you want to leave? Yes No", + "Log out of the other session? OK", + "Your changes will be deleted. OK", + "Quit the application", + "Stop the running import", + "Disconnect other user", + "Proceed anyway", + "Änderungen verwerfen OK", ], ) def test_a_dialog_whose_message_can_lose_or_commit_data_is_refused(message) -> None: from openadapt_flow.repair.self_repair import SelfRepairRefused - with pytest.raises(SelfRepairRefused, match="dialog's message"): + with pytest.raises(SelfRepairRefused, match="covering dialog"): _guard(at={"hwnd": 7, "pid": 42}, message=message) @@ -554,3 +563,58 @@ def test_a_covered_band_bridge_is_exactly_one_click(kinds) -> None: [False], None, ) + + +@pytest.mark.parametrize( + "text", + [ + "Session notice Your session expires in 5 minutes. OK", + "Information A new version is available. OK", + ], +) +def test_only_notice_words_pass_the_message_gate(text) -> None: + from openadapt_flow.repair.self_repair import _dialog_message_refusal + + assert _dialog_message_refusal(text) is None + + +def test_new_click_runs_the_cover_guard_before_admitting() -> None: + from types import SimpleNamespace + + from openadapt_flow.repair.self_repair import ( + SelfRepair, + SelfRepairRefused, + _CoveredHalt, + ) + + calls: list[tuple] = [] + + def guard(session, point, screen): + calls.append(tuple(point)) + raise SelfRepairRefused("guarded") + + me = SimpleNamespace( + _covered=_CoveredHalt(("s", "after", None), cover={}, pinned="search"), + vision=SimpleNamespace(ocr=lambda png: [], wait_settled=lambda b: None), + _dismisses_the_cover=guard, + ) + import io + + from PIL import Image + + buf = io.BytesIO() + Image.new("RGB", (800, 600), "white").save(buf, format="PNG") + structural = SimpleNamespace(name="OK") + with pytest.raises(SelfRepairRefused, match="guarded"): + SelfRepair._new_click( + me, + _workflow(), + "search_bridge_1", + SimpleNamespace(kind="click"), + (598, 440), + structural, + buf.getvalue(), + (0, 0, 40, 20), + SimpleNamespace(backend=object()), + ) + assert calls == [(598, 440)] From 4b3f5584e24501bee47a0c0a284d53fa50e43bda Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 12:27:28 -0500 Subject: [PATCH 3/5] Covered band: only a session notice with 'OK' may be dismissed; no question in any script --- packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md | 8 +- .../flow/openadapt_flow/repair/self_repair.py | 85 ++++++++++--------- .../flow/tests/test_repair_covered_band.py | 46 +++++++++- 3 files changed, 97 insertions(+), 42 deletions(-) diff --git a/packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md b/packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md index e0d810c62..8f158e2b1 100644 --- a/packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md +++ b/packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md @@ -170,9 +170,11 @@ when all of these hold: The dismiss click is held to that dialog. It must land in the very window `owned_window_over` reported (asked again at the click point), and the dialog's text must read as a notice only. Every word in it must be on a -short list of notice words (a session that expires, a version that is -available, "OK"), and it may not ask a question. Anything else refuses, -including a dialog in another language or with nothing readable in it. +short list of session-notice words ("Your session expires in 5 minutes", +"OK"), and it shows no punctuation beyond `. , : !`, so it asks no question +in any script. The click's label must be "OK". Anything else refuses, +including an update prompt, a dialog in another language, or one with +nothing readable in it. Answering "OK" to "Discard unsaved changes?" would lose data. The bridge is exactly that one click. A covered-band repair gets one attempt: a later attempt reopens the app, whose dialog is a new window, so its click is diff --git a/packages/flow/openadapt_flow/repair/self_repair.py b/packages/flow/openadapt_flow/repair/self_repair.py index bcdeec990..48266fc87 100644 --- a/packages/flow/openadapt_flow/repair/self_repair.py +++ b/packages/flow/openadapt_flow/repair/self_repair.py @@ -1521,6 +1521,7 @@ def _new_click( from openadapt_flow.ir import Anchor from openadapt_flow.risk import is_meaningful_label, is_write_shaped from openadapt_flow.runtime.heal import _crop_png, _reocr_text + from openadapt_flow.runtime.identity import squash region = _centered_region( point, (size_like[2], size_like[3]), _png_size(screen) @@ -1539,6 +1540,11 @@ def _new_click( "only lead to the recorded save" ) if self._covered is not None: + if squash(label).casefold() != _NOTICE_DISMISS_LABEL: + raise SelfRepairRefused( + f"a covered identity band is dismissed only with 'OK'; the " + f"bridge clicks {label!r}" + ) self._dismisses_the_cover(session, point, screen) try: admission = admit_new_click( @@ -2949,76 +2955,79 @@ def _cover_explains_cut( #: The only words a dismissable covering dialog may show (title, message -#: and button), casefolded: a notice that something expires, is available or -#: is informational, answered with "OK". Any other word, a question mark, or -#: no readable word refuses. An allowlist, so an unlisted "Log out of the -#: other session?", "Your changes will be deleted" or a dialog in another -#: language is refused rather than dismissed. +#: and button), casefolded after NFKC: a notice that the session expires, +#: answered with "OK". Any other word, any punctuation outside +#: ``_NOTICE_DIALOG_MARKS``, or no readable word refuses. An allowlist, so an +#: unlisted "Log out of the other session?", "Update the application now" or +#: a dialog in another language is refused rather than dismissed. _NOTICE_DIALOG_WORDS = frozenset( { "a", - "about", - "an", - "app", - "application", - "available", - "been", - "for", - "has", + "expire", + "expires", + "expiring", "hour", "hours", "in", - "info", "information", "is", "minute", "minutes", - "new", "notice", - "now", - "of", "ok", "reminder", "second", "seconds", "session", - "soon", "the", "this", - "update", - "version", "will", - "you", + "x", # the close glyph, as OCR may read it "your", - "expire", - "expires", - "expiring", } ) +#: Marks a notice may carry besides letters and digits: no question mark in +#: any script, nothing else. +_NOTICE_DIALOG_MARKS = frozenset(" \t\n.,:!×") + +#: The one label the dismiss click may have. +_NOTICE_DISMISS_LABEL = "ok" + def _dialog_message_refusal(text: str) -> Optional[str]: """Why a covering dialog's text forbids dismissing it, or None. - Only a notice may be dismissed by a bridge: every word must be in - ``_NOTICE_DIALOG_WORDS`` (numbers aside) and nothing may ask a question. - Answering "OK" to "Discard unsaved changes?" or "Log out of the other - session?" loses data, so anything not plainly a notice is refused, - including unreadable text. + Only a session notice may be dismissed by a bridge: after NFKC, every + word must be in ``_NOTICE_DIALOG_WORDS`` (numbers aside) and every other + character in ``_NOTICE_DIALOG_MARKS``, so nothing asks a question in any + script. Answering "OK" to "Discard unsaved changes?" or "Update the + application now" loses data, so anything else is refused, including + unreadable text. """ - words = [ - w for w in "".join(c if c.isalnum() else " " for c in text.casefold()).split() - ] - if not words: - return "the covering dialog's message can't be read" - if "?" in text: + import unicodedata + + folded = unicodedata.normalize("NFKC", text).casefold() + marks = sorted( + {c for c in folded if not c.isalnum() and c not in _NOTICE_DIALOG_MARKS} + ) + if marks: return ( - "the covering dialog asks a question; answering it may discard, end " - "or commit something" + "the covering dialog shows " + f"{' '.join(marks[:6])!r}, which a notice does not; answering it may " + "discard, end or commit something" ) + words = folded.replace("×", " ") + words_list = [w for w in "".join(c if c.isalnum() else " " for c in words).split()] + if not words_list: + return "the covering dialog's message can't be read" other = sorted( - {w for w in words if w not in _NOTICE_DIALOG_WORDS and not w.isdecimal()} + { + w + for w in words_list + if w not in _NOTICE_DIALOG_WORDS and not (w.isascii() and w.isdecimal()) + } ) if other: return ( diff --git a/packages/flow/tests/test_repair_covered_band.py b/packages/flow/tests/test_repair_covered_band.py index e361bd2a4..14040e818 100644 --- a/packages/flow/tests/test_repair_covered_band.py +++ b/packages/flow/tests/test_repair_covered_band.py @@ -482,6 +482,14 @@ def test_a_cover_without_an_id_never_matches_a_click_without_one() -> None: "Disconnect other user", "Proceed anyway", "Änderungen verwerfen OK", + # notice-like words that still act (re-review of this PR) + "Update the application now. OK", + "A new version of the application is available. Update now", + "This session will expire now. OK", + "Your session has expired. OK", + "Session notice Your session expires in 5 minutes？ OK", # fullwidth + "¿Session notice OK", + "Session notice; OK", ], ) def test_a_dialog_whose_message_can_lose_or_commit_data_is_refused(message) -> None: @@ -569,7 +577,8 @@ def test_a_covered_band_bridge_is_exactly_one_click(kinds) -> None: "text", [ "Session notice Your session expires in 5 minutes. OK", - "Information A new version is available. OK", + "Session notice × Your session expires in 5 minutes. OK", + "Session notice x Your session expires in 5 minutes. OK", ], ) def test_only_notice_words_pass_the_message_gate(text) -> None: @@ -618,3 +627,38 @@ def guard(session, point, screen): SimpleNamespace(backend=object()), ) assert calls == [(598, 440)] + + +@pytest.mark.parametrize("name", ["Yes", "Continue", "Log out", "Close"]) +def test_the_dismiss_click_must_be_ok(name) -> None: + from types import SimpleNamespace + + from openadapt_flow.repair.self_repair import ( + SelfRepair, + SelfRepairRefused, + _CoveredHalt, + ) + + me = SimpleNamespace( + _covered=_CoveredHalt(("s", "after", None), cover={}, pinned="search"), + vision=SimpleNamespace(ocr=lambda png: []), + _dismisses_the_cover=lambda *a: None, + ) + import io + + from PIL import Image + + buf = io.BytesIO() + Image.new("RGB", (800, 600), "white").save(buf, format="PNG") + with pytest.raises(SelfRepairRefused, match="only with 'OK'|reads as a save"): + SelfRepair._new_click( + me, + _workflow(), + "search_bridge_1", + SimpleNamespace(kind="click"), + (598, 440), + SimpleNamespace(name=name), + buf.getvalue(), + (0, 0, 40, 20), + SimpleNamespace(backend=object()), + ) From 1ecf4514d307cdbb777b4b7214208320f09c131e Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 12:31:23 -0500 Subject: [PATCH 4/5] Covered band: a dismissable dialog must say a session expires; text across its edge refuses --- packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md | 4 +- .../flow/openadapt_flow/repair/self_repair.py | 43 ++++++++++++---- .../flow/tests/test_repair_covered_band.py | 49 ++++++++++++++++++- 3 files changed, 84 insertions(+), 12 deletions(-) diff --git a/packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md b/packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md index 8f158e2b1..9faafda00 100644 --- a/packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md +++ b/packages/flow/docs/REPAIR_ON_EVERY_BACKEND.md @@ -172,7 +172,9 @@ The dismiss click is held to that dialog. It must land in the very window dialog's text must read as a notice only. Every word in it must be on a short list of session-notice words ("Your session expires in 5 minutes", "OK"), and it shows no punctuation beyond `. , : !`, so it asks no question -in any script. The click's label must be "OK". Anything else refuses, +in any script. It must also say that a session expires, so a dialog whose +message wasn't read (only its "OK" was) is refused, and so is any text that +runs across the dialog's edge. The click's label must be "OK". Anything else refuses, including an update prompt, a dialog in another language, or one with nothing readable in it. Answering "OK" to "Discard unsaved changes?" would lose data. The bridge is diff --git a/packages/flow/openadapt_flow/repair/self_repair.py b/packages/flow/openadapt_flow/repair/self_repair.py index 48266fc87..479a8228e 100644 --- a/packages/flow/openadapt_flow/repair/self_repair.py +++ b/packages/flow/openadapt_flow/repair/self_repair.py @@ -1629,12 +1629,23 @@ def _dismisses_the_cover( lines = self.vision.ocr(screen) except Exception: # noqa: BLE001 - unreadable is a refusal below lines = [] - text = " ".join( - str(line.text) - for line in lines - if x <= line.region[0] + line.region[2] / 2 < x + w - and y <= line.region[1] + line.region[3] / 2 < y + h - ) + inside: list[str] = [] + for line in lines: + lx, ly, lw, lh = (int(v) for v in line.region) + overlap_w = min(lx + lw, x + w) - max(lx, x) + overlap_h = min(ly + lh, y + h) - max(ly, y) + if overlap_w <= 0 or overlap_h <= 0: + continue + if lx >= x and ly >= y and lx + lw <= x + w and ly + lh <= y + h: + inside.append(str(line.text)) + elif overlap_w > _STRADDLE_SLACK_PX and overlap_h > _STRADDLE_SLACK_PX: + # A line reaching into the dialog from outside may carry + # its message merged with other text: it can't be read. + raise SelfRepairRefused( + "text runs across the covering dialog's edge, so its " + "message can't be read on its own" + ) + text = " ".join(inside) refusal = _dialog_message_refusal(text) if refusal is not None: raise SelfRepairRefused(refusal) @@ -2982,6 +2993,7 @@ def _cover_explains_cut( "the", "this", "will", + "i", # the information icon, as OCR may read it "x", # the close glyph, as OCR may read it "your", } @@ -2991,6 +3003,14 @@ def _cover_explains_cut( #: any script, nothing else. _NOTICE_DIALOG_MARKS = frozenset(" \t\n.,:!×") +#: How far, in pixels, an OCR line may reach over the dialog's edge before +#: it counts as text running across it (a glyph's antialiasing, a border). +_STRADDLE_SLACK_PX = 4 + +#: A notice must say both: what it is about, and that it expires. +_NOTICE_SUBJECT_WORDS = frozenset({"session"}) +_NOTICE_EXPIRY_WORDS = frozenset({"expire", "expires", "expiring"}) + #: The one label the dismiss click may have. _NOTICE_DISMISS_LABEL = "ok" @@ -3031,9 +3051,14 @@ def _dialog_message_refusal(text: str) -> Optional[str]: ) if other: return ( - "the covering dialog's message reads as more than a notice " - f"({', '.join(other[:6])}); dismissing it may discard, end or commit " - "something" + "the covering dialog shows words a session notice does not " + f"({', '.join(other[:6])}); it is not dismissed" + ) + present = set(words_list) + if not present & _NOTICE_SUBJECT_WORDS or not present & _NOTICE_EXPIRY_WORDS: + return ( + "the covering dialog's text does not say that a session expires; " + "only that notice is dismissed (its message may not have been read)" ) return None diff --git a/packages/flow/tests/test_repair_covered_band.py b/packages/flow/tests/test_repair_covered_band.py index 14040e818..e5015fb77 100644 --- a/packages/flow/tests/test_repair_covered_band.py +++ b/packages/flow/tests/test_repair_covered_band.py @@ -408,7 +408,7 @@ def _guard( _Line("Session notice", (395, 318, 80, 14)), _Line(message, (450, 374, 170, 14)), _Line("OK", (590, 433, 20, 14)), - _Line("Patient: Neil Sorenson (1984-", (230, 320, 150, 14)), # outside + _Line("Patient: Neil Sorenson (1984-", (230, 320, 153, 14)), # 3px in ] if not lines_in: lines = lines[-1:] @@ -490,12 +490,16 @@ def test_a_cover_without_an_id_never_matches_a_click_without_one() -> None: "Session notice Your session expires in 5 minutes？ OK", # fullwidth "¿Session notice OK", "Session notice; OK", + # nothing proves a notice: the message line was not read (third review) + "OK", + "This is the information. OK", + "Session notice OK", ], ) def test_a_dialog_whose_message_can_lose_or_commit_data_is_refused(message) -> None: from openadapt_flow.repair.self_repair import SelfRepairRefused - with pytest.raises(SelfRepairRefused, match="covering dialog"): + with pytest.raises(SelfRepairRefused, match="covering dialog|session expires"): _guard(at={"hwnd": 7, "pid": 42}, message=message) @@ -579,6 +583,7 @@ def test_a_covered_band_bridge_is_exactly_one_click(kinds) -> None: "Session notice Your session expires in 5 minutes. OK", "Session notice × Your session expires in 5 minutes. OK", "Session notice x Your session expires in 5 minutes. OK", + "Session notice i Your session expires in 5 minutes. OK", ], ) def test_only_notice_words_pass_the_message_gate(text) -> None: @@ -662,3 +667,43 @@ def test_the_dismiss_click_must_be_ok(name) -> None: (0, 0, 40, 20), SimpleNamespace(backend=object()), ) + + +def test_text_running_across_the_dialog_edge_is_refused() -> None: + from types import SimpleNamespace + + from openadapt_flow.repair.self_repair import ( + SelfRepair, + SelfRepairRefused, + _CoveredHalt, + ) + + cover = {"hwnd": 7, "pid": 42, "region": [380, 300, 270, 160]} + lines = [ + _Line("Session notice", (395, 318, 80, 14)), + # merged with text outside: its center is left of the dialog + _Line("Patient: Neil Sorenson Delete this record", (200, 374, 300, 14)), + _Line("Your session expires in 5 minutes.", (450, 400, 170, 14)), + _Line("OK", (590, 433, 20, 14)), + ] + me = SimpleNamespace( + _covered=_CoveredHalt(("s", "after", None), cover=cover, pinned="search"), + vision=SimpleNamespace(ocr=lambda png: lines), + ) + + class Here: + def owned_window_over(self, region, *, pinned_step_id): + return {"hwnd": 7, "pid": 42} + + with pytest.raises(SelfRepairRefused, match="across the covering dialog"): + SelfRepair._dismisses_the_cover( + me, SimpleNamespace(backend=Here()), (598, 440), b"png" + ) + + +def test_band_text_that_ends_at_the_dialog_edge_is_not_its_message() -> None: + # The cut band line ends where the dialog starts (within a few pixels). + queried = _guard( + at={"hwnd": 7, "pid": 42}, message="Your session expires in 5 minutes." + ) + assert queried From 15fd6266fddbc3a57a9f6d117bd88e5811a28f82 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 12:35:57 -0500 Subject: [PATCH 5/5] Covered band: accept OCR's run-together notice words only when they split fully into notice words --- .../flow/openadapt_flow/repair/self_repair.py | 38 +++++++++--- .../flow/tests/test_repair_covered_band.py | 58 +++++++++++++++++++ 2 files changed, 88 insertions(+), 8 deletions(-) diff --git a/packages/flow/openadapt_flow/repair/self_repair.py b/packages/flow/openadapt_flow/repair/self_repair.py index 479a8228e..02c4e5567 100644 --- a/packages/flow/openadapt_flow/repair/self_repair.py +++ b/packages/flow/openadapt_flow/repair/self_repair.py @@ -3015,6 +3015,34 @@ def _cover_explains_cut( _NOTICE_DISMISS_LABEL = "ok" +def _notice_parts(token: str) -> Optional[list[str]]: + """``token`` as allowlisted notice words and ASCII numbers, or None. + + OCR often runs a line's words together (``yoursession``, ``in5minutes``), + so a token may be several words; it is accepted only when it splits + completely into notice words of two or more letters and digit runs. A + one-letter word (``a``, ``i``, ``x``) counts only as a whole token. + """ + + if token in _NOTICE_DIALOG_WORDS or (token.isascii() and token.isdecimal()): + return [token] + if not token.isascii(): + return None + words = sorted((w for w in _NOTICE_DIALOG_WORDS if len(w) > 1), key=len) + best: list[Optional[list[str]]] = [None] * (len(token) + 1) + best[0] = [] + for end in range(1, len(token) + 1): + for start in range(end): + before = best[start] + if before is None: + continue + piece = token[start:end] + if piece in words or piece.isdecimal(): + best[end] = [*before, piece] + break + return best[len(token)] + + def _dialog_message_refusal(text: str) -> Optional[str]: """Why a covering dialog's text forbids dismissing it, or None. @@ -3042,19 +3070,13 @@ def _dialog_message_refusal(text: str) -> Optional[str]: words_list = [w for w in "".join(c if c.isalnum() else " " for c in words).split()] if not words_list: return "the covering dialog's message can't be read" - other = sorted( - { - w - for w in words_list - if w not in _NOTICE_DIALOG_WORDS and not (w.isascii() and w.isdecimal()) - } - ) + other = sorted({w for w in words_list if _notice_parts(w) is None}) if other: return ( "the covering dialog shows words a session notice does not " f"({', '.join(other[:6])}); it is not dismissed" ) - present = set(words_list) + present = {part for w in words_list for part in _notice_parts(w) or ()} if not present & _NOTICE_SUBJECT_WORDS or not present & _NOTICE_EXPIRY_WORDS: return ( "the covering dialog's text does not say that a session expires; " diff --git a/packages/flow/tests/test_repair_covered_band.py b/packages/flow/tests/test_repair_covered_band.py index e5015fb77..396dd1086 100644 --- a/packages/flow/tests/test_repair_covered_band.py +++ b/packages/flow/tests/test_repair_covered_band.py @@ -707,3 +707,61 @@ def test_band_text_that_ends_at_the_dialog_edge_is_not_its_message() -> None: at={"hwnd": 7, "pid": 42}, message="Your session expires in 5 minutes." ) assert queried + + +@pytest.mark.parametrize( + "text", + [ + "Sessionnotice Yoursession expires in5minutes. OK", + "Session notice Yoursessionexpiresin5minutes. OK ×", + ], +) +def test_words_ocr_runs_together_still_pass(text) -> None: + from openadapt_flow.repair.self_repair import _dialog_message_refusal + + assert _dialog_message_refusal(text) is None + + +@pytest.mark.parametrize( + "text", + [ + "Yoursession expires. Deleteall OK", + "Session notice Your session expires axdelete OK", + "Session notice Your session expiresnow OK", + ], +) +def test_a_run_together_token_must_split_into_notice_words(text) -> None: + from openadapt_flow.repair.self_repair import _dialog_message_refusal + + assert _dialog_message_refusal(text) is not None + + +def test_the_real_ocr_reads_a_session_notice_as_dismissable() -> None: + # The run's own OCR on a drawn "Session notice" dialog (the lab app's + # wording), so OCR quirks such as run-together words are covered. + pytest.importorskip("rapidocr_onnxruntime") + import io + + from PIL import Image, ImageDraw, ImageFont + + from openadapt_flow.repair.self_repair import _dialog_message_refusal + from openadapt_flow.vision import ocr + + img = Image.new("RGB", (520, 300), "white") + draw = ImageDraw.Draw(img) + try: + font = ImageFont.truetype("DejaVuSans.ttf", 18) + except OSError: + try: + font = ImageFont.truetype("Arial.ttf", 18) + except OSError: + pytest.skip("no TrueType font to draw the dialog") + draw.text((20, 20), "Session notice", fill="black", font=font) + draw.text((480, 20), "×", fill="black", font=font) + draw.text((70, 110), "Your session expires in 5 minutes.", fill="black", font=font) + draw.rectangle((380, 230, 490, 270), outline="black") + draw.text((420, 238), "OK", fill="black", font=font) + buf = io.BytesIO() + img.save(buf, format="PNG") + text = " ".join(line.text for line in ocr(buf.getvalue())) + assert _dialog_message_refusal(text) is None, text