Covered band: the dismiss step's dialog box is the proved covering window

packages/flow/CHANGELOG.md

@@ -25,6 +25,8 @@
 
 ### Fixes
 
+- A covered-band dismiss step takes its dialog box from the window the backend proved covers the band, before any pixel search, so a native message box that the accessibility tree doesn't label as a dialog can be dismissed. Without a proved window the step is refused as before.
+
 - Self-repair on a covered identity band dismisses only the proved dialog: the covering window must span the band from where the read stops to its right edge at full height, the dismiss click must land in that same window, and only a dialog whose every word is a notice word (no question) may be dismissed ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)).
 
 - Self-repair can take a stop where the app's own dialog covers the next step's identity band: when the backend proves the covering window belongs to the window the last verified step was in, and the part of the band read still matches the record, the dialog is dismissed after the previous click and the step re-checks its identity in full. Another record behind the dialog stays an identity halt. The runtime lets a backend remember a verified record's window (`record_verified_window`); macOS answers `owned_window_over` from the window's proved sheet ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)).

packages/flow/openadapt_flow/repair/self_repair.py

@@ -1562,7 +1562,16 @@ def _new_click(
         files: dict[str, bytes] = {}
         if admission.basis != BASIS_RECORD_IDENTITY:
             identity_template, identifier_region, band_png = _dialog_band(
-                workflow, admission, screen, point, self.vision
+                workflow,
+                admission,
+                screen,
+                point,
+                self.vision,
+                proved_region=(
+                    _cover_region(self._covered.cover)
+                    if self._covered is not None
+                    else None
+                ),
             )
             identifier_crop = f"templates/identifiers/{step_id}.png"
             files[identifier_crop] = band_png
@@ -1622,6 +1631,13 @@ def _dismisses_the_cover(
                 "band; only that dialog may be dismissed"
             )
         rect = _cover_region(covered.cover)
+        if "region" in here and _cover_region(here) != rect:
+            # The dismiss step's dialog box is the proved region; a dialog
+            # that moved since the halt would leave it stale.
+            raise SelfRepairRefused(
+                "the covering dialog moved since the halt; its box can't be "
+                "used as the new step's identity"
+            )
         text = ""
         if rect is not None:
             x, y, w, h = rect
@@ -3234,13 +3250,17 @@ def _dialog_band(
     screen: bytes,
     point: tuple[int, int],
     vision: Any,
+    *,
+    proved_region: Optional[tuple[int, int, int, int]] = None,
 ) -> tuple[Any, tuple[int, int, int, int], bytes]:
     """The dialog around a new click, as the step's identity band.
 
     Used for a click admitted on the record check: the band is a screen
     pre-check (the same dialog shows), never what makes the click safe. The
     dialog's box comes from the element's ancestry when the tree reads one,
-    otherwise from the drawn box around the click. Its text is hashed with
+    otherwise from the drawn box around the click. ``proved_region`` is a
+    box the backend already proved is the dialog's window (a covered band's
+    ``owned_window_over``); it is used first. Its text is hashed with
     the bundle's salt, as a recorded identity band is. Refuses when there is
     no dialog around the click or it has no readable text.
     """
@@ -3254,7 +3274,9 @@ def _dialog_band(
     )
 
     element = admission.element
-    region = element.dialog_region if element is not None else None
+    region = proved_region
+    if region is None and element is not None:
+        region = element.dialog_region
     if region is None:
         region = enclosing_dialog_region(screen, point)
     if region is None:

packages/flow/tests/test_repair_covered_band.py

@@ -391,7 +391,7 @@ def broken(step):
 
 class _Line:
     def __init__(self, text: str, region: tuple[int, int, int, int]) -> None:
-        self.text, self.region = text, region
+        self.text, self.region, self.confidence = text, region, 1.0
 
 
 def _guard(
@@ -765,3 +765,55 @@ def test_the_real_ocr_reads_a_session_notice_as_dismissable() -> None:
     img.save(buf, format="PNG")
     text = " ".join(line.text for line in ocr(buf.getvalue()))
     assert _dialog_message_refusal(text) is None, text
+
+
+def test_the_dialog_band_uses_the_proved_window_before_a_pixel_search() -> None:
+    import io
+    from types import SimpleNamespace
+
+    from PIL import Image
+
+    from openadapt_flow.repair.self_repair import SelfRepairRefused, _dialog_band
+
+    buf = io.BytesIO()
+    Image.new("RGB", (800, 600), "white").save(buf, format="PNG")  # no drawn box
+    lines = [_Line("Your session expires in 5 minutes.", (450, 374, 170, 14))]
+    vision = SimpleNamespace(ocr=lambda png: lines)
+    admission = SimpleNamespace(element=None)
+    with pytest.raises(SelfRepairRefused, match="no dialog was found"):
+        _dialog_band(_workflow(), admission, buf.getvalue(), (598, 440), vision)
+    template, region, crop = _dialog_band(
+        _workflow(),
+        admission,
+        buf.getvalue(),
+        (598, 440),
+        vision,
+        proved_region=(380, 300, 270, 160),
+    )
+    assert region == (380, 300, 270, 160) and template is not None and crop
+    with pytest.raises(SelfRepairRefused, match="outside the dialog"):
+        _dialog_band(
+            _workflow(),
+            admission,
+            buf.getvalue(),
+            (100, 100),
+            vision,
+            proved_region=(380, 300, 270, 160),
+        )
+
+
+def test_a_dialog_that_moved_since_the_halt_is_refused() -> None:
+    from openadapt_flow.repair.self_repair import SelfRepairRefused
+
+    cover = {"hwnd": 7, "pid": 42, "region": [380, 300, 270, 160]}
+    with pytest.raises(SelfRepairRefused, match="moved"):
+        _guard(
+            at={"hwnd": 7, "pid": 42, "region": [300, 300, 270, 160]},
+            message="Your session expires in 5 minutes.",
+            cover=cover,
+        )
+    assert _guard(
+        at={"hwnd": 7, "pid": 42, "region": [380, 300, 270, 160]},
+        message="Your session expires in 5 minutes.",
+        cover=cover,
+    )