From 5931239fdc796afb75f7e5d53137e2303fc128b1 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 14:07:32 -0500 Subject: [PATCH 1/5] No command falls back to the demo app on its own replay with no --url (and no backend.url) served the bundled MockMed app; it now needs --demo-app for that, and refuses otherwise. --drift needs --demo-app. The launcher help, quickstart docs, the quickstart lifecycle script, e2e tests and Desktop's frozen smoke test pass it. The variant router's first-screen probe returns nothing without a URL. qualify accept's break-it gate ran every workflow's proposed oracle against MockMed (its banner text, /api/encounter, /api/db). It now builds the lie from the workflow: an in-process store that did not change, and a screen that shows the workflow's own success text, every typed value, and whatever a screen read looks for. The reference Execute worker ran the MockMed tutorial driver against any admission's target_url (resetting and reading it as MockMed). It now refuses such an admission before anything runs. --- docs/cli.md | 2 +- docs/getting-started/quickstart.md | 2 +- openadapt/cli.py | 4 +- .../desktop/scripts/smoke_test_frozen_flow.py | 1 + packages/flow/CHANGELOG.md | 2 + packages/flow/docs/TUTORIAL.md | 4 +- packages/flow/openadapt_flow/__main__.py | 40 +++-- packages/flow/openadapt_flow/api.py | 6 +- packages/flow/openadapt_flow/cli_undo.py | 7 +- packages/flow/openadapt_flow/cli_variants.py | 15 +- .../flow/openadapt_flow/execute/dispatch.py | 59 ++++--- .../qualification_oracle_gate.py | 145 ++++++++---------- packages/flow/scripts/quickstart_lifecycle.py | 2 + packages/flow/tests/e2e/test_free_path_e2e.py | 10 +- .../tests/e2e/test_record_compile_replay.py | 5 +- .../test_execute_post_dispatch_faults.py | 21 +++ .../tests/test_qualification_oracle_gate.py | 53 +++++++ 17 files changed, 229 insertions(+), 149 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 2ac4aa1020..fb6b4d1016 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -75,7 +75,7 @@ openadapt flow demo-record --out rec openadapt flow compile rec --out bundle --name my-task openadapt flow lint bundle --strict openadapt flow certify bundle --policy permissive -openadapt flow replay bundle --run-dir run +openadapt flow replay bundle --demo-app --run-dir run ``` The strict lint step refuses the unarmed demo write. The permissive policy is a diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 21e0da90da..7a88d0fd84 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -50,7 +50,7 @@ openadapt flow demo-record --out rec openadapt flow compile rec --out bundle --name my-task openadapt flow lint bundle --strict openadapt flow certify bundle --policy permissive -openadapt flow replay bundle --run-dir run +openadapt flow replay bundle --demo-app --run-dir run ``` The strict lint step returns a nonzero exit code. The bundled manual demo has diff --git a/openadapt/cli.py b/openadapt/cli.py index d8ca1a1bce..d3082120bb 100644 --- a/openadapt/cli.py +++ b/openadapt/cli.py @@ -3,7 +3,7 @@ Usage: openadapt flow record --url --out rec # record a workflow openadapt flow compile rec --out bundle # compile it - openadapt flow replay bundle # run it, local, $0 + openadapt flow replay bundle --url # run it, local, $0 openadapt flow lint bundle openadapt flow certify bundle --policy clinical-write @@ -116,7 +116,7 @@ def main(): openadapt flow compile rec --out bundle --name demo openadapt flow lint bundle openadapt flow certify bundle --policy permissive - openadapt flow replay bundle + openadapt flow replay bundle --demo-app Lint lists the demo's coverage gaps. The demo has warnings, so `lint --strict` exits 1 on it. diff --git a/packages/desktop/scripts/smoke_test_frozen_flow.py b/packages/desktop/scripts/smoke_test_frozen_flow.py index e836bc71aa..18fab1f05a 100644 --- a/packages/desktop/scripts/smoke_test_frozen_flow.py +++ b/packages/desktop/scripts/smoke_test_frozen_flow.py @@ -668,6 +668,7 @@ def main() -> int: *flow, "replay", str(root / "bundle"), + "--demo-app", "--run-dir", str(root / "run"), ], diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md index 33467f5c83..639a876ad8 100644 --- a/packages/flow/CHANGELOG.md +++ b/packages/flow/CHANGELOG.md @@ -25,6 +25,8 @@ ### Fixes +- No command falls back to the demo app on its own. `replay` needs a target: `--url`, `backend.url` in `--config`, or `--demo-app` to serve the bundled MockMed (the tutorial and quickstart commands pass it; `--drift` needs it). The variant router's first-screen probe captures nothing without a URL. `qualify accept`'s break-it check no longer runs every workflow's oracle against MockMed: it builds the lie from the workflow itself (a store that did not change, and a screen showing the workflow's own success text and every value the run typed). The reference Execute worker refuses, before running anything, an admission whose `target_url` it cannot drive, instead of resetting and reading that app as if it were the demo. + - Self-repair on a covered identity band dismisses only the proved dialog: the covering window must span the band from where the read stops to its right edge at full height, the dismiss click must land in that same window, and only a dialog whose every word is a notice word (no question) may be dismissed ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). - Self-repair can take a stop where the app's own dialog covers the next step's identity band: when the backend proves the covering window belongs to the window the last verified step was in, and the part of the band read still matches the record, the dialog is dismissed after the previous click and the step re-checks its identity in full. Another record behind the dialog stays an identity halt. The runtime lets a backend remember a verified record's window (`record_verified_window`); macOS answers `owned_window_over` from the window's proved sheet ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). diff --git a/packages/flow/docs/TUTORIAL.md b/packages/flow/docs/TUTORIAL.md index 32b8dbec2a..63a18b54a9 100644 --- a/packages/flow/docs/TUTORIAL.md +++ b/packages/flow/docs/TUTORIAL.md @@ -125,8 +125,8 @@ openadapt-flow compile rec --out bundle --name my-task # compile it openadapt-flow lint bundle # expected: finds demo gaps openadapt-flow certify bundle --policy permissive # smoke-policy pass openadapt-flow certify bundle --policy clinical-write # expected: strict refusal -openadapt-flow replay bundle # replay: local, $0 -openadapt-flow replay bundle --drift theme \ +openadapt-flow replay bundle --demo-app # replay: local, $0 +openadapt-flow replay bundle --demo-app --drift theme \ --save-healed-to healed # deterministic repair openadapt-flow visualize bundle -o graph.html # see what compiled ``` diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index bb52425ba8..8dc15072d1 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -2834,12 +2834,24 @@ def _cmd_replay(args: argparse.Namespace) -> int: headed = args.headed or cfg.backend.headed url = args.url or cfg.backend.url - refuse_unqualified_skill(bundle, target_url=url) - if url and args.drift: + demo_app = bool(getattr(args, "demo_app", False)) + if url and demo_app: + raise SystemExit("--demo-app serves its own app; drop --url (or backend.url).") + if args.drift and not demo_app: + raise SystemExit( + "--drift only applies to the bundled MockMed demo app (--demo-app); " + "drift your own app for real." + ) + if ( + url is None + and not demo_app + and not (backend_cfg.browser_cdp_endpoint or backend_cfg.browser_page_url) + ): raise SystemExit( - "--drift only applies to the bundled MockMed demo app; " - "omit --url to use it (drift your own app for real)." + "replay needs a target: pass --url, set backend.url in --config, or " + "pass --demo-app to replay against the bundled MockMed demo app." ) + refuse_unqualified_skill(bundle, target_url=url) held_backend: list = [] hold_s = min(max(float(getattr(args, "hold_on_halt", 0.0) or 0.0), 0.0), 1800.0) @@ -2925,13 +2937,13 @@ def run_on_page(page, *, entry_url: Optional[str]): ) stop = None - if url is None: + if url is None and demo_app: from openadapt_flow.mockmed.server import serve url, stop = serve(port=0) url = _with_drift(url, args.drift) drift_note = f" (drift: {args.drift})" if args.drift else "" - print(f"No --url given; replaying against bundled MockMed{drift_note}") + print(f"--demo-app: replaying against bundled MockMed{drift_note}") video_dir = getattr(args, "record_video", None) from playwright.sync_api import sync_playwright @@ -9279,22 +9291,30 @@ def build_parser() -> argparse.ArgumentParser: p = sub.add_parser( "replay", help=( - "Replay a bundle (serves the bundled MockMed demo app when " - "no --url is given)" + "Replay a bundle against --url, the config's backend.url, or " + "with --demo-app the bundled demo app" ), ) p.add_argument("bundle", help="Workflow bundle directory") p.add_argument( "--url", default=None, - help=("URL of the target app (default: serve the bundled MockMed demo app)"), + help="URL of the target app", + ) + p.add_argument( + "--demo-app", + action="store_true", + help=( + "Serve the bundled MockMed demo app and replay against it (for a " + "bundle recorded on it, such as the tutorial's)" + ), ) p.add_argument( "--drift", default=None, help=( "Comma-separated MockMed drift modes (theme,move,rename,modal) " - "to demonstrate bounded drift resolution; only valid without --url" + "to demonstrate bounded drift resolution; only with --demo-app" ), ) p.add_argument( diff --git a/packages/flow/openadapt_flow/api.py b/packages/flow/openadapt_flow/api.py index 94e6cae9d0..5ccc4f6abc 100644 --- a/packages/flow/openadapt_flow/api.py +++ b/packages/flow/openadapt_flow/api.py @@ -78,16 +78,20 @@ def replay( *, params: Optional[Mapping[str, str]] = None, run_dir: str | Path = "run", + demo_app: bool = False, ) -> RunReport: """Rehearse ``bundle`` against ``url`` and return the run report. - Without ``url`` it serves the bundled MockMed demo app, as the CLI does. + ``demo_app=True`` (and no ``url``) serves the bundled demo app instead, as + ``--demo-app`` does on the CLI. With neither, the replay is refused. A halted run is not an exception: check ``report.transaction_outcome``. """ argv = ["replay", str(bundle), "--run-dir", str(run_dir)] if url: argv += ["--url", url] + if demo_app: + argv.append("--demo-app") for key, value in (params or {}).items(): argv += ["--param", f"{key}={value}"] from openadapt_flow.__main__ import main diff --git a/packages/flow/openadapt_flow/cli_undo.py b/packages/flow/openadapt_flow/cli_undo.py index 3159281628..a2a518098e 100644 --- a/packages/flow/openadapt_flow/cli_undo.py +++ b/packages/flow/openadapt_flow/cli_undo.py @@ -178,7 +178,7 @@ def register( p.add_argument( "--url", default=None, - help="URL of the target app (default: the bundled MockMed demo app)", + help="URL of the target app (required unless the config names one)", ) p.add_argument("--headed", action="store_true", help="Run the browser headed") for add in add_run_flags: @@ -535,9 +535,8 @@ def _cmd_undo_accept(args: argparse.Namespace) -> int: def _cmd_undo_run(args: argparse.Namespace) -> int: run_dir = Path(args.run_dir) workflow = Workflow.load(args.bundle) - # Without a target, replay would start the bundled MockMed demo app: the - # undo would act on the demo, leave the real record alone and use up the - # approval. Refuse before claiming it. + # Without a target, replay has nowhere to act: the undo would use up the + # approval and change nothing. Refuse before claiming it. problem = args._undo_target_problem(args, workflow) if problem: print( diff --git a/packages/flow/openadapt_flow/cli_variants.py b/packages/flow/openadapt_flow/cli_variants.py index a846cc17ff..c61ae4eeb0 100644 --- a/packages/flow/openadapt_flow/cli_variants.py +++ b/packages/flow/openadapt_flow/cli_variants.py @@ -128,8 +128,8 @@ def first_screen_probe( ``--first-screen PNG`` wins. Otherwise the probe reads one frame through the same backend selection ``run`` uses: the attached signed-in browser tab, the native or remote-display window, or a freshly launched browser - at ``--url`` (the bundled MockMed app when no URL is given). It only - reads; it never clicks or types. + at ``--url``. With no target it captures nothing. It only reads; it + never clicks or types. """ def capture() -> Optional[bytes]: @@ -148,7 +148,6 @@ def _probe() -> Optional[bytes]: _attached_web_page, _deployment_sections, _resolve_backend_config, - _with_drift, ) from openadapt_flow.admitted_composition import resolve_pointer from openadapt_flow.backends.factory import _normalize_kind, build_backend @@ -172,15 +171,7 @@ def _probe() -> Optional[bytes]: with _attached_web_page(backend_cfg, url=url, workflow=workflow) as page: return build_backend(backend_cfg, page=page).screenshot() if url is None: - from openadapt_flow.mockmed.server import serve - - served, stop = serve(port=0) - try: - return _launched_web_screen( - _with_drift(served, getattr(args, "drift", None)) - ) - finally: - stop() + return None return _launched_web_screen(url) return capture diff --git a/packages/flow/openadapt_flow/execute/dispatch.py b/packages/flow/openadapt_flow/execute/dispatch.py index d0178f3d8f..62a29aeea3 100644 --- a/packages/flow/openadapt_flow/execute/dispatch.py +++ b/packages/flow/openadapt_flow/execute/dispatch.py @@ -114,8 +114,23 @@ def live_replay( request: ExecuteRequestV1, run_dir: Path, ) -> DispatchResult: - """Replay a compiled bundle in this process under the standard profile.""" + """Replay a compiled bundle in this process under the standard profile. + The reference worker replays only on the demo app it serves itself: its + runner resets and reads that app's store. An admission naming another + target is refused before anything runs; such a bundle runs through + ``openadapt-flow run BUNDLE --config deploy.yaml``, which reads the app's + own deployment config. + """ + + if admission.target_url: + raise DispatchError( + "the reference Execute worker replays only on its bundled demo app; " + f"it does not drive {admission.target_url!r}. Run this bundle with " + "`openadapt-flow run BUNDLE --config deploy.yaml` or `openadapt-flow " + "connector run`. Nothing was executed.", + pre_effect=True, + ) bundle = Path(admission.bundle_dir or "") if not bundle.is_dir(): raise DispatchError( @@ -158,36 +173,18 @@ def live_replay( ) stop: Optional[Callable[[], None]] = None try: - if admission.target_url: - base_url = admission.target_url - entry_query = ( - TUTORIAL_BREAK_ENTRY_QUERY - if admission.break_it - else TUTORIAL_ENTRY_QUERY - ) - report = run_tutorial_workflow( - base_url=base_url, - workflow=workflow, - bundle_dir=bundle, - run_dir=run_dir, - headed=False, - entry_query=entry_query, - ) - else: - base_url, _db, stop = serve_mockmed(port=0) - entry_query = ( - TUTORIAL_BREAK_ENTRY_QUERY - if admission.break_it - else TUTORIAL_ENTRY_QUERY - ) - report = run_tutorial_workflow( - base_url=base_url, - workflow=workflow, - bundle_dir=bundle, - run_dir=run_dir, - headed=False, - entry_query=entry_query, - ) + base_url, _db, stop = serve_mockmed(port=0) + entry_query = ( + TUTORIAL_BREAK_ENTRY_QUERY if admission.break_it else TUTORIAL_ENTRY_QUERY + ) + report = run_tutorial_workflow( + base_url=base_url, + workflow=workflow, + bundle_dir=bundle, + run_dir=run_dir, + headed=False, + entry_query=entry_query, + ) except Exception as exc: # The replay may have dispatched a write before it failed. raise DispatchError(f"local replay failed: {exc}", pre_effect=False) from exc diff --git a/packages/flow/openadapt_flow/qualification_oracle_gate.py b/packages/flow/openadapt_flow/qualification_oracle_gate.py index be2a094c3d..c50574a690 100644 --- a/packages/flow/openadapt_flow/qualification_oracle_gate.py +++ b/packages/flow/openadapt_flow/qualification_oracle_gate.py @@ -1,8 +1,9 @@ """Execute the qualify-proposal effect oracle. Do not only list the fault. Before ``qualify accept`` can succeed, the proposed oracle is run against a -``--break-it`` world: the success banner says saved, the store did not change. -MockMed is the default fixture because it has a real persistence boundary. +``--break-it`` world: the screen says saved and shows every value the run +typed, and the store did not change. The world is built from the workflow +itself (its own success text and parameters), so it is the same for every app. A model must not invent an endpoint. Actor bytes and oracle bytes must be disjoint. Re-reading the acting screen or the same-session banner HALTs. @@ -10,11 +11,7 @@ from __future__ import annotations -import json -import urllib.error -import urllib.request from typing import Any, Literal, Optional -from urllib.parse import urlsplit from pydantic import BaseModel, ConfigDict, Field, ValidationError @@ -33,7 +30,7 @@ ORACLE_GATE_SCHEMA: Literal["openadapt.qualification-oracle-gate/v1"] = ( "openadapt.qualification-oracle-gate/v1" ) -DEFAULT_BREAK_IT_FIXTURE = "mockmed" +DEFAULT_BREAK_IT_FIXTURE = "unchanged_store" MISSING_SECOND_READ = "do not automate until a second read exists" BROKEN_CASE_ACCEPTED = ( "proposed oracle would have accepted the broken case: " @@ -67,7 +64,8 @@ } ) _BANNER_TOKENS = ("saved", "success", "banner") -_MOCKMED_BANNER_PREFIX = "Encounter saved — " +#: The break-it store's base URL: an in-process read, never a network call. +_BREAK_IT_STORE = "http://break-it.invalid" class BoundEffect(BaseModel): @@ -221,8 +219,17 @@ def _step_effect(workflow: Workflow, item: BoundEffect) -> Effect: def _banner_text(workflow: Workflow) -> str: - note = str(workflow.params.get("note") or "example") - return _MOCKMED_BANNER_PREFIX + note[:40] + """What the lying screen shows: the workflow's own success text, and every + value the run typed, so a screen read of any of them is fooled.""" + + success = [ + pc.text + for step in _write_steps(workflow) + for pc in step.expect + if pc.kind is PostconditionKind.TEXT_PRESENT and pc.text + ] + values = [str(v) for v in workflow.params.values() if v not in (None, "")] + return " — ".join([*(success or ["Saved"]), *values]) def _png() -> bytes: @@ -256,30 +263,23 @@ def ocr(self, png: bytes, *, region: Any = None) -> list[_BannerLine]: return [_BannerLine(self._text)] -def _http_json( - url: str, *, method: str = "GET", body: Optional[dict[str, Any]] = None -) -> tuple[int, dict[str, Any]]: - data = None if body is None else json.dumps(body).encode("utf-8") - request = urllib.request.Request(url, data=data, method=method) - if data is not None: - request.add_header("Content-Type", "application/json") - try: - with urllib.request.urlopen(request, timeout=2) as response: - payload = json.loads(response.read().decode("utf-8")) - status = int(response.status) - except urllib.error.HTTPError as exc: - raw = exc.read().decode("utf-8") if exc.fp is not None else "" - try: - payload = json.loads(raw) if raw else {} - except json.JSONDecodeError: - payload = {} - return int(exc.code), payload if isinstance(payload, dict) else {} - return status, payload if isinstance(payload, dict) else {} +class _UnchangedStore: + """A records read that returns the same empty store before and after: + the write the screen claims never landed.""" + + class _Response: + status_code = 200 + headers: dict[str, str] = {} + + def json(self) -> dict[str, Any]: + return {"records": []} + def raise_for_status(self) -> None: + return None -def _origin(url: str) -> str: - parts = urlsplit(url) - return f"{parts.scheme}://{parts.netloc}" + def get(self, url: str, **kwargs: Any) -> "_UnchangedStore._Response": + del url, kwargs + return self._Response() def _resolved_effect(workflow: Workflow, effect: Effect) -> Effect: @@ -293,13 +293,16 @@ def _resolved_effect(workflow: Workflow, effect: Effect) -> Effect: def _execute_screen_oracle( workflow: Workflow, effect: Effect, banner_text: str ) -> EffectVerdict: + resolved = _resolved_effect(workflow, effect) + # A lying screen can show anything, so it shows what this read looks for. + expected = getattr(resolved, "value", None) + if expected not in (None, "") and str(expected) not in banner_text: + banner_text = f"{banner_text} — {expected}" verifier = OnScreenReadbackVerifier( _BreakItBannerBackend(), vision=_BreakItBannerVision(banner_text), ) - return verifier.verify( - _resolved_effect(workflow, effect), verifier.capture_pre_state() - ) + return verifier.verify(resolved, verifier.capture_pre_state()) def _execute_sor_oracle( @@ -322,12 +325,12 @@ def evaluate_oracle_gate( *, fixture: str = DEFAULT_BREAK_IT_FIXTURE, ) -> OracleGateResult: - """Run channel disjointness and the MockMed --break-it fault. + """Run channel disjointness and the --break-it fault. Status stays draft/halted when the proposed oracle would have accepted the lie, when actor and oracle share a channel, or when no second read - exists. Nothing is guessed. The MockMed ``/api/db`` URL is the fixture's - persistence boundary, not a production endpoint invented for the pin. + exists. Nothing is guessed. The break-it store is read in process; it + names no app's endpoint. """ actor = actor_channel_ids(workflow) @@ -381,50 +384,28 @@ def evaluate_oracle_gate( ), ) - from openadapt_flow.mockmed.fault_server import serve - banner_text = _banner_text(workflow) - records: list[dict[str, Any]] = [] - store_unchanged = False - screen_claimed_success = False verdicts: list[EffectVerdict] = [] - status = 0 - base_url, db, stop = serve() - try: - verifier = RestRecordVerifier( - _origin(base_url), - records_path="/api/db", - records_key="records", - timeout_s=2.0, - poll_interval_s=0.05, - ) - before = verifier.capture_pre_state() - encounter_url = f"{base_url.rstrip('/')}/api/encounter?fault=optimistic" - status, _payload = _http_json( - encounter_url, - method="POST", - body={ - "patient_id": str(workflow.params.get("record_id") or "p1"), - "type": "Triage", - "note": str(workflow.params.get("note") or "example"), - }, - ) - snapshot = db.snapshot() - records = list(snapshot.get("records") or []) - store_unchanged = len(records) == 0 - screen_claimed_success = True - for item in effects: - effect = _step_effect(workflow, item) - if set(item.oracle_channels) & _SCREEN_CHANNELS: - verdicts.append(_execute_screen_oracle(workflow, effect, banner_text)) - else: - verdicts.append( - _execute_sor_oracle( - workflow, effect, verifier=verifier, before=before - ) - ) - finally: - stop() + verifier = RestRecordVerifier( + _BREAK_IT_STORE, + records_path="/records", + records_key="records", + session=_UnchangedStore(), + timeout_s=2.0, + poll_interval_s=0.05, + ) + before = verifier.capture_pre_state() + records = list(before.records or []) + store_unchanged = before.reachable and not records + screen_claimed_success = True + for item in effects: + effect = _step_effect(workflow, item) + if set(item.oracle_channels) & _SCREEN_CHANNELS: + verdicts.append(_execute_screen_oracle(workflow, effect, banner_text)) + else: + verdicts.append( + _execute_sor_oracle(workflow, effect, verifier=verifier, before=before) + ) dumped = [ { @@ -447,8 +428,8 @@ def evaluate_oracle_gate( "proposed oracle did not refute the lying backend; " "refusing to guess that it would catch --break-it" ) - if status not in {409, 200} and not store_unchanged: - reasons.append("MockMed optimistic write did not leave the store unchanged") + if not store_unchanged: + reasons.append("the break-it store could not be read as unchanged") halt_reason = "; ".join(reasons) if reasons else None return OracleGateResult( fixture=fixture, diff --git a/packages/flow/scripts/quickstart_lifecycle.py b/packages/flow/scripts/quickstart_lifecycle.py index 7ef099ac27..07cc31d397 100644 --- a/packages/flow/scripts/quickstart_lifecycle.py +++ b/packages/flow/scripts/quickstart_lifecycle.py @@ -408,6 +408,7 @@ def run_lifecycle( *cli, "replay", str(bundle), + "--demo-app", "--run-dir", str(artifacts / "baseline-run"), ], @@ -420,6 +421,7 @@ def run_lifecycle( *cli, "replay", str(bundle), + "--demo-app", "--drift", "theme", "--save-healed-to", diff --git a/packages/flow/tests/e2e/test_free_path_e2e.py b/packages/flow/tests/e2e/test_free_path_e2e.py index 63f2242112..9f4e2c580d 100644 --- a/packages/flow/tests/e2e/test_free_path_e2e.py +++ b/packages/flow/tests/e2e/test_free_path_e2e.py @@ -244,7 +244,15 @@ def test_demo_replay_still_cannot_reach_verified( run_dir = tmp_path / "demo-run" assert ( - cli_main(["replay", str(free_path.bundle_dir), "--run-dir", str(run_dir)]) == 1 + cli_main( + [ + "replay", + str(free_path.bundle_dir), + "--demo-app", + "--run-dir", + str(run_dir), + ] + ) == 1 ) report = _report(run_dir) assert report.execution_profile == "demo" diff --git a/packages/flow/tests/e2e/test_record_compile_replay.py b/packages/flow/tests/e2e/test_record_compile_replay.py index 64c32896ed..46fd5ed44d 100644 --- a/packages/flow/tests/e2e/test_record_compile_replay.py +++ b/packages/flow/tests/e2e/test_record_compile_replay.py @@ -460,13 +460,14 @@ def cli(*args: str) -> subprocess.CompletedProcess: assert NOTE_TEXT not in content assert "mockmed-demo-pass" not in content - # README quickstart contract: `replay` with no --url self-serves - # MockMed, and --drift demonstrates healing in one command. + # README quickstart contract: `replay --demo-app` serves MockMed, + # and --drift demonstrates healing in one command. selfserve_run = tmp_path / "run-selfserve" healed = tmp_path / "healed" proc = cli( "replay", str(bundle), + "--demo-app", "--drift", "theme", "--run-dir", diff --git a/packages/flow/tests/test_execute_post_dispatch_faults.py b/packages/flow/tests/test_execute_post_dispatch_faults.py index 069996a29d..14f36d5d05 100644 --- a/packages/flow/tests/test_execute_post_dispatch_faults.py +++ b/packages/flow/tests/test_execute_post_dispatch_faults.py @@ -243,3 +243,24 @@ def finalize_fails_once(self: Any, *args: Any, **kwargs: Any) -> None: receipt = store.get_receipt(accepted.execution_id) assert receipt.outcome is ExecuteTerminalOutcomeV1.RECONCILIATION_REQUIRED assert receipt.delivery_uncertain is True + + +def test_live_replay_refuses_a_target_it_cannot_drive(tmp_path: Path) -> None: + """The reference worker never resets or reads someone else's app.""" + from openadapt_flow.execute.dispatch import DispatchError, live_replay + from openadapt_flow.execute.models import AdmittedBundle + + admission = AdmittedBundle( + qualification_id="q", + workflow_version="v1", + workflow_digest="sha256:" + "0" * 64, + environment_id="env", + minimum_effect_strength="independent_system_of_record", + bundle_dir=str(tmp_path), + target_url="https://app.example/emr", + ) + with pytest.raises( + DispatchError, match="replays only on its bundled demo app" + ) as err: + live_replay(admission, None, tmp_path / "run") # type: ignore[arg-type] + assert err.value.pre_effect diff --git a/packages/flow/tests/test_qualification_oracle_gate.py b/packages/flow/tests/test_qualification_oracle_gate.py index 6a9cdae816..d54a82f64d 100644 --- a/packages/flow/tests/test_qualification_oracle_gate.py +++ b/packages/flow/tests/test_qualification_oracle_gate.py @@ -255,3 +255,56 @@ def propose(self, target: str, kind: str, context: dict[str, Any]) -> str: assert accepted.status == "accepted" # The sketch is not the pin. The structural SoR effect still is. assert accepted.pin("effect")["payload"]["effects"][0]["kind"] == "field_equals" + + +def _loan_workflow(effects: list[Effect]) -> Workflow: + """A Frappe-Lending-shaped write: nothing in it is MockMed's.""" + return Workflow( + name="loan-application", + params={"applicant": "Synthetic Applicant", "loan_amount": "125000"}, + steps=[_step(effects)], + ) + + +def test_break_it_on_another_apps_record_read_refutes() -> None: + """A record read for another app's write catches the unchanged store.""" + gate = evaluate_oracle_gate( + _loan_workflow( + [ + Effect( + kind=EffectKind.RECORD_WRITTEN, + match={ + "applicant": ValueExpr(param="applicant"), + "loan_amount": ValueExpr(param="loan_amount"), + }, + expected_count=1, + risk="irreversible", + ) + ] + ) + ) + assert gate.fixture == "unchanged_store" + assert gate.break_it_executed and gate.store_unchanged + assert gate.break_it_verdicts[0]["verdict"] == "refuted" + assert BROKEN_CASE_ACCEPTED not in (gate.halt_reason or "") + + +def test_break_it_fools_another_apps_screen_read() -> None: + """A screen read for another app's success text is accepted by the lie.""" + gate = evaluate_oracle_gate( + _loan_workflow( + [ + Effect( + kind=EffectKind.FIELD_EQUALS, + value="Loan Application saved", + readback=ReadbackSpec( + region=(10, 10, 40, 20), different_path=False + ), + risk="irreversible", + ) + ] + ) + ) + assert gate.break_it_verdicts[0]["verdict"] == "confirmed" + assert BROKEN_CASE_ACCEPTED in (gate.halt_reason or "") + assert not gate.passed From 4f3212d1c8e90780751e2b66fb6ee0543b49d094 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 15:13:07 -0500 Subject: [PATCH 2/5] Review fixes: --demo-app tests, attached-browser refusal, synthetic targets Adversarial review of the fallback change: two tests replayed without a target (now --demo-app); --demo-app with an attached browser was silently ignored (now refused); a synthetic Execute admission that names a target_url was reported as verified without running (now refused before anything runs); the bug template shows --demo-app. --- .../flow/.github/ISSUE_TEMPLATE/bug_report.yml | 2 +- packages/flow/openadapt_flow/__main__.py | 9 +++++++-- .../flow/openadapt_flow/execute/dispatch.py | 6 ++++++ packages/flow/tests/test_api.py | 2 +- .../tests/test_execute_post_dispatch_faults.py | 18 ++++++++++++++++++ packages/flow/tests/test_privacy.py | 2 +- 6 files changed, 34 insertions(+), 5 deletions(-) diff --git a/packages/flow/.github/ISSUE_TEMPLATE/bug_report.yml b/packages/flow/.github/ISSUE_TEMPLATE/bug_report.yml index be6258e74a..3c4e088c8e 100644 --- a/packages/flow/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/packages/flow/.github/ISSUE_TEMPLATE/bug_report.yml @@ -22,7 +22,7 @@ body: placeholder: | 1. openadapt-flow demo-record --out rec 2. openadapt-flow compile rec --out bundle --name my-task - 3. openadapt-flow replay bundle + 3. openadapt-flow replay bundle --demo-app validations: required: true - type: textarea diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index 8dc15072d1..0ef803ed4f 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -2835,8 +2835,13 @@ def _cmd_replay(args: argparse.Namespace) -> int: headed = args.headed or cfg.backend.headed url = args.url or cfg.backend.url demo_app = bool(getattr(args, "demo_app", False)) - if url and demo_app: - raise SystemExit("--demo-app serves its own app; drop --url (or backend.url).") + if demo_app and ( + url or backend_cfg.browser_cdp_endpoint or backend_cfg.browser_page_url + ): + raise SystemExit( + "--demo-app serves its own app; drop --url, backend.url or the " + "attached browser." + ) if args.drift and not demo_app: raise SystemExit( "--drift only applies to the bundled MockMed demo app (--demo-app); " diff --git a/packages/flow/openadapt_flow/execute/dispatch.py b/packages/flow/openadapt_flow/execute/dispatch.py index 62a29aeea3..b63d9793d3 100644 --- a/packages/flow/openadapt_flow/execute/dispatch.py +++ b/packages/flow/openadapt_flow/execute/dispatch.py @@ -64,6 +64,12 @@ def default_runner( ) -> DispatchResult: """Run synthetic MockMed or local governed replay.""" + if admission.synthetic and admission.target_url: + # A synthetic outcome describes the demo app, never another target. + raise DispatchError( + "a synthetic admission can't name a target_url; nothing was executed", + pre_effect=True, + ) if admission.synthetic or not admission.bundle_dir: return synthetic_mockmed(admission, request) return live_replay(admission, request, run_dir) diff --git a/packages/flow/tests/test_api.py b/packages/flow/tests/test_api.py index b02820265e..fd8d3ed434 100644 --- a/packages/flow/tests/test_api.py +++ b/packages/flow/tests/test_api.py @@ -18,7 +18,7 @@ def test_api_record_compile_replay_on_mockmed( assert main(["demo-record", "--out", "rec"]) == 0 bundle = api.compile("rec", out="bundle", name="api-test") - report = api.replay(bundle, params={"note": "from the API"}) + report = api.replay(bundle, params={"note": "from the API"}, demo_app=True) # replay is a rehearsal: it never reads the record back. assert report.transaction_outcome == "COMPLETED_UNVERIFIED" diff --git a/packages/flow/tests/test_execute_post_dispatch_faults.py b/packages/flow/tests/test_execute_post_dispatch_faults.py index 14f36d5d05..ec3b4a05bc 100644 --- a/packages/flow/tests/test_execute_post_dispatch_faults.py +++ b/packages/flow/tests/test_execute_post_dispatch_faults.py @@ -264,3 +264,21 @@ def test_live_replay_refuses_a_target_it_cannot_drive(tmp_path: Path) -> None: ) as err: live_replay(admission, None, tmp_path / "run") # type: ignore[arg-type] assert err.value.pre_effect + + +def test_a_synthetic_admission_cannot_name_a_target(tmp_path: Path) -> None: + from openadapt_flow.execute.dispatch import DispatchError, default_runner + from openadapt_flow.execute.models import AdmittedBundle + + admission = AdmittedBundle( + qualification_id="q", + workflow_version="v1", + workflow_digest="sha256:" + "0" * 64, + environment_id="env", + minimum_effect_strength="independent_system_of_record", + synthetic=True, + target_url="https://app.example/emr", + ) + with pytest.raises(DispatchError, match="synthetic admission") as err: + default_runner(admission, None, tmp_path / "run") # type: ignore[arg-type] + assert err.value.pre_effect diff --git a/packages/flow/tests/test_privacy.py b/packages/flow/tests/test_privacy.py index 1f1cab5d89..20b3f444dd 100644 --- a/packages/flow/tests/test_privacy.py +++ b/packages/flow/tests/test_privacy.py @@ -386,7 +386,7 @@ def fake_sync_playwright(): monkeypatch.setattr(cli, "_build_and_run_replayer", fake_run) args = cli.build_parser().parse_args( - ["replay", str(bundle), "--run-dir", str(run_dir)] + ["replay", str(bundle), "--demo-app", "--run-dir", str(run_dir)] ) with pytest.warns(PlaintextPHIWarning): From d2c8731eee9b0741dd9c5e49f6b00c1b3ddbf7d1 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 15:52:45 -0500 Subject: [PATCH 3/5] Frozen smoke test passes --demo-app only to a Flow that has it; format Desktop's frozen engine pins a released Flow, which serves the demo app whenever no URL is given and doesn't know --demo-app. --- packages/desktop/scripts/smoke_test_frozen_flow.py | 6 +++++- packages/flow/tests/e2e/test_free_path_e2e.py | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/desktop/scripts/smoke_test_frozen_flow.py b/packages/desktop/scripts/smoke_test_frozen_flow.py index 18fab1f05a..cb3eb1b3bb 100644 --- a/packages/desktop/scripts/smoke_test_frozen_flow.py +++ b/packages/desktop/scripts/smoke_test_frozen_flow.py @@ -663,12 +663,16 @@ def main() -> int: ], env=env, ) + # A Flow that serves the demo app only when asked takes --demo-app; + # an older pinned Flow serves it whenever no URL is given. + replay_help, _ = _run([*flow, "replay", "--help"], env=env) + demo_app = ["--demo-app"] if "--demo-app" in replay_help else [] _, replay_seconds = _run( [ *flow, "replay", str(root / "bundle"), - "--demo-app", + *demo_app, "--run-dir", str(root / "run"), ], diff --git a/packages/flow/tests/e2e/test_free_path_e2e.py b/packages/flow/tests/e2e/test_free_path_e2e.py index 9f4e2c580d..b988edc6f0 100644 --- a/packages/flow/tests/e2e/test_free_path_e2e.py +++ b/packages/flow/tests/e2e/test_free_path_e2e.py @@ -252,7 +252,8 @@ def test_demo_replay_still_cannot_reach_verified( "--run-dir", str(run_dir), ] - ) == 1 + ) + == 1 ) report = _report(run_dir) assert report.execution_profile == "demo" From cbaeea480814128a1abc382daa156c1c7eb32fa4 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 16:28:32 -0500 Subject: [PATCH 4/5] Re-run CI From 27b87c370569ac37d5610acd3d99acd8158a76c8 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 12:13:44 -0500 Subject: [PATCH 5/5] Lower the engine generality baseline: no command names the demo app now --- packages/flow/scripts/generality_baseline.json | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/packages/flow/scripts/generality_baseline.json b/packages/flow/scripts/generality_baseline.json index 50ebd1a16c..269cbec315 100644 --- a/packages/flow/scripts/generality_baseline.json +++ b/packages/flow/scripts/generality_baseline.json @@ -3,9 +3,6 @@ "mockmed": 43, "textedit": 1 }, - "api.py": { - "mockmed": 1 - }, "backend.py": { "mockmed-records": 1 }, @@ -21,12 +18,6 @@ "backends/win_agent/server.py": { "notepad": 1 }, - "cli_undo.py": { - "mockmed": 2 - }, - "cli_variants.py": { - "mockmed": 2 - }, "compile_agent/code_action.py": { "openemr": 1, "token_main": 1 @@ -141,11 +132,6 @@ "qualification_dev_signer.py": { "mockmed": 1 }, - "qualification_oracle_gate.py": { - "mockmed": 8, - "mockmed-records": 2, - "mockmed-write": 1 - }, "qualification_proposal.py": { "mockmed": 1 },