From bc98a3af23f4b245a0afeb25a14ffa68eae7b3a7 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 12:39:45 -0500 Subject: [PATCH 01/22] Engine generality audit; move the benchmark harnesses out of the wheel Audit every place the engine (packages/flow/openadapt_flow) names a specific application, bucketed as example, data, harness in the package, or real special-casing: docs/GENERALITY_AUDIT.md. Move openadapt_flow/benchmark and openadapt_flow/instrument to packages/flow/benchmark/harness (import benchmark.harness). The wheel packages only openadapt_flow, so it no longer ships them. The benchmark command loads the harness from a source checkout and says so from a wheel; instrument already failed to import from a wheel. Reword two comments that named an app on a generic branch. --- packages/flow/CHANGELOG.md | 2 + .../benchmark/agent_arm_verticals/README.md | 2 +- .../code_as_action/measure_openemr.py | 4 +- packages/flow/benchmark/dom/BENCHMARK.md | 4 +- .../flow/benchmark/effect_e2e/EFFECT_E2E.md | 2 +- .../benchmark/effect_e2e/record_service.py | 2 +- packages/flow/benchmark/effect_e2e/run.py | 8 +- .../effectbench/reference_fault_model.py | 10 +- .../benchmark/effectbench/task_pack/README.md | 2 +- .../effectbench/task_pack/__init__.py | 4 +- .../effectbench/task_pack/_authoring.py | 10 +- .../benchmark/effectbench/task_pack/audit.py | 2 +- .../benchmark/effectbench/task_pack/driver.py | 8 +- .../effectbench/task_pack/frappe_tasks.py | 8 +- .../effectbench/task_pack/mockmed_tasks.py | 8 +- .../effectbench/task_pack/openemr_tasks.py | 8 +- .../benchmark/effectbench/task_pack/pack.py | 6 +- .../flow/benchmark/frappe_lending/README.md | 2 +- .../harness}/__init__.py | 6 +- .../harness}/agent_baseline.py | 0 .../harness}/chart_fonts.py | 0 .../harness}/desktop_benchmark.py | 4 +- .../harness}/dom_arm.py | 20 +- .../harness}/effectbench/README.md | 2 +- .../harness}/effectbench/__init__.py | 8 +- .../harness}/effectbench/metrics.py | 4 +- .../harness}/effectbench/oracle.py | 6 +- .../harness}/effectbench/runner/README.md | 6 +- .../harness}/effectbench/runner/__init__.py | 26 +- .../harness}/effectbench/runner/__main__.py | 18 +- .../harness}/effectbench/runner/arms.py | 12 +- .../harness}/effectbench/runner/baselines.py | 8 +- .../harness}/effectbench/runner/compound.py | 8 +- .../harness}/effectbench/runner/harness.py | 14 +- .../effectbench/runner/reference_tasks.py | 12 +- .../harness}/effectbench/runner/substrate.py | 4 +- .../harness}/effectbench/schema.py | 4 +- .../harness}/frappe_lending.py | 0 .../harness}/hybrid_benchmark.py | 14 +- .../harness}/instrument/__init__.py | 4 +- .../harness}/instrument/competitor_drift.py | 6 +- .../harness}/openemr_benchmark.py | 12 +- .../harness}/openemr_local.py | 2 +- .../harness}/reliability.py | 0 .../harness}/run_benchmark.py | 10 +- .../harness}/silent_wrong_action.py | 8 +- .../benchmark => benchmark/harness}/verify.py | 0 packages/flow/benchmark/hybrid/BENCHMARK.md | 2 +- .../benchmark/lending_fault_model/README.md | 2 +- .../benchmark/lending_fault_model/swer.py | 4 +- .../openemr_api_tier/approval_durable.py | 2 +- .../flow/benchmark/openemr_api_tier/common.py | 2 +- .../benchmark/openemr_api_tier/feature_e2e.py | 4 +- .../benchmark/openemr_api_tier/spike/bench.py | 2 +- .../openemr_api_tier/spike/capture_har.py | 2 +- packages/flow/benchmark/openemr_e2e/README.md | 2 +- .../flow/benchmark/openemr_e2e/harness.py | 2 +- .../flow/benchmark/openemr_local/README.md | 2 +- .../benchmark/openemr_throughput/harness.py | 4 +- .../flow/benchmark/openimis_claims/README.md | 2 +- .../benchmark/scoreboard/tasks/openemr.py | 2 +- .../flow/benchmark/silent_saves/metrics.py | 2 +- .../SILENT_WRONG_ACTION.md | 2 +- .../flow/benchmark/vision_hardening/README.md | 2 +- packages/flow/docs/EFFECT_KIT.md | 2 +- packages/flow/docs/GENERALITY_AUDIT.md | 311 ++++++++++++++++++ packages/flow/docs/design/EFFECT_VERIFIER.md | 2 +- packages/flow/docs/desktop/PHASE2.md | 4 +- packages/flow/openadapt_flow/__main__.py | 25 +- .../flow/openadapt_flow/anthropic_defaults.py | 2 +- packages/flow/openadapt_flow/authoring.py | 2 +- packages/flow/openadapt_flow/deployment.py | 2 +- packages/flow/paper/REVIEW_ADVERSARIAL.md | 4 +- packages/flow/pyproject.toml | 2 +- packages/flow/scripts/desktop/waa_shim.py | 2 +- packages/flow/scripts/frappe_lending_demo.py | 4 +- packages/flow/scripts/openemr_demo.py | 10 +- packages/flow/scripts/openemr_local_demo.py | 4 +- packages/flow/scripts/openemr_param_depth.py | 2 +- .../test_region_stable_theme_regression.py | 2 +- packages/flow/tests/test_benchmark.py | 14 +- packages/flow/tests/test_chart_fonts.py | 4 +- .../tests/test_competitor_drift_instrument.py | 4 +- packages/flow/tests/test_desktop_benchmark.py | 4 +- packages/flow/tests/test_dom_arm.py | 8 +- packages/flow/tests/test_effectbench.py | 12 +- .../flow/tests/test_effectbench_runner.py | 16 +- .../flow/tests/test_effectbench_standalone.py | 6 +- .../flow/tests/test_effectbench_task_pack.py | 4 +- .../tests/test_frappe_lending_benchmark.py | 2 +- packages/flow/tests/test_hybrid_benchmark.py | 8 +- packages/flow/tests/test_lending_swer.py | 2 +- packages/flow/tests/test_openemr_benchmark.py | 12 +- .../tests/test_openemr_local_benchmark.py | 4 +- .../tests/test_openemr_retained_finals.py | 2 +- .../flow/tests/test_reliability_harness.py | 2 +- .../test_runtime_never_imports_benchmark.py | 58 ++-- .../test_silent_wrong_action_benchmark.py | 2 +- .../tests/test_ui_dismiss_live_openemr.py | 2 +- 99 files changed, 622 insertions(+), 274 deletions(-) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/__init__.py (63%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/agent_baseline.py (100%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/chart_fonts.py (100%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/desktop_benchmark.py (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/dom_arm.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/README.md (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/__init__.py (93%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/metrics.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/oracle.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/README.md (95%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/__init__.py (72%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/__main__.py (90%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/arms.py (95%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/baselines.py (97%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/compound.py (94%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/harness.py (92%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/reference_tasks.py (96%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/substrate.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/schema.py (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/frappe_lending.py (100%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/hybrid_benchmark.py (99%) rename packages/flow/{openadapt_flow => benchmark/harness}/instrument/__init__.py (91%) rename packages/flow/{openadapt_flow => benchmark/harness}/instrument/competitor_drift.py (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/openemr_benchmark.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/openemr_local.py (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/reliability.py (100%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/run_benchmark.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/silent_wrong_action.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/verify.py (100%) create mode 100644 packages/flow/docs/GENERALITY_AUDIT.md diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md index 99ec48b05e..978a0b337a 100644 --- a/packages/flow/CHANGELOG.md +++ b/packages/flow/CHANGELOG.md @@ -25,6 +25,8 @@ ### Fixes +- The wheel no longer ships the benchmark harnesses. `openadapt_flow.benchmark` and `openadapt_flow.instrument` moved to `packages/flow/benchmark/harness/` in the source tree (import `benchmark.harness`); `openadapt-flow benchmark` runs from a source checkout and says so from a wheel. The engine audit for application-specific code is in `docs/GENERALITY_AUDIT.md`. + - Self-repair can take a stop where the app's own dialog covers the next step's identity band: when the backend proves the covering window belongs to the window the last verified step was in, and the part of the band read still matches the record, the dialog is dismissed after the previous click and the step re-checks its identity in full. Another record behind the dialog stays an identity halt. The runtime lets a backend remember a verified record's window (`record_verified_window`); macOS answers `owned_window_over` from the window's proved sheet ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). - macOS: a run can now act in the pinned window's own AppKit sheet (an `NSAlert` attached to it), which is a separate window. `MacOSBackend` accepts it only when AX lists exactly one `AXSheet` on the window and exactly one window of the same process has its bounds inside it; the capture composites exactly that sheet over the window, and focus, pointer and click-target checks accept it only inside its bounds. Any other window is still refused ([#1711](https://github.com/OpenAdaptAI/OpenAdapt/issues/1711)). diff --git a/packages/flow/benchmark/agent_arm_verticals/README.md b/packages/flow/benchmark/agent_arm_verticals/README.md index fbfe9544dc..54526bd8fa 100644 --- a/packages/flow/benchmark/agent_arm_verticals/README.md +++ b/packages/flow/benchmark/agent_arm_verticals/README.md @@ -26,7 +26,7 @@ budget without completing the write in every trial. ## Method and public/private boundary The public, generic agent-baseline mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. It provides explicit paid-run +`benchmark/harness/agent_baseline.py`. It provides explicit paid-run opt-in, action and cost caps, usage accounting, and an agent self-report that is never trusted for scoring. The public benchmark schemas and aggregate failure taxonomy also remain available for independently authored tasks. diff --git a/packages/flow/benchmark/code_as_action/measure_openemr.py b/packages/flow/benchmark/code_as_action/measure_openemr.py index 74a34443a6..9e1b5e70f0 100644 --- a/packages/flow/benchmark/code_as_action/measure_openemr.py +++ b/packages/flow/benchmark/code_as_action/measure_openemr.py @@ -184,7 +184,7 @@ def spec_for(key: str) -> Any: - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec return SyntheticPatientSpec(**REFERRALS[key], **COMMON_FIELDS) @@ -390,7 +390,7 @@ def main() -> int: keys = [k for k in args.referrals.split(",") if k] from benchmark.openemr_local import qualify_hooks - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec fixture = qualify_hooks.fixture() demo = _demo() diff --git a/packages/flow/benchmark/dom/BENCHMARK.md b/packages/flow/benchmark/dom/BENCHMARK.md index 7111596c12..2b40374899 100644 --- a/packages/flow/benchmark/dom/BENCHMARK.md +++ b/packages/flow/benchmark/dom/BENCHMARK.md @@ -23,7 +23,7 @@ row's Open button — the literal reading — and **DOM (name-filtered)** clicks the Open button in the row named `Jane Sample` — the identity reading, hardcoding the demonstrated patient exactly as the compiled arm's recorded identity band does. Every selector choice is -documented in `openadapt_flow/benchmark/dom_arm.py`. +documented in `benchmark/harness/dom_arm.py`. ## Verdict @@ -208,7 +208,7 @@ Unmeasured variants, for completeness: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.dom_arm --out benchmark/dom --n-per-perturbation 2 +.venv/bin/python -m benchmark.harness.dom_arm --out benchmark/dom --n-per-perturbation 2 ``` (`--n-per-perturbation 2` matches the committed results.) No API key diff --git a/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md b/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md index dfa26c1e61..58868ee444 100644 --- a/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md +++ b/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md @@ -2,7 +2,7 @@ Date: 2026-07-21. This is the genuinely independent, end-to-end measurement of the silent-wrong-effect (SWER) result. Unlike the definitional -`openadapt_flow.benchmark.silent_wrong_action` (raw `requests.post`; effect +`benchmark.harness.silent_wrong_action` (raw `requests.post`; effect verifier and ground truth read the SAME in-process object; the effect contract restates the ground-truth definition -- so its `0/90` is circular by construction), every write below goes through the ACTUAL governed replay path diff --git a/packages/flow/benchmark/effect_e2e/record_service.py b/packages/flow/benchmark/effect_e2e/record_service.py index 2bb71f2e90..0f8f6e0fa3 100644 --- a/packages/flow/benchmark/effect_e2e/record_service.py +++ b/packages/flow/benchmark/effect_e2e/record_service.py @@ -3,7 +3,7 @@ This is the system of record the end-to-end silent-wrong-effect harness (:mod:`benchmark.effect_e2e.run`) writes to THROUGH the governed replay path. It exists to break the circularity of the older definitional benchmark -(``openadapt_flow.benchmark.silent_wrong_action``), whose effect verifier and +(``benchmark.harness.silent_wrong_action``), whose effect verifier and ground-truth judge read the SAME in-process Python object and whose write bypassed the :class:`~openadapt_flow.runtime.replayer.Replayer`. diff --git a/packages/flow/benchmark/effect_e2e/run.py b/packages/flow/benchmark/effect_e2e/run.py index d08601d691..7860d35b89 100644 --- a/packages/flow/benchmark/effect_e2e/run.py +++ b/packages/flow/benchmark/effect_e2e/run.py @@ -1,7 +1,7 @@ """End-to-end silent-wrong-effect (SWER) harness through the REAL replayer. This is the genuinely independent, end-to-end version of the silent-wrong-action -result. The older ``openadapt_flow.benchmark.silent_wrong_action`` is +result. The older ``benchmark.harness.silent_wrong_action`` is *definitional*: it issues raw ``requests.post`` (bypassing the :class:`~openadapt_flow.runtime.replayer.Replayer`), and its effect verifier and ground-truth judge both read the SAME in-process ``FaultDB`` object while the @@ -629,7 +629,7 @@ def render_markdown(results: dict[str, Any]) -> str: Date: {date}. This is the genuinely independent, end-to-end measurement of the silent-wrong-effect (SWER) result. Unlike the definitional -`openadapt_flow.benchmark.silent_wrong_action` (raw `requests.post`; effect +`benchmark.harness.silent_wrong_action` (raw `requests.post`; effect verifier and ground truth read the SAME in-process object; the effect contract restates the ground-truth definition -- so its `0/90` is circular by construction), every write below goes through the ACTUAL governed replay path @@ -774,7 +774,7 @@ def render_markdown(results: dict[str, Any]) -> str: def render_chart(results: dict[str, Any], out_png: Path) -> Path: """Bar chart: silent-wrong-effect rate per arm (large -> small -> zero).""" - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() m = results["metrics"]["per_arm"] @@ -827,7 +827,7 @@ def write_outputs(results: dict[str, Any], out_dir: Path) -> None: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") (out_dir / "EFFECT_E2E.md").write_text(render_markdown(results)) - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "effect_e2e.png") diff --git a/packages/flow/benchmark/effectbench/reference_fault_model.py b/packages/flow/benchmark/effectbench/reference_fault_model.py index 5ba4904208..0488c51dff 100644 --- a/packages/flow/benchmark/effectbench/reference_fault_model.py +++ b/packages/flow/benchmark/effectbench/reference_fault_model.py @@ -31,16 +31,16 @@ How it re-expresses the study, faithfully and without hardcoding verdicts: - Each of the study's nine scenarios (``benchmark.fault_model.faults.FAULTS``) - becomes a :class:`~openadapt_flow.benchmark.effectbench.schema.TaskSpec` plus + becomes a :class:`~benchmark.harness.effectbench.schema.TaskSpec` plus the KNOWN post-action system-of-record state that fault produces (the same DB states the study's own taxonomy tests assert). The states live in :data:`SCENARIOS` — the record shapes are MockMed ``FaultDB`` rows. - The TRUE effect is read by the substrate-agnostic - :class:`~openadapt_flow.benchmark.effectbench.oracle.RecordSnapshotOracle` + :class:`~benchmark.harness.effectbench.oracle.RecordSnapshotOracle` running the SAME shared ``judge_records`` decision logic the runtime uses — once for the at-most-once/collateral ``record_written`` contract and once for the ``field_equals`` note read-back — reduced by - :func:`~openadapt_flow.benchmark.effectbench.oracle.combine_true_states`. + :func:`~benchmark.harness.effectbench.oracle.combine_true_states`. - Two arms are scored against the identical independent oracle: ``screen_only`` (the weak banner oracle — reports success whenever the screen looks saved) and ``effect_verify`` (the agent halts unless the oracle CONFIRMS). Only the @@ -56,7 +56,7 @@ from typing import Optional from benchmark.fault_model import faults as F -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, DivergenceCategory, Effect, @@ -72,7 +72,7 @@ oracle_verdict_of, summarize, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel # The consequential-save contract the study verifies: exactly one Triage # encounter for the target patient (at-most-once + no collateral loss) whose diff --git a/packages/flow/benchmark/effectbench/task_pack/README.md b/packages/flow/benchmark/effectbench/task_pack/README.md index 88fa4d6c25..8846166f4c 100644 --- a/packages/flow/benchmark/effectbench/task_pack/README.md +++ b/packages/flow/benchmark/effectbench/task_pack/README.md @@ -68,7 +68,7 @@ python -m benchmark.effectbench.task_pack.pack # Live-run the MockMed anchor through the oracle (prints per-task classification) python -c "from benchmark.effectbench.task_pack.driver import run_mockmed_pack; \ -from openadapt_flow.benchmark.effectbench import summarize; \ +from benchmark.harness.effectbench import summarize; \ eps=run_mockmed_pack(trials=3); \ print('screen SWER', summarize(eps, arm='screen_only').swer.rate); \ print('effect SWER', summarize(eps, arm='effect_verify').swer.rate)" diff --git a/packages/flow/benchmark/effectbench/task_pack/__init__.py b/packages/flow/benchmark/effectbench/task_pack/__init__.py index e3f3537b7b..c823a82252 100644 --- a/packages/flow/benchmark/effectbench/task_pack/__init__.py +++ b/packages/flow/benchmark/effectbench/task_pack/__init__.py @@ -6,9 +6,9 @@ ``benchmark.environments`` registry (PR #173) indexes, authored against the frozen EffectBench schema/oracle/metrics contract (PR #178). -Each task is a :class:`~openadapt_flow.benchmark.effectbench.TaskSpec` (goal = +Each task is a :class:`~benchmark.harness.effectbench.TaskSpec` (goal = INTENT only, never a step list) plus an -:class:`~openadapt_flow.benchmark.effectbench.OracleSpec` bound to the +:class:`~benchmark.harness.effectbench.OracleSpec` bound to the environment's system-of-record READ recipe, DESIGNED so a green screen (task-success) diverges from the correct business effect. The oracle reads pre/post SoR state independently of the screen; the non-gameability attestations diff --git a/packages/flow/benchmark/effectbench/task_pack/_authoring.py b/packages/flow/benchmark/effectbench/task_pack/_authoring.py index 00e02bc078..aa0ba26354 100644 --- a/packages/flow/benchmark/effectbench/task_pack/_authoring.py +++ b/packages/flow/benchmark/effectbench/task_pack/_authoring.py @@ -3,10 +3,10 @@ These build the two things every consequential-write task needs: - the DECLARED expected effect (a typed - :class:`~openadapt_flow.benchmark.effectbench.Effect`), parameterized by run - params via :class:`~openadapt_flow.benchmark.effectbench.ValueExpr` so the + :class:`~benchmark.harness.effectbench.Effect`), parameterized by run + params via :class:`~benchmark.harness.effectbench.ValueExpr` so the record CHECKED is the record this trial WROTE (trial-unique payload); and -- the :class:`~openadapt_flow.benchmark.effectbench.OracleSpec` that declares +- the :class:`~benchmark.harness.effectbench.OracleSpec` that declares the independent read channel and the non-gameability attestations the adversarial audit signs off on. @@ -21,13 +21,13 @@ from typing import Any, Optional -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, EffectKind, OracleSpec, ValueExpr, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel # Canonical run-param names a trial binds a trial-unique value to. The driver # derives each from the trial seed so the oracle checks THIS run's exact write. diff --git a/packages/flow/benchmark/effectbench/task_pack/audit.py b/packages/flow/benchmark/effectbench/task_pack/audit.py index c44706010e..fcba2b8c15 100644 --- a/packages/flow/benchmark/effectbench/task_pack/audit.py +++ b/packages/flow/benchmark/effectbench/task_pack/audit.py @@ -41,7 +41,7 @@ _note_effect, _record_effect, ) -from openadapt_flow.benchmark.effectbench import RestRecordVerifier, Verdict +from benchmark.harness.effectbench import RestRecordVerifier, Verdict _HTTP_TIMEOUT_S = 5.0 TRIAL_NOTE = "triage note trial-unique-AUDIT" diff --git a/packages/flow/benchmark/effectbench/task_pack/driver.py b/packages/flow/benchmark/effectbench/task_pack/driver.py index 4145e46ae7..035a025097 100644 --- a/packages/flow/benchmark/effectbench/task_pack/driver.py +++ b/packages/flow/benchmark/effectbench/task_pack/driver.py @@ -8,9 +8,9 @@ mocked verdicts. It depends ONLY on the public EffectBench surface -(:func:`~openadapt_flow.benchmark.effectbench.score_episode`, -:class:`~openadapt_flow.benchmark.effectbench.RestRecordVerifier`, -:class:`~openadapt_flow.benchmark.effectbench.AgentReport`) and the MockMed +(:func:`~benchmark.harness.effectbench.score_episode`, +:class:`~benchmark.harness.effectbench.RestRecordVerifier`, +:class:`~benchmark.harness.effectbench.AgentReport`) and the MockMed fault server. It does not reimplement any real baseline "arm"; the two arms here are the two SCORING conditions the benchmark contrasts: @@ -46,7 +46,7 @@ MockMedDrive, MockMedTask, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, EpisodeRecord, RestRecordVerifier, diff --git a/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py b/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py index e2c22a2ef5..9219f750ea 100644 --- a/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py +++ b/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py @@ -17,7 +17,7 @@ ``oracle.config["query"]`` + ``["query_params"]`` is what the multi-baseline runner materializes into a -:class:`~openadapt_flow.benchmark.effectbench.SqlRecordVerifier` (read-only role, +:class:`~benchmark.harness.effectbench.SqlRecordVerifier` (read-only role, DB-API-bound params). The loan-application docname has a space (`` `tabLoan Application` ``), so the table is back-tick quoted in the SELECT. """ @@ -33,15 +33,15 @@ record_written_effect, ) from benchmark.effectbench.task_pack.openemr_tasks import ContainerTask -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( DivergenceCategory as DC, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, Substrate, TaskSpec, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel ENVIRONMENT = "frappe_lending" diff --git a/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py b/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py index 898f90db42..9f9b679bc3 100644 --- a/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py +++ b/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py @@ -8,7 +8,7 @@ path is exercised in CI (no mocked verdicts). Each entry is a :class:`MockMedTask`: an agent-agnostic -:class:`~openadapt_flow.benchmark.effectbench.TaskSpec` (goal = intent only) +:class:`~benchmark.harness.effectbench.TaskSpec` (goal = intent only) plus a :class:`MockMedDrive` — the environment recipe the live driver replays to MANIFEST the divergence (which ``?fault=`` mode to inject, the write the agent performs, decoy/concurrent rows to seed, and the documented banner @@ -34,9 +34,9 @@ param, record_written_effect, ) -from openadapt_flow.benchmark.effectbench import DivergenceCategory as DC -from openadapt_flow.benchmark.effectbench import Effect, Substrate, TaskSpec -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench import DivergenceCategory as DC +from benchmark.harness.effectbench import Effect, Substrate, TaskSpec +from benchmark.harness.effectbench.schema import OracleChannel TARGET_PATIENT = "p1" TARGET_TYPE = "Triage" diff --git a/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py b/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py index 2f6d61974d..7581aa2520 100644 --- a/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py +++ b/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py @@ -18,7 +18,7 @@ The SQL each oracle runs (``oracle.config["query"]`` + ``["query_params"]``) is what the multi-baseline runner materializes into a -:class:`~openadapt_flow.benchmark.effectbench.SqlRecordVerifier` (read-only role; +:class:`~benchmark.harness.effectbench.SqlRecordVerifier` (read-only role; values bound through DB-API params, never interpolated). Column names become record keys, so the same typed ``Effect`` the MockMed anchor uses judges these rows unchanged. @@ -36,15 +36,15 @@ param, record_written_effect, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( DivergenceCategory as DC, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, Substrate, TaskSpec, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel ENVIRONMENT = "openemr_local" diff --git a/packages/flow/benchmark/effectbench/task_pack/pack.py b/packages/flow/benchmark/effectbench/task_pack/pack.py index 2be0450827..fa15417f72 100644 --- a/packages/flow/benchmark/effectbench/task_pack/pack.py +++ b/packages/flow/benchmark/effectbench/task_pack/pack.py @@ -4,7 +4,7 @@ the container-gated OpenEMR and Frappe families). It: - assembles the unified list of :class:`PackEntry` (one per authored task), - each carrying its :class:`~openadapt_flow.benchmark.effectbench.TaskSpec`, the + each carrying its :class:`~benchmark.harness.effectbench.TaskSpec`, the pinned environment it targets, and whether it runs LIVE here or needs a container; - validates every task against the ``benchmark.environments`` registry (the @@ -29,8 +29,8 @@ from benchmark.effectbench.task_pack.frappe_tasks import FRAPPE_TASKS from benchmark.effectbench.task_pack.mockmed_tasks import MOCKMED_TASKS from benchmark.effectbench.task_pack.openemr_tasks import OPENEMR_TASKS -from openadapt_flow.benchmark.effectbench import TaskSpec -from openadapt_flow.benchmark.effectbench.schema import DivergenceCategory, Substrate +from benchmark.harness.effectbench import TaskSpec +from benchmark.harness.effectbench.schema import DivergenceCategory, Substrate MANIFEST_SCHEMA_VERSION = 1 diff --git a/packages/flow/benchmark/frappe_lending/README.md b/packages/flow/benchmark/frappe_lending/README.md index 1504ddf329..89ae27fa5a 100644 --- a/packages/flow/benchmark/frappe_lending/README.md +++ b/packages/flow/benchmark/frappe_lending/README.md @@ -30,7 +30,7 @@ trials per cell remains the publication bar. Raw per-run rows, environment fingerprints, application-specific adapter/oracle wiring, and the detailed cost ledger are retained in the private `OpenAdaptAI/openadapt-corpus` repository. The public generic cost-capped agent mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. See the +`benchmark/harness/agent_baseline.py`. See the [aggregate report](../agent_arm_verticals/README.md). The accepted run used baseline SHA-256 diff --git a/packages/flow/openadapt_flow/benchmark/__init__.py b/packages/flow/benchmark/harness/__init__.py similarity index 63% rename from packages/flow/openadapt_flow/benchmark/__init__.py rename to packages/flow/benchmark/harness/__init__.py index b157c4fda1..2eba51f30b 100644 --- a/packages/flow/openadapt_flow/benchmark/__init__.py +++ b/packages/flow/benchmark/harness/__init__.py @@ -1,4 +1,8 @@ -"""Benchmark: compiled replay vs. a frontier computer-use agent. +"""Benchmark harnesses: compiled replay vs. a frontier computer-use agent. + +This package lives in the source checkout, outside ``openadapt_flow``, so the +wheel does not ship it. Import it as ``benchmark.harness`` from +``packages/flow``. Submodules (import them directly; nothing heavy is imported here): diff --git a/packages/flow/openadapt_flow/benchmark/agent_baseline.py b/packages/flow/benchmark/harness/agent_baseline.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/agent_baseline.py rename to packages/flow/benchmark/harness/agent_baseline.py diff --git a/packages/flow/openadapt_flow/benchmark/chart_fonts.py b/packages/flow/benchmark/harness/chart_fonts.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/chart_fonts.py rename to packages/flow/benchmark/harness/chart_fonts.py diff --git a/packages/flow/openadapt_flow/benchmark/desktop_benchmark.py b/packages/flow/benchmark/harness/desktop_benchmark.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/desktop_benchmark.py rename to packages/flow/benchmark/harness/desktop_benchmark.py index 270605e54b..c581ca306d 100644 --- a/packages/flow/openadapt_flow/benchmark/desktop_benchmark.py +++ b/packages/flow/benchmark/harness/desktop_benchmark.py @@ -745,7 +745,7 @@ def _identity_transfer_verdict(results: dict) -> str: def render_chart(results: dict, path: Path) -> None: """Stacked outcome bars per arm (success / wrong-action / halt).""" - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() arms = list(results["arms"]) @@ -776,7 +776,7 @@ def write_outputs(results: dict, out_dir: str | Path) -> None: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") (out_dir / "BENCHMARK.md").write_text(render_markdown(results)) - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "outcomes.png") diff --git a/packages/flow/openadapt_flow/benchmark/dom_arm.py b/packages/flow/benchmark/harness/dom_arm.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/dom_arm.py rename to packages/flow/benchmark/harness/dom_arm.py index a6c0e598dd..27a1e277ef 100644 --- a/packages/flow/openadapt_flow/benchmark/dom_arm.py +++ b/packages/flow/benchmark/harness/dom_arm.py @@ -10,7 +10,7 @@ Three arms, one frozen schedule, one arm-independent success check: - **compiled**: the existing vision-anchored compiled replay - (:func:`openadapt_flow.benchmark.run_benchmark._compiled_run`), healing + (:func:`benchmark.harness.run_benchmark._compiled_run`), healing enabled as always. - **dom** (positional) and **dom_named** (name-filtered): :func:`dom_script` — a Playwright script written the way a competent @@ -29,7 +29,7 @@ drift modes (:data:`PERTURBATIONS`: lookalike, missing, grow, sort, theme, rename, move, typelabel). Success is judged for BOTH arms by the hybrid benchmark's own arm-independent check -(:func:`~openadapt_flow.benchmark.hybrid_benchmark.verify_hybrid_final`): +(:func:`~benchmark.harness.hybrid_benchmark.verify_hybrid_final`): OCR of the final screenshot must show the saved-encounter evidence AND the right patient, with wrong-type writes flagged — so this head-to-head is judged by the identical bar. Wrong-action events are recorded per arm: @@ -51,7 +51,7 @@ from pathlib import Path from typing import Any, Callable, Optional -from openadapt_flow.benchmark.hybrid_benchmark import ( +from benchmark.harness.hybrid_benchmark import ( _NOTE_POOL, DRIFT_TYPES, SCHEDULE, @@ -465,7 +465,7 @@ def dom_arm_aggregate(rows: list[dict[str, Any]]) -> dict[str, Any]: Aggregate dict: base stats plus clean/drift splits, outcome counts, wrong-action count, and (DOM arm) maintenance events. """ - from openadapt_flow.benchmark.run_benchmark import _arm_aggregate + from benchmark.harness.run_benchmark import _arm_aggregate agg = _arm_aggregate(rows) agg["clean"] = _split_rate([r for r in rows if r.get("condition") == "clean"]) @@ -621,7 +621,7 @@ def render_dom_chart(results: dict[str, Any], out_png: Path) -> Path: Returns: The written PNG path. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() from matplotlib.patches import Rectangle @@ -1177,7 +1177,7 @@ def stat_row(label: str, fmt: Callable[[dict[str, Any]], str]) -> str: clicks the Open button in the row named `{TARGET_PATIENT}` — the identity reading, hardcoding the demonstrated patient exactly as the compiled arm's recorded identity band does. Every selector choice is -documented in `openadapt_flow/benchmark/dom_arm.py`. +documented in `benchmark/harness/dom_arm.py`. ## Verdict @@ -1333,7 +1333,7 @@ def stat_row(label: str, fmt: Callable[[dict[str, Any]], str]) -> str: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.dom_arm --out benchmark/dom --n-per-perturbation 2 +.venv/bin/python -m benchmark.harness.dom_arm --out benchmark/dom --n-per-perturbation 2 ``` (`--n-per-perturbation 2` matches the committed results.) No API key @@ -1350,7 +1350,7 @@ def write_dom_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_dom_chart, results, out_dir / "outcome_matrix.png") (out_dir / "BENCHMARK.md").write_text(render_dom_markdown(results)) @@ -1388,7 +1388,7 @@ def run_dom_benchmark( Returns: The results dict (also written to ``results.json``). """ - from openadapt_flow.benchmark.run_benchmark import _compiled_run + from benchmark.harness.run_benchmark import _compiled_run from openadapt_flow.compiler import compile_recording from openadapt_flow.demo_driver import record_triage_demo from openadapt_flow.mockmed.server import serve @@ -1499,7 +1499,7 @@ def one_run( def main(argv: Optional[list[str]] = None) -> int: - """CLI entry point: ``python -m openadapt_flow.benchmark.dom_arm``.""" + """CLI entry point: ``python -m benchmark.harness.dom_arm``.""" parser = argparse.ArgumentParser( description=( "DOM-selector benchmark arm vs. compiled vision replay on " diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/README.md b/packages/flow/benchmark/harness/effectbench/README.md similarity index 99% rename from packages/flow/openadapt_flow/benchmark/effectbench/README.md rename to packages/flow/benchmark/harness/effectbench/README.md index c42718959f..08b926e621 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/README.md +++ b/packages/flow/benchmark/harness/effectbench/README.md @@ -5,7 +5,7 @@ > system-of-record oracle* disagrees. Reported jointly with **over-halt rate**, > **task success**, and the **success–effect gap**. -This package (`openadapt_flow.benchmark.effectbench`) is the **stable contract** +This package (`benchmark.harness.effectbench`) is the **stable contract** that two downstream efforts build on: 1. the **multi-baseline runner adapter** — drives each agent arm through a common diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/__init__.py b/packages/flow/benchmark/harness/effectbench/__init__.py similarity index 93% rename from packages/flow/openadapt_flow/benchmark/effectbench/__init__.py rename to packages/flow/benchmark/harness/effectbench/__init__.py index 7291bdbf04..62562d9441 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/__init__.py +++ b/packages/flow/benchmark/harness/effectbench/__init__.py @@ -31,10 +31,10 @@ bootstrap confidence intervals, all decomposed by category x substrate. The design doc is ``.private/benchmark_design_2026_07_20.md``; the operator -contract is ``openadapt_flow/benchmark/effectbench/README.md``. +contract is ``benchmark/harness/effectbench/README.md``. """ -from openadapt_flow.benchmark.effectbench.metrics import ( # noqa: F401 +from benchmark.harness.effectbench.metrics import ( # noqa: F401 BenchmarkSummary, CellSummary, Interval, @@ -44,7 +44,7 @@ summarize, wilson_interval, ) -from openadapt_flow.benchmark.effectbench.oracle import ( # noqa: F401 +from benchmark.harness.effectbench.oracle import ( # noqa: F401 RecordSnapshotOracle, TrueEffectState, classify_outcome, @@ -53,7 +53,7 @@ oracle_verdict_of, score_episode, ) -from openadapt_flow.benchmark.effectbench.schema import ( # noqa: F401 +from benchmark.harness.effectbench.schema import ( # noqa: F401 AgentReport, DivergenceCategory, EpisodeRecord, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/metrics.py b/packages/flow/benchmark/harness/effectbench/metrics.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/effectbench/metrics.py rename to packages/flow/benchmark/harness/effectbench/metrics.py index b7ad870e60..356119fde2 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/metrics.py +++ b/packages/flow/benchmark/harness/effectbench/metrics.py @@ -1,7 +1,7 @@ """EffectBench metrics — SWER and its co-metrics, with confidence intervals. Every headline is a pure aggregation over a list of -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord` rows, so a +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord` rows, so a run's numbers can be recomputed from the published raw rows by anyone. No single mean is reported alone: :func:`summarize` always returns the per (category x substrate) decomposition alongside the overall rate (design doc @@ -39,7 +39,7 @@ from pydantic import BaseModel, ConfigDict, Field -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( EpisodeRecord, OutcomeLabel, SwerVariant, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/oracle.py b/packages/flow/benchmark/harness/effectbench/oracle.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/effectbench/oracle.py rename to packages/flow/benchmark/harness/effectbench/oracle.py index f7f5ca031a..e490479d8b 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/oracle.py +++ b/packages/flow/benchmark/harness/effectbench/oracle.py @@ -17,7 +17,7 @@ 2. **The classifier** — :func:`classify_outcome`, which crosses the oracle's reading of the true effect with the agent's (untrusted) self-report to - assign exactly one :class:`~openadapt_flow.benchmark.effectbench.schema.OutcomeLabel`. + assign exactly one :class:`~benchmark.harness.effectbench.schema.OutcomeLabel`. This is ``fault_model.classify`` generalized: it adds the OVER_HALT vs SAFE_HALT split (was the correct action available?) and makes ``SILENT_WRONG_EFFECT`` the single home of ``reported_success AND @@ -25,7 +25,7 @@ :func:`score_episode` ties the two together for a runner: capture pre-state, run the arm's action, verify, classify, and emit an -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord`. +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord`. """ from __future__ import annotations @@ -34,7 +34,7 @@ from enum import Enum from typing import Any, Callable, Mapping, Optional -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( AgentReport, DivergenceCategory, EpisodeRecord, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/README.md b/packages/flow/benchmark/harness/effectbench/runner/README.md similarity index 95% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/README.md rename to packages/flow/benchmark/harness/effectbench/runner/README.md index a29834a364..a00d55f331 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/README.md +++ b/packages/flow/benchmark/harness/effectbench/runner/README.md @@ -1,7 +1,7 @@ # EffectBench runner adapter — one interface, every baseline The **multi-baseline runner adapter** (downstream effort #2 on the -`openadapt_flow.benchmark.effectbench` contract). It drives every agent *arm* +`benchmark.harness.effectbench` contract). It drives every agent *arm* through one common interface against the **identical task** and **identical independent oracle**, records one `EpisodeRecord` per (task × arm × trial), and summarizes per arm with `summarize`. A difference between arms is then the @@ -10,8 +10,8 @@ AGENT, never the harness. Run the CI-fast dry-run (real in-process HTTP, no Docker, no paid API, no spend): ```bash -python -m openadapt_flow.benchmark.effectbench.runner # all live arms -python -m openadapt_flow.benchmark.effectbench.runner --list-arms +python -m benchmark.harness.effectbench.runner # all live arms +python -m benchmark.harness.effectbench.runner --list-arms ``` ## The arm interface diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__init__.py b/packages/flow/benchmark/harness/effectbench/runner/__init__.py similarity index 72% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/__init__.py rename to packages/flow/benchmark/harness/effectbench/runner/__init__.py index 759ebae7d1..27fe3e39ff 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__init__.py +++ b/packages/flow/benchmark/harness/effectbench/runner/__init__.py @@ -2,20 +2,20 @@ Drives every agent *arm* through one common interface against the IDENTICAL task and INDEPENDENT oracle, records one -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord` per +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord` per (task × arm × trial), and summarizes per arm with -:func:`~openadapt_flow.benchmark.effectbench.metrics.summarize`. Built on the -stable contract in :mod:`openadapt_flow.benchmark.effectbench` (schema / oracle / +:func:`~benchmark.harness.effectbench.metrics.summarize`. Built on the +stable contract in :mod:`benchmark.harness.effectbench` (schema / oracle / metrics) — this package adds only the runner, never re-implements the taxonomy. Layers: - :mod:`.arms` — the ``AgentArm`` interface + the concrete in-repo arms: - :class:`~openadapt_flow.benchmark.effectbench.runner.arms.CompilerArm` + :class:`~benchmark.harness.effectbench.runner.arms.CompilerArm` (record→compile→replay, effect-gated), - :class:`~openadapt_flow.benchmark.effectbench.runner.arms.ScreenOnlyArm` + :class:`~benchmark.harness.effectbench.runner.arms.ScreenOnlyArm` (the silent-wrong-effect ablation), and a substrate-free - :class:`~openadapt_flow.benchmark.effectbench.runner.arms.MockArm` for CI. + :class:`~benchmark.harness.effectbench.runner.arms.MockArm` for CI. - :mod:`.substrate` — the arm-facing action+perception channel (the reference MockMed session), isolated from the oracle read path. - :mod:`.compound` — the compound consequential-save verifier (record + note). @@ -25,19 +25,19 @@ - :mod:`.harness` — ``run_episode`` / ``run_matrix``: wire each arm through ``score_episode`` with the task's oracle, fairly and identically. -Run the dry-run: ``python -m openadapt_flow.benchmark.effectbench.runner``. +Run the dry-run: ``python -m benchmark.harness.effectbench.runner``. """ from __future__ import annotations -from openadapt_flow.benchmark.effectbench.runner.arms import ( +from benchmark.harness.effectbench.runner.arms import ( AgentArm, ArmResult, CompilerArm, MockArm, ScreenOnlyArm, ) -from openadapt_flow.benchmark.effectbench.runner.baselines import ( +from benchmark.harness.effectbench.runner.baselines import ( SCAFFOLDED_ARMS, BaselineRequirements, ClaudeComputerUseArm, @@ -46,21 +46,21 @@ SkyvernArm, UITarsArm, ) -from openadapt_flow.benchmark.effectbench.runner.compound import ( +from benchmark.harness.effectbench.runner.compound import ( CompoundEffectVerifier, ) -from openadapt_flow.benchmark.effectbench.runner.harness import ( +from benchmark.harness.effectbench.runner.harness import ( EnvFactory, EpisodeEnv, run_episode, run_matrix, ) -from openadapt_flow.benchmark.effectbench.runner.reference_tasks import ( +from benchmark.harness.effectbench.runner.reference_tasks import ( MockMedEnvProvider, mockmed_env_factory, reference_tasks, ) -from openadapt_flow.benchmark.effectbench.runner.substrate import ( +from benchmark.harness.effectbench.runner.substrate import ( MockMedFault, MockMedSession, ScreenObservation, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__main__.py b/packages/flow/benchmark/harness/effectbench/runner/__main__.py similarity index 90% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/__main__.py rename to packages/flow/benchmark/harness/effectbench/runner/__main__.py index 4ba0d142eb..bc6c79734f 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__main__.py +++ b/packages/flow/benchmark/harness/effectbench/runner/__main__.py @@ -1,6 +1,6 @@ """EffectBench runner CLI — the CI-fast dry-run over the MockMed reference pack. -``python -m openadapt_flow.benchmark.effectbench.runner`` drives the nine MockMed +``python -m benchmark.harness.effectbench.runner`` drives the nine MockMed reference tasks through the in-repo LIVE arms (compiler, screen-only ablation, mock) over a real in-process HTTP fault server — no Docker, no paid API, no spend — and prints the per-arm summary. It proves the pipeline end-to-end: the @@ -25,16 +25,16 @@ import sys from typing import Optional, Sequence -from openadapt_flow.benchmark.effectbench.metrics import BenchmarkSummary, summarize -from openadapt_flow.benchmark.effectbench.runner import LIVE_ARMS -from openadapt_flow.benchmark.effectbench.runner.arms import AgentArm -from openadapt_flow.benchmark.effectbench.runner.baselines import SCAFFOLDED_ARMS -from openadapt_flow.benchmark.effectbench.runner.harness import run_matrix -from openadapt_flow.benchmark.effectbench.runner.reference_tasks import ( +from benchmark.harness.effectbench.metrics import BenchmarkSummary, summarize +from benchmark.harness.effectbench.runner import LIVE_ARMS +from benchmark.harness.effectbench.runner.arms import AgentArm +from benchmark.harness.effectbench.runner.baselines import SCAFFOLDED_ARMS +from benchmark.harness.effectbench.runner.harness import run_matrix +from benchmark.harness.effectbench.runner.reference_tasks import ( MockMedEnvProvider, reference_tasks, ) -from openadapt_flow.benchmark.effectbench.schema import EpisodeRecord +from benchmark.harness.effectbench.schema import EpisodeRecord def _select_arms(names: Optional[str]) -> list[AgentArm]: @@ -128,7 +128,7 @@ def run_dry_run( def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser( - prog="python -m openadapt_flow.benchmark.effectbench.runner", + prog="python -m benchmark.harness.effectbench.runner", description="EffectBench multi-baseline runner — MockMed dry-run.", ) parser.add_argument("--trials", type=int, default=3) diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/arms.py b/packages/flow/benchmark/harness/effectbench/runner/arms.py similarity index 95% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/arms.py rename to packages/flow/benchmark/harness/effectbench/runner/arms.py index 063389bcef..a63a5985c3 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/arms.py +++ b/packages/flow/benchmark/harness/effectbench/runner/arms.py @@ -18,14 +18,14 @@ def run(self, task: TaskSpec, session: SubstrateSession, An arm is handed ONLY the task's natural-language ``goal`` (intent, never a step list — the fairness requirement) via ``task``, a -:class:`~openadapt_flow.benchmark.effectbench.runner.substrate.SubstrateSession` +:class:`~benchmark.harness.effectbench.runner.substrate.SubstrateSession` to drive, and the run's ``params`` (the trial-unique payload it must write). It returns an :class:`ArmResult` carrying the untrusted -:class:`~openadapt_flow.benchmark.effectbench.schema.AgentReport` plus any -recorded :class:`~openadapt_flow.benchmark.effectbench.schema.ModelCall` rows so +:class:`~benchmark.harness.effectbench.schema.AgentReport` plus any +recorded :class:`~benchmark.harness.effectbench.schema.ModelCall` rows so cost is auditable. The harness (:mod:`.harness`) adapts ``run`` into the ``run_action: Callable[[], AgentReport]`` that -:func:`~openadapt_flow.benchmark.effectbench.oracle.score_episode` calls, and +:func:`~benchmark.harness.effectbench.oracle.score_episode` calls, and supplies the independent oracle — which the arm can NEVER reach. Three concrete in-repo arms live here: @@ -50,11 +50,11 @@ def run(self, task: TaskSpec, session: SubstrateSession, from dataclasses import dataclass, field from typing import Any, Callable, Mapping, Optional, Protocol, runtime_checkable -from openadapt_flow.benchmark.effectbench.runner.substrate import ( +from benchmark.harness.effectbench.runner.substrate import ( ScreenObservation, SubstrateSession, ) -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( AgentReport, ModelCall, TaskSpec, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/baselines.py b/packages/flow/benchmark/harness/effectbench/runner/baselines.py similarity index 97% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/baselines.py rename to packages/flow/benchmark/harness/effectbench/runner/baselines.py index 3674a508ce..5a43fbf73d 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/baselines.py +++ b/packages/flow/benchmark/harness/effectbench/runner/baselines.py @@ -21,7 +21,7 @@ side channel to the system of record; the independent oracle, not the agent, judges the effect. 3. **Record every model call.** Append one - :class:`~openadapt_flow.benchmark.effectbench.schema.ModelCall` per API + :class:`~benchmark.harness.effectbench.schema.ModelCall` per API request with token counts and the pinned list price, so ``cost_usd`` is auditable ``sum(model_calls)``. An RPA tool with a flat run fee records a single ``ModelCall`` carrying that fee. @@ -38,9 +38,9 @@ from dataclasses import dataclass from typing import Mapping -from openadapt_flow.benchmark.effectbench.runner.arms import ArmResult -from openadapt_flow.benchmark.effectbench.runner.substrate import SubstrateSession -from openadapt_flow.benchmark.effectbench.schema import TaskSpec +from benchmark.harness.effectbench.runner.arms import ArmResult +from benchmark.harness.effectbench.runner.substrate import SubstrateSession +from benchmark.harness.effectbench.schema import TaskSpec class ScaffoldNotWired(NotImplementedError): diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/compound.py b/packages/flow/benchmark/harness/effectbench/runner/compound.py similarity index 94% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/compound.py rename to packages/flow/benchmark/harness/effectbench/runner/compound.py index 1c6236449a..3c15f795f5 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/compound.py +++ b/packages/flow/benchmark/harness/effectbench/runner/compound.py @@ -9,16 +9,16 @@ * ``field_equals`` — that encounter's note reads back the run's exact value; catches a *partial save* that persists the row but drops a field. -:class:`~openadapt_flow.benchmark.effectbench.oracle.score_episode` verifies a +:class:`~benchmark.harness.effectbench.oracle.score_episode` verifies a SINGLE :class:`Effect` through the oracle. This module supplies a thin :class:`~openadapt_flow.runtime.effects.effect.EffectVerifier` that wraps ANY concrete base verifier (the in-process -:class:`~openadapt_flow.benchmark.effectbench.oracle.RecordSnapshotOracle`, a +:class:`~benchmark.harness.effectbench.oracle.RecordSnapshotOracle`, a :class:`~openadapt_flow.runtime.effects.rest.RestRecordVerifier`, …) and, when ``score_episode`` calls :meth:`verify` with the primary ``record_written`` effect, ALSO checks the authored extra sub-effects against the SAME pre-state snapshot, reducing the per-effect true-effect states with -:func:`~openadapt_flow.benchmark.effectbench.oracle.combine_true_states`. +:func:`~benchmark.harness.effectbench.oracle.combine_true_states`. It returns the single **deciding** :class:`EffectVerdict` whose ``effect_state`` equals the combined state (the fault-model precedence: the @@ -33,7 +33,7 @@ from typing import Any, Sequence -from openadapt_flow.benchmark.effectbench.oracle import ( +from benchmark.harness.effectbench.oracle import ( TrueEffectState, combine_true_states, effect_state, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/harness.py b/packages/flow/benchmark/harness/effectbench/runner/harness.py similarity index 92% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/harness.py rename to packages/flow/benchmark/harness/effectbench/runner/harness.py index f21991fed0..f1bafeefbf 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/harness.py +++ b/packages/flow/benchmark/harness/effectbench/runner/harness.py @@ -1,11 +1,11 @@ """The runner harness — wire each arm through ``score_episode`` identically. This is the multi-baseline runner's core: given a set of -:class:`~openadapt_flow.benchmark.effectbench.schema.TaskSpec` and a set of arms, +:class:`~benchmark.harness.effectbench.schema.TaskSpec` and a set of arms, it produces one -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord` per +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord` per (task × arm × trial), each scored by the task's INDEPENDENT oracle through -:func:`~openadapt_flow.benchmark.effectbench.oracle.score_episode`. Every arm +:func:`~benchmark.harness.effectbench.oracle.score_episode`. Every arm runs the SAME task and is judged by the SAME oracle — the harness guarantees the comparison is fair by construction: @@ -30,10 +30,10 @@ from dataclasses import dataclass, field from typing import Any, Callable, List, Mapping, Optional, Sequence -from openadapt_flow.benchmark.effectbench.oracle import score_episode -from openadapt_flow.benchmark.effectbench.runner.arms import AgentArm -from openadapt_flow.benchmark.effectbench.runner.substrate import SubstrateSession -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.oracle import score_episode +from benchmark.harness.effectbench.runner.arms import AgentArm +from benchmark.harness.effectbench.runner.substrate import SubstrateSession +from benchmark.harness.effectbench.schema import ( AgentReport, EpisodeRecord, ModelCall, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/reference_tasks.py b/packages/flow/benchmark/harness/effectbench/runner/reference_tasks.py similarity index 96% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/reference_tasks.py rename to packages/flow/benchmark/harness/effectbench/runner/reference_tasks.py index 6afe49e2be..9c4c8de44e 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/reference_tasks.py +++ b/packages/flow/benchmark/harness/effectbench/runner/reference_tasks.py @@ -3,7 +3,7 @@ These are the reference tasks the multi-baseline runner exercises end-to-end without Docker or spend: the bundled MockMed transactional-fault surface (``openadapt_flow.mockmed.fault_server``) turned into -:class:`~openadapt_flow.benchmark.effectbench.schema.TaskSpec` s across the +:class:`~benchmark.harness.effectbench.schema.TaskSpec` s across the divergence categories, plus a :func:`mockmed_env_factory` that provisions one isolated fault server per episode with an INDEPENDENT in-process oracle. @@ -24,17 +24,17 @@ from dataclasses import dataclass from typing import Mapping, Optional -from openadapt_flow.benchmark.effectbench.oracle import RecordSnapshotOracle -from openadapt_flow.benchmark.effectbench.runner.compound import ( +from benchmark.harness.effectbench.oracle import RecordSnapshotOracle +from benchmark.harness.effectbench.runner.compound import ( CompoundEffectVerifier, ) -from openadapt_flow.benchmark.effectbench.runner.harness import EpisodeEnv -from openadapt_flow.benchmark.effectbench.runner.substrate import ( +from benchmark.harness.effectbench.runner.harness import EpisodeEnv +from benchmark.harness.effectbench.runner.substrate import ( MockMedFault, MockMedSession, mockmed_session, ) -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( DivergenceCategory, OracleChannel, OracleSpec, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/substrate.py b/packages/flow/benchmark/harness/effectbench/runner/substrate.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/substrate.py rename to packages/flow/benchmark/harness/effectbench/runner/substrate.py index 3d4e525528..1322d730f6 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/substrate.py +++ b/packages/flow/benchmark/harness/effectbench/runner/substrate.py @@ -34,10 +34,10 @@ from dataclasses import dataclass, field from typing import Any, Mapping, Optional, Protocol, Sequence, runtime_checkable -from openadapt_flow.benchmark.effectbench.runner.compound import ( +from benchmark.harness.effectbench.runner.compound import ( CompoundEffectVerifier, ) -from openadapt_flow.benchmark.effectbench.schema import Substrate +from benchmark.harness.effectbench.schema import Substrate from openadapt_flow.runtime.effects.effect import Effect, EffectVerifier diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/schema.py b/packages/flow/benchmark/harness/effectbench/schema.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/effectbench/schema.py rename to packages/flow/benchmark/harness/effectbench/schema.py index 4e529ae716..d92fb0af90 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/schema.py +++ b/packages/flow/benchmark/harness/effectbench/schema.py @@ -135,7 +135,7 @@ class OracleChannel(str, Enum): Maps 1:1 onto the ``substrate`` attribute of the concrete :class:`~openadapt_flow.runtime.effects.effect.EffectVerifier` - implementations re-exported by :mod:`openadapt_flow.benchmark.effectbench`. + implementations re-exported by :mod:`benchmark.harness.effectbench`. """ SQL = "sql" # SqlRecordVerifier — read-only SELECT under a read-only role @@ -311,7 +311,7 @@ class EpisodeRecord(BaseModel): """One trial of one task by one arm — the atomic benchmark result row. Every headline metric is a pure aggregation over a list of these (see - :func:`openadapt_flow.benchmark.effectbench.metrics.summarize`). The runner + :func:`benchmark.harness.effectbench.metrics.summarize`). The runner produces one per (task x arm x trial); ``summarize`` never needs anything the row does not carry, so raw rows can be published for external replication. """ diff --git a/packages/flow/openadapt_flow/benchmark/frappe_lending.py b/packages/flow/benchmark/harness/frappe_lending.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/frappe_lending.py rename to packages/flow/benchmark/harness/frappe_lending.py diff --git a/packages/flow/openadapt_flow/benchmark/hybrid_benchmark.py b/packages/flow/benchmark/harness/hybrid_benchmark.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/hybrid_benchmark.py rename to packages/flow/benchmark/harness/hybrid_benchmark.py index c343d13b45..282bb79cbb 100644 --- a/packages/flow/openadapt_flow/benchmark/hybrid_benchmark.py +++ b/packages/flow/benchmark/harness/hybrid_benchmark.py @@ -67,12 +67,12 @@ from pydantic import BaseModel -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.openemr_benchmark import ( +from benchmark.harness import agent_baseline +from benchmark.harness.openemr_benchmark import ( BILLING_ABORT_AFTER, _looks_like_billing_error, ) -from openadapt_flow.benchmark.run_benchmark import ( +from benchmark.harness.run_benchmark import ( _agent_run, _arm_aggregate, _compiled_run, @@ -863,7 +863,7 @@ def render_hybrid_chart(results: dict[str, Any], out_png: Path) -> Path: Returns: The written PNG path. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() @@ -1379,7 +1379,7 @@ def failures(arm: str) -> str: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.hybrid_benchmark --out benchmark/hybrid +.venv/bin/python -m benchmark.harness.hybrid_benchmark --out benchmark/hybrid ``` Requires `ANTHROPIC_API_KEY` (or `~/.anthropic/api_key`). The paid arms @@ -1398,7 +1398,7 @@ def write_hybrid_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_hybrid_chart, results, out_dir / "success_cost.png") (out_dir / "BENCHMARK.md").write_text(render_hybrid_markdown(results)) @@ -1649,7 +1649,7 @@ def _error_row(arm: str, exc: Exception) -> dict[str, Any]: def main(argv: Optional[list[str]] = None) -> int: - """CLI entry point: ``python -m openadapt_flow.benchmark.hybrid_benchmark``.""" + """CLI entry point: ``python -m benchmark.harness.hybrid_benchmark``.""" parser = argparse.ArgumentParser( description=( "Four-arm hybrid benchmark on MockMed (compiled / agent / " diff --git a/packages/flow/openadapt_flow/instrument/__init__.py b/packages/flow/benchmark/harness/instrument/__init__.py similarity index 91% rename from packages/flow/openadapt_flow/instrument/__init__.py rename to packages/flow/benchmark/harness/instrument/__init__.py index bf92686661..dbf7469cc1 100644 --- a/packages/flow/openadapt_flow/instrument/__init__.py +++ b/packages/flow/benchmark/harness/instrument/__init__.py @@ -1,7 +1,7 @@ """Competitor-drift instrument: point our EffectVerifier at ANY external agent. The self-directed silent-wrong-action benchmark -(``openadapt_flow.benchmark.silent_wrong_action``, #67) measures how often our +(``benchmark.harness.silent_wrong_action``, #67) measures how often our OWN runtime lets a wrong / absent / duplicate business effect land while reporting success. This package generalizes that measurement to an *arbitrary external computer-use agent*: given any agent behind the @@ -19,7 +19,7 @@ user-gated step. """ -from openadapt_flow.instrument.competitor_drift import ( # noqa: F401 +from benchmark.harness.instrument.competitor_drift import ( # noqa: F401 AgentRunResult, ArchitectureClassError, CostGuard, diff --git a/packages/flow/openadapt_flow/instrument/competitor_drift.py b/packages/flow/benchmark/harness/instrument/competitor_drift.py similarity index 99% rename from packages/flow/openadapt_flow/instrument/competitor_drift.py rename to packages/flow/benchmark/harness/instrument/competitor_drift.py index 3daa5076c6..2abf9fe980 100644 --- a/packages/flow/openadapt_flow/instrument/competitor_drift.py +++ b/packages/flow/benchmark/harness/instrument/competitor_drift.py @@ -2,7 +2,7 @@ runner, cost-capped. This EXTENDS the self-directed silent-wrong-action benchmark -(``openadapt_flow.benchmark.silent_wrong_action``, #67) from "our own runtime" +(``benchmark.harness.silent_wrong_action``, #67) from "our own runtime" to "any external computer-use agent." The measurement is identical — did a wrong / absent / duplicate business effect land in the system of record while the actor reported success (a *silent wrong-action*)? — but the actor is now an @@ -53,7 +53,7 @@ # judge, and the #63 effect contract. We REUSE these from the self-directed # benchmark rather than re-deriving them so the external-agent instrument and # the self-directed instrument grade with byte-identical semantics. -from openadapt_flow.benchmark.silent_wrong_action import ( +from benchmark.harness.silent_wrong_action import ( NOTE, SCENARIOS, Scenario, @@ -888,7 +888,7 @@ def write_outputs(report: InstrumentReport, out_dir: Path | str) -> None: def main(argv: Optional[list[str]] = None) -> int: - """CLI: ``python -m openadapt_flow.instrument.competitor_drift``. + """CLI: ``python -m benchmark.harness.instrument.competitor_drift``. Ships only the offline stub adapter and a dry-run estimator — a real external adapter is wired programmatically (see diff --git a/packages/flow/openadapt_flow/benchmark/openemr_benchmark.py b/packages/flow/benchmark/harness/openemr_benchmark.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/openemr_benchmark.py rename to packages/flow/benchmark/harness/openemr_benchmark.py index f2482f2307..5bb2396f11 100644 --- a/packages/flow/openadapt_flow/benchmark/openemr_benchmark.py +++ b/packages/flow/benchmark/harness/openemr_benchmark.py @@ -7,7 +7,7 @@ recorded by ``scripts/openemr_demo.py`` (fake demo patients only). Both arms are judged by the same criterion, implemented once -(:func:`openadapt_flow.benchmark.verify.verify_note_saved`): OCR of the +(:func:`benchmark.harness.verify.verify_note_saved`): OCR of the final screen must show the run's parameterized note text in the patient message list. Each run — in BOTH arms — uses a distinct note value, so a pass proves parameter substitution against live state, not replay of a @@ -60,16 +60,16 @@ from pathlib import Path from typing import Any, Callable -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.agent_baseline import _looks_like_billing_error -from openadapt_flow.benchmark.run_benchmark import ( +from benchmark.harness import agent_baseline +from benchmark.harness.agent_baseline import _looks_like_billing_error +from benchmark.harness.run_benchmark import ( _agent_run, _arm_aggregate, _compiled_run, identity_coverage_block, render_chart, ) -from openadapt_flow.benchmark.verify import verify_note_saved +from benchmark.harness.verify import verify_note_saved DEMO_URL = "https://demo.openemr.io/openemr/index.php" N_COMPILED = 20 @@ -481,7 +481,7 @@ def write_openemr_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "latency_cost.png") (out_dir / "BENCHMARK.md").write_text(render_openemr_markdown(results)) diff --git a/packages/flow/openadapt_flow/benchmark/openemr_local.py b/packages/flow/benchmark/harness/openemr_local.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/openemr_local.py rename to packages/flow/benchmark/harness/openemr_local.py index 125f6d9c24..1a4e337de8 100644 --- a/packages/flow/openadapt_flow/benchmark/openemr_local.py +++ b/packages/flow/benchmark/harness/openemr_local.py @@ -18,7 +18,7 @@ from typing import Any, Iterable, Mapping, Sequence from urllib.parse import urlencode -from openadapt_flow.benchmark.frappe_lending import ( +from benchmark.harness.frappe_lending import ( ARMS, CONDITIONS, INITIAL_TRIALS_PER_CELL, diff --git a/packages/flow/openadapt_flow/benchmark/reliability.py b/packages/flow/benchmark/harness/reliability.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/reliability.py rename to packages/flow/benchmark/harness/reliability.py diff --git a/packages/flow/openadapt_flow/benchmark/run_benchmark.py b/packages/flow/benchmark/harness/run_benchmark.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/run_benchmark.py rename to packages/flow/benchmark/harness/run_benchmark.py index cd2c4c2d1e..3aba486faf 100644 --- a/packages/flow/openadapt_flow/benchmark/run_benchmark.py +++ b/packages/flow/benchmark/harness/run_benchmark.py @@ -29,8 +29,8 @@ from typing import Any, Callable from openadapt_flow.bench import _percentile -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.verify import verify_encounter_saved +from benchmark.harness import agent_baseline +from benchmark.harness.verify import verify_encounter_saved N_COMPILED = 100 N_AGENT = 20 @@ -153,7 +153,7 @@ def _agent_run( A per-run row dict. API failures are recorded as failed rows with ``error`` set, never raised; a failed row still carries whatever usage/cost the run paid for before crashing (via - :class:`~openadapt_flow.benchmark.agent_baseline.UsageLedger`), so + :class:`~benchmark.harness.agent_baseline.UsageLedger`), so crashed runs' real spend counts in aggregates and cost ceilings. """ from openadapt_flow.backends.playwright_backend import PlaywrightBackend @@ -364,7 +364,7 @@ def render_chart(results: dict[str, Any], out_png: Path) -> Path: Returns: The written PNG path. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() @@ -578,7 +578,7 @@ def write_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "latency_cost.png") (out_dir / "BENCHMARK.md").write_text(render_markdown(results)) diff --git a/packages/flow/openadapt_flow/benchmark/silent_wrong_action.py b/packages/flow/benchmark/harness/silent_wrong_action.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/silent_wrong_action.py rename to packages/flow/benchmark/harness/silent_wrong_action.py index 009bdda54c..daeb0c7c9f 100644 --- a/packages/flow/openadapt_flow/benchmark/silent_wrong_action.py +++ b/packages/flow/benchmark/harness/silent_wrong_action.py @@ -566,7 +566,7 @@ def render_markdown(results: dict[str, Any]) -> str: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.silent_wrong_action \\ +.venv/bin/python -m benchmark.harness.silent_wrong_action \\ --out {DEFAULT_OUT_DIR} --n {results["n_per_scenario"]} ``` @@ -585,7 +585,7 @@ def render_chart(results: dict[str, Any], out_png: Path) -> Path: false-abort rate. The metric a record system cares about — a wrong write that reports success — collapses from a large bar to zero. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() @@ -658,7 +658,7 @@ def write_outputs(results: dict[str, Any], out_dir: Path) -> None: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") (out_dir / "SILENT_WRONG_ACTION.md").write_text(render_markdown(results)) - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "silent_wrong_action.png") @@ -689,7 +689,7 @@ def run_and_write( def main(argv: Optional[list[str]] = None) -> int: - """CLI: ``python -m openadapt_flow.benchmark.silent_wrong_action``.""" + """CLI: ``python -m benchmark.harness.silent_wrong_action``.""" parser = argparse.ArgumentParser( description=( "Silent-wrong-action-rate benchmark: measure how often " diff --git a/packages/flow/openadapt_flow/benchmark/verify.py b/packages/flow/benchmark/harness/verify.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/verify.py rename to packages/flow/benchmark/harness/verify.py diff --git a/packages/flow/benchmark/hybrid/BENCHMARK.md b/packages/flow/benchmark/hybrid/BENCHMARK.md index bc64083f0d..d85401a2a7 100644 --- a/packages/flow/benchmark/hybrid/BENCHMARK.md +++ b/packages/flow/benchmark/hybrid/BENCHMARK.md @@ -207,7 +207,7 @@ $0.0967), and `a` = the agent-only mean cost per run ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.hybrid_benchmark --out benchmark/hybrid +.venv/bin/python -m benchmark.harness.hybrid_benchmark --out benchmark/hybrid ``` Requires `ANTHROPIC_API_KEY` (or `~/.anthropic/api_key`). The paid arms diff --git a/packages/flow/benchmark/lending_fault_model/README.md b/packages/flow/benchmark/lending_fault_model/README.md index 740b9134ff..b6de3a698a 100644 --- a/packages/flow/benchmark/lending_fault_model/README.md +++ b/packages/flow/benchmark/lending_fault_model/README.md @@ -39,7 +39,7 @@ judged by an independent ledger read, with zero model calls. 2. **Silent Wrong-Effect Rate, three-arm verification ladder** - `swer.py` scores the same faults through the shared EffectBench contract - (`openadapt_flow.benchmark.effectbench.score_episode` + `summarize`) under + (`benchmark.harness.effectbench.score_episode` + `summarize`) under screen-only, single-surface, and complete-read-path arms. -> `swer_results.json`, `SWER.md`. Headline: screen-only SWER is 24/36, a disbursements-only verifier leaves the diff --git a/packages/flow/benchmark/lending_fault_model/swer.py b/packages/flow/benchmark/lending_fault_model/swer.py index 9471d68615..312cc57e4a 100644 --- a/packages/flow/benchmark/lending_fault_model/swer.py +++ b/packages/flow/benchmark/lending_fault_model/swer.py @@ -5,7 +5,7 @@ screen-only postcondition contract (and measures how often that silently mishandles a transactional fault), THIS module measures the same faults through the shared EffectBench scoring contract -(:func:`openadapt_flow.benchmark.effectbench.score_episode`) under a THREE-arm +(:func:`benchmark.harness.effectbench.score_episode`) under a THREE-arm ladder that mirrors the clinical ``effect_e2e`` study (screen / single-surface / complete-read-path), so the two domains are directly comparable: @@ -71,7 +71,7 @@ import requests -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, DivergenceCategory, Effect, diff --git a/packages/flow/benchmark/openemr_api_tier/approval_durable.py b/packages/flow/benchmark/openemr_api_tier/approval_durable.py index 7b3e57f929..47dbe33709 100644 --- a/packages/flow/benchmark/openemr_api_tier/approval_durable.py +++ b/packages/flow/benchmark/openemr_api_tier/approval_durable.py @@ -157,7 +157,7 @@ def run_one( ) -> dict[str, Any]: from openadapt_flow.approval_mode.gate import ApprovalGate from openadapt_flow.approval_mode.item_approval import ItemApprovalStore - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer from openadapt_flow.runtime.durable.checkpoint import CheckpointStore diff --git a/packages/flow/benchmark/openemr_api_tier/common.py b/packages/flow/benchmark/openemr_api_tier/common.py index 19bb064547..8764d08f22 100644 --- a/packages/flow/benchmark/openemr_api_tier/common.py +++ b/packages/flow/benchmark/openemr_api_tier/common.py @@ -74,7 +74,7 @@ def load_demo() -> Any: def spec_for(ref: Referral) -> Any: - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec return SyntheticPatientSpec(**ref.kwargs) diff --git a/packages/flow/benchmark/openemr_api_tier/feature_e2e.py b/packages/flow/benchmark/openemr_api_tier/feature_e2e.py index 35754d3a4d..eccb0e8d1f 100644 --- a/packages/flow/benchmark/openemr_api_tier/feature_e2e.py +++ b/packages/flow/benchmark/openemr_api_tier/feature_e2e.py @@ -51,7 +51,7 @@ def cmd_record(a: argparse.Namespace) -> None: - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec from openadapt_flow.network_capture import NetworkCapture from openadapt_flow.recorder import Recorder @@ -190,7 +190,7 @@ def wait_quiet(max_s: float = float(os.environ.get("GATE_MAX_S", "1200"))) -> di def run_one(env: dict[str, Any], arm: str, ref_idx: int, bundle: Path, out: Path) -> dict[str, Any]: - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer diff --git a/packages/flow/benchmark/openemr_api_tier/spike/bench.py b/packages/flow/benchmark/openemr_api_tier/spike/bench.py index d3a74c2f53..138570dcca 100644 --- a/packages/flow/benchmark/openemr_api_tier/spike/bench.py +++ b/packages/flow/benchmark/openemr_api_tier/spike/bench.py @@ -62,7 +62,7 @@ def _refuse(self, *_a: Any, **_k: Any) -> Any: class Env: def __init__(self, browser: bool) -> None: - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer diff --git a/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py b/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py index 12929d4e07..5e6297d2ca 100644 --- a/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py +++ b/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py @@ -39,7 +39,7 @@ def main() -> None: from playwright.sync_api import sync_playwright - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer diff --git a/packages/flow/benchmark/openemr_e2e/README.md b/packages/flow/benchmark/openemr_e2e/README.md index 39fdb23bbe..68471ac1f1 100644 --- a/packages/flow/benchmark/openemr_e2e/README.md +++ b/packages/flow/benchmark/openemr_e2e/README.md @@ -10,7 +10,7 @@ compile → replay → effect-verify (system of record) → catch a silent wrong ``` It does **not** duplicate the existing OpenEMR *benchmark* -(`openadapt_flow.benchmark.openemr_benchmark`, which measures the compiled arm +(`benchmark.harness.openemr_benchmark`, which measures the compiled arm vs. a paid agent arm on the **live** public demo and is explicitly *not* CI-reproducible). This harness *orchestrates* the runtime components that benchmark assumes work — the compiler, the `Replayer`, the `EffectVerifier`, the diff --git a/packages/flow/benchmark/openemr_e2e/harness.py b/packages/flow/benchmark/openemr_e2e/harness.py index 23a10fc498..5110b31c77 100644 --- a/packages/flow/benchmark/openemr_e2e/harness.py +++ b/packages/flow/benchmark/openemr_e2e/harness.py @@ -8,7 +8,7 @@ inject drift -> HALT -> teach the fix -> re-run clean Unlike the existing OpenEMR *benchmark* -(:mod:`openadapt_flow.benchmark.openemr_benchmark`), which measures the compiled +(:mod:`benchmark.harness.openemr_benchmark`), which measures the compiled arm vs. a paid agent arm on the LIVE public demo and is explicitly NOT CI-reproducible, this harness proves the pipeline *wiring* end to end on a deterministic, offline fixture (the in-process MockMed ``fault_server`` as the diff --git a/packages/flow/benchmark/openemr_local/README.md b/packages/flow/benchmark/openemr_local/README.md index 6a9441c40a..19efb71a28 100644 --- a/packages/flow/benchmark/openemr_local/README.md +++ b/packages/flow/benchmark/openemr_local/README.md @@ -33,7 +33,7 @@ publication bar. Raw per-run rows, environment fingerprints, application-specific adapter/oracle wiring, and the detailed cost ledger are retained in the private `OpenAdaptAI/openadapt-corpus` repository. The public generic cost-capped agent mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. See the +`benchmark/harness/agent_baseline.py`. See the [aggregate report](../agent_arm_verticals/README.md). The accepted run used baseline SHA-256 diff --git a/packages/flow/benchmark/openemr_throughput/harness.py b/packages/flow/benchmark/openemr_throughput/harness.py index 4bcd15079b..fcf9a2e817 100644 --- a/packages/flow/benchmark/openemr_throughput/harness.py +++ b/packages/flow/benchmark/openemr_throughput/harness.py @@ -237,7 +237,7 @@ def _demo_module() -> Any: def build_worker(user: str, opts: dict[str, Any]) -> Any: """One worker signed in as ``user``. Module-level so a child process can build it (``ProcessWorker``); ``opts`` holds only plain values.""" - from openadapt_flow.benchmark.openemr_local import ( + from benchmark.harness.openemr_local import ( OpenEMRPatientOracle, SyntheticPatientSpec, ) @@ -345,7 +345,7 @@ def run_item(page: Any, idx: int, row: dict[str, Any], item_dir: Path, pipeline: def run(a: argparse.Namespace) -> dict[str, Any]: from openadapt_flow._browser_setup import ensure_chromium_installed - from openadapt_flow.benchmark.openemr_local import ( + from benchmark.harness.openemr_local import ( SyntheticPatientSpec, ) from openadapt_flow.ir import Workflow diff --git a/packages/flow/benchmark/openimis_claims/README.md b/packages/flow/benchmark/openimis_claims/README.md index 94e94ad017..e01f263e40 100644 --- a/packages/flow/benchmark/openimis_claims/README.md +++ b/packages/flow/benchmark/openimis_claims/README.md @@ -28,7 +28,7 @@ Raw per-run rows, environment fingerprints, application-specific adapter/oracle wiring, and the detailed cost ledger are retained in the private `OpenAdaptAI/openadapt-corpus` repository. The public generic cost-capped agent mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. See the +`benchmark/harness/agent_baseline.py`. See the [aggregate report](../agent_arm_verticals/README.md). There is still no full timing matrix and no publication protocol here: a diff --git a/packages/flow/benchmark/scoreboard/tasks/openemr.py b/packages/flow/benchmark/scoreboard/tasks/openemr.py index c2ffaf02d6..e93b7eab6d 100644 --- a/packages/flow/benchmark/scoreboard/tasks/openemr.py +++ b/packages/flow/benchmark/scoreboard/tasks/openemr.py @@ -207,7 +207,7 @@ def check(values: dict[str, str], before_ids: set[str]) -> str: def oracle_verifier(values: dict[str, str]) -> Any: - from openadapt_flow.benchmark.openemr_local import ( + from benchmark.harness.openemr_local import ( OpenEMRPatientOracle, SyntheticPatientSpec, ) diff --git a/packages/flow/benchmark/silent_saves/metrics.py b/packages/flow/benchmark/silent_saves/metrics.py index cb7f3ee9f1..2a8fa76483 100644 --- a/packages/flow/benchmark/silent_saves/metrics.py +++ b/packages/flow/benchmark/silent_saves/metrics.py @@ -6,7 +6,7 @@ import statistics from typing import Any, Iterable, Optional -from openadapt_flow.benchmark.effectbench.metrics import wilson_interval +from benchmark.harness.effectbench.metrics import wilson_interval def rate(k: int, n: int) -> dict[str, Any]: diff --git a/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md b/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md index c09304b4b5..ce1dfe56c9 100644 --- a/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md +++ b/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md @@ -66,7 +66,7 @@ run-to-run disagreement); N proves reproducibility. ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.silent_wrong_action \ +.venv/bin/python -m benchmark.harness.silent_wrong_action \ --out benchmark/silent_wrong_action --n 10 ``` diff --git a/packages/flow/benchmark/vision_hardening/README.md b/packages/flow/benchmark/vision_hardening/README.md index 273c387138..46644fdc38 100644 --- a/packages/flow/benchmark/vision_hardening/README.md +++ b/packages/flow/benchmark/vision_hardening/README.md @@ -36,7 +36,7 @@ and CLASSIFIES what the **real** resolver does on each generated case: go DOWN. A hardening fix regenerates the corpus and lowers the ratchet. The headline metric is **SWER** (Silent Wrong-Effect/Resolution Rate), the same -taxonomy as EffectBench (`openadapt_flow/benchmark/effectbench/`) — this sweep +taxonomy as EffectBench (`benchmark/harness/effectbench/`) — this sweep is the *resolution-tier* feeder for that benchmark. ## The two tiers diff --git a/packages/flow/docs/EFFECT_KIT.md b/packages/flow/docs/EFFECT_KIT.md index 3739752c74..cb02250ed9 100644 --- a/packages/flow/docs/EFFECT_KIT.md +++ b/packages/flow/docs/EFFECT_KIT.md @@ -1420,7 +1420,7 @@ Runs against the pinned reference fixture README documents the pinned compose bring-up and fixture bootstrap). The bundle's effect contracts are exactly the ones the reference matrix ships -(`openadapt_flow/benchmark/frappe_lending.py::loan_application_effects`): +(`benchmark/harness/frappe_lending.py::loan_application_effects`): one `record_written` (at-most-once for the synthetic applicant) plus a `field_equals` read-back per entered field, each bound to `{param: ...}`. diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md new file mode 100644 index 0000000000..7fb257ecdb --- /dev/null +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -0,0 +1,311 @@ +# Generality audit of the engine + +Audited 2026-11-03 on `main` at `213f3d9fe`. Scope: everything under +`packages/flow/openadapt_flow/`. + +The rule: the engine has no code written for one application. What one +application needs lives in data (`app_profiles/*.yaml`, generated contracts, +deployment configs) or in test fixtures and benchmark harnesses outside the +shipped package. `scripts/check_engine_generality.py` enforces it in CI; see +[CONTRIBUTING.md](../CONTRIBUTING.md#app-knowledge-lives-in-data). + +## Method + +1. Searched for application names, case-insensitive: OpenEMR, Medplum, + Dolibarr, EspoCRM, Redmine, Snipe-IT, InvenTree, Kimai, OpenMRS, Gitea, + Firefly III, Vikunja, Zammad, Open Dental, Odoo, openIMIS, Frappe/ERPNext, + Patient Notes, Epic, Stedi, TextEdit, Notepad, and OpenAdapt's own demo apps + MockMed and MockLoan. +2. Searched for identifiers that only make sense for one application: table and + column names (`pnotes`, `patient_data`, `uuid_registry`, `user_settings`, + `tabLoan Application`), URL paths (`/api/db`, `/api/encounter`, + `/apis/default/fhir`), form and query parameters (`token_main`, + `csrf_token_form`, `clearpass`), selectors, button labels, dialog titles, + field names, and thresholds tuned on one application's screens. +3. Read the code around each hit and put it in one of four buckets: + - **(a)** a comment or docstring example. Fine; reworded where it read as a + rule. + - **(b)** application knowledge stored as data and loaded generically. + - **(c)** a test fixture, benchmark harness or demo application inside the + shipped package. Moved out, unless it backs a user-facing command that + must work from a wheel. + - **(d)** special-casing in engine logic. Generalized, or turned into data. + +## Counts + +| Bucket | Rows | Distinct findings | +| --- | --- | --- | +| (a) comment or docstring example | 118 | — | +| (b) data loaded generically | 4 | 2 (`app_profiles/openemr.yaml`, the `clinical-write` policy pack as a default) | +| (c) fixture, harness or demo in the package | 35 rows, plus 27 files of `benchmark/` | 19 | +| (d) special-casing in engine logic | 27 | 15 | + +Rows group consecutive lines of one hit. The full table is at the end. + +## (d) findings and actions + +Each finding names the application that motivated it. "Second app" is the +existing fixture that shows the generalized code still works beyond the +motivating application. + +| # | Where | What | Motivating app | Action | Status | +| --- | --- | --- | --- | --- | --- | +| D1 | `report_redaction.py:68-70`, `:131-133` | Exact secret names `token_main`, `csrf_token_form` | OpenEMR | Match secret names by pattern (`token`, `csrf`, `xsrf`, `session`, ...) instead of a per-app list | planned | +| D2 | `network_capture.py:78` | `clearpass` in the secret-name pattern | OpenEMR | Drop it; the `pass\b` alternative already matches | planned | +| D3 | `runtime/effects/rest.py:114-115`, `deployment.py:387-390`, `reward/models.py:226`, `fault_gate.py:722` | REST reader defaults `records_path="/api/db"`, `records_key="records"` | MockMed | Required when the effect kind is `rest`; refused at config load when missing | planned | +| D4 | `runtime/effects/compensation.py:364` | `delete_path_template="/api/encounter/{id}"` default | MockMed | Required | planned | +| D5 | `compiler/binding_admission.py:114-119`, `:71-73` | Admission write path defaults to `/api/encounter`; body skips MockMed's `source` field | MockMed | Write path and method from the observed `api_write` event; skip only server-assigned fields | planned | +| D6 | `compiler/effect_mining.py:100-102`, `runtime/effects/effect.py:900-901` | Idempotency key field defaults to `key` | MockMed | No default; from the contract or deployment config | planned | +| D7 | `runtime/effects/fhir.py:59-67`, `:221`, `deployment.py:448-449` | FHIR defaults: resource type `Observation`, `note -> valueString` | OpenEMR | Required in the contract (`generate/ensure_contract.py` writes them) | planned | +| D8 | `generate/ensure_contract.py:123-147` | Translates the Medplum probe script's output keys | Medplum | The probe emits the generic fact names; translation removed | planned | +| D9 | `runtime/effects/frame.py:1937`, `rollback_state.py:781` | `"0"` means "no record" | OpenEMR | The value moves into the profile rule (`no_record_values`) | planned | +| D10 | `__main__.py:2838-2934`, `:9305-9319`; `cli_variants.py:131,173-182` | `replay` and the variant probe fall back to the bundled MockMed when no URL is given; `--drift` turns off structural grounding | MockMed | Target URL required outside the demo commands | planned | +| D11 | `qualification_oracle_gate.py` | `qualify accept` tests every workflow's oracle against MockMed (banner text, `/api/encounter`, `/api/db`) | MockMed | Build the fault fixture from the bundle's own effect binding | planned | +| D12 | `execute/dispatch.py:111-188` | The Execute service's live runner always runs the MockMed tutorial workflow | MockMed | Replay through the admission's deployment config; MockMed only for synthetic admissions | planned | +| D13 | `compiler/compile.py:151`, `runtime/replayer.py:394`, `runtime/identity.py:108` | Thresholds measured on MockMed only | MockMed | Generic logic, one-app numbers. Recalibrate on two or more apps | follow-up issue | +| D14 | `runtime/identity.py:275-302` | Hand-written list of non-identifying words drawn from EMR list screens | MockMed, EMRs | Derive from the recording (tokens shared by sibling rows) | follow-up issue | +| D15 | `eligibility/client.py:37-41`, `:1091-1110` | One vendor's eligibility API (URL, key name, body) in the engine | Stedi | Move behind a connector interface | follow-up issue | + +## (c) findings and actions + +| Where | What | Action | Status | +| --- | --- | --- | --- | +| `benchmark/` (27 Python files: `run_benchmark`, `openemr_benchmark`, `openemr_local`, `frappe_lending`, `hybrid_benchmark`, `dom_arm`, `silent_wrong_action`, `desktop_benchmark`, `verify`, `agent_baseline`, `reliability`, `chart_fonts`, `effectbench/`) | Benchmark harnesses shipped in the wheel | Moved to `packages/flow/benchmark/harness/` (import `benchmark.harness.*`). `openadapt-flow benchmark` loads it from a source checkout and says so from a wheel | done | +| `instrument/` | Competitor-drift harness; imported the benchmark at module load, so it was already broken in a wheel | Moved to `packages/flow/benchmark/harness/instrument/` | done | +| `integrations/harness2/` | Harness2 research harness on MockMed | Move to `packages/flow/benchmark/harness/harness2/` | planned | +| `mockloan/`, `demo_driver.record_disbursement_demo` | Lending demo app no product command uses | Move to the benchmark tree | planned | +| `guard_demo.py`, `agent/ledger_demo.py` | `python -m` demos, no CLI command | Move to the benchmark tree | planned | +| `mockmed/teach_scope.py` | Used only by tests | Move to `tests/fixtures` | planned | +| `learning/synth_stream.py:1-245` | MockMed note stream | Split out the generic `StructuralDiffInducer`, then move the stream | planned | +| `reward/seed.py:1-236` | MockMed reward seed | Split out the generic bundle helpers (`:238-410`), then move the seed behind the demo flag | planned | +| `record_check_proof.py:60-124` | `MOCKMED` fixture and `if name == "mockmed"` | Fixtures registered by name from the demo app's own module | planned | +| `validation/dense_surface.py` | Study harness rendering a MockMed table | Move to `benchmark/dense_surface` | planned | +| `__main__.py:5246-5275` (`bench`) | Always replays against MockMed | Accept `--url`, or load from a checkout like `benchmark` | planned | +| `execute/registry.py`, `execute/service.py`, `execute/app.py`, `execute/dispatch.py:60-108`, `--seed-mockmed` | Demo seeding inside the Execute service | Generic `seed_admissions`; the CLI passes the demo seed | planned | +| `eligibility/payer_routes.yaml`, `fleet/simulator.py`, `repair/dialog_fixture.py` | Synthetic test data and simulators in the package | Move to the benchmark tree or `tests/fixtures` | planned | + +### Kept in the package on purpose + +These back user-facing commands that must work from `pip install openadapt` +(`openadapt quickstart`, `openadapt-flow tutorial`, `demo`, `demo-record`, +`openadapt-agent serve` with no flags, `openadapt-agent example-app`). They are +OpenAdapt's own synthetic demo applications, not customer applications, and the +engine does not branch on them: + +- `mockmed/` (server, fault server, chart pages, static UI) +- `tutorial.py`, `demo_proof.py`, `demo_driver.record_triage_demo` +- `agent/sandbox.py`, `agent/tutorial.py`, `agent/example_app.py` + +The generality check allows MockMed names only in these modules. + +## (b) data + +- `app_profiles/openemr.yaml`: OpenEMR's bookkeeping writes, loaded by + `app_profiles.load_profile(name)`, which has no per-app code. +- The `clinical-write` policy pack is the default policy in + `execute/models.py:74`, `cli_config.py:164`, `integrations/agents/skills.py:351` + and `agent/teach.py:121`. It is data, but a healthcare default; whether to make + the policy required is a product decision, noted here. + +## (a) rewording done + +- `deployment.py:548`: the example environment variable in a user-facing + error is now `APP_READER_PASSWORD`. +- `authoring.py:807`: the comment on a generic branch no longer names MockMed. + +The remaining (a) rows are examples in comments and docstrings. The code they +describe is generic and configured per contract or profile. Healthcare words +such as MRN, DOB and patient appear in generic heuristics (for example +`risk.py:370`); that is the product's domain, not one application. + +## Full table + +Paths are relative to `packages/flow/openadapt_flow/` as of the audited commit. + +| Location | Bucket | App | Note and action | +| --- | --- | --- | --- | +| `__main__.py:2838-2934` | (d) | MockMed | replay: general code path defaults to serving bundled MockMed when no --url/config URL, and --drift flips use_structural=False (grounding ladder changes on a demo flag). Make --url required for replay (or route no-URL to `tutorial`/`demo`), rename --drift behavior to a generic --force-visual-grounding, keep drift injection in the demo command. | +| `__main__.py:9305-9319` | (d) | MockMed | replay parser: --url default = bundled MockMed; --drift MockMed modes (see 2838-2934). | +| `cli_variants.py:131,173-182` | (d) | MockMed | first-screen probe serves bundled MockMed when no --url (same default as replay). Refuse instead (first_screen_unavailable). | +| `compiler/binding_admission.py:114` | (d) | MockMed | AdmissionFixture.write_path default '/api/encounter' (line 119); compile.py:3379 never overrides it | +| `compiler/binding_admission.py:71` | (d) | MockMed | _BODY_SKIP_FIELDS adds MockMed's 'source' field (line 73) | +| `compiler/compile.py:146` | (d) | MockMed | REGION_STABLE_TOLERANCE=16 (line 151) calibrated on MockMed only; recalibrate on >=2 apps | +| `compiler/effect_mining.py:100` | (d) | MockMed | IDEMPOTENCY_KEY_FIELD='key' (line 102) is MockMed's field; take from contract/profile | +| `deployment.py:387` | (d) | MockMed | records_path default '/api/db' + records_key 'records' (lines 389-390) are MockMed's shape; make required for kind: rest | +| `deployment.py:448` | (d) | OpenEMR | fhir section default resource_type='Observation' (line 449) mirrors OpenEMR note-as-Observation; make required | +| `eligibility/client.py:37-41` | (d) | Stedi | Stedi 270/271 vendor client in engine (hard-coded URL, STEDI_API_KEY env, to_stedi_body); move to integrations/connector | +| `execute/dispatch.py:111-188` | (d) | MockMed | live_replay (the only real Execute runner) always calls tutorial.run_tutorial_workflow: POSTs /api/reset to target_url, appends MockMed ?fault=ok&idempotency=demo#tasks, verifies via /api/db 'records', forces clinical-write policy and 'note' param; with no target_url serves MockMed. Ignores AdmittedBundle.policy. Generalize: replay via the standard replay path with the admission's deployment.yaml (effects/backend/policy); keep MockMed only behind synthetic/demo admissions. | +| `fault_gate.py:722` | (d) | MockMed | rest_memory fault case default records_path '/api/db' (test harness; low) | +| `generate/ensure_contract.py:123` | (d) | Medplum | facts_from_probe translates Medplum probe key names (133-147); make the probe emit generic FACTS | +| `network_capture.py:78` | (d) | OpenEMR | SECRET_NAME regex lists 'clearpass' (OpenEMR login field); redundant with pass\b alternative -> drop | +| `qualification_oracle_gate.py:5,36,70,225,325-455` | (d) | MockMed | evaluate_oracle_gate (called by qualification_proposal.py:221,372 for every `qualify accept`) runs ANY workflow's proposed oracle against MockMed: hard-coded banner 'Encounter saved — ', POST /api/encounter?fault=optimistic with {patient_id, type:'Triage', note}, /api/db verifier. Generalize: a generic lying-store fixture driven by the bundle's own effect binding (write path/body/records path from the effect), or fixtures registered as data. | +| `report_redaction.py:131-133` | (d) | OpenEMR | _SECRET_KEYS lists token_main/csrf_token_form; same fix as DEFAULT_PARAMS | +| `report_redaction.py:68` | (d) | OpenEMR | token_main/csrf_token_form in DEFAULT_PARAMS (with :131-133 _SECRET_KEYS): replace with generic token/csrf name regex (reuse network_capture.TOKEN_NAME) or per-app secret_params in app profile | +| `reward/models.py:226` | (d) | MockMed | RecordReadRecipe.records_path default '/api/db' (MockMed's endpoint) in a general model; make it required for kind=rest. | +| `rollback_state.py:781` | (d) | OpenEMR | same '0' no-record sentinel as frame.py:1937 | +| `runtime/effects/compensation.py:364` | (d) | MockMed | RestCompensator delete_path_template default '/api/encounter/{id}'; make required | +| `runtime/effects/effect.py:900` | (d) | MockMed | Effect.key_field default 'key' (line 901) is MockMed's field | +| `runtime/effects/fhir.py:59` | (d) | OpenEMR | DEFAULT_OBSERVATION_PATHS (61-67) maps note->valueString per OpenEMR; require field_paths from contract | +| `runtime/effects/frame.py:1937` | (d) | OpenEMR | ref in (None,'','0',0): '0' = OpenEMR's 'no chart' (patient_id 0); gated by profile insert_entry_names_no_record; move sentinel into profile | +| `runtime/effects/rest.py:114-115` | (d) | MockMed | RestRecordVerifier defaults records_path='/api/db', records_key='records'; make required | +| `runtime/identity.py:104` | (d) | MockMed | COVERAGE_THRESHOLD=0.8 (line 108) calibrated on MockMed rows only | +| `runtime/identity.py:275-302` | (d) | MockMed/EMR list | _GENERIC_IDENTITY_WORDS hand-tuned EMR list vocabulary (incl. 'demo','chart','review','results'); derive from recording or app profile | +| `runtime/replayer.py:392` | (d) | MockMed | PC_STRUCTURAL_TEMPLATE_THRESHOLD=0.8 (line 394) calibrated on MockMed only | +| `__main__.py:10030-10037` | (c) | MockMed | bench parser. | +| `__main__.py:10166` | (c) | MockMed | verify-scaffold --prove choices ('mockmed','effect-e2e'): fixture names hard-coded in CLI; see record_check_proof.py. | +| `__main__.py:11940-11989,12341-12415` | (c) | MockMed | serve-execute/serve-reward --seed-mockmed flags and banner paths (data_dir/mockmed/records.db). Opt-in demo seeding; keep as demo flag or rename --seed-demo. | +| `__main__.py:1787-1806` | (c) | MockMed | demo-record command: serves MockMed, runs demo_driver.record_triage_demo. User-facing CLI; demo data could be a recorded script in data. | +| `__main__.py:1808-1998` | (c) | MockMed | tutorial command (via tutorial.py). Deliberate product feature; must work from wheel. | +| `__main__.py:1999-2040` | (c) | MockMed | demo command (demo_proof + tutorial). Deliberate product feature; must work from wheel. | +| `__main__.py:326-330` | (c) | MockMed | _with_drift: builds MockMed ?drift= query; only meaningful for the bundled demo. Move with demo helpers. | +| `__main__.py:5246-5275` | (c) | MockMed | bench command: always replays against bundled MockMed. Not generic; move to benchmark/ (source-checkout command, like `benchmark`) or accept --url. | +| `__main__.py:8773,8782` | (c) | MockMed | demo-record parser help/--drift. | +| `agent/example_app.py:1-400` | (c) | (own demo: reports portal) | `openadapt-agent example-app` local sales-order portal (/login clerk/portal-demo, /api/orders). User-facing demo command; importers agent/cli.py:1250 + 4 flow tests. | +| `agent/ledger_demo.py:1-720` | (c) | MockMed | scripted 30-referral worklist simulation (python -m). Not a CLI command; move to benchmark or packages/agent/tests fixtures; importer packages/agent/tests/test_ledger.py. | +| `agent/sandbox.py:1-530` | (c) | MockMed | `openadapt-agent serve` zero-flag sandbox: add_triage_note on MockMed, CASES with MockMed fault/drift queries, clinical-write. User-facing; must ship. Case table could be data (sandbox YAML). | +| `agent/tutorial.py:1-140` | (c) | MockMed | prepare_tutorial_session: hosts MockMed, writes deployment YAML with /api/db, clinical-write. Used by sandbox and agent/cli.py:760; packages/agent/tests/test_tutorial.py. | +| `demo_driver.py:1-96` | (c) | MockMed | record_triage_demo: MockMed selectors and creds (nurse.demo / mockmed-demo-pass). Used by demo-record CLI + benchmark (fault_model, undo_proposal, ui_readback, harness, cosmetic_drift), tests (recorder, e2e), scripts (demo_media, openemr_demo). Split: keep record_triage_demo with mockmed package. | +| `demo_driver.py:97-171` | (c) | MockLoan | record_disbursement_demo (mockloan-demo-pass): no product command uses it. Move to benchmark/lending_fault_model; importers: benchmark/undo_proposal/run.py, benchmark/lending_fault_model/{run,resolution_ladder}.py, tests/e2e/test_lending_fault_model.py. | +| `demo_proof.py:1-60,975-1110` | (c) | MockMed | `demo` page renderer; CROP 640x320 tuned to MockMed layout, 'Encounter saved', /api/db wording. User-facing; must ship. Crop could come from the demo-app spec. | +| `eligibility/payer_routes.yaml:1-` | (c) | Stedi/Cigna | synthetic TEST payer route data shipped in package | +| `execute/app.py:29,39,141,151` | (c) | MockMed | seed_mockmed passthrough; same as service. | +| `execute/dispatch.py:60-108` | (c) | MockMed | default_runner/synthetic_mockmed: synthetic admissions project canned MockMed tutorial outcomes (evidence_tag mockmed-*). Rename to synthetic_runner and move the canned projection to tests/fixtures or keep behind --seed-mockmed only. | +| `execute/registry.py:10-16,81-106` | (c) | MockMed | MOCKMED_* constants + seed_mockmed_admissions in engine registry. Move to execute/demo_seed.py (or tests/fixtures); importers: __main__ serve-execute, tests test_execute_ref/test_execute_post_dispatch_faults/test_jit_draft_execution/test_backend_parity_refusals. | +| `execute/service.py:45,74,84-85` | (c) | MockMed | ExecuteService(seed_mockmed=) flag. Replace with generic seed_admissions: Iterable[AdmittedBundle] and let the CLI pass the demo seed. | +| `fleet/simulator.py:1-` | (c) | synthetic payer portal | in-package simulator fixture; move to benchmark/examples | +| `guard_demo.py:1-296` | (c) | MockMed | `python -m openadapt_flow.guard_demo`: fake agent posts referral to MockMed with chart_pages READER creds; not a CLI subcommand. Move to packages/flow/examples or benchmark; importers: packages/agent/tests (1), packages/flow/tests (6). | +| `instrument/ (__init__.py, competitor_drift.py)` | (c) | MockMed | MockMed competitor-drift harness. Moved to packages/flow/benchmark/harness/instrument. | +| `integrations/harness2/ (all: tasks.py, apps.py, agent.py, adapter.py, traces.py, proposers.py, scorer.py, llm.py, pixels.py, rollout.py, selection.py, __init__.py)` | (c) | MockMed | Whole subpackage is a research harness for Harness2 on MockMed: MOCKMED_PATIENT/PATIENTS (tasks.py:71-148), MockMedTarget (apps.py:50-105, /api/db at :70), regex for MockMed task text and 'triage'/'New Encounter'/'encountersaved' (agent.py:221-324), MockMed banner regex Name — MRN — DOB (traces.py:55-57,128-138), clinical prompts (adapter.py:93-120,429-445; proposers.py:50-63). Move to packages/flow/benchmark/harness2; importers: benchmark/harness2_scorer/run.py, tests/test_harness2_{adapter,pixels,traces}.py, pyproject extra 'harness2', .github/workflows/packages.yml:451. | +| `learning/synth_stream.py:1-245` | (c) | MockMed | synthetic MockMed add-patient-note stream (intents, 'Consent Required', 'Belford, Phil DOB'). Move to benchmark/ or tests/fixtures. | +| `learning/synth_stream.py:246-525` | (c) | (generic) | StructuralDiffInducer is generic but lives in the MockMed harness module; imported by learning/teach.py:96, learning/branch_teach.py:76, benchmark/openemr_e2e/harness.py, 4 tests; pyproject mypy list (line 611). Move to learning/induce.py first, then move the stream out. | +| `mockloan/ (server.py, fault_server.py, static/)` | (c) | MockLoan | Own synthetic lending app shipped in wheel but used by no product command; only demo_driver.record_disbursement_demo, benchmark/lending_fault_model, benchmark/undo_proposal, tests (test_mockloan, test_lending_swer, e2e/test_lending_fault_model), scripts (1). Move to packages/flow/benchmark/apps/mockloan. | +| `mockmed/ (server.py, fault_server.py, chart_pages.py, teach_scope.py, static/)` | (c) | MockMed | Own synthetic demo app shipped in wheel. Required by tutorial/demo/demo-record/replay-default/sandbox and root openadapt/quickstart (engine.py, workers.py) + root tests/test_partner_kit.py. Keep shipping (or split to a demo-apps package). mockmed/teach_scope.py (EncounterWrites) is used only by tests -> move to tests/fixtures. chart_pages used by guard_demo + benchmark (agent_guard, effect_e2e, ui_readback). | +| `record_check_proof.py:1-11,60-124` | (c) | MockMed | --prove fixture registry hard-codes MOCKMED Fixture (judge on patient_id/note, write_path /api/encounter, body type Triage) and `if name == 'mockmed'`. Opt-in proof against our fixture. Move MOCKMED Fixture to mockmed/proof_fixture.py and resolve fixtures by entry point / dotted path. Importers: scaffold_verifier.py:594, benchmark/effect_e2e, tests (2). | +| `repair/dialog_fixture.py:1-` | (c) | synthetic | rendered test app in product package (no real app) | +| `reward/seed.py:1-236` | (c) | MockMed | seed_mockmed, MockMed encounters table/SQL (MOCKMED_TABLE 'encounters', SELECT id, patient_id, type, status), patients, episodes. Move to reward/demo_seed or tests/fixtures; CLI serve-reward --seed-mockmed uses it. | +| `reward/seed.py:238-410` | (c) | (generic) | write_bundle/calibrate_bundle/self_signed_certificate are generic helpers living in the MockMed seed module; split into reward/bundle.py. Importers: integrations/harness2/scorer.py, benchmark/undo_proposal/run.py, tests/test_probe_induction.py, test_undo_proposal.py, test_reward_worker.py, test_reward_trust_boundary.py. | +| `tutorial.py:1-560,630-841` | (c) | MockMed | tutorial engine: MockMed selectors (.open-btn,#new-encounter,#type-triage,#note,#save-encounter,#saved-banner), /api/reset, /api/db, ?fault=ok entry, TUTORIAL_POLICY clinical-write. User-facing (tutorial, demo, quickstart in root openadapt/quickstart/engine.py imports _records, run_tutorial_workflow, record_tutorial; agent/tutorial.py; execute/dispatch.py). Must ship in wheel; could be data-driven (demo-app spec YAML: entry query, reset/records endpoints, recorded click script). | +| `validation/dense_surface.py:418` | (c) | MockMed | validation study renders MockMed-branded table inside product package; move to benchmark/dense_surface | +| `__main__.py:5422,9499,10873` | (b) | (domain) | clinical-write as default/suggested policy name (built-in policy pack, data). | +| `app_profiles/openemr.yaml:1-` | (b) | OpenEMR | per-app bookkeeping profile loaded by load_profile(name); only profile shipped | +| `cli_config.py:164; integrations/agents/skills.py:351; agent/teach.py:121` | (b) | (domain) | default policy 'clinical-write' (shipped policy pack data). | +| `execute/models.py:74` | (b) | (domain) | AdmittedBundle.policy default 'clinical-write' (healthcare policy pack); consider requiring it. | +| `__main__.py:10053` | (a) | MockMed | benchmark help text (benchmark already moved to source-checkout harness). | +| `__main__.py:10536,10577` | (a) | MockMed | --admit-local help mentions MockMed; code path is generic. | +| `__main__.py:1149,1257-1266,1288` | (a) | MockMed | docstrings / refusal text about --drift being MockMed-only. | +| `__main__.py:3819` | (a) | MockMed | docstring: --drift forced off. | +| `__main__.py:7994` | (a) | TextEdit | help example for --backend macos owner app. | +| `__main__.py:9,27,39` | (a) | MockMed | module docstring command list. | +| `agent/cards.py:9-14; agent/skill.py:145-147; agent/bridge.py:132-133,325-326; agent/ledger.py:3,378; agent/delegated.py:11,78; agent/mailbox.py:130; agent/teach.py:281-283` | (a) | (domain) | referral / EMR / MRN examples in docstrings and tool descriptions; fine. | +| `api.py:84` | (a) | MockMed | docstring: replay without url serves MockMed (changes if replay default changes). | +| `app_profiles/__init__.py:315` | (a) | OpenEMR | comment/docstring example | +| `app_profiles/__init__.py:58` | (a) | OpenEMR | comment/docstring example | +| `authoring.py:807` | (a) | MockMed | comment 'Non-secret MockMed note' on a generic branch; reword to 'non-secret field'. | +| `backends/factory.py:117` | (a) | TextEdit | comment/docstring example | +| `backends/parallels_vm.py:403` | (a) | OpenDental | comment/docstring example | +| `backends/playwright_backend.py:890` | (a) | OpenEMR | comment/docstring example | +| `backends/win_agent/server.py:1578` | (a) | Notepad | comment/docstring example | +| `cli_api.py:71` | (a) | OpenEMR | help example 'lname'/'mrn' column names. | +| `cli_undo.py:181` | (a) | MockMed | help says default target is MockMed but _cmd_undo_run refuses that case; reword help. | +| `cli_undo.py:538-541` | (a) | MockMed | comment explaining refusal. | +| `compile_agent/code_action.py:135` | (a) | OpenEMR | comment/docstring example | +| `compiler/binding_admission.py:19` | (a) | MockMed | comment/docstring example | +| `compiler/compile.py:193` | (a) | OpenEMR | comment/docstring example | +| `compiler/compile.py:2184` | (a) | MockMed | comment/docstring example | +| `compiler/compile.py:717` | (a) | MockMed | comment/docstring example | +| `compiler/fixup_passes.py:7` | (a) | OpenEMR | history note; passes are generic | +| `compiler/local_edit.py:321` | (a) | MockMed | comment/docstring example | +| `deployment.py:1035` | (a) | OpenEMR | comment/docstring example | +| `deployment.py:112` | (a) | MockMed | comment/docstring example | +| `deployment.py:144` | (a) | TextEdit | comment/docstring example | +| `deployment.py:548` | (a) | OpenEMR | user-facing error example OPENEMR_READER_PASSWORD; reword to neutral APP_READER_PASSWORD | +| `ensure_record.py:114` | (a) | Medplum | comment/docstring example | +| `ensure_record.py:162` | (a) | Medplum | comment/docstring example | +| `ensure_record.py:97` | (a) | Medplum | comment/docstring example | +| `execute/dispatch.py:1-6` | (a) | MockMed | module docstring; update after generalizing live_replay. | +| `execution_profiles.py:2595` | (a) | MockMed | comment/docstring example | +| `fault_gate.py:822` | (a) | OpenEMR | comment/docstring example | +| `generate/demo_analysis.py:781` | (a) | Redmine | comment/docstring example | +| `generate/ensure_contract.py:4` | (a) | Medplum | comment/docstring example | +| `generate/verify_contract.py:1234` | (a) | Redmine | comment/docstring example | +| `generate/verify_contract.py:1275` | (a) | EspoCRM | comment/docstring example | +| `generate/verify_contract.py:1276` | (a) | Dolibarr | comment/docstring example | +| `generate/verify_contract.py:225` | (a) | Redmine | comment/docstring example | +| `generate/verify_contract.py:566` | (a) | EspoCRM | comment/docstring example | +| `integrations/agents/__init__.py:79` | (a) | (domain) | 'a clinic, a tenant' example. | +| `jit/authority.py:463` | (a) | TextEdit | comment/docstring example | +| `jit/authority.py:623` | (a) | TextEdit | comment/docstring example | +| `jit/compile_agent.py:354` | (a) | TextEdit | comment/docstring example | +| `jit/recorded.py:6` | (a) | MockMed | docstring example task id | +| `jit/signature.py:72` | (a) | MockMed | comment/docstring example | +| `learning/__init__.py:20; learning/halt_loop.py:8; learning/trace.py:60; learning/branch_teach.py:6,108; learning/library.py` | (a) | MockMed | docstring references. | +| `operator_profile.py:79` | (a) | OpenEMR | comment/docstring example | +| `policy_packs.py:41` | (a) | MockMed | user-facing pack summary names MockMed; fine as the demo app | +| `qualification_dev_signer.py:3` | (a) | MockMed | comment/docstring example | +| `qualification_proposal.py:10` | (a) | MockMed | comment/docstring example | +| `qualification_web_session.py:3` | (a) | OpenEMR,openIMIS | comment/docstring example | +| `qualify/config.py:303` | (a) | OpenEMR | comment/docstring example | +| `record_check_binding.py:48` | (a) | openIMIS | comment/docstring example | +| `record_check_identity.py:4` | (a) | OpenEMR | comment/docstring example | +| `record_check_synthesis.py:25` | (a) | MockMed | comment/docstring example | +| `registry.py:3` | (a) | OpenEMR | comment/docstring example | +| `registry.py:813` | (a) | OpenEMR | comment/docstring example | +| `report_redaction.py:5` | (a) | OpenEMR | comment/docstring example | +| `reward/models.py:497; reward/calibration.py:13` | (a) | (domain) | 'triage rows' wording in docstrings. | +| `risk.py:476` | (a) | openIMIS | comment/docstring example | +| `risk.py:565` | (a) | OpenEMR | comment/docstring example | +| `rollback_state.py:77` | (a) | OpenEMR | comment/docstring example | +| `runtime/actuators/web_form.py:214` | (a) | OpenEMR | comment/docstring example | +| `runtime/actuators/web_form.py:324` | (a) | validate.js | var constraints recognizer is a JS library convention (OpenEMR among users), not app-specific | +| `runtime/actuators/web_form.py:331` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/__init__.py:21` | (a) | MockMed | comment/docstring example | +| `runtime/effects/__init__.py:23` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/__init__.py:38` | (a) | Frappe | comment/docstring example | +| `runtime/effects/_common.py:143` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/auth.py:33` | (a) | Frappe | comment/docstring example | +| `runtime/effects/db_snapshot.py:713` | (a) | Dolibarr | comment/docstring example | +| `runtime/effects/effect.py:1536` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/effect.py:1541` | (a) | OpenEMR,MockMed | comment/docstring example | +| `runtime/effects/effect.py:206` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/effect.py:208` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/effect.py:223` | (a) | Redmine | comment/docstring example | +| `runtime/effects/effect.py:903` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:1` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:12` | (a) | MockMed | comment/docstring example | +| `runtime/effects/fhir.py:14` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:191` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:195` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:3` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:38` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:386` | (a) | Epic | comment/docstring example | +| `runtime/effects/fhir.py:39` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:42` | (a) | MockMed | comment/docstring example | +| `runtime/effects/frame.py:135` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:144` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:1499` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:1630` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:1944` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:242` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:46` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fresh_session.py:24` | (a) | MockMed | comment/docstring example | +| `runtime/effects/rest.py:237` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/rest.py:5` | (a) | MockMed | comment/docstring example | +| `runtime/effects/sql.py:12` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:215` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:215` | (a) | Frappe | comment; _TABLE_NAME allows space generically | +| `runtime/effects/sql.py:3` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:365` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/sql.py:4` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:480` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:534` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:535` | (a) | Frappe | comment/docstring example | +| `runtime/grounder.py:136-155` | (a) | healthcare domain | _INTENT_STOPWORDS include patient/mrn; domain wording, intent-only; low | +| `submit_binding.py:8` | (a) | openIMIS | comment/docstring example | +| `validation/dense_surface.py:1271` | (a) | OpenEMR | comment/docstring example | +| `validation/dense_surface.py:5` | (a) | OpenEMR | comment/docstring example | +| `verifier.py:2467` | (a) | OpenEMR | comment/docstring example | +| `vision/hashing.py:10` | (a) | MockMed | docstring for the MockMed-calibrated phash tolerance (see compile.py:151) | +| `volatility.py:100` | (a) | OpenEMR | comment/docstring example | +| `volatility.py:31` | (a) | OpenEMR | comment/docstring example | +| `volatility.py:6` | (a) | OpenEMR | comment/docstring example | +| `benchmark/` (27 files) | (c) | OpenEMR, Frappe, MockMed, Open Dental | Moved to `packages/flow/benchmark/harness/` | diff --git a/packages/flow/docs/design/EFFECT_VERIFIER.md b/packages/flow/docs/design/EFFECT_VERIFIER.md index b7364ebbcf..b54ec7cb59 100644 --- a/packages/flow/docs/design/EFFECT_VERIFIER.md +++ b/packages/flow/docs/design/EFFECT_VERIFIER.md @@ -80,7 +80,7 @@ the note-as-Observation write the fake models cannot be created over FHIR on a stock OpenEMR; the property the live test establishes is the one the fake could not — the verdicts are correct against a **real FHIR server**, not a fake. CI still runs vision-only against the **public demo** -(`openadapt_flow/benchmark/openemr_benchmark.py`, `scripts/openemr_demo.py`); +(`benchmark/harness/openemr_benchmark.py`, `scripts/openemr_demo.py`); the live FHIR test is gated behind `OPENEMR_FHIR_BASE_URL` and skipped otherwise. See `benchmark/openemr_live/README.md` for the one-command bring-up. diff --git a/packages/flow/docs/desktop/PHASE2.md b/packages/flow/docs/desktop/PHASE2.md index 6eb67329e8..16eadb2505 100644 --- a/packages/flow/docs/desktop/PHASE2.md +++ b/packages/flow/docs/desktop/PHASE2.md @@ -58,7 +58,7 @@ so `WindowsBackend` sees exactly what it drives). | `scripts/desktop/patient_notes.ps1` | WinForms target app (list-select → edit note → save); drift knobs via `pn_env.json` | | `scripts/desktop/pn_db.py` | SQLite ground-truth CLI (seed/list/save/get/all) | | `scripts/desktop/uia_arm.py` | UIA-selector arm (pywinauto), identity + positional | -| `openadapt_flow/benchmark/desktop_benchmark.py` | Orchestrator: 3 arms × drift matrix, DB judge, results.json/BENCHMARK.md/chart | +| `benchmark/harness/desktop_benchmark.py` | Orchestrator: 3 arms × drift matrix, DB judge, results.json/BENCHMARK.md/chart | ## Snapshots (reversibility + warm boot) @@ -78,7 +78,7 @@ state without a full VM revert. ```bash # one-off full matrix (6 conditions × 3 arms × n): -python -m openadapt_flow.benchmark.desktop_benchmark --out benchmark/desktop --n 3 +python -m benchmark.harness.desktop_benchmark --out benchmark/desktop --n 3 ``` `DesktopHarness.connect()` starts the VM, launches the session-1 shim, deploys diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index bb52425ba8..9a1f71f1d6 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -5337,8 +5337,31 @@ def _cmd_silent_saves(args: argparse.Namespace) -> int: return int(silent_saves_main(list(args.silent_saves_argv))) +def _checkout_harness_on_path(command: str) -> bool: + """Put the checkout's ``benchmark`` harness on ``sys.path``. + + The harness lives in ``packages/flow/benchmark/harness``, outside the + package, so a wheel install has none and the command says so. + """ + checkout = Path(__file__).resolve().parents[1] + if not (checkout / "benchmark" / "harness" / "__init__.py").is_file(): + print( + f"{command} runs from a source checkout of OpenAdapt " + "(packages/flow/benchmark/harness); the wheel does not ship it.", + file=sys.stderr, + ) + return False + if str(checkout) not in sys.path: + sys.path.insert(0, str(checkout)) + return True + + def _cmd_benchmark(args: argparse.Namespace) -> int: - from openadapt_flow.benchmark.run_benchmark import run_benchmark + if not _checkout_harness_on_path("benchmark"): + return 2 + from benchmark.harness.run_benchmark import ( # type: ignore[import-not-found] + run_benchmark, + ) results = run_benchmark( Path(args.out), diff --git a/packages/flow/openadapt_flow/anthropic_defaults.py b/packages/flow/openadapt_flow/anthropic_defaults.py index 22ec8ce45b..fc4201c29e 100644 --- a/packages/flow/openadapt_flow/anthropic_defaults.py +++ b/packages/flow/openadapt_flow/anthropic_defaults.py @@ -1,6 +1,6 @@ """Anthropic defaults that product code shares with the benchmark harness. -These live here, not in ``openadapt_flow.benchmark``, because the product +These live here, not in ``benchmark.harness`` (the checkout's harness), because the product imports them and a frozen build leaves the benchmark package out (``tests/test_runtime_never_imports_benchmark.py`` enforces the rule). """ diff --git a/packages/flow/openadapt_flow/authoring.py b/packages/flow/openadapt_flow/authoring.py index a1c4e0c2a1..24a3df3402 100644 --- a/packages/flow/openadapt_flow/authoring.py +++ b/packages/flow/openadapt_flow/authoring.py @@ -804,7 +804,7 @@ def continue_input(self, *, operator_confirmed: bool = True) -> dict[str, Any]: "reason": "unconfirmed_masked_secret", } if not pause.secret and (text is None or not text): - # Non-secret MockMed note: do not persist an empty TYPE/param. + # Non-secret field left empty: do not persist an empty TYPE/param. return { "recorded": False, "param": pause.param, diff --git a/packages/flow/openadapt_flow/deployment.py b/packages/flow/openadapt_flow/deployment.py index e8b54023bd..a5f3d698e3 100644 --- a/packages/flow/openadapt_flow/deployment.py +++ b/packages/flow/openadapt_flow/deployment.py @@ -545,7 +545,7 @@ def _validate_secret_env(cls, v: dict[str, str]) -> dict[str, str]: if not isinstance(env_name, str) or not _ENV_VAR_NAME.fullmatch(env_name): raise ValueError( f"effects.secret_env[{slot!r}] must be the NAME of an " - "environment variable (for example OPENEMR_READER_PASSWORD), " + "environment variable (for example APP_READER_PASSWORD), " "never the credential itself" ) return v diff --git a/packages/flow/paper/REVIEW_ADVERSARIAL.md b/packages/flow/paper/REVIEW_ADVERSARIAL.md index 4ebe33f3c6..9e31c59679 100644 --- a/packages/flow/paper/REVIEW_ADVERSARIAL.md +++ b/packages/flow/paper/REVIEW_ADVERSARIAL.md @@ -35,7 +35,7 @@ Every quantitative result is at a sample size where inference is not possible, a This is my single most serious concern, and it is not disclosed in the paper. -- The headline "screen 50/90, effect 0/90" comes from `openadapt_flow/benchmark/silent_wrong_action.py`, **not** from the browser study. That harness does not drive the browser, does not run OCR, and does not invoke the `Replayer`. It issues raw `requests.post` calls that replicate what the app's JavaScript would send, encodes the screen oracle as a hardcoded rule over HTTP status codes, and calls `RestRecordVerifier.verify(...)` directly. +- The headline "screen 50/90, effect 0/90" comes from `benchmark/harness/silent_wrong_action.py`, **not** from the browser study. That harness does not drive the browser, does not run OCR, and does not invoke the `Replayer`. It issues raw `requests.post` calls that replicate what the app's JavaScript would send, encodes the screen oracle as a hardcoded rule over HTTP status codes, and calls `RestRecordVerifier.verify(...)` directly. - In that harness the **effect oracle and the ground-truth judge read the same in-process store**: the effect verifier reads `GET /api/db` and the ground-truth `_business_effect` reads `FaultDB.snapshot()`, which is the same `FaultDB`. Worse, the effect contract (`record_written` exactly once AND `field_equals` on the note) is essentially a restatement of the ground-truth definition of "correct." So "effect silent-wrong = 0 of 90" is guaranteed by construction: the oracle and the ground truth are computed from the same bytes with near-identical logic. The number is a property of the fixture, not an empirical finding. - The genuinely end-to-end harness (`benchmark/fault_model/run.py`, real Playwright + OCR + `Replayer`) measures **only the screen-only baseline**. Its own recommendations section describes effect verification as a *proposed* fix that the reference postcondition system does not yet implement ("The reference postcondition system is vision-only today, so on this corpus it provides no transactional-safety coverage"). The effect-verifier-through-the-replayer path for these fault classes exists only in unit tests (`tests/test_effect_fault_matrix.py`), and those tests also call `verifier.verify(...)` directly rather than driving the `Replayer`. - Consequently the Results sentence "those five classes refuted and halted **through the live replayer**" overstates what was measured. The live replayer never ran the effect verifier on these 90 runs. The suspiciously clean integers (exactly 50, exactly 0, exactly 10) are deterministic-fixture artifacts, not a measured rate. @@ -48,7 +48,7 @@ The seven fault classes are hand-picked transactional hazards (partial save, dup ### 3.4 Baseline fairness: same-backend, but a vanilla single-shot agent -The agent baseline (`openadapt_flow/benchmark/agent_baseline.py`) is not a strawman in the crude sense: it drives the same Playwright backend, uses a current computer-use model and tool, and is handed generous task information (credentials, exact patient, exact note text, a hint that the app is slow). But it is a **single-attempt, un-scaffolded** loop: no system prompt, no planning/reflection/ReAct, no whole-task retry, and hard caps (25 actions, $1.50, last-3-screenshots context). Modern agent results lean heavily on scaffolding and retries; comparing a compiled program against a vanilla single-shot agent inflates the latency and cost deltas. The paper's hedges are appropriate, but the abstract and introduction still lead with the comparison in a way that implies a capability gap the design does not license. At minimum the baseline's single-shot, no-scaffold nature belongs in the main text next to the comparison table, not only implicitly in the code. +The agent baseline (`benchmark/harness/agent_baseline.py`) is not a strawman in the crude sense: it drives the same Playwright backend, uses a current computer-use model and tool, and is handed generous task information (credentials, exact patient, exact note text, a hint that the app is slow). But it is a **single-attempt, un-scaffolded** loop: no system prompt, no planning/reflection/ReAct, no whole-task retry, and hard caps (25 actions, $1.50, last-3-screenshots context). Modern agent results lean heavily on scaffolding and retries; comparing a compiled program against a vanilla single-shot agent inflates the latency and cost deltas. The paper's hedges are appropriate, but the abstract and introduction still lead with the comparison in a way that implies a capability gap the design does not license. At minimum the baseline's single-shot, no-scaffold nature belongs in the main text next to the comparison table, not only implicitly in the code. ### 3.5 The strong-oracle claim is ambiguous exactly where it matters most (pixel-only substrates) diff --git a/packages/flow/pyproject.toml b/packages/flow/pyproject.toml index b63cfa7435..b4bedc3057 100644 --- a/packages/flow/pyproject.toml +++ b/packages/flow/pyproject.toml @@ -605,7 +605,7 @@ module = [ "openadapt_flow.__main__", "openadapt_flow.backends.playwright_backend", "openadapt_flow.backends.rdp_backend", - "openadapt_flow.benchmark.desktop_benchmark", + "benchmark.harness.desktop_benchmark", "openadapt_flow.compiler.disambiguation", "openadapt_flow.interactive_recorder", "openadapt_flow.learning.synth_stream", diff --git a/packages/flow/scripts/desktop/waa_shim.py b/packages/flow/scripts/desktop/waa_shim.py index a15e66b1ae..e0a877e4af 100644 --- a/packages/flow/scripts/desktop/waa_shim.py +++ b/packages/flow/scripts/desktop/waa_shim.py @@ -1,7 +1,7 @@ """In-guest WAA-contract HTTP shim (runs INSIDE the Windows VM). Serves the desktop benchmark harness -(``openadapt_flow.benchmark.desktop_benchmark``): +(``benchmark.harness.desktop_benchmark``): GET /screenshot -> raw PNG bytes of the desktop (Flask send_file) GET /health -> ``{"status": "ok", ...}`` liveness probe diff --git a/packages/flow/scripts/frappe_lending_demo.py b/packages/flow/scripts/frappe_lending_demo.py index 8681b128b4..13877e4925 100644 --- a/packages/flow/scripts/frappe_lending_demo.py +++ b/packages/flow/scripts/frappe_lending_demo.py @@ -42,8 +42,8 @@ from openadapt_flow.backends.playwright_backend import ( # noqa: E402 PlaywrightBackend as _BasePlaywrightBackend, ) -from openadapt_flow.benchmark import agent_baseline # noqa: E402 -from openadapt_flow.benchmark.frappe_lending import ( # noqa: E402 +from benchmark.harness import agent_baseline # noqa: E402 +from benchmark.harness.frappe_lending import ( # noqa: E402 CONDITIONS, INITIAL_TRIALS_PER_CELL, PUBLICATION_TRIALS_PER_CELL, diff --git a/packages/flow/scripts/openemr_demo.py b/packages/flow/scripts/openemr_demo.py index 6a91a699fe..b9a75aee21 100644 --- a/packages/flow/scripts/openemr_demo.py +++ b/packages/flow/scripts/openemr_demo.py @@ -19,7 +19,7 @@ Showcase artifacts land in docs/showcase-openemr/; ``benchmark`` records a fresh demonstration into a temporary directory and writes results to -benchmark/openemr/ (see openadapt_flow.benchmark.openemr_benchmark). +benchmark/openemr/ (see benchmark.harness.openemr_benchmark). """ from __future__ import annotations @@ -189,7 +189,7 @@ def _note_visible(final_png: bytes, note: str) -> bool: Delegates to the benchmark's shared arm-independent check (``verify_note_saved``) so the criterion is implemented exactly once. """ - from openadapt_flow.benchmark.verify import verify_note_saved + from benchmark.harness.verify import verify_note_saved return verify_note_saved(final_png, note).success @@ -280,15 +280,15 @@ def benchmark() -> None: (incremental, per run) plus results.json / BENCHMARK.md / latency_cost.png to benchmark/openemr/. - Hard cost guardrails (see openadapt_flow.benchmark.openemr_benchmark): + Hard cost guardrails (see benchmark.harness.openemr_benchmark): an API preflight before any spend, a $1.50 per-run cap, an $8.00 total agent-arm ceiling, and an abort on consecutive billing errors. All at list price — billed cost under the intro rate is lower. """ import tempfile - from openadapt_flow.benchmark import agent_baseline - from openadapt_flow.benchmark.openemr_benchmark import ( + from benchmark.harness import agent_baseline + from benchmark.harness.openemr_benchmark import ( MAX_TOTAL_COST_USD, run_openemr_benchmark, ) diff --git a/packages/flow/scripts/openemr_local_demo.py b/packages/flow/scripts/openemr_local_demo.py index d9d8438cd1..0a62f7a4d8 100644 --- a/packages/flow/scripts/openemr_local_demo.py +++ b/packages/flow/scripts/openemr_local_demo.py @@ -40,8 +40,8 @@ from openadapt_flow.backends.playwright_backend import ( # noqa: E402 PlaywrightBackend as _BasePlaywrightBackend, ) -from openadapt_flow.benchmark import agent_baseline # noqa: E402 -from openadapt_flow.benchmark.openemr_local import ( # noqa: E402 +from benchmark.harness import agent_baseline # noqa: E402 +from benchmark.harness.openemr_local import ( # noqa: E402 CONDITIONS, INITIAL_TRIALS_PER_CELL, PUBLICATION_TRIALS_PER_CELL, diff --git a/packages/flow/scripts/openemr_param_depth.py b/packages/flow/scripts/openemr_param_depth.py index 6afd55d641..5c77b4c57f 100644 --- a/packages/flow/scripts/openemr_param_depth.py +++ b/packages/flow/scripts/openemr_param_depth.py @@ -148,7 +148,7 @@ def compile_bundle() -> None: def _replay_one(label: str, params: dict[str, str], url: str) -> dict: """One fresh-browser compiled replay; returns a summary row.""" - from openadapt_flow.benchmark.verify import verify_note_saved + from benchmark.harness.verify import verify_note_saved from openadapt_flow.ir import Workflow from openadapt_flow.runtime.replayer import Replayer from openadapt_flow.vision.ocr import normalize_text, ocr diff --git a/packages/flow/tests/e2e/test_region_stable_theme_regression.py b/packages/flow/tests/e2e/test_region_stable_theme_regression.py index 4ce3990130..152450bc7a 100644 --- a/packages/flow/tests/e2e/test_region_stable_theme_regression.py +++ b/packages/flow/tests/e2e/test_region_stable_theme_regression.py @@ -13,7 +13,7 @@ import pytest from openadapt_flow.backends.playwright_backend import PlaywrightBackend -from openadapt_flow.benchmark.dom_arm import verify_final_state +from benchmark.harness.dom_arm import verify_final_state from openadapt_flow.compiler import compile_recording from openadapt_flow.demo_driver import record_triage_demo from openadapt_flow.ir import Workflow diff --git a/packages/flow/tests/test_benchmark.py b/packages/flow/tests/test_benchmark.py index 6f903b79c9..68dbac3e54 100644 --- a/packages/flow/tests/test_benchmark.py +++ b/packages/flow/tests/test_benchmark.py @@ -15,20 +15,20 @@ import numpy as np import pytest -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.agent_baseline import ( +from benchmark.harness import agent_baseline +from benchmark.harness.agent_baseline import ( _truncate_screenshots, compute_cost, load_api_key, run_agent, triage_task_prompt, ) -from openadapt_flow.benchmark.run_benchmark import ( +from benchmark.harness.run_benchmark import ( aggregate_results, render_markdown, write_outputs, ) -from openadapt_flow.benchmark.verify import verify_encounter_saved +from benchmark.harness.verify import verify_encounter_saved NOTE = "Follow-up in 2 weeks; BP recheck." @@ -521,7 +521,7 @@ def test_right_type_passes_at_every_note_length( # line. TestVerify above retains a real-OCR smoke for the benchmark's # actual 32-character note; these four cases pin the verifier logic. monkeypatch.setattr( - "openadapt_flow.benchmark.verify.ocr", + "benchmark.harness.verify.ocr", lambda _screen: [SimpleNamespace(text=f"Triage - {note[:60]}")], ) verdict = verify_encounter_saved(screen, note) @@ -542,7 +542,7 @@ def test_wrong_type_fails_at_every_note_length( f"Consult - {note[:60]}", ) monkeypatch.setattr( - "openadapt_flow.benchmark.verify.ocr", + "benchmark.harness.verify.ocr", lambda _screen: [SimpleNamespace(text=f"Consult - {note[:60]}")], ) verdict = verify_encounter_saved(screen, note) @@ -599,7 +599,7 @@ def test_bare_note_starting_with_known_type_is_not_a_row( # the note itself starts with a known categorical value. screen = self.make_screen(f"Encounter saved - {note[:40]}", note[:60]) monkeypatch.setattr( - "openadapt_flow.benchmark.verify.ocr", + "benchmark.harness.verify.ocr", lambda _screen: [SimpleNamespace(text=note[:60])], ) verdict = verify_encounter_saved(screen, note) diff --git a/packages/flow/tests/test_chart_fonts.py b/packages/flow/tests/test_chart_fonts.py index c54ceb2bc1..d40847a4da 100644 --- a/packages/flow/tests/test_chart_fonts.py +++ b/packages/flow/tests/test_chart_fonts.py @@ -2,7 +2,7 @@ The benchmark chart is cosmetic; a font-lookup failure must never fail the suite. These tests cover the two guarantees in -``openadapt_flow.benchmark.chart_fonts``: the bundled DejaVuSans font is +``benchmark.harness.chart_fonts``: the bundled DejaVuSans font is registered so ``findfont`` cannot miss, and ``safe_render`` swallows any rendering failure. """ @@ -12,7 +12,7 @@ from pathlib import Path from typing import Any -from openadapt_flow.benchmark.chart_fonts import ( +from benchmark.harness.chart_fonts import ( BUNDLED_FONT_NAME, configure_bundled_font, safe_render, diff --git a/packages/flow/tests/test_competitor_drift_instrument.py b/packages/flow/tests/test_competitor_drift_instrument.py index 984b79c792..ec52bd8589 100644 --- a/packages/flow/tests/test_competitor_drift_instrument.py +++ b/packages/flow/tests/test_competitor_drift_instrument.py @@ -22,7 +22,7 @@ import pytest -from openadapt_flow.instrument import ( +from benchmark.harness.instrument import ( AgentRunResult, ArchitectureClassError, CostGuard, @@ -35,7 +35,7 @@ ensure_architecture_class, run_instrument, ) -from openadapt_flow.instrument.competitor_drift import render_markdown, write_outputs +from benchmark.harness.instrument.competitor_drift import render_markdown, write_outputs # Fault classes a screen-only report is BLIND to: a wrong effect lands while # the "saved" banner still paints, so a screen-blind agent claims success. diff --git a/packages/flow/tests/test_desktop_benchmark.py b/packages/flow/tests/test_desktop_benchmark.py index 45d605ed10..f54425cae2 100644 --- a/packages/flow/tests/test_desktop_benchmark.py +++ b/packages/flow/tests/test_desktop_benchmark.py @@ -9,8 +9,8 @@ import json -from openadapt_flow.benchmark import desktop_benchmark as db -from openadapt_flow.benchmark.desktop_benchmark import ( +from benchmark.harness import desktop_benchmark as db +from benchmark.harness.desktop_benchmark import ( _classify, run_desktop_benchmark, ) diff --git a/packages/flow/tests/test_dom_arm.py b/packages/flow/tests/test_dom_arm.py index c19314c790..6dd3a6e4b4 100644 --- a/packages/flow/tests/test_dom_arm.py +++ b/packages/flow/tests/test_dom_arm.py @@ -15,7 +15,7 @@ import numpy as np import pytest -from openadapt_flow.benchmark.dom_arm import ( +from benchmark.harness.dom_arm import ( PERTURBATIONS, SCHEDULE, aggregate_dom_results, @@ -191,7 +191,7 @@ def test_perturbation_menu(self) -> None: ) def test_notes_distinct_per_arm_and_slot(self) -> None: - from openadapt_flow.benchmark.dom_arm import ARMS + from benchmark.harness.dom_arm import ARMS notes = { note_for_slot(arm, slot) @@ -262,7 +262,7 @@ def test_maintenance_is_dom_loud_break_on_drift_only(self) -> None: assert not needs_maintenance(disputed) def test_verification_dispute(self) -> None: - from openadapt_flow.benchmark.dom_arm import verification_dispute + from benchmark.harness.dom_arm import verification_dispute # DOM: every step ran, no error, judge said no -> dispute. assert verification_dispute( @@ -546,7 +546,7 @@ class TestVerifyFinalState: exists to measure.""" def test_dom_check_is_the_hybrid_check(self) -> None: - from openadapt_flow.benchmark.hybrid_benchmark import ( + from benchmark.harness.hybrid_benchmark import ( verify_hybrid_final, ) diff --git a/packages/flow/tests/test_effectbench.py b/packages/flow/tests/test_effectbench.py index 435364cdcd..19419163d9 100644 --- a/packages/flow/tests/test_effectbench.py +++ b/packages/flow/tests/test_effectbench.py @@ -26,7 +26,7 @@ build_reference_episodes, ) from benchmark.fault_model import faults as F -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, DivergenceCategory, Effect, @@ -44,7 +44,7 @@ summarize, wilson_interval, ) -from openadapt_flow.benchmark.effectbench.metrics import bootstrap_ci, rate +from benchmark.harness.effectbench.metrics import bootstrap_ci, rate # --------------------------------------------------------------------------- # 1. Reference re-expression regression (the headline result) @@ -334,7 +334,7 @@ def test_record_and_note_effects_are_reexported() -> None: def test_score_episode_end_to_end() -> None: """The runner orchestrator: capture pre-state, run the arm, verify, label.""" - from openadapt_flow.benchmark.effectbench import AgentReport, score_episode + from benchmark.harness.effectbench import AgentReport, score_episode box = {"acted": False} @@ -367,7 +367,7 @@ def run_action() -> AgentReport: def test_score_episode_raises_on_unreadable_oracle() -> None: - from openadapt_flow.benchmark.effectbench import AgentReport, score_episode + from benchmark.harness.effectbench import AgentReport, score_episode with pytest.raises(ValueError): score_episode( @@ -433,8 +433,8 @@ def test_summary_filters_by_arm(episodes: list[EpisodeRecord]) -> None: def test_episode_record_predicates_match_outcome() -> None: - from openadapt_flow.benchmark.effectbench import Verdict - from openadapt_flow.benchmark.effectbench.schema import OracleVerdict + from benchmark.harness.effectbench import Verdict + from benchmark.harness.effectbench.schema import OracleVerdict ep = EpisodeRecord( episode_id="e1", diff --git a/packages/flow/tests/test_effectbench_runner.py b/packages/flow/tests/test_effectbench_runner.py index 64cd934c61..8abe524670 100644 --- a/packages/flow/tests/test_effectbench_runner.py +++ b/packages/flow/tests/test_effectbench_runner.py @@ -20,7 +20,7 @@ import pytest -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, EffectKind, EpisodeRecord, @@ -30,7 +30,7 @@ SwerVariant, summarize, ) -from openadapt_flow.benchmark.effectbench.runner import ( +from benchmark.harness.effectbench.runner import ( LIVE_ARMS, CompilerArm, CompoundEffectVerifier, @@ -43,19 +43,19 @@ run_episode, run_matrix, ) -from openadapt_flow.benchmark.effectbench.runner.arms import AgentArm, ArmResult -from openadapt_flow.benchmark.effectbench.runner.baselines import ( +from benchmark.harness.effectbench.runner.arms import AgentArm, ArmResult +from benchmark.harness.effectbench.runner.baselines import ( SCAFFOLDED_ARMS, ClaudeComputerUseArm, ScaffoldNotWired, ) -from openadapt_flow.benchmark.effectbench.runner.harness import EpisodeEnv -from openadapt_flow.benchmark.effectbench.runner.reference_tasks import ( +from benchmark.harness.effectbench.runner.harness import EpisodeEnv +from benchmark.harness.effectbench.runner.reference_tasks import ( GOAL, TARGET, make_params, ) -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( DivergenceCategory, OracleChannel, OracleSpec, @@ -343,7 +343,7 @@ def test_run_matrix_skips_scaffolded_arms_by_default() -> None: def test_cli_dry_run_runs_green() -> None: - from openadapt_flow.benchmark.effectbench.runner.__main__ import run_dry_run + from benchmark.harness.effectbench.runner.__main__ import run_dry_run episodes, summaries = run_dry_run( trials=1, arm_names="compiler,screen_only", include_scaffolded=False diff --git a/packages/flow/tests/test_effectbench_standalone.py b/packages/flow/tests/test_effectbench_standalone.py index dc0fb92526..56ac997158 100644 --- a/packages/flow/tests/test_effectbench_standalone.py +++ b/packages/flow/tests/test_effectbench_standalone.py @@ -6,7 +6,7 @@ dependency). Because the standalone package vendors the effect MECHANISM (the typed ``Effect`` contract, the substrate-independent judge, the outcome classifier) that is otherwise defined in ``openadapt_flow.runtime.effects`` and -``openadapt_flow.benchmark.effectbench``, this test guards against the two copies +``benchmark.harness.effectbench``, this test guards against the two copies drifting: it asserts the standalone classifier and judge agree BYTE-FOR-BYTE with the engine on the reference inputs, and that the standalone reference reproduces the published headline. It also re-runs the standalone source-availability @@ -62,7 +62,7 @@ def test_classifier_parity_with_engine_over_the_full_truth_table(): """The standalone classifier equals the engine classifier on every input.""" import effectbench.oracle as sa - from openadapt_flow.benchmark.effectbench import oracle as eng + from benchmark.harness.effectbench import oracle as eng scoreable = ["CORRECT", "WRONG_PERSISTED", "ABSENT"] for reported, state_name, avail in itertools.product( @@ -86,7 +86,7 @@ def test_divergence_category_parity_with_engine(): """The public standalone taxonomy must not lag the engine copy.""" from effectbench.schema import DivergenceCategory as StandaloneCategory - from openadapt_flow.benchmark.effectbench.schema import ( + from benchmark.harness.effectbench.schema import ( DivergenceCategory as EngineCategory, ) diff --git a/packages/flow/tests/test_effectbench_task_pack.py b/packages/flow/tests/test_effectbench_task_pack.py index bed77414ca..ddb2ea85c0 100644 --- a/packages/flow/tests/test_effectbench_task_pack.py +++ b/packages/flow/tests/test_effectbench_task_pack.py @@ -36,8 +36,8 @@ from benchmark.effectbench.task_pack.audit import audit_mockmed_oracle from benchmark.effectbench.task_pack.driver import run_mockmed_pack from benchmark.effectbench.task_pack.mockmed_tasks import MOCKMED_TASKS -from openadapt_flow.benchmark.effectbench import OutcomeLabel, Substrate, summarize -from openadapt_flow.benchmark.effectbench.schema import DivergenceCategory +from benchmark.harness.effectbench import OutcomeLabel, Substrate, summarize +from benchmark.harness.effectbench.schema import DivergenceCategory from openadapt_flow.runtime.effects.sql import assert_read_only_sql MANIFEST_PATH = Path(__file__).resolve().parents[1] / ( diff --git a/packages/flow/tests/test_frappe_lending_benchmark.py b/packages/flow/tests/test_frappe_lending_benchmark.py index ee3f50c080..1737d2a80b 100644 --- a/packages/flow/tests/test_frappe_lending_benchmark.py +++ b/packages/flow/tests/test_frappe_lending_benchmark.py @@ -19,7 +19,7 @@ FrappeFixture, audit_table_deltas, ) -from openadapt_flow.benchmark.frappe_lending import ( +from benchmark.harness.frappe_lending import ( ARMS, CONDITIONS, FrappeLoanApplicationOracle, diff --git a/packages/flow/tests/test_hybrid_benchmark.py b/packages/flow/tests/test_hybrid_benchmark.py index 9779311f40..11ca0244ba 100644 --- a/packages/flow/tests/test_hybrid_benchmark.py +++ b/packages/flow/tests/test_hybrid_benchmark.py @@ -15,8 +15,8 @@ import numpy as np import pytest -from openadapt_flow.benchmark import agent_baseline, hybrid_benchmark -from openadapt_flow.benchmark.hybrid_benchmark import ( +from benchmark.harness import agent_baseline, hybrid_benchmark +from benchmark.harness.hybrid_benchmark import ( AGENT_SLOTS, DRIFT_TYPES, FALLBACK_MAX_ACTIONS, @@ -610,7 +610,7 @@ def fake_run_agent(backend, task, **kwargs): assert "no failing step recorded" in seen["task"] def test_failed_result_index_edge_cases(self) -> None: - from openadapt_flow.benchmark.hybrid_benchmark import ( + from benchmark.harness.hybrid_benchmark import ( _failed_result_index, ) @@ -978,7 +978,7 @@ def test_arms_see_identical_conditions_per_slot( the run helpers), not on the orchestrator-stamped condition labels — the labels come from the same schedule lookup the orchestrator uses to build the URLs, so asserting on them alone is circular.""" - from openadapt_flow.benchmark.hybrid_benchmark import ( + from benchmark.harness.hybrid_benchmark import ( AGENT_SLOTS, condition_url, ) diff --git a/packages/flow/tests/test_lending_swer.py b/packages/flow/tests/test_lending_swer.py index b09dabeeed..b787960357 100644 --- a/packages/flow/tests/test_lending_swer.py +++ b/packages/flow/tests/test_lending_swer.py @@ -22,7 +22,7 @@ from benchmark.lending_fault_model import faults as F from benchmark.lending_fault_model import ground_truth as G from benchmark.lending_fault_model import swer as S -from openadapt_flow.benchmark.effectbench import summarize +from benchmark.harness.effectbench import summarize from openadapt_flow.mockloan.fault_server import LedgerDB diff --git a/packages/flow/tests/test_openemr_benchmark.py b/packages/flow/tests/test_openemr_benchmark.py index b2f10ba067..f32a93db12 100644 --- a/packages/flow/tests/test_openemr_benchmark.py +++ b/packages/flow/tests/test_openemr_benchmark.py @@ -16,20 +16,20 @@ import cv2 import numpy as np -import openadapt_flow.benchmark.verify as benchmark_verify -from openadapt_flow.benchmark import agent_baseline, openemr_benchmark -from openadapt_flow.benchmark.agent_baseline import ( +import benchmark.harness.verify as benchmark_verify +from benchmark.harness import agent_baseline, openemr_benchmark +from benchmark.harness.agent_baseline import ( openemr_task_prompt, run_agent, ) -from openadapt_flow.benchmark.openemr_benchmark import ( +from benchmark.harness.openemr_benchmark import ( aggregate_openemr_results, note_for, render_openemr_markdown, run_openemr_benchmark, write_openemr_outputs, ) -from openadapt_flow.benchmark.verify import verify_note_saved +from benchmark.harness.verify import verify_note_saved from openadapt_flow.vision import OcrLine NOTE = "Insurance card copied and coverage verified by phone." @@ -848,7 +848,7 @@ def crashing_client(self) -> CrashingClient: ) def test_mid_run_crash_row_carries_partial_cost(self, monkeypatch: Any) -> None: - from openadapt_flow.benchmark.run_benchmark import _agent_run + from benchmark.harness.run_benchmark import _agent_run _fake_launch(monkeypatch, FakeBackend()) row = _agent_run("http://x", NOTE, client=self.crashing_client(), task="task") diff --git a/packages/flow/tests/test_openemr_local_benchmark.py b/packages/flow/tests/test_openemr_local_benchmark.py index e637aa076b..c67f0b6c79 100644 --- a/packages/flow/tests/test_openemr_local_benchmark.py +++ b/packages/flow/tests/test_openemr_local_benchmark.py @@ -22,7 +22,7 @@ audit_table_deltas, unexpected_table_deltas, ) -from openadapt_flow.benchmark.openemr_local import ( +from benchmark.harness.openemr_local import ( ARMS, CONDITIONS, OpenEMRPatientOracle, @@ -1011,7 +1011,7 @@ def __init__(self, messages: Messages) -> None: "max_tokens": 4096, } ledger = __import__( - "openadapt_flow.benchmark.agent_baseline", fromlist=["UsageLedger"] + "benchmark.harness.agent_baseline", fromlist=["UsageLedger"] ).UsageLedger() allowed_messages = Messages(10_000) diff --git a/packages/flow/tests/test_openemr_retained_finals.py b/packages/flow/tests/test_openemr_retained_finals.py index 8fbdc22904..c1b6709429 100644 --- a/packages/flow/tests/test_openemr_retained_finals.py +++ b/packages/flow/tests/test_openemr_retained_finals.py @@ -16,7 +16,7 @@ import pytest -from openadapt_flow.benchmark.verify import verify_note_saved +from benchmark.harness.verify import verify_note_saved REPO_ROOT = Path(__file__).resolve().parents[1] RUN_RETAINED_FINALS = os.environ.get("OPENADAPT_OPENEMR_FINALS_DIR") diff --git a/packages/flow/tests/test_reliability_harness.py b/packages/flow/tests/test_reliability_harness.py index 06be924f83..e3dade0d41 100644 --- a/packages/flow/tests/test_reliability_harness.py +++ b/packages/flow/tests/test_reliability_harness.py @@ -1,6 +1,6 @@ """Small synthetic checks for the public reliability aggregation mechanism.""" -from openadapt_flow.benchmark.reliability import _outcome, aggregate +from benchmark.harness.reliability import _outcome, aggregate def _result(outcome: str, *, category: str | None = None) -> dict: diff --git a/packages/flow/tests/test_runtime_never_imports_benchmark.py b/packages/flow/tests/test_runtime_never_imports_benchmark.py index dc4fa57ffb..d59f3083d7 100644 --- a/packages/flow/tests/test_runtime_never_imports_benchmark.py +++ b/packages/flow/tests/test_runtime_never_imports_benchmark.py @@ -1,9 +1,10 @@ -"""Product code never imports ``openadapt_flow.benchmark``. +"""Product code never imports the benchmark harness. -A frozen build (Desktop's engine) leaves the benchmark package out, so a -product module that imports it, even lazily, fails there. Shared names -(``MODEL``, ``load_api_key``, ``SCROLL_PX_PER_UNIT``) live in -``openadapt_flow.anthropic_defaults``. +The harness lives in the source checkout (``packages/flow/benchmark/harness``), +outside the ``openadapt_flow`` package, so neither the wheel nor a frozen build +(Desktop's engine) carries it. A product module that imports it, even lazily, +fails there. Shared names (``MODEL``, ``load_api_key``, ``SCROLL_PX_PER_UNIT``) +live in ``openadapt_flow.anthropic_defaults``. """ from __future__ import annotations @@ -17,57 +18,64 @@ #: Modules whose job is the benchmark itself. Nothing else may import it. ALLOWED = { - # The benchmark CLI command loads the harness on demand. + # The benchmark CLI command loads the harness on demand from a checkout. "__main__.py", - # The competitor-drift harness reuses the benchmark's fault scenarios. - "instrument/competitor_drift.py", + # `record-check prove` loads its fixture from a checkout on demand. + "record_check_proof.py", } +def _is_benchmark(module: str) -> bool: + return any( + module == root or module.startswith(root + ".") + for root in ("benchmark", "openadapt_flow.benchmark") + ) + + def _imports_benchmark(tree: ast.AST) -> list[int]: lines: list[int] = [] for node in ast.walk(tree): if isinstance(node, ast.ImportFrom): module = node.module or "" - if node.level == 0 and ( - module == "openadapt_flow.benchmark" - or module.startswith("openadapt_flow.benchmark.") - ): + if node.level == 0 and _is_benchmark(module): lines.append(node.lineno) elif node.level == 0 and module == "openadapt_flow": if any(a.name == "benchmark" for a in node.names): lines.append(node.lineno) - elif node.level > 0 and ( - module == "benchmark" or module.startswith("benchmark.") - ): + elif node.level > 0 and _is_benchmark(module): lines.append(node.lineno) elif isinstance(node, ast.Import): - for alias in node.names: - if alias.name == "openadapt_flow.benchmark" or alias.name.startswith( - "openadapt_flow.benchmark." - ): - lines.append(node.lineno) + if any(_is_benchmark(alias.name) for alias in node.names): + lines.append(node.lineno) return lines +def test_the_harness_is_not_inside_the_package() -> None: + assert not (ROOT / "benchmark").exists(), ( + "benchmark harnesses belong in packages/flow/benchmark/, outside the " + "shipped openadapt_flow package" + ) + + def test_product_modules_never_import_the_benchmark_package() -> None: offenders: list[str] = [] for path in sorted(ROOT.rglob("*.py")): rel = path.relative_to(ROOT).as_posix() - if rel.startswith("benchmark/") or rel in ALLOWED: + if rel in ALLOWED: continue tree = ast.parse(path.read_text(encoding="utf-8")) offenders += [f"{rel}:{n}" for n in _imports_benchmark(tree)] assert not offenders, ( - "product code imports openadapt_flow.benchmark (a frozen build leaves " - "it out); move the shared name to openadapt_flow.anthropic_defaults " - f"or a runtime module: {offenders}" + "product code imports the benchmark harness (the wheel and a frozen " + "build leave it out); move the shared name to " + "openadapt_flow.anthropic_defaults or a runtime module: " + f"{offenders}" ) def test_shared_anthropic_names_have_one_home() -> None: + from benchmark.harness import agent_baseline from openadapt_flow import anthropic_defaults - from openadapt_flow.benchmark import agent_baseline assert agent_baseline.MODEL is anthropic_defaults.MODEL assert agent_baseline.SCROLL_PX_PER_UNIT == anthropic_defaults.SCROLL_PX_PER_UNIT diff --git a/packages/flow/tests/test_silent_wrong_action_benchmark.py b/packages/flow/tests/test_silent_wrong_action_benchmark.py index a340eea1ae..cadfdbcacc 100644 --- a/packages/flow/tests/test_silent_wrong_action_benchmark.py +++ b/packages/flow/tests/test_silent_wrong_action_benchmark.py @@ -16,7 +16,7 @@ from __future__ import annotations -from openadapt_flow.benchmark.silent_wrong_action import ( +from benchmark.harness.silent_wrong_action import ( SCENARIOS, aggregate, render_markdown, diff --git a/packages/flow/tests/test_ui_dismiss_live_openemr.py b/packages/flow/tests/test_ui_dismiss_live_openemr.py index 1468c56f6d..3caceffee1 100644 --- a/packages/flow/tests/test_ui_dismiss_live_openemr.py +++ b/packages/flow/tests/test_ui_dismiss_live_openemr.py @@ -47,7 +47,7 @@ def _record(tmp_path: Path) -> tuple[Path, Any]: from benchmark.openemr_local.qualify_hooks import fixture - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec from openadapt_flow.recorder import Recorder from scripts.openemr_local_demo import ( FORM_FRAME_NAME, From 22de5f6cc62f22b1de455563614c6c9ed16b2120 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 13:04:32 -0500 Subject: [PATCH 02/22] Generality audit: link CONTRIBUTING without the guard PR's anchor --- packages/flow/docs/GENERALITY_AUDIT.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md index 7fb257ecdb..f8f4b51798 100644 --- a/packages/flow/docs/GENERALITY_AUDIT.md +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -6,8 +6,8 @@ Audited 2026-11-03 on `main` at `213f3d9fe`. Scope: everything under The rule: the engine has no code written for one application. What one application needs lives in data (`app_profiles/*.yaml`, generated contracts, deployment configs) or in test fixtures and benchmark harnesses outside the -shipped package. `scripts/check_engine_generality.py` enforces it in CI; see -[CONTRIBUTING.md](../CONTRIBUTING.md#app-knowledge-lives-in-data). +shipped package. A CI check (`scripts/check_engine_generality.py`) keeps it +that way; [CONTRIBUTING.md](../CONTRIBUTING.md) states the rule. ## Method From d10bcc30206f71db4ad548a910c1db5024dd1cb6 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 12:41:22 -0500 Subject: [PATCH 03/22] CI: fail when the engine names a specific application scripts/check_engine_generality.py scans openadapt_flow/ for the application names and app-specific identifiers in scripts/generality.toml. app_profiles/ is data and passes; OpenAdapt's demo modules may name the demo app; a comment line may give an example under '# example:'. Hits that predate the check are counted in generality_baseline.json, which only shrinks: a rise fails, and so does a fall the baseline doesn't record. Runs in the flow lint job. CONTRIBUTING documents the rule: app knowledge lives in profiles, contracts and fixtures. --- .github/workflows/packages.yml | 4 + packages/flow/CONTRIBUTING.md | 25 ++ .../flow/scripts/check_engine_generality.py | 178 +++++++++++ packages/flow/scripts/generality.toml | 65 ++++ .../flow/scripts/generality_baseline.json | 296 ++++++++++++++++++ .../tests/test_engine_generality_check.py | 97 ++++++ 6 files changed, 665 insertions(+) create mode 100644 packages/flow/scripts/check_engine_generality.py create mode 100644 packages/flow/scripts/generality.toml create mode 100644 packages/flow/scripts/generality_baseline.json create mode 100644 packages/flow/tests/test_engine_generality_check.py diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 9daecc8dc2..0c6328a009 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -348,6 +348,10 @@ jobs: run: python scripts/check_release_consistency.py - name: Ruff lint run: ruff check openadapt_flow + - name: No app-specific code in the engine + # App knowledge lives in profiles, contracts and fixtures (CONTRIBUTING: + # "App knowledge lives in data"); the denylist is scripts/generality.toml. + run: python scripts/check_engine_generality.py - name: Install uv (for the oldest-Python syntax check) uses: astral-sh/setup-uv@18d369b23da25a81e9458561248e1aafff6a70fa # v10.3.1 - name: Syntax at the oldest supported Python (3.10) diff --git a/packages/flow/CONTRIBUTING.md b/packages/flow/CONTRIBUTING.md index da76cb2625..8641a68f76 100644 --- a/packages/flow/CONTRIBUTING.md +++ b/packages/flow/CONTRIBUTING.md @@ -23,6 +23,7 @@ Python 3.10–3.12 are supported and exercised in CI. ruff check openadapt_flow # lint ruff format --check openadapt_flow # format (drop --check to auto-apply) mypy # type-check (config in pyproject.toml) +python scripts/check_engine_generality.py # no app names in the engine pytest -q # tests ``` @@ -174,6 +175,30 @@ surface from permissively licensed package artifacts. The release-consistency gate inspects the actual wheel and sdist. A source-tree notice alone is not sufficient. +## App knowledge lives in data + +The engine (`openadapt_flow/`) works the same way for every application. Never +write code, a default, a constant or a heuristic for one application. What one +application needs goes in data: + +- its bookkeeping writes: an app profile in `openadapt_flow/app_profiles/`, + loaded by name; +- its endpoints, tables, keys and field paths: the generated contract or the + deployment config (`deploy.yaml`), which the engine requires rather than + defaulting; +- tests and benchmarks for it: fixtures under `tests/` and harnesses under + `benchmark/`, outside the shipped package. + +`scripts/check_engine_generality.py` runs in CI and fails when engine source +names an application or one of its identifiers from the denylist in +`scripts/generality.toml`. Add new applications and identifiers to that list +when you meet them. A comment that only gives an example may name an +application on a line marked `# example:`. Hits that predate the check are +counted in `scripts/generality_baseline.json`, which only shrinks: when you +remove one, lower it with `--update-baseline`. OpenAdapt's own demo apps +(MockMed) may be named only in the demo modules listed in the denylist. The +audit behind the rule is `docs/GENERALITY_AUDIT.md`. + ## Safety-sensitive areas The identity gate, the resolution ladder, and the postcondition/halt logic are diff --git a/packages/flow/scripts/check_engine_generality.py b/packages/flow/scripts/check_engine_generality.py new file mode 100644 index 0000000000..1200d93154 --- /dev/null +++ b/packages/flow/scripts/check_engine_generality.py @@ -0,0 +1,178 @@ +"""Fail when the engine names a specific application. + +The engine (``openadapt_flow/``) has no code written for one application. What +one application needs lives in data: ``app_profiles/*.yaml``, generated +contracts, deployment configs, or test fixtures and benchmark harnesses outside +the package. ``generality.toml`` beside this script holds the denylist of +application names and app-specific identifiers, and the few modules allowed to +name OpenAdapt's own demo apps. + +A comment line may name an application as an example when it carries an +explicit ``# example:`` marker. Every other hit fails, unless it is in the +baseline: the hits that were already there when the check went live +(``docs/GENERALITY_AUDIT.md``). The baseline only shrinks. A file whose count +for a term rises fails; a file whose count falls must have its baseline entry +lowered in the same change (``--update-baseline`` writes it). + +Usage:: + + python scripts/check_engine_generality.py + python scripts/check_engine_generality.py --update-baseline +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from collections import Counter +from pathlib import Path + +import tomllib + +HERE = Path(__file__).resolve().parent +FLOW = HERE.parent +ENGINE = FLOW / "openadapt_flow" +CONFIG = HERE / "generality.toml" +BASELINE = HERE / "generality_baseline.json" + +#: Engine files that hold text: code and the data shipped with it. +SUFFIXES = {".py", ".js", ".html", ".css", ".json", ".yaml", ".yml", ".toml", ".txt"} + +EXAMPLE_MARKER = re.compile(r"#\s*example:", re.IGNORECASE) + + +def load_config(path: Path = CONFIG) -> dict: + return tomllib.loads(path.read_text(encoding="utf-8")) + + +def _term_patterns(config: dict) -> dict[str, re.Pattern[str]]: + terms: dict[str, re.Pattern[str]] = {} + for section in ("names", "identifiers"): + for term, pattern in config[section].items(): + terms[term] = re.compile(pattern, re.IGNORECASE) + return terms + + +def _allowed(rel: str, term: str, config: dict) -> bool: + for prefix in config["allow"]["paths"]: + if rel == prefix or rel.startswith(prefix.rstrip("/") + "/"): + return True + for prefix, terms in config["allow"].get("terms_in", {}).items(): + if ( + rel == prefix or rel.startswith(prefix.rstrip("/") + "/") + ) and term in terms: + return True + return False + + +def scan( + engine: Path = ENGINE, config: dict | None = None +) -> dict[str, dict[str, list[int]]]: + """Return ``{relative path: {term: [line numbers]}}`` for every hit.""" + config = config if config is not None else load_config() + terms = _term_patterns(config) + hits: dict[str, dict[str, list[int]]] = {} + for path in sorted(engine.rglob("*")): + if not path.is_file() or path.suffix not in SUFFIXES: + continue + if "__pycache__" in path.parts: + continue + rel = path.relative_to(engine).as_posix() + try: + text = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + for number, line in enumerate(text.splitlines(), start=1): + stripped = line.lstrip() + if stripped.startswith("#") and EXAMPLE_MARKER.search(stripped): + continue + for term, pattern in terms.items(): + if pattern.search(line) and not _allowed(rel, term, config): + hits.setdefault(rel, {}).setdefault(term, []).append(number) + return hits + + +def counts(hits: dict[str, dict[str, list[int]]]) -> dict[str, dict[str, int]]: + return { + rel: {term: len(lines) for term, lines in sorted(by_term.items())} + for rel, by_term in sorted(hits.items()) + } + + +def compare( + hits: dict[str, dict[str, list[int]]], baseline: dict[str, dict[str, int]] +) -> tuple[list[str], list[str]]: + """Return (new hits, stale baseline entries).""" + new: list[str] = [] + stale: list[str] = [] + seen = Counter() + for rel, by_term in hits.items(): + for term, lines in by_term.items(): + allowed = baseline.get(rel, {}).get(term, 0) + seen[(rel, term)] = len(lines) + if len(lines) > allowed: + where = ", ".join(str(n) for n in lines) + new.append( + f"openadapt_flow/{rel}: {term!r} on line(s) {where} " + f"({len(lines)} found, {allowed} in the baseline)" + ) + for rel, by_term in baseline.items(): + for term, allowed in by_term.items(): + if seen[(rel, term)] < allowed: + stale.append( + f"openadapt_flow/{rel}: {term!r} baseline {allowed}, " + f"now {seen[(rel, term)]}" + ) + return new, stale + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + parser.add_argument( + "--update-baseline", + action="store_true", + help="Lower the baseline to the current hits (it never grows)", + ) + args = parser.parse_args(argv) + + hits = scan() + baseline = json.loads(BASELINE.read_text(encoding="utf-8")) + new, stale = compare(hits, baseline) + + if args.update_baseline: + if new: + print("New hits cannot enter the baseline:", file=sys.stderr) + print("\n".join(f" {line}" for line in new), file=sys.stderr) + return 1 + BASELINE.write_text(json.dumps(counts(hits), indent=2) + "\n", encoding="utf-8") + print(f"Baseline lowered: {BASELINE.relative_to(FLOW)}") + return 0 + + if new: + print( + "The engine names a specific application. Put what one app needs " + "in data (app_profiles/*.yaml, a generated contract, a deployment " + "config) or in a fixture outside openadapt_flow/; see " + "CONTRIBUTING.md, 'App knowledge lives in data'. A comment that " + "only gives an example may carry '# example:'.", + file=sys.stderr, + ) + print("\n".join(f" {line}" for line in new), file=sys.stderr) + if stale: + print( + "The baseline is higher than the engine now; lower it with " + "`python scripts/check_engine_generality.py --update-baseline`:", + file=sys.stderr, + ) + print("\n".join(f" {line}" for line in stale), file=sys.stderr) + if new or stale: + return 1 + total = sum(n for by_term in baseline.values() for n in by_term.values()) + print(f"Engine generality OK ({total} baseline hits left)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/scripts/generality.toml b/packages/flow/scripts/generality.toml new file mode 100644 index 0000000000..7b5b261273 --- /dev/null +++ b/packages/flow/scripts/generality.toml @@ -0,0 +1,65 @@ +# Denylist for scripts/check_engine_generality.py. +# +# The engine (openadapt_flow/) must not name a specific application. Add a term +# here when a new application is qualified or a new app-specific identifier is +# found. Patterns are Python regular expressions, matched case-insensitively +# unless they turn that off with (?-i:...). + +[names] +openemr = '\bopen[ _-]?emr' +openimis = '\bopen[ _-]?imis' +medplum = '\bmedplum' +dolibarr = '\bdolibarr' +espocrm = '\bespo[ _-]?crm' +redmine = '\bredmine' +snipe-it = '\bsnipe[ _-]?it\b' +inventree = '\binventree' +kimai = '\bkimai' +openmrs = '\bopen[ _-]?mrs' +gitea = '\bgitea' +firefly-iii = '\bfirefly[ _-]?(iii|3)\b' +vikunja = '\bvikunja' +zammad = '\bzammad' +open-dental = '\bopen[ _-]?dental' +odoo = '\bodoo' +frappe = '\b(frappe|erpnext)' +patient-notes = '\bpatient[ _-]notes\b' +epic = '(?-i:\bEpic\b)' +cerner = '\bcerner' +stedi = '\bstedi\b' +textedit = '\btextedit' +notepad = '\bnotepad\b' +mockmed = '\bmock[ _-]?med' +mockloan = '\bmock[ _-]?loan' + +[identifiers] +# OpenEMR +token_main = '\btoken_main\b' +csrf_token_form = '\bcsrf_token_form\b' +clearpass = '\bclearpass\b' +pnotes = '\bpnotes\b' +patient_data = '\bpatient_data\b' +uuid_registry = '\buuid_registry\b' +openemr-fhir-path = '/apis/default/fhir' +# Frappe and Dolibarr table prefixes +frappe-table = '\btabLoan\b' +dolibarr-table = '\bllx_[a-z]' +# MockMed's REST shape +mockmed-records = '/api/db\b' +mockmed-write = '/api/encounter\b' + +[allow] +# Data, loaded generically: an app profile is where one app's bookkeeping lives. +paths = ["app_profiles/"] + +# OpenAdapt's own demo apps and the user-facing commands that serve them +# (tutorial, demo, demo-record, quickstart, `openadapt-agent serve` with no +# flags). They may name the demo app and its REST shape; nothing else may. +[allow.terms_in] +"mockmed/" = ["mockmed", "mockmed-records", "mockmed-write"] +"tutorial.py" = ["mockmed", "mockmed-records", "mockmed-write"] +"demo_proof.py" = ["mockmed", "mockmed-records", "mockmed-write"] +"demo_driver.py" = ["mockmed", "mockmed-records", "mockmed-write"] +"agent/sandbox.py" = ["mockmed", "mockmed-records", "mockmed-write"] +"agent/tutorial.py" = ["mockmed", "mockmed-records", "mockmed-write"] +"agent/example_app.py" = ["mockmed", "mockmed-records", "mockmed-write"] diff --git a/packages/flow/scripts/generality_baseline.json b/packages/flow/scripts/generality_baseline.json new file mode 100644 index 0000000000..63467d2bf5 --- /dev/null +++ b/packages/flow/scripts/generality_baseline.json @@ -0,0 +1,296 @@ +{ + "__main__.py": { + "mockmed": 38, + "textedit": 1 + }, + "agent/ledger_demo.py": { + "mockmed": 4 + }, + "api.py": { + "mockmed": 1 + }, + "backend.py": { + "mockmed-records": 1 + }, + "backends/factory.py": { + "textedit": 1 + }, + "backends/parallels_vm.py": { + "open-dental": 1 + }, + "backends/playwright_backend.py": { + "openemr": 1 + }, + "backends/win_agent/server.py": { + "notepad": 1 + }, + "cli_undo.py": { + "mockmed": 2 + }, + "cli_variants.py": { + "mockmed": 2 + }, + "compile_agent/code_action.py": { + "openemr": 1, + "token_main": 1 + }, + "compiler/binding_admission.py": { + "mockmed": 3, + "mockmed-records": 1, + "mockmed-write": 2 + }, + "compiler/compile.py": { + "mockmed": 3, + "mockmed-records": 1, + "openemr": 1 + }, + "compiler/effect_mining.py": { + "mockmed": 1, + "mockmed-records": 1 + }, + "compiler/fixup_passes.py": { + "openemr": 1 + }, + "compiler/local_edit.py": { + "mockmed": 1 + }, + "demo_driver.py": { + "mockloan": 5 + }, + "deployment.py": { + "mockmed": 2, + "mockmed-records": 2, + "openemr": 2, + "textedit": 1 + }, + "eligibility/client.py": { + "stedi": 16 + }, + "eligibility/payer_routes.yaml": { + "stedi": 3 + }, + "eligibility/waterfall.py": { + "stedi": 2 + }, + "ensure_record.py": { + "medplum": 3 + }, + "execute/dispatch.py": { + "mockmed": 7 + }, + "execute/registry.py": { + "mockmed": 18 + }, + "execution_profiles.py": { + "mockmed": 1 + }, + "fault_gate.py": { + "mockmed-records": 1, + "openemr": 1 + }, + "generate/demo_analysis.py": { + "redmine": 1 + }, + "generate/verify_contract.py": { + "dolibarr": 1, + "espocrm": 2, + "redmine": 2 + }, + "guard_demo.py": { + "mockmed": 5, + "mockmed-records": 2, + "mockmed-write": 1 + }, + "integrations/harness2/__init__.py": { + "mockmed": 2 + }, + "integrations/harness2/adapter.py": { + "mockmed": 2 + }, + "integrations/harness2/agent.py": { + "mockmed": 1 + }, + "integrations/harness2/apps.py": { + "mockmed": 13, + "mockmed-records": 1 + }, + "integrations/harness2/tasks.py": { + "mockmed": 24 + }, + "ir.py": { + "token_main": 1 + }, + "jit/authority.py": { + "textedit": 2 + }, + "jit/compile_agent.py": { + "textedit": 1 + }, + "jit/recorded.py": { + "mockmed": 1 + }, + "jit/signature.py": { + "mockmed": 1 + }, + "learning/synth_stream.py": { + "mockmed": 5 + }, + "mockloan/__init__.py": { + "mockloan": 2, + "mockmed": 2 + }, + "mockloan/fault_server.py": { + "mockloan": 6, + "mockmed-records": 2 + }, + "mockloan/server.py": { + "mockloan": 4 + }, + "mockloan/static/app.js": { + "mockloan": 3, + "mockmed": 1 + }, + "mockloan/static/index.html": { + "mockloan": 2 + }, + "mockloan/static/styles.css": { + "mockloan": 1, + "mockmed": 1 + }, + "network_capture.py": { + "clearpass": 1 + }, + "operator_profile.py": { + "openemr": 1 + }, + "policy_packs.py": { + "mockmed": 1 + }, + "qualification_dev_signer.py": { + "mockmed": 1 + }, + "qualification_oracle_gate.py": { + "mockmed": 6, + "mockmed-records": 2, + "mockmed-write": 1 + }, + "qualification_proposal.py": { + "mockmed": 1 + }, + "qualification_web_session.py": { + "openemr": 1, + "openimis": 1 + }, + "qualify/config.py": { + "openemr": 1 + }, + "record_check_binding.py": { + "openimis": 1 + }, + "record_check_identity.py": { + "openemr": 1 + }, + "record_check_proof.py": { + "mockmed": 8, + "mockmed-records": 1, + "mockmed-write": 1 + }, + "record_check_synthesis.py": { + "mockmed": 1 + }, + "registry.py": { + "openemr": 2 + }, + "report_redaction.py": { + "csrf_token_form": 3, + "openemr": 2, + "token_main": 5 + }, + "reward/models.py": { + "mockmed-records": 1 + }, + "reward/seed.py": { + "mockmed": 33 + }, + "risk.py": { + "openemr": 1, + "openimis": 1 + }, + "rollback_state.py": { + "openemr": 1 + }, + "runtime/actuators/web_form.py": { + "openemr": 2 + }, + "runtime/effects/__init__.py": { + "frappe": 1, + "mockmed": 1, + "mockmed-records": 1, + "openemr": 1 + }, + "runtime/effects/_common.py": { + "openemr": 1 + }, + "runtime/effects/auth.py": { + "frappe": 1 + }, + "runtime/effects/compensation.py": { + "mockmed-write": 2 + }, + "runtime/effects/db_snapshot.py": { + "dolibarr": 1, + "dolibarr-table": 1 + }, + "runtime/effects/effect.py": { + "mockmed": 2, + "openemr": 5, + "redmine": 1 + }, + "runtime/effects/fhir.py": { + "epic": 1, + "mockmed": 2, + "openemr": 8, + "openemr-fhir-path": 2 + }, + "runtime/effects/frame.py": { + "openemr": 7, + "patient_data": 1, + "pnotes": 1, + "uuid_registry": 1 + }, + "runtime/effects/fresh_session.py": { + "mockmed": 1 + }, + "runtime/effects/rest.py": { + "mockmed": 1, + "mockmed-records": 2, + "openemr": 1 + }, + "runtime/effects/sql.py": { + "frappe": 7, + "frappe-table": 1, + "openemr": 1 + }, + "runtime/identity.py": { + "mockmed": 1 + }, + "runtime/replayer.py": { + "mockmed": 1 + }, + "submit_binding.py": { + "openimis": 1 + }, + "validation/dense_surface.py": { + "mockmed": 1, + "openemr": 2 + }, + "verifier.py": { + "openemr": 1 + }, + "vision/hashing.py": { + "mockmed": 1 + }, + "volatility.py": { + "openemr": 3 + } +} diff --git a/packages/flow/tests/test_engine_generality_check.py b/packages/flow/tests/test_engine_generality_check.py new file mode 100644 index 0000000000..40f0ac17c5 --- /dev/null +++ b/packages/flow/tests/test_engine_generality_check.py @@ -0,0 +1,97 @@ +"""The engine-generality check: app names fail, examples and data pass.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "check_engine_generality.py" + + +@pytest.fixture(scope="module") +def check(): + spec = importlib.util.spec_from_file_location("check_engine_generality", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _engine(tmp_path: Path, files: dict[str, str]) -> Path: + for rel, text in files.items(): + path = tmp_path / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return tmp_path + + +def test_a_name_in_code_is_a_hit(check, tmp_path: Path) -> None: + engine = _engine(tmp_path, {"runtime/x.py": 'if app == "OpenEMR":\n pass\n'}) + assert check.scan(engine) == {"runtime/x.py": {"openemr": [1]}} + + +def test_an_identifier_without_a_name_is_a_hit(check, tmp_path: Path) -> None: + engine = _engine(tmp_path, {"report.py": 'SECRET = {"token_main"}\n'}) + assert check.scan(engine) == {"report.py": {"token_main": [1]}} + + +def test_a_marked_example_comment_passes(check, tmp_path: Path) -> None: + engine = _engine( + tmp_path, + { + "a.py": "# example: OpenEMR keys rows by uuid\nx = 1\n", + # Without the marker the same comment is a hit. + "b.py": "# OpenEMR keys rows by uuid\nx = 1\n", + # The marker only counts on a comment line, not after code. + "c.py": 'x = "OpenEMR" # example: no\n', + }, + ) + assert check.scan(engine) == {"b.py": {"openemr": [1]}, "c.py": {"openemr": [1]}} + + +def test_app_profiles_are_data(check, tmp_path: Path) -> None: + engine = _engine(tmp_path, {"app_profiles/openemr.yaml": "app: openemr\n"}) + assert check.scan(engine) == {} + + +def test_the_demo_app_is_named_only_in_demo_modules(check, tmp_path: Path) -> None: + engine = _engine( + tmp_path, + { + "tutorial.py": 'URL = "/api/db" # the MockMed demo\n', + "runtime/effects/rest.py": 'records_path = "/api/db"\n', + # The demo modules may name the demo app, not a customer app. + "demo_proof.py": "# OpenEMR\n", + }, + ) + assert check.scan(engine) == { + "runtime/effects/rest.py": {"mockmed-records": [1]}, + "demo_proof.py": {"openemr": [1]}, + } + + +def test_epic_matches_the_product_not_the_word(check, tmp_path: Path) -> None: + engine = _engine( + tmp_path, {"a.py": "# an epic refactor\n", "b.py": "# Epic sends\n"} + ) + assert check.scan(engine) == {"b.py": {"epic": [1]}} + + +def test_the_baseline_only_shrinks(check) -> None: + hits = {"a.py": {"openemr": [3, 9]}, "b.py": {"medplum": [1]}} + baseline = {"a.py": {"openemr": 2}, "c.py": {"redmine": 1}} + + new, stale = check.compare(hits, baseline) + + assert len(new) == 1 and "b.py" in new[0] and "medplum" in new[0] + assert len(stale) == 1 and "c.py" in stale[0] + + +def test_the_engine_passes_against_its_baseline(check) -> None: + baseline = json.loads(check.BASELINE.read_text(encoding="utf-8")) + new, stale = check.compare(check.scan(), baseline) + assert not new, "\n".join(new) + assert not stale, "\n".join(stale) From 83300a021c6b5c98655ba39e662dead4ddd50b0d Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 13:48:49 -0500 Subject: [PATCH 04/22] Register the generality denylist and baseline as public artifacts --- packages/flow/public-artifacts.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 17b27ecdb2..84e66c296b 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -571,6 +571,8 @@ "schemas/sanitization-ingest-v1.json", "schemas/sanitized-artifact-approval-v1.json", "schemas/sanitized-artifact-manifest-v1.json", + "scripts/generality.toml", + "scripts/generality_baseline.json", "source-policy.public.json", "tests/golden/tutorial_run.json" ] From 31e5a27b4492c92cad5f6b2179da9c5e62797f54 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 13:51:00 -0500 Subject: [PATCH 05/22] Verifier: no app's REST paths or key field as engine defaults RestRecordVerifier, RestCompensator, fault_gate's memory copy, effects.kind rest and rest reward recipes defaulted to MockMed's /api/db, records and /api/encounter/{id}. The paths are now required; a rest config without records_path is refused when its verifier is built. Compile-time REST binding admission guessed MockMed's write path /api/encounter and skipped its server-assigned 'source' field. The AdmissionFixture now names the app's read path, write path, server-assigned fields and idempotency key field; with no captured api_write and no write path, a step gets no proposal. compile_recording takes admission_fixture instead of admission_fixture_url. Effect mining and scaffold-verifier mined an idempotency key from any record field named 'key' (MockMed's). They mine one only from the field the caller names. --- .../flow/benchmark/openemr_e2e/harness.py | 2 +- packages/flow/benchmark/undo_proposal/run.py | 8 +- packages/flow/docs/EFFECT_KIT.md | 2 + .../benchmark/silent_wrong_action.py | 2 +- .../compiler/binding_admission.py | 53 ++++++++---- .../flow/openadapt_flow/compiler/compile.py | 45 ++++++---- .../openadapt_flow/compiler/effect_mining.py | 32 ++++--- packages/flow/openadapt_flow/deployment.py | 12 ++- packages/flow/openadapt_flow/fault_gate.py | 4 +- .../flow/openadapt_flow/fleet/simulator.py | 6 +- .../instrument/competitor_drift.py | 6 +- .../flow/openadapt_flow/qualify/session.py | 5 +- packages/flow/openadapt_flow/reward/models.py | 6 +- .../flow/openadapt_flow/reward/oracles.py | 2 +- .../runtime/effects/compensation.py | 4 +- .../openadapt_flow/runtime/effects/effect.py | 6 +- .../openadapt_flow/runtime/effects/rest.py | 9 +- .../flow/openadapt_flow/scaffold_verifier.py | 11 +-- .../flow/tests/test_after_stop_mockmed.py | 20 +++-- packages/flow/tests/test_api_step_approval.py | 4 +- packages/flow/tests/test_api_step_durable.py | 5 +- .../tests/test_api_step_readback_repair.py | 12 ++- packages/flow/tests/test_api_tier_web_form.py | 8 +- .../tests/test_certified_binding_admission.py | 83 ++++++++++++++++++- packages/flow/tests/test_compose_runtime.py | 4 +- .../tests/test_decision_healthy_replay.py | 4 +- .../flow/tests/test_effect_fault_matrix.py | 10 +-- packages/flow/tests/test_effect_kit_config.py | 35 +++++++- packages/flow/tests/test_effect_mining.py | 12 ++- packages/flow/tests/test_effect_verifier.py | 30 ++++--- packages/flow/tests/test_fault_gate.py | 2 +- packages/flow/tests/test_loop_authoring.py | 12 ++- .../flow/tests/test_openemr_e2e_harness.py | 4 +- packages/flow/tests/test_readback_pipeline.py | 9 +- .../flow/tests/test_replayer_api_actuator.py | 36 ++++++-- .../test_replayer_effect_param_binding.py | 8 +- packages/flow/tests/test_replayer_effects.py | 48 +++++++---- .../tests/test_rest_verifier_transport.py | 15 +++- .../flow/tests/test_reward_trust_boundary.py | 13 +++ packages/flow/tests/test_rollback.py | 6 +- packages/flow/tests/test_rollback_qualify.py | 4 +- packages/flow/tests/test_sealed_at_rest.py | 4 +- .../flow/tests/test_ui_readback_verifier.py | 4 +- packages/flow/tests/test_undo_proposal.py | 16 +++- packages/flow/tests/test_variant_router.py | 4 +- .../tests/test_verifier_adapter_platform.py | 27 ++++-- 46 files changed, 495 insertions(+), 159 deletions(-) diff --git a/packages/flow/benchmark/openemr_e2e/harness.py b/packages/flow/benchmark/openemr_e2e/harness.py index 23a10fc498..418ab01544 100644 --- a/packages/flow/benchmark/openemr_e2e/harness.py +++ b/packages/flow/benchmark/openemr_e2e/harness.py @@ -231,7 +231,7 @@ def _replay( replayer = Replayer( backend, vision=vision, - effect_verifier=RestRecordVerifier(url.rstrip("/")), + effect_verifier=RestRecordVerifier(url.rstrip("/"), records_path="/api/db", records_key="records"), poll_interval_s=0.01, ) report = replayer.run( diff --git a/packages/flow/benchmark/undo_proposal/run.py b/packages/flow/benchmark/undo_proposal/run.py index 2dd6127ddd..03a5c4acfb 100644 --- a/packages/flow/benchmark/undo_proposal/run.py +++ b/packages/flow/benchmark/undo_proposal/run.py @@ -193,7 +193,7 @@ def _contract(directory: Path, base_url: str, fixture: Fixture, body: dict) -> P write_bundle( directory, contract_id=f"undo_propose_{fixture.name}", - oracle={"kind": "rest", "base_url": base_url}, + oracle={"kind": "rest", "base_url": base_url, "records_path": "/api/db"}, channel="api", key=Ed25519PrivateKey.generate(), issuer_key_id="self_signed:benchmark", @@ -218,7 +218,11 @@ def _deployment(path: Path, fixture: Fixture, base_url: str, contract: Path) -> { "name": f"{fixture.name}-test-copy", "backend": {"kind": "web", "url": base_url + "/" + fixture.query}, - "effects": {"kind": "rest", "base_url": base_url}, + "effects": { + "kind": "rest", + "base_url": base_url, + "records_path": "/api/db", + }, "actuation": {"api": True, "base_url": base_url}, "test_copy": { "declared": True, diff --git a/packages/flow/docs/EFFECT_KIT.md b/packages/flow/docs/EFFECT_KIT.md index 3739752c74..442559a1ba 100644 --- a/packages/flow/docs/EFFECT_KIT.md +++ b/packages/flow/docs/EFFECT_KIT.md @@ -1014,6 +1014,8 @@ Two kit-wide conventions: ```yaml effects: kind: rest + base_url: https://app.example + records_path: /api/records # required: the app's own records read auth: bearer_env: SOR_BEARER_TOKEN # or header+value_env, or basic_env ``` diff --git a/packages/flow/openadapt_flow/benchmark/silent_wrong_action.py b/packages/flow/openadapt_flow/benchmark/silent_wrong_action.py index 009bdda54c..4e34b4c7c4 100644 --- a/packages/flow/openadapt_flow/benchmark/silent_wrong_action.py +++ b/packages/flow/openadapt_flow/benchmark/silent_wrong_action.py @@ -322,7 +322,7 @@ def run_one(base: str, db: Any, scenario: Scenario) -> dict[str, Any]: db.reset(seed_concurrent=scenario.seed_concurrent) before_snapshot = db.snapshot()["records"] - verifier = RestRecordVerifier(base) + verifier = RestRecordVerifier(base, records_path="/api/db", records_key="records") before_state = verifier.capture_pre_state() statuses = _drive(base, scenario) diff --git a/packages/flow/openadapt_flow/compiler/binding_admission.py b/packages/flow/openadapt_flow/compiler/binding_admission.py index 416eafd6c9..6c2f092903 100644 --- a/packages/flow/openadapt_flow/compiler/binding_admission.py +++ b/packages/flow/openadapt_flow/compiler/binding_admission.py @@ -16,8 +16,8 @@ is REST only. A placeholder effect is not a proposal: there is no real endpoint in the recording, and we will not invent one (:mod:`compiler.effect_mining`). -2. **Admit.** Execute the proposal against a *held-out* fixture (MockMed's - in-process REST store is the built-in one) with the same +2. **Admit.** Execute the proposal against a *held-out* fixture (a REST + store whose read and write paths the caller names) with the same :class:`~openadapt_flow.runtime.effects.EffectVerifier` that gates replay. The binding is copied onto the step only when every proposed :class:`~openadapt_flow.runtime.effects.Effect` CONFIRMs. Anything else @@ -68,9 +68,9 @@ #: Event key for an observed REST write (method / url_template / body_template). API_WRITE_KEY = "api_write" -#: Record fields assigned by MockMed (and typical REST stores) at write time, -#: never sent in the POST body and never used as a proposal identity. -_BODY_SKIP_FIELDS = SURROGATE_ID_FIELDS | frozenset({"source"}) +#: Record fields every REST store assigns at write time: never sent in the +#: POST body. An app's own server-assigned fields come from the fixture. +_BODY_SKIP_FIELDS = SURROGATE_ID_FIELDS #: The only ``ApiBinding.kind`` this module will propose or admit. _REST_KIND: Literal["rest"] = "rest" @@ -111,26 +111,38 @@ class AdmissionDecision: class AdmissionFixture: """Held-out REST system of record used to certify a proposed binding. - MockMed (``openadapt_flow.mockmed.fault_server``) is the built-in fixture: - ``POST /api/encounter`` writes, ``GET /api/db`` is the verifier read. + Every app keeps its records at its own paths, so the caller names them: + ``records_path`` / ``records_key`` are the verifier read, ``write_path`` + the write a proposal makes when the recording captured no ``api_write`` + (unset: such a step gets no proposal). ``server_fields`` are record fields + the app assigns itself, never sent in a proposed body; + ``idempotency_key_field`` is the field carrying its at-most-once key. """ base_url: str - write_path: str = "/api/encounter" + records_path: str + records_key: Optional[str] + write_path: Optional[str] = None write_method: str = "POST" + server_fields: frozenset[str] = frozenset() + idempotency_key_field: Optional[str] = None params: dict[str, str] = field(default_factory=dict) timeout_s: float = 2.0 session: Any = None def _param_placeholders( - record: dict[str, Any], params: dict[str, str] + record: dict[str, Any], + params: dict[str, str], + skip: frozenset[str] = frozenset(), ) -> dict[str, str]: """Map observed record fields to ``{param}`` leaves or literal strings.""" inverse = {str(value): name for name, value in params.items() if value} body: dict[str, str] = {} for key, value in record.items(): - if key in _BODY_SKIP_FIELDS or value is None or str(value) == "": + if key in _BODY_SKIP_FIELDS or key in skip: + continue + if value is None or str(value) == "": continue text = str(value) if text in inverse: @@ -180,7 +192,12 @@ def propose_rest_binding( effects = list(step.effects) source = "step.effects" if not effects or any(eff.needs_operator_confirmation for eff in effects): - mined = mine_step_effects(event, step, exclude_texts=exclude_texts) + mined = mine_step_effects( + event, + step, + exclude_texts=exclude_texts, + idempotency_key_field=fixture.idempotency_key_field, + ) if not mined.derived: logger.debug( "binding-propose %s: no derived SoR effect (%s)", @@ -199,11 +216,12 @@ def propose_rest_binding( return None method = fixture.write_method - url_template = fixture.write_path + url_template: Optional[str] = fixture.write_path body: dict[str, Any] = {} if captured is not None: method = str(captured.get("method") or method).upper() - url_template = str(captured.get("url_template") or url_template) + captured_url = captured.get("url_template") + url_template = str(captured_url) if captured_url else url_template raw_body = captured.get("body_template", captured.get("body")) if isinstance(raw_body, dict): body = dict(raw_body) @@ -216,7 +234,7 @@ def propose_rest_binding( new_records = _new_records(before, after) if len(new_records) != 1: return None - body = _param_placeholders(new_records[0], params) + body = _param_placeholders(new_records[0], params, fixture.server_fields) source = "sor_delta" if not url_template or not body: @@ -269,7 +287,12 @@ def admit_rest_binding( proposal=proposal, ) - verifier = RestRecordVerifier(fixture.base_url, timeout_s=fixture.timeout_s) + verifier = RestRecordVerifier( + fixture.base_url, + records_path=fixture.records_path, + records_key=fixture.records_key, + timeout_s=fixture.timeout_s, + ) actuator = ApiActuator( fixture.base_url, session=fixture.session, diff --git a/packages/flow/openadapt_flow/compiler/compile.py b/packages/flow/openadapt_flow/compiler/compile.py index bd385d5e2e..4a4ef0a13e 100644 --- a/packages/flow/openadapt_flow/compiler/compile.py +++ b/packages/flow/openadapt_flow/compiler/compile.py @@ -8,6 +8,7 @@ from __future__ import annotations +import dataclasses import difflib import hashlib import json @@ -30,6 +31,7 @@ if TYPE_CHECKING: from openadapt_flow.compiler.annotate import StepAnnotator + from openadapt_flow.compiler.binding_admission import AdmissionFixture import numpy as np @@ -2078,7 +2080,7 @@ def _compile_recording( secret_param_steps: Optional[Iterable[str]] = None, mine_effects: bool = False, admit_api_bindings: bool = False, - admission_fixture_url: Optional[str] = None, + admission_fixture: Optional["AdmissionFixture"] = None, skill_library_root: Optional[Path | str] = None, annotate: bool = False, annotator: Optional["StepAnnotator"] = None, @@ -2177,13 +2179,15 @@ def _compile_recording( consequential write whose Effect was mined from an observed system-of-record delta, propose a REST binding and copy it onto the step only if the same Effect CONFIRMs on the held-out - fixture at ``admission_fixture_url``. A refused proposal leaves + ``admission_fixture``. A refused proposal leaves the step on the GUI ladder. FHIR / MCP / tool bindings are not synthesized. Default False keeps the bundle byte-identical. - admission_fixture_url: Base URL of the held-out REST fixture - (MockMed ``fault_server`` in tests). Required when - ``admit_api_bindings`` is True; without it the compiler logs - and leaves every step on the GUI ladder. + admission_fixture: The held-out REST fixture + (:class:`~openadapt_flow.compiler.binding_admission.AdmissionFixture`: + base URL, and the app's own read and write paths). Required when + ``admit_api_bindings`` is True; without it the compiler logs and + leaves every step on the GUI ladder. Its ``params`` default to the + recording's. skill_library_root: Optional :class:`~openadapt_flow.learning.library.SkillLibrary` root. When a binding is admitted, the lifted program is stored with @@ -3357,15 +3361,14 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: if admit_api_bindings: from openadapt_flow.compiler.binding_admission import ( - AdmissionFixture, certify_steps_on_fixture, ) from openadapt_flow.learning.library import SkillLibrary - if not admission_fixture_url: + if admission_fixture is None: logger.info( - "admit_api_bindings requested but admission_fixture_url is " - "empty -- leaving every step on the GUI ladder" + "admit_api_bindings requested but admission_fixture is " + "unset -- leaving every step on the GUI ladder" ) else: library = ( @@ -3376,9 +3379,10 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: decisions = certify_steps_on_fixture( steps, [event for _step, _sb, _sa, event in pending], - fixture=AdmissionFixture( - base_url=admission_fixture_url, - params=dict(params), + fixture=( + admission_fixture + if admission_fixture.params + else dataclasses.replace(admission_fixture, params=dict(params)) ), exclude_texts=exclude_texts, params=dict(params), @@ -3470,7 +3474,9 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: ), "mine_effects": mine_effects, "admit_api_bindings": admit_api_bindings, - "admission_fixture_url": admission_fixture_url, + "admission_fixture_url": ( + admission_fixture.base_url if admission_fixture is not None else None + ), "name": name, "param_overrides": dict(sorted((param_overrides or {}).items())), "risk_overrides": dict(sorted((risk_overrides or {}).items())), @@ -3480,6 +3486,17 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: if bind_effect_params: # Recorded only when set, so every existing compile keeps its digest. compiler_config["bind_effect_params"] = True + if admission_fixture is not None: + # Added only when set, so a compile without one hashes as before. + admission: dict[str, Any] = { + "records_path": admission_fixture.records_path, + "records_key": admission_fixture.records_key, + "write_path": admission_fixture.write_path, + "write_method": admission_fixture.write_method, + "server_fields": sorted(admission_fixture.server_fields), + "idempotency_key_field": admission_fixture.idempotency_key_field, + } + compiler_config["admission_fixture"] = admission compiler_config_sha256 = hashlib.sha256( json.dumps( compiler_config, diff --git a/packages/flow/openadapt_flow/compiler/effect_mining.py b/packages/flow/openadapt_flow/compiler/effect_mining.py index d16b9d26c8..ae7733e1da 100644 --- a/packages/flow/openadapt_flow/compiler/effect_mining.py +++ b/packages/flow/openadapt_flow/compiler/effect_mining.py @@ -96,11 +96,6 @@ def _vemap(selector: dict[str, str]) -> dict[str, ValueExpr]: # prune, rather than guess which app fields are "incidental". SURROGATE_ID_FIELDS = frozenset({"id"}) -# Field that commonly carries an at-most-once / idempotency key in a JSON -# system of record (the MockMed fault server uses ``key``). Only mined when the -# observed record actually carries a non-null value there — never invented. -IDEMPOTENCY_KEY_FIELD = "key" - # Sentinel selector for a PLACEHOLDER effect: a value no real record can carry, # so even if a misconfigured run tried to verify it (it must not — the replayer # HALTs on needs_operator_confirmation first), it could never falsely CONFIRM. @@ -288,6 +283,7 @@ def _mine_from_sor_delta( after: list[dict], exclude_texts: tuple[str, ...], param_bindings: Optional[Mapping[str, str]] = None, + idempotency_key_field: Optional[str] = None, ) -> Optional[StepEffectMining]: """Mine real effects from an OBSERVED before/after system-of-record delta. @@ -342,12 +338,12 @@ def _mine_from_sor_delta( "system-of-record delta)" ), ) - # Idempotency key: derivable ONLY when the observed record actually carries - # one. Never invented — §7 names the idempotency key as app-specific. - key_val = record.get(IDEMPOTENCY_KEY_FIELD) - if key_val is not None and str(key_val) != "": + # Idempotency key: only from the field the caller names for this app (§7: + # the key is app-specific), and only when the observed record carries it. + key_val = record.get(idempotency_key_field) if idempotency_key_field else None + if idempotency_key_field and key_val is not None and str(key_val) != "": written.idempotency_key = _ve(str(key_val)) - written.key_field = IDEMPOTENCY_KEY_FIELD + written.key_field = idempotency_key_field effects.append(written) # field_equals per payload field: the typed value actually persisted (the @@ -380,7 +376,7 @@ def _mine_from_sor_delta( f"derived {len(effects)} effect(s) from an observed system-of-record " f"delta (1 new record; selector {selector}" + ( - f"; idempotency key from field {IDEMPOTENCY_KEY_FIELD!r}" + f"; idempotency key from field {idempotency_key_field!r}" if written.idempotency_key is not None else "; no idempotency key observed" ) @@ -900,6 +896,7 @@ def mine_step_effects( param_bindings: Optional[Mapping[str, str]] = None, store_unchanged: Optional[bool] = None, write_requests: Optional[bool] = None, + idempotency_key_field: Optional[str] = None, ) -> StepEffectMining: """Derive candidate system-of-record effects for one compiled step. @@ -944,6 +941,10 @@ def mine_step_effects( ``None`` when the recording had no network capture, ``False`` when the capture showed no write request. Only used to word the reason. + idempotency_key_field: The record field that carries this app's + at-most-once key, from the contract or deployment config. Set, + and present on the observed record, the mined ``record_written`` + counts records bearing it. ``None``: no key is mined. Returns: A :class:`StepEffectMining` describing the decision. ``effects`` is @@ -952,7 +953,14 @@ def mine_step_effects( before = _as_records(event.get(SOR_BEFORE_KEY)) after = _as_records(event.get(SOR_AFTER_KEY)) if before is not None and after is not None: - mined = _mine_from_sor_delta(step, before, after, exclude_texts, param_bindings) + mined = _mine_from_sor_delta( + step, + before, + after, + exclude_texts, + param_bindings, + idempotency_key_field=idempotency_key_field, + ) if mined is not None: return mined # A snapshot existed but showed no single-record write. When the diff --git a/packages/flow/openadapt_flow/deployment.py b/packages/flow/openadapt_flow/deployment.py index e8b54023bd..7aeabc22d6 100644 --- a/packages/flow/openadapt_flow/deployment.py +++ b/packages/flow/openadapt_flow/deployment.py @@ -384,9 +384,11 @@ class EffectsConfig(BaseModel): #: ``runtime/effects/fresh_session.py`` and ``docs/READBACK_LADDER.md``. fresh_session: Optional[FreshSessionSpec] = None - # -- rest (JSON REST system of record, e.g. MockMed /api/db) ------------- + # -- rest (JSON REST system of record) ----------------------------------- base_url: Optional[str] = None - records_path: str = "/api/db" + #: ``rest`` only, and required there: the path whose GET returns the + #: records. Every app keeps them somewhere else, so there is no default. + records_path: Optional[str] = None records_key: Optional[str] = "records" #: ``rest`` only: dotted path in the response body to the next page's URL #: (``links.next``). Set, the read follows every page on the same origin @@ -627,6 +629,7 @@ def _serialize_compatible(self, handler: Any) -> dict[str, Any]: "deferred_report", "records_next_path", "records_total_path", + "records_path", ): if data.get(added, "") is None: data.pop(added, None) @@ -2223,6 +2226,11 @@ def _build_effect_verifier_unredacted( if kind == "rest": if not cfg.base_url: raise ValueError("effects.kind 'rest' requires effects.base_url") + if not cfg.records_path: + raise ValueError( + "effects.kind 'rest' requires effects.records_path (the path " + "whose GET returns the app's records)" + ) from openadapt_flow.runtime.effects import RestRecordVerifier headers = cfg.auth.resolve_headers() if cfg.auth is not None else None diff --git a/packages/flow/openadapt_flow/fault_gate.py b/packages/flow/openadapt_flow/fault_gate.py index c07e5e2332..3d275ca4ca 100644 --- a/packages/flow/openadapt_flow/fault_gate.py +++ b/packages/flow/openadapt_flow/fault_gate.py @@ -719,8 +719,8 @@ def __init__( seed: Sequence[Mapping[str, Any]], target: Mapping[str, Any], *, - records_path: str = "/api/db", - records_key: Optional[str] = "records", + records_path: str, + records_key: Optional[str], key_field: str = "id", ) -> None: self.key_field = key_field diff --git a/packages/flow/openadapt_flow/fleet/simulator.py b/packages/flow/openadapt_flow/fleet/simulator.py index 642544ea26..a59e096c8d 100644 --- a/packages/flow/openadapt_flow/fleet/simulator.py +++ b/packages/flow/openadapt_flow/fleet/simulator.py @@ -621,7 +621,11 @@ def replay( ) workflow = Workflow.load(bundle_dir) verifier = RestRecordVerifier( - "http://sor.sim", session=_RecordSession(sor), poll_interval_s=0.01 + "http://sor.sim", + records_path="/records", + records_key="records", + session=_RecordSession(sor), + poll_interval_s=0.01, ) gate = evaluate_run_gate( workflow, diff --git a/packages/flow/openadapt_flow/instrument/competitor_drift.py b/packages/flow/openadapt_flow/instrument/competitor_drift.py index 3daa5076c6..a6f861e04e 100644 --- a/packages/flow/openadapt_flow/instrument/competitor_drift.py +++ b/packages/flow/openadapt_flow/instrument/competitor_drift.py @@ -416,7 +416,9 @@ def run_task( self, task: DriftTask, target_url: str, *, max_steps: int ) -> AgentRunResult: base = self._base_of(target_url) - verifier = RestRecordVerifier(base) + verifier = RestRecordVerifier( + base, records_path="/api/db", records_key="records" + ) # Observe the system of record BEFORE acting (an agent may inspect # state around its own action) so the honest policy can detect # collateral loss, not just its own row. @@ -736,7 +738,7 @@ def _run_one( db.reset(seed_concurrent=task.scenario.seed_concurrent) before_snapshot = db.snapshot()["records"] - verifier = RestRecordVerifier(base) + verifier = RestRecordVerifier(base, records_path="/api/db", records_key="records") before_state = verifier.capture_pre_state() target_url = f"{base}/?fault={task.scenario.name}" diff --git a/packages/flow/openadapt_flow/qualify/session.py b/packages/flow/openadapt_flow/qualify/session.py index 1d66b53c59..e21d3a396e 100644 --- a/packages/flow/openadapt_flow/qualify/session.py +++ b/packages/flow/openadapt_flow/qualify/session.py @@ -144,11 +144,14 @@ def verifier_factory(config: VerifyConfig) -> Callable[[Mapping[str, object]], A if not cfg.base_url: raise QualifyConfigError("verifier.base_url is required for kind: rest") + if not cfg.records_path: + raise QualifyConfigError("verifier.records_path is required for kind: rest") + records_path = cfg.records_path def build_rest(params: Mapping[str, object]) -> Any: return RestRecordVerifier( cfg.base_url.rstrip("/"), # type: ignore[union-attr] - records_path=cfg.records_path, + records_path=records_path, records_key=cfg.records_key, session=hook(config), timeout_s=10.0, diff --git a/packages/flow/openadapt_flow/reward/models.py b/packages/flow/openadapt_flow/reward/models.py index 6223f843f0..8cac4bf805 100644 --- a/packages/flow/openadapt_flow/reward/models.py +++ b/packages/flow/openadapt_flow/reward/models.py @@ -222,8 +222,8 @@ class OracleRecipeV1(_Strict): records_key: Optional[StrictStr] = "records" #: ``rest`` / ``fhir``: base URL of the read endpoint. base_url: Optional[StrictStr] = None - #: ``rest``: path of the records document. - records_path: StrictStr = "/api/db" + #: ``rest``: path of the records document. Required there. + records_path: Optional[StrictStr] = None #: ``rest``: environment variable holding a JSON object of headers. headers_env: Optional[StrictStr] = None #: ``sqlite``: database file. ``query``: one read-only SELECT. @@ -245,6 +245,8 @@ def _kind_fields(self) -> "OracleRecipeV1": raise ValueError(f"oracle recipe {self.kind} requires path") if self.kind in {"rest", "fhir"} and not self.base_url: raise ValueError(f"oracle recipe {self.kind} requires base_url") + if self.kind == "rest" and not self.records_path: + raise ValueError("oracle recipe rest requires records_path") if self.kind == "sqlite" and not self.query: raise ValueError("oracle recipe sqlite requires query") return self diff --git a/packages/flow/openadapt_flow/reward/oracles.py b/packages/flow/openadapt_flow/reward/oracles.py index 20cdb85221..f4e6194d77 100644 --- a/packages/flow/openadapt_flow/reward/oracles.py +++ b/packages/flow/openadapt_flow/reward/oracles.py @@ -218,7 +218,7 @@ def _adapter_for(recipe: OracleRecipeV1, base_dir: Path) -> OracleAdapter: return VerifierOracle( RestRecordVerifier( str(recipe.base_url), - records_path=recipe.records_path, + records_path=str(recipe.records_path), records_key=recipe.records_key, headers=recipe.headers(), timeout_s=recipe.timeout_s, diff --git a/packages/flow/openadapt_flow/runtime/effects/compensation.py b/packages/flow/openadapt_flow/runtime/effects/compensation.py index 1572ff6184..9f1dcbf757 100644 --- a/packages/flow/openadapt_flow/runtime/effects/compensation.py +++ b/packages/flow/openadapt_flow/runtime/effects/compensation.py @@ -352,7 +352,7 @@ class RestCompensator: Args: base_url: Base URL of the system of record. delete_path_template: Path template with ``{id}`` for the per-record - DELETE (e.g. ``/api/encounter/{id}``). + DELETE (e.g. ``/records/{id}``). Required. session: Optional ``requests``-style session. timeout_s: Per-request timeout. """ @@ -361,7 +361,7 @@ def __init__( self, base_url: str, *, - delete_path_template: str = "/api/encounter/{id}", + delete_path_template: str, session: Any = None, timeout_s: float = 5.0, ) -> None: diff --git a/packages/flow/openadapt_flow/runtime/effects/effect.py b/packages/flow/openadapt_flow/runtime/effects/effect.py index f236588e8d..a25cbd46e0 100644 --- a/packages/flow/openadapt_flow/runtime/effects/effect.py +++ b/packages/flow/openadapt_flow/runtime/effects/effect.py @@ -896,8 +896,10 @@ class Effect(BaseModel): #: identity) so the key is PER-RUN, not the frozen demonstration literal #: that would collide across unrelated runs (P0-3). idempotency_key: Optional[ValueExpr] = None - #: Which record field carries the idempotency key (substrate-specific; - #: the MockMed system-of-record uses ``"key"``). + #: Which record field carries the idempotency key. App-specific: the + #: contract names it beside ``idempotency_key`` (the effect miner sets it + #: only from the field its caller names). The default is kept so effects + #: dumped before this note hash the same. key_field: str = "key" #: ``record_written`` only: a duplicate (idempotency) key made of several #: fields, such as OpenEMR's ``{lname, fname, DOB}``. Field -> value, each diff --git a/packages/flow/openadapt_flow/runtime/effects/rest.py b/packages/flow/openadapt_flow/runtime/effects/rest.py index 390d7ce6dc..4a02941934 100644 --- a/packages/flow/openadapt_flow/runtime/effects/rest.py +++ b/packages/flow/openadapt_flow/runtime/effects/rest.py @@ -74,9 +74,10 @@ class RestRecordVerifier(VerifierAdapterBase): Args: base_url: Base URL of the system of record (trailing slash optional). - records_path: Path whose GET returns the records document. + records_path: Path whose GET returns the records document. Required: + every app keeps its records somewhere else. records_key: Key in the JSON body holding the records list; when the - body is itself a JSON list, pass ``None``. + body is itself a JSON list, pass ``None``. Required. session: Optional ``requests``-style session (injectable for tests / auth headers); a module-level default is used when omitted. headers: Optional headers sent with every read (the kit's @@ -111,8 +112,8 @@ def __init__( self, base_url: str, *, - records_path: str = "/api/db", - records_key: Optional[str] = "records", + records_path: str, + records_key: Optional[str], session: Any = None, headers: Optional[dict[str, str]] = None, timeout_s: float = 5.0, diff --git a/packages/flow/openadapt_flow/scaffold_verifier.py b/packages/flow/openadapt_flow/scaffold_verifier.py index 200b9b8050..d7d052fc48 100644 --- a/packages/flow/openadapt_flow/scaffold_verifier.py +++ b/packages/flow/openadapt_flow/scaffold_verifier.py @@ -161,6 +161,7 @@ def _candidate_from_record( record: dict[str, Any], *, demonstrated_values: tuple[str, ...], + idempotency_key_field: Optional[str] = None, ) -> WriteCandidate: """Split one observed new record into identity vs payload fields. @@ -169,7 +170,6 @@ def _candidate_from_record( exclusion, same selector rules, same demonstrated-value payload split). """ from openadapt_flow.compiler.effect_mining import ( - IDEMPOTENCY_KEY_FIELD, _match_selector, _param_value_fields, ) @@ -177,13 +177,14 @@ def _candidate_from_record( payload = _param_value_fields(record, demonstrated_values) selector = _match_selector(record, set(payload)) idempotency = None - raw_key = record.get(IDEMPOTENCY_KEY_FIELD) - if raw_key not in (None, ""): - idempotency = (IDEMPOTENCY_KEY_FIELD, str(raw_key)) + # The app's at-most-once key field, when the caller names one. + raw_key = record.get(idempotency_key_field) if idempotency_key_field else None + if idempotency_key_field and raw_key not in (None, ""): + idempotency = (idempotency_key_field, str(raw_key)) # The at-most-once key is surfaced separately (it must be bound to a # per-run param, never the frozen demo literal), so it never doubles # as an identity selector. - selector.pop(IDEMPOTENCY_KEY_FIELD, None) + selector.pop(idempotency_key_field, None) return WriteCandidate( step_id=step_id, action=action, diff --git a/packages/flow/tests/test_after_stop_mockmed.py b/packages/flow/tests/test_after_stop_mockmed.py index 248e764269..d03dc33ac5 100644 --- a/packages/flow/tests/test_after_stop_mockmed.py +++ b/packages/flow/tests/test_after_stop_mockmed.py @@ -186,7 +186,9 @@ def _halted_run(tmp_path, url, *, with_undo=True): replayer = Replayer( backend, vision=_vision(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) run_dir = tmp_path / "run" @@ -199,7 +201,9 @@ def _undo_replayer(url): replayer = Replayer( backend, vision=_vision(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) return replayer, backend @@ -317,7 +321,9 @@ def press(self, key): replayer = Replayer( DeleteIgnored(mockmed), vision=_vision(), - effect_verifier=RestRecordVerifier(mockmed), + effect_verifier=RestRecordVerifier( + mockmed, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) result = run_undo( @@ -607,7 +613,9 @@ def test_compiler_probe_text_never_reaches_the_ledger(tmp_path, mockmed): replayer = Replayer( MockMedBackend(mockmed), vision=_vision(), - effect_verifier=RestRecordVerifier(mockmed), + effect_verifier=RestRecordVerifier( + mockmed, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) report = replayer.run( @@ -731,7 +739,9 @@ def test_a_second_undo_declared_after_the_stop_still_matches(tmp_path, mockmed): replayer = Replayer( MockMedBackend(mockmed), vision=_vision(), - effect_verifier=RestRecordVerifier(mockmed), + effect_verifier=RestRecordVerifier( + mockmed, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) run_dir = tmp_path / "run" diff --git a/packages/flow/tests/test_api_step_approval.py b/packages/flow/tests/test_api_step_approval.py index 7fcdeee8be..09d727187c 100644 --- a/packages/flow/tests/test_api_step_approval.py +++ b/packages/flow/tests/test_api_step_approval.py @@ -163,7 +163,9 @@ def _run(tmp_path, person: Person, *, transport_setup=None): report = Replayer( backend, vision=FakeVision(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), web_form_actuator=WebFormActuator(transport), approval_gate=gate, poll_interval_s=0.01, diff --git a/packages/flow/tests/test_api_step_durable.py b/packages/flow/tests/test_api_step_durable.py index 7e7d9ebf27..7fa248d1bd 100644 --- a/packages/flow/tests/test_api_step_durable.py +++ b/packages/flow/tests/test_api_step_durable.py @@ -148,7 +148,10 @@ def replayer( return Replayer( self.backend, vision=FakeVision(), - effect_verifier=verifier or RestRecordVerifier(verifier_url or self.url), + effect_verifier=verifier + or RestRecordVerifier( + verifier_url or self.url, records_path="/api/db", records_key="records" + ), web_form_actuator=actuator, poll_interval_s=0.01, durable=True, diff --git a/packages/flow/tests/test_api_step_readback_repair.py b/packages/flow/tests/test_api_step_readback_repair.py index ea566f0c32..a5dec0ca33 100644 --- a/packages/flow/tests/test_api_step_readback_repair.py +++ b/packages/flow/tests/test_api_step_readback_repair.py @@ -43,7 +43,9 @@ def _run(tmp_path, *, pipeline, durable=False, verifier_url=None): report = Replayer( EnterWritesBackend(url), vision=FakeVision(), - effect_verifier=RestRecordVerifier(verifier_url or url), + effect_verifier=RestRecordVerifier( + verifier_url or url, records_path="/api/db", records_key="records" + ), web_form_actuator=actuator, poll_interval_s=0.01, readback_pipeline=pipeline, @@ -112,7 +114,9 @@ def test_a_probe_or_resumed_run_never_sends_the_api_step(tmp_path): backend = EnterWritesBackend(url) target = ReplayTarget( lambda _fault: _Session(backend), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), policy="permissive", profile="demo", replayer_kwargs={ @@ -146,7 +150,9 @@ def test_a_standard_change_test_record_run_is_gui_only(tmp_path): backend = EnterWritesBackend(url) target = ReplayTarget( lambda _fault: _Session(backend), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), policy="permissive", profile="demo", replayer_kwargs={ diff --git a/packages/flow/tests/test_api_tier_web_form.py b/packages/flow/tests/test_api_tier_web_form.py index c45f0f6643..23ba1df02b 100644 --- a/packages/flow/tests/test_api_tier_web_form.py +++ b/packages/flow/tests/test_api_tier_web_form.py @@ -584,7 +584,9 @@ def _run(tmp_path, wf, status, **kw): report = Replayer( backend, vision=FakeVision(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), web_form_actuator=actuator, poll_interval_s=0.01, **kw, @@ -720,7 +722,9 @@ def test_an_uncertain_save_needs_reconciliation_and_is_never_redone(tmp_path): report = Replayer( backend, vision=FakeVision(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), web_form_actuator=actuator, poll_interval_s=0.01, ).run(_span_workflow(), bundle_dir=bundle, run_dir=tmp_path / "run") diff --git a/packages/flow/tests/test_certified_binding_admission.py b/packages/flow/tests/test_certified_binding_admission.py index 480b11df92..be662b0b7f 100644 --- a/packages/flow/tests/test_certified_binding_admission.py +++ b/packages/flow/tests/test_certified_binding_admission.py @@ -100,8 +100,13 @@ def _derived_event(record=None) -> dict: def _fixture(url: str, **over) -> AdmissionFixture: + # MockMed's own shape, named here: the engine assumes no app's paths. kw = dict( base_url=url, + records_path="/api/db", + records_key="records", + write_path="/api/encounter", + server_fields=frozenset({"source"}), params={"note": NOTE}, timeout_s=2.0, ) @@ -144,6 +149,68 @@ def test_propose_rest_binding_from_observed_sor_delta(): assert binding.on_unavailable == "gui" +def test_propose_needs_the_apps_write_path_when_none_was_captured(): + """No captured api_write and no write path named: no guessed endpoint.""" + fixture = AdmissionFixture( + base_url="http://held-out.example", + records_path="/api/db", + records_key="records", + params={"note": NOTE}, + ) + step = _save_step() + assert ( + propose_rest_binding( + step, _derived_event(), fixture=fixture, exclude_texts=(NOTE,) + ) + is None + ) + + +def test_propose_uses_another_apps_captured_write_and_server_fields(): + """A Frappe-shaped write: the captured endpoint, the app's own fields.""" + step = _save_step() + event = _derived_event() + event[API_WRITE_KEY] = { + "kind": "rest", + "method": "POST", + "url_template": "/api/resource/Loan Application", + "body": {"applicant": "p1", "description": "{note}"}, + } + fixture = AdmissionFixture( + base_url="http://held-out.example", + records_path="/api/resource/Loan Application", + records_key="data", + server_fields=frozenset({"owner", "creation", "modified"}), + params={"note": NOTE}, + ) + proposal = propose_rest_binding( + step, event, fixture=fixture, exclude_texts=(NOTE,), params={"note": NOTE} + ) + assert proposal is not None + assert proposal.binding.url_template == "/api/resource/Loan Application" + assert proposal.binding.body_template == { + "applicant": "p1", + "description": "{note}", + } + + # Without a capture, the body comes from the new record minus the fields + # this app assigns itself. + fixture = AdmissionFixture( + base_url="http://held-out.example", + records_path="/api/resource/Loan Application", + records_key="data", + write_path="/api/resource/Loan Application", + server_fields=frozenset({"source"}), + params={"note": NOTE}, + ) + proposal = propose_rest_binding( + step, _derived_event(), fixture=fixture, exclude_texts=(NOTE,) + ) + assert proposal is not None + assert "source" not in proposal.binding.body_template + assert "id" not in proposal.binding.body_template + + def test_propose_refuses_fhir_mcp_tool_captures(): step = _save_step() for kind in ("fhir", "mcp", "tool"): @@ -253,7 +320,9 @@ def test_refused_admission_leaves_gui_ladder(tmp_path): report = Replayer( backend, vision=_vision_that_confirms_saved(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ).run(_save_workflow(step), bundle_dir=bundle, run_dir=run_dir) @@ -339,7 +408,15 @@ def test_admit_requires_actuated_then_confirmed(): # build the request. Nothing is written; the step stays GUI-only. decision = admit_rest_binding( proposal, - fixture=AdmissionFixture(base_url=url, params={}, timeout_s=0.5), + fixture=AdmissionFixture( + records_path="/api/db", + records_key="records", + write_path="/api/encounter", + server_fields=frozenset({"source"}), + base_url=url, + params={}, + timeout_s=0.5, + ), ) assert decision.admitted is False assert decision.actuation is not None @@ -393,7 +470,7 @@ def test_compile_admits_rest_binding_on_held_out_fixture(tmp_path): name="certified-demo", risk_overrides={"step_001": "irreversible"}, admit_api_bindings=True, - admission_fixture_url=url, + admission_fixture=_fixture(url), skill_library_root=tmp_path / "skills", ) save = workflow.steps[1] diff --git a/packages/flow/tests/test_compose_runtime.py b/packages/flow/tests/test_compose_runtime.py index 33d1eee6b0..0df6c75cf7 100644 --- a/packages/flow/tests/test_compose_runtime.py +++ b/packages/flow/tests/test_compose_runtime.py @@ -230,7 +230,9 @@ def child_run( report = Replayer( backend, vision=_vision_confirms_saved(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ).run( workflow, diff --git a/packages/flow/tests/test_decision_healthy_replay.py b/packages/flow/tests/test_decision_healthy_replay.py index c8e4d036e7..6a03c1ee0c 100644 --- a/packages/flow/tests/test_decision_healthy_replay.py +++ b/packages/flow/tests/test_decision_healthy_replay.py @@ -54,7 +54,9 @@ def _replay(tmp_path: Path): replayer = Replayer( WritingBackend(url), vision=_vision_that_confirms_saved(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) return replayer.run(workflow, bundle_dir=bundle, run_dir=run_dir) diff --git a/packages/flow/tests/test_effect_fault_matrix.py b/packages/flow/tests/test_effect_fault_matrix.py index 4fc18606ed..756ec4b9e2 100644 --- a/packages/flow/tests/test_effect_fault_matrix.py +++ b/packages/flow/tests/test_effect_fault_matrix.py @@ -125,7 +125,7 @@ def test_screen_passes_but_effect_catches(sor, mode): seed = mode == "stale" db.reset(seed_concurrent=seed) - verifier = RestRecordVerifier(base) + verifier = RestRecordVerifier(base, records_path="/api/db", records_key="records") before = verifier.capture_pre_state() assert before.reachable @@ -148,7 +148,7 @@ def test_screen_passes_but_effect_catches(sor, mode): def test_control_clean_write_both_agree(sor): base, db = sor db.reset() - verifier = RestRecordVerifier(base) + verifier = RestRecordVerifier(base, records_path="/api/db", records_key="records") before = verifier.capture_pre_state() _drive_persistence_boundary(base, db, "ok") assert SCREEN_SHOWS_SUCCESS["ok"] is True @@ -161,7 +161,7 @@ def test_idempotency_key_neutralizes_duplicate(sor): # refuted -- proving the at-most-once check. base, db = sor db.reset() - verifier = RestRecordVerifier(base) + verifier = RestRecordVerifier(base, records_path="/api/db", records_key="records") before = verifier.capture_pre_state() _drive_persistence_boundary(base, db, "idempotent") written = Effect( @@ -181,7 +181,7 @@ def test_timeout_effect_verify_is_more_correct_than_screen(sor): # would double-write. base, db = sor db.reset() - verifier = RestRecordVerifier(base) + verifier = RestRecordVerifier(base, records_path="/api/db", records_key="records") before = verifier.capture_pre_state() _drive_persistence_boundary(base, db, "timeout") assert SCREEN_SHOWS_SUCCESS["timeout"] is False # screen: false-abort @@ -199,7 +199,7 @@ def test_session_expiry_both_safe(sor): # no banner -> the screen halts, and effect-verify agrees nothing landed. base, db = sor db.reset() - verifier = RestRecordVerifier(base) + verifier = RestRecordVerifier(base, records_path="/api/db", records_key="records") before = verifier.capture_pre_state() _drive_persistence_boundary(base, db, "session") assert SCREEN_SHOWS_SUCCESS["session"] is False diff --git a/packages/flow/tests/test_effect_kit_config.py b/packages/flow/tests/test_effect_kit_config.py index 4a43a4bdbd..cd1bb44304 100644 --- a/packages/flow/tests/test_effect_kit_config.py +++ b/packages/flow/tests/test_effect_kit_config.py @@ -114,7 +114,11 @@ def json(): with_auth = _Session() RestRecordVerifier( - "http://x", session=with_auth, headers={"Authorization": "Bearer t"} + "http://x", + records_path="/api/db", + records_key="records", + session=with_auth, + headers={"Authorization": "Bearer t"}, ).capture_pre_state() assert with_auth.calls == [{"Authorization": "Bearer t"}] @@ -125,7 +129,9 @@ def get(self, url, *, timeout): # no headers kwarg at all return _Session.get(self, url, timeout=timeout) narrow = _NarrowSession() - state = RestRecordVerifier("http://x", session=narrow).capture_pre_state() + state = RestRecordVerifier( + "http://x", records_path="/api/db", records_key="records", session=narrow + ).capture_pre_state() assert state.reachable assert "narrow" in narrow.calls @@ -370,3 +376,28 @@ def test_onscreen_explicit_region_honored(self, tmp_path: Path): def test_onscreen_listed_in_unknown_kind_message(self): with pytest.raises(ValueError, match="onscreen"): build_effect_verifier(EffectsConfig(kind="bogus")) + + +def test_rest_needs_the_apps_records_path(): + """No app's records path is assumed: kind rest names its own.""" + cfg = EffectsConfig(kind="rest", base_url="http://app.local") + with pytest.raises(ValueError, match="records_path"): + build_effect_verifier(cfg) + # A config without a rest read does not carry the field at all. + assert "records_path" not in EffectsConfig(kind="none").model_dump() + + # MockMed's shape and Frappe's both work once named. + for path, key in ( + ("/api/db", "records"), + ("/api/resource/Loan Application", "data"), + ): + v = build_effect_verifier( + EffectsConfig( + kind="rest", + base_url="http://app.local", + records_path=path, + records_key=key, + ) + ) + assert isinstance(v, RestRecordVerifier) + assert (v.records_path, v.records_key) == (path, key) diff --git a/packages/flow/tests/test_effect_mining.py b/packages/flow/tests/test_effect_mining.py index 95f8db7bd9..ab60f8bd6d 100644 --- a/packages/flow/tests/test_effect_mining.py +++ b/packages/flow/tests/test_effect_mining.py @@ -196,7 +196,9 @@ def test_dom_field_delta_mines_flagged_field_equals(): def test_mined_effects_confirmed_by_verifier_on_real_sor(): base, db, stop = _fault_server() try: - verifier = RestRecordVerifier(base) + verifier = RestRecordVerifier( + base, records_path="/api/db", records_key="records" + ) before = db.snapshot()["records"] requests.post( f"{base}/api/encounter", @@ -220,7 +222,9 @@ def test_mined_effects_confirmed_by_verifier_on_real_sor(): def test_mined_record_written_refutes_a_duplicate_on_the_real_sor(): base, db, stop = _fault_server() try: - verifier = RestRecordVerifier(base) + verifier = RestRecordVerifier( + base, records_path="/api/db", records_key="records" + ) before = db.snapshot()["records"] requests.post( f"{base}/api/encounter", @@ -266,7 +270,9 @@ def test_placeholder_effect_halts_the_run(tmp_path): replayer = Replayer( backend, vision=vision, - effect_verifier=RestRecordVerifier(base), + effect_verifier=RestRecordVerifier( + base, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) report = replayer.run(workflow, bundle_dir=bundle, run_dir=run_dir) diff --git a/packages/flow/tests/test_effect_verifier.py b/packages/flow/tests/test_effect_verifier.py index 7229cda3ce..04b0b12afe 100644 --- a/packages/flow/tests/test_effect_verifier.py +++ b/packages/flow/tests/test_effect_verifier.py @@ -8,7 +8,6 @@ from __future__ import annotations - import pytest import requests @@ -59,7 +58,10 @@ def _post_encounter(base, *, note=NOTE, fault="", key=None): def test_all_verifiers_satisfy_protocol(tmp_path): - assert isinstance(RestRecordVerifier("http://x"), EffectVerifier) + assert isinstance( + RestRecordVerifier("http://x", records_path="/api/db", records_key="records"), + EffectVerifier, + ) assert isinstance(DocumentHashVerifier(tmp_path), EffectVerifier) from openadapt_flow.runtime.effects import FhirEffectVerifier @@ -149,7 +151,7 @@ def test_record_matches_string_coercion(): def test_rest_verifier_confirms_real_write(sor): base, _db = sor - verifier = RestRecordVerifier(base) + verifier = RestRecordVerifier(base, records_path="/api/db", records_key="records") before = verifier.capture_pre_state() assert before.reachable and before.records == [] _post_encounter(base) @@ -164,7 +166,9 @@ def test_rest_verifier_confirms_real_write(sor): def test_rest_verifier_indeterminate_when_unreachable(): - verifier = RestRecordVerifier("http://127.0.0.1:1") # nothing listening + verifier = RestRecordVerifier( + "http://127.0.0.1:1", records_path="/api/db", records_key="records" + ) # nothing listening before = verifier.capture_pre_state() assert not before.reachable eff = Effect(kind=EffectKind.RECORD_WRITTEN, match=TARGET, timeout_s=0.1) @@ -175,7 +179,7 @@ def test_rest_verifier_indeterminate_when_unreachable(): def test_rest_idempotent_write_is_at_most_once(sor): base, _db = sor - verifier = RestRecordVerifier(base) + verifier = RestRecordVerifier(base, records_path="/api/db", records_key="records") before = verifier.capture_pre_state() # Two submissions carrying the SAME idempotency key -> server dedupes. _post_encounter(base, fault="idempotent", key="run-42") @@ -246,7 +250,7 @@ def test_document_hash_indeterminate_when_store_absent(tmp_path): def test_compensation_reconciles_detected_duplicate(sor): base, db = sor - verifier = RestRecordVerifier(base) + verifier = RestRecordVerifier(base, records_path="/api/db", records_key="records") before = verifier.capture_pre_state() _post_encounter(base) # two non-idempotent submissions -> two rows _post_encounter(base) @@ -260,7 +264,7 @@ def test_compensation_reconciles_detected_duplicate(sor): verdict = verifier.verify(eff, before) assert verdict.verdict is Verdict.REFUTED and verdict.observed_count == 2 - compensator = RestCompensator(base) + compensator = RestCompensator(base, delete_path_template="/api/encounter/{id}") result = reconcile_or_escalate( eff, verdict, verifier=verifier, before=before, compensator=compensator ) @@ -272,7 +276,7 @@ def test_compensation_reconciles_detected_duplicate(sor): def test_compensation_escalates_partial_save(sor): base, _db = sor - verifier = RestRecordVerifier(base) + verifier = RestRecordVerifier(base, records_path="/api/db", records_key="records") before = verifier.capture_pre_state() _post_encounter(base, fault="partial") # row persists, note dropped eff = Effect( @@ -290,7 +294,7 @@ def test_compensation_escalates_partial_save(sor): verdict, verifier=verifier, before=before, - compensator=RestCompensator(base), + compensator=RestCompensator(base, delete_path_template="/api/encounter/{id}"), ) assert result.outcome is CompensationOutcome.ESCALATED assert not result.proceed @@ -298,7 +302,9 @@ def test_compensation_escalates_partial_save(sor): def test_compensation_escalates_when_indeterminate(): - verifier = RestRecordVerifier("http://127.0.0.1:1") + verifier = RestRecordVerifier( + "http://127.0.0.1:1", records_path="/api/db", records_key="records" + ) before = EffectState(substrate="rest", reachable=False) eff = Effect( kind=EffectKind.RECORD_WRITTEN, @@ -313,7 +319,9 @@ def test_compensation_escalates_when_indeterminate(): verdict, verifier=verifier, before=before, - compensator=RestCompensator("http://127.0.0.1:1"), + compensator=RestCompensator( + "http://127.0.0.1:1", delete_path_template="/api/encounter/{id}" + ), ) assert result.outcome is CompensationOutcome.ESCALATED assert not result.proceed diff --git a/packages/flow/tests/test_fault_gate.py b/packages/flow/tests/test_fault_gate.py index 1945c04a2a..99eb5b9ec8 100644 --- a/packages/flow/tests/test_fault_gate.py +++ b/packages/flow/tests/test_fault_gate.py @@ -58,7 +58,7 @@ def _rest_cfg(base: str = "http://127.0.0.1:1"): @pytest.fixture def rest_copy(): - store = MemoryRestCopy(SEED, TARGET) + store = MemoryRestCopy(SEED, records_path="/api/db", records_key="records", TARGET) try: yield store finally: diff --git a/packages/flow/tests/test_loop_authoring.py b/packages/flow/tests/test_loop_authoring.py index 1e9c87ac03..bf7f5949bb 100644 --- a/packages/flow/tests/test_loop_authoring.py +++ b/packages/flow/tests/test_loop_authoring.py @@ -379,7 +379,9 @@ def test_healthy_loop_confirms_each_record_zero_model_calls(dirs): report = Replayer( backend, vision=_vision_confirms_saved(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ).run(looped, bundle_dir=bundle, run_dir=run_dir) @@ -425,7 +427,9 @@ def test_poisoned_record_effect_refute_halts_no_silent_skip(dirs): report = Replayer( backend, vision=_vision_confirms_saved(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ).run(looped, bundle_dir=bundle, run_dir=run_dir) @@ -500,7 +504,9 @@ def test_committed_encounter_showcase_loop_replays_through_the_interpreter(tmp_p report = Replayer( backend, vision=_vision_confirms_saved(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ).run(wf, bundle_dir=bundle, run_dir=run_dir) diff --git a/packages/flow/tests/test_openemr_e2e_harness.py b/packages/flow/tests/test_openemr_e2e_harness.py index 52062f0f4f..7348343989 100644 --- a/packages/flow/tests/test_openemr_e2e_harness.py +++ b/packages/flow/tests/test_openemr_e2e_harness.py @@ -40,7 +40,9 @@ def _replay(workflow, *, bundle_dir, run_dir, note, fault="", modal=None): replayer = Replayer( backend, vision=vision, - effect_verifier=RestRecordVerifier(url.rstrip("/")), + effect_verifier=RestRecordVerifier( + url.rstrip("/"), records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) report = replayer.run( diff --git a/packages/flow/tests/test_readback_pipeline.py b/packages/flow/tests/test_readback_pipeline.py index 7c20079f88..2309c9bc0d 100644 --- a/packages/flow/tests/test_readback_pipeline.py +++ b/packages/flow/tests/test_readback_pipeline.py @@ -134,8 +134,13 @@ def factory(url): if kind == "screen_attr": return ScreenVerifier(url) if kind == "candidate": - return CandidateEffectVerifier([RestRecordVerifier(url)]) - return RedactingVerifier(RestRecordVerifier(url), policy=None) + return CandidateEffectVerifier( + [RestRecordVerifier(url, records_path="/api/db", records_key="records")] + ) + return RedactingVerifier( + RestRecordVerifier(url, records_path="/api/db", records_key="records"), + policy=None, + ) pipeline = ReadbackPipeline() report, settled, backend, _ = _run( diff --git a/packages/flow/tests/test_replayer_api_actuator.py b/packages/flow/tests/test_replayer_api_actuator.py index fad040ea6b..4ce9747673 100644 --- a/packages/flow/tests/test_replayer_api_actuator.py +++ b/packages/flow/tests/test_replayer_api_actuator.py @@ -236,7 +236,9 @@ def test_api_binding_actuates_and_confirms_skipping_gui(tmp_path): replayer = Replayer( backend, vision=vision, - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), api_actuator=ApiActuator(url), poll_interval_s=0.01, ) @@ -281,7 +283,9 @@ def test_unreachable_api_halts_without_gui_fallback(tmp_path): replayer = Replayer( backend, vision=vision, - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), api_actuator=ApiActuator(dead, timeout_s=1.0), poll_interval_s=0.01, ) @@ -307,7 +311,9 @@ def test_unreachable_effect_pre_state_refuses_api_actuation(tmp_path): replayer = Replayer( backend, vision=_vision_that_confirms_saved(), - effect_verifier=RestRecordVerifier("http://127.0.0.1:1"), + effect_verifier=RestRecordVerifier( + "http://127.0.0.1:1", records_path="/api/db", records_key="records" + ), api_actuator=ApiActuator(url), poll_interval_s=0.01, ) @@ -376,7 +382,9 @@ def test_post_send_protocol_error_is_proven_by_the_complete_contract(tmp_path): report = Replayer( backend, vision=_vision_that_confirms_saved(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), api_actuator=ApiActuator(url, session=session), poll_interval_s=0.01, ).run(workflow, bundle_dir=bundle, run_dir=run_dir) @@ -502,7 +510,9 @@ def test_post_send_protocol_error_with_refuted_effect_requires_reconciliation( report = Replayer( backend, vision=_vision_that_confirms_saved(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), api_actuator=ApiActuator(url, session=session), poll_interval_s=0.01, ).run(workflow, bundle_dir=bundle, run_dir=run_dir) @@ -574,7 +584,9 @@ def test_api_write_refuted_by_record_halts(tmp_path): replayer = Replayer( backend, vision=vision, - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), api_actuator=ApiActuator(url), poll_interval_s=0.01, ) @@ -614,7 +626,9 @@ def test_api_non_2xx_halts_never_double_writes(tmp_path): replayer = Replayer( backend, vision=vision, - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), api_actuator=ApiActuator(url), poll_interval_s=0.01, ) @@ -648,7 +662,9 @@ def test_api_binding_without_effects_is_config_error_halt(tmp_path): replayer = Replayer( backend, vision=vision, - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), api_actuator=ApiActuator(url), poll_interval_s=0.01, ) @@ -710,7 +726,9 @@ def test_binding_present_but_no_actuator_uses_gui_unchanged(tmp_path): replayer = Replayer( backend, vision=vision, - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) report = replayer.run(workflow, bundle_dir=bundle, run_dir=run_dir) diff --git a/packages/flow/tests/test_replayer_effect_param_binding.py b/packages/flow/tests/test_replayer_effect_param_binding.py index 498ac5c3a5..e62d0ff905 100644 --- a/packages/flow/tests/test_replayer_effect_param_binding.py +++ b/packages/flow/tests/test_replayer_effect_param_binding.py @@ -504,7 +504,9 @@ def test_parameterized_effect_confirmed_against_real_sor(tmp_path): replayer = Replayer( backend, vision=_vision_confirms_saved(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) report = replayer.run( @@ -544,7 +546,9 @@ def test_demo_literal_effect_refutes_when_run_writes_different_patient(tmp_path) replayer = Replayer( backend, vision=_vision_confirms_saved(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) report = replayer.run(workflow, bundle_dir=bundle, run_dir=run_dir) diff --git a/packages/flow/tests/test_replayer_effects.py b/packages/flow/tests/test_replayer_effects.py index 976f9ee005..c90989cede 100644 --- a/packages/flow/tests/test_replayer_effects.py +++ b/packages/flow/tests/test_replayer_effects.py @@ -162,7 +162,9 @@ def test_effect_confirmed_proceeds(tmp_path): replayer = Replayer( backend, vision=vision, - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) report = replayer.run(workflow, bundle_dir=bundle, run_dir=run_dir) @@ -204,7 +206,9 @@ def test_effect_refuted_halts_despite_green_screen(tmp_path): replayer = Replayer( backend, vision=vision, - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) report = replayer.run(workflow, bundle_dir=bundle, run_dir=run_dir) @@ -251,8 +255,12 @@ def test_effect_duplicate_irreversible_reconciles_as_rolled_back(tmp_path): replayer = Replayer( backend, vision=vision, - effect_verifier=RestRecordVerifier(url), - effect_compensator=RestCompensator(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), + effect_compensator=RestCompensator( + url, delete_path_template="/api/encounter/{id}" + ), poll_interval_s=0.01, ) report = replayer.run(workflow, bundle_dir=bundle, run_dir=run_dir) @@ -295,7 +303,9 @@ def test_effect_duplicate_irreversible_without_compensator_halts(tmp_path): replayer = Replayer( backend, vision=vision, - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), # no compensator configured poll_interval_s=0.01, ) @@ -487,7 +497,9 @@ def test_effect_evidence_records_a_loopback_verifier_endpoint(tmp_path): report = Replayer( WritingBackend(url), vision=_vision_that_confirms_saved(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ).run(workflow, bundle_dir=bundle, run_dir=run_dir) @@ -503,15 +515,21 @@ def test_effect_evidence_records_a_loopback_verifier_endpoint(tmp_path): def test_effect_evidence_marks_a_remote_verifier_endpoint_external(): from openadapt_flow.runtime.replayer import effect_endpoint_scope - assert effect_endpoint_scope(RestRecordVerifier("https://emr.example.com")) == ( - "external" - ) - assert effect_endpoint_scope(RestRecordVerifier("http://[::1]:8080")) == ( - "loopback" - ) - assert effect_endpoint_scope(RestRecordVerifier("http://localhost:8080")) == ( - "loopback" - ) + assert effect_endpoint_scope( + RestRecordVerifier( + "https://emr.example.com", records_path="/api/db", records_key="records" + ) + ) == ("external") + assert effect_endpoint_scope( + RestRecordVerifier( + "http://[::1]:8080", records_path="/api/db", records_key="records" + ) + ) == ("loopback") + assert effect_endpoint_scope( + RestRecordVerifier( + "http://localhost:8080", records_path="/api/db", records_key="records" + ) + ) == ("loopback") assert effect_endpoint_scope(object()) == "unknown" diff --git a/packages/flow/tests/test_rest_verifier_transport.py b/packages/flow/tests/test_rest_verifier_transport.py index 00376c0cd3..d8a0bc623f 100644 --- a/packages/flow/tests/test_rest_verifier_transport.py +++ b/packages/flow/tests/test_rest_verifier_transport.py @@ -36,13 +36,20 @@ def test_session_falls_back_to_httpx_without_requests( import httpx _hide(monkeypatch, "requests") - session = RestRecordVerifier("http://127.0.0.1:1")._get_session() + session = RestRecordVerifier( + "http://127.0.0.1:1", records_path="/api/db", records_key="records" + )._get_session() assert isinstance(session, httpx.Client) def test_injected_session_is_always_preferred() -> None: sentinel = object() - verifier = RestRecordVerifier("http://127.0.0.1:1", session=sentinel) + verifier = RestRecordVerifier( + "http://127.0.0.1:1", + records_path="/api/db", + records_key="records", + session=sentinel, + ) assert verifier._get_session() is sentinel @@ -57,7 +64,9 @@ def test_records_are_read_over_the_fallback_transport( base_url, db, stop = serve() try: db.add("p1", "Triage", "note", key="k1") - state = RestRecordVerifier(base_url).capture_pre_state() + state = RestRecordVerifier( + base_url, records_path="/api/db", records_key="records" + ).capture_pre_state() assert state.reachable is True assert [record["patient_id"] for record in state.records] == ["p1"] finally: diff --git a/packages/flow/tests/test_reward_trust_boundary.py b/packages/flow/tests/test_reward_trust_boundary.py index d92a0ad65a..50ae47a333 100644 --- a/packages/flow/tests/test_reward_trust_boundary.py +++ b/packages/flow/tests/test_reward_trust_boundary.py @@ -529,3 +529,16 @@ def test_the_calibration_records_which_corpus_it_ran_on( calibration["calibration_corpus_digest"] == contract["certificate_policy"]["calibration_corpus_digest"] ) + + +def test_a_rest_recipe_names_its_records_path() -> None: + """No app's records path is assumed for a REST oracle.""" + with pytest.raises(ValueError, match="records_path"): + OracleRecipeV1(kind="rest", base_url="http://app.local") + recipe = OracleRecipeV1( + kind="rest", + base_url="http://app.local", + records_path="/api/resource/Loan Application", + records_key="data", + ) + assert recipe.records_path == "/api/resource/Loan Application" diff --git a/packages/flow/tests/test_rollback.py b/packages/flow/tests/test_rollback.py index f98fa2f082..c56ae9581e 100644 --- a/packages/flow/tests/test_rollback.py +++ b/packages/flow/tests/test_rollback.py @@ -208,7 +208,9 @@ def _mockmed_replayer(url: str) -> Replayer: return Replayer( MockMedBackend(url), vision=_vision(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), api_actuator=ApiActuator(url), poll_interval_s=0.01, ) @@ -220,7 +222,7 @@ def _run_with_rollback( bundle = tmp_path / "bundle" workflow.save(bundle) workflow = Workflow.load(bundle) - verifier = RestRecordVerifier(url) + verifier = RestRecordVerifier(url, records_path="/api/db", records_key="records") from openadapt_flow.after_stop import records_reader plan = RollbackPlan( diff --git a/packages/flow/tests/test_rollback_qualify.py b/packages/flow/tests/test_rollback_qualify.py index e55515ac70..f3f4ca8818 100644 --- a/packages/flow/tests/test_rollback_qualify.py +++ b/packages/flow/tests/test_rollback_qualify.py @@ -55,7 +55,9 @@ def test_matrix_qualifies_a_compensation_for_every_save_on_mockmed(tmp_path, moc Workflow.load(bundle), bundle_dir=bundle, target=target, - reader=VerifierReader(RestRecordVerifier(mockmed)), + reader=VerifierReader( + RestRecordVerifier(mockmed, records_path="/api/db", records_key="records") + ), sources=Sources(network_lines=_network()), work_dir=tmp_path / "work", allow_delete=True, diff --git a/packages/flow/tests/test_sealed_at_rest.py b/packages/flow/tests/test_sealed_at_rest.py index 7324a6956e..97bef6e06e 100644 --- a/packages/flow/tests/test_sealed_at_rest.py +++ b/packages/flow/tests/test_sealed_at_rest.py @@ -544,7 +544,9 @@ def test_halted_sealed_run_is_undone_without_plaintext( report = Replayer( MockMedBackend(url), vision=_vision(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ).run(workflow, bundle_dir=bundle, run_dir=run_dir) assert report.execution_outcome == "HALTED" diff --git a/packages/flow/tests/test_ui_readback_verifier.py b/packages/flow/tests/test_ui_readback_verifier.py index 70070f3d9c..cf725558f0 100644 --- a/packages/flow/tests/test_ui_readback_verifier.py +++ b/packages/flow/tests/test_ui_readback_verifier.py @@ -432,7 +432,9 @@ def test_existing_verifier_identities_do_not_change() -> None: # Digests computed with origin/main (before ``ui_readback`` existed). The # new optional field must not enter the identity of any verifier that # does not use it, or every issued admission bound to one would break. - rest = RestRecordVerifier("http://sor.test") + rest = RestRecordVerifier( + "http://sor.test", records_path="/api/db", records_key="records" + ) cfg = EffectsConfig(kind="rest", base_url="http://sor.test") assert _effect_verifier_config_identity(cfg, rest) == ( "sha256:9f288a5b15b66001565c6d5bbd0f0a09bf92485280fe5de37b421802c3e0e663" diff --git a/packages/flow/tests/test_undo_proposal.py b/packages/flow/tests/test_undo_proposal.py index 30c493e9a8..10d0522027 100644 --- a/packages/flow/tests/test_undo_proposal.py +++ b/packages/flow/tests/test_undo_proposal.py @@ -178,7 +178,9 @@ def run(self, workflow, *, bundle_dir, run_dir, params): replayer = Replayer( SavingBackend(self.url), vision=_vision(), - effect_verifier=RestRecordVerifier(self.url), + effect_verifier=RestRecordVerifier( + self.url, records_path="/api/db", records_key="records" + ), api_actuator=ApiActuator(self.url), poll_interval_s=0.01, ) @@ -255,7 +257,9 @@ def test_network_undo_is_verified_written_as_a_proposal_and_waits_for_a_person( stopping = Replayer( _RejectSecond(mockmed), vision=_vision(), - effect_verifier=RestRecordVerifier(mockmed), + effect_verifier=RestRecordVerifier( + mockmed, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) stopped = stopping.run(run_wf, bundle_dir=bundle, run_dir=run_dir) @@ -323,7 +327,9 @@ def _undo_replayer(url: str) -> Replayer: return Replayer( SavingBackend(url), vision=_vision(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), api_actuator=ApiActuator(url), poll_interval_s=0.01, ) @@ -581,7 +587,9 @@ def run(self, workflow, *, bundle_dir, run_dir, params): replayer = Replayer( EditingBackend(self.url, also_touch_p2=self.also_touch_p2), vision=_vision(), - effect_verifier=RestRecordVerifier(self.url), + effect_verifier=RestRecordVerifier( + self.url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) return replayer.run( diff --git a/packages/flow/tests/test_variant_router.py b/packages/flow/tests/test_variant_router.py index 3451656bd3..a98941ef31 100644 --- a/packages/flow/tests/test_variant_router.py +++ b/packages/flow/tests/test_variant_router.py @@ -513,7 +513,9 @@ def child_run( report = Replayer( _RowWritingBackend(url), vision=_vision_confirms_saved(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ).run( workflow, diff --git a/packages/flow/tests/test_verifier_adapter_platform.py b/packages/flow/tests/test_verifier_adapter_platform.py index ee1d680766..c1d82c30eb 100644 --- a/packages/flow/tests/test_verifier_adapter_platform.py +++ b/packages/flow/tests/test_verifier_adapter_platform.py @@ -104,7 +104,9 @@ def reachable_state(records=(), substrate="test"): class TestAdapterInterface: def test_builtin_adapters_conform_to_protocol(self, tmp_path): adapters = [ - RestRecordVerifier("http://sor"), + RestRecordVerifier( + "http://sor", records_path="/api/db", records_key="records" + ), GraphQLRecordVerifier("http://sor/graphql", query="query { r { id } }"), FhirEffectVerifier("http://sor/fhir"), SqlRecordVerifier(lambda: None, "SELECT id FROM t"), @@ -127,7 +129,12 @@ def test_screen_readback_is_demoted_not_independent_proof(self): assert onscreen.verification_tier is VerificationTier.IMMEDIATE_SCREEN assert confidence_label(onscreen.verification_tier) == "screen-consistency" # Every system-of-record adapter IS independent proof. - assert RestRecordVerifier("http://sor").independent_system_of_record is True + assert ( + RestRecordVerifier( + "http://sor", records_path="/api/db", records_key="records" + ).independent_system_of_record + is True + ) def test_confidence_labels_cover_every_tier(self): labels = {confidence_label(tier) for tier in VerificationTier} @@ -140,7 +147,9 @@ def test_confidence_labels_cover_every_tier(self): def test_test_connection_ok(self): session = FakeSession([FakeResponse(200, {"records": []})]) - verifier = RestRecordVerifier("http://sor", session=session) + verifier = RestRecordVerifier( + "http://sor", records_path="/api/db", records_key="records", session=session + ) probe = verifier.test_connection() assert isinstance(probe, ConnectionProbe) assert probe.ok is True @@ -148,7 +157,9 @@ def test_test_connection_ok(self): def test_test_connection_credential_failure_not_ok(self): session = FakeSession([FakeResponse(401, {"error": "bad token"})]) - probe = RestRecordVerifier("http://sor", session=session).test_connection() + probe = RestRecordVerifier( + "http://sor", records_path="/api/db", records_key="records", session=session + ).test_connection() assert probe.ok is False def test_test_connection_never_raises(self): @@ -169,7 +180,9 @@ def test_capture_post_state_defaults_to_fresh_snapshot(self): FakeResponse(200, {"records": [{"id": 1}]}), ] ) - verifier = RestRecordVerifier("http://sor", session=session) + verifier = RestRecordVerifier( + "http://sor", records_path="/api/db", records_key="records", session=session + ) before = verifier.capture_pre_state() after = verifier.capture_post_state() assert before.records == [] @@ -244,7 +257,9 @@ def test_no_non_confirmed_result_maps_to_a_pass(self): def test_unreachable_sor_classifies_unavailable(self): session = FakeSession([FakeResponse(500)]) - verifier = RestRecordVerifier("http://sor", session=session) + verifier = RestRecordVerifier( + "http://sor", records_path="/api/db", records_key="records", session=session + ) before = verifier.capture_pre_state() verdict = verifier.verify(make_effect(), before) assert verdict.unavailable is True From 2c78f016943e05022f1f8e2112e27af585934a0d Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 13:44:41 -0500 Subject: [PATCH 06/22] Verifier: FHIR layout, the no-record value and server facts come from data Three engine defaults encoded one app: - FhirEffectVerifier, effects.kind fhir and fhir reward recipes defaulted to OpenEMR's note-as-Observation layout (resource Observation, note in valueString). They now require resource_type and field_paths; a config without them is refused when the verifier is built. - The frame and rollback attribution read an audit entry for record "0" as naming no record, which is how OpenEMR files bookkeeping inserts. An app profile now lists such values (no_record_values); the OpenEMR profile says "0" (version 2026-11-03.1). An empty field still names no record in any app. - ensure-contract generation translated the Medplum probe's own output keys. It reads only the generic fact names now; the translation moved beside the probe, and the committed teach fixtures hold generic facts. --- packages/flow/CHANGELOG.md | 2 + .../flow/benchmark/contract_autogen/blind.py | 4 +- .../contract_autogen/build_patient_teach.py | 10 +- .../teach/immunization/facts.json | 91 ++----------------- .../teach/medplum_patient_rebuilt/facts.json | 91 ++----------------- .../benchmark/fhir_medplum/readpath/facts.py | 47 ++++++++++ .../run3/teach/allergy/facts.json | 91 ++----------------- .../run3/teach/appointment/facts.json | 91 ++----------------- .../run3/teach/patient/facts.json | 91 ++----------------- .../flow/benchmark/openemr_e2e/harness.py | 9 +- packages/flow/docs/EFFECT_KIT.md | 1 + packages/flow/docs/deployment.example.yaml | 8 +- .../openadapt_flow/app_profiles/__init__.py | 8 +- .../openadapt_flow/app_profiles/openemr.yaml | 4 +- packages/flow/openadapt_flow/deployment.py | 13 ++- .../generate/ensure_contract.py | 36 ++------ packages/flow/openadapt_flow/reward/models.py | 6 +- .../flow/openadapt_flow/reward/oracles.py | 4 +- packages/flow/openadapt_flow/rollback.py | 12 ++- .../flow/openadapt_flow/rollback_state.py | 6 +- .../openadapt_flow/runtime/effects/fhir.py | 23 +++-- .../openadapt_flow/runtime/effects/frame.py | 9 +- packages/flow/openadapt_flow/verify_run.py | 4 + packages/flow/tests/test_app_profile.py | 6 ++ packages/flow/tests/test_effect_fhir.py | 6 ++ packages/flow/tests/test_effect_kit_config.py | 33 +++++++ packages/flow/tests/test_effect_verifier.py | 7 +- .../tests/test_effect_verifier_candidates.py | 20 +++- .../tests/test_effect_verifier_transport.py | 29 +++++- packages/flow/tests/test_frame_conditions.py | 10 ++ .../tests/test_generate_ensure_contract.py | 2 +- packages/flow/tests/test_rollback.py | 11 ++- .../flow/tests/test_smart_backend_auth.py | 13 ++- .../tests/test_verifier_adapter_platform.py | 8 +- 34 files changed, 313 insertions(+), 493 deletions(-) create mode 100644 packages/flow/benchmark/fhir_medplum/readpath/facts.py diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md index 33467f5c83..ed6573cc1e 100644 --- a/packages/flow/CHANGELOG.md +++ b/packages/flow/CHANGELOG.md @@ -25,6 +25,8 @@ ### Fixes +- A FHIR read names its resource and field map: `effects.kind: fhir`, a reward recipe of kind `fhir` and `FhirEffectVerifier` require `resource_type` and `field_paths` instead of defaulting to one app's note-as-Observation layout (`DEFAULT_OBSERVATION_PATHS` stays importable for callers whose app stores notes that way). An app profile's `no_record_values` lists the record-field values its audit entries use for "no record" (OpenEMR's profile: `"0"`; profile version `openemr-frame-profile/2026-11-03.1`); the frame and rollback no longer treat `0` as "no record" in every app. `generate ensure-contract` reads only the generic server-fact names from `facts.json`; the Medplum probe's translation moved to `benchmark/fhir_medplum/readpath/facts.py`. + - Self-repair on a covered identity band dismisses only the proved dialog: the covering window must span the band from where the read stops to its right edge at full height, the dismiss click must land in that same window, and only a dialog whose every word is a notice word (no question) may be dismissed ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). - Self-repair can take a stop where the app's own dialog covers the next step's identity band: when the backend proves the covering window belongs to the window the last verified step was in, and the part of the band read still matches the record, the dialog is dismissed after the previous click and the step re-checks its identity in full. Another record behind the dialog stays an identity halt. The runtime lets a backend remember a verified record's window (`record_verified_window`); macOS answers `owned_window_over` from the window's proved sheet ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). diff --git a/packages/flow/benchmark/contract_autogen/blind.py b/packages/flow/benchmark/contract_autogen/blind.py index a3be44a130..f2fb98ac65 100644 --- a/packages/flow/benchmark/contract_autogen/blind.py +++ b/packages/flow/benchmark/contract_autogen/blind.py @@ -25,7 +25,6 @@ import hashlib import json import os -import shutil import sys import time from pathlib import Path @@ -33,6 +32,7 @@ from benchmark.fhir_medplum.fixture import FixtureError, MedplumFixture, _http from benchmark.fhir_medplum.readpath import suite as runner +from benchmark.fhir_medplum.readpath.facts import generic_facts HERE = Path(__file__).resolve().parent PROBE = HERE.parent / "fhir_medplum" / "readpath" / "probe.json" @@ -101,7 +101,7 @@ def cmd_teach(args: argparse.Namespace) -> int: out / "search_parameters.json", read_all(f"{fx.fhir_url}/SearchParameter?base={rtype}&_count=200", reader), ) - shutil.copyfile(PROBE, out / "facts.json") + _dump(out / "facts.json", generic_facts(json.loads(PROBE.read_text()))) _dump( out / "teach.json", { diff --git a/packages/flow/benchmark/contract_autogen/build_patient_teach.py b/packages/flow/benchmark/contract_autogen/build_patient_teach.py index 5cdfb085be..40c584cad5 100644 --- a/packages/flow/benchmark/contract_autogen/build_patient_teach.py +++ b/packages/flow/benchmark/contract_autogen/build_patient_teach.py @@ -18,6 +18,7 @@ import json from pathlib import Path +from benchmark.fhir_medplum.readpath.facts import generic_facts HERE = Path(__file__).resolve().parent SRC = HERE.parent / "fhir_medplum" / "readpath" @@ -123,7 +124,14 @@ def main() -> None: (OUT / "search_parameters.json").write_text( json.dumps(params_bundle, indent=1) + "\n" ) - (OUT / "facts.json").write_text((SRC / "probe.json").read_text()) + (OUT / "facts.json").write_text( + json.dumps( + generic_facts(json.loads((SRC / "probe.json").read_text())), + indent=2, + sort_keys=True, + ) + + "\n" + ) teach = { "resource_type": "Patient", "deployment": { diff --git a/packages/flow/benchmark/contract_autogen/teach/immunization/facts.json b/packages/flow/benchmark/contract_autogen/teach/immunization/facts.json index ed88066a65..a6c9b649c5 100644 --- a/packages/flow/benchmark/contract_autogen/teach/immunization/facts.json +++ b/packages/flow/benchmark/contract_autogen/teach/immunization/facts.json @@ -1,87 +1,8 @@ { - "audit_lag_s": { - "C": { - "max": 0.043, - "median": 0.036 - }, - "D": { - "max": 0.049, - "median": 0.036 - }, - "U": { - "max": 0.043, - "median": 0.031 - } - }, - "audit_names_the_user_profile": true, - "audit_seen_on_an_immediate_read": [ - "C by bot", - "C by bot", - "D by bot", - "U by other" - ], - "audit_writes_are_logged": false, - "bot_admin_endpoint": "HTTP 403", - "bot_is_project_admin": false, - "bot_is_super_admin": false, - "bundle_and_graphql_writes_logged": [ - "C Observation by bot", - "C Observation by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot" - ], - "date_header": true, - "delete_in_history": true, - "meta_author_returned": false, - "policy": "AccessPolicy/f348be14-df4d-47cc-a49b-1f45e08dd554", - "restricted_bot_audit_read_only": true, - "restricted_bot_audit_writes": { - "create": "HTTP 403", - "delete": "HTTP 403", - "update": "HTTP 403" - }, - "restricted_bot_can_write": { - "Observation": "allowed", - "Patient": "allowed", - "Task": "allowed" - }, - "sweep_sees_delete": false, - "type_history": "HTTP 404", - "unrestricted_bot_audit_writes": { - "create": null, - "delete": null, - "update": null - } + "agent_cannot_write_audit": true, + "audit_lag_max_s": 0.049, + "audit_logs_writes": true, + "audit_names_actor": true, + "history_lists_versions": true, + "meta_author_returned": false } diff --git a/packages/flow/benchmark/contract_autogen/teach/medplum_patient_rebuilt/facts.json b/packages/flow/benchmark/contract_autogen/teach/medplum_patient_rebuilt/facts.json index ed88066a65..a6c9b649c5 100644 --- a/packages/flow/benchmark/contract_autogen/teach/medplum_patient_rebuilt/facts.json +++ b/packages/flow/benchmark/contract_autogen/teach/medplum_patient_rebuilt/facts.json @@ -1,87 +1,8 @@ { - "audit_lag_s": { - "C": { - "max": 0.043, - "median": 0.036 - }, - "D": { - "max": 0.049, - "median": 0.036 - }, - "U": { - "max": 0.043, - "median": 0.031 - } - }, - "audit_names_the_user_profile": true, - "audit_seen_on_an_immediate_read": [ - "C by bot", - "C by bot", - "D by bot", - "U by other" - ], - "audit_writes_are_logged": false, - "bot_admin_endpoint": "HTTP 403", - "bot_is_project_admin": false, - "bot_is_super_admin": false, - "bundle_and_graphql_writes_logged": [ - "C Observation by bot", - "C Observation by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot" - ], - "date_header": true, - "delete_in_history": true, - "meta_author_returned": false, - "policy": "AccessPolicy/f348be14-df4d-47cc-a49b-1f45e08dd554", - "restricted_bot_audit_read_only": true, - "restricted_bot_audit_writes": { - "create": "HTTP 403", - "delete": "HTTP 403", - "update": "HTTP 403" - }, - "restricted_bot_can_write": { - "Observation": "allowed", - "Patient": "allowed", - "Task": "allowed" - }, - "sweep_sees_delete": false, - "type_history": "HTTP 404", - "unrestricted_bot_audit_writes": { - "create": null, - "delete": null, - "update": null - } + "agent_cannot_write_audit": true, + "audit_lag_max_s": 0.049, + "audit_logs_writes": true, + "audit_names_actor": true, + "history_lists_versions": true, + "meta_author_returned": false } diff --git a/packages/flow/benchmark/fhir_medplum/readpath/facts.py b/packages/flow/benchmark/fhir_medplum/readpath/facts.py new file mode 100644 index 0000000000..402a152d29 --- /dev/null +++ b/packages/flow/benchmark/fhir_medplum/readpath/facts.py @@ -0,0 +1,47 @@ +"""Translate this probe's output into the generic server facts. + +``openadapt_flow.generate.ensure_contract`` reads only the generic names in +``FACTS`` from a teach directory's ``facts.json``; what each Medplum probe +measured, and under which name, stays here with the probe. + + PYTHONPATH=. python -m benchmark.fhir_medplum.readpath.facts probe.json > facts.json +""" + +from __future__ import annotations + +import json +import sys +from typing import Any, Mapping + +from openadapt_flow.generate.ensure_contract import FACTS + + +def generic_facts(probe: Mapping[str, Any]) -> dict[str, Any]: + """The generic facts this probe's output supports, by their generic names.""" + + if all(k in probe for k in FACTS): + return {k: probe[k] for k in FACTS} + out: dict[str, Any] = {} + logged = probe.get("bundle_and_graphql_writes_logged") + if isinstance(logged, list): + out["audit_logs_writes"] = len(logged) > 0 + for name, key in ( + ("audit_names_actor", "audit_names_the_user_profile"), + ("agent_cannot_write_audit", "restricted_bot_audit_read_only"), + ("history_lists_versions", "delete_in_history"), + ("meta_author_returned", "meta_author_returned"), + ): + if key in probe: + out[name] = probe[key] + lags = probe.get("audit_lag_s") + if isinstance(lags, Mapping) and lags: + out["audit_lag_max_s"] = max( + float(v.get("max", 0)) for v in lags.values() if isinstance(v, Mapping) + ) + return out + + +if __name__ == "__main__": + with open(sys.argv[1], encoding="utf-8") as f: + json.dump(generic_facts(json.load(f)), sys.stdout, indent=2, sort_keys=True) + sys.stdout.write("\n") diff --git a/packages/flow/benchmark/fhir_medplum/run3/teach/allergy/facts.json b/packages/flow/benchmark/fhir_medplum/run3/teach/allergy/facts.json index ed88066a65..a6c9b649c5 100644 --- a/packages/flow/benchmark/fhir_medplum/run3/teach/allergy/facts.json +++ b/packages/flow/benchmark/fhir_medplum/run3/teach/allergy/facts.json @@ -1,87 +1,8 @@ { - "audit_lag_s": { - "C": { - "max": 0.043, - "median": 0.036 - }, - "D": { - "max": 0.049, - "median": 0.036 - }, - "U": { - "max": 0.043, - "median": 0.031 - } - }, - "audit_names_the_user_profile": true, - "audit_seen_on_an_immediate_read": [ - "C by bot", - "C by bot", - "D by bot", - "U by other" - ], - "audit_writes_are_logged": false, - "bot_admin_endpoint": "HTTP 403", - "bot_is_project_admin": false, - "bot_is_super_admin": false, - "bundle_and_graphql_writes_logged": [ - "C Observation by bot", - "C Observation by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot" - ], - "date_header": true, - "delete_in_history": true, - "meta_author_returned": false, - "policy": "AccessPolicy/f348be14-df4d-47cc-a49b-1f45e08dd554", - "restricted_bot_audit_read_only": true, - "restricted_bot_audit_writes": { - "create": "HTTP 403", - "delete": "HTTP 403", - "update": "HTTP 403" - }, - "restricted_bot_can_write": { - "Observation": "allowed", - "Patient": "allowed", - "Task": "allowed" - }, - "sweep_sees_delete": false, - "type_history": "HTTP 404", - "unrestricted_bot_audit_writes": { - "create": null, - "delete": null, - "update": null - } + "agent_cannot_write_audit": true, + "audit_lag_max_s": 0.049, + "audit_logs_writes": true, + "audit_names_actor": true, + "history_lists_versions": true, + "meta_author_returned": false } diff --git a/packages/flow/benchmark/fhir_medplum/run3/teach/appointment/facts.json b/packages/flow/benchmark/fhir_medplum/run3/teach/appointment/facts.json index ed88066a65..a6c9b649c5 100644 --- a/packages/flow/benchmark/fhir_medplum/run3/teach/appointment/facts.json +++ b/packages/flow/benchmark/fhir_medplum/run3/teach/appointment/facts.json @@ -1,87 +1,8 @@ { - "audit_lag_s": { - "C": { - "max": 0.043, - "median": 0.036 - }, - "D": { - "max": 0.049, - "median": 0.036 - }, - "U": { - "max": 0.043, - "median": 0.031 - } - }, - "audit_names_the_user_profile": true, - "audit_seen_on_an_immediate_read": [ - "C by bot", - "C by bot", - "D by bot", - "U by other" - ], - "audit_writes_are_logged": false, - "bot_admin_endpoint": "HTTP 403", - "bot_is_project_admin": false, - "bot_is_super_admin": false, - "bundle_and_graphql_writes_logged": [ - "C Observation by bot", - "C Observation by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot" - ], - "date_header": true, - "delete_in_history": true, - "meta_author_returned": false, - "policy": "AccessPolicy/f348be14-df4d-47cc-a49b-1f45e08dd554", - "restricted_bot_audit_read_only": true, - "restricted_bot_audit_writes": { - "create": "HTTP 403", - "delete": "HTTP 403", - "update": "HTTP 403" - }, - "restricted_bot_can_write": { - "Observation": "allowed", - "Patient": "allowed", - "Task": "allowed" - }, - "sweep_sees_delete": false, - "type_history": "HTTP 404", - "unrestricted_bot_audit_writes": { - "create": null, - "delete": null, - "update": null - } + "agent_cannot_write_audit": true, + "audit_lag_max_s": 0.049, + "audit_logs_writes": true, + "audit_names_actor": true, + "history_lists_versions": true, + "meta_author_returned": false } diff --git a/packages/flow/benchmark/fhir_medplum/run3/teach/patient/facts.json b/packages/flow/benchmark/fhir_medplum/run3/teach/patient/facts.json index ed88066a65..a6c9b649c5 100644 --- a/packages/flow/benchmark/fhir_medplum/run3/teach/patient/facts.json +++ b/packages/flow/benchmark/fhir_medplum/run3/teach/patient/facts.json @@ -1,87 +1,8 @@ { - "audit_lag_s": { - "C": { - "max": 0.043, - "median": 0.036 - }, - "D": { - "max": 0.049, - "median": 0.036 - }, - "U": { - "max": 0.043, - "median": 0.031 - } - }, - "audit_names_the_user_profile": true, - "audit_seen_on_an_immediate_read": [ - "C by bot", - "C by bot", - "D by bot", - "U by other" - ], - "audit_writes_are_logged": false, - "bot_admin_endpoint": "HTTP 403", - "bot_is_project_admin": false, - "bot_is_super_admin": false, - "bundle_and_graphql_writes_logged": [ - "C Observation by bot", - "C Observation by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "C Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "D Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot", - "U Patient by bot" - ], - "date_header": true, - "delete_in_history": true, - "meta_author_returned": false, - "policy": "AccessPolicy/f348be14-df4d-47cc-a49b-1f45e08dd554", - "restricted_bot_audit_read_only": true, - "restricted_bot_audit_writes": { - "create": "HTTP 403", - "delete": "HTTP 403", - "update": "HTTP 403" - }, - "restricted_bot_can_write": { - "Observation": "allowed", - "Patient": "allowed", - "Task": "allowed" - }, - "sweep_sees_delete": false, - "type_history": "HTTP 404", - "unrestricted_bot_audit_writes": { - "create": null, - "delete": null, - "update": null - } + "agent_cannot_write_audit": true, + "audit_lag_max_s": 0.049, + "audit_logs_writes": true, + "audit_names_actor": true, + "history_lists_versions": true, + "meta_author_returned": false } diff --git a/packages/flow/benchmark/openemr_e2e/harness.py b/packages/flow/benchmark/openemr_e2e/harness.py index 418ab01544..0c9ff8e057 100644 --- a/packages/flow/benchmark/openemr_e2e/harness.py +++ b/packages/flow/benchmark/openemr_e2e/harness.py @@ -40,6 +40,8 @@ from __future__ import annotations +from openadapt_flow.runtime.effects.fhir import DEFAULT_OBSERVATION_PATHS + import json import os import time @@ -272,7 +274,12 @@ def _live_probe( from openadapt_flow.runtime.effects import FhirEffectVerifier token = os.environ.get("OPENEMR_FHIR_TOKEN") - verifier = FhirEffectVerifier(base.rstrip("/"), access_token=token) + verifier = FhirEffectVerifier( + base.rstrip("/"), + resource_type="Observation", + field_paths=dict(DEFAULT_OBSERVATION_PATHS), + access_token=token, + ) try: state = verifier.capture_pre_state() reachable = bool(state.reachable) diff --git a/packages/flow/docs/EFFECT_KIT.md b/packages/flow/docs/EFFECT_KIT.md index 442559a1ba..d78ae3f25f 100644 --- a/packages/flow/docs/EFFECT_KIT.md +++ b/packages/flow/docs/EFFECT_KIT.md @@ -1096,6 +1096,7 @@ effects: kind: fhir base_url: https://fhir.example.org/r4 resource_type: Observation + field_paths: {id: id, patient: subject.reference, note: valueString} smart_backend: client_id: my-registered-client scope: system/Observation.rs diff --git a/packages/flow/docs/deployment.example.yaml b/packages/flow/docs/deployment.example.yaml index 52e088f96a..1afb43b39c 100644 --- a/packages/flow/docs/deployment.example.yaml +++ b/packages/flow/docs/deployment.example.yaml @@ -128,8 +128,12 @@ effects: # field: patient_id # added to every row for matching # fhir (used when kind: fhir) ---------------------------------------------- - # base_url: https://openemr.example.org/apis/default/fhir - # resource_type: Observation + # base_url: https://fhir.example.org/r4 + # resource_type: Observation # required: the app's resource + # field_paths: # required: where each field lives + # id: id + # patient: subject.reference + # note: valueString # search_params: { category: vital-signs } # search_param_exprs: # patient: { param: patient_ref } # bound per-run diff --git a/packages/flow/openadapt_flow/app_profiles/__init__.py b/packages/flow/openadapt_flow/app_profiles/__init__.py index e559b6aeee..88747bef5f 100644 --- a/packages/flow/openadapt_flow/app_profiles/__init__.py +++ b/packages/flow/openadapt_flow/app_profiles/__init__.py @@ -10,7 +10,10 @@ rule's ``columns`` (none, by default) and may never go. With ``insert_entry_names_no_record``, the table is an id map or link row the app inserts with a save: the acting user's own insert entry for it that - names no record is the app's bookkeeping, not a write the run hid. + names no record is the app's bookkeeping, not a write the run hid. An entry + names no record when its record field is empty, or holds one of the + profile's ``no_record_values`` (an app may file such entries under record + ``0``). - ``columns``: an existing row may change in the named columns, and no row may be added or removed. Narrowed by ``fill_only`` (only from NULL or empty to a value: a backfill, never a swap, overwrite or clear), ``with_fill`` @@ -147,6 +150,9 @@ class AppProfile(BaseModel): system_columns: list[str] = Field(default_factory=list) #: Where the acting user's id is looked up from their sign-in name. users: Optional[UsersTable] = None + #: Record-field values with which the app's audit entries name no record + #: (besides an empty field), for ``insert_entry_names_no_record`` rules. + no_record_values: list[str] = Field(default_factory=list) rules: list[ProfileRule] @model_validator(mode="after") diff --git a/packages/flow/openadapt_flow/app_profiles/openemr.yaml b/packages/flow/openadapt_flow/app_profiles/openemr.yaml index 49c7fda99e..e72e319e4d 100644 --- a/packages/flow/openadapt_flow/app_profiles/openemr.yaml +++ b/packages/flow/openadapt_flow/app_profiles/openemr.yaml @@ -11,13 +11,15 @@ # excused. schema_version: openadapt.app-profile/v2 app: openemr -version: openemr-frame-profile/2026-10-29.5 +version: openemr-frame-profile/2026-11-03.1 bookkeeping_tables: [log, log_comment_encrypt, api_log, extended_log, audit_master, audit_details, clinical_rules_log, uuid_registry, uuid_mapping, contact, session_tracker, ip_tracking, background_services, sequences, users_secure, user_settings, recent_patients, users] system_columns: [uuid] users: {table: users, id_column: id, name_column: username} +# OpenEMR files an entry that names no chart under patient_id 0. +no_record_values: ["0"] rules: # Logs and id maps: rows are added, never changed or removed. On the four # marked insert_entry_names_no_record, OpenEMR inserts a row with a save or diff --git a/packages/flow/openadapt_flow/deployment.py b/packages/flow/openadapt_flow/deployment.py index 7aeabc22d6..717ebbe232 100644 --- a/packages/flow/openadapt_flow/deployment.py +++ b/packages/flow/openadapt_flow/deployment.py @@ -447,8 +447,10 @@ class EffectsConfig(BaseModel): #: Allowlist variant: when set, every evidence field NOT named is redacted. evidence_keep_fields: Optional[list[str]] = None - # -- fhir (FHIR R4 search, e.g. OpenEMR) --------------------------------- - resource_type: str = "Observation" + # -- fhir (FHIR R4 search) ----------------------------------------------- + #: ``fhir`` only, and required there with ``field_paths``: which resource + #: the app keeps the write in, and where each field lives in it. + resource_type: Optional[str] = None search_params: dict[str, str] = Field(default_factory=dict) field_paths: Optional[dict[str, str]] = None #: OAuth2 bearer token literal. DEPRECATED for committed configs — prefer @@ -630,6 +632,7 @@ def _serialize_compatible(self, handler: Any) -> dict[str, Any]: "records_next_path", "records_total_path", "records_path", + "resource_type", ): if data.get(added, "") is None: data.pop(added, None) @@ -2266,6 +2269,12 @@ def _build_effect_verifier_unredacted( if kind == "fhir": if not cfg.base_url: raise ValueError("effects.kind 'fhir' requires effects.base_url") + if not cfg.resource_type or not cfg.field_paths: + raise ValueError( + "effects.kind 'fhir' requires effects.resource_type and " + "effects.field_paths (which resource the app keeps the write " + "in, and where each field lives in it)" + ) from openadapt_flow.runtime.effects import FhirEffectVerifier access_token = cfg.access_token diff --git a/packages/flow/openadapt_flow/generate/ensure_contract.py b/packages/flow/openadapt_flow/generate/ensure_contract.py index a34a3747bb..4ecabfbbaf 100644 --- a/packages/flow/openadapt_flow/generate/ensure_contract.py +++ b/packages/flow/openadapt_flow/generate/ensure_contract.py @@ -1,7 +1,7 @@ """Generate an ``openadapt.ensure-record/v1`` contract for a FHIR write. A person used to write about half of a FHIR ensure-record contract by hand -(``benchmark/fhir_medplum/readpath/MINUTES.md``): the ``telecom[system=...]`` +(for example, for a FHIR server's patient write): the ``telecom[system=...]`` paths, the comparison kinds, the state map, the identity rules, the search parameter per identity value, the closed record and the change feed. This module derives each of them from what a teach session already has: @@ -119,32 +119,16 @@ def _load(path: Path) -> Any: raise GenerateError(f"cannot read {path}: {exc}") from exc -def facts_from_probe(probe: Mapping[str, Any]) -> dict[str, Any]: - """Server facts from ``benchmark/fhir_medplum/readpath/probe.py`` output. +def server_facts(doc: Mapping[str, Any]) -> dict[str, Any]: + """The generic server facts in a teach directory's ``facts.json``. - A probe that already reports the generic names passes through. + Only the names in :data:`FACTS` are read. A probe written for one server + translates its own output to these names, beside the probe in the + benchmark tree. A fact that is missing is not measured, and the contract + it would back is refused. """ - if all(k in probe for k in FACTS): - return {k: probe[k] for k in FACTS} - out: dict[str, Any] = {} - logged = probe.get("bundle_and_graphql_writes_logged") - if isinstance(logged, list): - out["audit_logs_writes"] = len(logged) > 0 - for name, key in ( - ("audit_names_actor", "audit_names_the_user_profile"), - ("agent_cannot_write_audit", "restricted_bot_audit_read_only"), - ("history_lists_versions", "delete_in_history"), - ("meta_author_returned", "meta_author_returned"), - ): - if key in probe: - out[name] = probe[key] - lags = probe.get("audit_lag_s") - if isinstance(lags, Mapping) and lags: - out["audit_lag_max_s"] = max( - float(v.get("max", 0)) for v in lags.values() if isinstance(v, Mapping) - ) - return out + return {k: doc[k] for k in FACTS if k in doc} def load_teach(root: str | Path, confirmed: Sequence[str] = ()) -> Teach: @@ -152,7 +136,7 @@ def load_teach(root: str | Path, confirmed: Sequence[str] = ()) -> Teach: ``teach.json`` (resource type and deployment settings), ``capability.json``, ``search_parameters.json`` (Bundle), ``facts.json`` - (generic facts or a probe's output) and ``demos//{params,before, + (the generic facts in :data:`FACTS`) and ``demos//{params,before, after}.json``. """ @@ -174,7 +158,7 @@ def load_teach(root: str | Path, confirmed: Sequence[str] = ()) -> Teach: demos=demos, capability=_load(base / "capability.json"), search_parameters=_resources(_load(sp_path)) if sp_path.exists() else [], - facts=facts_from_probe(_load(base / "facts.json")), + facts=server_facts(_load(base / "facts.json")), deployment=dict(meta.get("deployment") or {}), # A confirmation is a person's, given to the command, never read # from the teach data. diff --git a/packages/flow/openadapt_flow/reward/models.py b/packages/flow/openadapt_flow/reward/models.py index 8cac4bf805..4ebaa3ce2d 100644 --- a/packages/flow/openadapt_flow/reward/models.py +++ b/packages/flow/openadapt_flow/reward/models.py @@ -230,7 +230,7 @@ class OracleRecipeV1(_Strict): query: Optional[StrictStr] = None #: ``fhir``: resource type and search parameters; ``token_env`` names the #: bearer token variable. - resource_type: StrictStr = "Observation" + resource_type: Optional[StrictStr] = None search_params: dict[StrictStr, StrictStr] = Field(default_factory=dict) field_paths: Optional[dict[StrictStr, StrictStr]] = None token_env: Optional[StrictStr] = None @@ -247,6 +247,10 @@ def _kind_fields(self) -> "OracleRecipeV1": raise ValueError(f"oracle recipe {self.kind} requires base_url") if self.kind == "rest" and not self.records_path: raise ValueError("oracle recipe rest requires records_path") + if self.kind == "fhir" and not (self.resource_type and self.field_paths): + raise ValueError( + "oracle recipe fhir requires resource_type and field_paths" + ) if self.kind == "sqlite" and not self.query: raise ValueError("oracle recipe sqlite requires query") return self diff --git a/packages/flow/openadapt_flow/reward/oracles.py b/packages/flow/openadapt_flow/reward/oracles.py index f4e6194d77..90bdc31775 100644 --- a/packages/flow/openadapt_flow/reward/oracles.py +++ b/packages/flow/openadapt_flow/reward/oracles.py @@ -246,9 +246,9 @@ def connect() -> sqlite3.Connection: return VerifierOracle( FhirEffectVerifier( str(recipe.base_url), - resource_type=recipe.resource_type, + resource_type=str(recipe.resource_type), search_params=dict(recipe.search_params), - field_paths=recipe.field_paths, + field_paths=dict(recipe.field_paths or {}), access_token=recipe.token(), timeout_s=recipe.timeout_s, ), diff --git a/packages/flow/openadapt_flow/rollback.py b/packages/flow/openadapt_flow/rollback.py index 83a04167d3..cb85614240 100644 --- a/packages/flow/openadapt_flow/rollback.py +++ b/packages/flow/openadapt_flow/rollback.py @@ -345,6 +345,12 @@ def _bookkeeping_inserts(reader: Any) -> set[str]: return set(profile.no_record_insert_tables()) if profile is not None else set() +def _no_record_values(reader: Any) -> set[str]: + """Record-field values that name no record, from the app profile.""" + profile = _profile_of(reader) + return set(profile.no_record_values) if profile is not None else set() + + def _own_row_tables(reader: Any) -> set[str]: """Tables of the acting user's own rows the reader's app profile excuses.""" from openadapt_flow.app_profiles import table_name @@ -559,7 +565,11 @@ def hidden_writes( if attributor is None: return None return attributor.unaccounted_writes( - rows, window, _bookkeeping_inserts(reader), _own_row_tables(reader) + rows, + window, + _bookkeeping_inserts(reader), + _own_row_tables(reader), + _no_record_values(reader), ) record.after_stop_diff = landed.summary() diff --git a/packages/flow/openadapt_flow/rollback_state.py b/packages/flow/openadapt_flow/rollback_state.py index 06524d65a7..fcc624cab4 100644 --- a/packages/flow/openadapt_flow/rollback_state.py +++ b/packages/flow/openadapt_flow/rollback_state.py @@ -754,11 +754,13 @@ def unaccounted_writes( window: Optional[list[dict[str, Any]]], bookkeeping_inserts: set[str] = frozenset(), # type: ignore[assignment] own_tables: set[str] = frozenset(), # type: ignore[assignment] + no_record_values: set[str] = frozenset(), # type: ignore[assignment] ) -> Optional[list[dict[str, Any]]]: """The run's own logged writes that no changed row accounts for (a write to a table the read keeps append-only, say), or None when there is no audit window to tell. As the frame does: an insert entry - that names no record (none, or patient 0) in a table the app profile + that names no record (none, or a profile ``no_record_values`` value) + in a table the app profile calls bookkeeping (``no_record_insert_tables``) is the app's own, and so is an entry for a table of the acting user's own rows the profile excuses (``own_rows``: UI settings, recent charts).""" @@ -778,7 +780,7 @@ def unaccounted_writes( ): continue if ( - record in (None, "", "0", 0) + (record in (None, "") or str(record) in no_record_values) and self._event(e) in set(self.spec.insert_events) and table not in (None, "") and any(_same_table(table, t) for t in bookkeeping_inserts) diff --git a/packages/flow/openadapt_flow/runtime/effects/fhir.py b/packages/flow/openadapt_flow/runtime/effects/fhir.py index 2384806e9e..45b4a93e32 100644 --- a/packages/flow/openadapt_flow/runtime/effects/fhir.py +++ b/packages/flow/openadapt_flow/runtime/effects/fhir.py @@ -56,8 +56,9 @@ ) from openadapt_flow.verification import VerificationTier -#: OpenEMR's FHIR field paths for the canonical patient-note-as-Observation -#: write. Callers override for DocumentReference / Encounter / etc. +#: A field map for a note kept as a FHIR R4 Observation (the note in +#: ``valueString``). Not a default: a caller whose app stores its writes this +#: way passes it as ``field_paths``; any other resource or app names its own. DEFAULT_OBSERVATION_PATHS: dict[str, str] = { "id": "id", "patient": "subject.reference", @@ -188,18 +189,18 @@ def extract_path(resource: dict[str, Any], dotted: str) -> Any: class FhirEffectVerifier(VerifierAdapterBase): - """Verify effects against an OpenEMR FHIR R4 system of record. + """Verify effects against a FHIR R4 system of record. Args: - base_url: FHIR base URL (e.g. - ``https://demo.openemr.io/apis/default/fhir/R4``). + base_url: FHIR base URL (e.g. ``https://fhir.example.org/R4``). resource_type: FHIR resource type to search (``Observation``, - ``DocumentReference``, ``Encounter``, ...). + ``DocumentReference``, ``Encounter``, ...). Required. search_params: Query params identifying the candidate resource set (e.g. ``{"patient": "9", "category": "social-history"}``); used for both the pre-state snapshot and the post-action read. field_paths: Flat Effect key -> dotted FHIRPath-lite for flattening a - resource into the dict :func:`judge_records` matches on. + resource into the dict :func:`judge_records` matches on. Required: + where a field lives depends on the app and the resource. access_token: OAuth2 bearer token (``Authorization: Bearer ...``). token_source: Gets a token per read instead of ``access_token`` (mutually exclusive with it). @@ -218,9 +219,9 @@ def __init__( self, base_url: str, *, - resource_type: str = "Observation", + resource_type: str, + field_paths: dict[str, str], search_params: Optional[dict[str, str]] = None, - field_paths: Optional[dict[str, str]] = None, access_token: Optional[str] = None, token_source: Optional[AccessTokenSource] = None, session: Any = None, @@ -233,7 +234,9 @@ def __init__( self.drop_fields = tuple(drop_fields) self.resource_type = resource_type self.search_params = dict(search_params or {}) - self.field_paths = dict(field_paths or DEFAULT_OBSERVATION_PATHS) + if not resource_type or not field_paths: + raise ValueError("a FHIR read needs its resource_type and field_paths") + self.field_paths = dict(field_paths) if access_token and token_source is not None: raise ValueError("give access_token or token_source, not both") self.access_token = access_token diff --git a/packages/flow/openadapt_flow/runtime/effects/frame.py b/packages/flow/openadapt_flow/runtime/effects/frame.py index 1f31b56662..fa44829da5 100644 --- a/packages/flow/openadapt_flow/runtime/effects/frame.py +++ b/packages/flow/openadapt_flow/runtime/effects/frame.py @@ -826,6 +826,7 @@ def judge_frame( new_row_defaults: Optional[Mapping[str, Mapping[str, Optional[str]]]] = None, unread_changes: Sequence[tuple[str, str]] = (), no_record_inserts: Collection[str] = (), + no_record_values: Collection[str] = (), hashers: Optional[Mapping[str, Callable[[str, Any], str]]] = None, clock: tuple[str, str] = ("", ""), database_evidence: Optional[dict[str, Any]] = None, @@ -864,6 +865,9 @@ def judge_frame( no_record_inserts: Tables both database reads read as append-only whose profile rule marks the acting user's insert entry naming no record as the app's bookkeeping (an id map, a link row). + no_record_values: Record-field values that name no record in such an + entry, besides an empty field (the app profile's + ``no_record_values``). hashers: Slice -> (column, value) -> the keyed MAC its baseline records hold instead of values (database tables). A post-save record is plain; it is hashed to compare with the baseline. @@ -1934,14 +1938,15 @@ def names_moved(entry_ref: Any) -> bool: else "" ) if ( - ref in (None, "", "0", 0) + (ref in (None, "") or str(ref) in no_record_values) and attribution is not None and event in attribution.insert_events and entry_table not in (None, "") and any(same_table(entry_table, t) for t in no_record_inserts) ): # The bot's insert into an id map or link row a save - # makes (OpenEMR's uuid_registry), filed under no chart: + # makes (an id map such as a uuid registry), filed under + # no record: # both reads read the table, which allows the row and # judges every existing row. An update or delete, an # entry naming a record, or any other table still counts. diff --git a/packages/flow/openadapt_flow/verify_run.py b/packages/flow/openadapt_flow/verify_run.py index 4c8c516aa0..3f3f10c387 100644 --- a/packages/flow/openadapt_flow/verify_run.py +++ b/packages/flow/openadapt_flow/verify_run.py @@ -782,6 +782,8 @@ class _DatabaseFrame: #: Append-only tables in both reads whose profile rule excuses the acting #: user's insert entries that name no record. no_record_inserts: list[str] = field(default_factory=list) + #: The profile's record-field values that name no record. + no_record_values: list[str] = field(default_factory=list) #: The settled read after the save, and when it was taken: what a later #: recheck compares with (``recheck_after_s``). post: Any = None @@ -933,6 +935,7 @@ def row_key(table: str, record: Mapping[str, Any]) -> Optional[str]: profile = database.loaded_profile() no_record = profile.no_record_insert_tables() if profile is not None else set() + out.no_record_values = list(profile.no_record_values) if profile is not None else [] for name in sorted(set(old.tables) | set(new.tables)): a, b = old.tables.get(name), new.tables.get(name) try: @@ -1286,6 +1289,7 @@ def excuse(name: str, before: Any, after: Any, fields: list[str]) -> Any: new_row_defaults=None if db is None else db.defaults, unread_changes=() if db is None else db.unread, no_record_inserts=() if db is None else db.no_record_inserts, + no_record_values=() if db is None else db.no_record_values, hashers=None if db is None else db.hashers, clock=("", "") if db is None else db.clock, database_evidence=None if db is None else db.evidence, diff --git a/packages/flow/tests/test_app_profile.py b/packages/flow/tests/test_app_profile.py index 194a370995..7e71cd028f 100644 --- a/packages/flow/tests/test_app_profile.py +++ b/packages/flow/tests/test_app_profile.py @@ -282,3 +282,9 @@ def test_a_colleagues_own_ui_state_is_theirs_only_by_their_entry_for_it( "UPDATE users_secure SET login_fail_counter = 3 WHERE username = 'clinic2'", ) assert _frame(_run(tmp_path / "d", password)) == "refuted" + + +def test_the_openemr_profile_names_its_no_record_value() -> None: + """Record 0 names no chart in OpenEMR's data, not in the engine.""" + profile = load_profile("openemr") + assert profile.no_record_values == ["0"] diff --git a/packages/flow/tests/test_effect_fhir.py b/packages/flow/tests/test_effect_fhir.py index d49cc545d0..676e854ec4 100644 --- a/packages/flow/tests/test_effect_fhir.py +++ b/packages/flow/tests/test_effect_fhir.py @@ -9,6 +9,8 @@ from __future__ import annotations +from openadapt_flow.runtime.effects.fhir import DEFAULT_OBSERVATION_PATHS + import os import pytest @@ -38,6 +40,7 @@ def fhir(): def _verifier(base, *, token=None): return FhirEffectVerifier( base, + field_paths=dict(DEFAULT_OBSERVATION_PATHS), resource_type="Observation", search_params={"patient": PATIENT}, access_token=token, @@ -152,6 +155,7 @@ def test_fhir_expired_token_is_indeterminate_not_absent(fhir): try: v = FhirEffectVerifier( _base, + field_paths=dict(DEFAULT_OBSERVATION_PATHS), resource_type="Observation", search_params={"patient": PATIENT}, access_token="WRONG", @@ -172,6 +176,7 @@ def test_fhir_expired_token_is_indeterminate_not_absent(fhir): def test_fhir_unreachable_is_indeterminate(): v = FhirEffectVerifier( "http://127.0.0.1:1", + field_paths=dict(DEFAULT_OBSERVATION_PATHS), resource_type="Observation", search_params={"patient": PATIENT}, timeout_s=0.2, @@ -199,6 +204,7 @@ def test_live_openemr_fhir_reachable(): patient = os.environ.get("OPENEMR_FHIR_PATIENT", "1") v = FhirEffectVerifier( base, + field_paths=dict(DEFAULT_OBSERVATION_PATHS), resource_type="Observation", search_params={"patient": patient}, access_token=token, diff --git a/packages/flow/tests/test_effect_kit_config.py b/packages/flow/tests/test_effect_kit_config.py index cd1bb44304..3723260a85 100644 --- a/packages/flow/tests/test_effect_kit_config.py +++ b/packages/flow/tests/test_effect_kit_config.py @@ -8,6 +8,8 @@ from __future__ import annotations +from openadapt_flow.runtime.effects.fhir import DEFAULT_OBSERVATION_PATHS + import sqlite3 from pathlib import Path @@ -181,6 +183,8 @@ class TestFhirKitConfig: def test_search_param_exprs_and_token_env(self, monkeypatch): monkeypatch.setenv("KIT_FHIR_TOKEN", "oauth-token") cfg = EffectsConfig( + resource_type="Observation", + field_paths=dict(DEFAULT_OBSERVATION_PATHS), kind="fhir", base_url="https://emr.local/fhir", search_params={"category": "vital-signs"}, @@ -198,6 +202,8 @@ def test_search_param_exprs_and_token_env(self, monkeypatch): def test_token_env_missing_fails_loud(self, monkeypatch): monkeypatch.delenv("KIT_FHIR_ABSENT", raising=False) cfg = EffectsConfig( + resource_type="Observation", + field_paths=dict(DEFAULT_OBSERVATION_PATHS), kind="fhir", base_url="https://emr.local/fhir", access_token_env="KIT_FHIR_ABSENT", @@ -208,6 +214,8 @@ def test_token_env_missing_fails_loud(self, monkeypatch): def test_token_env_wins_over_literal(self, monkeypatch): monkeypatch.setenv("KIT_FHIR_TOKEN", "from-env") cfg = EffectsConfig( + resource_type="Observation", + field_paths=dict(DEFAULT_OBSERVATION_PATHS), kind="fhir", base_url="https://emr.local/fhir", access_token="literal-should-lose", @@ -401,3 +409,28 @@ def test_rest_needs_the_apps_records_path(): ) assert isinstance(v, RestRecordVerifier) assert (v.records_path, v.records_key) == (path, key) + + +def test_fhir_needs_the_apps_resource_and_field_paths(): + """No app's FHIR layout is assumed: kind fhir names its resource and paths.""" + with pytest.raises(ValueError, match="resource_type and"): + build_effect_verifier(EffectsConfig(kind="fhir", base_url="https://fhir.local")) + assert "resource_type" not in EffectsConfig(kind="none").model_dump() + # A note kept as an Observation (OpenEMR) and a Patient (Epic, Medplum) + # both work once named. + for resource, paths in ( + ("Observation", dict(DEFAULT_OBSERVATION_PATHS)), + ("Patient", {"id": "id", "family": "name.family", "birthDate": "birthDate"}), + ): + v = build_effect_verifier( + EffectsConfig( + kind="fhir", + base_url="https://fhir.local", + resource_type=resource, + field_paths=paths, + ) + ) + assert isinstance(v, FhirEffectVerifier) + assert (v.resource_type, v.field_paths) == (resource, paths) + with pytest.raises(TypeError): + FhirEffectVerifier("https://fhir.local") # type: ignore[call-arg] diff --git a/packages/flow/tests/test_effect_verifier.py b/packages/flow/tests/test_effect_verifier.py index 04b0b12afe..15542bd0a5 100644 --- a/packages/flow/tests/test_effect_verifier.py +++ b/packages/flow/tests/test_effect_verifier.py @@ -65,7 +65,12 @@ def test_all_verifiers_satisfy_protocol(tmp_path): assert isinstance(DocumentHashVerifier(tmp_path), EffectVerifier) from openadapt_flow.runtime.effects import FhirEffectVerifier - assert isinstance(FhirEffectVerifier("http://x"), EffectVerifier) + assert isinstance( + FhirEffectVerifier( + "http://x", resource_type="Observation", field_paths={"id": "id"} + ), + EffectVerifier, + ) # -- judge_records unit logic (no network) ---------------------------------- diff --git a/packages/flow/tests/test_effect_verifier_candidates.py b/packages/flow/tests/test_effect_verifier_candidates.py index 8a16767849..a01a1c0cdc 100644 --- a/packages/flow/tests/test_effect_verifier_candidates.py +++ b/packages/flow/tests/test_effect_verifier_candidates.py @@ -2,6 +2,8 @@ from __future__ import annotations +from openadapt_flow.runtime.effects.fhir import DEFAULT_OBSERVATION_PATHS + import pytest from openadapt_flow.deployment import ( @@ -368,17 +370,29 @@ def verify(self): def test_verifier_identity_binds_config_but_not_literal_secret(): first = build_effect_verifier( EffectsConfig( - kind="fhir", base_url="https://records.example/a", access_token="one" + resource_type="Observation", + field_paths=dict(DEFAULT_OBSERVATION_PATHS), + kind="fhir", + base_url="https://records.example/a", + access_token="one", ) ) rotated_secret = build_effect_verifier( EffectsConfig( - kind="fhir", base_url="https://records.example/a", access_token="two" + resource_type="Observation", + field_paths=dict(DEFAULT_OBSERVATION_PATHS), + kind="fhir", + base_url="https://records.example/a", + access_token="two", ) ) changed_boundary = build_effect_verifier( EffectsConfig( - kind="fhir", base_url="https://records.example/b", access_token="two" + resource_type="Observation", + field_paths=dict(DEFAULT_OBSERVATION_PATHS), + kind="fhir", + base_url="https://records.example/b", + access_token="two", ) ) diff --git a/packages/flow/tests/test_effect_verifier_transport.py b/packages/flow/tests/test_effect_verifier_transport.py index ee30982fef..de6176d57e 100644 --- a/packages/flow/tests/test_effect_verifier_transport.py +++ b/packages/flow/tests/test_effect_verifier_transport.py @@ -17,6 +17,8 @@ from __future__ import annotations +from openadapt_flow.runtime.effects.fhir import DEFAULT_OBSERVATION_PATHS + import builtins import json import threading @@ -95,12 +97,22 @@ def test_session_falls_back_to_httpx_without_requests( _hide(monkeypatch, "requests") assert isinstance( - FhirEffectVerifier("http://127.0.0.1:1")._get_session(), httpx.Client + FhirEffectVerifier( + "http://127.0.0.1:1", + resource_type="Observation", + field_paths=dict(DEFAULT_OBSERVATION_PATHS), + )._get_session(), + httpx.Client, ) def test_injected_session_is_always_preferred(self) -> None: sentinel = object() - verifier = FhirEffectVerifier("http://127.0.0.1:1", session=sentinel) + verifier = FhirEffectVerifier( + "http://127.0.0.1:1", + resource_type="Observation", + field_paths=dict(DEFAULT_OBSERVATION_PATHS), + session=sentinel, + ) assert verifier._get_session() is sentinel def test_records_are_read_over_the_fallback_transport( @@ -108,7 +120,11 @@ def test_records_are_read_over_the_fallback_transport( ) -> None: """The fallback is wired end to end, not merely constructible.""" _hide(monkeypatch, "requests") - state = FhirEffectVerifier(sor, resource_type="Observation").capture_pre_state() + state = FhirEffectVerifier( + sor, + field_paths=dict(DEFAULT_OBSERVATION_PATHS), + resource_type="Observation", + ).capture_pre_state() assert state.reachable is True assert [record["id"] for record in state.records] == ["obs-1"] @@ -121,7 +137,12 @@ def test_tls_verification_is_bound_on_the_fallback_client( the blanket except reads as an unreadable system of record. """ _hide(monkeypatch, "requests") - verifier = FhirEffectVerifier(sor, verify_tls=False) + verifier = FhirEffectVerifier( + sor, + resource_type="Observation", + field_paths=dict(DEFAULT_OBSERVATION_PATHS), + verify_tls=False, + ) assert verifier._get_session() is not None assert verifier._verify_per_request is False assert verifier.capture_pre_state().reachable is True diff --git a/packages/flow/tests/test_frame_conditions.py b/packages/flow/tests/test_frame_conditions.py index a1d50ba519..9160af0ac8 100644 --- a/packages/flow/tests/test_frame_conditions.py +++ b/packages/flow/tests/test_frame_conditions.py @@ -282,6 +282,12 @@ def test_a_bot_log_entry_on_an_undeclared_record_is_collateral() -> None: # Both reads read uuid_registry as append-only, and the profile marks # it: the bot's insert there, filed under no chart, is bookkeeping. (["openemr.uuid_registry"], {}, "held"), + # An empty record field names no record in any app. + (["openemr.uuid_registry"], {"record": None, "values": ()}, "held"), + (["openemr.uuid_registry"], {"record": "", "values": ()}, "held"), + # Record 0 names no record only where the app profile says so; in an + # app whose profile doesn't, 0 is a record id and the entry counts. + (["openemr.uuid_registry"], {"values": ()}, "refuted"), # No read of it, or a read of another table: the entry still refutes. ([], {}, "refuted"), (["openemr.log"], {}, "refuted"), @@ -294,6 +300,9 @@ def test_a_bot_log_entry_on_an_undeclared_record_is_collateral() -> None: def test_a_bot_insert_entry_naming_no_record_needs_that_table_read( read: list[str], entry: dict, status: str ) -> None: + entry = dict(entry) + # OpenEMR's profile: an entry filed under patient 0 names no chart. + values = entry.pop("values", ("0",)) spec = _audit_spec() assert spec.attribution is not None spec.attribution.table_field = "tbl" @@ -308,6 +317,7 @@ def test_a_bot_insert_entry_naming_no_record_needs_that_table_read( audit_before=[], audit_after=[logged], no_record_inserts=read, + no_record_values=values, ) assert verdict.status == status, verdict.reason diff --git a/packages/flow/tests/test_generate_ensure_contract.py b/packages/flow/tests/test_generate_ensure_contract.py index f52da87ec8..ed32381f62 100644 --- a/packages/flow/tests/test_generate_ensure_contract.py +++ b/packages/flow/tests/test_generate_ensure_contract.py @@ -303,7 +303,7 @@ def test_an_unexplained_element_of_a_told_apart_list_refuses(teach): "bad", [{"agent_cannot_write_audit": "false"}, {"audit_lag_max_s": "0.1"}] ) def test_server_facts_of_the_wrong_type_refuse(teach, bad): - teach.facts = dict(gen.facts_from_probe(dict(teach.facts)), **bad) + teach.facts = dict(gen.server_facts(dict(teach.facts)), **bad) out = generate(teach) assert any("wrong type" in r for r in out.refusals) diff --git a/packages/flow/tests/test_rollback.py b/packages/flow/tests/test_rollback.py index c56ae9581e..dd9cc5ae70 100644 --- a/packages/flow/tests/test_rollback.py +++ b/packages/flow/tests/test_rollback.py @@ -1780,8 +1780,9 @@ def test_a_rollback_that_isnt_back_is_always_reconciliation(tmp_path): def test_the_apps_own_logged_bookkeeping_is_accounted_for(): - # Round 9, as the frame decides: an insert naming no record (none, or - # OpenEMR's patient 0) in a bookkeeping table, and the acting user's own + # Round 9, as the frame decides: an insert naming no record (none, or a + # value the app profile lists, such as OpenEMR's patient 0) in a + # bookkeeping table, and the acting user's own # per-user rows, are the app's; a write to a disclosure log isn't. from openadapt_flow.runtime.effects.frame import FrameAttribution @@ -1810,10 +1811,14 @@ def entry(event, table, record): ] assert ( attributor.unaccounted_writes( - [], window, {"uuid_registry"}, {"recent_patients"} + [], window, {"uuid_registry"}, {"recent_patients"}, {"0"} ) == [] ) + # Without that profile value, record 0 is a record like any other. + assert attributor.unaccounted_writes( + [], window, {"uuid_registry"}, {"recent_patients"} + ) == [window[0]] hidden = [entry("replace", "extended_log", "4")] assert ( attributor.unaccounted_writes( diff --git a/packages/flow/tests/test_smart_backend_auth.py b/packages/flow/tests/test_smart_backend_auth.py index 3bcbe2235a..2595785b4e 100644 --- a/packages/flow/tests/test_smart_backend_auth.py +++ b/packages/flow/tests/test_smart_backend_auth.py @@ -9,6 +9,8 @@ from __future__ import annotations +from openadapt_flow.runtime.effects.fhir import DEFAULT_OBSERVATION_PATHS + import base64 import json import logging @@ -282,6 +284,7 @@ def _fhir_cfg(server: Server, smart: dict) -> EffectsConfig: return EffectsConfig.model_validate( { "kind": "fhir", + "field_paths": dict(DEFAULT_OBSERVATION_PATHS), "base_url": server.fhir, "resource_type": "Observation", "timeout_s": 2.0, @@ -548,6 +551,8 @@ def test_smart_backend_needs_fhir_and_no_static_token(): EffectsConfig.model_validate( { "kind": "fhir", + "resource_type": "Observation", + "field_paths": dict(DEFAULT_OBSERVATION_PATHS), "base_url": "https://x.test/fhir", "access_token_env": "TOKEN", "smart_backend": _smart(), @@ -614,7 +619,13 @@ def test_static_token_and_token_source_are_exclusive(rsa_key): token_url="https://auth.example.test/token", ) with pytest.raises(ValueError, match="not both"): - FhirEffectVerifier("https://x.test/fhir", access_token="t", token_source=source) + FhirEffectVerifier( + "https://x.test/fhir", + resource_type="Observation", + field_paths=dict(DEFAULT_OBSERVATION_PATHS), + access_token="t", + token_source=source, + ) @pytest.mark.parametrize( diff --git a/packages/flow/tests/test_verifier_adapter_platform.py b/packages/flow/tests/test_verifier_adapter_platform.py index c1d82c30eb..e437c7217e 100644 --- a/packages/flow/tests/test_verifier_adapter_platform.py +++ b/packages/flow/tests/test_verifier_adapter_platform.py @@ -11,6 +11,8 @@ from __future__ import annotations +from openadapt_flow.runtime.effects.fhir import DEFAULT_OBSERVATION_PATHS + import importlib.metadata from datetime import datetime, timedelta, timezone @@ -108,7 +110,11 @@ def test_builtin_adapters_conform_to_protocol(self, tmp_path): "http://sor", records_path="/api/db", records_key="records" ), GraphQLRecordVerifier("http://sor/graphql", query="query { r { id } }"), - FhirEffectVerifier("http://sor/fhir"), + FhirEffectVerifier( + "http://sor/fhir", + resource_type="Observation", + field_paths=dict(DEFAULT_OBSERVATION_PATHS), + ), SqlRecordVerifier(lambda: None, "SELECT id FROM t"), FileArrivalVerifier(str(tmp_path)), MaildirDeliveryVerifier(str(tmp_path)), From 764339b136dc0c970b8f960ea5ee5fdc19e06163 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 12:39:45 -0500 Subject: [PATCH 07/22] Engine generality audit; move the benchmark harnesses out of the wheel Audit every place the engine (packages/flow/openadapt_flow) names a specific application, bucketed as example, data, harness in the package, or real special-casing: docs/GENERALITY_AUDIT.md. Move openadapt_flow/benchmark and openadapt_flow/instrument to packages/flow/benchmark/harness (import benchmark.harness). The wheel packages only openadapt_flow, so it no longer ships them. The benchmark command loads the harness from a source checkout and says so from a wheel; instrument already failed to import from a wheel. Reword two comments that named an app on a generic branch. --- packages/flow/CHANGELOG.md | 2 + .../benchmark/agent_arm_verticals/README.md | 2 +- .../code_as_action/measure_openemr.py | 4 +- packages/flow/benchmark/dom/BENCHMARK.md | 4 +- .../flow/benchmark/effect_e2e/EFFECT_E2E.md | 2 +- .../benchmark/effect_e2e/record_service.py | 2 +- packages/flow/benchmark/effect_e2e/run.py | 8 +- .../effectbench/reference_fault_model.py | 10 +- .../benchmark/effectbench/task_pack/README.md | 2 +- .../effectbench/task_pack/__init__.py | 4 +- .../effectbench/task_pack/_authoring.py | 10 +- .../benchmark/effectbench/task_pack/audit.py | 2 +- .../benchmark/effectbench/task_pack/driver.py | 8 +- .../effectbench/task_pack/frappe_tasks.py | 8 +- .../effectbench/task_pack/mockmed_tasks.py | 8 +- .../effectbench/task_pack/openemr_tasks.py | 8 +- .../benchmark/effectbench/task_pack/pack.py | 6 +- .../flow/benchmark/frappe_lending/README.md | 2 +- .../harness}/__init__.py | 6 +- .../harness}/agent_baseline.py | 0 .../harness}/chart_fonts.py | 0 .../harness}/desktop_benchmark.py | 4 +- .../harness}/dom_arm.py | 20 +- .../harness}/effectbench/README.md | 2 +- .../harness}/effectbench/__init__.py | 8 +- .../harness}/effectbench/metrics.py | 4 +- .../harness}/effectbench/oracle.py | 6 +- .../harness}/effectbench/runner/README.md | 6 +- .../harness}/effectbench/runner/__init__.py | 26 +- .../harness}/effectbench/runner/__main__.py | 18 +- .../harness}/effectbench/runner/arms.py | 12 +- .../harness}/effectbench/runner/baselines.py | 8 +- .../harness}/effectbench/runner/compound.py | 8 +- .../harness}/effectbench/runner/harness.py | 14 +- .../effectbench/runner/reference_tasks.py | 12 +- .../harness}/effectbench/runner/substrate.py | 4 +- .../harness}/effectbench/schema.py | 4 +- .../harness}/frappe_lending.py | 0 .../harness}/hybrid_benchmark.py | 14 +- .../harness}/instrument/__init__.py | 4 +- .../harness}/instrument/competitor_drift.py | 6 +- .../harness}/openemr_benchmark.py | 12 +- .../harness}/openemr_local.py | 2 +- .../harness}/reliability.py | 0 .../harness}/run_benchmark.py | 10 +- .../harness}/silent_wrong_action.py | 8 +- .../benchmark => benchmark/harness}/verify.py | 0 packages/flow/benchmark/hybrid/BENCHMARK.md | 2 +- .../benchmark/lending_fault_model/README.md | 2 +- .../benchmark/lending_fault_model/swer.py | 4 +- .../openemr_api_tier/approval_durable.py | 2 +- .../flow/benchmark/openemr_api_tier/common.py | 2 +- .../benchmark/openemr_api_tier/feature_e2e.py | 4 +- .../benchmark/openemr_api_tier/spike/bench.py | 2 +- .../openemr_api_tier/spike/capture_har.py | 2 +- packages/flow/benchmark/openemr_e2e/README.md | 2 +- .../flow/benchmark/openemr_e2e/harness.py | 2 +- .../flow/benchmark/openemr_local/README.md | 2 +- .../benchmark/openemr_throughput/harness.py | 4 +- .../flow/benchmark/openimis_claims/README.md | 2 +- .../benchmark/scoreboard/tasks/openemr.py | 2 +- .../flow/benchmark/silent_saves/metrics.py | 2 +- .../SILENT_WRONG_ACTION.md | 2 +- .../flow/benchmark/vision_hardening/README.md | 2 +- packages/flow/docs/EFFECT_KIT.md | 2 +- packages/flow/docs/GENERALITY_AUDIT.md | 311 ++++++++++++++++++ packages/flow/docs/design/EFFECT_VERIFIER.md | 2 +- packages/flow/docs/desktop/PHASE2.md | 4 +- packages/flow/openadapt_flow/__main__.py | 25 +- .../flow/openadapt_flow/anthropic_defaults.py | 2 +- packages/flow/openadapt_flow/authoring.py | 2 +- packages/flow/openadapt_flow/deployment.py | 2 +- packages/flow/paper/REVIEW_ADVERSARIAL.md | 4 +- packages/flow/pyproject.toml | 2 +- packages/flow/scripts/desktop/waa_shim.py | 2 +- packages/flow/scripts/frappe_lending_demo.py | 4 +- packages/flow/scripts/openemr_demo.py | 10 +- packages/flow/scripts/openemr_local_demo.py | 4 +- packages/flow/scripts/openemr_param_depth.py | 2 +- .../test_region_stable_theme_regression.py | 2 +- packages/flow/tests/test_benchmark.py | 14 +- packages/flow/tests/test_chart_fonts.py | 4 +- .../tests/test_competitor_drift_instrument.py | 4 +- packages/flow/tests/test_desktop_benchmark.py | 4 +- packages/flow/tests/test_dom_arm.py | 8 +- packages/flow/tests/test_effectbench.py | 12 +- .../flow/tests/test_effectbench_runner.py | 16 +- .../flow/tests/test_effectbench_standalone.py | 6 +- .../flow/tests/test_effectbench_task_pack.py | 4 +- .../tests/test_frappe_lending_benchmark.py | 2 +- packages/flow/tests/test_hybrid_benchmark.py | 8 +- packages/flow/tests/test_lending_swer.py | 2 +- packages/flow/tests/test_openemr_benchmark.py | 12 +- .../tests/test_openemr_local_benchmark.py | 4 +- .../tests/test_openemr_retained_finals.py | 2 +- .../flow/tests/test_reliability_harness.py | 2 +- .../test_runtime_never_imports_benchmark.py | 58 ++-- .../test_silent_wrong_action_benchmark.py | 2 +- .../tests/test_ui_dismiss_live_openemr.py | 2 +- 99 files changed, 622 insertions(+), 274 deletions(-) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/__init__.py (63%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/agent_baseline.py (100%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/chart_fonts.py (100%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/desktop_benchmark.py (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/dom_arm.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/README.md (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/__init__.py (93%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/metrics.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/oracle.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/README.md (95%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/__init__.py (72%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/__main__.py (90%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/arms.py (95%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/baselines.py (97%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/compound.py (94%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/harness.py (92%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/reference_tasks.py (96%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/substrate.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/schema.py (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/frappe_lending.py (100%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/hybrid_benchmark.py (99%) rename packages/flow/{openadapt_flow => benchmark/harness}/instrument/__init__.py (91%) rename packages/flow/{openadapt_flow => benchmark/harness}/instrument/competitor_drift.py (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/openemr_benchmark.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/openemr_local.py (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/reliability.py (100%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/run_benchmark.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/silent_wrong_action.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/verify.py (100%) create mode 100644 packages/flow/docs/GENERALITY_AUDIT.md diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md index 33467f5c83..37dd88ec85 100644 --- a/packages/flow/CHANGELOG.md +++ b/packages/flow/CHANGELOG.md @@ -25,6 +25,8 @@ ### Fixes +- The wheel no longer ships the benchmark harnesses. `openadapt_flow.benchmark` and `openadapt_flow.instrument` moved to `packages/flow/benchmark/harness/` in the source tree (import `benchmark.harness`); `openadapt-flow benchmark` runs from a source checkout and says so from a wheel. The engine audit for application-specific code is in `docs/GENERALITY_AUDIT.md`. + - Self-repair on a covered identity band dismisses only the proved dialog: the covering window must span the band from where the read stops to its right edge at full height, the dismiss click must land in that same window, and only a dialog whose every word is a notice word (no question) may be dismissed ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). - Self-repair can take a stop where the app's own dialog covers the next step's identity band: when the backend proves the covering window belongs to the window the last verified step was in, and the part of the band read still matches the record, the dialog is dismissed after the previous click and the step re-checks its identity in full. Another record behind the dialog stays an identity halt. The runtime lets a backend remember a verified record's window (`record_verified_window`); macOS answers `owned_window_over` from the window's proved sheet ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). diff --git a/packages/flow/benchmark/agent_arm_verticals/README.md b/packages/flow/benchmark/agent_arm_verticals/README.md index fbfe9544dc..54526bd8fa 100644 --- a/packages/flow/benchmark/agent_arm_verticals/README.md +++ b/packages/flow/benchmark/agent_arm_verticals/README.md @@ -26,7 +26,7 @@ budget without completing the write in every trial. ## Method and public/private boundary The public, generic agent-baseline mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. It provides explicit paid-run +`benchmark/harness/agent_baseline.py`. It provides explicit paid-run opt-in, action and cost caps, usage accounting, and an agent self-report that is never trusted for scoring. The public benchmark schemas and aggregate failure taxonomy also remain available for independently authored tasks. diff --git a/packages/flow/benchmark/code_as_action/measure_openemr.py b/packages/flow/benchmark/code_as_action/measure_openemr.py index 74a34443a6..9e1b5e70f0 100644 --- a/packages/flow/benchmark/code_as_action/measure_openemr.py +++ b/packages/flow/benchmark/code_as_action/measure_openemr.py @@ -184,7 +184,7 @@ def spec_for(key: str) -> Any: - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec return SyntheticPatientSpec(**REFERRALS[key], **COMMON_FIELDS) @@ -390,7 +390,7 @@ def main() -> int: keys = [k for k in args.referrals.split(",") if k] from benchmark.openemr_local import qualify_hooks - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec fixture = qualify_hooks.fixture() demo = _demo() diff --git a/packages/flow/benchmark/dom/BENCHMARK.md b/packages/flow/benchmark/dom/BENCHMARK.md index 7111596c12..2b40374899 100644 --- a/packages/flow/benchmark/dom/BENCHMARK.md +++ b/packages/flow/benchmark/dom/BENCHMARK.md @@ -23,7 +23,7 @@ row's Open button — the literal reading — and **DOM (name-filtered)** clicks the Open button in the row named `Jane Sample` — the identity reading, hardcoding the demonstrated patient exactly as the compiled arm's recorded identity band does. Every selector choice is -documented in `openadapt_flow/benchmark/dom_arm.py`. +documented in `benchmark/harness/dom_arm.py`. ## Verdict @@ -208,7 +208,7 @@ Unmeasured variants, for completeness: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.dom_arm --out benchmark/dom --n-per-perturbation 2 +.venv/bin/python -m benchmark.harness.dom_arm --out benchmark/dom --n-per-perturbation 2 ``` (`--n-per-perturbation 2` matches the committed results.) No API key diff --git a/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md b/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md index dfa26c1e61..58868ee444 100644 --- a/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md +++ b/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md @@ -2,7 +2,7 @@ Date: 2026-07-21. This is the genuinely independent, end-to-end measurement of the silent-wrong-effect (SWER) result. Unlike the definitional -`openadapt_flow.benchmark.silent_wrong_action` (raw `requests.post`; effect +`benchmark.harness.silent_wrong_action` (raw `requests.post`; effect verifier and ground truth read the SAME in-process object; the effect contract restates the ground-truth definition -- so its `0/90` is circular by construction), every write below goes through the ACTUAL governed replay path diff --git a/packages/flow/benchmark/effect_e2e/record_service.py b/packages/flow/benchmark/effect_e2e/record_service.py index 2bb71f2e90..0f8f6e0fa3 100644 --- a/packages/flow/benchmark/effect_e2e/record_service.py +++ b/packages/flow/benchmark/effect_e2e/record_service.py @@ -3,7 +3,7 @@ This is the system of record the end-to-end silent-wrong-effect harness (:mod:`benchmark.effect_e2e.run`) writes to THROUGH the governed replay path. It exists to break the circularity of the older definitional benchmark -(``openadapt_flow.benchmark.silent_wrong_action``), whose effect verifier and +(``benchmark.harness.silent_wrong_action``), whose effect verifier and ground-truth judge read the SAME in-process Python object and whose write bypassed the :class:`~openadapt_flow.runtime.replayer.Replayer`. diff --git a/packages/flow/benchmark/effect_e2e/run.py b/packages/flow/benchmark/effect_e2e/run.py index d08601d691..7860d35b89 100644 --- a/packages/flow/benchmark/effect_e2e/run.py +++ b/packages/flow/benchmark/effect_e2e/run.py @@ -1,7 +1,7 @@ """End-to-end silent-wrong-effect (SWER) harness through the REAL replayer. This is the genuinely independent, end-to-end version of the silent-wrong-action -result. The older ``openadapt_flow.benchmark.silent_wrong_action`` is +result. The older ``benchmark.harness.silent_wrong_action`` is *definitional*: it issues raw ``requests.post`` (bypassing the :class:`~openadapt_flow.runtime.replayer.Replayer`), and its effect verifier and ground-truth judge both read the SAME in-process ``FaultDB`` object while the @@ -629,7 +629,7 @@ def render_markdown(results: dict[str, Any]) -> str: Date: {date}. This is the genuinely independent, end-to-end measurement of the silent-wrong-effect (SWER) result. Unlike the definitional -`openadapt_flow.benchmark.silent_wrong_action` (raw `requests.post`; effect +`benchmark.harness.silent_wrong_action` (raw `requests.post`; effect verifier and ground truth read the SAME in-process object; the effect contract restates the ground-truth definition -- so its `0/90` is circular by construction), every write below goes through the ACTUAL governed replay path @@ -774,7 +774,7 @@ def render_markdown(results: dict[str, Any]) -> str: def render_chart(results: dict[str, Any], out_png: Path) -> Path: """Bar chart: silent-wrong-effect rate per arm (large -> small -> zero).""" - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() m = results["metrics"]["per_arm"] @@ -827,7 +827,7 @@ def write_outputs(results: dict[str, Any], out_dir: Path) -> None: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") (out_dir / "EFFECT_E2E.md").write_text(render_markdown(results)) - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "effect_e2e.png") diff --git a/packages/flow/benchmark/effectbench/reference_fault_model.py b/packages/flow/benchmark/effectbench/reference_fault_model.py index 5ba4904208..0488c51dff 100644 --- a/packages/flow/benchmark/effectbench/reference_fault_model.py +++ b/packages/flow/benchmark/effectbench/reference_fault_model.py @@ -31,16 +31,16 @@ How it re-expresses the study, faithfully and without hardcoding verdicts: - Each of the study's nine scenarios (``benchmark.fault_model.faults.FAULTS``) - becomes a :class:`~openadapt_flow.benchmark.effectbench.schema.TaskSpec` plus + becomes a :class:`~benchmark.harness.effectbench.schema.TaskSpec` plus the KNOWN post-action system-of-record state that fault produces (the same DB states the study's own taxonomy tests assert). The states live in :data:`SCENARIOS` — the record shapes are MockMed ``FaultDB`` rows. - The TRUE effect is read by the substrate-agnostic - :class:`~openadapt_flow.benchmark.effectbench.oracle.RecordSnapshotOracle` + :class:`~benchmark.harness.effectbench.oracle.RecordSnapshotOracle` running the SAME shared ``judge_records`` decision logic the runtime uses — once for the at-most-once/collateral ``record_written`` contract and once for the ``field_equals`` note read-back — reduced by - :func:`~openadapt_flow.benchmark.effectbench.oracle.combine_true_states`. + :func:`~benchmark.harness.effectbench.oracle.combine_true_states`. - Two arms are scored against the identical independent oracle: ``screen_only`` (the weak banner oracle — reports success whenever the screen looks saved) and ``effect_verify`` (the agent halts unless the oracle CONFIRMS). Only the @@ -56,7 +56,7 @@ from typing import Optional from benchmark.fault_model import faults as F -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, DivergenceCategory, Effect, @@ -72,7 +72,7 @@ oracle_verdict_of, summarize, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel # The consequential-save contract the study verifies: exactly one Triage # encounter for the target patient (at-most-once + no collateral loss) whose diff --git a/packages/flow/benchmark/effectbench/task_pack/README.md b/packages/flow/benchmark/effectbench/task_pack/README.md index 88fa4d6c25..8846166f4c 100644 --- a/packages/flow/benchmark/effectbench/task_pack/README.md +++ b/packages/flow/benchmark/effectbench/task_pack/README.md @@ -68,7 +68,7 @@ python -m benchmark.effectbench.task_pack.pack # Live-run the MockMed anchor through the oracle (prints per-task classification) python -c "from benchmark.effectbench.task_pack.driver import run_mockmed_pack; \ -from openadapt_flow.benchmark.effectbench import summarize; \ +from benchmark.harness.effectbench import summarize; \ eps=run_mockmed_pack(trials=3); \ print('screen SWER', summarize(eps, arm='screen_only').swer.rate); \ print('effect SWER', summarize(eps, arm='effect_verify').swer.rate)" diff --git a/packages/flow/benchmark/effectbench/task_pack/__init__.py b/packages/flow/benchmark/effectbench/task_pack/__init__.py index e3f3537b7b..c823a82252 100644 --- a/packages/flow/benchmark/effectbench/task_pack/__init__.py +++ b/packages/flow/benchmark/effectbench/task_pack/__init__.py @@ -6,9 +6,9 @@ ``benchmark.environments`` registry (PR #173) indexes, authored against the frozen EffectBench schema/oracle/metrics contract (PR #178). -Each task is a :class:`~openadapt_flow.benchmark.effectbench.TaskSpec` (goal = +Each task is a :class:`~benchmark.harness.effectbench.TaskSpec` (goal = INTENT only, never a step list) plus an -:class:`~openadapt_flow.benchmark.effectbench.OracleSpec` bound to the +:class:`~benchmark.harness.effectbench.OracleSpec` bound to the environment's system-of-record READ recipe, DESIGNED so a green screen (task-success) diverges from the correct business effect. The oracle reads pre/post SoR state independently of the screen; the non-gameability attestations diff --git a/packages/flow/benchmark/effectbench/task_pack/_authoring.py b/packages/flow/benchmark/effectbench/task_pack/_authoring.py index 00e02bc078..aa0ba26354 100644 --- a/packages/flow/benchmark/effectbench/task_pack/_authoring.py +++ b/packages/flow/benchmark/effectbench/task_pack/_authoring.py @@ -3,10 +3,10 @@ These build the two things every consequential-write task needs: - the DECLARED expected effect (a typed - :class:`~openadapt_flow.benchmark.effectbench.Effect`), parameterized by run - params via :class:`~openadapt_flow.benchmark.effectbench.ValueExpr` so the + :class:`~benchmark.harness.effectbench.Effect`), parameterized by run + params via :class:`~benchmark.harness.effectbench.ValueExpr` so the record CHECKED is the record this trial WROTE (trial-unique payload); and -- the :class:`~openadapt_flow.benchmark.effectbench.OracleSpec` that declares +- the :class:`~benchmark.harness.effectbench.OracleSpec` that declares the independent read channel and the non-gameability attestations the adversarial audit signs off on. @@ -21,13 +21,13 @@ from typing import Any, Optional -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, EffectKind, OracleSpec, ValueExpr, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel # Canonical run-param names a trial binds a trial-unique value to. The driver # derives each from the trial seed so the oracle checks THIS run's exact write. diff --git a/packages/flow/benchmark/effectbench/task_pack/audit.py b/packages/flow/benchmark/effectbench/task_pack/audit.py index c44706010e..fcba2b8c15 100644 --- a/packages/flow/benchmark/effectbench/task_pack/audit.py +++ b/packages/flow/benchmark/effectbench/task_pack/audit.py @@ -41,7 +41,7 @@ _note_effect, _record_effect, ) -from openadapt_flow.benchmark.effectbench import RestRecordVerifier, Verdict +from benchmark.harness.effectbench import RestRecordVerifier, Verdict _HTTP_TIMEOUT_S = 5.0 TRIAL_NOTE = "triage note trial-unique-AUDIT" diff --git a/packages/flow/benchmark/effectbench/task_pack/driver.py b/packages/flow/benchmark/effectbench/task_pack/driver.py index 4145e46ae7..035a025097 100644 --- a/packages/flow/benchmark/effectbench/task_pack/driver.py +++ b/packages/flow/benchmark/effectbench/task_pack/driver.py @@ -8,9 +8,9 @@ mocked verdicts. It depends ONLY on the public EffectBench surface -(:func:`~openadapt_flow.benchmark.effectbench.score_episode`, -:class:`~openadapt_flow.benchmark.effectbench.RestRecordVerifier`, -:class:`~openadapt_flow.benchmark.effectbench.AgentReport`) and the MockMed +(:func:`~benchmark.harness.effectbench.score_episode`, +:class:`~benchmark.harness.effectbench.RestRecordVerifier`, +:class:`~benchmark.harness.effectbench.AgentReport`) and the MockMed fault server. It does not reimplement any real baseline "arm"; the two arms here are the two SCORING conditions the benchmark contrasts: @@ -46,7 +46,7 @@ MockMedDrive, MockMedTask, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, EpisodeRecord, RestRecordVerifier, diff --git a/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py b/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py index e2c22a2ef5..9219f750ea 100644 --- a/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py +++ b/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py @@ -17,7 +17,7 @@ ``oracle.config["query"]`` + ``["query_params"]`` is what the multi-baseline runner materializes into a -:class:`~openadapt_flow.benchmark.effectbench.SqlRecordVerifier` (read-only role, +:class:`~benchmark.harness.effectbench.SqlRecordVerifier` (read-only role, DB-API-bound params). The loan-application docname has a space (`` `tabLoan Application` ``), so the table is back-tick quoted in the SELECT. """ @@ -33,15 +33,15 @@ record_written_effect, ) from benchmark.effectbench.task_pack.openemr_tasks import ContainerTask -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( DivergenceCategory as DC, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, Substrate, TaskSpec, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel ENVIRONMENT = "frappe_lending" diff --git a/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py b/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py index 898f90db42..9f9b679bc3 100644 --- a/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py +++ b/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py @@ -8,7 +8,7 @@ path is exercised in CI (no mocked verdicts). Each entry is a :class:`MockMedTask`: an agent-agnostic -:class:`~openadapt_flow.benchmark.effectbench.TaskSpec` (goal = intent only) +:class:`~benchmark.harness.effectbench.TaskSpec` (goal = intent only) plus a :class:`MockMedDrive` — the environment recipe the live driver replays to MANIFEST the divergence (which ``?fault=`` mode to inject, the write the agent performs, decoy/concurrent rows to seed, and the documented banner @@ -34,9 +34,9 @@ param, record_written_effect, ) -from openadapt_flow.benchmark.effectbench import DivergenceCategory as DC -from openadapt_flow.benchmark.effectbench import Effect, Substrate, TaskSpec -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench import DivergenceCategory as DC +from benchmark.harness.effectbench import Effect, Substrate, TaskSpec +from benchmark.harness.effectbench.schema import OracleChannel TARGET_PATIENT = "p1" TARGET_TYPE = "Triage" diff --git a/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py b/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py index 2f6d61974d..7581aa2520 100644 --- a/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py +++ b/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py @@ -18,7 +18,7 @@ The SQL each oracle runs (``oracle.config["query"]`` + ``["query_params"]``) is what the multi-baseline runner materializes into a -:class:`~openadapt_flow.benchmark.effectbench.SqlRecordVerifier` (read-only role; +:class:`~benchmark.harness.effectbench.SqlRecordVerifier` (read-only role; values bound through DB-API params, never interpolated). Column names become record keys, so the same typed ``Effect`` the MockMed anchor uses judges these rows unchanged. @@ -36,15 +36,15 @@ param, record_written_effect, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( DivergenceCategory as DC, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, Substrate, TaskSpec, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel ENVIRONMENT = "openemr_local" diff --git a/packages/flow/benchmark/effectbench/task_pack/pack.py b/packages/flow/benchmark/effectbench/task_pack/pack.py index 2be0450827..fa15417f72 100644 --- a/packages/flow/benchmark/effectbench/task_pack/pack.py +++ b/packages/flow/benchmark/effectbench/task_pack/pack.py @@ -4,7 +4,7 @@ the container-gated OpenEMR and Frappe families). It: - assembles the unified list of :class:`PackEntry` (one per authored task), - each carrying its :class:`~openadapt_flow.benchmark.effectbench.TaskSpec`, the + each carrying its :class:`~benchmark.harness.effectbench.TaskSpec`, the pinned environment it targets, and whether it runs LIVE here or needs a container; - validates every task against the ``benchmark.environments`` registry (the @@ -29,8 +29,8 @@ from benchmark.effectbench.task_pack.frappe_tasks import FRAPPE_TASKS from benchmark.effectbench.task_pack.mockmed_tasks import MOCKMED_TASKS from benchmark.effectbench.task_pack.openemr_tasks import OPENEMR_TASKS -from openadapt_flow.benchmark.effectbench import TaskSpec -from openadapt_flow.benchmark.effectbench.schema import DivergenceCategory, Substrate +from benchmark.harness.effectbench import TaskSpec +from benchmark.harness.effectbench.schema import DivergenceCategory, Substrate MANIFEST_SCHEMA_VERSION = 1 diff --git a/packages/flow/benchmark/frappe_lending/README.md b/packages/flow/benchmark/frappe_lending/README.md index 1504ddf329..89ae27fa5a 100644 --- a/packages/flow/benchmark/frappe_lending/README.md +++ b/packages/flow/benchmark/frappe_lending/README.md @@ -30,7 +30,7 @@ trials per cell remains the publication bar. Raw per-run rows, environment fingerprints, application-specific adapter/oracle wiring, and the detailed cost ledger are retained in the private `OpenAdaptAI/openadapt-corpus` repository. The public generic cost-capped agent mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. See the +`benchmark/harness/agent_baseline.py`. See the [aggregate report](../agent_arm_verticals/README.md). The accepted run used baseline SHA-256 diff --git a/packages/flow/openadapt_flow/benchmark/__init__.py b/packages/flow/benchmark/harness/__init__.py similarity index 63% rename from packages/flow/openadapt_flow/benchmark/__init__.py rename to packages/flow/benchmark/harness/__init__.py index b157c4fda1..2eba51f30b 100644 --- a/packages/flow/openadapt_flow/benchmark/__init__.py +++ b/packages/flow/benchmark/harness/__init__.py @@ -1,4 +1,8 @@ -"""Benchmark: compiled replay vs. a frontier computer-use agent. +"""Benchmark harnesses: compiled replay vs. a frontier computer-use agent. + +This package lives in the source checkout, outside ``openadapt_flow``, so the +wheel does not ship it. Import it as ``benchmark.harness`` from +``packages/flow``. Submodules (import them directly; nothing heavy is imported here): diff --git a/packages/flow/openadapt_flow/benchmark/agent_baseline.py b/packages/flow/benchmark/harness/agent_baseline.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/agent_baseline.py rename to packages/flow/benchmark/harness/agent_baseline.py diff --git a/packages/flow/openadapt_flow/benchmark/chart_fonts.py b/packages/flow/benchmark/harness/chart_fonts.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/chart_fonts.py rename to packages/flow/benchmark/harness/chart_fonts.py diff --git a/packages/flow/openadapt_flow/benchmark/desktop_benchmark.py b/packages/flow/benchmark/harness/desktop_benchmark.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/desktop_benchmark.py rename to packages/flow/benchmark/harness/desktop_benchmark.py index 270605e54b..c581ca306d 100644 --- a/packages/flow/openadapt_flow/benchmark/desktop_benchmark.py +++ b/packages/flow/benchmark/harness/desktop_benchmark.py @@ -745,7 +745,7 @@ def _identity_transfer_verdict(results: dict) -> str: def render_chart(results: dict, path: Path) -> None: """Stacked outcome bars per arm (success / wrong-action / halt).""" - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() arms = list(results["arms"]) @@ -776,7 +776,7 @@ def write_outputs(results: dict, out_dir: str | Path) -> None: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") (out_dir / "BENCHMARK.md").write_text(render_markdown(results)) - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "outcomes.png") diff --git a/packages/flow/openadapt_flow/benchmark/dom_arm.py b/packages/flow/benchmark/harness/dom_arm.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/dom_arm.py rename to packages/flow/benchmark/harness/dom_arm.py index a6c0e598dd..27a1e277ef 100644 --- a/packages/flow/openadapt_flow/benchmark/dom_arm.py +++ b/packages/flow/benchmark/harness/dom_arm.py @@ -10,7 +10,7 @@ Three arms, one frozen schedule, one arm-independent success check: - **compiled**: the existing vision-anchored compiled replay - (:func:`openadapt_flow.benchmark.run_benchmark._compiled_run`), healing + (:func:`benchmark.harness.run_benchmark._compiled_run`), healing enabled as always. - **dom** (positional) and **dom_named** (name-filtered): :func:`dom_script` — a Playwright script written the way a competent @@ -29,7 +29,7 @@ drift modes (:data:`PERTURBATIONS`: lookalike, missing, grow, sort, theme, rename, move, typelabel). Success is judged for BOTH arms by the hybrid benchmark's own arm-independent check -(:func:`~openadapt_flow.benchmark.hybrid_benchmark.verify_hybrid_final`): +(:func:`~benchmark.harness.hybrid_benchmark.verify_hybrid_final`): OCR of the final screenshot must show the saved-encounter evidence AND the right patient, with wrong-type writes flagged — so this head-to-head is judged by the identical bar. Wrong-action events are recorded per arm: @@ -51,7 +51,7 @@ from pathlib import Path from typing import Any, Callable, Optional -from openadapt_flow.benchmark.hybrid_benchmark import ( +from benchmark.harness.hybrid_benchmark import ( _NOTE_POOL, DRIFT_TYPES, SCHEDULE, @@ -465,7 +465,7 @@ def dom_arm_aggregate(rows: list[dict[str, Any]]) -> dict[str, Any]: Aggregate dict: base stats plus clean/drift splits, outcome counts, wrong-action count, and (DOM arm) maintenance events. """ - from openadapt_flow.benchmark.run_benchmark import _arm_aggregate + from benchmark.harness.run_benchmark import _arm_aggregate agg = _arm_aggregate(rows) agg["clean"] = _split_rate([r for r in rows if r.get("condition") == "clean"]) @@ -621,7 +621,7 @@ def render_dom_chart(results: dict[str, Any], out_png: Path) -> Path: Returns: The written PNG path. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() from matplotlib.patches import Rectangle @@ -1177,7 +1177,7 @@ def stat_row(label: str, fmt: Callable[[dict[str, Any]], str]) -> str: clicks the Open button in the row named `{TARGET_PATIENT}` — the identity reading, hardcoding the demonstrated patient exactly as the compiled arm's recorded identity band does. Every selector choice is -documented in `openadapt_flow/benchmark/dom_arm.py`. +documented in `benchmark/harness/dom_arm.py`. ## Verdict @@ -1333,7 +1333,7 @@ def stat_row(label: str, fmt: Callable[[dict[str, Any]], str]) -> str: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.dom_arm --out benchmark/dom --n-per-perturbation 2 +.venv/bin/python -m benchmark.harness.dom_arm --out benchmark/dom --n-per-perturbation 2 ``` (`--n-per-perturbation 2` matches the committed results.) No API key @@ -1350,7 +1350,7 @@ def write_dom_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_dom_chart, results, out_dir / "outcome_matrix.png") (out_dir / "BENCHMARK.md").write_text(render_dom_markdown(results)) @@ -1388,7 +1388,7 @@ def run_dom_benchmark( Returns: The results dict (also written to ``results.json``). """ - from openadapt_flow.benchmark.run_benchmark import _compiled_run + from benchmark.harness.run_benchmark import _compiled_run from openadapt_flow.compiler import compile_recording from openadapt_flow.demo_driver import record_triage_demo from openadapt_flow.mockmed.server import serve @@ -1499,7 +1499,7 @@ def one_run( def main(argv: Optional[list[str]] = None) -> int: - """CLI entry point: ``python -m openadapt_flow.benchmark.dom_arm``.""" + """CLI entry point: ``python -m benchmark.harness.dom_arm``.""" parser = argparse.ArgumentParser( description=( "DOM-selector benchmark arm vs. compiled vision replay on " diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/README.md b/packages/flow/benchmark/harness/effectbench/README.md similarity index 99% rename from packages/flow/openadapt_flow/benchmark/effectbench/README.md rename to packages/flow/benchmark/harness/effectbench/README.md index c42718959f..08b926e621 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/README.md +++ b/packages/flow/benchmark/harness/effectbench/README.md @@ -5,7 +5,7 @@ > system-of-record oracle* disagrees. Reported jointly with **over-halt rate**, > **task success**, and the **success–effect gap**. -This package (`openadapt_flow.benchmark.effectbench`) is the **stable contract** +This package (`benchmark.harness.effectbench`) is the **stable contract** that two downstream efforts build on: 1. the **multi-baseline runner adapter** — drives each agent arm through a common diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/__init__.py b/packages/flow/benchmark/harness/effectbench/__init__.py similarity index 93% rename from packages/flow/openadapt_flow/benchmark/effectbench/__init__.py rename to packages/flow/benchmark/harness/effectbench/__init__.py index 7291bdbf04..62562d9441 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/__init__.py +++ b/packages/flow/benchmark/harness/effectbench/__init__.py @@ -31,10 +31,10 @@ bootstrap confidence intervals, all decomposed by category x substrate. The design doc is ``.private/benchmark_design_2026_07_20.md``; the operator -contract is ``openadapt_flow/benchmark/effectbench/README.md``. +contract is ``benchmark/harness/effectbench/README.md``. """ -from openadapt_flow.benchmark.effectbench.metrics import ( # noqa: F401 +from benchmark.harness.effectbench.metrics import ( # noqa: F401 BenchmarkSummary, CellSummary, Interval, @@ -44,7 +44,7 @@ summarize, wilson_interval, ) -from openadapt_flow.benchmark.effectbench.oracle import ( # noqa: F401 +from benchmark.harness.effectbench.oracle import ( # noqa: F401 RecordSnapshotOracle, TrueEffectState, classify_outcome, @@ -53,7 +53,7 @@ oracle_verdict_of, score_episode, ) -from openadapt_flow.benchmark.effectbench.schema import ( # noqa: F401 +from benchmark.harness.effectbench.schema import ( # noqa: F401 AgentReport, DivergenceCategory, EpisodeRecord, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/metrics.py b/packages/flow/benchmark/harness/effectbench/metrics.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/effectbench/metrics.py rename to packages/flow/benchmark/harness/effectbench/metrics.py index b7ad870e60..356119fde2 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/metrics.py +++ b/packages/flow/benchmark/harness/effectbench/metrics.py @@ -1,7 +1,7 @@ """EffectBench metrics — SWER and its co-metrics, with confidence intervals. Every headline is a pure aggregation over a list of -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord` rows, so a +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord` rows, so a run's numbers can be recomputed from the published raw rows by anyone. No single mean is reported alone: :func:`summarize` always returns the per (category x substrate) decomposition alongside the overall rate (design doc @@ -39,7 +39,7 @@ from pydantic import BaseModel, ConfigDict, Field -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( EpisodeRecord, OutcomeLabel, SwerVariant, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/oracle.py b/packages/flow/benchmark/harness/effectbench/oracle.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/effectbench/oracle.py rename to packages/flow/benchmark/harness/effectbench/oracle.py index f7f5ca031a..e490479d8b 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/oracle.py +++ b/packages/flow/benchmark/harness/effectbench/oracle.py @@ -17,7 +17,7 @@ 2. **The classifier** — :func:`classify_outcome`, which crosses the oracle's reading of the true effect with the agent's (untrusted) self-report to - assign exactly one :class:`~openadapt_flow.benchmark.effectbench.schema.OutcomeLabel`. + assign exactly one :class:`~benchmark.harness.effectbench.schema.OutcomeLabel`. This is ``fault_model.classify`` generalized: it adds the OVER_HALT vs SAFE_HALT split (was the correct action available?) and makes ``SILENT_WRONG_EFFECT`` the single home of ``reported_success AND @@ -25,7 +25,7 @@ :func:`score_episode` ties the two together for a runner: capture pre-state, run the arm's action, verify, classify, and emit an -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord`. +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord`. """ from __future__ import annotations @@ -34,7 +34,7 @@ from enum import Enum from typing import Any, Callable, Mapping, Optional -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( AgentReport, DivergenceCategory, EpisodeRecord, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/README.md b/packages/flow/benchmark/harness/effectbench/runner/README.md similarity index 95% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/README.md rename to packages/flow/benchmark/harness/effectbench/runner/README.md index a29834a364..a00d55f331 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/README.md +++ b/packages/flow/benchmark/harness/effectbench/runner/README.md @@ -1,7 +1,7 @@ # EffectBench runner adapter — one interface, every baseline The **multi-baseline runner adapter** (downstream effort #2 on the -`openadapt_flow.benchmark.effectbench` contract). It drives every agent *arm* +`benchmark.harness.effectbench` contract). It drives every agent *arm* through one common interface against the **identical task** and **identical independent oracle**, records one `EpisodeRecord` per (task × arm × trial), and summarizes per arm with `summarize`. A difference between arms is then the @@ -10,8 +10,8 @@ AGENT, never the harness. Run the CI-fast dry-run (real in-process HTTP, no Docker, no paid API, no spend): ```bash -python -m openadapt_flow.benchmark.effectbench.runner # all live arms -python -m openadapt_flow.benchmark.effectbench.runner --list-arms +python -m benchmark.harness.effectbench.runner # all live arms +python -m benchmark.harness.effectbench.runner --list-arms ``` ## The arm interface diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__init__.py b/packages/flow/benchmark/harness/effectbench/runner/__init__.py similarity index 72% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/__init__.py rename to packages/flow/benchmark/harness/effectbench/runner/__init__.py index 759ebae7d1..27fe3e39ff 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__init__.py +++ b/packages/flow/benchmark/harness/effectbench/runner/__init__.py @@ -2,20 +2,20 @@ Drives every agent *arm* through one common interface against the IDENTICAL task and INDEPENDENT oracle, records one -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord` per +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord` per (task × arm × trial), and summarizes per arm with -:func:`~openadapt_flow.benchmark.effectbench.metrics.summarize`. Built on the -stable contract in :mod:`openadapt_flow.benchmark.effectbench` (schema / oracle / +:func:`~benchmark.harness.effectbench.metrics.summarize`. Built on the +stable contract in :mod:`benchmark.harness.effectbench` (schema / oracle / metrics) — this package adds only the runner, never re-implements the taxonomy. Layers: - :mod:`.arms` — the ``AgentArm`` interface + the concrete in-repo arms: - :class:`~openadapt_flow.benchmark.effectbench.runner.arms.CompilerArm` + :class:`~benchmark.harness.effectbench.runner.arms.CompilerArm` (record→compile→replay, effect-gated), - :class:`~openadapt_flow.benchmark.effectbench.runner.arms.ScreenOnlyArm` + :class:`~benchmark.harness.effectbench.runner.arms.ScreenOnlyArm` (the silent-wrong-effect ablation), and a substrate-free - :class:`~openadapt_flow.benchmark.effectbench.runner.arms.MockArm` for CI. + :class:`~benchmark.harness.effectbench.runner.arms.MockArm` for CI. - :mod:`.substrate` — the arm-facing action+perception channel (the reference MockMed session), isolated from the oracle read path. - :mod:`.compound` — the compound consequential-save verifier (record + note). @@ -25,19 +25,19 @@ - :mod:`.harness` — ``run_episode`` / ``run_matrix``: wire each arm through ``score_episode`` with the task's oracle, fairly and identically. -Run the dry-run: ``python -m openadapt_flow.benchmark.effectbench.runner``. +Run the dry-run: ``python -m benchmark.harness.effectbench.runner``. """ from __future__ import annotations -from openadapt_flow.benchmark.effectbench.runner.arms import ( +from benchmark.harness.effectbench.runner.arms import ( AgentArm, ArmResult, CompilerArm, MockArm, ScreenOnlyArm, ) -from openadapt_flow.benchmark.effectbench.runner.baselines import ( +from benchmark.harness.effectbench.runner.baselines import ( SCAFFOLDED_ARMS, BaselineRequirements, ClaudeComputerUseArm, @@ -46,21 +46,21 @@ SkyvernArm, UITarsArm, ) -from openadapt_flow.benchmark.effectbench.runner.compound import ( +from benchmark.harness.effectbench.runner.compound import ( CompoundEffectVerifier, ) -from openadapt_flow.benchmark.effectbench.runner.harness import ( +from benchmark.harness.effectbench.runner.harness import ( EnvFactory, EpisodeEnv, run_episode, run_matrix, ) -from openadapt_flow.benchmark.effectbench.runner.reference_tasks import ( +from benchmark.harness.effectbench.runner.reference_tasks import ( MockMedEnvProvider, mockmed_env_factory, reference_tasks, ) -from openadapt_flow.benchmark.effectbench.runner.substrate import ( +from benchmark.harness.effectbench.runner.substrate import ( MockMedFault, MockMedSession, ScreenObservation, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__main__.py b/packages/flow/benchmark/harness/effectbench/runner/__main__.py similarity index 90% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/__main__.py rename to packages/flow/benchmark/harness/effectbench/runner/__main__.py index 4ba0d142eb..bc6c79734f 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__main__.py +++ b/packages/flow/benchmark/harness/effectbench/runner/__main__.py @@ -1,6 +1,6 @@ """EffectBench runner CLI — the CI-fast dry-run over the MockMed reference pack. -``python -m openadapt_flow.benchmark.effectbench.runner`` drives the nine MockMed +``python -m benchmark.harness.effectbench.runner`` drives the nine MockMed reference tasks through the in-repo LIVE arms (compiler, screen-only ablation, mock) over a real in-process HTTP fault server — no Docker, no paid API, no spend — and prints the per-arm summary. It proves the pipeline end-to-end: the @@ -25,16 +25,16 @@ import sys from typing import Optional, Sequence -from openadapt_flow.benchmark.effectbench.metrics import BenchmarkSummary, summarize -from openadapt_flow.benchmark.effectbench.runner import LIVE_ARMS -from openadapt_flow.benchmark.effectbench.runner.arms import AgentArm -from openadapt_flow.benchmark.effectbench.runner.baselines import SCAFFOLDED_ARMS -from openadapt_flow.benchmark.effectbench.runner.harness import run_matrix -from openadapt_flow.benchmark.effectbench.runner.reference_tasks import ( +from benchmark.harness.effectbench.metrics import BenchmarkSummary, summarize +from benchmark.harness.effectbench.runner import LIVE_ARMS +from benchmark.harness.effectbench.runner.arms import AgentArm +from benchmark.harness.effectbench.runner.baselines import SCAFFOLDED_ARMS +from benchmark.harness.effectbench.runner.harness import run_matrix +from benchmark.harness.effectbench.runner.reference_tasks import ( MockMedEnvProvider, reference_tasks, ) -from openadapt_flow.benchmark.effectbench.schema import EpisodeRecord +from benchmark.harness.effectbench.schema import EpisodeRecord def _select_arms(names: Optional[str]) -> list[AgentArm]: @@ -128,7 +128,7 @@ def run_dry_run( def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser( - prog="python -m openadapt_flow.benchmark.effectbench.runner", + prog="python -m benchmark.harness.effectbench.runner", description="EffectBench multi-baseline runner — MockMed dry-run.", ) parser.add_argument("--trials", type=int, default=3) diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/arms.py b/packages/flow/benchmark/harness/effectbench/runner/arms.py similarity index 95% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/arms.py rename to packages/flow/benchmark/harness/effectbench/runner/arms.py index 063389bcef..a63a5985c3 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/arms.py +++ b/packages/flow/benchmark/harness/effectbench/runner/arms.py @@ -18,14 +18,14 @@ def run(self, task: TaskSpec, session: SubstrateSession, An arm is handed ONLY the task's natural-language ``goal`` (intent, never a step list — the fairness requirement) via ``task``, a -:class:`~openadapt_flow.benchmark.effectbench.runner.substrate.SubstrateSession` +:class:`~benchmark.harness.effectbench.runner.substrate.SubstrateSession` to drive, and the run's ``params`` (the trial-unique payload it must write). It returns an :class:`ArmResult` carrying the untrusted -:class:`~openadapt_flow.benchmark.effectbench.schema.AgentReport` plus any -recorded :class:`~openadapt_flow.benchmark.effectbench.schema.ModelCall` rows so +:class:`~benchmark.harness.effectbench.schema.AgentReport` plus any +recorded :class:`~benchmark.harness.effectbench.schema.ModelCall` rows so cost is auditable. The harness (:mod:`.harness`) adapts ``run`` into the ``run_action: Callable[[], AgentReport]`` that -:func:`~openadapt_flow.benchmark.effectbench.oracle.score_episode` calls, and +:func:`~benchmark.harness.effectbench.oracle.score_episode` calls, and supplies the independent oracle — which the arm can NEVER reach. Three concrete in-repo arms live here: @@ -50,11 +50,11 @@ def run(self, task: TaskSpec, session: SubstrateSession, from dataclasses import dataclass, field from typing import Any, Callable, Mapping, Optional, Protocol, runtime_checkable -from openadapt_flow.benchmark.effectbench.runner.substrate import ( +from benchmark.harness.effectbench.runner.substrate import ( ScreenObservation, SubstrateSession, ) -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( AgentReport, ModelCall, TaskSpec, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/baselines.py b/packages/flow/benchmark/harness/effectbench/runner/baselines.py similarity index 97% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/baselines.py rename to packages/flow/benchmark/harness/effectbench/runner/baselines.py index 3674a508ce..5a43fbf73d 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/baselines.py +++ b/packages/flow/benchmark/harness/effectbench/runner/baselines.py @@ -21,7 +21,7 @@ side channel to the system of record; the independent oracle, not the agent, judges the effect. 3. **Record every model call.** Append one - :class:`~openadapt_flow.benchmark.effectbench.schema.ModelCall` per API + :class:`~benchmark.harness.effectbench.schema.ModelCall` per API request with token counts and the pinned list price, so ``cost_usd`` is auditable ``sum(model_calls)``. An RPA tool with a flat run fee records a single ``ModelCall`` carrying that fee. @@ -38,9 +38,9 @@ from dataclasses import dataclass from typing import Mapping -from openadapt_flow.benchmark.effectbench.runner.arms import ArmResult -from openadapt_flow.benchmark.effectbench.runner.substrate import SubstrateSession -from openadapt_flow.benchmark.effectbench.schema import TaskSpec +from benchmark.harness.effectbench.runner.arms import ArmResult +from benchmark.harness.effectbench.runner.substrate import SubstrateSession +from benchmark.harness.effectbench.schema import TaskSpec class ScaffoldNotWired(NotImplementedError): diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/compound.py b/packages/flow/benchmark/harness/effectbench/runner/compound.py similarity index 94% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/compound.py rename to packages/flow/benchmark/harness/effectbench/runner/compound.py index 1c6236449a..3c15f795f5 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/compound.py +++ b/packages/flow/benchmark/harness/effectbench/runner/compound.py @@ -9,16 +9,16 @@ * ``field_equals`` — that encounter's note reads back the run's exact value; catches a *partial save* that persists the row but drops a field. -:class:`~openadapt_flow.benchmark.effectbench.oracle.score_episode` verifies a +:class:`~benchmark.harness.effectbench.oracle.score_episode` verifies a SINGLE :class:`Effect` through the oracle. This module supplies a thin :class:`~openadapt_flow.runtime.effects.effect.EffectVerifier` that wraps ANY concrete base verifier (the in-process -:class:`~openadapt_flow.benchmark.effectbench.oracle.RecordSnapshotOracle`, a +:class:`~benchmark.harness.effectbench.oracle.RecordSnapshotOracle`, a :class:`~openadapt_flow.runtime.effects.rest.RestRecordVerifier`, …) and, when ``score_episode`` calls :meth:`verify` with the primary ``record_written`` effect, ALSO checks the authored extra sub-effects against the SAME pre-state snapshot, reducing the per-effect true-effect states with -:func:`~openadapt_flow.benchmark.effectbench.oracle.combine_true_states`. +:func:`~benchmark.harness.effectbench.oracle.combine_true_states`. It returns the single **deciding** :class:`EffectVerdict` whose ``effect_state`` equals the combined state (the fault-model precedence: the @@ -33,7 +33,7 @@ from typing import Any, Sequence -from openadapt_flow.benchmark.effectbench.oracle import ( +from benchmark.harness.effectbench.oracle import ( TrueEffectState, combine_true_states, effect_state, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/harness.py b/packages/flow/benchmark/harness/effectbench/runner/harness.py similarity index 92% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/harness.py rename to packages/flow/benchmark/harness/effectbench/runner/harness.py index f21991fed0..f1bafeefbf 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/harness.py +++ b/packages/flow/benchmark/harness/effectbench/runner/harness.py @@ -1,11 +1,11 @@ """The runner harness — wire each arm through ``score_episode`` identically. This is the multi-baseline runner's core: given a set of -:class:`~openadapt_flow.benchmark.effectbench.schema.TaskSpec` and a set of arms, +:class:`~benchmark.harness.effectbench.schema.TaskSpec` and a set of arms, it produces one -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord` per +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord` per (task × arm × trial), each scored by the task's INDEPENDENT oracle through -:func:`~openadapt_flow.benchmark.effectbench.oracle.score_episode`. Every arm +:func:`~benchmark.harness.effectbench.oracle.score_episode`. Every arm runs the SAME task and is judged by the SAME oracle — the harness guarantees the comparison is fair by construction: @@ -30,10 +30,10 @@ from dataclasses import dataclass, field from typing import Any, Callable, List, Mapping, Optional, Sequence -from openadapt_flow.benchmark.effectbench.oracle import score_episode -from openadapt_flow.benchmark.effectbench.runner.arms import AgentArm -from openadapt_flow.benchmark.effectbench.runner.substrate import SubstrateSession -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.oracle import score_episode +from benchmark.harness.effectbench.runner.arms import AgentArm +from benchmark.harness.effectbench.runner.substrate import SubstrateSession +from benchmark.harness.effectbench.schema import ( AgentReport, EpisodeRecord, ModelCall, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/reference_tasks.py b/packages/flow/benchmark/harness/effectbench/runner/reference_tasks.py similarity index 96% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/reference_tasks.py rename to packages/flow/benchmark/harness/effectbench/runner/reference_tasks.py index 6afe49e2be..9c4c8de44e 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/reference_tasks.py +++ b/packages/flow/benchmark/harness/effectbench/runner/reference_tasks.py @@ -3,7 +3,7 @@ These are the reference tasks the multi-baseline runner exercises end-to-end without Docker or spend: the bundled MockMed transactional-fault surface (``openadapt_flow.mockmed.fault_server``) turned into -:class:`~openadapt_flow.benchmark.effectbench.schema.TaskSpec` s across the +:class:`~benchmark.harness.effectbench.schema.TaskSpec` s across the divergence categories, plus a :func:`mockmed_env_factory` that provisions one isolated fault server per episode with an INDEPENDENT in-process oracle. @@ -24,17 +24,17 @@ from dataclasses import dataclass from typing import Mapping, Optional -from openadapt_flow.benchmark.effectbench.oracle import RecordSnapshotOracle -from openadapt_flow.benchmark.effectbench.runner.compound import ( +from benchmark.harness.effectbench.oracle import RecordSnapshotOracle +from benchmark.harness.effectbench.runner.compound import ( CompoundEffectVerifier, ) -from openadapt_flow.benchmark.effectbench.runner.harness import EpisodeEnv -from openadapt_flow.benchmark.effectbench.runner.substrate import ( +from benchmark.harness.effectbench.runner.harness import EpisodeEnv +from benchmark.harness.effectbench.runner.substrate import ( MockMedFault, MockMedSession, mockmed_session, ) -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( DivergenceCategory, OracleChannel, OracleSpec, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/substrate.py b/packages/flow/benchmark/harness/effectbench/runner/substrate.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/substrate.py rename to packages/flow/benchmark/harness/effectbench/runner/substrate.py index 3d4e525528..1322d730f6 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/substrate.py +++ b/packages/flow/benchmark/harness/effectbench/runner/substrate.py @@ -34,10 +34,10 @@ from dataclasses import dataclass, field from typing import Any, Mapping, Optional, Protocol, Sequence, runtime_checkable -from openadapt_flow.benchmark.effectbench.runner.compound import ( +from benchmark.harness.effectbench.runner.compound import ( CompoundEffectVerifier, ) -from openadapt_flow.benchmark.effectbench.schema import Substrate +from benchmark.harness.effectbench.schema import Substrate from openadapt_flow.runtime.effects.effect import Effect, EffectVerifier diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/schema.py b/packages/flow/benchmark/harness/effectbench/schema.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/effectbench/schema.py rename to packages/flow/benchmark/harness/effectbench/schema.py index 4e529ae716..d92fb0af90 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/schema.py +++ b/packages/flow/benchmark/harness/effectbench/schema.py @@ -135,7 +135,7 @@ class OracleChannel(str, Enum): Maps 1:1 onto the ``substrate`` attribute of the concrete :class:`~openadapt_flow.runtime.effects.effect.EffectVerifier` - implementations re-exported by :mod:`openadapt_flow.benchmark.effectbench`. + implementations re-exported by :mod:`benchmark.harness.effectbench`. """ SQL = "sql" # SqlRecordVerifier — read-only SELECT under a read-only role @@ -311,7 +311,7 @@ class EpisodeRecord(BaseModel): """One trial of one task by one arm — the atomic benchmark result row. Every headline metric is a pure aggregation over a list of these (see - :func:`openadapt_flow.benchmark.effectbench.metrics.summarize`). The runner + :func:`benchmark.harness.effectbench.metrics.summarize`). The runner produces one per (task x arm x trial); ``summarize`` never needs anything the row does not carry, so raw rows can be published for external replication. """ diff --git a/packages/flow/openadapt_flow/benchmark/frappe_lending.py b/packages/flow/benchmark/harness/frappe_lending.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/frappe_lending.py rename to packages/flow/benchmark/harness/frappe_lending.py diff --git a/packages/flow/openadapt_flow/benchmark/hybrid_benchmark.py b/packages/flow/benchmark/harness/hybrid_benchmark.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/hybrid_benchmark.py rename to packages/flow/benchmark/harness/hybrid_benchmark.py index c343d13b45..282bb79cbb 100644 --- a/packages/flow/openadapt_flow/benchmark/hybrid_benchmark.py +++ b/packages/flow/benchmark/harness/hybrid_benchmark.py @@ -67,12 +67,12 @@ from pydantic import BaseModel -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.openemr_benchmark import ( +from benchmark.harness import agent_baseline +from benchmark.harness.openemr_benchmark import ( BILLING_ABORT_AFTER, _looks_like_billing_error, ) -from openadapt_flow.benchmark.run_benchmark import ( +from benchmark.harness.run_benchmark import ( _agent_run, _arm_aggregate, _compiled_run, @@ -863,7 +863,7 @@ def render_hybrid_chart(results: dict[str, Any], out_png: Path) -> Path: Returns: The written PNG path. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() @@ -1379,7 +1379,7 @@ def failures(arm: str) -> str: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.hybrid_benchmark --out benchmark/hybrid +.venv/bin/python -m benchmark.harness.hybrid_benchmark --out benchmark/hybrid ``` Requires `ANTHROPIC_API_KEY` (or `~/.anthropic/api_key`). The paid arms @@ -1398,7 +1398,7 @@ def write_hybrid_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_hybrid_chart, results, out_dir / "success_cost.png") (out_dir / "BENCHMARK.md").write_text(render_hybrid_markdown(results)) @@ -1649,7 +1649,7 @@ def _error_row(arm: str, exc: Exception) -> dict[str, Any]: def main(argv: Optional[list[str]] = None) -> int: - """CLI entry point: ``python -m openadapt_flow.benchmark.hybrid_benchmark``.""" + """CLI entry point: ``python -m benchmark.harness.hybrid_benchmark``.""" parser = argparse.ArgumentParser( description=( "Four-arm hybrid benchmark on MockMed (compiled / agent / " diff --git a/packages/flow/openadapt_flow/instrument/__init__.py b/packages/flow/benchmark/harness/instrument/__init__.py similarity index 91% rename from packages/flow/openadapt_flow/instrument/__init__.py rename to packages/flow/benchmark/harness/instrument/__init__.py index bf92686661..dbf7469cc1 100644 --- a/packages/flow/openadapt_flow/instrument/__init__.py +++ b/packages/flow/benchmark/harness/instrument/__init__.py @@ -1,7 +1,7 @@ """Competitor-drift instrument: point our EffectVerifier at ANY external agent. The self-directed silent-wrong-action benchmark -(``openadapt_flow.benchmark.silent_wrong_action``, #67) measures how often our +(``benchmark.harness.silent_wrong_action``, #67) measures how often our OWN runtime lets a wrong / absent / duplicate business effect land while reporting success. This package generalizes that measurement to an *arbitrary external computer-use agent*: given any agent behind the @@ -19,7 +19,7 @@ user-gated step. """ -from openadapt_flow.instrument.competitor_drift import ( # noqa: F401 +from benchmark.harness.instrument.competitor_drift import ( # noqa: F401 AgentRunResult, ArchitectureClassError, CostGuard, diff --git a/packages/flow/openadapt_flow/instrument/competitor_drift.py b/packages/flow/benchmark/harness/instrument/competitor_drift.py similarity index 99% rename from packages/flow/openadapt_flow/instrument/competitor_drift.py rename to packages/flow/benchmark/harness/instrument/competitor_drift.py index 3daa5076c6..2abf9fe980 100644 --- a/packages/flow/openadapt_flow/instrument/competitor_drift.py +++ b/packages/flow/benchmark/harness/instrument/competitor_drift.py @@ -2,7 +2,7 @@ runner, cost-capped. This EXTENDS the self-directed silent-wrong-action benchmark -(``openadapt_flow.benchmark.silent_wrong_action``, #67) from "our own runtime" +(``benchmark.harness.silent_wrong_action``, #67) from "our own runtime" to "any external computer-use agent." The measurement is identical — did a wrong / absent / duplicate business effect land in the system of record while the actor reported success (a *silent wrong-action*)? — but the actor is now an @@ -53,7 +53,7 @@ # judge, and the #63 effect contract. We REUSE these from the self-directed # benchmark rather than re-deriving them so the external-agent instrument and # the self-directed instrument grade with byte-identical semantics. -from openadapt_flow.benchmark.silent_wrong_action import ( +from benchmark.harness.silent_wrong_action import ( NOTE, SCENARIOS, Scenario, @@ -888,7 +888,7 @@ def write_outputs(report: InstrumentReport, out_dir: Path | str) -> None: def main(argv: Optional[list[str]] = None) -> int: - """CLI: ``python -m openadapt_flow.instrument.competitor_drift``. + """CLI: ``python -m benchmark.harness.instrument.competitor_drift``. Ships only the offline stub adapter and a dry-run estimator — a real external adapter is wired programmatically (see diff --git a/packages/flow/openadapt_flow/benchmark/openemr_benchmark.py b/packages/flow/benchmark/harness/openemr_benchmark.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/openemr_benchmark.py rename to packages/flow/benchmark/harness/openemr_benchmark.py index f2482f2307..5bb2396f11 100644 --- a/packages/flow/openadapt_flow/benchmark/openemr_benchmark.py +++ b/packages/flow/benchmark/harness/openemr_benchmark.py @@ -7,7 +7,7 @@ recorded by ``scripts/openemr_demo.py`` (fake demo patients only). Both arms are judged by the same criterion, implemented once -(:func:`openadapt_flow.benchmark.verify.verify_note_saved`): OCR of the +(:func:`benchmark.harness.verify.verify_note_saved`): OCR of the final screen must show the run's parameterized note text in the patient message list. Each run — in BOTH arms — uses a distinct note value, so a pass proves parameter substitution against live state, not replay of a @@ -60,16 +60,16 @@ from pathlib import Path from typing import Any, Callable -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.agent_baseline import _looks_like_billing_error -from openadapt_flow.benchmark.run_benchmark import ( +from benchmark.harness import agent_baseline +from benchmark.harness.agent_baseline import _looks_like_billing_error +from benchmark.harness.run_benchmark import ( _agent_run, _arm_aggregate, _compiled_run, identity_coverage_block, render_chart, ) -from openadapt_flow.benchmark.verify import verify_note_saved +from benchmark.harness.verify import verify_note_saved DEMO_URL = "https://demo.openemr.io/openemr/index.php" N_COMPILED = 20 @@ -481,7 +481,7 @@ def write_openemr_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "latency_cost.png") (out_dir / "BENCHMARK.md").write_text(render_openemr_markdown(results)) diff --git a/packages/flow/openadapt_flow/benchmark/openemr_local.py b/packages/flow/benchmark/harness/openemr_local.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/openemr_local.py rename to packages/flow/benchmark/harness/openemr_local.py index 125f6d9c24..1a4e337de8 100644 --- a/packages/flow/openadapt_flow/benchmark/openemr_local.py +++ b/packages/flow/benchmark/harness/openemr_local.py @@ -18,7 +18,7 @@ from typing import Any, Iterable, Mapping, Sequence from urllib.parse import urlencode -from openadapt_flow.benchmark.frappe_lending import ( +from benchmark.harness.frappe_lending import ( ARMS, CONDITIONS, INITIAL_TRIALS_PER_CELL, diff --git a/packages/flow/openadapt_flow/benchmark/reliability.py b/packages/flow/benchmark/harness/reliability.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/reliability.py rename to packages/flow/benchmark/harness/reliability.py diff --git a/packages/flow/openadapt_flow/benchmark/run_benchmark.py b/packages/flow/benchmark/harness/run_benchmark.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/run_benchmark.py rename to packages/flow/benchmark/harness/run_benchmark.py index cd2c4c2d1e..3aba486faf 100644 --- a/packages/flow/openadapt_flow/benchmark/run_benchmark.py +++ b/packages/flow/benchmark/harness/run_benchmark.py @@ -29,8 +29,8 @@ from typing import Any, Callable from openadapt_flow.bench import _percentile -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.verify import verify_encounter_saved +from benchmark.harness import agent_baseline +from benchmark.harness.verify import verify_encounter_saved N_COMPILED = 100 N_AGENT = 20 @@ -153,7 +153,7 @@ def _agent_run( A per-run row dict. API failures are recorded as failed rows with ``error`` set, never raised; a failed row still carries whatever usage/cost the run paid for before crashing (via - :class:`~openadapt_flow.benchmark.agent_baseline.UsageLedger`), so + :class:`~benchmark.harness.agent_baseline.UsageLedger`), so crashed runs' real spend counts in aggregates and cost ceilings. """ from openadapt_flow.backends.playwright_backend import PlaywrightBackend @@ -364,7 +364,7 @@ def render_chart(results: dict[str, Any], out_png: Path) -> Path: Returns: The written PNG path. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() @@ -578,7 +578,7 @@ def write_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "latency_cost.png") (out_dir / "BENCHMARK.md").write_text(render_markdown(results)) diff --git a/packages/flow/openadapt_flow/benchmark/silent_wrong_action.py b/packages/flow/benchmark/harness/silent_wrong_action.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/silent_wrong_action.py rename to packages/flow/benchmark/harness/silent_wrong_action.py index 009bdda54c..daeb0c7c9f 100644 --- a/packages/flow/openadapt_flow/benchmark/silent_wrong_action.py +++ b/packages/flow/benchmark/harness/silent_wrong_action.py @@ -566,7 +566,7 @@ def render_markdown(results: dict[str, Any]) -> str: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.silent_wrong_action \\ +.venv/bin/python -m benchmark.harness.silent_wrong_action \\ --out {DEFAULT_OUT_DIR} --n {results["n_per_scenario"]} ``` @@ -585,7 +585,7 @@ def render_chart(results: dict[str, Any], out_png: Path) -> Path: false-abort rate. The metric a record system cares about — a wrong write that reports success — collapses from a large bar to zero. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() @@ -658,7 +658,7 @@ def write_outputs(results: dict[str, Any], out_dir: Path) -> None: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") (out_dir / "SILENT_WRONG_ACTION.md").write_text(render_markdown(results)) - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "silent_wrong_action.png") @@ -689,7 +689,7 @@ def run_and_write( def main(argv: Optional[list[str]] = None) -> int: - """CLI: ``python -m openadapt_flow.benchmark.silent_wrong_action``.""" + """CLI: ``python -m benchmark.harness.silent_wrong_action``.""" parser = argparse.ArgumentParser( description=( "Silent-wrong-action-rate benchmark: measure how often " diff --git a/packages/flow/openadapt_flow/benchmark/verify.py b/packages/flow/benchmark/harness/verify.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/verify.py rename to packages/flow/benchmark/harness/verify.py diff --git a/packages/flow/benchmark/hybrid/BENCHMARK.md b/packages/flow/benchmark/hybrid/BENCHMARK.md index bc64083f0d..d85401a2a7 100644 --- a/packages/flow/benchmark/hybrid/BENCHMARK.md +++ b/packages/flow/benchmark/hybrid/BENCHMARK.md @@ -207,7 +207,7 @@ $0.0967), and `a` = the agent-only mean cost per run ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.hybrid_benchmark --out benchmark/hybrid +.venv/bin/python -m benchmark.harness.hybrid_benchmark --out benchmark/hybrid ``` Requires `ANTHROPIC_API_KEY` (or `~/.anthropic/api_key`). The paid arms diff --git a/packages/flow/benchmark/lending_fault_model/README.md b/packages/flow/benchmark/lending_fault_model/README.md index 740b9134ff..b6de3a698a 100644 --- a/packages/flow/benchmark/lending_fault_model/README.md +++ b/packages/flow/benchmark/lending_fault_model/README.md @@ -39,7 +39,7 @@ judged by an independent ledger read, with zero model calls. 2. **Silent Wrong-Effect Rate, three-arm verification ladder** - `swer.py` scores the same faults through the shared EffectBench contract - (`openadapt_flow.benchmark.effectbench.score_episode` + `summarize`) under + (`benchmark.harness.effectbench.score_episode` + `summarize`) under screen-only, single-surface, and complete-read-path arms. -> `swer_results.json`, `SWER.md`. Headline: screen-only SWER is 24/36, a disbursements-only verifier leaves the diff --git a/packages/flow/benchmark/lending_fault_model/swer.py b/packages/flow/benchmark/lending_fault_model/swer.py index 9471d68615..312cc57e4a 100644 --- a/packages/flow/benchmark/lending_fault_model/swer.py +++ b/packages/flow/benchmark/lending_fault_model/swer.py @@ -5,7 +5,7 @@ screen-only postcondition contract (and measures how often that silently mishandles a transactional fault), THIS module measures the same faults through the shared EffectBench scoring contract -(:func:`openadapt_flow.benchmark.effectbench.score_episode`) under a THREE-arm +(:func:`benchmark.harness.effectbench.score_episode`) under a THREE-arm ladder that mirrors the clinical ``effect_e2e`` study (screen / single-surface / complete-read-path), so the two domains are directly comparable: @@ -71,7 +71,7 @@ import requests -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, DivergenceCategory, Effect, diff --git a/packages/flow/benchmark/openemr_api_tier/approval_durable.py b/packages/flow/benchmark/openemr_api_tier/approval_durable.py index 7b3e57f929..47dbe33709 100644 --- a/packages/flow/benchmark/openemr_api_tier/approval_durable.py +++ b/packages/flow/benchmark/openemr_api_tier/approval_durable.py @@ -157,7 +157,7 @@ def run_one( ) -> dict[str, Any]: from openadapt_flow.approval_mode.gate import ApprovalGate from openadapt_flow.approval_mode.item_approval import ItemApprovalStore - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer from openadapt_flow.runtime.durable.checkpoint import CheckpointStore diff --git a/packages/flow/benchmark/openemr_api_tier/common.py b/packages/flow/benchmark/openemr_api_tier/common.py index 19bb064547..8764d08f22 100644 --- a/packages/flow/benchmark/openemr_api_tier/common.py +++ b/packages/flow/benchmark/openemr_api_tier/common.py @@ -74,7 +74,7 @@ def load_demo() -> Any: def spec_for(ref: Referral) -> Any: - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec return SyntheticPatientSpec(**ref.kwargs) diff --git a/packages/flow/benchmark/openemr_api_tier/feature_e2e.py b/packages/flow/benchmark/openemr_api_tier/feature_e2e.py index 35754d3a4d..eccb0e8d1f 100644 --- a/packages/flow/benchmark/openemr_api_tier/feature_e2e.py +++ b/packages/flow/benchmark/openemr_api_tier/feature_e2e.py @@ -51,7 +51,7 @@ def cmd_record(a: argparse.Namespace) -> None: - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec from openadapt_flow.network_capture import NetworkCapture from openadapt_flow.recorder import Recorder @@ -190,7 +190,7 @@ def wait_quiet(max_s: float = float(os.environ.get("GATE_MAX_S", "1200"))) -> di def run_one(env: dict[str, Any], arm: str, ref_idx: int, bundle: Path, out: Path) -> dict[str, Any]: - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer diff --git a/packages/flow/benchmark/openemr_api_tier/spike/bench.py b/packages/flow/benchmark/openemr_api_tier/spike/bench.py index d3a74c2f53..138570dcca 100644 --- a/packages/flow/benchmark/openemr_api_tier/spike/bench.py +++ b/packages/flow/benchmark/openemr_api_tier/spike/bench.py @@ -62,7 +62,7 @@ def _refuse(self, *_a: Any, **_k: Any) -> Any: class Env: def __init__(self, browser: bool) -> None: - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer diff --git a/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py b/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py index 12929d4e07..5e6297d2ca 100644 --- a/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py +++ b/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py @@ -39,7 +39,7 @@ def main() -> None: from playwright.sync_api import sync_playwright - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer diff --git a/packages/flow/benchmark/openemr_e2e/README.md b/packages/flow/benchmark/openemr_e2e/README.md index 39fdb23bbe..68471ac1f1 100644 --- a/packages/flow/benchmark/openemr_e2e/README.md +++ b/packages/flow/benchmark/openemr_e2e/README.md @@ -10,7 +10,7 @@ compile → replay → effect-verify (system of record) → catch a silent wrong ``` It does **not** duplicate the existing OpenEMR *benchmark* -(`openadapt_flow.benchmark.openemr_benchmark`, which measures the compiled arm +(`benchmark.harness.openemr_benchmark`, which measures the compiled arm vs. a paid agent arm on the **live** public demo and is explicitly *not* CI-reproducible). This harness *orchestrates* the runtime components that benchmark assumes work — the compiler, the `Replayer`, the `EffectVerifier`, the diff --git a/packages/flow/benchmark/openemr_e2e/harness.py b/packages/flow/benchmark/openemr_e2e/harness.py index 23a10fc498..5110b31c77 100644 --- a/packages/flow/benchmark/openemr_e2e/harness.py +++ b/packages/flow/benchmark/openemr_e2e/harness.py @@ -8,7 +8,7 @@ inject drift -> HALT -> teach the fix -> re-run clean Unlike the existing OpenEMR *benchmark* -(:mod:`openadapt_flow.benchmark.openemr_benchmark`), which measures the compiled +(:mod:`benchmark.harness.openemr_benchmark`), which measures the compiled arm vs. a paid agent arm on the LIVE public demo and is explicitly NOT CI-reproducible, this harness proves the pipeline *wiring* end to end on a deterministic, offline fixture (the in-process MockMed ``fault_server`` as the diff --git a/packages/flow/benchmark/openemr_local/README.md b/packages/flow/benchmark/openemr_local/README.md index 6a9441c40a..19efb71a28 100644 --- a/packages/flow/benchmark/openemr_local/README.md +++ b/packages/flow/benchmark/openemr_local/README.md @@ -33,7 +33,7 @@ publication bar. Raw per-run rows, environment fingerprints, application-specific adapter/oracle wiring, and the detailed cost ledger are retained in the private `OpenAdaptAI/openadapt-corpus` repository. The public generic cost-capped agent mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. See the +`benchmark/harness/agent_baseline.py`. See the [aggregate report](../agent_arm_verticals/README.md). The accepted run used baseline SHA-256 diff --git a/packages/flow/benchmark/openemr_throughput/harness.py b/packages/flow/benchmark/openemr_throughput/harness.py index 4bcd15079b..fcf9a2e817 100644 --- a/packages/flow/benchmark/openemr_throughput/harness.py +++ b/packages/flow/benchmark/openemr_throughput/harness.py @@ -237,7 +237,7 @@ def _demo_module() -> Any: def build_worker(user: str, opts: dict[str, Any]) -> Any: """One worker signed in as ``user``. Module-level so a child process can build it (``ProcessWorker``); ``opts`` holds only plain values.""" - from openadapt_flow.benchmark.openemr_local import ( + from benchmark.harness.openemr_local import ( OpenEMRPatientOracle, SyntheticPatientSpec, ) @@ -345,7 +345,7 @@ def run_item(page: Any, idx: int, row: dict[str, Any], item_dir: Path, pipeline: def run(a: argparse.Namespace) -> dict[str, Any]: from openadapt_flow._browser_setup import ensure_chromium_installed - from openadapt_flow.benchmark.openemr_local import ( + from benchmark.harness.openemr_local import ( SyntheticPatientSpec, ) from openadapt_flow.ir import Workflow diff --git a/packages/flow/benchmark/openimis_claims/README.md b/packages/flow/benchmark/openimis_claims/README.md index 94e94ad017..e01f263e40 100644 --- a/packages/flow/benchmark/openimis_claims/README.md +++ b/packages/flow/benchmark/openimis_claims/README.md @@ -28,7 +28,7 @@ Raw per-run rows, environment fingerprints, application-specific adapter/oracle wiring, and the detailed cost ledger are retained in the private `OpenAdaptAI/openadapt-corpus` repository. The public generic cost-capped agent mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. See the +`benchmark/harness/agent_baseline.py`. See the [aggregate report](../agent_arm_verticals/README.md). There is still no full timing matrix and no publication protocol here: a diff --git a/packages/flow/benchmark/scoreboard/tasks/openemr.py b/packages/flow/benchmark/scoreboard/tasks/openemr.py index c2ffaf02d6..e93b7eab6d 100644 --- a/packages/flow/benchmark/scoreboard/tasks/openemr.py +++ b/packages/flow/benchmark/scoreboard/tasks/openemr.py @@ -207,7 +207,7 @@ def check(values: dict[str, str], before_ids: set[str]) -> str: def oracle_verifier(values: dict[str, str]) -> Any: - from openadapt_flow.benchmark.openemr_local import ( + from benchmark.harness.openemr_local import ( OpenEMRPatientOracle, SyntheticPatientSpec, ) diff --git a/packages/flow/benchmark/silent_saves/metrics.py b/packages/flow/benchmark/silent_saves/metrics.py index cb7f3ee9f1..2a8fa76483 100644 --- a/packages/flow/benchmark/silent_saves/metrics.py +++ b/packages/flow/benchmark/silent_saves/metrics.py @@ -6,7 +6,7 @@ import statistics from typing import Any, Iterable, Optional -from openadapt_flow.benchmark.effectbench.metrics import wilson_interval +from benchmark.harness.effectbench.metrics import wilson_interval def rate(k: int, n: int) -> dict[str, Any]: diff --git a/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md b/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md index c09304b4b5..ce1dfe56c9 100644 --- a/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md +++ b/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md @@ -66,7 +66,7 @@ run-to-run disagreement); N proves reproducibility. ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.silent_wrong_action \ +.venv/bin/python -m benchmark.harness.silent_wrong_action \ --out benchmark/silent_wrong_action --n 10 ``` diff --git a/packages/flow/benchmark/vision_hardening/README.md b/packages/flow/benchmark/vision_hardening/README.md index 273c387138..46644fdc38 100644 --- a/packages/flow/benchmark/vision_hardening/README.md +++ b/packages/flow/benchmark/vision_hardening/README.md @@ -36,7 +36,7 @@ and CLASSIFIES what the **real** resolver does on each generated case: go DOWN. A hardening fix regenerates the corpus and lowers the ratchet. The headline metric is **SWER** (Silent Wrong-Effect/Resolution Rate), the same -taxonomy as EffectBench (`openadapt_flow/benchmark/effectbench/`) — this sweep +taxonomy as EffectBench (`benchmark/harness/effectbench/`) — this sweep is the *resolution-tier* feeder for that benchmark. ## The two tiers diff --git a/packages/flow/docs/EFFECT_KIT.md b/packages/flow/docs/EFFECT_KIT.md index 3739752c74..cb02250ed9 100644 --- a/packages/flow/docs/EFFECT_KIT.md +++ b/packages/flow/docs/EFFECT_KIT.md @@ -1420,7 +1420,7 @@ Runs against the pinned reference fixture README documents the pinned compose bring-up and fixture bootstrap). The bundle's effect contracts are exactly the ones the reference matrix ships -(`openadapt_flow/benchmark/frappe_lending.py::loan_application_effects`): +(`benchmark/harness/frappe_lending.py::loan_application_effects`): one `record_written` (at-most-once for the synthetic applicant) plus a `field_equals` read-back per entered field, each bound to `{param: ...}`. diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md new file mode 100644 index 0000000000..7fb257ecdb --- /dev/null +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -0,0 +1,311 @@ +# Generality audit of the engine + +Audited 2026-11-03 on `main` at `213f3d9fe`. Scope: everything under +`packages/flow/openadapt_flow/`. + +The rule: the engine has no code written for one application. What one +application needs lives in data (`app_profiles/*.yaml`, generated contracts, +deployment configs) or in test fixtures and benchmark harnesses outside the +shipped package. `scripts/check_engine_generality.py` enforces it in CI; see +[CONTRIBUTING.md](../CONTRIBUTING.md#app-knowledge-lives-in-data). + +## Method + +1. Searched for application names, case-insensitive: OpenEMR, Medplum, + Dolibarr, EspoCRM, Redmine, Snipe-IT, InvenTree, Kimai, OpenMRS, Gitea, + Firefly III, Vikunja, Zammad, Open Dental, Odoo, openIMIS, Frappe/ERPNext, + Patient Notes, Epic, Stedi, TextEdit, Notepad, and OpenAdapt's own demo apps + MockMed and MockLoan. +2. Searched for identifiers that only make sense for one application: table and + column names (`pnotes`, `patient_data`, `uuid_registry`, `user_settings`, + `tabLoan Application`), URL paths (`/api/db`, `/api/encounter`, + `/apis/default/fhir`), form and query parameters (`token_main`, + `csrf_token_form`, `clearpass`), selectors, button labels, dialog titles, + field names, and thresholds tuned on one application's screens. +3. Read the code around each hit and put it in one of four buckets: + - **(a)** a comment or docstring example. Fine; reworded where it read as a + rule. + - **(b)** application knowledge stored as data and loaded generically. + - **(c)** a test fixture, benchmark harness or demo application inside the + shipped package. Moved out, unless it backs a user-facing command that + must work from a wheel. + - **(d)** special-casing in engine logic. Generalized, or turned into data. + +## Counts + +| Bucket | Rows | Distinct findings | +| --- | --- | --- | +| (a) comment or docstring example | 118 | — | +| (b) data loaded generically | 4 | 2 (`app_profiles/openemr.yaml`, the `clinical-write` policy pack as a default) | +| (c) fixture, harness or demo in the package | 35 rows, plus 27 files of `benchmark/` | 19 | +| (d) special-casing in engine logic | 27 | 15 | + +Rows group consecutive lines of one hit. The full table is at the end. + +## (d) findings and actions + +Each finding names the application that motivated it. "Second app" is the +existing fixture that shows the generalized code still works beyond the +motivating application. + +| # | Where | What | Motivating app | Action | Status | +| --- | --- | --- | --- | --- | --- | +| D1 | `report_redaction.py:68-70`, `:131-133` | Exact secret names `token_main`, `csrf_token_form` | OpenEMR | Match secret names by pattern (`token`, `csrf`, `xsrf`, `session`, ...) instead of a per-app list | planned | +| D2 | `network_capture.py:78` | `clearpass` in the secret-name pattern | OpenEMR | Drop it; the `pass\b` alternative already matches | planned | +| D3 | `runtime/effects/rest.py:114-115`, `deployment.py:387-390`, `reward/models.py:226`, `fault_gate.py:722` | REST reader defaults `records_path="/api/db"`, `records_key="records"` | MockMed | Required when the effect kind is `rest`; refused at config load when missing | planned | +| D4 | `runtime/effects/compensation.py:364` | `delete_path_template="/api/encounter/{id}"` default | MockMed | Required | planned | +| D5 | `compiler/binding_admission.py:114-119`, `:71-73` | Admission write path defaults to `/api/encounter`; body skips MockMed's `source` field | MockMed | Write path and method from the observed `api_write` event; skip only server-assigned fields | planned | +| D6 | `compiler/effect_mining.py:100-102`, `runtime/effects/effect.py:900-901` | Idempotency key field defaults to `key` | MockMed | No default; from the contract or deployment config | planned | +| D7 | `runtime/effects/fhir.py:59-67`, `:221`, `deployment.py:448-449` | FHIR defaults: resource type `Observation`, `note -> valueString` | OpenEMR | Required in the contract (`generate/ensure_contract.py` writes them) | planned | +| D8 | `generate/ensure_contract.py:123-147` | Translates the Medplum probe script's output keys | Medplum | The probe emits the generic fact names; translation removed | planned | +| D9 | `runtime/effects/frame.py:1937`, `rollback_state.py:781` | `"0"` means "no record" | OpenEMR | The value moves into the profile rule (`no_record_values`) | planned | +| D10 | `__main__.py:2838-2934`, `:9305-9319`; `cli_variants.py:131,173-182` | `replay` and the variant probe fall back to the bundled MockMed when no URL is given; `--drift` turns off structural grounding | MockMed | Target URL required outside the demo commands | planned | +| D11 | `qualification_oracle_gate.py` | `qualify accept` tests every workflow's oracle against MockMed (banner text, `/api/encounter`, `/api/db`) | MockMed | Build the fault fixture from the bundle's own effect binding | planned | +| D12 | `execute/dispatch.py:111-188` | The Execute service's live runner always runs the MockMed tutorial workflow | MockMed | Replay through the admission's deployment config; MockMed only for synthetic admissions | planned | +| D13 | `compiler/compile.py:151`, `runtime/replayer.py:394`, `runtime/identity.py:108` | Thresholds measured on MockMed only | MockMed | Generic logic, one-app numbers. Recalibrate on two or more apps | follow-up issue | +| D14 | `runtime/identity.py:275-302` | Hand-written list of non-identifying words drawn from EMR list screens | MockMed, EMRs | Derive from the recording (tokens shared by sibling rows) | follow-up issue | +| D15 | `eligibility/client.py:37-41`, `:1091-1110` | One vendor's eligibility API (URL, key name, body) in the engine | Stedi | Move behind a connector interface | follow-up issue | + +## (c) findings and actions + +| Where | What | Action | Status | +| --- | --- | --- | --- | +| `benchmark/` (27 Python files: `run_benchmark`, `openemr_benchmark`, `openemr_local`, `frappe_lending`, `hybrid_benchmark`, `dom_arm`, `silent_wrong_action`, `desktop_benchmark`, `verify`, `agent_baseline`, `reliability`, `chart_fonts`, `effectbench/`) | Benchmark harnesses shipped in the wheel | Moved to `packages/flow/benchmark/harness/` (import `benchmark.harness.*`). `openadapt-flow benchmark` loads it from a source checkout and says so from a wheel | done | +| `instrument/` | Competitor-drift harness; imported the benchmark at module load, so it was already broken in a wheel | Moved to `packages/flow/benchmark/harness/instrument/` | done | +| `integrations/harness2/` | Harness2 research harness on MockMed | Move to `packages/flow/benchmark/harness/harness2/` | planned | +| `mockloan/`, `demo_driver.record_disbursement_demo` | Lending demo app no product command uses | Move to the benchmark tree | planned | +| `guard_demo.py`, `agent/ledger_demo.py` | `python -m` demos, no CLI command | Move to the benchmark tree | planned | +| `mockmed/teach_scope.py` | Used only by tests | Move to `tests/fixtures` | planned | +| `learning/synth_stream.py:1-245` | MockMed note stream | Split out the generic `StructuralDiffInducer`, then move the stream | planned | +| `reward/seed.py:1-236` | MockMed reward seed | Split out the generic bundle helpers (`:238-410`), then move the seed behind the demo flag | planned | +| `record_check_proof.py:60-124` | `MOCKMED` fixture and `if name == "mockmed"` | Fixtures registered by name from the demo app's own module | planned | +| `validation/dense_surface.py` | Study harness rendering a MockMed table | Move to `benchmark/dense_surface` | planned | +| `__main__.py:5246-5275` (`bench`) | Always replays against MockMed | Accept `--url`, or load from a checkout like `benchmark` | planned | +| `execute/registry.py`, `execute/service.py`, `execute/app.py`, `execute/dispatch.py:60-108`, `--seed-mockmed` | Demo seeding inside the Execute service | Generic `seed_admissions`; the CLI passes the demo seed | planned | +| `eligibility/payer_routes.yaml`, `fleet/simulator.py`, `repair/dialog_fixture.py` | Synthetic test data and simulators in the package | Move to the benchmark tree or `tests/fixtures` | planned | + +### Kept in the package on purpose + +These back user-facing commands that must work from `pip install openadapt` +(`openadapt quickstart`, `openadapt-flow tutorial`, `demo`, `demo-record`, +`openadapt-agent serve` with no flags, `openadapt-agent example-app`). They are +OpenAdapt's own synthetic demo applications, not customer applications, and the +engine does not branch on them: + +- `mockmed/` (server, fault server, chart pages, static UI) +- `tutorial.py`, `demo_proof.py`, `demo_driver.record_triage_demo` +- `agent/sandbox.py`, `agent/tutorial.py`, `agent/example_app.py` + +The generality check allows MockMed names only in these modules. + +## (b) data + +- `app_profiles/openemr.yaml`: OpenEMR's bookkeeping writes, loaded by + `app_profiles.load_profile(name)`, which has no per-app code. +- The `clinical-write` policy pack is the default policy in + `execute/models.py:74`, `cli_config.py:164`, `integrations/agents/skills.py:351` + and `agent/teach.py:121`. It is data, but a healthcare default; whether to make + the policy required is a product decision, noted here. + +## (a) rewording done + +- `deployment.py:548`: the example environment variable in a user-facing + error is now `APP_READER_PASSWORD`. +- `authoring.py:807`: the comment on a generic branch no longer names MockMed. + +The remaining (a) rows are examples in comments and docstrings. The code they +describe is generic and configured per contract or profile. Healthcare words +such as MRN, DOB and patient appear in generic heuristics (for example +`risk.py:370`); that is the product's domain, not one application. + +## Full table + +Paths are relative to `packages/flow/openadapt_flow/` as of the audited commit. + +| Location | Bucket | App | Note and action | +| --- | --- | --- | --- | +| `__main__.py:2838-2934` | (d) | MockMed | replay: general code path defaults to serving bundled MockMed when no --url/config URL, and --drift flips use_structural=False (grounding ladder changes on a demo flag). Make --url required for replay (or route no-URL to `tutorial`/`demo`), rename --drift behavior to a generic --force-visual-grounding, keep drift injection in the demo command. | +| `__main__.py:9305-9319` | (d) | MockMed | replay parser: --url default = bundled MockMed; --drift MockMed modes (see 2838-2934). | +| `cli_variants.py:131,173-182` | (d) | MockMed | first-screen probe serves bundled MockMed when no --url (same default as replay). Refuse instead (first_screen_unavailable). | +| `compiler/binding_admission.py:114` | (d) | MockMed | AdmissionFixture.write_path default '/api/encounter' (line 119); compile.py:3379 never overrides it | +| `compiler/binding_admission.py:71` | (d) | MockMed | _BODY_SKIP_FIELDS adds MockMed's 'source' field (line 73) | +| `compiler/compile.py:146` | (d) | MockMed | REGION_STABLE_TOLERANCE=16 (line 151) calibrated on MockMed only; recalibrate on >=2 apps | +| `compiler/effect_mining.py:100` | (d) | MockMed | IDEMPOTENCY_KEY_FIELD='key' (line 102) is MockMed's field; take from contract/profile | +| `deployment.py:387` | (d) | MockMed | records_path default '/api/db' + records_key 'records' (lines 389-390) are MockMed's shape; make required for kind: rest | +| `deployment.py:448` | (d) | OpenEMR | fhir section default resource_type='Observation' (line 449) mirrors OpenEMR note-as-Observation; make required | +| `eligibility/client.py:37-41` | (d) | Stedi | Stedi 270/271 vendor client in engine (hard-coded URL, STEDI_API_KEY env, to_stedi_body); move to integrations/connector | +| `execute/dispatch.py:111-188` | (d) | MockMed | live_replay (the only real Execute runner) always calls tutorial.run_tutorial_workflow: POSTs /api/reset to target_url, appends MockMed ?fault=ok&idempotency=demo#tasks, verifies via /api/db 'records', forces clinical-write policy and 'note' param; with no target_url serves MockMed. Ignores AdmittedBundle.policy. Generalize: replay via the standard replay path with the admission's deployment.yaml (effects/backend/policy); keep MockMed only behind synthetic/demo admissions. | +| `fault_gate.py:722` | (d) | MockMed | rest_memory fault case default records_path '/api/db' (test harness; low) | +| `generate/ensure_contract.py:123` | (d) | Medplum | facts_from_probe translates Medplum probe key names (133-147); make the probe emit generic FACTS | +| `network_capture.py:78` | (d) | OpenEMR | SECRET_NAME regex lists 'clearpass' (OpenEMR login field); redundant with pass\b alternative -> drop | +| `qualification_oracle_gate.py:5,36,70,225,325-455` | (d) | MockMed | evaluate_oracle_gate (called by qualification_proposal.py:221,372 for every `qualify accept`) runs ANY workflow's proposed oracle against MockMed: hard-coded banner 'Encounter saved — ', POST /api/encounter?fault=optimistic with {patient_id, type:'Triage', note}, /api/db verifier. Generalize: a generic lying-store fixture driven by the bundle's own effect binding (write path/body/records path from the effect), or fixtures registered as data. | +| `report_redaction.py:131-133` | (d) | OpenEMR | _SECRET_KEYS lists token_main/csrf_token_form; same fix as DEFAULT_PARAMS | +| `report_redaction.py:68` | (d) | OpenEMR | token_main/csrf_token_form in DEFAULT_PARAMS (with :131-133 _SECRET_KEYS): replace with generic token/csrf name regex (reuse network_capture.TOKEN_NAME) or per-app secret_params in app profile | +| `reward/models.py:226` | (d) | MockMed | RecordReadRecipe.records_path default '/api/db' (MockMed's endpoint) in a general model; make it required for kind=rest. | +| `rollback_state.py:781` | (d) | OpenEMR | same '0' no-record sentinel as frame.py:1937 | +| `runtime/effects/compensation.py:364` | (d) | MockMed | RestCompensator delete_path_template default '/api/encounter/{id}'; make required | +| `runtime/effects/effect.py:900` | (d) | MockMed | Effect.key_field default 'key' (line 901) is MockMed's field | +| `runtime/effects/fhir.py:59` | (d) | OpenEMR | DEFAULT_OBSERVATION_PATHS (61-67) maps note->valueString per OpenEMR; require field_paths from contract | +| `runtime/effects/frame.py:1937` | (d) | OpenEMR | ref in (None,'','0',0): '0' = OpenEMR's 'no chart' (patient_id 0); gated by profile insert_entry_names_no_record; move sentinel into profile | +| `runtime/effects/rest.py:114-115` | (d) | MockMed | RestRecordVerifier defaults records_path='/api/db', records_key='records'; make required | +| `runtime/identity.py:104` | (d) | MockMed | COVERAGE_THRESHOLD=0.8 (line 108) calibrated on MockMed rows only | +| `runtime/identity.py:275-302` | (d) | MockMed/EMR list | _GENERIC_IDENTITY_WORDS hand-tuned EMR list vocabulary (incl. 'demo','chart','review','results'); derive from recording or app profile | +| `runtime/replayer.py:392` | (d) | MockMed | PC_STRUCTURAL_TEMPLATE_THRESHOLD=0.8 (line 394) calibrated on MockMed only | +| `__main__.py:10030-10037` | (c) | MockMed | bench parser. | +| `__main__.py:10166` | (c) | MockMed | verify-scaffold --prove choices ('mockmed','effect-e2e'): fixture names hard-coded in CLI; see record_check_proof.py. | +| `__main__.py:11940-11989,12341-12415` | (c) | MockMed | serve-execute/serve-reward --seed-mockmed flags and banner paths (data_dir/mockmed/records.db). Opt-in demo seeding; keep as demo flag or rename --seed-demo. | +| `__main__.py:1787-1806` | (c) | MockMed | demo-record command: serves MockMed, runs demo_driver.record_triage_demo. User-facing CLI; demo data could be a recorded script in data. | +| `__main__.py:1808-1998` | (c) | MockMed | tutorial command (via tutorial.py). Deliberate product feature; must work from wheel. | +| `__main__.py:1999-2040` | (c) | MockMed | demo command (demo_proof + tutorial). Deliberate product feature; must work from wheel. | +| `__main__.py:326-330` | (c) | MockMed | _with_drift: builds MockMed ?drift= query; only meaningful for the bundled demo. Move with demo helpers. | +| `__main__.py:5246-5275` | (c) | MockMed | bench command: always replays against bundled MockMed. Not generic; move to benchmark/ (source-checkout command, like `benchmark`) or accept --url. | +| `__main__.py:8773,8782` | (c) | MockMed | demo-record parser help/--drift. | +| `agent/example_app.py:1-400` | (c) | (own demo: reports portal) | `openadapt-agent example-app` local sales-order portal (/login clerk/portal-demo, /api/orders). User-facing demo command; importers agent/cli.py:1250 + 4 flow tests. | +| `agent/ledger_demo.py:1-720` | (c) | MockMed | scripted 30-referral worklist simulation (python -m). Not a CLI command; move to benchmark or packages/agent/tests fixtures; importer packages/agent/tests/test_ledger.py. | +| `agent/sandbox.py:1-530` | (c) | MockMed | `openadapt-agent serve` zero-flag sandbox: add_triage_note on MockMed, CASES with MockMed fault/drift queries, clinical-write. User-facing; must ship. Case table could be data (sandbox YAML). | +| `agent/tutorial.py:1-140` | (c) | MockMed | prepare_tutorial_session: hosts MockMed, writes deployment YAML with /api/db, clinical-write. Used by sandbox and agent/cli.py:760; packages/agent/tests/test_tutorial.py. | +| `demo_driver.py:1-96` | (c) | MockMed | record_triage_demo: MockMed selectors and creds (nurse.demo / mockmed-demo-pass). Used by demo-record CLI + benchmark (fault_model, undo_proposal, ui_readback, harness, cosmetic_drift), tests (recorder, e2e), scripts (demo_media, openemr_demo). Split: keep record_triage_demo with mockmed package. | +| `demo_driver.py:97-171` | (c) | MockLoan | record_disbursement_demo (mockloan-demo-pass): no product command uses it. Move to benchmark/lending_fault_model; importers: benchmark/undo_proposal/run.py, benchmark/lending_fault_model/{run,resolution_ladder}.py, tests/e2e/test_lending_fault_model.py. | +| `demo_proof.py:1-60,975-1110` | (c) | MockMed | `demo` page renderer; CROP 640x320 tuned to MockMed layout, 'Encounter saved', /api/db wording. User-facing; must ship. Crop could come from the demo-app spec. | +| `eligibility/payer_routes.yaml:1-` | (c) | Stedi/Cigna | synthetic TEST payer route data shipped in package | +| `execute/app.py:29,39,141,151` | (c) | MockMed | seed_mockmed passthrough; same as service. | +| `execute/dispatch.py:60-108` | (c) | MockMed | default_runner/synthetic_mockmed: synthetic admissions project canned MockMed tutorial outcomes (evidence_tag mockmed-*). Rename to synthetic_runner and move the canned projection to tests/fixtures or keep behind --seed-mockmed only. | +| `execute/registry.py:10-16,81-106` | (c) | MockMed | MOCKMED_* constants + seed_mockmed_admissions in engine registry. Move to execute/demo_seed.py (or tests/fixtures); importers: __main__ serve-execute, tests test_execute_ref/test_execute_post_dispatch_faults/test_jit_draft_execution/test_backend_parity_refusals. | +| `execute/service.py:45,74,84-85` | (c) | MockMed | ExecuteService(seed_mockmed=) flag. Replace with generic seed_admissions: Iterable[AdmittedBundle] and let the CLI pass the demo seed. | +| `fleet/simulator.py:1-` | (c) | synthetic payer portal | in-package simulator fixture; move to benchmark/examples | +| `guard_demo.py:1-296` | (c) | MockMed | `python -m openadapt_flow.guard_demo`: fake agent posts referral to MockMed with chart_pages READER creds; not a CLI subcommand. Move to packages/flow/examples or benchmark; importers: packages/agent/tests (1), packages/flow/tests (6). | +| `instrument/ (__init__.py, competitor_drift.py)` | (c) | MockMed | MockMed competitor-drift harness. Moved to packages/flow/benchmark/harness/instrument. | +| `integrations/harness2/ (all: tasks.py, apps.py, agent.py, adapter.py, traces.py, proposers.py, scorer.py, llm.py, pixels.py, rollout.py, selection.py, __init__.py)` | (c) | MockMed | Whole subpackage is a research harness for Harness2 on MockMed: MOCKMED_PATIENT/PATIENTS (tasks.py:71-148), MockMedTarget (apps.py:50-105, /api/db at :70), regex for MockMed task text and 'triage'/'New Encounter'/'encountersaved' (agent.py:221-324), MockMed banner regex Name — MRN — DOB (traces.py:55-57,128-138), clinical prompts (adapter.py:93-120,429-445; proposers.py:50-63). Move to packages/flow/benchmark/harness2; importers: benchmark/harness2_scorer/run.py, tests/test_harness2_{adapter,pixels,traces}.py, pyproject extra 'harness2', .github/workflows/packages.yml:451. | +| `learning/synth_stream.py:1-245` | (c) | MockMed | synthetic MockMed add-patient-note stream (intents, 'Consent Required', 'Belford, Phil DOB'). Move to benchmark/ or tests/fixtures. | +| `learning/synth_stream.py:246-525` | (c) | (generic) | StructuralDiffInducer is generic but lives in the MockMed harness module; imported by learning/teach.py:96, learning/branch_teach.py:76, benchmark/openemr_e2e/harness.py, 4 tests; pyproject mypy list (line 611). Move to learning/induce.py first, then move the stream out. | +| `mockloan/ (server.py, fault_server.py, static/)` | (c) | MockLoan | Own synthetic lending app shipped in wheel but used by no product command; only demo_driver.record_disbursement_demo, benchmark/lending_fault_model, benchmark/undo_proposal, tests (test_mockloan, test_lending_swer, e2e/test_lending_fault_model), scripts (1). Move to packages/flow/benchmark/apps/mockloan. | +| `mockmed/ (server.py, fault_server.py, chart_pages.py, teach_scope.py, static/)` | (c) | MockMed | Own synthetic demo app shipped in wheel. Required by tutorial/demo/demo-record/replay-default/sandbox and root openadapt/quickstart (engine.py, workers.py) + root tests/test_partner_kit.py. Keep shipping (or split to a demo-apps package). mockmed/teach_scope.py (EncounterWrites) is used only by tests -> move to tests/fixtures. chart_pages used by guard_demo + benchmark (agent_guard, effect_e2e, ui_readback). | +| `record_check_proof.py:1-11,60-124` | (c) | MockMed | --prove fixture registry hard-codes MOCKMED Fixture (judge on patient_id/note, write_path /api/encounter, body type Triage) and `if name == 'mockmed'`. Opt-in proof against our fixture. Move MOCKMED Fixture to mockmed/proof_fixture.py and resolve fixtures by entry point / dotted path. Importers: scaffold_verifier.py:594, benchmark/effect_e2e, tests (2). | +| `repair/dialog_fixture.py:1-` | (c) | synthetic | rendered test app in product package (no real app) | +| `reward/seed.py:1-236` | (c) | MockMed | seed_mockmed, MockMed encounters table/SQL (MOCKMED_TABLE 'encounters', SELECT id, patient_id, type, status), patients, episodes. Move to reward/demo_seed or tests/fixtures; CLI serve-reward --seed-mockmed uses it. | +| `reward/seed.py:238-410` | (c) | (generic) | write_bundle/calibrate_bundle/self_signed_certificate are generic helpers living in the MockMed seed module; split into reward/bundle.py. Importers: integrations/harness2/scorer.py, benchmark/undo_proposal/run.py, tests/test_probe_induction.py, test_undo_proposal.py, test_reward_worker.py, test_reward_trust_boundary.py. | +| `tutorial.py:1-560,630-841` | (c) | MockMed | tutorial engine: MockMed selectors (.open-btn,#new-encounter,#type-triage,#note,#save-encounter,#saved-banner), /api/reset, /api/db, ?fault=ok entry, TUTORIAL_POLICY clinical-write. User-facing (tutorial, demo, quickstart in root openadapt/quickstart/engine.py imports _records, run_tutorial_workflow, record_tutorial; agent/tutorial.py; execute/dispatch.py). Must ship in wheel; could be data-driven (demo-app spec YAML: entry query, reset/records endpoints, recorded click script). | +| `validation/dense_surface.py:418` | (c) | MockMed | validation study renders MockMed-branded table inside product package; move to benchmark/dense_surface | +| `__main__.py:5422,9499,10873` | (b) | (domain) | clinical-write as default/suggested policy name (built-in policy pack, data). | +| `app_profiles/openemr.yaml:1-` | (b) | OpenEMR | per-app bookkeeping profile loaded by load_profile(name); only profile shipped | +| `cli_config.py:164; integrations/agents/skills.py:351; agent/teach.py:121` | (b) | (domain) | default policy 'clinical-write' (shipped policy pack data). | +| `execute/models.py:74` | (b) | (domain) | AdmittedBundle.policy default 'clinical-write' (healthcare policy pack); consider requiring it. | +| `__main__.py:10053` | (a) | MockMed | benchmark help text (benchmark already moved to source-checkout harness). | +| `__main__.py:10536,10577` | (a) | MockMed | --admit-local help mentions MockMed; code path is generic. | +| `__main__.py:1149,1257-1266,1288` | (a) | MockMed | docstrings / refusal text about --drift being MockMed-only. | +| `__main__.py:3819` | (a) | MockMed | docstring: --drift forced off. | +| `__main__.py:7994` | (a) | TextEdit | help example for --backend macos owner app. | +| `__main__.py:9,27,39` | (a) | MockMed | module docstring command list. | +| `agent/cards.py:9-14; agent/skill.py:145-147; agent/bridge.py:132-133,325-326; agent/ledger.py:3,378; agent/delegated.py:11,78; agent/mailbox.py:130; agent/teach.py:281-283` | (a) | (domain) | referral / EMR / MRN examples in docstrings and tool descriptions; fine. | +| `api.py:84` | (a) | MockMed | docstring: replay without url serves MockMed (changes if replay default changes). | +| `app_profiles/__init__.py:315` | (a) | OpenEMR | comment/docstring example | +| `app_profiles/__init__.py:58` | (a) | OpenEMR | comment/docstring example | +| `authoring.py:807` | (a) | MockMed | comment 'Non-secret MockMed note' on a generic branch; reword to 'non-secret field'. | +| `backends/factory.py:117` | (a) | TextEdit | comment/docstring example | +| `backends/parallels_vm.py:403` | (a) | OpenDental | comment/docstring example | +| `backends/playwright_backend.py:890` | (a) | OpenEMR | comment/docstring example | +| `backends/win_agent/server.py:1578` | (a) | Notepad | comment/docstring example | +| `cli_api.py:71` | (a) | OpenEMR | help example 'lname'/'mrn' column names. | +| `cli_undo.py:181` | (a) | MockMed | help says default target is MockMed but _cmd_undo_run refuses that case; reword help. | +| `cli_undo.py:538-541` | (a) | MockMed | comment explaining refusal. | +| `compile_agent/code_action.py:135` | (a) | OpenEMR | comment/docstring example | +| `compiler/binding_admission.py:19` | (a) | MockMed | comment/docstring example | +| `compiler/compile.py:193` | (a) | OpenEMR | comment/docstring example | +| `compiler/compile.py:2184` | (a) | MockMed | comment/docstring example | +| `compiler/compile.py:717` | (a) | MockMed | comment/docstring example | +| `compiler/fixup_passes.py:7` | (a) | OpenEMR | history note; passes are generic | +| `compiler/local_edit.py:321` | (a) | MockMed | comment/docstring example | +| `deployment.py:1035` | (a) | OpenEMR | comment/docstring example | +| `deployment.py:112` | (a) | MockMed | comment/docstring example | +| `deployment.py:144` | (a) | TextEdit | comment/docstring example | +| `deployment.py:548` | (a) | OpenEMR | user-facing error example OPENEMR_READER_PASSWORD; reword to neutral APP_READER_PASSWORD | +| `ensure_record.py:114` | (a) | Medplum | comment/docstring example | +| `ensure_record.py:162` | (a) | Medplum | comment/docstring example | +| `ensure_record.py:97` | (a) | Medplum | comment/docstring example | +| `execute/dispatch.py:1-6` | (a) | MockMed | module docstring; update after generalizing live_replay. | +| `execution_profiles.py:2595` | (a) | MockMed | comment/docstring example | +| `fault_gate.py:822` | (a) | OpenEMR | comment/docstring example | +| `generate/demo_analysis.py:781` | (a) | Redmine | comment/docstring example | +| `generate/ensure_contract.py:4` | (a) | Medplum | comment/docstring example | +| `generate/verify_contract.py:1234` | (a) | Redmine | comment/docstring example | +| `generate/verify_contract.py:1275` | (a) | EspoCRM | comment/docstring example | +| `generate/verify_contract.py:1276` | (a) | Dolibarr | comment/docstring example | +| `generate/verify_contract.py:225` | (a) | Redmine | comment/docstring example | +| `generate/verify_contract.py:566` | (a) | EspoCRM | comment/docstring example | +| `integrations/agents/__init__.py:79` | (a) | (domain) | 'a clinic, a tenant' example. | +| `jit/authority.py:463` | (a) | TextEdit | comment/docstring example | +| `jit/authority.py:623` | (a) | TextEdit | comment/docstring example | +| `jit/compile_agent.py:354` | (a) | TextEdit | comment/docstring example | +| `jit/recorded.py:6` | (a) | MockMed | docstring example task id | +| `jit/signature.py:72` | (a) | MockMed | comment/docstring example | +| `learning/__init__.py:20; learning/halt_loop.py:8; learning/trace.py:60; learning/branch_teach.py:6,108; learning/library.py` | (a) | MockMed | docstring references. | +| `operator_profile.py:79` | (a) | OpenEMR | comment/docstring example | +| `policy_packs.py:41` | (a) | MockMed | user-facing pack summary names MockMed; fine as the demo app | +| `qualification_dev_signer.py:3` | (a) | MockMed | comment/docstring example | +| `qualification_proposal.py:10` | (a) | MockMed | comment/docstring example | +| `qualification_web_session.py:3` | (a) | OpenEMR,openIMIS | comment/docstring example | +| `qualify/config.py:303` | (a) | OpenEMR | comment/docstring example | +| `record_check_binding.py:48` | (a) | openIMIS | comment/docstring example | +| `record_check_identity.py:4` | (a) | OpenEMR | comment/docstring example | +| `record_check_synthesis.py:25` | (a) | MockMed | comment/docstring example | +| `registry.py:3` | (a) | OpenEMR | comment/docstring example | +| `registry.py:813` | (a) | OpenEMR | comment/docstring example | +| `report_redaction.py:5` | (a) | OpenEMR | comment/docstring example | +| `reward/models.py:497; reward/calibration.py:13` | (a) | (domain) | 'triage rows' wording in docstrings. | +| `risk.py:476` | (a) | openIMIS | comment/docstring example | +| `risk.py:565` | (a) | OpenEMR | comment/docstring example | +| `rollback_state.py:77` | (a) | OpenEMR | comment/docstring example | +| `runtime/actuators/web_form.py:214` | (a) | OpenEMR | comment/docstring example | +| `runtime/actuators/web_form.py:324` | (a) | validate.js | var constraints recognizer is a JS library convention (OpenEMR among users), not app-specific | +| `runtime/actuators/web_form.py:331` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/__init__.py:21` | (a) | MockMed | comment/docstring example | +| `runtime/effects/__init__.py:23` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/__init__.py:38` | (a) | Frappe | comment/docstring example | +| `runtime/effects/_common.py:143` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/auth.py:33` | (a) | Frappe | comment/docstring example | +| `runtime/effects/db_snapshot.py:713` | (a) | Dolibarr | comment/docstring example | +| `runtime/effects/effect.py:1536` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/effect.py:1541` | (a) | OpenEMR,MockMed | comment/docstring example | +| `runtime/effects/effect.py:206` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/effect.py:208` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/effect.py:223` | (a) | Redmine | comment/docstring example | +| `runtime/effects/effect.py:903` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:1` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:12` | (a) | MockMed | comment/docstring example | +| `runtime/effects/fhir.py:14` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:191` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:195` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:3` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:38` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:386` | (a) | Epic | comment/docstring example | +| `runtime/effects/fhir.py:39` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:42` | (a) | MockMed | comment/docstring example | +| `runtime/effects/frame.py:135` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:144` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:1499` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:1630` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:1944` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:242` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:46` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fresh_session.py:24` | (a) | MockMed | comment/docstring example | +| `runtime/effects/rest.py:237` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/rest.py:5` | (a) | MockMed | comment/docstring example | +| `runtime/effects/sql.py:12` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:215` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:215` | (a) | Frappe | comment; _TABLE_NAME allows space generically | +| `runtime/effects/sql.py:3` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:365` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/sql.py:4` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:480` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:534` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:535` | (a) | Frappe | comment/docstring example | +| `runtime/grounder.py:136-155` | (a) | healthcare domain | _INTENT_STOPWORDS include patient/mrn; domain wording, intent-only; low | +| `submit_binding.py:8` | (a) | openIMIS | comment/docstring example | +| `validation/dense_surface.py:1271` | (a) | OpenEMR | comment/docstring example | +| `validation/dense_surface.py:5` | (a) | OpenEMR | comment/docstring example | +| `verifier.py:2467` | (a) | OpenEMR | comment/docstring example | +| `vision/hashing.py:10` | (a) | MockMed | docstring for the MockMed-calibrated phash tolerance (see compile.py:151) | +| `volatility.py:100` | (a) | OpenEMR | comment/docstring example | +| `volatility.py:31` | (a) | OpenEMR | comment/docstring example | +| `volatility.py:6` | (a) | OpenEMR | comment/docstring example | +| `benchmark/` (27 files) | (c) | OpenEMR, Frappe, MockMed, Open Dental | Moved to `packages/flow/benchmark/harness/` | diff --git a/packages/flow/docs/design/EFFECT_VERIFIER.md b/packages/flow/docs/design/EFFECT_VERIFIER.md index b7364ebbcf..b54ec7cb59 100644 --- a/packages/flow/docs/design/EFFECT_VERIFIER.md +++ b/packages/flow/docs/design/EFFECT_VERIFIER.md @@ -80,7 +80,7 @@ the note-as-Observation write the fake models cannot be created over FHIR on a stock OpenEMR; the property the live test establishes is the one the fake could not — the verdicts are correct against a **real FHIR server**, not a fake. CI still runs vision-only against the **public demo** -(`openadapt_flow/benchmark/openemr_benchmark.py`, `scripts/openemr_demo.py`); +(`benchmark/harness/openemr_benchmark.py`, `scripts/openemr_demo.py`); the live FHIR test is gated behind `OPENEMR_FHIR_BASE_URL` and skipped otherwise. See `benchmark/openemr_live/README.md` for the one-command bring-up. diff --git a/packages/flow/docs/desktop/PHASE2.md b/packages/flow/docs/desktop/PHASE2.md index 6eb67329e8..16eadb2505 100644 --- a/packages/flow/docs/desktop/PHASE2.md +++ b/packages/flow/docs/desktop/PHASE2.md @@ -58,7 +58,7 @@ so `WindowsBackend` sees exactly what it drives). | `scripts/desktop/patient_notes.ps1` | WinForms target app (list-select → edit note → save); drift knobs via `pn_env.json` | | `scripts/desktop/pn_db.py` | SQLite ground-truth CLI (seed/list/save/get/all) | | `scripts/desktop/uia_arm.py` | UIA-selector arm (pywinauto), identity + positional | -| `openadapt_flow/benchmark/desktop_benchmark.py` | Orchestrator: 3 arms × drift matrix, DB judge, results.json/BENCHMARK.md/chart | +| `benchmark/harness/desktop_benchmark.py` | Orchestrator: 3 arms × drift matrix, DB judge, results.json/BENCHMARK.md/chart | ## Snapshots (reversibility + warm boot) @@ -78,7 +78,7 @@ state without a full VM revert. ```bash # one-off full matrix (6 conditions × 3 arms × n): -python -m openadapt_flow.benchmark.desktop_benchmark --out benchmark/desktop --n 3 +python -m benchmark.harness.desktop_benchmark --out benchmark/desktop --n 3 ``` `DesktopHarness.connect()` starts the VM, launches the session-1 shim, deploys diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index bb52425ba8..9a1f71f1d6 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -5337,8 +5337,31 @@ def _cmd_silent_saves(args: argparse.Namespace) -> int: return int(silent_saves_main(list(args.silent_saves_argv))) +def _checkout_harness_on_path(command: str) -> bool: + """Put the checkout's ``benchmark`` harness on ``sys.path``. + + The harness lives in ``packages/flow/benchmark/harness``, outside the + package, so a wheel install has none and the command says so. + """ + checkout = Path(__file__).resolve().parents[1] + if not (checkout / "benchmark" / "harness" / "__init__.py").is_file(): + print( + f"{command} runs from a source checkout of OpenAdapt " + "(packages/flow/benchmark/harness); the wheel does not ship it.", + file=sys.stderr, + ) + return False + if str(checkout) not in sys.path: + sys.path.insert(0, str(checkout)) + return True + + def _cmd_benchmark(args: argparse.Namespace) -> int: - from openadapt_flow.benchmark.run_benchmark import run_benchmark + if not _checkout_harness_on_path("benchmark"): + return 2 + from benchmark.harness.run_benchmark import ( # type: ignore[import-not-found] + run_benchmark, + ) results = run_benchmark( Path(args.out), diff --git a/packages/flow/openadapt_flow/anthropic_defaults.py b/packages/flow/openadapt_flow/anthropic_defaults.py index 22ec8ce45b..fc4201c29e 100644 --- a/packages/flow/openadapt_flow/anthropic_defaults.py +++ b/packages/flow/openadapt_flow/anthropic_defaults.py @@ -1,6 +1,6 @@ """Anthropic defaults that product code shares with the benchmark harness. -These live here, not in ``openadapt_flow.benchmark``, because the product +These live here, not in ``benchmark.harness`` (the checkout's harness), because the product imports them and a frozen build leaves the benchmark package out (``tests/test_runtime_never_imports_benchmark.py`` enforces the rule). """ diff --git a/packages/flow/openadapt_flow/authoring.py b/packages/flow/openadapt_flow/authoring.py index a1c4e0c2a1..24a3df3402 100644 --- a/packages/flow/openadapt_flow/authoring.py +++ b/packages/flow/openadapt_flow/authoring.py @@ -804,7 +804,7 @@ def continue_input(self, *, operator_confirmed: bool = True) -> dict[str, Any]: "reason": "unconfirmed_masked_secret", } if not pause.secret and (text is None or not text): - # Non-secret MockMed note: do not persist an empty TYPE/param. + # Non-secret field left empty: do not persist an empty TYPE/param. return { "recorded": False, "param": pause.param, diff --git a/packages/flow/openadapt_flow/deployment.py b/packages/flow/openadapt_flow/deployment.py index e8b54023bd..a5f3d698e3 100644 --- a/packages/flow/openadapt_flow/deployment.py +++ b/packages/flow/openadapt_flow/deployment.py @@ -545,7 +545,7 @@ def _validate_secret_env(cls, v: dict[str, str]) -> dict[str, str]: if not isinstance(env_name, str) or not _ENV_VAR_NAME.fullmatch(env_name): raise ValueError( f"effects.secret_env[{slot!r}] must be the NAME of an " - "environment variable (for example OPENEMR_READER_PASSWORD), " + "environment variable (for example APP_READER_PASSWORD), " "never the credential itself" ) return v diff --git a/packages/flow/paper/REVIEW_ADVERSARIAL.md b/packages/flow/paper/REVIEW_ADVERSARIAL.md index 4ebe33f3c6..9e31c59679 100644 --- a/packages/flow/paper/REVIEW_ADVERSARIAL.md +++ b/packages/flow/paper/REVIEW_ADVERSARIAL.md @@ -35,7 +35,7 @@ Every quantitative result is at a sample size where inference is not possible, a This is my single most serious concern, and it is not disclosed in the paper. -- The headline "screen 50/90, effect 0/90" comes from `openadapt_flow/benchmark/silent_wrong_action.py`, **not** from the browser study. That harness does not drive the browser, does not run OCR, and does not invoke the `Replayer`. It issues raw `requests.post` calls that replicate what the app's JavaScript would send, encodes the screen oracle as a hardcoded rule over HTTP status codes, and calls `RestRecordVerifier.verify(...)` directly. +- The headline "screen 50/90, effect 0/90" comes from `benchmark/harness/silent_wrong_action.py`, **not** from the browser study. That harness does not drive the browser, does not run OCR, and does not invoke the `Replayer`. It issues raw `requests.post` calls that replicate what the app's JavaScript would send, encodes the screen oracle as a hardcoded rule over HTTP status codes, and calls `RestRecordVerifier.verify(...)` directly. - In that harness the **effect oracle and the ground-truth judge read the same in-process store**: the effect verifier reads `GET /api/db` and the ground-truth `_business_effect` reads `FaultDB.snapshot()`, which is the same `FaultDB`. Worse, the effect contract (`record_written` exactly once AND `field_equals` on the note) is essentially a restatement of the ground-truth definition of "correct." So "effect silent-wrong = 0 of 90" is guaranteed by construction: the oracle and the ground truth are computed from the same bytes with near-identical logic. The number is a property of the fixture, not an empirical finding. - The genuinely end-to-end harness (`benchmark/fault_model/run.py`, real Playwright + OCR + `Replayer`) measures **only the screen-only baseline**. Its own recommendations section describes effect verification as a *proposed* fix that the reference postcondition system does not yet implement ("The reference postcondition system is vision-only today, so on this corpus it provides no transactional-safety coverage"). The effect-verifier-through-the-replayer path for these fault classes exists only in unit tests (`tests/test_effect_fault_matrix.py`), and those tests also call `verifier.verify(...)` directly rather than driving the `Replayer`. - Consequently the Results sentence "those five classes refuted and halted **through the live replayer**" overstates what was measured. The live replayer never ran the effect verifier on these 90 runs. The suspiciously clean integers (exactly 50, exactly 0, exactly 10) are deterministic-fixture artifacts, not a measured rate. @@ -48,7 +48,7 @@ The seven fault classes are hand-picked transactional hazards (partial save, dup ### 3.4 Baseline fairness: same-backend, but a vanilla single-shot agent -The agent baseline (`openadapt_flow/benchmark/agent_baseline.py`) is not a strawman in the crude sense: it drives the same Playwright backend, uses a current computer-use model and tool, and is handed generous task information (credentials, exact patient, exact note text, a hint that the app is slow). But it is a **single-attempt, un-scaffolded** loop: no system prompt, no planning/reflection/ReAct, no whole-task retry, and hard caps (25 actions, $1.50, last-3-screenshots context). Modern agent results lean heavily on scaffolding and retries; comparing a compiled program against a vanilla single-shot agent inflates the latency and cost deltas. The paper's hedges are appropriate, but the abstract and introduction still lead with the comparison in a way that implies a capability gap the design does not license. At minimum the baseline's single-shot, no-scaffold nature belongs in the main text next to the comparison table, not only implicitly in the code. +The agent baseline (`benchmark/harness/agent_baseline.py`) is not a strawman in the crude sense: it drives the same Playwright backend, uses a current computer-use model and tool, and is handed generous task information (credentials, exact patient, exact note text, a hint that the app is slow). But it is a **single-attempt, un-scaffolded** loop: no system prompt, no planning/reflection/ReAct, no whole-task retry, and hard caps (25 actions, $1.50, last-3-screenshots context). Modern agent results lean heavily on scaffolding and retries; comparing a compiled program against a vanilla single-shot agent inflates the latency and cost deltas. The paper's hedges are appropriate, but the abstract and introduction still lead with the comparison in a way that implies a capability gap the design does not license. At minimum the baseline's single-shot, no-scaffold nature belongs in the main text next to the comparison table, not only implicitly in the code. ### 3.5 The strong-oracle claim is ambiguous exactly where it matters most (pixel-only substrates) diff --git a/packages/flow/pyproject.toml b/packages/flow/pyproject.toml index b63cfa7435..b4bedc3057 100644 --- a/packages/flow/pyproject.toml +++ b/packages/flow/pyproject.toml @@ -605,7 +605,7 @@ module = [ "openadapt_flow.__main__", "openadapt_flow.backends.playwright_backend", "openadapt_flow.backends.rdp_backend", - "openadapt_flow.benchmark.desktop_benchmark", + "benchmark.harness.desktop_benchmark", "openadapt_flow.compiler.disambiguation", "openadapt_flow.interactive_recorder", "openadapt_flow.learning.synth_stream", diff --git a/packages/flow/scripts/desktop/waa_shim.py b/packages/flow/scripts/desktop/waa_shim.py index a15e66b1ae..e0a877e4af 100644 --- a/packages/flow/scripts/desktop/waa_shim.py +++ b/packages/flow/scripts/desktop/waa_shim.py @@ -1,7 +1,7 @@ """In-guest WAA-contract HTTP shim (runs INSIDE the Windows VM). Serves the desktop benchmark harness -(``openadapt_flow.benchmark.desktop_benchmark``): +(``benchmark.harness.desktop_benchmark``): GET /screenshot -> raw PNG bytes of the desktop (Flask send_file) GET /health -> ``{"status": "ok", ...}`` liveness probe diff --git a/packages/flow/scripts/frappe_lending_demo.py b/packages/flow/scripts/frappe_lending_demo.py index 8681b128b4..13877e4925 100644 --- a/packages/flow/scripts/frappe_lending_demo.py +++ b/packages/flow/scripts/frappe_lending_demo.py @@ -42,8 +42,8 @@ from openadapt_flow.backends.playwright_backend import ( # noqa: E402 PlaywrightBackend as _BasePlaywrightBackend, ) -from openadapt_flow.benchmark import agent_baseline # noqa: E402 -from openadapt_flow.benchmark.frappe_lending import ( # noqa: E402 +from benchmark.harness import agent_baseline # noqa: E402 +from benchmark.harness.frappe_lending import ( # noqa: E402 CONDITIONS, INITIAL_TRIALS_PER_CELL, PUBLICATION_TRIALS_PER_CELL, diff --git a/packages/flow/scripts/openemr_demo.py b/packages/flow/scripts/openemr_demo.py index 6a91a699fe..b9a75aee21 100644 --- a/packages/flow/scripts/openemr_demo.py +++ b/packages/flow/scripts/openemr_demo.py @@ -19,7 +19,7 @@ Showcase artifacts land in docs/showcase-openemr/; ``benchmark`` records a fresh demonstration into a temporary directory and writes results to -benchmark/openemr/ (see openadapt_flow.benchmark.openemr_benchmark). +benchmark/openemr/ (see benchmark.harness.openemr_benchmark). """ from __future__ import annotations @@ -189,7 +189,7 @@ def _note_visible(final_png: bytes, note: str) -> bool: Delegates to the benchmark's shared arm-independent check (``verify_note_saved``) so the criterion is implemented exactly once. """ - from openadapt_flow.benchmark.verify import verify_note_saved + from benchmark.harness.verify import verify_note_saved return verify_note_saved(final_png, note).success @@ -280,15 +280,15 @@ def benchmark() -> None: (incremental, per run) plus results.json / BENCHMARK.md / latency_cost.png to benchmark/openemr/. - Hard cost guardrails (see openadapt_flow.benchmark.openemr_benchmark): + Hard cost guardrails (see benchmark.harness.openemr_benchmark): an API preflight before any spend, a $1.50 per-run cap, an $8.00 total agent-arm ceiling, and an abort on consecutive billing errors. All at list price — billed cost under the intro rate is lower. """ import tempfile - from openadapt_flow.benchmark import agent_baseline - from openadapt_flow.benchmark.openemr_benchmark import ( + from benchmark.harness import agent_baseline + from benchmark.harness.openemr_benchmark import ( MAX_TOTAL_COST_USD, run_openemr_benchmark, ) diff --git a/packages/flow/scripts/openemr_local_demo.py b/packages/flow/scripts/openemr_local_demo.py index d9d8438cd1..0a62f7a4d8 100644 --- a/packages/flow/scripts/openemr_local_demo.py +++ b/packages/flow/scripts/openemr_local_demo.py @@ -40,8 +40,8 @@ from openadapt_flow.backends.playwright_backend import ( # noqa: E402 PlaywrightBackend as _BasePlaywrightBackend, ) -from openadapt_flow.benchmark import agent_baseline # noqa: E402 -from openadapt_flow.benchmark.openemr_local import ( # noqa: E402 +from benchmark.harness import agent_baseline # noqa: E402 +from benchmark.harness.openemr_local import ( # noqa: E402 CONDITIONS, INITIAL_TRIALS_PER_CELL, PUBLICATION_TRIALS_PER_CELL, diff --git a/packages/flow/scripts/openemr_param_depth.py b/packages/flow/scripts/openemr_param_depth.py index 6afd55d641..5c77b4c57f 100644 --- a/packages/flow/scripts/openemr_param_depth.py +++ b/packages/flow/scripts/openemr_param_depth.py @@ -148,7 +148,7 @@ def compile_bundle() -> None: def _replay_one(label: str, params: dict[str, str], url: str) -> dict: """One fresh-browser compiled replay; returns a summary row.""" - from openadapt_flow.benchmark.verify import verify_note_saved + from benchmark.harness.verify import verify_note_saved from openadapt_flow.ir import Workflow from openadapt_flow.runtime.replayer import Replayer from openadapt_flow.vision.ocr import normalize_text, ocr diff --git a/packages/flow/tests/e2e/test_region_stable_theme_regression.py b/packages/flow/tests/e2e/test_region_stable_theme_regression.py index 4ce3990130..152450bc7a 100644 --- a/packages/flow/tests/e2e/test_region_stable_theme_regression.py +++ b/packages/flow/tests/e2e/test_region_stable_theme_regression.py @@ -13,7 +13,7 @@ import pytest from openadapt_flow.backends.playwright_backend import PlaywrightBackend -from openadapt_flow.benchmark.dom_arm import verify_final_state +from benchmark.harness.dom_arm import verify_final_state from openadapt_flow.compiler import compile_recording from openadapt_flow.demo_driver import record_triage_demo from openadapt_flow.ir import Workflow diff --git a/packages/flow/tests/test_benchmark.py b/packages/flow/tests/test_benchmark.py index 6f903b79c9..68dbac3e54 100644 --- a/packages/flow/tests/test_benchmark.py +++ b/packages/flow/tests/test_benchmark.py @@ -15,20 +15,20 @@ import numpy as np import pytest -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.agent_baseline import ( +from benchmark.harness import agent_baseline +from benchmark.harness.agent_baseline import ( _truncate_screenshots, compute_cost, load_api_key, run_agent, triage_task_prompt, ) -from openadapt_flow.benchmark.run_benchmark import ( +from benchmark.harness.run_benchmark import ( aggregate_results, render_markdown, write_outputs, ) -from openadapt_flow.benchmark.verify import verify_encounter_saved +from benchmark.harness.verify import verify_encounter_saved NOTE = "Follow-up in 2 weeks; BP recheck." @@ -521,7 +521,7 @@ def test_right_type_passes_at_every_note_length( # line. TestVerify above retains a real-OCR smoke for the benchmark's # actual 32-character note; these four cases pin the verifier logic. monkeypatch.setattr( - "openadapt_flow.benchmark.verify.ocr", + "benchmark.harness.verify.ocr", lambda _screen: [SimpleNamespace(text=f"Triage - {note[:60]}")], ) verdict = verify_encounter_saved(screen, note) @@ -542,7 +542,7 @@ def test_wrong_type_fails_at_every_note_length( f"Consult - {note[:60]}", ) monkeypatch.setattr( - "openadapt_flow.benchmark.verify.ocr", + "benchmark.harness.verify.ocr", lambda _screen: [SimpleNamespace(text=f"Consult - {note[:60]}")], ) verdict = verify_encounter_saved(screen, note) @@ -599,7 +599,7 @@ def test_bare_note_starting_with_known_type_is_not_a_row( # the note itself starts with a known categorical value. screen = self.make_screen(f"Encounter saved - {note[:40]}", note[:60]) monkeypatch.setattr( - "openadapt_flow.benchmark.verify.ocr", + "benchmark.harness.verify.ocr", lambda _screen: [SimpleNamespace(text=note[:60])], ) verdict = verify_encounter_saved(screen, note) diff --git a/packages/flow/tests/test_chart_fonts.py b/packages/flow/tests/test_chart_fonts.py index c54ceb2bc1..d40847a4da 100644 --- a/packages/flow/tests/test_chart_fonts.py +++ b/packages/flow/tests/test_chart_fonts.py @@ -2,7 +2,7 @@ The benchmark chart is cosmetic; a font-lookup failure must never fail the suite. These tests cover the two guarantees in -``openadapt_flow.benchmark.chart_fonts``: the bundled DejaVuSans font is +``benchmark.harness.chart_fonts``: the bundled DejaVuSans font is registered so ``findfont`` cannot miss, and ``safe_render`` swallows any rendering failure. """ @@ -12,7 +12,7 @@ from pathlib import Path from typing import Any -from openadapt_flow.benchmark.chart_fonts import ( +from benchmark.harness.chart_fonts import ( BUNDLED_FONT_NAME, configure_bundled_font, safe_render, diff --git a/packages/flow/tests/test_competitor_drift_instrument.py b/packages/flow/tests/test_competitor_drift_instrument.py index 984b79c792..ec52bd8589 100644 --- a/packages/flow/tests/test_competitor_drift_instrument.py +++ b/packages/flow/tests/test_competitor_drift_instrument.py @@ -22,7 +22,7 @@ import pytest -from openadapt_flow.instrument import ( +from benchmark.harness.instrument import ( AgentRunResult, ArchitectureClassError, CostGuard, @@ -35,7 +35,7 @@ ensure_architecture_class, run_instrument, ) -from openadapt_flow.instrument.competitor_drift import render_markdown, write_outputs +from benchmark.harness.instrument.competitor_drift import render_markdown, write_outputs # Fault classes a screen-only report is BLIND to: a wrong effect lands while # the "saved" banner still paints, so a screen-blind agent claims success. diff --git a/packages/flow/tests/test_desktop_benchmark.py b/packages/flow/tests/test_desktop_benchmark.py index 45d605ed10..f54425cae2 100644 --- a/packages/flow/tests/test_desktop_benchmark.py +++ b/packages/flow/tests/test_desktop_benchmark.py @@ -9,8 +9,8 @@ import json -from openadapt_flow.benchmark import desktop_benchmark as db -from openadapt_flow.benchmark.desktop_benchmark import ( +from benchmark.harness import desktop_benchmark as db +from benchmark.harness.desktop_benchmark import ( _classify, run_desktop_benchmark, ) diff --git a/packages/flow/tests/test_dom_arm.py b/packages/flow/tests/test_dom_arm.py index c19314c790..6dd3a6e4b4 100644 --- a/packages/flow/tests/test_dom_arm.py +++ b/packages/flow/tests/test_dom_arm.py @@ -15,7 +15,7 @@ import numpy as np import pytest -from openadapt_flow.benchmark.dom_arm import ( +from benchmark.harness.dom_arm import ( PERTURBATIONS, SCHEDULE, aggregate_dom_results, @@ -191,7 +191,7 @@ def test_perturbation_menu(self) -> None: ) def test_notes_distinct_per_arm_and_slot(self) -> None: - from openadapt_flow.benchmark.dom_arm import ARMS + from benchmark.harness.dom_arm import ARMS notes = { note_for_slot(arm, slot) @@ -262,7 +262,7 @@ def test_maintenance_is_dom_loud_break_on_drift_only(self) -> None: assert not needs_maintenance(disputed) def test_verification_dispute(self) -> None: - from openadapt_flow.benchmark.dom_arm import verification_dispute + from benchmark.harness.dom_arm import verification_dispute # DOM: every step ran, no error, judge said no -> dispute. assert verification_dispute( @@ -546,7 +546,7 @@ class TestVerifyFinalState: exists to measure.""" def test_dom_check_is_the_hybrid_check(self) -> None: - from openadapt_flow.benchmark.hybrid_benchmark import ( + from benchmark.harness.hybrid_benchmark import ( verify_hybrid_final, ) diff --git a/packages/flow/tests/test_effectbench.py b/packages/flow/tests/test_effectbench.py index 435364cdcd..19419163d9 100644 --- a/packages/flow/tests/test_effectbench.py +++ b/packages/flow/tests/test_effectbench.py @@ -26,7 +26,7 @@ build_reference_episodes, ) from benchmark.fault_model import faults as F -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, DivergenceCategory, Effect, @@ -44,7 +44,7 @@ summarize, wilson_interval, ) -from openadapt_flow.benchmark.effectbench.metrics import bootstrap_ci, rate +from benchmark.harness.effectbench.metrics import bootstrap_ci, rate # --------------------------------------------------------------------------- # 1. Reference re-expression regression (the headline result) @@ -334,7 +334,7 @@ def test_record_and_note_effects_are_reexported() -> None: def test_score_episode_end_to_end() -> None: """The runner orchestrator: capture pre-state, run the arm, verify, label.""" - from openadapt_flow.benchmark.effectbench import AgentReport, score_episode + from benchmark.harness.effectbench import AgentReport, score_episode box = {"acted": False} @@ -367,7 +367,7 @@ def run_action() -> AgentReport: def test_score_episode_raises_on_unreadable_oracle() -> None: - from openadapt_flow.benchmark.effectbench import AgentReport, score_episode + from benchmark.harness.effectbench import AgentReport, score_episode with pytest.raises(ValueError): score_episode( @@ -433,8 +433,8 @@ def test_summary_filters_by_arm(episodes: list[EpisodeRecord]) -> None: def test_episode_record_predicates_match_outcome() -> None: - from openadapt_flow.benchmark.effectbench import Verdict - from openadapt_flow.benchmark.effectbench.schema import OracleVerdict + from benchmark.harness.effectbench import Verdict + from benchmark.harness.effectbench.schema import OracleVerdict ep = EpisodeRecord( episode_id="e1", diff --git a/packages/flow/tests/test_effectbench_runner.py b/packages/flow/tests/test_effectbench_runner.py index 64cd934c61..8abe524670 100644 --- a/packages/flow/tests/test_effectbench_runner.py +++ b/packages/flow/tests/test_effectbench_runner.py @@ -20,7 +20,7 @@ import pytest -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, EffectKind, EpisodeRecord, @@ -30,7 +30,7 @@ SwerVariant, summarize, ) -from openadapt_flow.benchmark.effectbench.runner import ( +from benchmark.harness.effectbench.runner import ( LIVE_ARMS, CompilerArm, CompoundEffectVerifier, @@ -43,19 +43,19 @@ run_episode, run_matrix, ) -from openadapt_flow.benchmark.effectbench.runner.arms import AgentArm, ArmResult -from openadapt_flow.benchmark.effectbench.runner.baselines import ( +from benchmark.harness.effectbench.runner.arms import AgentArm, ArmResult +from benchmark.harness.effectbench.runner.baselines import ( SCAFFOLDED_ARMS, ClaudeComputerUseArm, ScaffoldNotWired, ) -from openadapt_flow.benchmark.effectbench.runner.harness import EpisodeEnv -from openadapt_flow.benchmark.effectbench.runner.reference_tasks import ( +from benchmark.harness.effectbench.runner.harness import EpisodeEnv +from benchmark.harness.effectbench.runner.reference_tasks import ( GOAL, TARGET, make_params, ) -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( DivergenceCategory, OracleChannel, OracleSpec, @@ -343,7 +343,7 @@ def test_run_matrix_skips_scaffolded_arms_by_default() -> None: def test_cli_dry_run_runs_green() -> None: - from openadapt_flow.benchmark.effectbench.runner.__main__ import run_dry_run + from benchmark.harness.effectbench.runner.__main__ import run_dry_run episodes, summaries = run_dry_run( trials=1, arm_names="compiler,screen_only", include_scaffolded=False diff --git a/packages/flow/tests/test_effectbench_standalone.py b/packages/flow/tests/test_effectbench_standalone.py index dc0fb92526..56ac997158 100644 --- a/packages/flow/tests/test_effectbench_standalone.py +++ b/packages/flow/tests/test_effectbench_standalone.py @@ -6,7 +6,7 @@ dependency). Because the standalone package vendors the effect MECHANISM (the typed ``Effect`` contract, the substrate-independent judge, the outcome classifier) that is otherwise defined in ``openadapt_flow.runtime.effects`` and -``openadapt_flow.benchmark.effectbench``, this test guards against the two copies +``benchmark.harness.effectbench``, this test guards against the two copies drifting: it asserts the standalone classifier and judge agree BYTE-FOR-BYTE with the engine on the reference inputs, and that the standalone reference reproduces the published headline. It also re-runs the standalone source-availability @@ -62,7 +62,7 @@ def test_classifier_parity_with_engine_over_the_full_truth_table(): """The standalone classifier equals the engine classifier on every input.""" import effectbench.oracle as sa - from openadapt_flow.benchmark.effectbench import oracle as eng + from benchmark.harness.effectbench import oracle as eng scoreable = ["CORRECT", "WRONG_PERSISTED", "ABSENT"] for reported, state_name, avail in itertools.product( @@ -86,7 +86,7 @@ def test_divergence_category_parity_with_engine(): """The public standalone taxonomy must not lag the engine copy.""" from effectbench.schema import DivergenceCategory as StandaloneCategory - from openadapt_flow.benchmark.effectbench.schema import ( + from benchmark.harness.effectbench.schema import ( DivergenceCategory as EngineCategory, ) diff --git a/packages/flow/tests/test_effectbench_task_pack.py b/packages/flow/tests/test_effectbench_task_pack.py index bed77414ca..ddb2ea85c0 100644 --- a/packages/flow/tests/test_effectbench_task_pack.py +++ b/packages/flow/tests/test_effectbench_task_pack.py @@ -36,8 +36,8 @@ from benchmark.effectbench.task_pack.audit import audit_mockmed_oracle from benchmark.effectbench.task_pack.driver import run_mockmed_pack from benchmark.effectbench.task_pack.mockmed_tasks import MOCKMED_TASKS -from openadapt_flow.benchmark.effectbench import OutcomeLabel, Substrate, summarize -from openadapt_flow.benchmark.effectbench.schema import DivergenceCategory +from benchmark.harness.effectbench import OutcomeLabel, Substrate, summarize +from benchmark.harness.effectbench.schema import DivergenceCategory from openadapt_flow.runtime.effects.sql import assert_read_only_sql MANIFEST_PATH = Path(__file__).resolve().parents[1] / ( diff --git a/packages/flow/tests/test_frappe_lending_benchmark.py b/packages/flow/tests/test_frappe_lending_benchmark.py index ee3f50c080..1737d2a80b 100644 --- a/packages/flow/tests/test_frappe_lending_benchmark.py +++ b/packages/flow/tests/test_frappe_lending_benchmark.py @@ -19,7 +19,7 @@ FrappeFixture, audit_table_deltas, ) -from openadapt_flow.benchmark.frappe_lending import ( +from benchmark.harness.frappe_lending import ( ARMS, CONDITIONS, FrappeLoanApplicationOracle, diff --git a/packages/flow/tests/test_hybrid_benchmark.py b/packages/flow/tests/test_hybrid_benchmark.py index 9779311f40..11ca0244ba 100644 --- a/packages/flow/tests/test_hybrid_benchmark.py +++ b/packages/flow/tests/test_hybrid_benchmark.py @@ -15,8 +15,8 @@ import numpy as np import pytest -from openadapt_flow.benchmark import agent_baseline, hybrid_benchmark -from openadapt_flow.benchmark.hybrid_benchmark import ( +from benchmark.harness import agent_baseline, hybrid_benchmark +from benchmark.harness.hybrid_benchmark import ( AGENT_SLOTS, DRIFT_TYPES, FALLBACK_MAX_ACTIONS, @@ -610,7 +610,7 @@ def fake_run_agent(backend, task, **kwargs): assert "no failing step recorded" in seen["task"] def test_failed_result_index_edge_cases(self) -> None: - from openadapt_flow.benchmark.hybrid_benchmark import ( + from benchmark.harness.hybrid_benchmark import ( _failed_result_index, ) @@ -978,7 +978,7 @@ def test_arms_see_identical_conditions_per_slot( the run helpers), not on the orchestrator-stamped condition labels — the labels come from the same schedule lookup the orchestrator uses to build the URLs, so asserting on them alone is circular.""" - from openadapt_flow.benchmark.hybrid_benchmark import ( + from benchmark.harness.hybrid_benchmark import ( AGENT_SLOTS, condition_url, ) diff --git a/packages/flow/tests/test_lending_swer.py b/packages/flow/tests/test_lending_swer.py index b09dabeeed..b787960357 100644 --- a/packages/flow/tests/test_lending_swer.py +++ b/packages/flow/tests/test_lending_swer.py @@ -22,7 +22,7 @@ from benchmark.lending_fault_model import faults as F from benchmark.lending_fault_model import ground_truth as G from benchmark.lending_fault_model import swer as S -from openadapt_flow.benchmark.effectbench import summarize +from benchmark.harness.effectbench import summarize from openadapt_flow.mockloan.fault_server import LedgerDB diff --git a/packages/flow/tests/test_openemr_benchmark.py b/packages/flow/tests/test_openemr_benchmark.py index b2f10ba067..f32a93db12 100644 --- a/packages/flow/tests/test_openemr_benchmark.py +++ b/packages/flow/tests/test_openemr_benchmark.py @@ -16,20 +16,20 @@ import cv2 import numpy as np -import openadapt_flow.benchmark.verify as benchmark_verify -from openadapt_flow.benchmark import agent_baseline, openemr_benchmark -from openadapt_flow.benchmark.agent_baseline import ( +import benchmark.harness.verify as benchmark_verify +from benchmark.harness import agent_baseline, openemr_benchmark +from benchmark.harness.agent_baseline import ( openemr_task_prompt, run_agent, ) -from openadapt_flow.benchmark.openemr_benchmark import ( +from benchmark.harness.openemr_benchmark import ( aggregate_openemr_results, note_for, render_openemr_markdown, run_openemr_benchmark, write_openemr_outputs, ) -from openadapt_flow.benchmark.verify import verify_note_saved +from benchmark.harness.verify import verify_note_saved from openadapt_flow.vision import OcrLine NOTE = "Insurance card copied and coverage verified by phone." @@ -848,7 +848,7 @@ def crashing_client(self) -> CrashingClient: ) def test_mid_run_crash_row_carries_partial_cost(self, monkeypatch: Any) -> None: - from openadapt_flow.benchmark.run_benchmark import _agent_run + from benchmark.harness.run_benchmark import _agent_run _fake_launch(monkeypatch, FakeBackend()) row = _agent_run("http://x", NOTE, client=self.crashing_client(), task="task") diff --git a/packages/flow/tests/test_openemr_local_benchmark.py b/packages/flow/tests/test_openemr_local_benchmark.py index e637aa076b..c67f0b6c79 100644 --- a/packages/flow/tests/test_openemr_local_benchmark.py +++ b/packages/flow/tests/test_openemr_local_benchmark.py @@ -22,7 +22,7 @@ audit_table_deltas, unexpected_table_deltas, ) -from openadapt_flow.benchmark.openemr_local import ( +from benchmark.harness.openemr_local import ( ARMS, CONDITIONS, OpenEMRPatientOracle, @@ -1011,7 +1011,7 @@ def __init__(self, messages: Messages) -> None: "max_tokens": 4096, } ledger = __import__( - "openadapt_flow.benchmark.agent_baseline", fromlist=["UsageLedger"] + "benchmark.harness.agent_baseline", fromlist=["UsageLedger"] ).UsageLedger() allowed_messages = Messages(10_000) diff --git a/packages/flow/tests/test_openemr_retained_finals.py b/packages/flow/tests/test_openemr_retained_finals.py index 8fbdc22904..c1b6709429 100644 --- a/packages/flow/tests/test_openemr_retained_finals.py +++ b/packages/flow/tests/test_openemr_retained_finals.py @@ -16,7 +16,7 @@ import pytest -from openadapt_flow.benchmark.verify import verify_note_saved +from benchmark.harness.verify import verify_note_saved REPO_ROOT = Path(__file__).resolve().parents[1] RUN_RETAINED_FINALS = os.environ.get("OPENADAPT_OPENEMR_FINALS_DIR") diff --git a/packages/flow/tests/test_reliability_harness.py b/packages/flow/tests/test_reliability_harness.py index 06be924f83..e3dade0d41 100644 --- a/packages/flow/tests/test_reliability_harness.py +++ b/packages/flow/tests/test_reliability_harness.py @@ -1,6 +1,6 @@ """Small synthetic checks for the public reliability aggregation mechanism.""" -from openadapt_flow.benchmark.reliability import _outcome, aggregate +from benchmark.harness.reliability import _outcome, aggregate def _result(outcome: str, *, category: str | None = None) -> dict: diff --git a/packages/flow/tests/test_runtime_never_imports_benchmark.py b/packages/flow/tests/test_runtime_never_imports_benchmark.py index dc4fa57ffb..d59f3083d7 100644 --- a/packages/flow/tests/test_runtime_never_imports_benchmark.py +++ b/packages/flow/tests/test_runtime_never_imports_benchmark.py @@ -1,9 +1,10 @@ -"""Product code never imports ``openadapt_flow.benchmark``. +"""Product code never imports the benchmark harness. -A frozen build (Desktop's engine) leaves the benchmark package out, so a -product module that imports it, even lazily, fails there. Shared names -(``MODEL``, ``load_api_key``, ``SCROLL_PX_PER_UNIT``) live in -``openadapt_flow.anthropic_defaults``. +The harness lives in the source checkout (``packages/flow/benchmark/harness``), +outside the ``openadapt_flow`` package, so neither the wheel nor a frozen build +(Desktop's engine) carries it. A product module that imports it, even lazily, +fails there. Shared names (``MODEL``, ``load_api_key``, ``SCROLL_PX_PER_UNIT``) +live in ``openadapt_flow.anthropic_defaults``. """ from __future__ import annotations @@ -17,57 +18,64 @@ #: Modules whose job is the benchmark itself. Nothing else may import it. ALLOWED = { - # The benchmark CLI command loads the harness on demand. + # The benchmark CLI command loads the harness on demand from a checkout. "__main__.py", - # The competitor-drift harness reuses the benchmark's fault scenarios. - "instrument/competitor_drift.py", + # `record-check prove` loads its fixture from a checkout on demand. + "record_check_proof.py", } +def _is_benchmark(module: str) -> bool: + return any( + module == root or module.startswith(root + ".") + for root in ("benchmark", "openadapt_flow.benchmark") + ) + + def _imports_benchmark(tree: ast.AST) -> list[int]: lines: list[int] = [] for node in ast.walk(tree): if isinstance(node, ast.ImportFrom): module = node.module or "" - if node.level == 0 and ( - module == "openadapt_flow.benchmark" - or module.startswith("openadapt_flow.benchmark.") - ): + if node.level == 0 and _is_benchmark(module): lines.append(node.lineno) elif node.level == 0 and module == "openadapt_flow": if any(a.name == "benchmark" for a in node.names): lines.append(node.lineno) - elif node.level > 0 and ( - module == "benchmark" or module.startswith("benchmark.") - ): + elif node.level > 0 and _is_benchmark(module): lines.append(node.lineno) elif isinstance(node, ast.Import): - for alias in node.names: - if alias.name == "openadapt_flow.benchmark" or alias.name.startswith( - "openadapt_flow.benchmark." - ): - lines.append(node.lineno) + if any(_is_benchmark(alias.name) for alias in node.names): + lines.append(node.lineno) return lines +def test_the_harness_is_not_inside_the_package() -> None: + assert not (ROOT / "benchmark").exists(), ( + "benchmark harnesses belong in packages/flow/benchmark/, outside the " + "shipped openadapt_flow package" + ) + + def test_product_modules_never_import_the_benchmark_package() -> None: offenders: list[str] = [] for path in sorted(ROOT.rglob("*.py")): rel = path.relative_to(ROOT).as_posix() - if rel.startswith("benchmark/") or rel in ALLOWED: + if rel in ALLOWED: continue tree = ast.parse(path.read_text(encoding="utf-8")) offenders += [f"{rel}:{n}" for n in _imports_benchmark(tree)] assert not offenders, ( - "product code imports openadapt_flow.benchmark (a frozen build leaves " - "it out); move the shared name to openadapt_flow.anthropic_defaults " - f"or a runtime module: {offenders}" + "product code imports the benchmark harness (the wheel and a frozen " + "build leave it out); move the shared name to " + "openadapt_flow.anthropic_defaults or a runtime module: " + f"{offenders}" ) def test_shared_anthropic_names_have_one_home() -> None: + from benchmark.harness import agent_baseline from openadapt_flow import anthropic_defaults - from openadapt_flow.benchmark import agent_baseline assert agent_baseline.MODEL is anthropic_defaults.MODEL assert agent_baseline.SCROLL_PX_PER_UNIT == anthropic_defaults.SCROLL_PX_PER_UNIT diff --git a/packages/flow/tests/test_silent_wrong_action_benchmark.py b/packages/flow/tests/test_silent_wrong_action_benchmark.py index a340eea1ae..cadfdbcacc 100644 --- a/packages/flow/tests/test_silent_wrong_action_benchmark.py +++ b/packages/flow/tests/test_silent_wrong_action_benchmark.py @@ -16,7 +16,7 @@ from __future__ import annotations -from openadapt_flow.benchmark.silent_wrong_action import ( +from benchmark.harness.silent_wrong_action import ( SCENARIOS, aggregate, render_markdown, diff --git a/packages/flow/tests/test_ui_dismiss_live_openemr.py b/packages/flow/tests/test_ui_dismiss_live_openemr.py index 1468c56f6d..3caceffee1 100644 --- a/packages/flow/tests/test_ui_dismiss_live_openemr.py +++ b/packages/flow/tests/test_ui_dismiss_live_openemr.py @@ -47,7 +47,7 @@ def _record(tmp_path: Path) -> tuple[Path, Any]: from benchmark.openemr_local.qualify_hooks import fixture - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec from openadapt_flow.recorder import Recorder from scripts.openemr_local_demo import ( FORM_FRAME_NAME, From 7d884c01b42462e6f087bdad4296814e74a89121 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 13:04:32 -0500 Subject: [PATCH 08/22] Generality audit: link CONTRIBUTING without the guard PR's anchor --- packages/flow/docs/GENERALITY_AUDIT.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md index 7fb257ecdb..f8f4b51798 100644 --- a/packages/flow/docs/GENERALITY_AUDIT.md +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -6,8 +6,8 @@ Audited 2026-11-03 on `main` at `213f3d9fe`. Scope: everything under The rule: the engine has no code written for one application. What one application needs lives in data (`app_profiles/*.yaml`, generated contracts, deployment configs) or in test fixtures and benchmark harnesses outside the -shipped package. `scripts/check_engine_generality.py` enforces it in CI; see -[CONTRIBUTING.md](../CONTRIBUTING.md#app-knowledge-lives-in-data). +shipped package. A CI check (`scripts/check_engine_generality.py`) keeps it +that way; [CONTRIBUTING.md](../CONTRIBUTING.md) states the rule. ## Method From e97d0dfba49c11eed0b3f82f0386455771d6fc8d Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 14:07:54 -0500 Subject: [PATCH 09/22] Verifier: keep issued identities and check digests; --effects-records-path Adversarial review found that dropping the records_path and resource_type defaults changed the verifier identity of every existing config, and the fault-gate check digest of an explicit resource_type: Observation. Both now hash the old values (LEGACY_HASHED_DEFAULTS), for hashing only; a read without its own path or resource is still refused. run --effects-kind rest takes --effects-records-path, and the CLI tests that relied on the old default pass it. Test configs that read the demo app name its path. --- packages/agent/tests/test_approval_flow.py | 2 + .../tests/test_engine/test_check_setup.py | 2 +- packages/flow/CHANGELOG.md | 4 +- .../flow/benchmark/openemr_e2e/harness.py | 4 +- packages/flow/openadapt_flow/__main__.py | 14 +++++-- packages/flow/openadapt_flow/deployment.py | 37 ++++++++++++++++++- packages/flow/openadapt_flow/fault_gate.py | 3 ++ packages/flow/tests/test_approval_mode_cli.py | 2 + packages/flow/tests/test_cli_deployment.py | 2 +- packages/flow/tests/test_cli_new_commands.py | 2 + .../test_desktop_check_it_cli_mockmed.py | 4 ++ packages/flow/tests/test_effect_kit_config.py | 1 + .../flow/tests/test_effect_plugin_options.py | 4 +- packages/flow/tests/test_fault_gate.py | 2 +- packages/flow/tests/test_probe_induction.py | 6 ++- .../flow/tests/test_qualification_drills.py | 2 + .../flow/tests/test_record_check_synthesis.py | 5 ++- packages/flow/tests/test_run_gate.py | 14 ++++++- .../flow/tests/test_smart_backend_auth.py | 7 +++- .../flow/tests/test_ui_readback_verifier.py | 2 +- packages/flow/tests/test_undo_proposal.py | 2 +- .../tests/test_verifier_adapter_platform.py | 2 +- 22 files changed, 106 insertions(+), 17 deletions(-) diff --git a/packages/agent/tests/test_approval_flow.py b/packages/agent/tests/test_approval_flow.py index e3ffa7e510..acc71b9e25 100644 --- a/packages/agent/tests/test_approval_flow.py +++ b/packages/agent/tests/test_approval_flow.py @@ -91,6 +91,8 @@ def _bridge( "web", "--effects-kind", "rest", + "--effects-records-path", + "/api/db", "--effects-base-url", mockmed["base_url"], ), diff --git a/packages/desktop/tests/test_engine/test_check_setup.py b/packages/desktop/tests/test_engine/test_check_setup.py index 12ca9ea5ad..40de9e7096 100644 --- a/packages/desktop/tests/test_engine/test_check_setup.py +++ b/packages/desktop/tests/test_engine/test_check_setup.py @@ -279,7 +279,7 @@ def test_fixed_values_are_the_recorded_ones_and_follow_the_sealed_version(tmp_pa with pytest.raises(CheckSetupError, match="Unknown value"): answer_values(dispatcher, "wf1", ["member_id"], answered_by="person") - store.write_record_check_config("wf1", {"kind": "rest", "base_url": BASE}) + store.write_record_check_config("wf1", {"kind": "rest", "records_path": "/api/db", "base_url": BASE}) saved = store.load("wf1") saved["record_check_config"] = str(store.root / "wf1.record-check.yaml") store.save("wf1", saved) diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md index ed6573cc1e..7efc631110 100644 --- a/packages/flow/CHANGELOG.md +++ b/packages/flow/CHANGELOG.md @@ -25,7 +25,9 @@ ### Fixes -- A FHIR read names its resource and field map: `effects.kind: fhir`, a reward recipe of kind `fhir` and `FhirEffectVerifier` require `resource_type` and `field_paths` instead of defaulting to one app's note-as-Observation layout (`DEFAULT_OBSERVATION_PATHS` stays importable for callers whose app stores notes that way). An app profile's `no_record_values` lists the record-field values its audit entries use for "no record" (OpenEMR's profile: `"0"`; profile version `openemr-frame-profile/2026-11-03.1`); the frame and rollback no longer treat `0` as "no record" in every app. `generate ensure-contract` reads only the generic server-fact names from `facts.json`; the Medplum probe's translation moved to `benchmark/fhir_medplum/readpath/facts.py`. +- A REST read names its own records path: `effects.kind: rest`, a reward recipe of kind `rest`, `RestRecordVerifier`, `RestCompensator` and the fault gate's in-memory copy require `records_path` (and `records_key`, `delete_path_template`) instead of defaulting to the demo app's `/api/db`; `run --effects-kind rest` takes `--effects-records-path`. Compile-time REST binding admission (`compile_recording(admit_api_bindings=True, admission_fixture=...)`) takes the app's read path, write path, server-assigned fields and idempotency key field from an `AdmissionFixture`, and proposes nothing when no write was captured and no write path is named. Effect mining and `scaffold-verifier` mine an idempotency key only from a field the caller names. Verifier identities and fault-gate check digests of existing configs are unchanged. + +- A FHIR read names its resource and field map: `effects.kind: fhir`, a reward recipe of kind `fhir` and `FhirEffectVerifier` require `resource_type` and `field_paths` instead of defaulting to one app's note-as-Observation layout (`DEFAULT_OBSERVATION_PATHS` stays importable for callers whose app stores notes that way). An app profile's `no_record_values` lists the record-field values its audit entries use for "no record" (OpenEMR's profile: `"0"`; profile version `openemr-frame-profile/2026-11-03.1`, so a config that pins the old profile digest is refused until it pins the new one); the frame and rollback no longer treat `0` as "no record" in every app. `generate ensure-contract` reads only the generic server-fact names from `facts.json`; the Medplum probe's translation moved to `benchmark/fhir_medplum/readpath/facts.py`. - Self-repair on a covered identity band dismisses only the proved dialog: the covering window must span the band from where the read stops to its right edge at full height, the dismiss click must land in that same window, and only a dialog whose every word is a notice word (no question) may be dismissed ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). diff --git a/packages/flow/benchmark/openemr_e2e/harness.py b/packages/flow/benchmark/openemr_e2e/harness.py index 0c9ff8e057..b2afef1ea5 100644 --- a/packages/flow/benchmark/openemr_e2e/harness.py +++ b/packages/flow/benchmark/openemr_e2e/harness.py @@ -233,7 +233,9 @@ def _replay( replayer = Replayer( backend, vision=vision, - effect_verifier=RestRecordVerifier(url.rstrip("/"), records_path="/api/db", records_key="records"), + effect_verifier=RestRecordVerifier( + url.rstrip("/"), records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) report = replayer.run( diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index bb52425ba8..5f68142465 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -420,6 +420,8 @@ def _deployment_sections(args: argparse.Namespace): effects = effects.model_copy(update={"kind": args.effects_kind}) if getattr(args, "effects_base_url", None): effects = effects.model_copy(update={"base_url": args.effects_base_url}) + if getattr(args, "effects_records_path", None): + effects = effects.model_copy(update={"records_path": args.effects_records_path}) if getattr(args, "effects_root", None): effects = effects.model_copy(update={"root": args.effects_root}) @@ -8232,9 +8234,10 @@ def _add_deployment_flags( default=None, help=( "System-of-record EffectVerifier to wire so consequential writes " - "are verified against the real record (not the screen). The sql/" - "file kinds need their config fields (sql_query, root, ...) from " - "a --config deployment YAML" + "are verified against the real record (not the screen). rest " + "needs --effects-records-path; the fhir/sql/file kinds need their " + "config fields (resource_type and field_paths, sql_query, root, " + "...) from a --config deployment YAML" ), ) p.add_argument( @@ -8242,6 +8245,11 @@ def _add_deployment_flags( default=None, help="Base URL for the rest / fhir effect verifier", ) + p.add_argument( + "--effects-records-path", + default=None, + help="Path whose GET returns the app's records (rest effect verifier)", + ) p.add_argument( "--effects-root", default=None, diff --git a/packages/flow/openadapt_flow/deployment.py b/packages/flow/openadapt_flow/deployment.py index 717ebbe232..500824f58a 100644 --- a/packages/flow/openadapt_flow/deployment.py +++ b/packages/flow/openadapt_flow/deployment.py @@ -26,7 +26,7 @@ import os import re from pathlib import Path -from typing import Any, Callable, Literal, Mapping, Optional +from typing import Any, Callable, Final, Literal, Mapping, Optional from pydantic import ( BaseModel, @@ -2049,6 +2049,40 @@ def build_effect_verifier( _PLUGIN_ONLY_FIELDS = ("options", "option_params", "secret_env") +#: What these fields held when they still had defaults. Verifier identities +#: and check digests issued before then hashed these values, so for hashing +#: only, an unset field still reads as its old value. Never used to read a +#: store: a read without its own records_path or resource_type is refused. +LEGACY_HASHED_DEFAULTS: Final[dict[str, str]] = { + "records_path": "/api/db", + "resource_type": "Observation", +} + + +def legacy_hash_dump(dumped: dict[str, Any], *, defaults_excluded: bool) -> None: + """Make a config dump hash as it did when those fields had defaults. + + ``defaults_excluded``: the dump left defaults out (``exclude_defaults``), + so a field holding its old default is dropped; otherwise an unset field + takes its old default. + """ + + for name, legacy in LEGACY_HASHED_DEFAULTS.items(): + if defaults_excluded: + if dumped.get(name) == legacy: + dumped.pop(name) + elif dumped.get(name) is None: + dumped[name] = legacy + if defaults_excluded: + # Nested reads (the identity path prunes its own nesting). + for item in dumped.get("candidates") or []: + if isinstance(item, dict): + legacy_hash_dump(item, defaults_excluded=True) + for item in (dumped.get("surfaces") or {}).values(): + if isinstance(item, dict): + legacy_hash_dump(item, defaults_excluded=True) + + def _identity_config_dump(cfg: EffectsConfig) -> dict[str, Any]: """Serialize ``cfg`` for the verifier identity. @@ -2057,6 +2091,7 @@ def _identity_config_dump(cfg: EffectsConfig) -> dict[str, Any]: """ def prune(dumped: dict[str, Any]) -> dict[str, Any]: + legacy_hash_dump(dumped, defaults_excluded=False) for name in _PLUGIN_ONLY_FIELDS: if not dumped.get(name): dumped.pop(name, None) diff --git a/packages/flow/openadapt_flow/fault_gate.py b/packages/flow/openadapt_flow/fault_gate.py index 3d275ca4ca..47a8f51dab 100644 --- a/packages/flow/openadapt_flow/fault_gate.py +++ b/packages/flow/openadapt_flow/fault_gate.py @@ -374,7 +374,10 @@ def check_digest(cfg: Any) -> str: check as production. """ + from openadapt_flow.deployment import legacy_hash_dump + verifier = cfg.verifier.model_dump(mode="json", exclude_defaults=True) + legacy_hash_dump(verifier, defaults_excluded=True) frame = cfg.frame.model_dump(mode="json", by_alias=True) if cfg.frame else None payload = { "effects": [e.model_dump(mode="json") for e in cfg.effects], diff --git a/packages/flow/tests/test_approval_mode_cli.py b/packages/flow/tests/test_approval_mode_cli.py index 481c71420f..dbcb5dc281 100644 --- a/packages/flow/tests/test_approval_mode_cli.py +++ b/packages/flow/tests/test_approval_mode_cli.py @@ -186,6 +186,8 @@ def _run_case_argv( str(tmp_path / "case-run"), "--effects-kind", "rest", + "--effects-records-path", + "/api/db", "--effects-base-url", "http://sor.local", "--backend", diff --git a/packages/flow/tests/test_cli_deployment.py b/packages/flow/tests/test_cli_deployment.py index d5d3aedc63..8898f5bafd 100644 --- a/packages/flow/tests/test_cli_deployment.py +++ b/packages/flow/tests/test_cli_deployment.py @@ -109,7 +109,7 @@ def test_build_verifier_none_kind_is_none() -> None: def test_rest_requires_base_url() -> None: with pytest.raises(ValueError, match="base_url"): - build_effect_verifier(EffectsConfig(kind="rest")) + build_effect_verifier(EffectsConfig(kind="rest", records_path="/api/db")) def test_document_hash_requires_root() -> None: diff --git a/packages/flow/tests/test_cli_new_commands.py b/packages/flow/tests/test_cli_new_commands.py index dd9f06fdc3..a399d06803 100644 --- a/packages/flow/tests/test_cli_new_commands.py +++ b/packages/flow/tests/test_cli_new_commands.py @@ -460,6 +460,8 @@ def test_replay_injects_effects_actuator_durable_and_worklist( "http://app.example", "--effects-kind", "rest", + "--effects-records-path", + "/api/db", "--effects-base-url", "http://sor", "--api-base-url", diff --git a/packages/flow/tests/test_desktop_check_it_cli_mockmed.py b/packages/flow/tests/test_desktop_check_it_cli_mockmed.py index 47ac633a7d..874ca385ba 100644 --- a/packages/flow/tests/test_desktop_check_it_cli_mockmed.py +++ b/packages/flow/tests/test_desktop_check_it_cli_mockmed.py @@ -365,6 +365,8 @@ def run_case(case_id: str) -> SimpleNamespace: entry_url, "--effects-kind", "rest", + "--effects-records-path", + "/api/db", "--effects-base-url", base_url, "--approval-wait", @@ -580,6 +582,8 @@ def test_run_it_saves_once_after_approval_and_never_saves_a_form_nobody_sees( check_it.entry_url, "--effects-kind", "rest", + "--effects-records-path", + "/api/db", "--effects-base-url", base_url, "--params-file", diff --git a/packages/flow/tests/test_effect_kit_config.py b/packages/flow/tests/test_effect_kit_config.py index 3723260a85..4cd0c4ccb2 100644 --- a/packages/flow/tests/test_effect_kit_config.py +++ b/packages/flow/tests/test_effect_kit_config.py @@ -343,6 +343,7 @@ def test_candidate_construction_never_skips_a_broken_stronger_proof( candidates=[ EffectsConfig( kind="rest", + records_path="/api/db", base_url="https://oracle.invalid", auth={"bearer_env": "MISSING_ORACLE_TOKEN"}, ), diff --git a/packages/flow/tests/test_effect_plugin_options.py b/packages/flow/tests/test_effect_plugin_options.py index cd2c78885a..5383db1ca5 100644 --- a/packages/flow/tests/test_effect_plugin_options.py +++ b/packages/flow/tests/test_effect_plugin_options.py @@ -250,7 +250,9 @@ def test_changed_options_change_the_verifier_identity(pack): def test_identity_of_configs_without_plugin_fields_is_unchanged(): """Existing deployments keep a byte-identical verifier identity.""" - cfg = EffectsConfig(kind="rest", base_url="http://127.0.0.1:9") + cfg = EffectsConfig( + kind="rest", records_path="/api/db", base_url="http://127.0.0.1:9" + ) verifier = build_effect_verifier(cfg) legacy_config = cfg.model_dump(mode="json") # Neither the plugin fields, ui_readback, smart_backend, surfaces, diff --git a/packages/flow/tests/test_fault_gate.py b/packages/flow/tests/test_fault_gate.py index 99eb5b9ec8..1af8751c35 100644 --- a/packages/flow/tests/test_fault_gate.py +++ b/packages/flow/tests/test_fault_gate.py @@ -58,7 +58,7 @@ def _rest_cfg(base: str = "http://127.0.0.1:1"): @pytest.fixture def rest_copy(): - store = MemoryRestCopy(SEED, records_path="/api/db", records_key="records", TARGET) + store = MemoryRestCopy(SEED, TARGET, records_path="/api/db", records_key="records") try: yield store finally: diff --git a/packages/flow/tests/test_probe_induction.py b/packages/flow/tests/test_probe_induction.py index 40939eb2fe..9dcf501338 100644 --- a/packages/flow/tests/test_probe_induction.py +++ b/packages/flow/tests/test_probe_induction.py @@ -272,7 +272,11 @@ def _contract(directory: Path, base_url: str) -> Path: write_bundle( directory, contract_id="reward_contract_mockmed_probe", - oracle={"kind": "rest", "base_url": base_url.rstrip("/")}, + oracle={ + "kind": "rest", + "records_path": "/api/db", + "base_url": base_url.rstrip("/"), + }, channel="api", key=Ed25519PrivateKey.generate(), issuer_key_id="self_signed:test", diff --git a/packages/flow/tests/test_qualification_drills.py b/packages/flow/tests/test_qualification_drills.py index a3d9a19704..4f9a64d555 100644 --- a/packages/flow/tests/test_qualification_drills.py +++ b/packages/flow/tests/test_qualification_drills.py @@ -287,6 +287,8 @@ def _drill_run_case_argv(bundle, inputs_path, tmp_path, case_id, *extra): str(tmp_path / f"{case_id}-run"), "--effects-kind", "rest", + "--effects-records-path", + "/api/db", "--effects-base-url", "http://sor.local", "--backend", diff --git a/packages/flow/tests/test_record_check_synthesis.py b/packages/flow/tests/test_record_check_synthesis.py index 409d8457dd..f2c846b4d2 100644 --- a/packages/flow/tests/test_record_check_synthesis.py +++ b/packages/flow/tests/test_record_check_synthesis.py @@ -534,6 +534,7 @@ def test_surfaces_config_validation_and_identity_unchanged(): EffectsConfig.model_validate( { "kind": "rest", + "records_path": "/api/db", "base_url": "http://x", "surfaces": {"a": {"kind": "rest"}}, } @@ -546,7 +547,9 @@ def test_surfaces_config_validation_and_identity_unchanged(): from openadapt_flow.deployment import _identity_config_dump dumped = _identity_config_dump( - EffectsConfig.model_validate({"kind": "rest", "base_url": "http://x"}) + EffectsConfig.model_validate( + {"kind": "rest", "records_path": "/api/db", "base_url": "http://x"} + ) ) assert "surfaces" not in dumped diff --git a/packages/flow/tests/test_run_gate.py b/packages/flow/tests/test_run_gate.py index 83be1a2e2b..39ad39aefe 100644 --- a/packages/flow/tests/test_run_gate.py +++ b/packages/flow/tests/test_run_gate.py @@ -206,7 +206,7 @@ def _deployment( *, verifier: bool = True, policy: str = "clinical-write" ) -> DeploymentConfig: effects = ( - EffectsConfig(kind="rest", base_url="http://sor.local") + EffectsConfig(kind="rest", records_path="/api/db", base_url="http://sor.local") if verifier else EffectsConfig(kind="none") ) @@ -810,6 +810,8 @@ def test_cli_run_dry_run_reports_without_executing(tmp_path, monkeypatch, capsys "clinical-write", "--effects-kind", "rest", + "--effects-records-path", + "/api/db", "--effects-base-url", "http://sor.local", "--dry-run", @@ -953,6 +955,8 @@ def capture(args): str(tmp_path / "case-run"), "--effects-kind", "rest", + "--effects-records-path", + "/api/db", "--effects-base-url", "http://sor.local", "--backend", @@ -1022,6 +1026,8 @@ def test_qualification_run_case_tightens_owned_legacy_state_root(tmp_path, monke str(tmp_path / "case-run"), "--effects-kind", "rest", + "--effects-records-path", + "/api/db", "--effects-base-url", "http://sor.local", "--backend", @@ -1063,6 +1069,8 @@ def test_qualification_run_case_refuses_changed_inputs_and_duplicate_attempt( str(tmp_path / "case-run"), "--effects-kind", "rest", + "--effects-records-path", + "/api/db", "--effects-base-url", "http://sor.local", "--backend", @@ -1144,6 +1152,8 @@ def run(self, *_args, **_kwargs): "http://127.0.0.1:5001", "--effects-kind", "rest", + "--effects-records-path", + "/api/db", "--effects-base-url", "http://sor.local", ] @@ -1471,6 +1481,8 @@ def test_cli_run_encryption_key_required_for_encrypted_bundle( "clinical-write", "--effects-kind", "rest", + "--effects-records-path", + "/api/db", "--effects-base-url", "http://sor.local", "--allow-unencrypted", # isolate the KEY-at-load behavior from gate 6 diff --git a/packages/flow/tests/test_smart_backend_auth.py b/packages/flow/tests/test_smart_backend_auth.py index 2595785b4e..11535daf9c 100644 --- a/packages/flow/tests/test_smart_backend_auth.py +++ b/packages/flow/tests/test_smart_backend_auth.py @@ -545,7 +545,12 @@ def test_bad_smart_config_is_refused(overrides, message): def test_smart_backend_needs_fhir_and_no_static_token(): with pytest.raises(ValueError, match="only to kind fhir"): EffectsConfig.model_validate( - {"kind": "rest", "base_url": "https://x.test", "smart_backend": _smart()} + { + "kind": "rest", + "records_path": "/api/db", + "base_url": "https://x.test", + "smart_backend": _smart(), + } ) with pytest.raises(ValueError, match="mutually exclusive"): EffectsConfig.model_validate( diff --git a/packages/flow/tests/test_ui_readback_verifier.py b/packages/flow/tests/test_ui_readback_verifier.py index cf725558f0..8f594530b4 100644 --- a/packages/flow/tests/test_ui_readback_verifier.py +++ b/packages/flow/tests/test_ui_readback_verifier.py @@ -435,7 +435,7 @@ def test_existing_verifier_identities_do_not_change() -> None: rest = RestRecordVerifier( "http://sor.test", records_path="/api/db", records_key="records" ) - cfg = EffectsConfig(kind="rest", base_url="http://sor.test") + cfg = EffectsConfig(kind="rest", records_path="/api/db", base_url="http://sor.test") assert _effect_verifier_config_identity(cfg, rest) == ( "sha256:9f288a5b15b66001565c6d5bbd0f0a09bf92485280fe5de37b421802c3e0e663" ) diff --git a/packages/flow/tests/test_undo_proposal.py b/packages/flow/tests/test_undo_proposal.py index 10d0522027..b670da8f0b 100644 --- a/packages/flow/tests/test_undo_proposal.py +++ b/packages/flow/tests/test_undo_proposal.py @@ -905,7 +905,7 @@ def test_deployment_test_copy_resets_and_reads_every_record(tmp_path, mockmed): write_bundle( tmp_path / "contract", contract_id="undo_copy", - oracle={"kind": "rest", "base_url": mockmed}, + oracle={"kind": "rest", "records_path": "/api/db", "base_url": mockmed}, channel="api", key=Ed25519PrivateKey.generate(), issuer_key_id="self_signed:test", diff --git a/packages/flow/tests/test_verifier_adapter_platform.py b/packages/flow/tests/test_verifier_adapter_platform.py index e437c7217e..2fe631156e 100644 --- a/packages/flow/tests/test_verifier_adapter_platform.py +++ b/packages/flow/tests/test_verifier_adapter_platform.py @@ -555,6 +555,6 @@ def test_builtin_kinds_are_not_shadowed_by_plugins(self, clean_registry, tmp_pat # first inside build_effect_verifier. register_verifier_factory("rest", build_csv_ledger_verifier) verifier = build_effect_verifier( - EffectsConfig(kind="rest", base_url="http://sor") + EffectsConfig(kind="rest", records_path="/api/db", base_url="http://sor") ) assert isinstance(verifier, RestRecordVerifier) From cc145da332b549305f2c50b44c74256b791fe4f5 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 14:08:46 -0500 Subject: [PATCH 10/22] CONTRIBUTING: state the app-knowledge rule positively --- packages/flow/CONTRIBUTING.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/flow/CONTRIBUTING.md b/packages/flow/CONTRIBUTING.md index 8641a68f76..c91ee968f6 100644 --- a/packages/flow/CONTRIBUTING.md +++ b/packages/flow/CONTRIBUTING.md @@ -177,15 +177,13 @@ notice alone is not sufficient. ## App knowledge lives in data -The engine (`openadapt_flow/`) works the same way for every application. Never -write code, a default, a constant or a heuristic for one application. What one -application needs goes in data: +The engine (`openadapt_flow/`) works the same way for every application. App +knowledge lives in data: - its bookkeeping writes: an app profile in `openadapt_flow/app_profiles/`, loaded by name; - its endpoints, tables, keys and field paths: the generated contract or the - deployment config (`deploy.yaml`), which the engine requires rather than - defaulting; + deployment config (`deploy.yaml`); - tests and benchmarks for it: fixtures under `tests/` and harnesses under `benchmark/`, outside the shipped package. From 2bdcfa2e1721f6f3d89ceb206a316f53d46aa78e Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 14:10:32 -0500 Subject: [PATCH 11/22] Move test fixtures and demo harnesses out of the openadapt-flow wheel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wheel ships only the engine (see packages/flow/docs/GENERALITY_AUDIT.md). Moved out of openadapt_flow: - integrations/harness2 -> benchmark/harness/harness2 (the harness2 extra is dropped; install [browser] beside Harness² from a checkout) - mockloan/ and demo_driver.record_disbursement_demo -> benchmark/harness/mockloan/ (recorder in demo.py) - guard_demo.py -> benchmark/harness/guard_demo.py - agent/ledger_demo.py -> packages/agent/tests/ledger_demo.py - mockmed/teach_scope.py -> packages/flow/tests/_teach_scope.py - learning/synth_stream.py: StructuralDiffInducer -> learning/induce.py; the MockMed drift stream -> benchmark/harness/synth_stream.py reward/seed.py is split: the bundle helpers (write_bundle, calibrate_bundle, self_signed_certificate) are now reward/bundle.py. The MockMed seed stays, because serve-reward --seed-mockmed needs it from a wheel. --- .github/workflows/packages.yml | 2 +- CAPABILITIES.md | 2 +- llms-full.txt | 2 +- packages/agent/README.md | 2 +- .../agent => agent/tests}/ledger_demo.py | 5 +- packages/agent/tests/test_guard_tools.py | 12 +- packages/agent/tests/test_ledger.py | 3 +- packages/flow/CHANGELOG.md | 2 + .../harness}/guard_demo.py | 8 +- .../harness}/harness2/__init__.py | 0 .../harness}/harness2/adapter.py | 19 +- .../harness}/harness2/agent.py | 2 +- .../harness}/harness2/apps.py | 2 +- .../harness}/harness2/llm.py | 0 .../harness}/harness2/pixels.py | 2 +- .../harness}/harness2/proposers.py | 4 +- .../harness}/harness2/rollout.py | 8 +- .../harness}/harness2/scorer.py | 4 +- .../harness}/harness2/selection.py | 4 +- .../harness}/harness2/tasks.py | 0 .../harness}/harness2/traces.py | 8 +- .../harness}/mockloan/__init__.py | 2 +- .../flow/benchmark/harness/mockloan/demo.py | 97 +++++++ .../harness}/mockloan/fault_server.py | 0 .../harness}/mockloan/server.py | 2 +- .../harness}/mockloan/static/app.js | 2 +- .../harness}/mockloan/static/index.html | 0 .../harness}/mockloan/static/styles.css | 0 .../flow/benchmark/harness/synth_stream.py | 222 +++++++++++++++ .../flow/benchmark/harness2_scorer/RESULTS.md | 2 +- .../flow/benchmark/harness2_scorer/run.py | 14 +- .../LENDING_FAULT_MODEL.md | 2 +- .../benchmark/lending_fault_model/README.md | 6 +- .../benchmark/lending_fault_model/faults.py | 2 +- .../lending_fault_model/resolution_ladder.py | 4 +- .../flow/benchmark/lending_fault_model/run.py | 6 +- .../benchmark/lending_fault_model/swer.py | 2 +- .../flow/benchmark/openemr_e2e/harness.py | 2 +- packages/flow/benchmark/undo_proposal/run.py | 9 +- packages/flow/docs/GENERALITY_AUDIT.md | 14 +- packages/flow/docs/GUARDED_SAVE.md | 4 +- packages/flow/docs/GUARD_AN_AGENT.md | 2 +- packages/flow/docs/HARNESS2.md | 24 +- packages/flow/docs/JIT.md | 6 +- packages/flow/openadapt_flow/demo_driver.py | 86 +----- .../flow/openadapt_flow/learning/__init__.py | 5 +- .../openadapt_flow/learning/branch_teach.py | 2 +- .../flow/openadapt_flow/learning/halt_loop.py | 2 +- .../learning/{synth_stream.py => induce.py} | 199 ++------------ .../flow/openadapt_flow/learning/teach.py | 4 +- packages/flow/openadapt_flow/reward/bundle.py | 254 ++++++++++++++++++ packages/flow/openadapt_flow/reward/seed.py | 234 +--------------- packages/flow/public-artifacts.json | 6 +- packages/flow/pyproject.toml | 10 +- .../teach_scope.py => tests/_teach_scope.py} | 2 + .../flow/tests/e2e/test_injection_live.py | 2 +- .../tests/e2e/test_lending_fault_model.py | 4 +- packages/flow/tests/test_agent_guard.py | 4 +- .../flow/tests/test_continuous_learning.py | 12 +- packages/flow/tests/test_guarded_save.py | 20 +- packages/flow/tests/test_halt_learn_loop.py | 10 +- packages/flow/tests/test_harness2_adapter.py | 28 +- packages/flow/tests/test_harness2_pixels.py | 14 +- packages/flow/tests/test_harness2_traces.py | 10 +- packages/flow/tests/test_injection_paths.py | 4 +- packages/flow/tests/test_injection_suite.py | 2 +- .../tests/test_learning_gate_semantics.py | 12 +- packages/flow/tests/test_lending_swer.py | 4 +- packages/flow/tests/test_mockloan.py | 4 +- packages/flow/tests/test_probe_induction.py | 2 +- packages/flow/tests/test_provenance.py | 4 +- .../flow/tests/test_reward_trust_boundary.py | 6 +- packages/flow/tests/test_reward_worker.py | 8 +- packages/flow/tests/test_teach_placement.py | 2 +- packages/flow/tests/test_teach_scope.py | 8 +- packages/flow/tests/test_undo_proposal.py | 2 +- packages/flow/tests/test_verify_tools.py | 2 +- packages/flow/tests/test_verify_write.py | 2 +- packages/flow/tests/test_write_ledger.py | 4 +- packages/flow/uv.lock | 6 +- 80 files changed, 813 insertions(+), 682 deletions(-) rename packages/{flow/openadapt_flow/agent => agent/tests}/ledger_demo.py (99%) rename packages/flow/{openadapt_flow => benchmark/harness}/guard_demo.py (96%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/__init__.py (100%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/adapter.py (96%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/agent.py (99%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/apps.py (97%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/llm.py (100%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/pixels.py (98%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/proposers.py (98%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/rollout.py (96%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/scorer.py (98%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/selection.py (98%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/tasks.py (100%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/traces.py (98%) rename packages/flow/{openadapt_flow => benchmark/harness}/mockloan/__init__.py (89%) create mode 100644 packages/flow/benchmark/harness/mockloan/demo.py rename packages/flow/{openadapt_flow => benchmark/harness}/mockloan/fault_server.py (100%) rename packages/flow/{openadapt_flow => benchmark/harness}/mockloan/server.py (95%) rename packages/flow/{openadapt_flow => benchmark/harness}/mockloan/static/app.js (99%) rename packages/flow/{openadapt_flow => benchmark/harness}/mockloan/static/index.html (100%) rename packages/flow/{openadapt_flow => benchmark/harness}/mockloan/static/styles.css (100%) create mode 100644 packages/flow/benchmark/harness/synth_stream.py rename packages/flow/openadapt_flow/learning/{synth_stream.py => induce.py} (64%) create mode 100644 packages/flow/openadapt_flow/reward/bundle.py rename packages/flow/{openadapt_flow/mockmed/teach_scope.py => tests/_teach_scope.py} (97%) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 9daecc8dc2..397dbd2c87 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -448,7 +448,7 @@ jobs: python ../../scripts/ci/locked_install.py --lock 'packages/flow[dev,capture,agent]' --lock packages/capture --editable packages/types --editable packages/capture --editable packages/flow - name: Install Harness² for the integration tests - # Not on PyPI. Pinned to the commit openadapt_flow.integrations.harness2 + # Not on PyPI. Pinned to the commit benchmark.harness.harness2 # is written against; it has no runtime dependencies. run: >- uv pip install --system --no-deps diff --git a/CAPABILITIES.md b/CAPABILITIES.md index 1f9d0d7c2d..ead5238109 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -285,7 +285,7 @@ improvising or retrying when something is off. | Task ledger per job: rules locked when the job opens and enforced on every run, item status set only from OpenAdapt's own results, and yes-or-no questions a person answers on the phone or in Desktop, including when a run stops | On main ([#1332](https://github.com/OpenAdaptAI/OpenAdapt/pull/1332)) | Synthetic (30-item MockMed worklist, scripted agent, no model calls) | MCP tools `open_job`, `get_job`, `add_items`, `record_evidence`, `ask_user`, `get_answer`, `change_constraint`, `close_job`; [Track a long job](packages/agent/README.md#track-a-long-job) | | Limit a teaching agent to the declared task on every teach path (`compile_agent`, `compile-agent`, teach over MCP, the airbag guard): typed text must be a declared value, links and browser requests to other origins are refused, writes are checked before they are sent, and the record is read back before a draft exists. Injected instructions in record notes, banners, tooltips, alt text and server responses are blocked or caught on MockMed through each path (scripted agent; a live-model mode is off by default). Self-repair refuses a control that leaves the app | On main ([#1359](https://github.com/OpenAdaptAI/OpenAdapt/pull/1359), [#1386](https://github.com/OpenAdaptAI/OpenAdapt/pull/1386)) | Synthetic | `openadapt_flow.teach_scope`, [THREAT_MODEL.md](packages/flow/docs/THREAT_MODEL.md) | | Compile an agent's first verified run from a prompt (`compile-agent`), and a code-as-action authoring agent | On main ([#1277](https://github.com/OpenAdaptAI/OpenAdapt/pull/1277), [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)); the agent harness behind it runs on every backend, on main ([#1283](https://github.com/OpenAdaptAI/OpenAdapt/pull/1283)) | Synthetic | `compile-agent` | -| Score harness optimizers (Harness², Meta-Harness) on GUI tasks with the record read-back as the reward, with full trace export | On main ([#1299](https://github.com/OpenAdaptAI/OpenAdapt/pull/1299), [#1300](https://github.com/OpenAdaptAI/OpenAdapt/pull/1300), [#1301](https://github.com/OpenAdaptAI/OpenAdapt/pull/1301)) | Synthetic (scripted proposer) | `integrations/harness2/` | +| Score harness optimizers (Harness², Meta-Harness) on GUI tasks with the record read-back as the reward, with full trace export | On main ([#1299](https://github.com/OpenAdaptAI/OpenAdapt/pull/1299), [#1300](https://github.com/OpenAdaptAI/OpenAdapt/pull/1300), [#1301](https://github.com/OpenAdaptAI/OpenAdapt/pull/1301)) | Synthetic (scripted proposer) | `benchmark/harness/harness2/` (source checkout) | | One scoreboard comparing the agent alone with OpenAdapt across tasks (correctness by read-back, cost, time, items per hour) | On main ([#1303](https://github.com/OpenAdaptAI/OpenAdapt/pull/1303)) | Synthetic and local OpenEMR and openIMIS test copies | `benchmark/scoreboard/` | | Pull a skill from the registry (`pull app/task@version`) and run it only after qualifying it on your own instance: `pull` marks it and expires any certification it carries, `skill bind` records your addresses and the names of the variables that hold credentials (never their values), `skill qualify` runs its bundled record check there, and `run`, `replay`, `emit-skill` and `emit-mcp` refuse it until then, and on any other instance. The index and its archives carry a keyless Sigstore attestation from the registry release workflow, checked offline against a pinned identity, and expire and can't be rolled back; a shipped record check runs no code and reads only your binding | On main | Unit tests | `openadapt-flow skill`, [SKILLS_REGISTRY.md](packages/flow/docs/design/SKILLS_REGISTRY.md) | diff --git a/llms-full.txt b/llms-full.txt index 46b17ec00a..affba2443a 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -682,7 +682,7 @@ improvising or retrying when something is off. | Task ledger per job: rules locked when the job opens and enforced on every run, item status set only from OpenAdapt's own results, and yes-or-no questions a person answers on the phone or in Desktop, including when a run stops | On main ([#1332](https://github.com/OpenAdaptAI/OpenAdapt/pull/1332)) | Synthetic (30-item MockMed worklist, scripted agent, no model calls) | MCP tools `open_job`, `get_job`, `add_items`, `record_evidence`, `ask_user`, `get_answer`, `change_constraint`, `close_job`; [Track a long job](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/agent/README.md#track-a-long-job) | | Limit a teaching agent to the declared task on every teach path (`compile_agent`, `compile-agent`, teach over MCP, the airbag guard): typed text must be a declared value, links and browser requests to other origins are refused, writes are checked before they are sent, and the record is read back before a draft exists. Injected instructions in record notes, banners, tooltips, alt text and server responses are blocked or caught on MockMed through each path (scripted agent; a live-model mode is off by default). Self-repair refuses a control that leaves the app | On main ([#1359](https://github.com/OpenAdaptAI/OpenAdapt/pull/1359), [#1386](https://github.com/OpenAdaptAI/OpenAdapt/pull/1386)) | Synthetic | `openadapt_flow.teach_scope`, [THREAT_MODEL.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/THREAT_MODEL.md) | | Compile an agent's first verified run from a prompt (`compile-agent`), and a code-as-action authoring agent | On main ([#1277](https://github.com/OpenAdaptAI/OpenAdapt/pull/1277), [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)); the agent harness behind it runs on every backend, on main ([#1283](https://github.com/OpenAdaptAI/OpenAdapt/pull/1283)) | Synthetic | `compile-agent` | -| Score harness optimizers (Harness², Meta-Harness) on GUI tasks with the record read-back as the reward, with full trace export | On main ([#1299](https://github.com/OpenAdaptAI/OpenAdapt/pull/1299), [#1300](https://github.com/OpenAdaptAI/OpenAdapt/pull/1300), [#1301](https://github.com/OpenAdaptAI/OpenAdapt/pull/1301)) | Synthetic (scripted proposer) | `integrations/harness2/` | +| Score harness optimizers (Harness², Meta-Harness) on GUI tasks with the record read-back as the reward, with full trace export | On main ([#1299](https://github.com/OpenAdaptAI/OpenAdapt/pull/1299), [#1300](https://github.com/OpenAdaptAI/OpenAdapt/pull/1300), [#1301](https://github.com/OpenAdaptAI/OpenAdapt/pull/1301)) | Synthetic (scripted proposer) | `benchmark/harness/harness2/` (source checkout) | | One scoreboard comparing the agent alone with OpenAdapt across tasks (correctness by read-back, cost, time, items per hour) | On main ([#1303](https://github.com/OpenAdaptAI/OpenAdapt/pull/1303)) | Synthetic and local OpenEMR and openIMIS test copies | `benchmark/scoreboard/` | | Pull a skill from the registry (`pull app/task@version`) and run it only after qualifying it on your own instance: `pull` marks it and expires any certification it carries, `skill bind` records your addresses and the names of the variables that hold credentials (never their values), `skill qualify` runs its bundled record check there, and `run`, `replay`, `emit-skill` and `emit-mcp` refuse it until then, and on any other instance. The index and its archives carry a keyless Sigstore attestation from the registry release workflow, checked offline against a pinned identity, and expire and can't be rolled back; a shipped record check runs no code and reads only your binding | On main | Unit tests | `openadapt-flow skill`, [SKILLS_REGISTRY.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/design/SKILLS_REGISTRY.md) | diff --git a/packages/agent/README.md b/packages/agent/README.md index 3b65776b02..11dd2ff83e 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -498,7 +498,7 @@ A scripted agent worked a 30-item synthetic MockMed worklist. The user changes a In the middle arm, the agent opens a job and uses each item's `request_id` but keeps its flaws. The ledger stops every rule break, but it can't stop an agent from guessing a value or misreporting its own tally. `close_job` still lists what was verified, so a caller can check the agent's report. The scripted agent stands for a policy, not a model, so these numbers test the mechanism. They aren't a model benchmark. To reproduce them (about 3 seconds): ```bash -python -m openadapt_flow.agent.ledger_demo +python packages/agent/tests/ledger_demo.py ``` Results: [docs/task-ledger-measurement.json](docs/task-ledger-measurement.json). diff --git a/packages/flow/openadapt_flow/agent/ledger_demo.py b/packages/agent/tests/ledger_demo.py similarity index 99% rename from packages/flow/openadapt_flow/agent/ledger_demo.py rename to packages/agent/tests/ledger_demo.py index 8ba2f1fe81..2f7c45c10a 100644 --- a/packages/flow/openadapt_flow/agent/ledger_demo.py +++ b/packages/agent/tests/ledger_demo.py @@ -1,8 +1,9 @@ """A scripted agent works a 30-item synthetic MockMed worklist, with and without the task ledger. -Run it:: +It lives beside the test that uses it (``test_ledger.py``). Run it from the +repository root:: - python -m openadapt_flow.agent.ledger_demo [--out results.json] + python packages/agent/tests/ledger_demo.py [--out results.json] Everything is synthetic and local: the referrals, the patients, the app (an in-memory record store behind the real ``run_workflow`` service) and the diff --git a/packages/agent/tests/test_guard_tools.py b/packages/agent/tests/test_guard_tools.py index 6d10a04748..2d7064cff9 100644 --- a/packages/agent/tests/test_guard_tools.py +++ b/packages/agent/tests/test_guard_tools.py @@ -4,6 +4,8 @@ import base64 import json +import sys +from pathlib import Path import anyio import pytest @@ -13,15 +15,19 @@ from openadapt_flow.approval_mode.demo_approver import demo_approver, demo_root from openadapt_flow.agent.mcp import build_server from openadapt_flow.guard import GUARD_TASK_SCHEMA, GuardedSave -from openadapt_flow.guard_demo import ( +from openadapt_flow.mockmed.fault_server import serve as fault_serve +from openadapt_flow.verify_run import parse_verify_config + +# The fake agent and its helpers are Flow's guarded-save demo, which lives in +# Flow's source checkout (packages/flow/benchmark/harness), outside the wheel. +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "flow")) +from benchmark.harness.guard_demo import ( # noqa: E402 NOTE, FakeAgent, person_approves, tiny_png, verify_yaml, ) -from openadapt_flow.mockmed.fault_server import serve as fault_serve -from openadapt_flow.verify_run import parse_verify_config @pytest.fixture diff --git a/packages/agent/tests/test_ledger.py b/packages/agent/tests/test_ledger.py index bfd588d543..c8b4260f1d 100644 --- a/packages/agent/tests/test_ledger.py +++ b/packages/agent/tests/test_ledger.py @@ -7,6 +7,8 @@ import anyio import pytest +# The scripted 30-referral simulation is test data; it lives beside this file. +from ledger_demo import WORKFLOW, SyntheticReferralApp, measure from openadapt_flow.agent.bridge import AgentBridge, BridgeError from openadapt_flow.agent.cli import main from openadapt_flow.agent.ledger import ( @@ -14,7 +16,6 @@ LEDGER_TOOLS, TaskLedger, ) -from openadapt_flow.agent.ledger_demo import WORKFLOW, SyntheticReferralApp, measure from openadapt_flow.agent.mcp import build_server from openadapt_flow.agent.runner import RunnerConfig from openadapt_flow.approval_mode.demo_approver import demo_approver, demo_root diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md index 978a0b337a..6a5b595ddc 100644 --- a/packages/flow/CHANGELOG.md +++ b/packages/flow/CHANGELOG.md @@ -27,6 +27,8 @@ - The wheel no longer ships the benchmark harnesses. `openadapt_flow.benchmark` and `openadapt_flow.instrument` moved to `packages/flow/benchmark/harness/` in the source tree (import `benchmark.harness`); `openadapt-flow benchmark` runs from a source checkout and says so from a wheel. The engine audit for application-specific code is in `docs/GENERALITY_AUDIT.md`. +- The wheel no longer ships test fixtures and demo harnesses: the Harness² integration, the MockLoan app and its recorder, `guard_demo`, the MockMed teach-scope adapter, the synthetic drift stream (the `StructuralDiffInducer` it used now lives in `openadapt_flow.learning.induce`) and the task-ledger simulation moved to `benchmark/harness/` or beside their tests, the reward bundle helpers moved to `openadapt_flow.reward.bundle`, and the `harness2` extra is gone. + - Self-repair can take a stop where the app's own dialog covers the next step's identity band: when the backend proves the covering window belongs to the window the last verified step was in, and the part of the band read still matches the record, the dialog is dismissed after the previous click and the step re-checks its identity in full. Another record behind the dialog stays an identity halt. The runtime lets a backend remember a verified record's window (`record_verified_window`); macOS answers `owned_window_over` from the window's proved sheet ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). - macOS: a run can now act in the pinned window's own AppKit sheet (an `NSAlert` attached to it), which is a separate window. `MacOSBackend` accepts it only when AX lists exactly one `AXSheet` on the window and exactly one window of the same process has its bounds inside it; the capture composites exactly that sheet over the window, and focus, pointer and click-target checks accept it only inside its bounds. Any other window is still refused ([#1711](https://github.com/OpenAdaptAI/OpenAdapt/issues/1711)). diff --git a/packages/flow/openadapt_flow/guard_demo.py b/packages/flow/benchmark/harness/guard_demo.py similarity index 96% rename from packages/flow/openadapt_flow/guard_demo.py rename to packages/flow/benchmark/harness/guard_demo.py index a7578d1bb3..4d1fb788d1 100644 --- a/packages/flow/openadapt_flow/guard_demo.py +++ b/packages/flow/benchmark/harness/guard_demo.py @@ -1,9 +1,9 @@ """Guarded-save demo: a fake agent saves a referral to MockMed through the guard. -Run it with ``python -m openadapt_flow.guard_demo``. It starts the MockMed -fault server on localhost, and for each case a fake agent opens a chart page, -asks the guard, and posts the referral to MockMed's API the way a -computer-use agent's click on Save would. The guard reads ``GET /api/db`` +Run it from ``packages/flow`` with ``python -m benchmark.harness.guard_demo``. +It starts the MockMed fault server on localhost, and for each case a fake +agent opens a chart page, asks the guard, and posts the referral to MockMed's +API the way a computer-use agent's click on Save would. The guard reads ``GET /api/db`` (the system of record) to judge the write. All data is synthetic, nothing leaves localhost, and no model is called. diff --git a/packages/flow/openadapt_flow/integrations/harness2/__init__.py b/packages/flow/benchmark/harness/harness2/__init__.py similarity index 100% rename from packages/flow/openadapt_flow/integrations/harness2/__init__.py rename to packages/flow/benchmark/harness/harness2/__init__.py diff --git a/packages/flow/openadapt_flow/integrations/harness2/adapter.py b/packages/flow/benchmark/harness/harness2/adapter.py similarity index 96% rename from packages/flow/openadapt_flow/integrations/harness2/adapter.py rename to packages/flow/benchmark/harness/harness2/adapter.py index ba92686434..a6d9c456a3 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/adapter.py +++ b/packages/flow/benchmark/harness/harness2/adapter.py @@ -7,8 +7,9 @@ only measured afterwards. This adapter gives it one: every rollout is scored by the reward worker reading the system of record of a fresh app copy. -This module imports Harness²; it copies none of it. Install it beside -``openadapt-flow[harness2]`` (it is not on PyPI):: +This module imports Harness²; it copies none of it. It lives in the source +checkout (run from ``packages/flow``). Install Harness² beside +``openadapt-flow[browser]`` (it is not on PyPI):: pip install 'harness2 @ git+https://github.com/google-research/harness2@1cfe254e8d25b15f37048c3730134686abe60335' @@ -19,7 +20,7 @@ This is Harness² as published, with no scorer. * ``readback``: each trajectory and report also carries what the system of record shows after the run (which records were added, whose they are, the - outcome). With :func:`~openadapt_flow.integrations.harness2.selection.reward_selection` + outcome). With :func:`~benchmark.harness.harness2.selection.reward_selection` the final is chosen by the read-back reward instead of a model. In both modes ``score()`` is the read-back. Harness² calls it only after @@ -34,17 +35,17 @@ from harness2.core.spec import BenchPrompts, BenchSpec, ChannelSpec from harness2.core.tree import SurfaceSpec, read_tree, write_tree -from openadapt_flow.integrations.harness2.agent import GuiAgent -from openadapt_flow.integrations.harness2.apps import AppTarget, default_targets -from openadapt_flow.integrations.harness2.rollout import ( +from benchmark.harness.harness2.agent import GuiAgent +from benchmark.harness.harness2.apps import AppTarget, default_targets +from benchmark.harness.harness2.rollout import ( REPORT_FILE, load_run, load_score, load_steps, run_rollout, ) -from openadapt_flow.integrations.harness2.scorer import EpisodeScore, Penalties -from openadapt_flow.integrations.harness2.tasks import MOCKMED_PATIENTS, GuiTask +from benchmark.harness.harness2.scorer import EpisodeScore, Penalties +from benchmark.harness.harness2.tasks import MOCKMED_PATIENTS, GuiTask EVIDENCE_MODES = ("screen", "readback") @@ -307,7 +308,7 @@ def trajectory(self, run_dir: Path, *, obs_trunc: int) -> str: if self.evidence == "readback" and run.get("task_id") in self._tasks: # OpenAdapt's identity check at each write: was the record on # screen the requested one? Part of the read-back evidence. - from openadapt_flow.integrations.harness2.traces import from_rollout + from benchmark.harness.harness2.traces import from_rollout trace = from_rollout(run_dir, self._tasks[run["task_id"]]) identity = {s.index: s.identity for s in trace.steps if s.identity} diff --git a/packages/flow/openadapt_flow/integrations/harness2/agent.py b/packages/flow/benchmark/harness/harness2/agent.py similarity index 99% rename from packages/flow/openadapt_flow/integrations/harness2/agent.py rename to packages/flow/benchmark/harness/harness2/agent.py index 3894249ba6..68fe348227 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/agent.py +++ b/packages/flow/benchmark/harness/harness2/agent.py @@ -27,7 +27,7 @@ from dataclasses import asdict, dataclass, field from typing import Any, Optional, Protocol, runtime_checkable -from openadapt_flow.integrations.harness2.llm import ChatModel +from benchmark.harness.harness2.llm import ChatModel #: The final report statuses and the runtime signal each maps to. STATUS_SIGNAL = {"saved": "completed", "stopped": "halted_before_effect"} diff --git a/packages/flow/openadapt_flow/integrations/harness2/apps.py b/packages/flow/benchmark/harness/harness2/apps.py similarity index 97% rename from packages/flow/openadapt_flow/integrations/harness2/apps.py rename to packages/flow/benchmark/harness/harness2/apps.py index 4f82c27e00..05a1867927 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/apps.py +++ b/packages/flow/benchmark/harness/harness2/apps.py @@ -15,7 +15,7 @@ from typing import Any, Callable, Optional, Protocol, runtime_checkable from urllib.parse import urlencode -from openadapt_flow.integrations.harness2.tasks import MOCKMED_PATIENTS, GuiTask +from benchmark.harness.harness2.tasks import MOCKMED_PATIENTS, GuiTask @runtime_checkable diff --git a/packages/flow/openadapt_flow/integrations/harness2/llm.py b/packages/flow/benchmark/harness/harness2/llm.py similarity index 100% rename from packages/flow/openadapt_flow/integrations/harness2/llm.py rename to packages/flow/benchmark/harness/harness2/llm.py diff --git a/packages/flow/openadapt_flow/integrations/harness2/pixels.py b/packages/flow/benchmark/harness/harness2/pixels.py similarity index 98% rename from packages/flow/openadapt_flow/integrations/harness2/pixels.py rename to packages/flow/benchmark/harness/harness2/pixels.py index af0ea96df2..0d6d9ea6c1 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/pixels.py +++ b/packages/flow/benchmark/harness/harness2/pixels.py @@ -26,7 +26,7 @@ import time from typing import Any, Callable, Optional -from openadapt_flow.integrations.harness2.agent import Element, Observation +from benchmark.harness.harness2.agent import Element, Observation #: Seconds between frames while waiting for the screen to stop changing. _FRAME_GAP_S = 0.25 diff --git a/packages/flow/openadapt_flow/integrations/harness2/proposers.py b/packages/flow/benchmark/harness/harness2/proposers.py similarity index 98% rename from packages/flow/openadapt_flow/integrations/harness2/proposers.py rename to packages/flow/benchmark/harness/harness2/proposers.py index 26840b7002..f08fae6352 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/proposers.py +++ b/packages/flow/benchmark/harness/harness2/proposers.py @@ -21,12 +21,12 @@ from harness2.core.proposer import Proposer -from openadapt_flow.integrations.harness2.agent import ( +from benchmark.harness.harness2.agent import ( RULE_IDENTITY, RULE_SAVE_ONCE, RULE_TYPE_LABEL, ) -from openadapt_flow.integrations.harness2.llm import ChatModel +from benchmark.harness.harness2.llm import ChatModel _CHOICE_HINT = "```choice" diff --git a/packages/flow/openadapt_flow/integrations/harness2/rollout.py b/packages/flow/benchmark/harness/harness2/rollout.py similarity index 96% rename from packages/flow/openadapt_flow/integrations/harness2/rollout.py rename to packages/flow/benchmark/harness/harness2/rollout.py index c404a95717..4b85761492 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/rollout.py +++ b/packages/flow/benchmark/harness/harness2/rollout.py @@ -27,14 +27,14 @@ from pathlib import Path from typing import Any, Callable, Optional -from openadapt_flow.integrations.harness2.agent import AgentResult, GuiAgent, PageDriver -from openadapt_flow.integrations.harness2.apps import AppTarget -from openadapt_flow.integrations.harness2.scorer import ( +from benchmark.harness.harness2.agent import AgentResult, GuiAgent, PageDriver +from benchmark.harness.harness2.apps import AppTarget +from benchmark.harness.harness2.scorer import ( EpisodeScore, Penalties, ReadBackScorer, ) -from openadapt_flow.integrations.harness2.tasks import GuiTask +from benchmark.harness.harness2.tasks import GuiTask RUN_FILE = "run.json" STEPS_FILE = "steps.jsonl" diff --git a/packages/flow/openadapt_flow/integrations/harness2/scorer.py b/packages/flow/benchmark/harness/harness2/scorer.py similarity index 98% rename from packages/flow/openadapt_flow/integrations/harness2/scorer.py rename to packages/flow/benchmark/harness/harness2/scorer.py index 7c08719030..ce86a38157 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/scorer.py +++ b/packages/flow/benchmark/harness/harness2/scorer.py @@ -32,8 +32,8 @@ from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey -from openadapt_flow.integrations.harness2.tasks import GuiTask -from openadapt_flow.reward.seed import write_bundle +from benchmark.harness.harness2.tasks import GuiTask +from openadapt_flow.reward.bundle import write_bundle from openadapt_flow.reward.worker import RewardWorker #: Runtime signals an agent's final report can map to. diff --git a/packages/flow/openadapt_flow/integrations/harness2/selection.py b/packages/flow/benchmark/harness/harness2/selection.py similarity index 98% rename from packages/flow/openadapt_flow/integrations/harness2/selection.py rename to packages/flow/benchmark/harness/harness2/selection.py index f98d0e5397..d2ee73597e 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/selection.py +++ b/packages/flow/benchmark/harness/harness2/selection.py @@ -24,8 +24,8 @@ from harness2.core.proposer import Proposer from harness2.core.spec import RunView -from openadapt_flow.integrations.harness2 import traces -from openadapt_flow.integrations.harness2.adapter import OpenAdaptGuiAdapter +from benchmark.harness.harness2 import traces +from benchmark.harness.harness2.adapter import OpenAdaptGuiAdapter #: Where an unscored candidate ranks: below a safe stop, above any penalty. _UNSCORED_RANK = -0.01 diff --git a/packages/flow/openadapt_flow/integrations/harness2/tasks.py b/packages/flow/benchmark/harness/harness2/tasks.py similarity index 100% rename from packages/flow/openadapt_flow/integrations/harness2/tasks.py rename to packages/flow/benchmark/harness/harness2/tasks.py diff --git a/packages/flow/openadapt_flow/integrations/harness2/traces.py b/packages/flow/benchmark/harness/harness2/traces.py similarity index 98% rename from packages/flow/openadapt_flow/integrations/harness2/traces.py rename to packages/flow/benchmark/harness/harness2/traces.py index 81eb053394..0d5fbb7b35 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/traces.py +++ b/packages/flow/benchmark/harness/harness2/traces.py @@ -39,14 +39,14 @@ from pathlib import Path from typing import Any, Iterable, Optional, Sequence -from openadapt_flow import privacy -from openadapt_flow.integrations.harness2.rollout import ( +from benchmark.harness.harness2.rollout import ( load_run, load_score, load_steps, ) -from openadapt_flow.integrations.harness2.scorer import EpisodeScore -from openadapt_flow.integrations.harness2.tasks import GuiTask +from benchmark.harness.harness2.scorer import EpisodeScore +from benchmark.harness.harness2.tasks import GuiTask +from openadapt_flow import privacy SCHEMA = "openadapt.trace-export/v1" diff --git a/packages/flow/openadapt_flow/mockloan/__init__.py b/packages/flow/benchmark/harness/mockloan/__init__.py similarity index 89% rename from packages/flow/openadapt_flow/mockloan/__init__.py rename to packages/flow/benchmark/harness/mockloan/__init__.py index 1279fb3536..a5e02f767a 100644 --- a/packages/flow/openadapt_flow/mockloan/__init__.py +++ b/packages/flow/benchmark/harness/mockloan/__init__.py @@ -8,6 +8,6 @@ transactional fault injection via ``?fault=`` (see ``fault_server``). """ -from openadapt_flow.mockloan.server import serve # noqa: F401 +from benchmark.harness.mockloan.server import serve # noqa: F401 __all__ = ["serve"] diff --git a/packages/flow/benchmark/harness/mockloan/demo.py b/packages/flow/benchmark/harness/mockloan/demo.py new file mode 100644 index 0000000000..e56fc280dd --- /dev/null +++ b/packages/flow/benchmark/harness/mockloan/demo.py @@ -0,0 +1,97 @@ +"""Record the canonical MockLoan disbursement demonstration. + +Record time is allowed to cheat with Playwright locators (to find pixel +coordinates via ``bounding_box()``), but every action is performed through +`Recorder` so frames and events are captured exactly as a human demo would +be. Replay never uses selectors. + +The non-healthcare companion to +:func:`openadapt_flow.demo_driver.record_triage_demo`; both drive the same +governed record -> compile -> replay path. +""" + +from __future__ import annotations + +from pathlib import Path + +from openadapt_flow.backends.playwright_backend import PlaywrightBackend +from openadapt_flow.recorder import Recorder + +__all__ = ["record_disbursement_demo"] + + +def record_disbursement_demo( + url: str, + out_dir: Path | str, + *, + memo_text: str, + param_name: str = "memo", + headed: bool = False, + record_video_dir: Path | str | None = None, +) -> Path: + """Record the canonical MockLoan disbursement demo against a running app. + + The non-healthcare companion to :func:`record_triage_demo`. Flow: + login -> pipeline -> Open first loan -> New Disbursement -> Personal product + -> click Funding memo field -> type memo (as a parameter) -> Authorize + Disbursement. The consequential final step authorizes moving money to a + borrower, the lending analog of clinically saving an encounter. + + Args: + url: MockLoan base URL (from `benchmark.harness.mockloan.server.serve`). + out_dir: Directory to write the recording into. + memo_text: The funding memo typed during the demo (recorded as a param). + param_name: Parameter name for the typed memo (default ``"memo"``). + headed: Run the browser headed (visible) instead of headless. + record_video_dir: OPT-IN (default ``None`` = off). When set, a WebM + video of the recording session is captured into this directory; it + does not change what is recorded to ``out_dir``. + + Returns: + The recording directory (contains meta.json, events.jsonl, frames/). + """ + backend, close = PlaywrightBackend.launch( + url, + headless=not headed, + record_video_dir=( + str(record_video_dir) if record_video_dir is not None else None + ), + ) + try: + page = backend.page + recorder = Recorder(backend, out_dir, app_url=url) + + def center(selector: str) -> tuple[int, int]: + """Center pixel coordinates of the first element matching CSS.""" + locator = page.locator(selector).first + locator.wait_for(state="visible") + box = locator.bounding_box() + if box is None: # pragma: no cover - visible => box exists + raise RuntimeError(f"no bounding box for {selector!r}") + return ( + int(box["x"] + box["width"] / 2), + int(box["y"] + box["height"] / 2), + ) + + # Login + recorder.click(*center("#username")) + recorder.type_text("officer.demo") + recorder.click(*center("#password")) + recorder.type_text("mockloan-demo-pass") + recorder.click(*center("#signin")) + + # Pipeline -> open first loan + recorder.click(*center(".open-btn")) + + # Loan -> new disbursement + recorder.click(*center("#new-disbursement")) + + # Disbursement: product chooser, memo (parameterized), authorize + recorder.click(*center("#product-personal")) + recorder.click(*center("#memo")) + recorder.type_text(memo_text, param=param_name) + recorder.click(*center("#authorize")) + + return recorder.finish() + finally: + close() diff --git a/packages/flow/openadapt_flow/mockloan/fault_server.py b/packages/flow/benchmark/harness/mockloan/fault_server.py similarity index 100% rename from packages/flow/openadapt_flow/mockloan/fault_server.py rename to packages/flow/benchmark/harness/mockloan/fault_server.py diff --git a/packages/flow/openadapt_flow/mockloan/server.py b/packages/flow/benchmark/harness/mockloan/server.py similarity index 95% rename from packages/flow/openadapt_flow/mockloan/server.py rename to packages/flow/benchmark/harness/mockloan/server.py index 0bc09c6603..f20c68e5b4 100644 --- a/packages/flow/openadapt_flow/mockloan/server.py +++ b/packages/flow/benchmark/harness/mockloan/server.py @@ -1,6 +1,6 @@ """Threaded static-file server for the MockLoan demo app. -Serves ``openadapt_flow/mockloan/static`` on localhost. No external resources +Serves ``benchmark/harness/mockloan/static`` on localhost. No external resources are referenced by the app, so tests never touch the network beyond localhost. """ diff --git a/packages/flow/openadapt_flow/mockloan/static/app.js b/packages/flow/benchmark/harness/mockloan/static/app.js similarity index 99% rename from packages/flow/openadapt_flow/mockloan/static/app.js rename to packages/flow/benchmark/harness/mockloan/static/app.js index ed305c1714..7f5bce22c9 100644 --- a/packages/flow/openadapt_flow/mockloan/static/app.js +++ b/packages/flow/benchmark/harness/mockloan/static/app.js @@ -29,7 +29,7 @@ var LABEL_OPEN = DRIFT.has('rename') ? 'View' : 'Open'; // Transactional fault-injection hook (flag-gated, exactly like ?drift=). // When the page is loaded with ?fault= the Authorize handler routes the -// write through a REAL backend API (openadapt_flow/mockloan/fault_server.py) +// write through a REAL backend API (benchmark/harness/mockloan/fault_server.py) // so the fault-model study can judge a replay against DB ground truth rather // than the on-screen banner. With no ?fault query this is inert and Authorize // behaves byte-for-byte as before (the normal benchmark is unaffected). diff --git a/packages/flow/openadapt_flow/mockloan/static/index.html b/packages/flow/benchmark/harness/mockloan/static/index.html similarity index 100% rename from packages/flow/openadapt_flow/mockloan/static/index.html rename to packages/flow/benchmark/harness/mockloan/static/index.html diff --git a/packages/flow/openadapt_flow/mockloan/static/styles.css b/packages/flow/benchmark/harness/mockloan/static/styles.css similarity index 100% rename from packages/flow/openadapt_flow/mockloan/static/styles.css rename to packages/flow/benchmark/harness/mockloan/static/styles.css diff --git a/packages/flow/benchmark/harness/synth_stream.py b/packages/flow/benchmark/harness/synth_stream.py new file mode 100644 index 0000000000..a516971965 --- /dev/null +++ b/packages/flow/benchmark/harness/synth_stream.py @@ -0,0 +1,222 @@ +"""Synthetic trace-stream harness: a MockMed skill drifting over "time". + +To exercise the learn/promote loop end-to-end without a live app, this module + +1. builds a base MockMed skill program (``add-patient-note`` from + ``docs/design/WORKFLOW_PROGRAM_IR.md`` section 2.5: sign in -> open the + patient record -> new encounter -> type note -> SAVE), and +2. emits a STREAM of :class:`~openadapt_flow.learning.trace.ExecutionTrace`s over + a logical time axis with INJECTED DRIFT at a chosen trace index -- e.g. a new + consent dialog starts appearing mid-stream, so the active program stops + reproducing the new successful traces and the loop must detect the novelty, + induce a revision, validate it, and promote it. + +The reference inducer that revises the program, +:class:`~openadapt_flow.learning.induce.StructuralDiffInducer`, ships in the +engine; this stream is a test and benchmark fixture and lives in the source +checkout (``packages/flow/benchmark/harness``), outside the wheel. + +All deterministic, ``$0``, no model calls. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Optional + +from openadapt_flow.ir import ( + ActionKind, + Guard, + ProgramGraph, + State, + StateKind, +) +from openadapt_flow.learning.induce import StructuralDiffInducer +from openadapt_flow.learning.induce import _action_state as _induce_action_state +from openadapt_flow.learning.trace import ExecutionTrace, TraceStep +from openadapt_flow.runtime.effects.effect import Effect, EffectKind + +__all__ = [ + "Drift", + "StructuralDiffInducer", + "generate_stream", + "mockmed_base_program", +] + +# -- the MockMed skill's canonical intents ------------------------------------ + +INTENT_LOGIN = "Sign in" +INTENT_OPEN = "Open patient record" +INTENT_ENCOUNTER = "Start new encounter" +INTENT_NOTE = "Enter note" +INTENT_SAVE = "Save encounter" +INTENT_CONSENT = "Acknowledge consent notice" + +#: The screen fact that co-occurs with the injected consent dialog -- the text a +#: ``TEXT_PRESENT`` guard keys on. +FACT_CONSENT = "Consent Required" + +#: A representative identity band for the patient row (name + DOB) -- the +#: evidence the pre-click identity gate keys on; makes the "open patient" step +#: identity-ARMED so an identity regression is meaningful. +PATIENT_BAND = "Belford, Phil DOB 1990-01-01" + + +def _action_state( + sid: str, + intent: str, + target: str, + *, + action: ActionKind = ActionKind.CLICK, + armed: bool = False, + risk: str = "reversible", + effects: Optional[list[Effect]] = None, + guard: Optional[Guard] = None, + text: Optional[str] = None, +) -> State: + return _induce_action_state( + sid, + intent, + target, + action=action, + armed=armed, + risk=risk, + effects=effects, + guard=guard, + text=text, + context_text=PATIENT_BAND, + ) + + +def mockmed_base_program() -> ProgramGraph: + """The base ``add-patient-note`` program (no consent dialog). + + A straight-line state machine: sign in -> open patient (identity-armed) -> + new encounter -> type note -> SAVE (irreversible, with a ``record_written`` + system-of-record effect) -> success. The identity band + effect + risk on + the sensitive steps are what a rigged revision could weaken, and what the + regression gate protects. + """ + save_effect = Effect( + kind=EffectKind.RECORD_WRITTEN, + match={"type": "encounter", "patient": "p1"}, + expected_count=1, + risk="irreversible", + probe="encounter record written for patient", + ) + states = { + "s_login": _action_state("s_login", INTENT_LOGIN, "s_open"), + "s_open": _action_state("s_open", INTENT_OPEN, "s_encounter", armed=True), + "s_encounter": _action_state("s_encounter", INTENT_ENCOUNTER, "s_note"), + "s_note": _action_state( + "s_note", INTENT_NOTE, "s_save", action=ActionKind.TYPE, text="Follow-up" + ), + "s_save": _action_state( + "s_save", + INTENT_SAVE, + "__end__", + risk="irreversible", + effects=[save_effect], + ), + "__end__": State(id="__end__", kind=StateKind.TERMINAL, outcome="success"), + } + return ProgramGraph(entry="s_login", states=states) + + +# -- the drift stream --------------------------------------------------------- + + +class Drift(str, Enum): + """The injected drift/variant a stream can carry from trace K onward.""" + + NONE = "none" + #: A new consent dialog appears after sign-in (a new OPTIONAL step + branch). + CONSENT_DIALOG = "consent_dialog" + #: The note field is renamed (the observed intent changes) -- a structural + #: change the additive reference inducer CANNOT cover (used to show the loop + #: correctly refuses an inadequate revision). + FIELD_RENAME = "field_rename" + + +def _baseline_steps() -> list[TraceStep]: + return [ + TraceStep(intent=INTENT_LOGIN), + TraceStep(intent=INTENT_OPEN, identity=PATIENT_BAND), + TraceStep(intent=INTENT_ENCOUNTER), + TraceStep(intent=INTENT_NOTE, action=ActionKind.TYPE), + TraceStep(intent=INTENT_SAVE), + ] + + +def _drifted_steps(drift: Drift) -> tuple[list[TraceStep], dict[str, bool]]: + steps = _baseline_steps() + facts: dict[str, bool] = {} + if drift is Drift.CONSENT_DIALOG: + # The consent dialog appears right after sign-in. + steps.insert(1, TraceStep(intent=INTENT_CONSENT)) + facts[FACT_CONSENT] = True + elif drift is Drift.FIELD_RENAME: + for s in steps: + if s.intent == INTENT_NOTE: + s.intent = "Enter clinical note" + return steps, facts + + +def generate_stream( + *, + n_baseline: int = 6, + n_drift: int = 6, + drift: Drift = Drift.CONSENT_DIALOG, + n_failures: int = 0, + prefix: str = "mockmed", +) -> list[ExecutionTrace]: + """Emit a deterministic stream of traces with drift injected at trace + ``n_baseline``. + + The first ``n_baseline`` traces are clean successful runs of the base skill; + from index ``n_baseline`` the injected ``drift`` appears in every successful + run. ``n_failures`` failure traces (noise) are appended -- runs that halted + partway with no novel successful structure, so a batch of them alone must NOT + trigger a revision. + """ + traces: list[ExecutionTrace] = [] + t = 0 + for _ in range(n_baseline): + traces.append( + ExecutionTrace( + trace_id=f"{prefix}-{t:04d}", + t=t, + outcome="success", + steps=_baseline_steps(), + params={"encounter_type": "Triage"}, + ) + ) + t += 1 + for _ in range(n_drift): + steps, facts = _drifted_steps(drift) + traces.append( + ExecutionTrace( + trace_id=f"{prefix}-{t:04d}", + t=t, + outcome="success", + steps=steps, + facts=facts, + params={"encounter_type": "Triage"}, + ) + ) + t += 1 + for _ in range(n_failures): + # A run that halted after opening the patient (e.g. a transient + # resolution failure): truncated, no novel successful structure. + traces.append( + ExecutionTrace( + trace_id=f"{prefix}-{t:04d}", + t=t, + outcome="failure", + steps=_baseline_steps()[:2], + failure_reason="transient resolution failure on 'Start new encounter'", + params={"encounter_type": "Triage"}, + ) + ) + t += 1 + return traces diff --git a/packages/flow/benchmark/harness2_scorer/RESULTS.md b/packages/flow/benchmark/harness2_scorer/RESULTS.md index eff54f7ebb..975843b034 100644 --- a/packages/flow/benchmark/harness2_scorer/RESULTS.md +++ b/packages/flow/benchmark/harness2_scorer/RESULTS.md @@ -16,7 +16,7 @@ reasons under [What went wrong](#what-went-wrong). ## Setup - **Tasks:** the seven MockMed variants in - `openadapt_flow.integrations.harness2.tasks`: `plain`, `lookalike`, `sort`, + `benchmark.harness.harness2.tasks`: `plain`, `lookalike`, `sort`, `typelabel`, `timeout`, `missing`, `slow`. Every task asks for a Triage encounter for Jane Sample (born 1980-01-01) with a fixed note. - **Agent:** `ModelGuiAgent` on `gpt-4.1-nano`, reading the DOM text and diff --git a/packages/flow/benchmark/harness2_scorer/run.py b/packages/flow/benchmark/harness2_scorer/run.py index fd7d17cd6d..e96f0dff64 100644 --- a/packages/flow/benchmark/harness2_scorer/run.py +++ b/packages/flow/benchmark/harness2_scorer/run.py @@ -34,21 +34,21 @@ from harness2.core.spec import ChannelSpec from harness2.core.tree import write_tree -from openadapt_flow import privacy -from openadapt_flow.integrations.harness2.adapter import ( +from benchmark.harness.harness2.adapter import ( GUI_SURFACE, OpenAdaptGuiAdapter, ) -from openadapt_flow.integrations.harness2.agent import ModelGuiAgent -from openadapt_flow.integrations.harness2.llm import ( +from benchmark.harness.harness2.agent import ModelGuiAgent +from benchmark.harness.harness2.llm import ( BudgetExceeded, CostLedger, HttpChatModel, Price, ) -from openadapt_flow.integrations.harness2.proposers import ModelProposer -from openadapt_flow.integrations.harness2.selection import run_task -from openadapt_flow.integrations.harness2.tasks import mockmed_tasks +from benchmark.harness.harness2.proposers import ModelProposer +from benchmark.harness.harness2.selection import run_task +from benchmark.harness.harness2.tasks import mockmed_tasks +from openadapt_flow import privacy HERE = Path(__file__).resolve().parent diff --git a/packages/flow/benchmark/lending_fault_model/LENDING_FAULT_MODEL.md b/packages/flow/benchmark/lending_fault_model/LENDING_FAULT_MODEL.md index 8ec43c4fe9..9aa741b969 100644 --- a/packages/flow/benchmark/lending_fault_model/LENDING_FAULT_MODEL.md +++ b/packages/flow/benchmark/lending_fault_model/LENDING_FAULT_MODEL.md @@ -10,7 +10,7 @@ Model calls: **0** (compiled replay; `ANTHROPIC_API_KEY` unset, grounder rung ne ## Method -The bundled MockLoan app is a client-side SPA with no backend, so an "authorize" only mutates in-page state - the UI *is* the source of truth. This study adds a real persistence boundary (`openadapt_flow/mockloan/fault_server.py`): a flag-gated `?fault=` hook in the app (mirroring the `?drift=` hooks) routes the Authorize write through a backend API with an isolated SQLite ledger. **With no `?fault` query the app never calls the API and the normal benchmark is byte-for-byte unaffected** (pinned by a test). Each fault class is injected at that boundary; the SAME recorded bundle is replayed through the REAL `Replayer` against it, and the outcome is judged by `GET /api/db` (ground truth) plus whether the replay halted - never by the replay's self-report. `/api/db` is a read path the SPA itself never calls, so the oracle cannot be gamed by the screen. +The bundled MockLoan app is a client-side SPA with no backend, so an "authorize" only mutates in-page state - the UI *is* the source of truth. This study adds a real persistence boundary (`benchmark/harness/mockloan/fault_server.py`): a flag-gated `?fault=` hook in the app (mirroring the `?drift=` hooks) routes the Authorize write through a backend API with an isolated SQLite ledger. **With no `?fault` query the app never calls the API and the normal benchmark is byte-for-byte unaffected** (pinned by a test). Each fault class is injected at that boundary; the SAME recorded bundle is replayed through the REAL `Replayer` against it, and the outcome is judged by `GET /api/db` (ground truth) plus whether the replay halted - never by the replay's self-report. `/api/db` is a read path the SPA itself never calls, so the oracle cannot be gamed by the screen. ### Outcome taxonomy diff --git a/packages/flow/benchmark/lending_fault_model/README.md b/packages/flow/benchmark/lending_fault_model/README.md index b6de3a698a..f2b801ca09 100644 --- a/packages/flow/benchmark/lending_fault_model/README.md +++ b/packages/flow/benchmark/lending_fault_model/README.md @@ -6,7 +6,7 @@ All prior transactional evidence (`benchmark/fault_model`, `benchmark/effect_readback`, `benchmark/silent_wrong_action`) ran against **MockMed**, a clinical fixture. This study replicates the same rigorous method on a **second, non-healthcare system of record with a distinct UI**: **MockLoan** -(`openadapt_flow/mockloan/`), a loan-origination console whose consequential +(`benchmark/harness/mockloan/`), a loan-origination console whose consequential write **authorizes a disbursement of funds to a borrower** - an irreversible money-movement write. @@ -16,11 +16,11 @@ judged by an independent ledger read, with zero model calls. ## The fixture -- `openadapt_flow/mockloan/` - a self-contained, deterministic static SPA +- `benchmark/harness/mockloan/` - a self-contained, deterministic static SPA (login -> funding pipeline -> loan -> new disbursement -> **Authorize Disbursement**), a distinct finance UI from MockMed. All data is fake/synthetic (no real PII). -- `openadapt_flow/mockloan/fault_server.py` - adds a real persistence boundary +- `benchmark/harness/mockloan/fault_server.py` - adds a real persistence boundary (an in-process **ledger**) and a flag-gated `?fault=` hook that mirrors MockMed's. **With no `?fault` query the app never calls the API** and the normal benchmark is byte-for-byte unaffected (pinned by a test). The ledger is diff --git a/packages/flow/benchmark/lending_fault_model/faults.py b/packages/flow/benchmark/lending_fault_model/faults.py index b549bab918..b9a1b726ea 100644 --- a/packages/flow/benchmark/lending_fault_model/faults.py +++ b/packages/flow/benchmark/lending_fault_model/faults.py @@ -6,7 +6,7 @@ (``tests/e2e/test_lending_fault_model.py``). It defines: - the transactional fault classes injected at the persistence boundary - (realized by ``openadapt_flow/mockloan/fault_server.py`` + the flag-gated + (realized by ``benchmark/harness/mockloan/fault_server.py`` + the flag-gated ``?fault=`` hook in ``mockloan/static/app.js``), and - ``classify()``, which assigns each replay an outcome judged ONLY by the backend ledger (ground truth) and whether the replay halted - never by the diff --git a/packages/flow/benchmark/lending_fault_model/resolution_ladder.py b/packages/flow/benchmark/lending_fault_model/resolution_ladder.py index 2a998611f7..fc4b120d08 100644 --- a/packages/flow/benchmark/lending_fault_model/resolution_ladder.py +++ b/packages/flow/benchmark/lending_fault_model/resolution_ladder.py @@ -38,12 +38,12 @@ import requests +from benchmark.harness.mockloan.demo import record_disbursement_demo +from benchmark.harness.mockloan.fault_server import serve as fault_serve from benchmark.lending_fault_model import faults as F from openadapt_flow.backends.playwright_backend import PlaywrightBackend from openadapt_flow.compiler import compile_recording -from openadapt_flow.demo_driver import record_disbursement_demo from openadapt_flow.ir import Workflow -from openadapt_flow.mockloan.fault_server import serve as fault_serve from openadapt_flow.runtime import Replayer HERE = Path(__file__).resolve().parent diff --git a/packages/flow/benchmark/lending_fault_model/run.py b/packages/flow/benchmark/lending_fault_model/run.py index 94fd661270..6603f5afc8 100644 --- a/packages/flow/benchmark/lending_fault_model/run.py +++ b/packages/flow/benchmark/lending_fault_model/run.py @@ -44,12 +44,12 @@ import requests +from benchmark.harness.mockloan.demo import record_disbursement_demo +from benchmark.harness.mockloan.fault_server import serve as fault_serve from benchmark.lending_fault_model import faults as F from openadapt_flow.backends.playwright_backend import PlaywrightBackend from openadapt_flow.compiler import compile_recording -from openadapt_flow.demo_driver import record_disbursement_demo from openadapt_flow.ir import RunReport, Workflow -from openadapt_flow.mockloan.fault_server import serve as fault_serve from openadapt_flow.runtime import Replayer HERE = Path(__file__).resolve().parent @@ -261,7 +261,7 @@ def render_markdown(results: list[ClassResult], meta: dict) -> str: "The bundled MockLoan app is a client-side SPA with no backend, so an " '"authorize" only mutates in-page state - the UI *is* the source of ' "truth. This study adds a real persistence boundary " - "(`openadapt_flow/mockloan/fault_server.py`): a flag-gated `?fault=` " + "(`benchmark/harness/mockloan/fault_server.py`): a flag-gated `?fault=` " "hook in the app (mirroring the `?drift=` hooks) routes the Authorize " "write through a backend API with an isolated SQLite ledger. **With no " "`?fault` query the app never calls the API and the normal benchmark " diff --git a/packages/flow/benchmark/lending_fault_model/swer.py b/packages/flow/benchmark/lending_fault_model/swer.py index 312cc57e4a..716bce000f 100644 --- a/packages/flow/benchmark/lending_fault_model/swer.py +++ b/packages/flow/benchmark/lending_fault_model/swer.py @@ -83,7 +83,7 @@ score_episode, summarize, ) -from openadapt_flow.mockloan.fault_server import LedgerDB, serve +from benchmark.harness.mockloan.fault_server import LedgerDB, serve from .ground_truth import SQLiteGroundTruthVerifier diff --git a/packages/flow/benchmark/openemr_e2e/harness.py b/packages/flow/benchmark/openemr_e2e/harness.py index 5110b31c77..114e17d82d 100644 --- a/packages/flow/benchmark/openemr_e2e/harness.py +++ b/packages/flow/benchmark/openemr_e2e/harness.py @@ -57,7 +57,7 @@ promoted_workflow, resolution_demonstration, ) -from openadapt_flow.learning.synth_stream import StructuralDiffInducer +from openadapt_flow.learning.induce import StructuralDiffInducer from openadapt_flow.mockmed.fault_server import serve as fault_serve from openadapt_flow.runtime.effects import RestRecordVerifier from openadapt_flow.runtime.replayer import Replayer diff --git a/packages/flow/benchmark/undo_proposal/run.py b/packages/flow/benchmark/undo_proposal/run.py index 2dd6127ddd..169941bafe 100644 --- a/packages/flow/benchmark/undo_proposal/run.py +++ b/packages/flow/benchmark/undo_proposal/run.py @@ -55,8 +55,9 @@ class Fixture: def _fixtures() -> dict[str, Fixture]: + from benchmark.harness.mockloan.demo import record_disbursement_demo + from benchmark.harness.mockloan.fault_server import serve as serve_loan from openadapt_flow import demo_driver - from openadapt_flow.mockloan.fault_server import serve as serve_loan from openadapt_flow.mockmed.fault_server import serve as serve_med return { @@ -74,7 +75,7 @@ def _fixtures() -> dict[str, Fixture]: "mockloan": Fixture( name="mockloan", serve=serve_loan, - record=demo_driver.record_disbursement_demo, + record=record_disbursement_demo, record_kwargs={"memo_text": "Synthetic funding memo"}, query="?fault=ok", match_fields=("loan_id", "product"), @@ -86,7 +87,7 @@ def _fixtures() -> dict[str, Fixture]: "mockloan-fee-row": Fixture( name="mockloan-fee-row", serve=serve_loan, - record=demo_driver.record_disbursement_demo, + record=record_disbursement_demo, record_kwargs={"memo_text": "Synthetic funding memo"}, query="?fault=collateral", match_fields=("loan_id", "product"), @@ -185,7 +186,7 @@ def _bind_save_effect( def _contract(directory: Path, base_url: str, fixture: Fixture, body: dict) -> Path: from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey - from openadapt_flow.reward.seed import write_bundle + from openadapt_flow.reward.bundle import write_bundle match: dict[str, Any] = {k: str(body[k]) for k in fixture.match_fields} match.update(dict(fixture.extra_match)) diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md index f8f4b51798..ece04e3645 100644 --- a/packages/flow/docs/GENERALITY_AUDIT.md +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -72,14 +72,14 @@ motivating application. | --- | --- | --- | --- | | `benchmark/` (27 Python files: `run_benchmark`, `openemr_benchmark`, `openemr_local`, `frappe_lending`, `hybrid_benchmark`, `dom_arm`, `silent_wrong_action`, `desktop_benchmark`, `verify`, `agent_baseline`, `reliability`, `chart_fonts`, `effectbench/`) | Benchmark harnesses shipped in the wheel | Moved to `packages/flow/benchmark/harness/` (import `benchmark.harness.*`). `openadapt-flow benchmark` loads it from a source checkout and says so from a wheel | done | | `instrument/` | Competitor-drift harness; imported the benchmark at module load, so it was already broken in a wheel | Moved to `packages/flow/benchmark/harness/instrument/` | done | -| `integrations/harness2/` | Harness2 research harness on MockMed | Move to `packages/flow/benchmark/harness/harness2/` | planned | -| `mockloan/`, `demo_driver.record_disbursement_demo` | Lending demo app no product command uses | Move to the benchmark tree | planned | -| `guard_demo.py`, `agent/ledger_demo.py` | `python -m` demos, no CLI command | Move to the benchmark tree | planned | -| `mockmed/teach_scope.py` | Used only by tests | Move to `tests/fixtures` | planned | -| `learning/synth_stream.py:1-245` | MockMed note stream | Split out the generic `StructuralDiffInducer`, then move the stream | planned | -| `reward/seed.py:1-236` | MockMed reward seed | Split out the generic bundle helpers (`:238-410`), then move the seed behind the demo flag | planned | +| `integrations/harness2/` | Harness2 research harness on MockMed | Moved to `packages/flow/benchmark/harness/harness2/`; the `harness2` extra is gone (install `[browser]`) | done | +| `mockloan/`, `demo_driver.record_disbursement_demo` | Lending demo app no product command uses | Moved to `packages/flow/benchmark/harness/mockloan/` (recorder in `mockloan/demo.py`) | done | +| `guard_demo.py`, `agent/ledger_demo.py` | `python -m` demos, no CLI command | Moved to `packages/flow/benchmark/harness/guard_demo.py` and `packages/agent/tests/ledger_demo.py` | done | +| `mockmed/teach_scope.py` | Used only by tests | Moved to `packages/flow/tests/_teach_scope.py` | done | +| `learning/synth_stream.py:1-245` | MockMed note stream | `StructuralDiffInducer` moved to `openadapt_flow/learning/induce.py`; the stream moved to `packages/flow/benchmark/harness/synth_stream.py` | done | +| `reward/seed.py:1-236` | MockMed reward seed | Bundle helpers split out to `openadapt_flow/reward/bundle.py`. The seed stays: `serve-reward --seed-mockmed` needs it from a wheel | done | | `record_check_proof.py:60-124` | `MOCKMED` fixture and `if name == "mockmed"` | Fixtures registered by name from the demo app's own module | planned | -| `validation/dense_surface.py` | Study harness rendering a MockMed table | Move to `benchmark/dense_surface` | planned | +| `validation/dense_surface.py` | Study harness rendering a MockMed table | Move to the benchmark tree together with the identity studies that import it (`identity_ladder`, `identity_looks_dense`, `pixel_identity_probe`, `known_ids_dense`) | planned | | `__main__.py:5246-5275` (`bench`) | Always replays against MockMed | Accept `--url`, or load from a checkout like `benchmark` | planned | | `execute/registry.py`, `execute/service.py`, `execute/app.py`, `execute/dispatch.py:60-108`, `--seed-mockmed` | Demo seeding inside the Execute service | Generic `seed_admissions`; the CLI passes the demo seed | planned | | `eligibility/payer_routes.yaml`, `fleet/simulator.py`, `repair/dialog_fixture.py` | Synthetic test data and simulators in the package | Move to the benchmark tree or `tests/fixtures` | planned | diff --git a/packages/flow/docs/GUARDED_SAVE.md b/packages/flow/docs/GUARDED_SAVE.md index 9ede4e7e2e..7b6a56b104 100644 --- a/packages/flow/docs/GUARDED_SAVE.md +++ b/packages/flow/docs/GUARDED_SAVE.md @@ -208,8 +208,10 @@ second commit. ## Try it +From `packages/flow` in a source checkout: + ```bash -python -m openadapt_flow.guard_demo +python -m benchmark.harness.guard_demo ``` The demo starts MockMed on localhost. A fake agent signs in to the chart diff --git a/packages/flow/docs/GUARD_AN_AGENT.md b/packages/flow/docs/GUARD_AN_AGENT.md index b46adf5752..77b78b0d7d 100644 --- a/packages/flow/docs/GUARD_AN_AGENT.md +++ b/packages/flow/docs/GUARD_AN_AGENT.md @@ -44,7 +44,7 @@ Nothing leaves your computer, and the guard makes no model calls. - {method: POST, path: /chart/signin} approval: not_gated # each_item: a person approves each save allow_origins: [] # optional: other origins the tab may reach - write_adapter: openadapt_flow.mockmed.teach_scope:EncounterWrites # optional + write_adapter: myapp.writes:EncounterWrites # optional: module:Class on your path allow_text: [Triage] # optional: values a save may carry besides the task's teach: # optional: make run 2 a program dir: .openadapt-teach diff --git a/packages/flow/docs/HARNESS2.md b/packages/flow/docs/HARNESS2.md index 73e37febf4..526dbb5cfe 100644 --- a/packages/flow/docs/HARNESS2.md +++ b/packages/flow/docs/HARNESS2.md @@ -9,7 +9,7 @@ is another model's judgement. GUI tasks on a system of record do have a scorer. After the agent stops, the store either holds the requested record on the requested patient, exactly -once, or it does not. `openadapt_flow.integrations.harness2` connects the two: +once, or it does not. `benchmark.harness.harness2` connects the two: Harness² supplies the search, and OpenAdapt supplies the task environment, the score, and the safety envelope. @@ -100,7 +100,7 @@ read-back of the chosen run. Meta-Harness ([arXiv:2603.28052](https://arxiv.org/abs/2603.28052)) gives its proposer a filesystem with every earlier candidate's harness, full execution traces and scores. In its ablation, full traces beat summaries, -56.7 to 38.7. `openadapt_flow.integrations.harness2.traces` writes that +56.7 to 38.7. `benchmark.harness.harness2.traces` writes that filesystem for GUI work. `run_task` writes one for every task under `/traces`. @@ -143,12 +143,14 @@ rollout. The raw data is committed beside it. ## Install -Harness² needs Python 3.12 and is not on PyPI. Install it from its -repository at the commit this integration is written against, beside the -extra: +Harness² needs Python 3.12 and is not on PyPI. The integration lives in the +source checkout, in `packages/flow/benchmark/harness/harness2/`, and is not in +the `openadapt-flow` wheel. From `packages/flow`, install Flow with the browser +extra, then Harness² from its repository at the commit this integration is +written against: ```bash -pip install 'openadapt-flow[harness2]' +pip install -e '.[browser]' ``` ```bash @@ -170,11 +172,11 @@ commit is pinned. from pathlib import Path from harness2.core.spec import ChannelSpec -from openadapt_flow.integrations.harness2.adapter import OpenAdaptGuiAdapter -from openadapt_flow.integrations.harness2.agent import ScriptedGuiAgent -from openadapt_flow.integrations.harness2.proposers import ScriptedProposer -from openadapt_flow.integrations.harness2.selection import run_task -from openadapt_flow.integrations.harness2.tasks import mockmed_tasks +from benchmark.harness.harness2.adapter import OpenAdaptGuiAdapter +from benchmark.harness.harness2.agent import ScriptedGuiAgent +from benchmark.harness.harness2.proposers import ScriptedProposer +from benchmark.harness.harness2.selection import run_task +from benchmark.harness.harness2.tasks import mockmed_tasks adapter = OpenAdaptGuiAdapter( mockmed_tasks(("lookalike",)), ScriptedGuiAgent(), Path("runs"), evidence="readback" diff --git a/packages/flow/docs/JIT.md b/packages/flow/docs/JIT.md index 6cc6119127..070cb7fc3b 100644 --- a/packages/flow/docs/JIT.md +++ b/packages/flow/docs/JIT.md @@ -307,8 +307,12 @@ it. Pass `teach_scope=` to `compile_agent` to fix what the recording run may do before the agent reads a page: the app origins, the record, the fields and exact values the save may write, and any other text it may type. +The example uses MockMed's write adapter, a test fixture in the source +checkout (`packages/flow/tests/_teach_scope.py`); write the same few lines for +your app. + ```python -from openadapt_flow.mockmed.teach_scope import encounter_scope +from tests._teach_scope import encounter_scope jit = compile_agent( agent, diff --git a/packages/flow/openadapt_flow/demo_driver.py b/packages/flow/openadapt_flow/demo_driver.py index 9ee4d3586f..5ca0e8be37 100644 --- a/packages/flow/openadapt_flow/demo_driver.py +++ b/packages/flow/openadapt_flow/demo_driver.py @@ -1,14 +1,13 @@ -"""Demo driver: records the canonical MockMed and MockLoan demonstrations. +"""Demo driver: records the canonical MockMed demonstration. Record time is allowed to cheat with Playwright locators (to find pixel coordinates via ``bounding_box()``), but every action is performed through `Recorder` so frames and events are captured exactly as a human demo would be. Replay never uses selectors. -Two self-contained demonstration targets are supported: the healthcare -MockMed triage-save (``record_triage_demo``) and the non-healthcare MockLoan -loan-disbursement (``record_disbursement_demo``). Both drive the identical -governed record -> compile -> replay path. +``record_triage_demo`` records the MockMed triage save that ``openadapt-flow +demo-record`` uses. The MockLoan disbursement recorder is a benchmark fixture +in the source checkout (``benchmark/harness/mockloan/demo.py``). """ from __future__ import annotations @@ -92,80 +91,3 @@ def center(selector: str) -> tuple[int, int]: return recorder.finish() finally: close() - - -def record_disbursement_demo( - url: str, - out_dir: Path | str, - *, - memo_text: str, - param_name: str = "memo", - headed: bool = False, - record_video_dir: Path | str | None = None, -) -> Path: - """Record the canonical MockLoan disbursement demo against a running app. - - The non-healthcare companion to :func:`record_triage_demo`. Flow: - login -> pipeline -> Open first loan -> New Disbursement -> Personal product - -> click Funding memo field -> type memo (as a parameter) -> Authorize - Disbursement. The consequential final step authorizes moving money to a - borrower, the lending analog of clinically saving an encounter. - - Args: - url: MockLoan base URL (from `openadapt_flow.mockloan.server.serve`). - out_dir: Directory to write the recording into. - memo_text: The funding memo typed during the demo (recorded as a param). - param_name: Parameter name for the typed memo (default ``"memo"``). - headed: Run the browser headed (visible) instead of headless. - record_video_dir: OPT-IN (default ``None`` = off). When set, a WebM - video of the recording session is captured into this directory; it - does not change what is recorded to ``out_dir``. - - Returns: - The recording directory (contains meta.json, events.jsonl, frames/). - """ - backend, close = PlaywrightBackend.launch( - url, - headless=not headed, - record_video_dir=( - str(record_video_dir) if record_video_dir is not None else None - ), - ) - try: - page = backend.page - recorder = Recorder(backend, out_dir, app_url=url) - - def center(selector: str) -> tuple[int, int]: - """Center pixel coordinates of the first element matching CSS.""" - locator = page.locator(selector).first - locator.wait_for(state="visible") - box = locator.bounding_box() - if box is None: # pragma: no cover - visible => box exists - raise RuntimeError(f"no bounding box for {selector!r}") - return ( - int(box["x"] + box["width"] / 2), - int(box["y"] + box["height"] / 2), - ) - - # Login - recorder.click(*center("#username")) - recorder.type_text("officer.demo") - recorder.click(*center("#password")) - recorder.type_text("mockloan-demo-pass") - recorder.click(*center("#signin")) - - # Pipeline -> open first loan - recorder.click(*center(".open-btn")) - - # Loan -> new disbursement - recorder.click(*center("#new-disbursement")) - - # Disbursement: product chooser, memo (parameterized), authorize - recorder.click(*center("#product-personal")) - recorder.click(*center("#memo")) - recorder.type_text(memo_text, param=param_name) - recorder.click(*center("#authorize")) - - return recorder.finish() - finally: - close() diff --git a/packages/flow/openadapt_flow/learning/__init__.py b/packages/flow/openadapt_flow/learning/__init__.py index 963729c2c8..610a3b0bf2 100644 --- a/packages/flow/openadapt_flow/learning/__init__.py +++ b/packages/flow/openadapt_flow/learning/__init__.py @@ -17,8 +17,9 @@ Multi-trace INDUCTION (turning traces into a revised graph) is a sibling PR, depended on only through the :class:`~openadapt_flow.learning.loop.Inducer` -Protocol; :mod:`openadapt_flow.learning.synth_stream` provides a synthetic -drift-stream harness and a deterministic reference inducer to exercise the loop. +Protocol; :mod:`openadapt_flow.learning.induce` provides a deterministic +reference inducer. The synthetic drift stream that exercises the loop is a +fixture in the source checkout (``benchmark/harness/synth_stream.py``). No model calls anywhere on the runtime path. """ diff --git a/packages/flow/openadapt_flow/learning/branch_teach.py b/packages/flow/openadapt_flow/learning/branch_teach.py index 7301b8abeb..811ac469b5 100644 --- a/packages/flow/openadapt_flow/learning/branch_teach.py +++ b/packages/flow/openadapt_flow/learning/branch_teach.py @@ -70,10 +70,10 @@ execution_trace_from_halt, promoted_workflow, ) +from openadapt_flow.learning.induce import StructuralDiffInducer from openadapt_flow.learning.interpreter import program_reproduces from openadapt_flow.learning.library import SkillLibrary from openadapt_flow.learning.loop import LearnOutcome, RegressionGate, learn_from_traces -from openadapt_flow.learning.synth_stream import StructuralDiffInducer from openadapt_flow.learning.teach import ( CorrectionSpec, TeachError, diff --git a/packages/flow/openadapt_flow/learning/halt_loop.py b/packages/flow/openadapt_flow/learning/halt_loop.py index 6599706958..fd17b308b9 100644 --- a/packages/flow/openadapt_flow/learning/halt_loop.py +++ b/packages/flow/openadapt_flow/learning/halt_loop.py @@ -5,7 +5,7 @@ The continuous-learning scaffold (:mod:`openadapt_flow.learning.loop`, :mod:`~openadapt_flow.learning.gate`, :mod:`~openadapt_flow.learning.library`) and the multi-trace inducer (:mod:`openadapt_flow.compiler.induction`, -:mod:`openadapt_flow.learning.synth_stream`) already EXIST but no real run feeds +:mod:`openadapt_flow.learning.induce`) already EXIST but no real run feeds them. This module is the thin BRIDGE that makes the loop real: 1. **A halt emits a learnable trace.** ``Replayer.run`` now populates diff --git a/packages/flow/openadapt_flow/learning/synth_stream.py b/packages/flow/openadapt_flow/learning/induce.py similarity index 64% rename from packages/flow/openadapt_flow/learning/synth_stream.py rename to packages/flow/openadapt_flow/learning/induce.py index 323194500c..67f6f1b58c 100644 --- a/packages/flow/openadapt_flow/learning/synth_stream.py +++ b/packages/flow/openadapt_flow/learning/induce.py @@ -1,26 +1,11 @@ -"""Synthetic trace-stream harness: a MockMed skill drifting over "time". - -To exercise the learn/promote loop end-to-end without a live app, this module - -1. builds a base MockMed skill program (``add-patient-note`` from - ``docs/design/WORKFLOW_PROGRAM_IR.md`` section 2.5: sign in -> open the - patient record -> new encounter -> type note -> SAVE), and -2. emits a STREAM of :class:`~openadapt_flow.learning.trace.ExecutionTrace`s over - a logical time axis with INJECTED DRIFT at a chosen trace index -- e.g. a new - consent dialog starts appearing mid-stream, so the active program stops - reproducing the new successful traces and the loop must detect the novelty, - induce a revision, validate it, and promote it. - -It also provides :class:`StructuralDiffInducer`, a deterministic, model-free -REFERENCE inducer used by the harness/tests. It is a stand-in for the real -multi-trace-induction sibling PR (wired in at merge via the -:class:`~openadapt_flow.learning.loop.Inducer` Protocol): it induces exactly the -"a new optional step appeared" revision -- detect an intent present in some -successful traces but absent from the base program, and splice it in as a -GUARDED optional step (a Phase-1 ``skip`` guard keyed on the screen fact that -co-occurs with it). It does NOT invent identity/effect/risk -- a dialog -dismissal is reversible and unarmed -- so the regression gate has nothing to -refuse on the happy path, exactly as intended. +"""A deterministic, model-free inducer: splice a newly observed step in. + +:class:`StructuralDiffInducer` diffs successful traces against a base program +and splices in the one step they show that the program lacks, guarded on the +screen fact that co-occurs with it. ``openadapt-flow teach`` and the branch +teach use it; it plugs into the learn/promote loop through the +:class:`~openadapt_flow.learning.loop.Inducer` Protocol. It does not invent +identity, effects or risk, so the regression gate still decides promotion. All deterministic, ``$0``, no model calls. """ @@ -28,7 +13,6 @@ from __future__ import annotations from collections.abc import Mapping -from enum import Enum from typing import Optional from openadapt_flow.ir import ( @@ -44,34 +28,20 @@ Step, Transition, ) -from openadapt_flow.learning.trace import ExecutionTrace, TraceStep -from openadapt_flow.runtime.effects.effect import Effect, EffectKind - -# -- the MockMed skill's canonical intents ------------------------------------ - -INTENT_LOGIN = "Sign in" -INTENT_OPEN = "Open patient record" -INTENT_ENCOUNTER = "Start new encounter" -INTENT_NOTE = "Enter note" -INTENT_SAVE = "Save encounter" -INTENT_CONSENT = "Acknowledge consent notice" - -#: The screen fact that co-occurs with the injected consent dialog -- the text a -#: ``TEXT_PRESENT`` guard keys on. -FACT_CONSENT = "Consent Required" +from openadapt_flow.learning.trace import ExecutionTrace +from openadapt_flow.runtime.effects.effect import Effect -#: A representative identity band for the patient row (name + DOB) -- the -#: evidence the pre-click identity gate keys on; makes the "open patient" step -#: identity-ARMED so an identity regression is meaningful. -PATIENT_BAND = "Belford, Phil DOB 1990-01-01" +__all__ = ["StructuralDiffInducer"] -def _anchor(*, armed: bool = False, cp: tuple[int, int] = (100, 100)) -> Anchor: +def _anchor( + *, context_text: Optional[str] = None, cp: tuple[int, int] = (100, 100) +) -> Anchor: return Anchor( template="templates/x.png", region=(cp[0] - 10, cp[1] - 10, 20, 20), click_point=cp, - context_text=PATIENT_BAND if armed else None, + context_text=context_text, ) @@ -86,14 +56,17 @@ def _action_state( effects: Optional[list[Effect]] = None, guard: Optional[Guard] = None, text: Optional[str] = None, + context_text: Optional[str] = None, ) -> State: + """An action state with a placeholder anchor; ``context_text`` is the + identity band an armed step's anchor carries.""" step = Step( id=sid, intent=intent, action=action, anchor=None if action in (ActionKind.KEY, ActionKind.WAIT) - else _anchor(armed=armed), + else _anchor(context_text=context_text if armed else None), text=text, risk=risk, # type: ignore[arg-type] effects=effects or [], @@ -108,140 +81,6 @@ def _action_state( ) -def mockmed_base_program() -> ProgramGraph: - """The base ``add-patient-note`` program (no consent dialog). - - A straight-line state machine: sign in -> open patient (identity-armed) -> - new encounter -> type note -> SAVE (irreversible, with a ``record_written`` - system-of-record effect) -> success. The identity band + effect + risk on - the sensitive steps are what a rigged revision could weaken, and what the - regression gate protects. - """ - save_effect = Effect( - kind=EffectKind.RECORD_WRITTEN, - match={"type": "encounter", "patient": "p1"}, - expected_count=1, - risk="irreversible", - probe="encounter record written for patient", - ) - states = { - "s_login": _action_state("s_login", INTENT_LOGIN, "s_open"), - "s_open": _action_state("s_open", INTENT_OPEN, "s_encounter", armed=True), - "s_encounter": _action_state("s_encounter", INTENT_ENCOUNTER, "s_note"), - "s_note": _action_state( - "s_note", INTENT_NOTE, "s_save", action=ActionKind.TYPE, text="Follow-up" - ), - "s_save": _action_state( - "s_save", - INTENT_SAVE, - "__end__", - risk="irreversible", - effects=[save_effect], - ), - "__end__": State(id="__end__", kind=StateKind.TERMINAL, outcome="success"), - } - return ProgramGraph(entry="s_login", states=states) - - -# -- the drift stream --------------------------------------------------------- - - -class Drift(str, Enum): - """The injected drift/variant a stream can carry from trace K onward.""" - - NONE = "none" - #: A new consent dialog appears after sign-in (a new OPTIONAL step + branch). - CONSENT_DIALOG = "consent_dialog" - #: The note field is renamed (the observed intent changes) -- a structural - #: change the additive reference inducer CANNOT cover (used to show the loop - #: correctly refuses an inadequate revision). - FIELD_RENAME = "field_rename" - - -def _baseline_steps() -> list[TraceStep]: - return [ - TraceStep(intent=INTENT_LOGIN), - TraceStep(intent=INTENT_OPEN, identity=PATIENT_BAND), - TraceStep(intent=INTENT_ENCOUNTER), - TraceStep(intent=INTENT_NOTE, action=ActionKind.TYPE), - TraceStep(intent=INTENT_SAVE), - ] - - -def _drifted_steps(drift: Drift) -> tuple[list[TraceStep], dict[str, bool]]: - steps = _baseline_steps() - facts: dict[str, bool] = {} - if drift is Drift.CONSENT_DIALOG: - # The consent dialog appears right after sign-in. - steps.insert(1, TraceStep(intent=INTENT_CONSENT)) - facts[FACT_CONSENT] = True - elif drift is Drift.FIELD_RENAME: - for s in steps: - if s.intent == INTENT_NOTE: - s.intent = "Enter clinical note" - return steps, facts - - -def generate_stream( - *, - n_baseline: int = 6, - n_drift: int = 6, - drift: Drift = Drift.CONSENT_DIALOG, - n_failures: int = 0, - prefix: str = "mockmed", -) -> list[ExecutionTrace]: - """Emit a deterministic stream of traces with drift injected at trace - ``n_baseline``. - - The first ``n_baseline`` traces are clean successful runs of the base skill; - from index ``n_baseline`` the injected ``drift`` appears in every successful - run. ``n_failures`` failure traces (noise) are appended -- runs that halted - partway with no novel successful structure, so a batch of them alone must NOT - trigger a revision. - """ - traces: list[ExecutionTrace] = [] - t = 0 - for _ in range(n_baseline): - traces.append( - ExecutionTrace( - trace_id=f"{prefix}-{t:04d}", - t=t, - outcome="success", - steps=_baseline_steps(), - params={"encounter_type": "Triage"}, - ) - ) - t += 1 - for _ in range(n_drift): - steps, facts = _drifted_steps(drift) - traces.append( - ExecutionTrace( - trace_id=f"{prefix}-{t:04d}", - t=t, - outcome="success", - steps=steps, - facts=facts, - params={"encounter_type": "Triage"}, - ) - ) - t += 1 - for _ in range(n_failures): - # A run that halted after opening the patient (e.g. a transient - # resolution failure): truncated, no novel successful structure. - traces.append( - ExecutionTrace( - trace_id=f"{prefix}-{t:04d}", - t=t, - outcome="failure", - steps=_baseline_steps()[:2], - failure_reason="transient resolution failure on 'Start new encounter'", - params={"encounter_type": "Triage"}, - ) - ) - t += 1 - return traces - - # -- the reference (stand-in) inducer ----------------------------------------- #: Postconditions about what the screen shows, which stay meaningful when diff --git a/packages/flow/openadapt_flow/learning/teach.py b/packages/flow/openadapt_flow/learning/teach.py index fdca14a554..7f11d6eecc 100644 --- a/packages/flow/openadapt_flow/learning/teach.py +++ b/packages/flow/openadapt_flow/learning/teach.py @@ -31,7 +31,7 @@ Deterministic and ``$0`` on the shipped path: the resolution is induced by the model-free reference inducer -(:class:`~openadapt_flow.learning.synth_stream.StructuralDiffInducer`), which +(:class:`~openadapt_flow.learning.induce.StructuralDiffInducer`), which handles the "an unexpected dialog intercepted the workflow" resolution class the halt->learn loop was built for. Three things make that work on a real bundle, not only on the synthetic stream: @@ -86,6 +86,7 @@ promoted_workflow, resolution_demonstration, ) +from openadapt_flow.learning.induce import StructuralDiffInducer from openadapt_flow.learning.library import SkillLibrary from openadapt_flow.learning.loop import ( CanaryContext, @@ -93,7 +94,6 @@ LearnOutcome, ProgramCanaryFn, ) -from openadapt_flow.learning.synth_stream import StructuralDiffInducer from openadapt_flow.learning.trace import ExecutionTrace, TraceStep from openadapt_flow.runtime.authorization import runtime_params_for_gui diff --git a/packages/flow/openadapt_flow/reward/bundle.py b/packages/flow/openadapt_flow/reward/bundle.py new file mode 100644 index 0000000000..1f79155b28 --- /dev/null +++ b/packages/flow/openadapt_flow/reward/bundle.py @@ -0,0 +1,254 @@ +"""Write, calibrate and self-sign a synthetic reward bundle. + +A reward bundle is the directory ``openadapt_flow.reward.models.RewardBundle`` +loads: the contract, its required and forbidden effects, the oracle recipe, +and (when certified) a self-signed certificate with the calibration counts +beside it. :func:`write_bundle` writes one from the effects and oracle a +caller passes; tests, the benchmark harnesses and the ``serve-reward`` demo +seed (:mod:`openadapt_flow.reward.seed`) all build their bundles with it. + +Nothing here is a production recipe. A production-scope certificate is +calibrated on a held-out corpus by the OpenAdapt control service. +""" + +from __future__ import annotations + +import base64 +import json +from pathlib import Path +from typing import Any + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from openadapt_types.process_capability import ( + _digest_payload, + canonical_json_bytes, +) +from openadapt_types.reward import RewardCertificateV1, RewardContractV1 + +from openadapt_flow.reward.calibration import ( + CalibrationResult, + confidence_delta, + corpus_digest_for, + corpus_from_effects, + extradup_trials, +) +from openadapt_flow.reward.models import ( + CERTIFICATE_FILE, + CONTRACT_FILE, + FORBIDDEN_EFFECTS_FILE, + ORACLE_FILE, + REQUIRED_EFFECTS_FILE, + RewardBundle, +) +from openadapt_flow.runtime.effects.effect import Effect + +#: The task and environment a synthetic bundle's contract names. The values +#: are the ones the ``serve-reward`` demo seed has always used, so a bundle's +#: digests do not change with the module it was written from. +SYNTHETIC_TASK_ID = "task_mockmed_encounter_note" +SYNTHETIC_ENVIRONMENT_ID = "environment_mockmed_synthetic" +CERTIFICATE_EXPIRY_UPDATES = 1000 +CALIBRATION_FILE = "calibration.json" +#: Trials run before the synthetic certificate is signed. 300 trials with +#: zero false accepts bound the rate at 0.0099 (95%). +CALIBRATION_TRIALS = 300 +CALIBRATION_SEED = 20260901 +CALIBRATION_CONFIDENCE = 0.95 + +#: Effects a bundle requires and forbids when the caller passes none. +DEFAULT_REQUIRED_EFFECTS: list[dict[str, Any]] = [ + { + "kind": "record_written", + "match": {"patient_id": {"param": "patient_id"}, "type": "Triage"}, + "expected_count": 1, + # The claim is about what this episode added, not about what the + # store happens to hold. Without it a rollout that did nothing + # scores the full reward whenever the subject already had a row. + "count_new_only": True, + } +] + +DEFAULT_FORBIDDEN_EFFECTS: list[dict[str, Any]] = [ + { + "kind": "record_written", + "match": {"patient_id": {"param": "patient_id"}, "type": "Discharge"}, + "expected_count": 1, + } +] + + +def write_bundle( + directory: Path, + *, + contract_id: str, + oracle: dict[str, Any], + channel: str, + key: Ed25519PrivateKey, + issuer_key_id: str, + certify: bool, + required_effects: list[dict[str, Any]] | None = None, + forbidden_effects: list[dict[str, Any]] | None = None, + identity_keys: list[str] | None = None, +) -> None: + """Write one synthetic bundle. Tests pass their own effects and keys.""" + + directory.mkdir(parents=True, exist_ok=True) + required = _canonical( + DEFAULT_REQUIRED_EFFECTS if required_effects is None else required_effects + ) + forbidden = _canonical( + DEFAULT_FORBIDDEN_EFFECTS if forbidden_effects is None else forbidden_effects + ) + oracle_doc = _canonical(oracle) + keys = ["patient_id"] if identity_keys is None else list(identity_keys) + # The corpus a certificate is calibrated on comes from the contract's own + # effects, so its digest is computed here and not chosen. + corpus_digest = corpus_digest_for( + [Effect.model_validate(item) for item in required], + [Effect.model_validate(item) for item in forbidden], + keys, + ) + contract = RewardContractV1.model_validate( + { + "contract_id": contract_id, + "contract_version": "version_0001", + "task_id": SYNTHETIC_TASK_ID, + "task_digest": _digest_payload({"task": SYNTHETIC_TASK_ID}), + "environment_id": SYNTHETIC_ENVIRONMENT_ID, + "environment_digest": _digest_payload( + {"environment": SYNTHETIC_ENVIRONMENT_ID} + ), + "required_effect_contract_digest": _digest_payload(required), + "forbidden_effect_contract_digest": _digest_payload(forbidden), + "oracle": { + "channel": channel, + "identity_keys": keys, + "oracle_contract_digest": _digest_payload(oracle_doc), + }, + "components": [{"name": "terminal_effect", "weight": 1.0}], + # The synthetic banner lie yields 0, not the default -1 penalty: + # this fixture demonstrates "no reward", not a tuned penalty. + "scoring": {"wrong_effect_reward": 0.0}, + "certificate_policy": { + "epsilon": 0.05, + "delta": 0.05, + "threshold": 0.5, + "calibration_corpus_digest": corpus_digest, + "expiry_policy_updates": CERTIFICATE_EXPIRY_UPDATES, + }, + } + ) + _write(directory / CONTRACT_FILE, contract.model_dump(mode="json")) + _write(directory / REQUIRED_EFFECTS_FILE, required) + _write(directory / FORBIDDEN_EFFECTS_FILE, forbidden) + _write(directory / ORACLE_FILE, oracle_doc) + if certify: + calibration = calibrate_bundle(directory) + certificate = self_signed_certificate( + contract, + key=key, + issuer_key_id=issuer_key_id, + issued_at_policy_update=0, + expiry_policy_updates=CERTIFICATE_EXPIRY_UPDATES, + epsilon=calibration.epsilon, + delta=confidence_delta(calibration.confidence), + ) + _write(directory / CERTIFICATE_FILE, certificate.model_dump(mode="json")) + _write(directory / CALIBRATION_FILE, calibration.as_metadata()) + + +def calibrate_bundle(directory: Path) -> CalibrationResult: + """Run the seeded trials through this bundle's judge. + + The corpus is read off the bundle's own required and forbidden effects, + so the faults it plants are faults in the records this contract talks + about. The certificate's ``epsilon`` is the exact one-sided + Clopper-Pearson bound from these counts, and ``calibration.json`` beside + the certificate records the counts, the corpus digest, and which fault + classes applied, so a reader can recompute the bound and see what it + covers. + """ + + from openadapt_types.oracle import OracleChannel + + from openadapt_flow.reward.oracles import observation + from openadapt_flow.reward.worker import judge_episode + from openadapt_flow.runtime.effects.effect import EffectState + + bundle = RewardBundle.load(directory) + channel = OracleChannel(bundle.contract.oracle.channel) + corpus = corpus_from_effects( + bundle.required_effects, + bundle.forbidden_effects, + bundle.identity_keys, + ) + + def checker(before: Any, current: Any, identity: dict[str, str]) -> Any: + pre = EffectState(substrate=channel.value, reachable=True, records=list(before)) + observed = observation(channel, identity, list(current)) + return judge_episode(bundle, identity, "completed", pre, observed).outcome + + return extradup_trials( + checker, + corpus, + trials=CALIBRATION_TRIALS, + generator_seed=CALIBRATION_SEED, + confidence=CALIBRATION_CONFIDENCE, + corpus_digest=bundle.contract.certificate_policy.calibration_corpus_digest, + ) + + +def self_signed_certificate( + contract: RewardContractV1, + *, + key: Ed25519PrivateKey, + issuer_key_id: str, + issued_at_policy_update: int, + expiry_policy_updates: int, + epsilon: float, + delta: float, + certificate_id: str = "reward_certificate_mockmed", +) -> RewardCertificateV1: + """A certificate signed by the local key, synthetic scope. + + ``epsilon`` comes from :func:`calibrate_bundle`, never from a constant. + A production-scope certificate is calibrated on a held-out corpus by + the OpenAdapt control service and is not published. This one exists so + the seeded run can show a certified receipt next to an uncertified one. + """ + + from openadapt_flow.reward.worker import _now + + unsigned = { + "schema_version": "openadapt.reward-certificate/v1", + "certificate_id": certificate_id, + "reward_contract_digest": contract.digest, + "checker_configuration_digest": _digest_payload({"checker": "synthetic"}), + "epsilon": epsilon, + "delta": delta, + "threshold": contract.certificate_policy.threshold, + "calibration_corpus_digest": contract.certificate_policy.calibration_corpus_digest, + "calibration_scope": "synthetic", + "issued_at_policy_update": issued_at_policy_update, + "expiry_policy_updates": expiry_policy_updates, + "issued_at": _now(), + "issuer": "self_signed", + "issuer_key_id": issuer_key_id, + } + signature = base64.b64encode(key.sign(canonical_json_bytes(unsigned))).decode( + "ascii" + ) + return RewardCertificateV1.model_validate( + {**unsigned, "signature_algorithm": "ed25519", "signature": signature} + ) + + +def _canonical(value: Any) -> Any: + return json.loads(json.dumps(value, sort_keys=True)) + + +def _write(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) diff --git a/packages/flow/openadapt_flow/reward/seed.py b/packages/flow/openadapt_flow/reward/seed.py index d0c3856835..eeec87e987 100644 --- a/packages/flow/openadapt_flow/reward/seed.py +++ b/packages/flow/openadapt_flow/reward/seed.py @@ -27,51 +27,28 @@ from __future__ import annotations -import base64 import json import sqlite3 from pathlib import Path from typing import Any from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey -from openadapt_types.process_capability import ( - _digest_payload, - canonical_json_bytes, -) -from openadapt_types.reward import RewardCertificateV1, RewardContractV1 -from openadapt_flow.reward.calibration import ( - CalibrationResult, - confidence_delta, - corpus_digest_for, - corpus_from_effects, - extradup_trials, -) -from openadapt_flow.reward.models import ( - CERTIFICATE_FILE, - CONTRACT_FILE, - FORBIDDEN_EFFECTS_FILE, - ORACLE_FILE, - REQUIRED_EFFECTS_FILE, - RewardBundle, +from openadapt_flow.reward.bundle import ( + SYNTHETIC_ENVIRONMENT_ID, + SYNTHETIC_TASK_ID, + _write, + write_bundle, ) -from openadapt_flow.runtime.effects.effect import Effect -MOCKMED_TASK_ID = "task_mockmed_encounter_note" -MOCKMED_ENVIRONMENT_ID = "environment_mockmed_synthetic" +MOCKMED_TASK_ID = SYNTHETIC_TASK_ID +MOCKMED_ENVIRONMENT_ID = SYNTHETIC_ENVIRONMENT_ID MOCKMED_CONTRACT_ID = "reward_contract_mockmed" MOCKMED_TIER0_CONTRACT_ID = "reward_contract_mockmed_tier0" MOCKMED_HONEST_PATIENT = "patient-honest-0001" MOCKMED_LIE_PATIENT = "patient-lie-0002" MOCKMED_DUPLICATE_PATIENT = "patient-dup-0003" MOCKMED_CHECKPOINT = "policy_checkpoint_mockmed_0" -CERTIFICATE_EXPIRY_UPDATES = 1000 -CALIBRATION_FILE = "calibration.json" -#: Trials the seed runs before it signs the synthetic certificate. 300 trials -#: with zero false accepts bound the rate at 0.0099 (95%). -CALIBRATION_TRIALS = 300 -CALIBRATION_SEED = 20260901 -CALIBRATION_CONFIDENCE = 0.95 #: The tier-2 store's one table and the read-only SELECT the oracle runs. MOCKMED_TABLE = "encounters" @@ -90,26 +67,6 @@ {"id": 2, "patient_id": MOCKMED_DUPLICATE_PATIENT, "type": "Triage"}, ] -_REQUIRED_EFFECTS: list[dict[str, Any]] = [ - { - "kind": "record_written", - "match": {"patient_id": {"param": "patient_id"}, "type": "Triage"}, - "expected_count": 1, - # The claim is about what this episode added, not about what the - # store happens to hold. Without it a rollout that did nothing - # scores the full reward whenever the subject already had a row. - "count_new_only": True, - } -] - -_FORBIDDEN_EFFECTS: list[dict[str, Any]] = [ - { - "kind": "record_written", - "match": {"patient_id": {"param": "patient_id"}, "type": "Discharge"}, - "expected_count": 1, - } -] - def seed_mockmed( data_dir: Path, key: Ed25519PrivateKey, issuer_key_id: str @@ -233,180 +190,3 @@ def mockmed_episode( "runtime_signal": runtime_signal, }, } - - -def write_bundle( - directory: Path, - *, - contract_id: str, - oracle: dict[str, Any], - channel: str, - key: Ed25519PrivateKey, - issuer_key_id: str, - certify: bool, - required_effects: list[dict[str, Any]] | None = None, - forbidden_effects: list[dict[str, Any]] | None = None, - identity_keys: list[str] | None = None, -) -> None: - """Write one synthetic bundle. Tests pass their own effects and keys.""" - - directory.mkdir(parents=True, exist_ok=True) - required = _canonical( - _REQUIRED_EFFECTS if required_effects is None else required_effects - ) - forbidden = _canonical( - _FORBIDDEN_EFFECTS if forbidden_effects is None else forbidden_effects - ) - oracle_doc = _canonical(oracle) - keys = ["patient_id"] if identity_keys is None else list(identity_keys) - # The corpus a certificate is calibrated on comes from the contract's own - # effects, so its digest is computed here and not chosen. - corpus_digest = corpus_digest_for( - [Effect.model_validate(item) for item in required], - [Effect.model_validate(item) for item in forbidden], - keys, - ) - contract = RewardContractV1.model_validate( - { - "contract_id": contract_id, - "contract_version": "version_0001", - "task_id": MOCKMED_TASK_ID, - "task_digest": _digest_payload({"task": MOCKMED_TASK_ID}), - "environment_id": MOCKMED_ENVIRONMENT_ID, - "environment_digest": _digest_payload( - {"environment": MOCKMED_ENVIRONMENT_ID} - ), - "required_effect_contract_digest": _digest_payload(required), - "forbidden_effect_contract_digest": _digest_payload(forbidden), - "oracle": { - "channel": channel, - "identity_keys": keys, - "oracle_contract_digest": _digest_payload(oracle_doc), - }, - "components": [{"name": "terminal_effect", "weight": 1.0}], - # The synthetic banner lie yields 0, not the default -1 penalty: - # this fixture demonstrates "no reward", not a tuned penalty. - "scoring": {"wrong_effect_reward": 0.0}, - "certificate_policy": { - "epsilon": 0.05, - "delta": 0.05, - "threshold": 0.5, - "calibration_corpus_digest": corpus_digest, - "expiry_policy_updates": CERTIFICATE_EXPIRY_UPDATES, - }, - } - ) - _write(directory / CONTRACT_FILE, contract.model_dump(mode="json")) - _write(directory / REQUIRED_EFFECTS_FILE, required) - _write(directory / FORBIDDEN_EFFECTS_FILE, forbidden) - _write(directory / ORACLE_FILE, oracle_doc) - if certify: - calibration = calibrate_bundle(directory) - certificate = self_signed_certificate( - contract, - key=key, - issuer_key_id=issuer_key_id, - issued_at_policy_update=0, - expiry_policy_updates=CERTIFICATE_EXPIRY_UPDATES, - epsilon=calibration.epsilon, - delta=confidence_delta(calibration.confidence), - ) - _write(directory / CERTIFICATE_FILE, certificate.model_dump(mode="json")) - _write(directory / CALIBRATION_FILE, calibration.as_metadata()) - - -def calibrate_bundle(directory: Path) -> CalibrationResult: - """Run the seeded trials through this bundle's judge. - - The corpus is read off the bundle's own required and forbidden effects, - so the faults it plants are faults in the records this contract talks - about. The certificate's ``epsilon`` is the exact one-sided - Clopper-Pearson bound from these counts, and ``calibration.json`` beside - the certificate records the counts, the corpus digest, and which fault - classes applied, so a reader can recompute the bound and see what it - covers. - """ - - from openadapt_types.oracle import OracleChannel - - from openadapt_flow.reward.oracles import observation - from openadapt_flow.reward.worker import judge_episode - from openadapt_flow.runtime.effects.effect import EffectState - - bundle = RewardBundle.load(directory) - channel = OracleChannel(bundle.contract.oracle.channel) - corpus = corpus_from_effects( - bundle.required_effects, - bundle.forbidden_effects, - bundle.identity_keys, - ) - - def checker(before: Any, current: Any, identity: dict[str, str]) -> Any: - pre = EffectState(substrate=channel.value, reachable=True, records=list(before)) - observed = observation(channel, identity, list(current)) - return judge_episode(bundle, identity, "completed", pre, observed).outcome - - return extradup_trials( - checker, - corpus, - trials=CALIBRATION_TRIALS, - generator_seed=CALIBRATION_SEED, - confidence=CALIBRATION_CONFIDENCE, - corpus_digest=bundle.contract.certificate_policy.calibration_corpus_digest, - ) - - -def self_signed_certificate( - contract: RewardContractV1, - *, - key: Ed25519PrivateKey, - issuer_key_id: str, - issued_at_policy_update: int, - expiry_policy_updates: int, - epsilon: float, - delta: float, - certificate_id: str = "reward_certificate_mockmed", -) -> RewardCertificateV1: - """A certificate signed by the local key, synthetic scope. - - ``epsilon`` comes from :func:`calibrate_bundle`, never from a constant. - A production-scope certificate is calibrated on a held-out corpus by - the OpenAdapt control service and is not published. This one exists so - the seeded run can show a certified receipt next to an uncertified one. - """ - - from openadapt_flow.reward.worker import _now - - unsigned = { - "schema_version": "openadapt.reward-certificate/v1", - "certificate_id": certificate_id, - "reward_contract_digest": contract.digest, - "checker_configuration_digest": _digest_payload({"checker": "synthetic"}), - "epsilon": epsilon, - "delta": delta, - "threshold": contract.certificate_policy.threshold, - "calibration_corpus_digest": contract.certificate_policy.calibration_corpus_digest, - "calibration_scope": "synthetic", - "issued_at_policy_update": issued_at_policy_update, - "expiry_policy_updates": expiry_policy_updates, - "issued_at": _now(), - "issuer": "self_signed", - "issuer_key_id": issuer_key_id, - } - signature = base64.b64encode(key.sign(canonical_json_bytes(unsigned))).decode( - "ascii" - ) - return RewardCertificateV1.model_validate( - {**unsigned, "signature_algorithm": "ed25519", "signature": signature} - ) - - -def _canonical(value: Any) -> Any: - return json.loads(json.dumps(value, sort_keys=True)) - - -def _write(path: Path, payload: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text( - json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 17b27ecdb2..49131b61ee 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -286,6 +286,9 @@ "benchmark/grounding_eval/results.json", "benchmark/grounding_eval/results_sweep.json", "benchmark/halt_cleanup/results/openemr.json", + "benchmark/harness/mockloan/static/app.js", + "benchmark/harness/mockloan/static/index.html", + "benchmark/harness/mockloan/static/styles.css", "benchmark/harness2_scorer/evidence/base/results.json", "benchmark/harness2_scorer/evidence/harness2/results.json", "benchmark/harness2_scorer/evidence/harness2_readback/results.json", @@ -542,9 +545,6 @@ "openadapt_flow/console/static/console.js", "openadapt_flow/console/static/index.html", "openadapt_flow/eligibility/payer_routes.yaml", - "openadapt_flow/mockloan/static/app.js", - "openadapt_flow/mockloan/static/index.html", - "openadapt_flow/mockloan/static/styles.css", "openadapt_flow/mockmed/static/app.js", "openadapt_flow/mockmed/static/index.html", "openadapt_flow/mockmed/static/styles.css", diff --git a/packages/flow/pyproject.toml b/packages/flow/pyproject.toml index b4bedc3057..a45b45596e 100644 --- a/packages/flow/pyproject.toml +++ b/packages/flow/pyproject.toml @@ -197,12 +197,6 @@ reward = [ "fastapi>=0.110", "uvicorn>=0.29", ] -# Harness² integration (openadapt_flow.integrations.harness2): GUI tasks on -# resettable apps, scored by reading the system of record, for -# google-research/harness2's harness search. Harness² (Apache-2.0, Python -# 3.12+) is not on PyPI, and a published wheel cannot name a git URL, so it is -# installed beside this extra; docs/HARNESS2.md gives the pinned command. -harness2 = ["playwright>=1.44"] # Native macOS window capture/input. Imported lazily; other platforms never # install or import these framework bindings. macos = [ @@ -592,7 +586,7 @@ follow_imports_for_stubs = true # - alternate backends: backends.playwright_backend, backends.rdp_backend # - benchmark & interactive tooling: benchmark.desktop_benchmark, # interactive_recorder -# - learning-loop synthesis: learning.synth_stream +# - learning-loop synthesis: learning.induce # - disambiguation heuristics: compiler.disambiguation # - runtime helpers still on untyped deps: runtime.effects._common, # runtime.grounder, runtime.healing.perturbation @@ -608,7 +602,7 @@ module = [ "benchmark.harness.desktop_benchmark", "openadapt_flow.compiler.disambiguation", "openadapt_flow.interactive_recorder", - "openadapt_flow.learning.synth_stream", + "openadapt_flow.learning.induce", "openadapt_flow.report", "openadapt_flow.runtime.effects._common", "openadapt_flow.runtime.grounder", diff --git a/packages/flow/openadapt_flow/mockmed/teach_scope.py b/packages/flow/tests/_teach_scope.py similarity index 97% rename from packages/flow/openadapt_flow/mockmed/teach_scope.py rename to packages/flow/tests/_teach_scope.py index 170bc0b60c..811cac5052 100644 --- a/packages/flow/openadapt_flow/mockmed/teach_scope.py +++ b/packages/flow/tests/_teach_scope.py @@ -1,5 +1,7 @@ """MockMed's write adapter for a teach scope (:mod:`openadapt_flow.teach_scope`). +A test fixture: it lives in the source checkout, outside the wheel. + MockMed's fault server (:mod:`openadapt_flow.mockmed.fault_server`) takes one kind of record write: ``POST /api/encounter`` with a JSON body naming the patient, the encounter type and the note. :class:`EncounterWrites` reads that diff --git a/packages/flow/tests/e2e/test_injection_live.py b/packages/flow/tests/e2e/test_injection_live.py index 59fe7e9986..d9ff0ae339 100644 --- a/packages/flow/tests/e2e/test_injection_live.py +++ b/packages/flow/tests/e2e/test_injection_live.py @@ -42,9 +42,9 @@ from openadapt_flow.jit import TaskSignature, WebApp, compile_agent from openadapt_flow.jit.adapters import CaptureComputer, UnsupportedAction from openadapt_flow.mockmed.fault_server import serve -from openadapt_flow.mockmed.teach_scope import encounter_scope from openadapt_flow.runtime.effects import RestRecordVerifier from openadapt_flow.tutorial import _http_json, _records +from tests._teach_scope import encounter_scope LIVE = os.environ.get("OPENADAPT_INJECTION_LIVE") == "1" APPROVED_BY = os.environ.get("OPENADAPT_INJECTION_LIVE_APPROVED_BY", "").strip() diff --git a/packages/flow/tests/e2e/test_lending_fault_model.py b/packages/flow/tests/e2e/test_lending_fault_model.py index 1923442d22..374f3f6bd1 100644 --- a/packages/flow/tests/e2e/test_lending_fault_model.py +++ b/packages/flow/tests/e2e/test_lending_fault_model.py @@ -28,12 +28,12 @@ import pytest import requests +from benchmark.harness.mockloan.demo import record_disbursement_demo +from benchmark.harness.mockloan.fault_server import serve as fault_serve from benchmark.lending_fault_model import faults as F from openadapt_flow.backends.playwright_backend import PlaywrightBackend from openadapt_flow.compiler import compile_recording -from openadapt_flow.demo_driver import record_disbursement_demo from openadapt_flow.ir import Workflow -from openadapt_flow.mockloan.fault_server import serve as fault_serve from openadapt_flow.runtime import Replayer from .conftest import VIEWPORT diff --git a/packages/flow/tests/test_agent_guard.py b/packages/flow/tests/test_agent_guard.py index b57057b97e..21696760e9 100644 --- a/packages/flow/tests/test_agent_guard.py +++ b/packages/flow/tests/test_agent_guard.py @@ -14,6 +14,7 @@ import pytest +from benchmark.harness.guard_demo import NOTE, FakeAgent, person_approves, verify_yaml from openadapt_flow.agent_guard.cdp import CdpGuardPage from openadapt_flow.agent_guard.commit_guard import BLOCK_STATUS, CommitGuard from openadapt_flow.agent_guard.config import ConfigError, parse_config @@ -24,7 +25,6 @@ origin_of, ) from openadapt_flow.guard import GuardedSave, GuardTask -from openadapt_flow.guard_demo import NOTE, FakeAgent, person_approves, verify_yaml from openadapt_flow.mockmed.fault_server import serve as fault_serve from openadapt_flow.verify_run import parse_verify_config @@ -155,7 +155,7 @@ def screen_text(self) -> Optional[str]: return self.agent.open_chart(self.patient) def screenshot(self) -> Optional[bytes]: - from openadapt_flow.guard_demo import tiny_png + from benchmark.harness.guard_demo import tiny_png return tiny_png() diff --git a/packages/flow/tests/test_continuous_learning.py b/packages/flow/tests/test_continuous_learning.py index 74606077e3..ef2bf78ded 100644 --- a/packages/flow/tests/test_continuous_learning.py +++ b/packages/flow/tests/test_continuous_learning.py @@ -25,6 +25,11 @@ import pytest +from benchmark.harness.synth_stream import ( + Drift, + generate_stream, + mockmed_base_program, +) from openadapt_flow.ir import ( ProgramGraph, StateKind, @@ -38,13 +43,8 @@ program_regression_gate, program_reproduces, ) +from openadapt_flow.learning.induce import StructuralDiffInducer from openadapt_flow.learning.loop import Inducer -from openadapt_flow.learning.synth_stream import ( - Drift, - StructuralDiffInducer, - generate_stream, - mockmed_base_program, -) from openadapt_flow.runtime.identity_template import build_identity_template # -- stub inducers ------------------------------------------------------------ diff --git a/packages/flow/tests/test_guarded_save.py b/packages/flow/tests/test_guarded_save.py index 3349372137..439054f52e 100644 --- a/packages/flow/tests/test_guarded_save.py +++ b/packages/flow/tests/test_guarded_save.py @@ -1,6 +1,6 @@ """Guarded save: a fake agent writes a referral to MockMed through the guard. -The agent here is test code (``openadapt_flow.guard_demo.FakeAgent``) that +The agent here is test code (``benchmark.harness.guard_demo.FakeAgent``) that opens MockMed's server-rendered chart page and posts to its encounter API, the way a computer-use agent's click on Save would. The guard judges the write from ``GET /api/db`` alone. No model calls, nothing beyond localhost. @@ -14,15 +14,7 @@ import pytest -from openadapt_flow.agent.guard_tools import GuardBridge -from openadapt_flow.guard import ( - GUARD_TASK_SCHEMA, - GuardedSave, - GuardRefused, - GuardVerdict, - check_identity, -) -from openadapt_flow.guard_demo import ( +from benchmark.harness.guard_demo import ( NOTE, FakeAgent, person_approves, @@ -31,6 +23,14 @@ tiny_png, verify_yaml, ) +from openadapt_flow.agent.guard_tools import GuardBridge +from openadapt_flow.guard import ( + GUARD_TASK_SCHEMA, + GuardedSave, + GuardRefused, + GuardVerdict, + check_identity, +) from openadapt_flow.mockmed.fault_server import serve as fault_serve from openadapt_flow.verify_run import ( evidence_problem, diff --git a/packages/flow/tests/test_halt_learn_loop.py b/packages/flow/tests/test_halt_learn_loop.py index 458c9f81aa..5175c57a0a 100644 --- a/packages/flow/tests/test_halt_learn_loop.py +++ b/packages/flow/tests/test_halt_learn_loop.py @@ -29,6 +29,10 @@ import pytest +from benchmark.harness.synth_stream import ( + INTENT_OPEN, + mockmed_base_program, +) from openadapt_flow.ir import ( ActionKind, Anchor, @@ -51,11 +55,7 @@ promoted_workflow, resolution_demonstration, ) -from openadapt_flow.learning.synth_stream import ( - INTENT_OPEN, - StructuralDiffInducer, - mockmed_base_program, -) +from openadapt_flow.learning.induce import StructuralDiffInducer from openadapt_flow.runtime.replayer import Replayer from openadapt_flow.vision.ocr import OcrLine from tests.test_replayer import FakeBackend, FakeVision, Match, make_png diff --git a/packages/flow/tests/test_harness2_adapter.py b/packages/flow/tests/test_harness2_adapter.py index 09f0b1fed4..9f5bb149dd 100644 --- a/packages/flow/tests/test_harness2_adapter.py +++ b/packages/flow/tests/test_harness2_adapter.py @@ -13,25 +13,25 @@ import pytest -from openadapt_flow.integrations.harness2.agent import ( +from benchmark.harness.harness2.agent import ( RULE_IDENTITY, RULE_SAVE_ONCE, RULE_TYPE_LABEL, ScriptedGuiAgent, ) -from openadapt_flow.integrations.harness2.apps import MockMedTarget -from openadapt_flow.integrations.harness2.llm import ( +from benchmark.harness.harness2.apps import MockMedTarget +from benchmark.harness.harness2.llm import ( BudgetExceeded, Completion, CostLedger, ) -from openadapt_flow.integrations.harness2.rollout import ( +from benchmark.harness.harness2.rollout import ( load_run, load_score, run_rollout, ) -from openadapt_flow.integrations.harness2.scorer import ReadBackScorer -from openadapt_flow.integrations.harness2.tasks import mockmed_tasks +from benchmark.harness.harness2.scorer import ReadBackScorer +from benchmark.harness.harness2.tasks import mockmed_tasks CAREFUL = ( f"Check the {RULE_IDENTITY}. Choose the type {RULE_TYPE_LABEL}. {RULE_SAVE_ONCE}." @@ -192,7 +192,7 @@ def test_a_rollout_is_redone_when_the_harness_changed(tmp_path): def test_reward_selection_ranks_the_right_outcome_first(tmp_path, h2): from harness2.core.spec import RunView - from openadapt_flow.integrations.harness2.selection import reward_selection + from benchmark.harness.harness2.selection import reward_selection scores = { # A single save a person reconciles: right on the timeout task, unscored. @@ -214,8 +214,8 @@ def score(self, run_dir, task_id): def test_model_requests_pass_the_privacy_gate(monkeypatch, tmp_path): + from benchmark.harness.harness2.llm import HttpChatModel, Price from openadapt_flow import privacy - from openadapt_flow.integrations.harness2.llm import HttpChatModel, Price monkeypatch.setenv("OPENADAPT_HOME", str(tmp_path)) monkeypatch.delenv(privacy.OUTBOUND_OPT_OUT_ENV, raising=False) @@ -252,7 +252,7 @@ def h2(): def _adapter(tmp_path, variants, evidence="readback"): - from openadapt_flow.integrations.harness2.adapter import OpenAdaptGuiAdapter + from benchmark.harness.harness2.adapter import OpenAdaptGuiAdapter return OpenAdaptGuiAdapter( mockmed_tasks(variants), @@ -279,8 +279,8 @@ def test_adapter_is_a_harness2_benchmark_adapter(tmp_path, h2): def test_reward_selection_finds_the_safe_harness(tmp_path, h2): pytest.importorskip("playwright.sync_api") - from openadapt_flow.integrations.harness2.proposers import ScriptedProposer - from openadapt_flow.integrations.harness2.selection import run_task + from benchmark.harness.harness2.proposers import ScriptedProposer + from benchmark.harness.harness2.selection import run_task adapter = _adapter(tmp_path, ("lookalike",)) prop = ScriptedProposer( @@ -331,8 +331,8 @@ def test_screen_evidence_hides_the_readback(tmp_path, h2): def test_model_selector_path_runs_harness2_selection(tmp_path, h2): pytest.importorskip("playwright.sync_api") - from openadapt_flow.integrations.harness2.proposers import ScriptedProposer - from openadapt_flow.integrations.harness2.selection import run_task + from benchmark.harness.harness2.proposers import ScriptedProposer + from benchmark.harness.harness2.selection import run_task adapter = _adapter(tmp_path, ("plain",), evidence="screen") prop = ScriptedProposer( @@ -365,7 +365,7 @@ def complete(self, system: str, user: str, *, max_tokens: int = 800) -> Completi def test_model_proposer_writes_only_harness_files(tmp_path, h2): - from openadapt_flow.integrations.harness2.proposers import ModelProposer + from benchmark.harness.harness2.proposers import ModelProposer reply = ( '\n# New\n- check the date of birth\n\n' diff --git a/packages/flow/tests/test_harness2_pixels.py b/packages/flow/tests/test_harness2_pixels.py index 40d24b216a..774e2f1917 100644 --- a/packages/flow/tests/test_harness2_pixels.py +++ b/packages/flow/tests/test_harness2_pixels.py @@ -11,19 +11,19 @@ import pytest -from openadapt_flow.integrations.harness2.agent import ( +from benchmark.harness.harness2.agent import ( RULE_IDENTITY, RULE_TYPE_LABEL, ScriptedGuiAgent, ) -from openadapt_flow.integrations.harness2.apps import MockMedTarget -from openadapt_flow.integrations.harness2.pixels import PixelDriver, web_pixels -from openadapt_flow.integrations.harness2.rollout import ( +from benchmark.harness.harness2.apps import MockMedTarget +from benchmark.harness.harness2.pixels import PixelDriver, web_pixels +from benchmark.harness.harness2.rollout import ( load_run, load_score, run_rollout, ) -from openadapt_flow.integrations.harness2.tasks import mockmed_tasks +from benchmark.harness.harness2.tasks import mockmed_tasks CAREFUL = f"Check the {RULE_IDENTITY}. Choose the type {RULE_TYPE_LABEL}." @@ -138,8 +138,8 @@ def rdp(page: Any) -> PixelDriver: def test_model_agent_is_told_typing_appends_on_pixels(): - from openadapt_flow.integrations.harness2.agent import ModelGuiAgent, Observation - from openadapt_flow.integrations.harness2.llm import Completion + from benchmark.harness.harness2.agent import ModelGuiAgent, Observation + from benchmark.harness.harness2.llm import Completion class Model: name = "fake/model" diff --git a/packages/flow/tests/test_harness2_traces.py b/packages/flow/tests/test_harness2_traces.py index abdcec1daf..5d4ca9d60a 100644 --- a/packages/flow/tests/test_harness2_traces.py +++ b/packages/flow/tests/test_harness2_traces.py @@ -6,12 +6,12 @@ import pytest +from benchmark.harness.harness2 import traces +from benchmark.harness.harness2.agent import RULE_IDENTITY, ScriptedGuiAgent +from benchmark.harness.harness2.apps import MockMedTarget +from benchmark.harness.harness2.rollout import run_rollout +from benchmark.harness.harness2.tasks import mockmed_tasks from openadapt_flow import privacy -from openadapt_flow.integrations.harness2 import traces -from openadapt_flow.integrations.harness2.agent import RULE_IDENTITY, ScriptedGuiAgent -from openadapt_flow.integrations.harness2.apps import MockMedTarget -from openadapt_flow.integrations.harness2.rollout import run_rollout -from openadapt_flow.integrations.harness2.tasks import mockmed_tasks @pytest.fixture(autouse=True) diff --git a/packages/flow/tests/test_injection_paths.py b/packages/flow/tests/test_injection_paths.py index 21fa304d03..3fb5ffaf6d 100644 --- a/packages/flow/tests/test_injection_paths.py +++ b/packages/flow/tests/test_injection_paths.py @@ -32,8 +32,8 @@ from openadapt_flow.compile_agent import FirstRunCompiler # noqa: E402 from openadapt_flow.compile_agent.agents import AgentLedger # noqa: E402 from openadapt_flow.mockmed.fault_server import serve # noqa: E402 -from openadapt_flow.mockmed.teach_scope import EncounterWrites # noqa: E402 from openadapt_flow.tutorial import _http_json # noqa: E402 +from tests._teach_scope import EncounterWrites # noqa: E402 from tests.test_injection_suite import _Attacker, _intent, _untrusted_text # noqa: E402 NOTE = "Synthetic follow-up in two weeks" @@ -470,7 +470,7 @@ def _guarded_run( "verify": str(_guard_verify(tmp_path, base)), "allow_text": ["Triage"], **( - {"write_adapter": "openadapt_flow.mockmed.teach_scope:EncounterWrites"} + {"write_adapter": "tests._teach_scope:EncounterWrites"} if write_adapter else {} ), diff --git a/packages/flow/tests/test_injection_suite.py b/packages/flow/tests/test_injection_suite.py index 3561f02e4f..3fa9f63eb5 100644 --- a/packages/flow/tests/test_injection_suite.py +++ b/packages/flow/tests/test_injection_suite.py @@ -40,7 +40,6 @@ from openadapt_flow.jit import TaskSignature, WebApp, compile_agent # noqa: E402 from openadapt_flow.jit.adapters import CaptureComputer, UnsupportedAction # noqa: E402 from openadapt_flow.mockmed.fault_server import serve # noqa: E402 -from openadapt_flow.mockmed.teach_scope import encounter_scope # noqa: E402 from openadapt_flow.runtime.effects import RestRecordVerifier # noqa: E402 from openadapt_flow.teach_scope import ( # noqa: E402 ScopeViolation, @@ -48,6 +47,7 @@ instruction_like, ) from openadapt_flow.tutorial import _http_json, _records # noqa: E402 +from tests._teach_scope import encounter_scope # noqa: E402 NOTE = "Synthetic follow-up in two weeks" SURFACES = ("note", "banner", "tooltip", "alt", "network") diff --git a/packages/flow/tests/test_learning_gate_semantics.py b/packages/flow/tests/test_learning_gate_semantics.py index 5fa506c9b0..47d3432ab2 100644 --- a/packages/flow/tests/test_learning_gate_semantics.py +++ b/packages/flow/tests/test_learning_gate_semantics.py @@ -26,6 +26,11 @@ import pytest +from benchmark.harness.synth_stream import ( + Drift, + generate_stream, + mockmed_base_program, +) from openadapt_flow.ir import ( ActionKind, Anchor, @@ -43,12 +48,7 @@ learn_from_traces, program_regression_gate, ) -from openadapt_flow.learning.synth_stream import ( - Drift, - StructuralDiffInducer, - generate_stream, - mockmed_base_program, -) +from openadapt_flow.learning.induce import StructuralDiffInducer # -- program builders --------------------------------------------------------- diff --git a/packages/flow/tests/test_lending_swer.py b/packages/flow/tests/test_lending_swer.py index b787960357..2b62a724f9 100644 --- a/packages/flow/tests/test_lending_swer.py +++ b/packages/flow/tests/test_lending_swer.py @@ -19,11 +19,11 @@ import inspect import sqlite3 +from benchmark.harness.effectbench import summarize +from benchmark.harness.mockloan.fault_server import LedgerDB from benchmark.lending_fault_model import faults as F from benchmark.lending_fault_model import ground_truth as G from benchmark.lending_fault_model import swer as S -from benchmark.harness.effectbench import summarize -from openadapt_flow.mockloan.fault_server import LedgerDB def _row(memo: str = F.MEMO_TEXT, source: str = "replay", **kw) -> dict: diff --git a/packages/flow/tests/test_mockloan.py b/packages/flow/tests/test_mockloan.py index a401562850..629e3169c4 100644 --- a/packages/flow/tests/test_mockloan.py +++ b/packages/flow/tests/test_mockloan.py @@ -14,8 +14,8 @@ import requests from playwright.sync_api import Browser, Page, sync_playwright -from openadapt_flow.mockloan.fault_server import serve as fault_serve -from openadapt_flow.mockloan.server import serve +from benchmark.harness.mockloan.fault_server import serve as fault_serve +from benchmark.harness.mockloan.server import serve MEMO = "Funding released per approval, first advance" diff --git a/packages/flow/tests/test_probe_induction.py b/packages/flow/tests/test_probe_induction.py index 40939eb2fe..9457f96f31 100644 --- a/packages/flow/tests/test_probe_induction.py +++ b/packages/flow/tests/test_probe_induction.py @@ -41,7 +41,7 @@ from openadapt_flow.deployment import DeploymentConfig from openadapt_flow.ir import ActionKind, Anchor, StateKind, Step, Workflow from openadapt_flow.mockmed.fault_server import serve as serve_fault -from openadapt_flow.reward.seed import write_bundle +from openadapt_flow.reward.bundle import write_bundle from openadapt_flow.runtime.replayer import Replayer from tests.test_replayer import FakeBackend, FakeVision, Match, make_png diff --git a/packages/flow/tests/test_provenance.py b/packages/flow/tests/test_provenance.py index e7886cf68d..41cae68e9a 100644 --- a/packages/flow/tests/test_provenance.py +++ b/packages/flow/tests/test_provenance.py @@ -225,7 +225,7 @@ def _note_provenance(sha: str, value: str) -> ProvenanceFile: def test_verify_run_refutes_a_faithfully_saved_misread(sor, tmp_path: Path) -> None: import requests - from openadapt_flow.guard_demo import verify_yaml + from benchmark.harness.guard_demo import verify_yaml from openadapt_flow.provenance import read_sources from openadapt_flow.verify_run import ( capture_baseline, @@ -288,8 +288,8 @@ def _task_file(tmp_path: Path, note: str) -> Path: def test_the_guard_sends_a_disagreement_to_a_person(sor, tmp_path: Path) -> None: + from benchmark.harness.guard_demo import FakeAgent, verify_yaml from openadapt_flow.guard import GuardedSave, GuardRefused, load_task - from openadapt_flow.guard_demo import FakeAgent, verify_yaml from openadapt_flow.verify_run import parse_verify_config base, _db = sor diff --git a/packages/flow/tests/test_reward_trust_boundary.py b/packages/flow/tests/test_reward_trust_boundary.py index d92a0ad65a..328887651d 100644 --- a/packages/flow/tests/test_reward_trust_boundary.py +++ b/packages/flow/tests/test_reward_trust_boundary.py @@ -21,6 +21,10 @@ from openadapt_types.oracle import OracleChannel # noqa: E402 from openadapt_types.reward import RewardOutcomeV1 # noqa: E402 +from openadapt_flow.reward.bundle import ( # noqa: E402 + calibrate_bundle, + write_bundle, +) from openadapt_flow.reward.calibration import ( # noqa: E402 FAULT_CLASSES, CalibrationRefused, @@ -44,10 +48,8 @@ MOCKMED_HONEST_PATIENT, MOCKMED_LIE_PATIENT, MOCKMED_QUERY, - calibrate_bundle, mockmed_episode, seed_mockmed, - write_bundle, write_mockmed_encounter, ) from openadapt_flow.reward.worker import RewardWorker, RewardWorkerError # noqa: E402 diff --git a/packages/flow/tests/test_reward_worker.py b/packages/flow/tests/test_reward_worker.py index 08b020c6aa..67f912df70 100644 --- a/packages/flow/tests/test_reward_worker.py +++ b/packages/flow/tests/test_reward_worker.py @@ -24,6 +24,11 @@ ) import openadapt_flow.reward.callables as callables # noqa: E402 +from openadapt_flow.reward.bundle import ( # noqa: E402 + CALIBRATION_FILE, + CALIBRATION_TRIALS, + write_bundle, +) from openadapt_flow.reward.calibration import ( # noqa: E402 CorpusRecipe, clopper_pearson_upper, @@ -46,15 +51,12 @@ observation, ) from openadapt_flow.reward.seed import ( # noqa: E402 - CALIBRATION_FILE, - CALIBRATION_TRIALS, MOCKMED_DUPLICATE_PATIENT, MOCKMED_HONEST_PATIENT, MOCKMED_LIE_PATIENT, MOCKMED_QUERY, mockmed_episode, seed_mockmed, - write_bundle, write_mockmed_banner, write_mockmed_encounter, ) diff --git a/packages/flow/tests/test_teach_placement.py b/packages/flow/tests/test_teach_placement.py index 3367fc1cd7..03014ed541 100644 --- a/packages/flow/tests/test_teach_placement.py +++ b/packages/flow/tests/test_teach_placement.py @@ -45,8 +45,8 @@ Workflow, ) from openadapt_flow.learning.gate import program_regression_gate +from openadapt_flow.learning.induce import StructuralDiffInducer from openadapt_flow.learning.interpreter import program_reproduces -from openadapt_flow.learning.synth_stream import StructuralDiffInducer from openadapt_flow.learning.teach import _choose_guard_fact from openadapt_flow.learning.trace import ExecutionTrace, TraceStep from openadapt_flow.runtime.effects.effect import Effect, EffectKind diff --git a/packages/flow/tests/test_teach_scope.py b/packages/flow/tests/test_teach_scope.py index 588cf3bc92..a631e9ade1 100644 --- a/packages/flow/tests/test_teach_scope.py +++ b/packages/flow/tests/test_teach_scope.py @@ -8,7 +8,6 @@ from openadapt_flow.guard import GuardTask from openadapt_flow.jit.adapters import Op -from openadapt_flow.mockmed.teach_scope import EncounterWrites, encounter_scope from openadapt_flow.teach_scope import ( ScopeViolation, TeachGuard, @@ -17,6 +16,7 @@ origin_of, provenance_for, ) +from tests._teach_scope import EncounterWrites, encounter_scope BASE = "http://127.0.0.1:8765" @@ -349,9 +349,9 @@ def test_a_scope_with_no_origin_lets_a_browser_reach_nothing() -> None: def test_write_adapters_load_by_name() -> None: from openadapt_flow.teach_scope import load_write_adapter - adapter = load_write_adapter("openadapt_flow.mockmed.teach_scope:EncounterWrites") + adapter = load_write_adapter("tests._teach_scope:EncounterWrites") assert isinstance(adapter, EncounterWrites) with pytest.raises(ValueError, match="module:attr"): - load_write_adapter("openadapt_flow.mockmed.teach_scope") + load_write_adapter("tests._teach_scope") with pytest.raises(ValueError, match="not a write adapter"): - load_write_adapter("openadapt_flow.mockmed.teach_scope:encounter_scope") + load_write_adapter("tests._teach_scope:encounter_scope") diff --git a/packages/flow/tests/test_undo_proposal.py b/packages/flow/tests/test_undo_proposal.py index 30c493e9a8..a7f4f01a40 100644 --- a/packages/flow/tests/test_undo_proposal.py +++ b/packages/flow/tests/test_undo_proposal.py @@ -891,7 +891,7 @@ def test_deployment_test_copy_resets_and_reads_every_record(tmp_path, mockmed): from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey from openadapt_flow.deployment import load_deployment - from openadapt_flow.reward.seed import write_bundle + from openadapt_flow.reward.bundle import write_bundle from openadapt_flow.undo_proposal import DeploymentUndoTestCopy write_bundle( diff --git a/packages/flow/tests/test_verify_tools.py b/packages/flow/tests/test_verify_tools.py index a681cb68b7..5d81e31009 100644 --- a/packages/flow/tests/test_verify_tools.py +++ b/packages/flow/tests/test_verify_tools.py @@ -21,12 +21,12 @@ from mcp import ClientSession # noqa: E402 from mcp.shared.memory import create_client_server_memory_streams # noqa: E402 +from benchmark.harness.guard_demo import FakeAgent, verify_yaml # noqa: E402 from openadapt_flow import verifier # noqa: E402 from openadapt_flow.agent.bridge import BridgeError # noqa: E402 from openadapt_flow.agent.cli import main as agent_main # noqa: E402 from openadapt_flow.agent.mcp import build_server # noqa: E402 from openadapt_flow.agent.verify_tools import VerifyBridge # noqa: E402 -from openadapt_flow.guard_demo import FakeAgent, verify_yaml # noqa: E402 from openadapt_flow.mockmed.fault_server import serve as fault_serve # noqa: E402 NOTE = "Follow-up in two weeks" diff --git a/packages/flow/tests/test_verify_write.py b/packages/flow/tests/test_verify_write.py index 62ab78fc3b..cb12b3a7ee 100644 --- a/packages/flow/tests/test_verify_write.py +++ b/packages/flow/tests/test_verify_write.py @@ -14,7 +14,7 @@ import pytest -from openadapt_flow.guard_demo import FakeAgent, verify_yaml +from benchmark.harness.guard_demo import FakeAgent, verify_yaml from openadapt_flow.mockmed.fault_server import serve as fault_serve from openadapt_flow.verify import ( BaselineUnreadable, diff --git a/packages/flow/tests/test_write_ledger.py b/packages/flow/tests/test_write_ledger.py index b2cd21b90c..d2639de45f 100644 --- a/packages/flow/tests/test_write_ledger.py +++ b/packages/flow/tests/test_write_ledger.py @@ -18,7 +18,7 @@ import pytest -from openadapt_flow.guard_demo import FakeAgent, verify_yaml +from benchmark.harness.guard_demo import FakeAgent, verify_yaml from openadapt_flow.mockmed.fault_server import serve as fault_serve from openadapt_flow.verifier import ( Actor, @@ -73,7 +73,7 @@ def _landed(db) -> int: """ import json, os, signal, sys, time from pathlib import Path - from openadapt_flow.guard_demo import FakeAgent, verify_yaml + from benchmark.harness.guard_demo import FakeAgent, verify_yaml from openadapt_flow.verifier import ( Actor, AlreadyWritten, Contract, WriteHeld, begin_write, verify_write, ) diff --git a/packages/flow/uv.lock b/packages/flow/uv.lock index 349fa70602..930fb9713b 100644 --- a/packages/flow/uv.lock +++ b/packages/flow/uv.lock @@ -2372,9 +2372,6 @@ grounder = [ grounding = [ { name = "openadapt-grounding" }, ] -harness2 = [ - { name = "playwright" }, -] hosted = [ { name = "keyring" }, ] @@ -2495,7 +2492,6 @@ requires-dist = [ { name = "pillow", specifier = ">=10.0" }, { name = "playwright", marker = "extra == 'browser'", specifier = ">=1.44" }, { name = "playwright", marker = "extra == 'dev'", specifier = ">=1.44" }, - { name = "playwright", marker = "extra == 'harness2'", specifier = ">=1.44" }, { name = "pydantic", specifier = ">=2.5" }, { name = "pygobject", marker = "sys_platform == 'linux' and extra == 'linux'", specifier = ">=3.46" }, { name = "pyobjc-core", marker = "sys_platform == 'darwin' and extra == 'native-ocr'", specifier = ">=10.0" }, @@ -2534,7 +2530,7 @@ requires-dist = [ { name = "winrt-windows-media-ocr", marker = "sys_platform == 'win32' and extra == 'native-ocr'", specifier = ">=2.0" }, { name = "winrt-windows-storage-streams", marker = "sys_platform == 'win32' and extra == 'native-ocr'", specifier = ">=2.0" }, ] -provides-extras = ["native-ocr", "browser", "dev", "grounder", "grounding", "local-grounding", "console", "countersign", "receipts", "keychain", "pkcs11", "execute", "windows", "reward", "harness2", "macos", "linux", "rdp", "registry", "privacy", "hosted", "service", "service-mlx", "capture", "interop", "agent", "claude-agent-sdk", "openai-agents", "langgraph"] +provides-extras = ["native-ocr", "browser", "dev", "grounder", "grounding", "local-grounding", "console", "countersign", "receipts", "keychain", "pkcs11", "execute", "windows", "reward", "macos", "linux", "rdp", "registry", "privacy", "hosted", "service", "service-mlx", "capture", "interop", "agent", "claude-agent-sdk", "openai-agents", "langgraph"] [[package]] name = "openadapt-grounding" From d85e82f1e5a02123df359cb9a474403211abcccb Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 15:34:46 -0500 Subject: [PATCH 12/22] Verifier: contract hashes keep their old values; name the key field Second review: Contract.digest, store and identity hashed the verifier dump directly, so every sql and fhir contract changed its hash (write ledger keys, reward certificates). They go through the same legacy shim now; a test pins main's values for sql, rest and fhir contracts. compile_recording and scaffold-verifier take idempotency_key_field, so a caller can name the app's at-most-once key field. Test verifier subclasses name the demo app's records path. --- .../flow/openadapt_flow/compiler/compile.py | 8 +++ packages/flow/openadapt_flow/deployment.py | 14 ++-- .../flow/openadapt_flow/scaffold_verifier.py | 5 +- packages/flow/openadapt_flow/verifier.py | 18 +++-- packages/flow/tests/test_api_step_durable.py | 2 +- .../tests/test_certified_binding_admission.py | 4 +- packages/flow/tests/test_effect_mining.py | 25 ++++++- packages/flow/tests/test_readback_pipeline.py | 13 +++- .../flow/tests/test_replayer_api_actuator.py | 8 ++- packages/flow/tests/test_scaffold_verifier.py | 8 ++- packages/flow/tests/test_verifier_api.py | 69 +++++++++++++++++++ 11 files changed, 151 insertions(+), 23 deletions(-) diff --git a/packages/flow/openadapt_flow/compiler/compile.py b/packages/flow/openadapt_flow/compiler/compile.py index 4a4ef0a13e..698656f8a0 100644 --- a/packages/flow/openadapt_flow/compiler/compile.py +++ b/packages/flow/openadapt_flow/compiler/compile.py @@ -2081,6 +2081,7 @@ def _compile_recording( mine_effects: bool = False, admit_api_bindings: bool = False, admission_fixture: Optional["AdmissionFixture"] = None, + idempotency_key_field: Optional[str] = None, skill_library_root: Optional[Path | str] = None, annotate: bool = False, annotator: Optional["StepAnnotator"] = None, @@ -2182,6 +2183,9 @@ def _compile_recording( ``admission_fixture``. A refused proposal leaves the step on the GUI ladder. FHIR / MCP / tool bindings are not synthesized. Default False keeps the bundle byte-identical. + idempotency_key_field: With ``mine_effects``, the record field that + carries this app's at-most-once key (from its contract or + deployment config). ``None``: no key is mined. admission_fixture: The held-out REST fixture (:class:`~openadapt_flow.compiler.binding_admission.AdmissionFixture`: base URL, and the app's own read and write paths). Required when @@ -3308,6 +3312,7 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: param_bindings=bindings or None, store_unchanged=unchanged, write_requests=write_requests, + idempotency_key_field=idempotency_key_field, ) if mined.effects: step.effects = mined.effects @@ -3486,6 +3491,9 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: if bind_effect_params: # Recorded only when set, so every existing compile keeps its digest. compiler_config["bind_effect_params"] = True + if idempotency_key_field: + # Recorded only when set, so every existing compile keeps its digest. + compiler_config["idempotency_key_field"] = idempotency_key_field if admission_fixture is not None: # Added only when set, so a compile without one hashes as before. admission: dict[str, Any] = { diff --git a/packages/flow/openadapt_flow/deployment.py b/packages/flow/openadapt_flow/deployment.py index 500824f58a..b9546ba4dd 100644 --- a/packages/flow/openadapt_flow/deployment.py +++ b/packages/flow/openadapt_flow/deployment.py @@ -2073,14 +2073,12 @@ def legacy_hash_dump(dumped: dict[str, Any], *, defaults_excluded: bool) -> None dumped.pop(name) elif dumped.get(name) is None: dumped[name] = legacy - if defaults_excluded: - # Nested reads (the identity path prunes its own nesting). - for item in dumped.get("candidates") or []: - if isinstance(item, dict): - legacy_hash_dump(item, defaults_excluded=True) - for item in (dumped.get("surfaces") or {}).values(): - if isinstance(item, dict): - legacy_hash_dump(item, defaults_excluded=True) + for item in dumped.get("candidates") or []: + if isinstance(item, dict): + legacy_hash_dump(item, defaults_excluded=defaults_excluded) + for item in (dumped.get("surfaces") or {}).values(): + if isinstance(item, dict): + legacy_hash_dump(item, defaults_excluded=defaults_excluded) def _identity_config_dump(cfg: EffectsConfig) -> dict[str, Any]: diff --git a/packages/flow/openadapt_flow/scaffold_verifier.py b/packages/flow/openadapt_flow/scaffold_verifier.py index d7d052fc48..7d5236c6e2 100644 --- a/packages/flow/openadapt_flow/scaffold_verifier.py +++ b/packages/flow/openadapt_flow/scaffold_verifier.py @@ -196,7 +196,9 @@ def _candidate_from_record( ) -def candidates_from_recording(recording_dir: Path) -> list[WriteCandidate]: +def candidates_from_recording( + recording_dir: Path, *, idempotency_key_field: Optional[str] = None +) -> list[WriteCandidate]: """Write-shaped candidates straight from ``events.jsonl`` evidence.""" import json @@ -245,6 +247,7 @@ def candidates_from_recording(recording_dir: Path) -> list[WriteCandidate]: label, new_records[0], demonstrated_values=demonstrated_values, + idempotency_key_field=idempotency_key_field, ) ) continue diff --git a/packages/flow/openadapt_flow/verifier.py b/packages/flow/openadapt_flow/verifier.py index b52cd42c2c..7af2112770 100644 --- a/packages/flow/openadapt_flow/verifier.py +++ b/packages/flow/openadapt_flow/verifier.py @@ -290,9 +290,11 @@ def from_dict(cls, data: Mapping[str, Any], *, name: str = "") -> "Contract": def digest(self) -> str: """SHA-256 of the contract as parsed: part of a write's identity.""" + config = self.config.model_dump(mode="json") + config["verifier"] = self._verifier_dump() body = json.dumps( { - "config": self.config.model_dump(mode="json"), + "config": config, "writes": list(self.writes), }, sort_keys=True, @@ -310,12 +312,20 @@ def store(self) -> str: return _digest( { "environment": self.config.environment_id, - "verifier": _strip( - self.config.verifier.model_dump(mode="json"), drop_secrets=True - ), + "verifier": _strip(self._verifier_dump(), drop_secrets=True), } ) + def _verifier_dump(self) -> dict[str, Any]: + """The verifier as hashed: fields that lost their defaults still hash + as their old values, so contracts written before keep their digests.""" + + from openadapt_flow.deployment import legacy_hash_dump + + dumped = self.config.verifier.model_dump(mode="json") + legacy_hash_dump(dumped, defaults_excluded=False) + return dumped + @property def shape(self) -> str: """SHA-256 of the effects, the verifier kind, the ``environment_id`` diff --git a/packages/flow/tests/test_api_step_durable.py b/packages/flow/tests/test_api_step_durable.py index 7fa248d1bd..27606aed44 100644 --- a/packages/flow/tests/test_api_step_durable.py +++ b/packages/flow/tests/test_api_step_durable.py @@ -122,7 +122,7 @@ class SecondReadFails(RestRecordVerifier): """Reads the record check's pre-state once, then can't reach it.""" def __init__(self, base_url: str) -> None: - super().__init__(base_url) + super().__init__(base_url, records_path="/api/db", records_key="records") self.reads = 0 def capture_pre_state(self, context: Any = None): diff --git a/packages/flow/tests/test_certified_binding_admission.py b/packages/flow/tests/test_certified_binding_admission.py index be662b0b7f..8527763100 100644 --- a/packages/flow/tests/test_certified_binding_admission.py +++ b/packages/flow/tests/test_certified_binding_admission.py @@ -375,7 +375,9 @@ def test_admitted_binding_halts_on_indeterminate_without_gui_retry(tmp_path): report = Replayer( backend, vision=_vision_that_confirms_saved(), - effect_verifier=_IndeterminateVerifier(url), + effect_verifier=_IndeterminateVerifier( + url, records_path="/api/db", records_key="records" + ), api_actuator=ApiActuator(url), poll_interval_s=0.01, ).run(_save_workflow(step), bundle_dir=bundle, run_dir=run_dir) diff --git a/packages/flow/tests/test_effect_mining.py b/packages/flow/tests/test_effect_mining.py index ab60f8bd6d..ef67e03633 100644 --- a/packages/flow/tests/test_effect_mining.py +++ b/packages/flow/tests/test_effect_mining.py @@ -112,20 +112,43 @@ def test_mines_record_written_and_field_equals_from_delta(): assert "note" not in field.match -def test_mines_idempotency_key_only_when_observed(): +def test_mines_idempotency_key_only_from_the_named_field_when_observed(): + # MockMed keeps its at-most-once key in "key". with_key = mine_step_effects( {SOR_BEFORE_KEY: [], SOR_AFTER_KEY: [_record(key="idem-abc")]}, _key_step(), exclude_texts=(NOTE,), + idempotency_key_field="key", ) written = next(e for e in with_key.effects if e.kind is EffectKind.RECORD_WRITTEN) assert written.idempotency_key == "idem-abc" assert written.key_field == "key" + # Another app names another field (a client request id, say). + other = mine_step_effects( + {SOR_BEFORE_KEY: [], SOR_AFTER_KEY: [_record(client_request_id="req-7")]}, + _key_step(), + exclude_texts=(NOTE,), + idempotency_key_field="client_request_id", + ) + written3 = next(e for e in other.effects if e.kind is EffectKind.RECORD_WRITTEN) + assert written3.idempotency_key == "req-7" + assert written3.key_field == "client_request_id" + + # No field named: no key mined, whatever the record holds. + unnamed = mine_step_effects( + {SOR_BEFORE_KEY: [], SOR_AFTER_KEY: [_record(key="idem-abc")]}, + _key_step(), + exclude_texts=(NOTE,), + ) + written4 = next(e for e in unnamed.effects if e.kind is EffectKind.RECORD_WRITTEN) + assert written4.idempotency_key is None + without = mine_step_effects( {SOR_BEFORE_KEY: [], SOR_AFTER_KEY: [_record()]}, _key_step(), exclude_texts=(NOTE,), + idempotency_key_field="key", ) written2 = next(e for e in without.effects if e.kind is EffectKind.RECORD_WRITTEN) # Never invented: §7 names the idempotency key as app-specific. diff --git a/packages/flow/tests/test_readback_pipeline.py b/packages/flow/tests/test_readback_pipeline.py index 2309c9bc0d..1396249468 100644 --- a/packages/flow/tests/test_readback_pipeline.py +++ b/packages/flow/tests/test_readback_pipeline.py @@ -27,6 +27,11 @@ def dirs(tmp_path): return bundle, tmp_path / "run" +def _demo_verifier(url): + """The demo app's records read.""" + return RestRecordVerifier(url, records_path="/api/db", records_key="records") + + def _run(records, dirs, *, corrupt=None, pipeline=None, verifier_factory=None): bundle, run_dir = dirs looped = author_data_driven_loop(_encounter_body(), records) @@ -35,7 +40,7 @@ def _run(records, dirs, *, corrupt=None, pipeline=None, verifier_factory=None): replayer = Replayer( backend, vision=_vision_confirms_saved(), - effect_verifier=(verifier_factory or RestRecordVerifier)(url), + effect_verifier=(verifier_factory or _demo_verifier)(url), poll_interval_s=0.01, readback_pipeline=pipeline, ) @@ -132,7 +137,7 @@ class ScreenVerifier(RestRecordVerifier): def factory(url): if kind == "screen_attr": - return ScreenVerifier(url) + return ScreenVerifier(url, records_path="/api/db", records_key="records") if kind == "candidate": return CandidateEffectVerifier( [RestRecordVerifier(url, records_path="/api/db", records_key="records")] @@ -177,7 +182,9 @@ def verify(self, *a, **k): report = Replayer( _RowWritingBackend(url), vision=_vision_confirms_saved(), - effect_verifier=SlowVerifier(url), + effect_verifier=SlowVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, readback_pipeline=pipeline, ).run(looped, bundle_dir=bundle, run_dir=run_dir) diff --git a/packages/flow/tests/test_replayer_api_actuator.py b/packages/flow/tests/test_replayer_api_actuator.py index 4ce9747673..388dfd5a2d 100644 --- a/packages/flow/tests/test_replayer_api_actuator.py +++ b/packages/flow/tests/test_replayer_api_actuator.py @@ -185,7 +185,7 @@ def __init__( fail_verify=False, tier_after_verify=None, ): - super().__init__(url) + super().__init__(url, records_path="/api/db", records_key="records") self.substrate = substrate self.verification_tier = tier self.fail_verify = fail_verify @@ -214,7 +214,7 @@ def __init__(self, reachable): class _LegacyReachabilityVerifier(RestRecordVerifier): def __init__(self, url, reachable): - super().__init__(url) + super().__init__(url, records_path="/api/db", records_key="records") self._reachable = reachable def capture_pre_state(self): @@ -543,7 +543,9 @@ def test_post_send_protocol_error_with_unavailable_verifier_requires_reconciliat report = Replayer( backend, vision=_vision_that_confirms_saved(), - effect_verifier=_UnavailableVerifier(url), + effect_verifier=_UnavailableVerifier( + url, records_path="/api/db", records_key="records" + ), api_actuator=ApiActuator(url, session=session), poll_interval_s=0.01, ).run(workflow, bundle_dir=bundle, run_dir=run_dir) diff --git a/packages/flow/tests/test_scaffold_verifier.py b/packages/flow/tests/test_scaffold_verifier.py index cff4078781..40b2e63fac 100644 --- a/packages/flow/tests/test_scaffold_verifier.py +++ b/packages/flow/tests/test_scaffold_verifier.py @@ -175,13 +175,19 @@ def test_recording_candidates_split_identity_and_payload( self, tmp_path: Path ) -> None: rec = _recording_dir(tmp_path) - candidates = candidates_from_recording(rec) + # The demo app keeps its at-most-once key in "key". + candidates = candidates_from_recording(rec, idempotency_key_field="key") assert [candidate.step_id for candidate in candidates] == ["step_001"] candidate = candidates[0] assert candidate.observed_delta assert candidate.match == {"patient_id": "p2", "type": "Triage"} assert candidate.payload == {"note": "Synthetic follow-up in two weeks"} assert candidate.idempotency == ("key", "demo-key-1") + # Unnamed, no field is guessed to be the key: it stays an identity + # field for a person to review. + (plain,) = candidates_from_recording(rec) + assert plain.idempotency is None + assert plain.match["key"] == "demo-key-1" def test_write_draft_from_recording_is_todo_marked(self, tmp_path: Path) -> None: rec = _recording_dir(tmp_path) diff --git a/packages/flow/tests/test_verifier_api.py b/packages/flow/tests/test_verifier_api.py index e3383a848e..0ce75a1f79 100644 --- a/packages/flow/tests/test_verifier_api.py +++ b/packages/flow/tests/test_verifier_api.py @@ -409,3 +409,72 @@ def test_a_pending_write_repr_shows_no_values(tmp_path: Path) -> None: text = repr(pending) assert "Ada" not in text and "example.test" not in text pending.release() + + +#: Contract hashes as openadapt-flow 2.5.0 computed them. A field that lost +#: its default (records_path, resource_type) must not move them: the write +#: ledger and reward certificates are keyed on these. +_PINNED_CONTRACT_HASHES = { + "fhir": { + "digest": "91c4a3ebe007fb29ff109ab420e004c69d18344776d91b833275cafa0764fd20", + "identity": "5035aab8eb04079931ced609318a886d463b270031edb967650e3966206919e3", + "store": "20d025f2007789ef2271b5cef933e38fd47d4114d559fc820aca9283d6d827fd", + }, + "rest": { + "digest": "5ab1d6ebc5bae6c5fba13a3f2ce60e2b65292c2126e6f858ef95190ee4b1104f", + "identity": "f287cf9ea6af74fd36dba0e42ae64a3024518c4c45ac7e063a1b4fee7dd2e678", + "store": "32521d4b64cce55ed24443f4d35ec1e73f472cd88ebc4d57e6a92d06161611a7", + }, + "sql": { + "digest": "45ea6035db9bf63b39e7bf31db07dab820e42a9a218ba39906ea54e283cb5b22", + "identity": "60cf99eaa0beed0335bff2e85e23ac1a29b90882075d728eb8952a71305b52a2", + "store": "f333364923dd35c0e086046de0494ee84df7c766bc5c9a53209e58cad4594a9c", + }, +} + + +def test_contract_hashes_do_not_change() -> None: + record = { + "name": "patient", + "match": ["email", "lname"], + "fields": {"fname": "fname", "lname": "lname", "email": "email"}, + } + verifiers = { + "sql": { + "kind": "sql", + "sqlite_database": "/data/app.db", + "sql_query": "SELECT * FROM patient", + }, + "rest": { + "kind": "rest", + "base_url": "http://app.local", + "records_path": "/api/db", + "records_key": "records", + }, + "fhir": { + "kind": "fhir", + "base_url": "https://fhir.local/r4", + "resource_type": "Observation", + "field_paths": { + "id": "id", + "email": "email", + "lname": "lname", + "fname": "fname", + }, + }, + } + for kind, verifier in verifiers.items(): + contract = Contract.from_dict( + { + "schema_version": "openadapt.verify/v1", + "verifier": verifier, + "record": record, + "frame": {"identity_field": "id"}, + } + ) + got = { + "digest": contract.digest, + "store": contract.store, + "identity": contract.identity, + } + assert got == _PINNED_CONTRACT_HASHES[kind], kind From 89fb0a61b4b1da777ff5bf2f6fa44a2c3e8b4a7b Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 19:22:33 -0500 Subject: [PATCH 13/22] Mine the 'key' convention as the default idempotency field; give the new tests their record paths --- .../tests/test_engine/test_check_setup.py | 4 +++- packages/flow/openadapt_flow/compiler/compile.py | 12 ++++++++---- .../openadapt_flow/compiler/effect_mining.py | 8 +++++--- .../openadapt_flow/runtime/effects/effect.py | 15 ++++++++++----- .../flow/openadapt_flow/scaffold_verifier.py | 7 +++++-- packages/flow/tests/test_cli_deployment.py | 16 ++++++++++++++-- packages/flow/tests/test_cli_new_commands.py | 3 +++ packages/flow/tests/test_effect_mining.py | 14 ++++++++++++-- .../flow/tests/test_effect_plugin_options.py | 2 ++ packages/flow/tests/test_scaffold_verifier.py | 6 +++--- packages/flow/tests/test_undo_proposal.py | 1 + 11 files changed, 66 insertions(+), 22 deletions(-) diff --git a/packages/desktop/tests/test_engine/test_check_setup.py b/packages/desktop/tests/test_engine/test_check_setup.py index 40de9e7096..c0a994b0be 100644 --- a/packages/desktop/tests/test_engine/test_check_setup.py +++ b/packages/desktop/tests/test_engine/test_check_setup.py @@ -279,7 +279,9 @@ def test_fixed_values_are_the_recorded_ones_and_follow_the_sealed_version(tmp_pa with pytest.raises(CheckSetupError, match="Unknown value"): answer_values(dispatcher, "wf1", ["member_id"], answered_by="person") - store.write_record_check_config("wf1", {"kind": "rest", "records_path": "/api/db", "base_url": BASE}) + store.write_record_check_config( + "wf1", {"kind": "rest", "records_path": "/api/db", "base_url": BASE} + ) saved = store.load("wf1") saved["record_check_config"] = str(store.root / "wf1.record-check.yaml") store.save("wf1", saved) diff --git a/packages/flow/openadapt_flow/compiler/compile.py b/packages/flow/openadapt_flow/compiler/compile.py index 934fd8e5f9..e097c566e5 100644 --- a/packages/flow/openadapt_flow/compiler/compile.py +++ b/packages/flow/openadapt_flow/compiler/compile.py @@ -74,6 +74,7 @@ is_meaningful_label, presumed_commit_step_ids, ) +from openadapt_flow.runtime.effects.effect import DEFAULT_IDEMPOTENCY_KEY_FIELD from openadapt_flow.runtime.identity import ( band_region, context_from_lines, @@ -2095,7 +2096,7 @@ def _compile_recording( mine_effects: bool = False, admit_api_bindings: bool = False, admission_fixture: Optional["AdmissionFixture"] = None, - idempotency_key_field: Optional[str] = None, + idempotency_key_field: Optional[str] = DEFAULT_IDEMPOTENCY_KEY_FIELD, skill_library_root: Optional[Path | str] = None, annotate: bool = False, annotator: Optional["StepAnnotator"] = None, @@ -2199,7 +2200,9 @@ def _compile_recording( synthesized. Default False keeps the bundle byte-identical. idempotency_key_field: With ``mine_effects``, the record field that carries this app's at-most-once key (from its contract or - deployment config). ``None``: no key is mined. + deployment config). Defaults to the ``key`` convention + (:data:`~openadapt_flow.compiler.effect_mining.DEFAULT_IDEMPOTENCY_KEY_FIELD`); + ``None``: no key is mined. admission_fixture: The held-out REST fixture (:class:`~openadapt_flow.compiler.binding_admission.AdmissionFixture`: base URL, and the app's own read and write paths). Required when @@ -3555,8 +3558,9 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: if bind_effect_params: # Recorded only when set, so every existing compile keeps its digest. compiler_config["bind_effect_params"] = True - if idempotency_key_field: - # Recorded only when set, so every existing compile keeps its digest. + if idempotency_key_field != DEFAULT_IDEMPOTENCY_KEY_FIELD: + # Recorded only when not the default, so every existing compile keeps + # its digest. compiler_config["idempotency_key_field"] = idempotency_key_field if admission_fixture is not None: # Added only when set, so a compile without one hashes as before. diff --git a/packages/flow/openadapt_flow/compiler/effect_mining.py b/packages/flow/openadapt_flow/compiler/effect_mining.py index ae7733e1da..8bf6117e0f 100644 --- a/packages/flow/openadapt_flow/compiler/effect_mining.py +++ b/packages/flow/openadapt_flow/compiler/effect_mining.py @@ -55,6 +55,7 @@ from openadapt_flow.ir import ActionKind, PostconditionKind, Region, Step from openadapt_flow.runtime.effects.effect import ( + DEFAULT_IDEMPOTENCY_KEY_FIELD, MIN_RENAV_ACTIONS, Effect, EffectKind, @@ -283,7 +284,7 @@ def _mine_from_sor_delta( after: list[dict], exclude_texts: tuple[str, ...], param_bindings: Optional[Mapping[str, str]] = None, - idempotency_key_field: Optional[str] = None, + idempotency_key_field: Optional[str] = DEFAULT_IDEMPOTENCY_KEY_FIELD, ) -> Optional[StepEffectMining]: """Mine real effects from an OBSERVED before/after system-of-record delta. @@ -896,7 +897,7 @@ def mine_step_effects( param_bindings: Optional[Mapping[str, str]] = None, store_unchanged: Optional[bool] = None, write_requests: Optional[bool] = None, - idempotency_key_field: Optional[str] = None, + idempotency_key_field: Optional[str] = DEFAULT_IDEMPOTENCY_KEY_FIELD, ) -> StepEffectMining: """Derive candidate system-of-record effects for one compiled step. @@ -944,7 +945,8 @@ def mine_step_effects( idempotency_key_field: The record field that carries this app's at-most-once key, from the contract or deployment config. Set, and present on the observed record, the mined ``record_written`` - counts records bearing it. ``None``: no key is mined. + counts records bearing it. Defaults to + :data:`DEFAULT_IDEMPOTENCY_KEY_FIELD`; ``None``: no key is mined. Returns: A :class:`StepEffectMining` describing the decision. ``effects`` is diff --git a/packages/flow/openadapt_flow/runtime/effects/effect.py b/packages/flow/openadapt_flow/runtime/effects/effect.py index a25cbd46e0..5685c2d3a7 100644 --- a/packages/flow/openadapt_flow/runtime/effects/effect.py +++ b/packages/flow/openadapt_flow/runtime/effects/effect.py @@ -825,6 +825,12 @@ def _resolve_readback( # contract cannot claim stronger evidence than the compiler would emit. MIN_RENAV_ACTIONS = 2 +#: The record field read as an at-most-once key when nothing names another: +#: :attr:`Effect.key_field`'s default, and the field the effect miner reads +#: unless its caller names another (or ``None`` for no key). The miner sets a +#: key only when the observed record carries a value there. +DEFAULT_IDEMPOTENCY_KEY_FIELD = "key" + class Effect(BaseModel): """A typed system-of-record effect (RFC ``Effect``, concrete runtime form). @@ -896,11 +902,10 @@ class Effect(BaseModel): #: identity) so the key is PER-RUN, not the frozen demonstration literal #: that would collide across unrelated runs (P0-3). idempotency_key: Optional[ValueExpr] = None - #: Which record field carries the idempotency key. App-specific: the - #: contract names it beside ``idempotency_key`` (the effect miner sets it - #: only from the field its caller names). The default is kept so effects - #: dumped before this note hash the same. - key_field: str = "key" + #: Which record field carries the idempotency key. The contract names it + #: beside ``idempotency_key``; the effect miner sets it from the field its + #: caller names (:data:`DEFAULT_IDEMPOTENCY_KEY_FIELD` unless told). + key_field: str = DEFAULT_IDEMPOTENCY_KEY_FIELD #: ``record_written`` only: a duplicate (idempotency) key made of several #: fields, such as OpenEMR's ``{lname, fname, DOB}``. Field -> value, each #: a :class:`ValueExpr` like :attr:`match`. The count reads only records diff --git a/packages/flow/openadapt_flow/scaffold_verifier.py b/packages/flow/openadapt_flow/scaffold_verifier.py index 7d5236c6e2..98a34bc963 100644 --- a/packages/flow/openadapt_flow/scaffold_verifier.py +++ b/packages/flow/openadapt_flow/scaffold_verifier.py @@ -44,6 +44,7 @@ from typing import Any, Optional from openadapt_flow import sealed_set as _sealed +from openadapt_flow.runtime.effects.effect import DEFAULT_IDEMPOTENCY_KEY_FIELD BUNDLE_MARKERS = ("workflow.json", "workflow.json.enc") RECORDING_MARKER = "events.jsonl" @@ -161,7 +162,7 @@ def _candidate_from_record( record: dict[str, Any], *, demonstrated_values: tuple[str, ...], - idempotency_key_field: Optional[str] = None, + idempotency_key_field: Optional[str] = DEFAULT_IDEMPOTENCY_KEY_FIELD, ) -> WriteCandidate: """Split one observed new record into identity vs payload fields. @@ -197,7 +198,9 @@ def _candidate_from_record( def candidates_from_recording( - recording_dir: Path, *, idempotency_key_field: Optional[str] = None + recording_dir: Path, + *, + idempotency_key_field: Optional[str] = DEFAULT_IDEMPOTENCY_KEY_FIELD, ) -> list[WriteCandidate]: """Write-shaped candidates straight from ``events.jsonl`` evidence.""" import json diff --git a/packages/flow/tests/test_cli_deployment.py b/packages/flow/tests/test_cli_deployment.py index 8898f5bafd..d2b1932300 100644 --- a/packages/flow/tests/test_cli_deployment.py +++ b/packages/flow/tests/test_cli_deployment.py @@ -93,8 +93,20 @@ def test_load_non_mapping_raises(tmp_path: Path) -> None: @pytest.mark.parametrize( "kind,kwargs,expected", [ - ("rest", {"base_url": "http://sor"}, "RestRecordVerifier"), - ("fhir", {"base_url": "http://sor/fhir"}, "FhirEffectVerifier"), + ( + "rest", + {"base_url": "http://sor", "records_path": "/api/db"}, + "RestRecordVerifier", + ), + ( + "fhir", + { + "base_url": "http://sor/fhir", + "resource_type": "Observation", + "field_paths": {"value": "valueString"}, + }, + "FhirEffectVerifier", + ), ("document-hash", {"root": "/tmp/store"}, "DocumentHashVerifier"), ], ) diff --git a/packages/flow/tests/test_cli_new_commands.py b/packages/flow/tests/test_cli_new_commands.py index a399d06803..620650efd8 100644 --- a/packages/flow/tests/test_cli_new_commands.py +++ b/packages/flow/tests/test_cli_new_commands.py @@ -126,6 +126,7 @@ def _replay_ns(**over) -> argparse.Namespace: config=None, effects_kind=None, effects_base_url=None, + effects_records_path=None, effects_root=None, api_actuator=False, api_base_url=None, @@ -145,6 +146,7 @@ def test_deployment_runtime_flags_wire_objects() -> None: ns = _replay_ns( effects_kind="rest", effects_base_url="http://sor", + effects_records_path="/api/db", api_base_url="http://api", durable=True, ) @@ -535,6 +537,7 @@ def _armed_click(sid, ocr, *, risk="reversible", effects=None): cfg.write_text( "backend:\n url: http://from-config\n" "effects:\n kind: rest\n base_url: http://sor\n" + " records_path: /api/db\n" "runtime:\n durable: true\n" "policy:\n policy: clinical-write\n" ) diff --git a/packages/flow/tests/test_effect_mining.py b/packages/flow/tests/test_effect_mining.py index ef67e03633..d0d14c4468 100644 --- a/packages/flow/tests/test_effect_mining.py +++ b/packages/flow/tests/test_effect_mining.py @@ -135,14 +135,24 @@ def test_mines_idempotency_key_only_from_the_named_field_when_observed(): assert written3.idempotency_key == "req-7" assert written3.key_field == "client_request_id" - # No field named: no key mined, whatever the record holds. + # No field named: the "key" convention, the default of Effect.key_field. unnamed = mine_step_effects( {SOR_BEFORE_KEY: [], SOR_AFTER_KEY: [_record(key="idem-abc")]}, _key_step(), exclude_texts=(NOTE,), ) written4 = next(e for e in unnamed.effects if e.kind is EffectKind.RECORD_WRITTEN) - assert written4.idempotency_key is None + assert written4.idempotency_key == "idem-abc" + + # None: no key mined, whatever the record holds. + none = mine_step_effects( + {SOR_BEFORE_KEY: [], SOR_AFTER_KEY: [_record(key="idem-abc")]}, + _key_step(), + exclude_texts=(NOTE,), + idempotency_key_field=None, + ) + written5 = next(e for e in none.effects if e.kind is EffectKind.RECORD_WRITTEN) + assert written5.idempotency_key is None without = mine_step_effects( {SOR_BEFORE_KEY: [], SOR_AFTER_KEY: [_record()]}, diff --git a/packages/flow/tests/test_effect_plugin_options.py b/packages/flow/tests/test_effect_plugin_options.py index 5383db1ca5..035a51084f 100644 --- a/packages/flow/tests/test_effect_plugin_options.py +++ b/packages/flow/tests/test_effect_plugin_options.py @@ -270,6 +270,8 @@ def test_identity_of_configs_without_plugin_fields_is_unchanged(): ): legacy_config.pop(name, None) legacy_config["candidates"] = [] + # resource_type had a default then and was always in the dump. + legacy_config.setdefault("resource_type", "Observation") def sanitized(value, *, key=""): # the identity's own redaction, verbatim lowered = key.lower() diff --git a/packages/flow/tests/test_scaffold_verifier.py b/packages/flow/tests/test_scaffold_verifier.py index 40b2e63fac..72b84b6c26 100644 --- a/packages/flow/tests/test_scaffold_verifier.py +++ b/packages/flow/tests/test_scaffold_verifier.py @@ -183,9 +183,9 @@ def test_recording_candidates_split_identity_and_payload( assert candidate.match == {"patient_id": "p2", "type": "Triage"} assert candidate.payload == {"note": "Synthetic follow-up in two weeks"} assert candidate.idempotency == ("key", "demo-key-1") - # Unnamed, no field is guessed to be the key: it stays an identity - # field for a person to review. - (plain,) = candidates_from_recording(rec) + # None: no field is taken as the key, so it stays an identity field + # for a person to review. + (plain,) = candidates_from_recording(rec, idempotency_key_field=None) assert plain.idempotency is None assert plain.match["key"] == "demo-key-1" diff --git a/packages/flow/tests/test_undo_proposal.py b/packages/flow/tests/test_undo_proposal.py index b670da8f0b..8edc923f16 100644 --- a/packages/flow/tests/test_undo_proposal.py +++ b/packages/flow/tests/test_undo_proposal.py @@ -925,6 +925,7 @@ def test_deployment_test_copy_resets_and_reads_every_record(tmp_path, mockmed): "name: copy\n" f"backend:\n kind: web\n url: {mockmed}/\n" f"effects:\n kind: rest\n base_url: {mockmed}\n" + " records_path: /api/db\n" "test_copy:\n declared: true\n" f" reset_url: {mockmed}/api/reset\n" " reward_contract: contract\n" From 3ccd99c59410531bd58ab4faf94b1a08aeb21101 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 20:18:33 -0500 Subject: [PATCH 14/22] Guard: reword new app-name examples from main; lower the baseline to 365 --- packages/flow/openadapt_flow/ir.py | 6 +++--- packages/flow/openadapt_flow/report_redaction.py | 2 +- .../flow/openadapt_flow/runtime/actuators/form_eval.py | 6 +++--- packages/flow/scripts/generality_baseline.json | 8 ++------ 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/packages/flow/openadapt_flow/ir.py b/packages/flow/openadapt_flow/ir.py index 6366891200..5922d72681 100644 --- a/packages/flow/openadapt_flow/ir.py +++ b/packages/flow/openadapt_flow/ir.py @@ -835,7 +835,7 @@ class WebFormWrite(BaseModel): - ``ambient``: the page sent it on its own. It came from a script of another document than the form, one that was loaded before the form - opened (for example OpenEMR's tab shell, which posts + opened (for example an app's tab shell that posts ``execute_background_services.php`` on a timer), and the form page's source doesn't name it. No replaced step caused it, so the API step neither sends nor waits for it; the live page goes on as it does under @@ -886,7 +886,7 @@ class WebFormSpec(BaseModel): #: Page script the browser runs on this save that the API step can't #: evaluate by reading the form: the form's ``on*`` handlers, the ``on*`` #: handlers of the fields it sends, a page script listening for submit, - #: and validate.js rule kinds such as OpenEMR's ``pastDate``. Before each + #: and validate.js rule kinds such as a ``pastDate`` rule. Before each #: save the runtime runs that script itself, in a scratch page of the live #: session with every write blocked, on the run's values, and sends only #: when the page would submit exactly the body the API step built. When @@ -901,7 +901,7 @@ class WebFormSpec(BaseModel): submitter: Optional[str] = None submitter_requests: list[str] = Field(default_factory=list) #: The control the person clicked to confirm the save when it isn't the - #: submitter (for example OpenEMR's ``#confirmCreate`` in the duplicate + #: submitter (for example a ``#confirmCreate`` button in a duplicate #: check dialog). Running the page's script clicks it too, in whichever #: frame has it, and the page must then submit the form. confirm: Optional[str] = None diff --git a/packages/flow/openadapt_flow/report_redaction.py b/packages/flow/openadapt_flow/report_redaction.py index 1770cb75b7..5755138fc5 100644 --- a/packages/flow/openadapt_flow/report_redaction.py +++ b/packages/flow/openadapt_flow/report_redaction.py @@ -234,7 +234,7 @@ def is_credential_name(name: str, *, header: bool = False) -> bool: The name is split into parts on anything but letters and digits; a part that is a token, CSRF/XSRF token or session id makes it one - (``csrf_token_form``, ``X-Frappe-CSRF-Token``, ``app_sessid``). For a + (``csrf_token_form``, ``X-CSRF-Token``, ``app_sessid``). For a header (``header=True``) an ``authorization`` part does too (``Espo-Authorization``); elsewhere that word names an approval, not a credential. A parameter or header named so is redacted in every app and diff --git a/packages/flow/openadapt_flow/runtime/actuators/form_eval.py b/packages/flow/openadapt_flow/runtime/actuators/form_eval.py index dd12e6c5c0..2c3b776732 100644 --- a/packages/flow/openadapt_flow/runtime/actuators/form_eval.py +++ b/packages/flow/openadapt_flow/runtime/actuators/form_eval.py @@ -4,7 +4,7 @@ and submit a form. Most forms run script on the way: key and change handlers on the fields, an ``onsubmit`` validator, a click handler on the save button, a page listener for ``submit``, custom validate.js rules such -as OpenEMR's ``pastDate``. The API step can't read what that script does, +as a ``pastDate`` rule. The API step can't read what that script does, so it runs it, before each save, the way the person's steps would: 1. open the form page in a scratch copy of the live session: a separate @@ -14,8 +14,8 @@ 2. while the page loads, let only same-origin GET requests through (no other origin at all). Once it has loaded, block every request except the exact scripts and stylesheets the page loaded, the reads the person's - click made in the recording (``submitter_requests``, such as OpenEMR's - duplicate search) and the static ``.js``/``.css`` files of a recorded + click made in the recording (``submitter_requests``, such as a + duplicate-record search) and the static ``.js``/``.css`` files of a recorded read's own frame, so nothing the page does can write; a form submit or a blocked channel in any frame counts as the page sending; 3. type the run's values into the fields the person typed or picked, with diff --git a/packages/flow/scripts/generality_baseline.json b/packages/flow/scripts/generality_baseline.json index 63467d2bf5..d7e0e8672c 100644 --- a/packages/flow/scripts/generality_baseline.json +++ b/packages/flow/scripts/generality_baseline.json @@ -157,9 +157,6 @@ "mockloan": 1, "mockmed": 1 }, - "network_capture.py": { - "clearpass": 1 - }, "operator_profile.py": { "openemr": 1 }, @@ -202,9 +199,8 @@ "openemr": 2 }, "report_redaction.py": { - "csrf_token_form": 3, - "openemr": 2, - "token_main": 5 + "csrf_token_form": 2, + "token_main": 1 }, "reward/models.py": { "mockmed-records": 1 From e47cdf11598eecfa74507bbcdb5b06b31cca90db Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 20:20:36 -0500 Subject: [PATCH 15/22] Point the new teach-authorization test at tests._teach_scope --- packages/flow/tests/_teach_scope.py | 2 +- packages/flow/tests/test_teach_authorization.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/flow/tests/_teach_scope.py b/packages/flow/tests/_teach_scope.py index d34ef23de1..ad66529a15 100644 --- a/packages/flow/tests/_teach_scope.py +++ b/packages/flow/tests/_teach_scope.py @@ -109,7 +109,7 @@ def encounter_authorization_spec( "identity": {"patient_id": {"value": patient_id}}, "fields": {"type": {"value": encounter_type}, "note": {"input": "note"}}, "expected_rows": 1, - "write_adapter": "openadapt_flow.mockmed.teach_scope:EncounterWrites", + "write_adapter": "tests._teach_scope:EncounterWrites", }, "forbidden": { "actions": [ diff --git a/packages/flow/tests/test_teach_authorization.py b/packages/flow/tests/test_teach_authorization.py index 2536cfe5c5..112e0c597e 100644 --- a/packages/flow/tests/test_teach_authorization.py +++ b/packages/flow/tests/test_teach_authorization.py @@ -16,13 +16,13 @@ from openadapt_flow import teach_authorization as ta from openadapt_flow.jit.adapters import Op -from openadapt_flow.mockmed.teach_scope import ( +from openadapt_flow.teach_authorization_fixture import LocalAuthorizer +from openadapt_flow.teach_scope import ScopeViolation, TeachGuard, TeachScope +from tests._teach_scope import ( EncounterWrites, encounter_authorization_spec, encounter_scope, ) -from openadapt_flow.teach_authorization_fixture import LocalAuthorizer -from openadapt_flow.teach_scope import ScopeViolation, TeachGuard, TeachScope BASE = "http://127.0.0.1:8765" NOTE = "Synthetic follow-up in two weeks" From 10e3734b83541b5304740d6829a8f0d0e8efa006 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 20:21:28 -0500 Subject: [PATCH 16/22] Guard: take main's harness import order; reword a new MockMed example --- packages/flow/benchmark/harness/run_benchmark.py | 2 +- packages/flow/openadapt_flow/teach_authorization_fixture.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/flow/benchmark/harness/run_benchmark.py b/packages/flow/benchmark/harness/run_benchmark.py index 3aba486faf..cdfc68a791 100644 --- a/packages/flow/benchmark/harness/run_benchmark.py +++ b/packages/flow/benchmark/harness/run_benchmark.py @@ -28,9 +28,9 @@ from pathlib import Path from typing import Any, Callable -from openadapt_flow.bench import _percentile from benchmark.harness import agent_baseline from benchmark.harness.verify import verify_encounter_saved +from openadapt_flow.bench import _percentile N_COMPILED = 100 N_AGENT = 20 diff --git a/packages/flow/openadapt_flow/teach_authorization_fixture.py b/packages/flow/openadapt_flow/teach_authorization_fixture.py index 6ef5c2d289..2aa82a9c0d 100644 --- a/packages/flow/openadapt_flow/teach_authorization_fixture.py +++ b/packages/flow/openadapt_flow/teach_authorization_fixture.py @@ -1,7 +1,7 @@ """A throwaway approver for tests, demos and evaluations. A real teach authorization is signed by a person with their approver key -(:mod:`openadapt_flow.teach_authorization`). Tests, the MockMed demos and the +(:mod:`openadapt_flow.teach_authorization`). Tests, the demos and the injection evaluation need one without a person at the keyboard, so :class:`LocalAuthorizer` makes an Ed25519 approver key in memory, starts (or joins) a real signed approver roster in a key directory, and signs statements From 5e3742e08ff5b11d1c3e31d491e82bcebff14011 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 20:32:35 -0500 Subject: [PATCH 17/22] Guard: count app names inside identifiers; restore main's lint, packaging and import-guard lines --- .github/workflows/packages.yml | 5 +- packages/flow/docs/GENERALITY_AUDIT.md | 16 ++- packages/flow/openadapt_flow/__main__.py | 3 +- packages/flow/pyproject.toml | 7 +- .../build_showcase_encounter_loop_bundle.py | 4 +- .../flow/scripts/check_engine_generality.py | 25 +++- packages/flow/scripts/demo_media.py | 128 ++++++++++++------ packages/flow/scripts/desktop/pn_db.py | 15 +- .../flow/scripts/desktop/session1_launch.py | 13 +- packages/flow/scripts/desktop/uia_arm.py | 28 ++-- packages/flow/scripts/frappe_lending_demo.py | 24 +--- packages/flow/scripts/generality.toml | 56 ++++---- .../flow/scripts/generality_baseline.json | 35 +++-- packages/flow/scripts/make_demo_gif.py | 41 +++--- packages/flow/scripts/openemr_demo.py | 67 ++++----- packages/flow/scripts/openemr_param_depth.py | 52 +++---- .../flow/scripts/openimis_eligibility_demo.py | 1 + .../flow/scripts/qualify_macos_ax_identity.py | 24 ++-- .../tests/test_engine_generality_check.py | 29 ++++ .../test_runtime_never_imports_benchmark.py | 21 ++- 20 files changed, 367 insertions(+), 227 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 2eec7df276..72880b1d77 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -357,7 +357,7 @@ jobs: # every public data/config asset listed in public-artifacts.json. run: python scripts/check_release_consistency.py - name: Ruff lint - run: ruff check openadapt_flow + run: ruff check openadapt_flow benchmark/harness - name: No app-specific code in the engine # App knowledge lives in profiles, contracts and fixtures (CONTRIBUTING: # "App knowledge lives in data"); the denylist is scripts/generality.toml. @@ -376,7 +376,7 @@ jobs: ruff check --isolated --select E9 --target-version py310 . uv run --no-project --python 3.10 python -m compileall -q -x '(^|/)(\.venv|node_modules|build|dist)/' . - name: Ruff format check - run: ruff format --check openadapt_flow tests + run: ruff format --check openadapt_flow tests benchmark/harness - name: Mypy run: mypy - name: Mypy (strict, safety path only) @@ -580,6 +580,7 @@ jobs: tests/e2e/test_structural_action.py \ tests/e2e/test_compile_agent_e2e.py \ tests/e2e/test_code_action_e2e.py \ + tests/e2e/test_code_agent_default_e2e.py \ tests/e2e/test_mcp_teach_e2e.py \ tests/e2e/test_browser_teach_e2e.py \ tests/e2e/test_browser_teach_write_e2e.py \ diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md index f8f4b51798..24507e5974 100644 --- a/packages/flow/docs/GENERALITY_AUDIT.md +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -7,7 +7,9 @@ The rule: the engine has no code written for one application. What one application needs lives in data (`app_profiles/*.yaml`, generated contracts, deployment configs) or in test fixtures and benchmark harnesses outside the shipped package. A CI check (`scripts/check_engine_generality.py`) keeps it -that way; [CONTRIBUTING.md](../CONTRIBUTING.md) states the rule. +that way; [CONTRIBUTING.md](../CONTRIBUTING.md) states the rule. It counts an +app name inside an identifier too (`_openemr_base`, `isOpenEMR`); its baseline +started at 426 hits and only shrinks. ## Method @@ -37,7 +39,7 @@ that way; [CONTRIBUTING.md](../CONTRIBUTING.md) states the rule. | --- | --- | --- | | (a) comment or docstring example | 118 | — | | (b) data loaded generically | 4 | 2 (`app_profiles/openemr.yaml`, the `clinical-write` policy pack as a default) | -| (c) fixture, harness or demo in the package | 35 rows, plus 27 files of `benchmark/` | 19 | +| (c) fixture, harness or demo in the package | 35 rows, plus the 25 Python files of `benchmark/` and 2 of `instrument/` | 19 | | (d) special-casing in engine logic | 27 | 15 | Rows group consecutive lines of one hit. The full table is at the end. @@ -62,7 +64,7 @@ motivating application. | D10 | `__main__.py:2838-2934`, `:9305-9319`; `cli_variants.py:131,173-182` | `replay` and the variant probe fall back to the bundled MockMed when no URL is given; `--drift` turns off structural grounding | MockMed | Target URL required outside the demo commands | planned | | D11 | `qualification_oracle_gate.py` | `qualify accept` tests every workflow's oracle against MockMed (banner text, `/api/encounter`, `/api/db`) | MockMed | Build the fault fixture from the bundle's own effect binding | planned | | D12 | `execute/dispatch.py:111-188` | The Execute service's live runner always runs the MockMed tutorial workflow | MockMed | Replay through the admission's deployment config; MockMed only for synthetic admissions | planned | -| D13 | `compiler/compile.py:151`, `runtime/replayer.py:394`, `runtime/identity.py:108` | Thresholds measured on MockMed only | MockMed | Generic logic, one-app numbers. Recalibrate on two or more apps | follow-up issue | +| D13 | `compiler/compile.py:148`, `runtime/replayer.py:394`, `runtime/identity.py:108` | Thresholds measured on MockMed only | MockMed | Generic logic, one-app numbers. Recalibrate on two or more apps | follow-up issue | | D14 | `runtime/identity.py:275-302` | Hand-written list of non-identifying words drawn from EMR list screens | MockMed, EMRs | Derive from the recording (tokens shared by sibling rows) | follow-up issue | | D15 | `eligibility/client.py:37-41`, `:1091-1110` | One vendor's eligibility API (URL, key name, body) in the engine | Stedi | Move behind a connector interface | follow-up issue | @@ -70,7 +72,7 @@ motivating application. | Where | What | Action | Status | | --- | --- | --- | --- | -| `benchmark/` (27 Python files: `run_benchmark`, `openemr_benchmark`, `openemr_local`, `frappe_lending`, `hybrid_benchmark`, `dom_arm`, `silent_wrong_action`, `desktop_benchmark`, `verify`, `agent_baseline`, `reliability`, `chart_fonts`, `effectbench/`) | Benchmark harnesses shipped in the wheel | Moved to `packages/flow/benchmark/harness/` (import `benchmark.harness.*`). `openadapt-flow benchmark` loads it from a source checkout and says so from a wheel | done | +| `benchmark/` (25 Python files: `run_benchmark`, `openemr_benchmark`, `openemr_local`, `frappe_lending`, `hybrid_benchmark`, `dom_arm`, `silent_wrong_action`, `desktop_benchmark`, `verify`, `agent_baseline`, `reliability`, `chart_fonts`, `effectbench/`) | Benchmark harnesses shipped in the wheel | Moved to `packages/flow/benchmark/harness/` (import `benchmark.harness.*`). `openadapt-flow benchmark` loads it from a source checkout and says so from a wheel | done | | `instrument/` | Competitor-drift harness; imported the benchmark at module load, so it was already broken in a wheel | Moved to `packages/flow/benchmark/harness/instrument/` | done | | `integrations/harness2/` | Harness2 research harness on MockMed | Move to `packages/flow/benchmark/harness/harness2/` | planned | | `mockloan/`, `demo_driver.record_disbursement_demo` | Lending demo app no product command uses | Move to the benchmark tree | planned | @@ -175,7 +177,7 @@ Paths are relative to `packages/flow/openadapt_flow/` as of the audited commit. | `fleet/simulator.py:1-` | (c) | synthetic payer portal | in-package simulator fixture; move to benchmark/examples | | `guard_demo.py:1-296` | (c) | MockMed | `python -m openadapt_flow.guard_demo`: fake agent posts referral to MockMed with chart_pages READER creds; not a CLI subcommand. Move to packages/flow/examples or benchmark; importers: packages/agent/tests (1), packages/flow/tests (6). | | `instrument/ (__init__.py, competitor_drift.py)` | (c) | MockMed | MockMed competitor-drift harness. Moved to packages/flow/benchmark/harness/instrument. | -| `integrations/harness2/ (all: tasks.py, apps.py, agent.py, adapter.py, traces.py, proposers.py, scorer.py, llm.py, pixels.py, rollout.py, selection.py, __init__.py)` | (c) | MockMed | Whole subpackage is a research harness for Harness2 on MockMed: MOCKMED_PATIENT/PATIENTS (tasks.py:71-148), MockMedTarget (apps.py:50-105, /api/db at :70), regex for MockMed task text and 'triage'/'New Encounter'/'encountersaved' (agent.py:221-324), MockMed banner regex Name — MRN — DOB (traces.py:55-57,128-138), clinical prompts (adapter.py:93-120,429-445; proposers.py:50-63). Move to packages/flow/benchmark/harness2; importers: benchmark/harness2_scorer/run.py, tests/test_harness2_{adapter,pixels,traces}.py, pyproject extra 'harness2', .github/workflows/packages.yml:451. | +| `integrations/harness2/ (all: tasks.py, apps.py, agent.py, adapter.py, traces.py, proposers.py, scorer.py, llm.py, pixels.py, rollout.py, selection.py, __init__.py)` | (c) | MockMed | Whole subpackage is a research harness for Harness2 on MockMed: MOCKMED_PATIENT/PATIENTS (tasks.py:71-148), MockMedTarget (apps.py:50-105, /api/db at :70), regex for MockMed task text and 'triage'/'New Encounter'/'encountersaved' (agent.py:221-324), MockMed banner regex Name — MRN — DOB (traces.py:55-57,128-138), clinical prompts (adapter.py:93-120,429-445; proposers.py:50-63). Move to packages/flow/benchmark/harness/harness2; importers: benchmark/harness2_scorer/run.py, tests/test_harness2_{adapter,pixels,traces}.py, pyproject extra 'harness2', .github/workflows/packages.yml:451. | | `learning/synth_stream.py:1-245` | (c) | MockMed | synthetic MockMed add-patient-note stream (intents, 'Consent Required', 'Belford, Phil DOB'). Move to benchmark/ or tests/fixtures. | | `learning/synth_stream.py:246-525` | (c) | (generic) | StructuralDiffInducer is generic but lives in the MockMed harness module; imported by learning/teach.py:96, learning/branch_teach.py:76, benchmark/openemr_e2e/harness.py, 4 tests; pyproject mypy list (line 611). Move to learning/induce.py first, then move the stream out. | | `mockloan/ (server.py, fault_server.py, static/)` | (c) | MockLoan | Own synthetic lending app shipped in wheel but used by no product command; only demo_driver.record_disbursement_demo, benchmark/lending_fault_model, benchmark/undo_proposal, tests (test_mockloan, test_lending_swer, e2e/test_lending_fault_model), scripts (1). Move to packages/flow/benchmark/apps/mockloan. | @@ -304,8 +306,8 @@ Paths are relative to `packages/flow/openadapt_flow/` as of the audited commit. | `validation/dense_surface.py:1271` | (a) | OpenEMR | comment/docstring example | | `validation/dense_surface.py:5` | (a) | OpenEMR | comment/docstring example | | `verifier.py:2467` | (a) | OpenEMR | comment/docstring example | -| `vision/hashing.py:10` | (a) | MockMed | docstring for the MockMed-calibrated phash tolerance (see compile.py:151) | +| `vision/hashing.py:10` | (a) | MockMed | docstring for the MockMed-calibrated phash tolerance (see compile.py:148) | | `volatility.py:100` | (a) | OpenEMR | comment/docstring example | | `volatility.py:31` | (a) | OpenEMR | comment/docstring example | | `volatility.py:6` | (a) | OpenEMR | comment/docstring example | -| `benchmark/` (27 files) | (c) | OpenEMR, Frappe, MockMed, Open Dental | Moved to `packages/flow/benchmark/harness/` | +| `benchmark/` (25 files) | (c) | OpenEMR, Frappe, MockMed, Open Dental | Moved to `packages/flow/benchmark/harness/` | diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index 0a15c63874..2abfb44aa3 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -5458,7 +5458,8 @@ def _checkout_harness_on_path(command: str) -> bool: ) return False if str(checkout) not in sys.path: - sys.path.insert(0, str(checkout)) + # Appended, so nothing in the checkout shadows an installed module. + sys.path.append(str(checkout)) return True diff --git a/packages/flow/pyproject.toml b/packages/flow/pyproject.toml index b4bedc3057..f8723b8307 100644 --- a/packages/flow/pyproject.toml +++ b/packages/flow/pyproject.toml @@ -353,6 +353,7 @@ exclude = [ "/docs/validation/identity_roc.json", "/docs/validation/identity_roc.png", "/openadapt_flow/benchmark/reliability_corpus.py", + "/benchmark/harness/reliability_corpus.py", "/openadapt_flow/validation/adversary_corpus.py", "/openadapt_flow/validation/adversary_corpus_v2.py", "/openadapt_flow/validation/adversary_corpus_v3.py", @@ -416,6 +417,7 @@ exclude = [ "/docs/validation/identity_roc.json", "/docs/validation/identity_roc.png", "/openadapt_flow/benchmark/reliability_corpus.py", + "/benchmark/harness/reliability_corpus.py", "/openadapt_flow/validation/adversary_corpus.py", "/openadapt_flow/validation/adversary_corpus_v2.py", "/openadapt_flow/validation/adversary_corpus_v3.py", @@ -501,8 +503,8 @@ required-environments = [ target-version = "py310" line-length = 88 # Generated / non-package trees at the repo root are not part of the lint -# surface. Anchored with a leading slash so the package's own -# openadapt_flow/benchmark/ IS still linted (a bare "benchmark" would match it). +# surface. The benchmark harness (benchmark/harness/) is linted by naming it +# explicitly in CI (`ruff check openadapt_flow benchmark/harness`). extend-exclude = ["/benchmark", "/docs", "/runs", "/dist"] [tool.ruff.lint] @@ -605,7 +607,6 @@ module = [ "openadapt_flow.__main__", "openadapt_flow.backends.playwright_backend", "openadapt_flow.backends.rdp_backend", - "benchmark.harness.desktop_benchmark", "openadapt_flow.compiler.disambiguation", "openadapt_flow.interactive_recorder", "openadapt_flow.learning.synth_stream", diff --git a/packages/flow/scripts/build_showcase_encounter_loop_bundle.py b/packages/flow/scripts/build_showcase_encounter_loop_bundle.py index d829ac27d1..29e5372093 100644 --- a/packages/flow/scripts/build_showcase_encounter_loop_bundle.py +++ b/packages/flow/scripts/build_showcase_encounter_loop_bundle.py @@ -158,9 +158,7 @@ def main() -> int: _reset(LOOP_OUT) looped.save(LOOP_OUT) - print( - f"Wrote {BODY_OUT} (program:false): {len(body.steps)} demonstrated steps." - ) + print(f"Wrote {BODY_OUT} (program:false): {len(body.steps)} demonstrated steps.") print( f"Wrote {LOOP_OUT} (program:true): {len(records)} records over relation " f"'worklist', body = {len(body.steps)} demonstrated steps." diff --git a/packages/flow/scripts/check_engine_generality.py b/packages/flow/scripts/check_engine_generality.py index 536a2ee77a..d5b3976f2d 100644 --- a/packages/flow/scripts/check_engine_generality.py +++ b/packages/flow/scripts/check_engine_generality.py @@ -29,7 +29,10 @@ from collections import Counter from pathlib import Path -import tomllib +try: + import tomllib +except ModuleNotFoundError: # Python 3.10 + import tomli as tomllib HERE = Path(__file__).resolve().parent FLOW = HERE.parent @@ -40,18 +43,24 @@ #: Engine files that hold text: code and the data shipped with it. SUFFIXES = {".py", ".js", ".html", ".css", ".json", ".yaml", ".yml", ".toml", ".txt"} +#: The marker opens the comment: ``# example: ...`` or ``#: example: ...``. EXAMPLE_MARKER = re.compile(r"#:?\s*example:", re.IGNORECASE) +#: A lower-case letter or digit followed by a capital: where a camelCase word +#: splits, so ``isOpenEMR`` reads as ``is Open EMR`` for the name patterns. +_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") + def load_config(path: Path = CONFIG) -> dict: return tomllib.loads(path.read_text(encoding="utf-8")) -def _term_patterns(config: dict) -> dict[str, re.Pattern[str]]: - terms: dict[str, re.Pattern[str]] = {} +def _term_patterns(config: dict) -> dict[str, tuple[re.Pattern[str], bool]]: + """``{term: (pattern, is_name)}``; names also match inside camelCase words.""" + terms: dict[str, tuple[re.Pattern[str], bool]] = {} for section in ("names", "identifiers"): for term, pattern in config[section].items(): - terms[term] = re.compile(pattern, re.IGNORECASE) + terms[term] = (re.compile(pattern, re.IGNORECASE), section == "names") return terms @@ -86,10 +95,12 @@ def scan( continue for number, line in enumerate(text.splitlines(), start=1): stripped = line.lstrip() - if stripped.startswith("#") and EXAMPLE_MARKER.search(stripped): + if EXAMPLE_MARKER.match(stripped): continue - for term, pattern in terms.items(): - if pattern.search(line) and not _allowed(rel, term, config): + split = _CAMEL_BOUNDARY.sub(" ", line) + for term, (pattern, is_name) in terms.items(): + found = pattern.search(split if is_name else line) + if found and not _allowed(rel, term, config): hits.setdefault(rel, {}).setdefault(term, []).append(number) return hits diff --git a/packages/flow/scripts/demo_media.py b/packages/flow/scripts/demo_media.py index 701167749c..72bd91f903 100644 --- a/packages/flow/scripts/demo_media.py +++ b/packages/flow/scripts/demo_media.py @@ -247,12 +247,8 @@ def _build_master( for idx, ov in enumerate(overlays): inputs += ["-i", str(ov.png)] label = f"o{idx}" - enable = ( - ":enable='lt(mod(t,1),0.55)'" if ov.blink else "" - ) - parts.append( - f"[{last}][{idx + 1}:v]overlay={ov.x}:{ov.y}{enable}[{label}]" - ) + enable = ":enable='lt(mod(t,1),0.55)'" if ov.blink else "" + parts.append(f"[{last}][{idx + 1}:v]overlay={ov.x}:{ov.y}{enable}[{label}]") last = label filtergraph = ";".join(parts) _run( @@ -705,9 +701,7 @@ def _diff_card(patch: dict, heal: dict, base_frame: Path, card: Path) -> None: # Diff panel px, py, pw, ph = 80, 190, W - 160, 250 d.rounded_rectangle((px, py, px + pw, py + ph), radius=16, fill=CARD) - d.rounded_rectangle( - (px, py, px + pw, py + ph), radius=16, outline=LINE, width=1 - ) + d.rounded_rectangle((px, py, px + pw, py + ph), radius=16, outline=LINE, width=1) y = py + 26 changes = patch.get("changes", []) for ch in changes: @@ -733,7 +727,9 @@ def _diff_card(patch: dict, heal: dict, base_frame: Path, card: Path) -> None: continue d.text((px + 30, y), f"- {field_name}: {old!r}", font=f_mono, fill=DIFF_DEL) y += 42 - d.text((px + 30, y), f"+ {field_name}: {new!r}{ident}", font=f_mono, fill=DIFF_ADD) + d.text( + (px + 30, y), f"+ {field_name}: {new!r}{ident}", font=f_mono, fill=DIFF_ADD + ) y += 52 d.text( @@ -815,8 +811,7 @@ def build_heal(work: Path, out: Path, bundle: Path, *, headed: bool) -> Path: "-i", str(card), "-vf", - f"scale={OUT_W}:-2:flags=lanczos,fps={FPS}," - "fade=t=in:st=0:d=0.3", + f"scale={OUT_W}:-2:flags=lanczos,fps={FPS},fade=t=in:st=0:d=0.3", "-c:v", "libx264", "-profile:v", @@ -914,7 +909,9 @@ def _label( d.rounded_rectangle( (x, y, x + ww + 24, y + h), radius=10, outline=color + (255,), width=2 ) - d.rounded_rectangle((x + 10, y + 9, x + 10 + tw, y + h - 9), radius=6, fill=color + (255,)) + d.rounded_rectangle( + (x + 10, y + 9, x + 10 + tw, y + h - 9), radius=6, fill=color + (255,) + ) d.text((x + 20, y + 12), tag, font=f_tag, fill=(10, 13, 20, 255)) d.text((x + 22 + tw, y + 10), text, font=f_txt, fill=(232, 236, 244, 255)) @@ -925,7 +922,12 @@ def _anchor_box(size, region, color, alpha_line) -> Image.Image: x, y, w, h = region d.rectangle((x - 3, y - 3, x + w + 3, y + h + 3), outline=color + (255,), width=4) # corner ticks - for cx, cy in ((x - 3, y - 3), (x + w + 3, y - 3), (x - 3, y + h + 3), (x + w + 3, y + h + 3)): + for cx, cy in ( + (x - 3, y - 3), + (x + w + 3, y - 3), + (x - 3, y + h + 3), + (x + w + 3, y + h + 3), + ): d.rectangle((cx - 5, cy - 5, cx + 5, cy + 5), fill=color + (255,)) return layer @@ -939,9 +941,7 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None save_region = save["anchor"]["region"] note_region = note_click["anchor"]["region"] save_sel = save["anchor"]["structural"]["selector"] - assertion = next( - (e for e in save["expect"] if e["kind"] == "text_present"), None - ) + assertion = next((e for e in save["expect"] if e["kind"] == "text_present"), None) assert_text = assertion["text"] if assertion else "Encounter saved-" risk = save["risk"] @@ -954,21 +954,41 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None box_save = _anchor_box((1280, 800), save_region, ACCENT, 255) lbl_anchor = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) - _label(lbl_anchor, (save_region[0] + save_region[2] + 24, save_region[1] - 6), - "anchor", save_sel, ACCENT) + _label( + lbl_anchor, + (save_region[0] + save_region[2] + 24, save_region[1] - 6), + "anchor", + save_sel, + ACCENT, + ) lbl_assert = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) - _label(lbl_assert, (save_region[0] + save_region[2] + 24, save_region[1] + 54), - "assert", f'text_present "{assert_text}"', WARN) + _label( + lbl_assert, + (save_region[0] + save_region[2] + 24, save_region[1] + 54), + "assert", + f'text_present "{assert_text}"', + WARN, + ) # risk pill under the assert dr = ImageDraw.Draw(lbl_assert) rx, ry = save_region[0] + save_region[2] + 24, save_region[1] + 110 dr.rounded_rectangle((rx, ry, rx + 210, ry + 40), radius=20, fill=BAD + (255,)) - dr.text((rx + 18, ry + 8), f"risk · {risk}", font=_font(_FONT_SANS_B, 22), fill=(10, 13, 20, 255)) + dr.text( + (rx + 18, ry + 8), + f"risk · {risk}", + font=_font(_FONT_SANS_B, 22), + fill=(10, 13, 20, 255), + ) lbl_param = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) - _label(lbl_param, (note_region[0], note_region[1] - 58), - "param", f'note = "{param_val}"', GOOD) + _label( + lbl_param, + (note_region[0], note_region[1] - 58), + "param", + f'note = "{param_val}"', + GOOD, + ) # Code panel (real workflow.py step_010 excerpt), wrapped to fit. cp = tuple(save["anchor"]["click_point"]) @@ -1005,8 +1025,12 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None head = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) hd = ImageDraw.Draw(head) hd.rectangle((0, 0, 1280, 96), fill=(10, 13, 20, 210)) - hd.text((40, 24), "The demonstration becomes an editable script", - font=title_f, fill=(232, 236, 244, 255)) + hd.text( + (40, 24), + "The demonstration becomes an editable script", + font=title_f, + fill=(232, 236, 244, 255), + ) _paste_alpha(img, head, _ramp(t, 0.1, 0.5)) pulse = 0.55 + 0.45 * abs(((t * 1.6) % 1.0) - 0.5) * 2 @@ -1019,7 +1043,9 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None slide = _ramp(t, 4.6, 0.9) if slide > 0: shade = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) - ImageDraw.Draw(shade).rectangle((0, 96, 1280, 800), fill=(8, 11, 18, int(150 * slide))) + ImageDraw.Draw(shade).rectangle( + (0, 96, 1280, 800), fill=(8, 11, 18, int(150 * slide)) + ) _paste_alpha(img, shade, 1.0) px = int(1280 - (560 + 60) * slide) img.alpha_composite(panel, (px, 150)) @@ -1037,8 +1063,12 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None def _code_panel(lines: Sequence[str], width: int, height: int) -> Image.Image: panel = Image.new("RGBA", (width, height), (0, 0, 0, 0)) d = ImageDraw.Draw(panel) - d.rounded_rectangle((0, 0, width - 1, height - 1), radius=18, fill=(18, 22, 32, 255)) - d.rounded_rectangle((0, 0, width - 1, height - 1), radius=18, outline=LINE + (255,), width=1) + d.rounded_rectangle( + (0, 0, width - 1, height - 1), radius=18, fill=(18, 22, 32, 255) + ) + d.rounded_rectangle( + (0, 0, width - 1, height - 1), radius=18, outline=LINE + (255,), width=1 + ) # title bar dots for k, col in enumerate(((248, 113, 113), (251, 191, 36), (52, 211, 153))): d.ellipse((22 + k * 26, 20, 36 + k * 26, 34), fill=col + (255,)) @@ -1093,8 +1123,10 @@ def build_audit(work: Path, out: Path, run_dir: Path, *, headed: bool) -> None: with sync_playwright() as p: browser = p.chromium.launch(headless=not headed) context = browser.new_context( - viewport=view, device_scale_factor=1, - record_video_dir=str(vid_dir), record_video_size=view, + viewport=view, + device_scale_factor=1, + record_video_dir=str(vid_dir), + record_video_size=view, ) page = context.new_page() page.goto(file_url) @@ -1117,7 +1149,10 @@ def build_audit(work: Path, out: Path, run_dir: Path, *, headed: bool) -> None: _badge_png(badge, "illustrated run report", dot=ACCENT) master = work / "audit_master.mp4" _build_master( - raw, master, trim_start=0.5, speed=1.0, + raw, + master, + trim_start=0.5, + speed=1.0, overlays=[Overlay(badge, x="W-w-28", y="26")], ) # The audit clip is long and text-dense; bound the GIF fallback hard @@ -1131,9 +1166,19 @@ def build_audit(work: Path, out: Path, run_dir: Path, *, headed: bool) -> None: gif_max_s=3.0, ) # Prefer the crisp full-res top-of-report screenshot as the poster. - _run(["ffmpeg", "-y", "-i", str(poster_png), "-vf", - f"scale={OUT_W}:-2:flags=lanczos", "-q:v", "3", - str((out / "audit.jpg"))]) + _run( + [ + "ffmpeg", + "-y", + "-i", + str(poster_png), + "-vf", + f"scale={OUT_W}:-2:flags=lanczos", + "-q:v", + "3", + str((out / "audit.jpg")), + ] + ) sizes["jpg"] = (out / "audit.jpg").stat().st_size sizes["app"] = "mockmed" _update_manifest(out, "audit", sizes, "real") @@ -1226,9 +1271,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: # Everything else falls back to MockMed. No real PHI: the stock docker install # ships throwaway admin/pass creds and zero patient data. -OPENEMR_LOGIN = ( - "https://localhost:9390/interface/login/login.php?site=default" -) +OPENEMR_LOGIN = "https://localhost:9390/interface/login/login.php?site=default" def capture_record_openemr(work: Path, *, headed: bool) -> Optional[Path]: @@ -1290,9 +1333,7 @@ def center(getter) -> tuple[int, int]: page.wait_for_timeout(1200) # Open the Patient workflow via the real top nav. - rec.click( - *center(lambda: page.get_by_text("Patient", exact=True).first) - ) + rec.click(*center(lambda: page.get_by_text("Patient", exact=True).first)) page.wait_for_timeout(700) ns = page.get_by_text("New/Search", exact=False) if ns.count(): @@ -1412,7 +1453,10 @@ def build_run_openemr(work: Path, out: Path, *, headed: bool) -> bool: # would double up the indicator). See openadapt-web ReplayHero + PR #246. master = work / "run_openemr_master.mp4" _build_master( - raw, master, trim_start=0.6, speed=1.4, + raw, + master, + trim_start=0.6, + speed=1.4, overlays=[], ) sizes = _finalize(master, out / "run_openemr", poster_at=1.0) diff --git a/packages/flow/scripts/desktop/pn_db.py b/packages/flow/scripts/desktop/pn_db.py index 434f61c000..68c04e17be 100644 --- a/packages/flow/scripts/desktop/pn_db.py +++ b/packages/flow/scripts/desktop/pn_db.py @@ -86,8 +86,13 @@ def seed(drift: str = "none") -> None: def _patient_dict(r: sqlite3.Row) -> dict: - return {"id": r["id"], "first": r["first"], "last": r["last"], - "dob": r["dob"], "note": r["note"] or ""} + return { + "id": r["id"], + "first": r["first"], + "last": r["last"], + "dob": r["dob"], + "note": r["note"] or "", + } def list_patients(filt: str = "") -> None: @@ -125,8 +130,10 @@ def get(pid: int) -> None: def dump_all() -> None: con = _conn() - rows = [_patient_dict(r) for r in - con.execute("SELECT * FROM patients ORDER BY id").fetchall()] + rows = [ + _patient_dict(r) + for r in con.execute("SELECT * FROM patients ORDER BY id").fetchall() + ] con.close() print(json.dumps(rows)) diff --git a/packages/flow/scripts/desktop/session1_launch.py b/packages/flow/scripts/desktop/session1_launch.py index 13934b794d..5c6d33d52d 100644 --- a/packages/flow/scripts/desktop/session1_launch.py +++ b/packages/flow/scripts/desktop/session1_launch.py @@ -168,15 +168,20 @@ def main() -> None: import os if len(sys.argv) < 2: - print("usage: session1_launch.py [args...]", - file=sys.stderr) + print("usage: session1_launch.py [args...]", file=sys.stderr) sys.exit(2) script = os.path.abspath(sys.argv[1]) extra = list(sys.argv[2:]) if script.lower().endswith(".ps1"): # A WinForms/PowerShell UI target: run STA so WinForms works. - parts = ["powershell.exe", "-STA", "-ExecutionPolicy", "Bypass", - "-File", f'"{script}"'] + extra + parts = [ + "powershell.exe", + "-STA", + "-ExecutionPolicy", + "Bypass", + "-File", + f'"{script}"', + ] + extra else: pythonw = os.path.join(os.path.dirname(sys.executable), "pythonw.exe") if not os.path.exists(pythonw): diff --git a/packages/flow/scripts/desktop/uia_arm.py b/packages/flow/scripts/desktop/uia_arm.py index 3123032d4e..52bd4f06c3 100644 --- a/packages/flow/scripts/desktop/uia_arm.py +++ b/packages/flow/scripts/desktop/uia_arm.py @@ -35,8 +35,13 @@ import time from typing import Optional -WORKFLOW_TARGET_IDS = ["searchBox", "searchButton", "patientGrid", - "noteBox", "saveButton"] +WORKFLOW_TARGET_IDS = [ + "searchBox", + "searchButton", + "patientGrid", + "noteBox", + "saveButton", +] # The patient ROW is the identity-critical target and has NO usable id. IDENTITY_TARGET = "patient_row" @@ -149,15 +154,18 @@ def run(search: str, first: str, last: str, note: str, mode: str) -> dict: if rows: _select_row(rows[0].get("_cell")) selected_index = rows[0]["row_index"] - selected_name = f"{rows[0].get('first','')} {rows[0].get('last','')}".strip() + selected_name = ( + f"{rows[0].get('first', '')} {rows[0].get('last', '')}".strip() + ) else: # identity: exact match on rendered first AND last name text for row in rows: - if (row.get("first", "").strip().lower() == first.strip().lower() - and row.get("last", "").strip().lower() - == last.strip().lower()): + if ( + row.get("first", "").strip().lower() == first.strip().lower() + and row.get("last", "").strip().lower() == last.strip().lower() + ): _select_row(row.get("_cell")) selected_index = row["row_index"] - selected_name = f"{row.get('first','')} {row.get('last','')}".strip() + selected_name = f"{row.get('first', '')} {row.get('last', '')}".strip() break # 3. note + 4. save @@ -182,16 +190,14 @@ def main() -> None: ap.add_argument("--first", required=True) ap.add_argument("--last", required=True) ap.add_argument("--note", required=True) - ap.add_argument("--mode", choices=["positional", "identity"], - default="identity") + ap.add_argument("--mode", choices=["positional", "identity"], default="identity") args = ap.parse_args() try: out = run(args.search, args.first, args.last, args.note, args.mode) except Exception as e: # noqa: BLE001 import traceback - out = {"status": "error", "error": str(e), - "trace": traceback.format_exc()} + out = {"status": "error", "error": str(e), "trace": traceback.format_exc()} print(json.dumps(out)) diff --git a/packages/flow/scripts/frappe_lending_demo.py b/packages/flow/scripts/frappe_lending_demo.py index 13877e4925..a174d83c43 100644 --- a/packages/flow/scripts/frappe_lending_demo.py +++ b/packages/flow/scripts/frappe_lending_demo.py @@ -347,9 +347,7 @@ def record(fixture: FrappeFixture, out_dir: Path, *, headed: bool) -> Path: _preauthenticate_browser(backend, fixture) page = backend.page before_counts = fixture.table_counts() - before_non_target_loan_sha256 = ( - fixture.non_target_loan_applications_sha256() - ) + before_non_target_loan_sha256 = fixture.non_target_loan_applications_sha256() recorder = Recorder( backend, out_dir, @@ -451,9 +449,7 @@ def link_field(field: str, value: str, *, param: str) -> None: "primary_outcome": verdict.primary_outcome.value, "rest_records_sha256": _safe_records_sha256(evidence.rest_records), "db_records_sha256": _safe_records_sha256(evidence.db_records), - "before_non_target_loan_sha256": ( - evidence.before_non_target_loan_sha256 - ), + "before_non_target_loan_sha256": (evidence.before_non_target_loan_sha256), "after_non_target_loan_sha256": evidence.after_non_target_loan_sha256, "all_table_deltas": evidence.all_table_deltas, "delta_violations": evidence.delta_violations, @@ -585,9 +581,7 @@ def _capture_post_evidence( errors.append("pre-write non-target Loan Application digest unavailable") else: try: - after_non_target_loan_sha256 = ( - fixture.non_target_loan_applications_sha256() - ) + after_non_target_loan_sha256 = fixture.non_target_loan_applications_sha256() if after_non_target_loan_sha256 != before_non_target_loan_sha256: violations.append("tabLoan Application:non-target-content-changed") except Exception as exc: # noqa: BLE001 - unreadable is not clean @@ -826,9 +820,7 @@ def _persist_evidence( "before_table_counts": evidence.before_table_counts, "after_table_counts": evidence.after_table_counts, "all_table_deltas": evidence.all_table_deltas, - "before_non_target_loan_sha256": ( - evidence.before_non_target_loan_sha256 - ), + "before_non_target_loan_sha256": (evidence.before_non_target_loan_sha256), "after_non_target_loan_sha256": evidence.after_non_target_loan_sha256, "delta_violations": evidence.delta_violations, "oracle_errors": evidence.errors, @@ -1064,9 +1056,7 @@ def run_compiled_trial( # Authentication/form navigation are declared unmeasured setup; take # the collateral-delta baseline only after they finish. before_counts = fixture.table_counts() - before_non_target_loan_sha256 = ( - fixture.non_target_loan_applications_sha256() - ) + before_non_target_loan_sha256 = fixture.non_target_loan_applications_sha256() start = time.monotonic() report = Replayer(backend, effect_verifier=oracle.verifier).run( Workflow.load(bundle_dir), @@ -1155,9 +1145,7 @@ def run_agent_trial( _preauthenticate_browser(backend, fixture) _apply_condition(backend, condition) before_counts = fixture.table_counts() - before_non_target_loan_sha256 = ( - fixture.non_target_loan_applications_sha256() - ) + before_non_target_loan_sha256 = fixture.non_target_loan_applications_sha256() start = time.monotonic() result = agent_baseline.run_agent( backend, diff --git a/packages/flow/scripts/generality.toml b/packages/flow/scripts/generality.toml index 7b5b261273..8c382c226d 100644 --- a/packages/flow/scripts/generality.toml +++ b/packages/flow/scripts/generality.toml @@ -3,34 +3,36 @@ # The engine (openadapt_flow/) must not name a specific application. Add a term # here when a new application is qualified or a new app-specific identifier is # found. Patterns are Python regular expressions, matched case-insensitively -# unless they turn that off with (?-i:...). +# unless they turn that off with (?-i:...). A name matches inside an +# identifier too (`_openemr_base`, `isOpenEMR`, `to_stedi_body`): only a letter +# or digit next to it stops a match. [names] -openemr = '\bopen[ _-]?emr' -openimis = '\bopen[ _-]?imis' -medplum = '\bmedplum' -dolibarr = '\bdolibarr' -espocrm = '\bespo[ _-]?crm' -redmine = '\bredmine' -snipe-it = '\bsnipe[ _-]?it\b' -inventree = '\binventree' -kimai = '\bkimai' -openmrs = '\bopen[ _-]?mrs' -gitea = '\bgitea' -firefly-iii = '\bfirefly[ _-]?(iii|3)\b' -vikunja = '\bvikunja' -zammad = '\bzammad' -open-dental = '\bopen[ _-]?dental' -odoo = '\bodoo' -frappe = '\b(frappe|erpnext)' -patient-notes = '\bpatient[ _-]notes\b' -epic = '(?-i:\bEpic\b)' -cerner = '\bcerner' -stedi = '\bstedi\b' -textedit = '\btextedit' -notepad = '\bnotepad\b' -mockmed = '\bmock[ _-]?med' -mockloan = '\bmock[ _-]?loan' +openemr = '(? tuple[int, int]: @@ -73,8 +70,7 @@ def _center(locator) -> tuple[int, int]: box = locator.bounding_box() if box is None: raise RuntimeError(f"no bounding box for {locator}") - return (int(box["x"] + box["width"] / 2), - int(box["y"] + box["height"] / 2)) + return (int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2)) def _frame(page, url_fragment: str): @@ -153,14 +149,14 @@ def record(headed: bool = False, recording_dir: Path = RECORDING_DIR) -> Path: recorder.click(*_center(pencil)) page.wait_for_timeout(3000) pnotes = _frame(page, "pnotes_full") - recorder.click(*_center(pnotes.locator("button, a").filter( - has_text="Add").first)) + recorder.click( + *_center(pnotes.locator("button, a").filter(has_text="Add").first) + ) page.wait_for_timeout(2000) # The add-note form opens as a second nested pnotes_full iframe. recorder.click(*_center(_locate_any_frame(page, "textarea#note"))) recorder.type_text(RECORDED_NOTE, param="note") - recorder.click(*_center(_locate_any_frame( - page, "text=Save as new message"))) + recorder.click(*_center(_locate_any_frame(page, "text=Save as new message"))) page.wait_for_timeout(3000) return recorder.finish() @@ -232,35 +228,44 @@ def replay(n: int = 5) -> dict: (run_dir / "final.png").write_bytes(final_png) note_visible = _note_visible(final_png, note) except Exception as exc: # noqa: BLE001 - record and continue - summary["runs"].append({ - "run": i + 1, "success": False, "error": f"{type(exc).__name__}: {exc}", - }) + summary["runs"].append( + { + "run": i + 1, + "success": False, + "error": f"{type(exc).__name__}: {exc}", + } + ) print(f"run {i + 1}: EXCEPTION {exc}") continue finally: close() failed = [r for r in report.results if not r.ok] - summary["runs"].append({ - "run": i + 1, - "success": report.success, - "note_visible": note_visible, - "steps_ok": sum(1 for r in report.results if r.ok), - "steps_total": len(workflow.steps), - "rung_counts": report.rung_counts, - "heal_count": report.heal_count, - "total_s": round(elapsed, 1), - "first_failure": ( - {"step": failed[0].step_id, "error": failed[0].error} - if failed else None - ), - }) + summary["runs"].append( + { + "run": i + 1, + "success": report.success, + "note_visible": note_visible, + "steps_ok": sum(1 for r in report.results if r.ok), + "steps_total": len(workflow.steps), + "rung_counts": report.rung_counts, + "heal_count": report.heal_count, + "total_s": round(elapsed, 1), + "first_failure": ( + {"step": failed[0].step_id, "error": failed[0].error} + if failed + else None + ), + } + ) try: render_run_report(run_dir) except Exception as exc: # noqa: BLE001 print(f"run {i + 1}: report render failed: {exc}") - print(f"run {i + 1}: success={report.success} " - f"note_visible={note_visible} rungs={report.rung_counts} " - f"heals={report.heal_count} {elapsed:.0f}s") + print( + f"run {i + 1}: success={report.success} " + f"note_visible={note_visible} rungs={report.rung_counts} " + f"heals={report.heal_count} {elapsed:.0f}s" + ) ok = sum(1 for r in summary["runs"] if r.get("success")) verified = sum( 1 for r in summary["runs"] if r.get("success") and r.get("note_visible") diff --git a/packages/flow/scripts/openemr_param_depth.py b/packages/flow/scripts/openemr_param_depth.py index 5c77b4c57f..bafc965f1f 100644 --- a/packages/flow/scripts/openemr_param_depth.py +++ b/packages/flow/scripts/openemr_param_depth.py @@ -66,9 +66,7 @@ RUNS_DIR = WORK / "runs" RECORDED_PATIENT = "Phil" # demonstrated value; unique match "Belford, Phil" -RECORDED_NOTE = ( - "Validation baseline: reviewed home readings, plan unchanged today." -) +RECORDED_NOTE = "Validation baseline: reviewed home readings, plan unchanged today." PACING_S = 30.0 # public-demo courtesy between browser sessions @@ -99,9 +97,7 @@ def record() -> Path: page.wait_for_timeout(3000) finder = _frame(page, "dynamic_finder") - recorder.click( - *_center(finder.locator("a", has_text=RECORDED_PATIENT).first) - ) + recorder.click(*_center(finder.locator("a", has_text=RECORDED_PATIENT).first)) page.wait_for_timeout(4000) demographics = _frame(page, "demographics.php") @@ -118,14 +114,13 @@ def record() -> Path: recorder.click(*_center(pencil)) page.wait_for_timeout(3000) pnotes = _frame(page, "pnotes_full") - recorder.click(*_center( - pnotes.locator("button, a").filter(has_text="Add").first - )) + recorder.click( + *_center(pnotes.locator("button, a").filter(has_text="Add").first) + ) page.wait_for_timeout(2000) recorder.click(*_center(_locate_any_frame(page, "textarea#note"))) recorder.type_text(RECORDED_NOTE, param="note") - recorder.click(*_center(_locate_any_frame( - page, "text=Save as new message"))) + recorder.click(*_center(_locate_any_frame(page, "text=Save as new message"))) page.wait_for_timeout(3000) return recorder.finish() finally: @@ -137,9 +132,7 @@ def compile_bundle() -> None: if BUNDLE_DIR.exists(): shutil.rmtree(BUNDLE_DIR) - wf = compile_recording( - RECORDING_DIR, BUNDLE_DIR, name="openemr-param-depth" - ) + wf = compile_recording(RECORDING_DIR, BUNDLE_DIR, name="openemr-param-depth") print(f"compiled {len(wf.steps)} steps -> {BUNDLE_DIR}") for step in wf.steps: texts = [pc.text for pc in step.expect if pc.text] @@ -172,9 +165,7 @@ def _replay_one(label: str, params: dict[str, str], url: str) -> dict: note = params["note"] note_check = verify_note_saved(final_png, note) - final_text = normalize_text( - " ".join(line.text for line in ocr(final_png)) - ) + final_text = normalize_text(" ".join(line.text for line in ocr(final_png))) failed = [r for r in report.results if not r.ok] row = { "label": label, @@ -192,7 +183,8 @@ def _replay_one(label: str, params: dict[str, str], url: str) -> dict: "underwood_on_final": "underwood" in final_text, "first_failure": ( {"step": failed[0].step_id, "error": (failed[0].error or "")[:300]} - if failed else None + if failed + else None ), } print(json.dumps(row)) @@ -203,14 +195,22 @@ def replay() -> None: RUNS_DIR.mkdir(parents=True, exist_ok=True) rows = [] plan = [ - ("phil-control", { - "patient": "Phil", - "note": "Param depth control: confirmed refill pickup scheduled.", - }, DEMO_URL), - ("susan-drift", { - "patient": "Susan", - "note": "Param depth drift: dietary counseling summary delivered.", - }, DEMO_URL), + ( + "phil-control", + { + "patient": "Phil", + "note": "Param depth control: confirmed refill pickup scheduled.", + }, + DEMO_URL, + ), + ( + "susan-drift", + { + "patient": "Susan", + "note": "Param depth drift: dietary counseling summary delivered.", + }, + DEMO_URL, + ), ] for i, (label, params, url) in enumerate(plan): if i: diff --git a/packages/flow/scripts/openimis_eligibility_demo.py b/packages/flow/scripts/openimis_eligibility_demo.py index d1241f57ca..d2dbd15b77 100644 --- a/packages/flow/scripts/openimis_eligibility_demo.py +++ b/packages/flow/scripts/openimis_eligibility_demo.py @@ -212,6 +212,7 @@ def structured_text_at(self, x: int, y: int) -> str | None: context["insurance_no"] = dialogs[0][0] return json.dumps(context, separators=(",", ":")) + def eligibility_effects() -> list[Any]: """The eligibility outcome bound to the run's ``insurance_no``. diff --git a/packages/flow/scripts/qualify_macos_ax_identity.py b/packages/flow/scripts/qualify_macos_ax_identity.py index 1eb902da81..a575c9f8fe 100644 --- a/packages/flow/scripts/qualify_macos_ax_identity.py +++ b/packages/flow/scripts/qualify_macos_ax_identity.py @@ -161,7 +161,9 @@ def _terminate(pid: int) -> dict[str, Any]: return receipt -def _blocked(reason: str, missing: list[str], candidate: dict[str, Any]) -> dict[str, Any]: +def _blocked( + reason: str, missing: list[str], candidate: dict[str, Any] +) -> dict[str, Any]: return { "task": "macos_ax_structured_identity", "status": "blocked", @@ -182,7 +184,9 @@ def qualify(*, client: Optional[MacWindowClient] = None) -> dict[str, Any]: if not client.input_trusted(): missing.append("accessibility") if missing: - return _blocked("required macOS permissions are not granted", missing, candidate) + return _blocked( + "required macOS permissions are not granted", missing, candidate + ) pids_before = _textedit_pids() root = Path(tempfile.mkdtemp(prefix="oa-axid-")) @@ -264,7 +268,9 @@ def qualify(*, client: Optional[MacWindowClient] = None) -> dict[str, Any]: # file's exact content (or a timeout, recording whatever was observed). observed: Optional[str] = None px = py = 0 - enum = ax.find_candidates(window.pid, window.title, text_view_locator, limit=4000) + enum = ax.find_candidates( + window.pid, window.title, text_view_locator, limit=4000 + ) deadline = time.monotonic() + 6.0 while time.monotonic() < deadline: enum = ax.find_candidates( @@ -303,9 +309,7 @@ def qualify(*, client: Optional[MacWindowClient] = None) -> dict[str, Any]: ) locator = backend.structural_locator_at(px, py) - report["structural_locator"] = ( - None if locator is None else locator.model_dump() - ) + report["structural_locator"] = None if locator is None else locator.model_dump() handle = backend.locate_structural(locator) if locator is not None else None report["structural_handle"] = ( @@ -364,8 +368,8 @@ def qualify(*, client: Optional[MacWindowClient] = None) -> dict[str, Any]: cleanup["terminate"] = _terminate(launched_pid) pids_after = _textedit_pids() harness = {launched_pid} if launched_pid is not None else set() - cleanup["unrelated_textedit_pids_preserved"] = ( - (pids_before - harness) == (pids_after - harness) + cleanup["unrelated_textedit_pids_preserved"] = (pids_before - harness) == ( + pids_after - harness ) cleanup["unrelated_textedit_pids_after"] = sorted(pids_after - harness) try: @@ -407,7 +411,9 @@ def _write_evidence(report: dict[str, Any]) -> dict[str, Any]: ), "refusal_contract_is_unit_covered_in": "tests/test_macos_structural.py", } - ADJUDICATION_PATH.write_text(json.dumps(adjudication, indent=2, sort_keys=True) + "\n") + ADJUDICATION_PATH.write_text( + json.dumps(adjudication, indent=2, sort_keys=True) + "\n" + ) return {"sha256": digest, "evidence": str(EVIDENCE_PATH)} diff --git a/packages/flow/tests/test_engine_generality_check.py b/packages/flow/tests/test_engine_generality_check.py index 9bf2f5a22d..b794d132fd 100644 --- a/packages/flow/tests/test_engine_generality_check.py +++ b/packages/flow/tests/test_engine_generality_check.py @@ -53,6 +53,35 @@ def test_a_marked_example_comment_passes(check, tmp_path: Path) -> None: assert check.scan(engine) == {"b.py": {"openemr": [1]}, "c.py": {"openemr": [1]}} +def test_a_name_inside_an_identifier_is_a_hit(check, tmp_path: Path) -> None: + engine = _engine( + tmp_path, + { + "a.py": "def _openemr_fixup():\n pass\n", + "b.py": "APP_OPENEMR = 1\n", + "c.py": "isOpenEMR = True\n", + "d.py": "def to_stedi_body():\n pass\n", + "e.py": "EPIC_BASE = 1\n", + "f.py": "row = 'tabLoanApplication'\n", + # A name only stops at a letter or digit beside it. + "g.py": "reopenemrs = 1\n", + }, + ) + assert check.scan(engine) == { + "a.py": {"openemr": [1]}, + "b.py": {"openemr": [1]}, + "c.py": {"openemr": [1]}, + "d.py": {"stedi": [1]}, + "e.py": {"epic": [1]}, + "f.py": {"frappe-table": [1]}, + } + + +def test_the_marker_must_open_the_comment(check, tmp_path: Path) -> None: + engine = _engine(tmp_path, {"a.py": "# OpenEMR-only hack # example: no\n"}) + assert check.scan(engine) == {"a.py": {"openemr": [1]}} + + def test_app_profiles_are_data(check, tmp_path: Path) -> None: engine = _engine(tmp_path, {"app_profiles/openemr.yaml": "app: openemr\n"}) assert check.scan(engine) == {} diff --git a/packages/flow/tests/test_runtime_never_imports_benchmark.py b/packages/flow/tests/test_runtime_never_imports_benchmark.py index d59f3083d7..7d052ea956 100644 --- a/packages/flow/tests/test_runtime_never_imports_benchmark.py +++ b/packages/flow/tests/test_runtime_never_imports_benchmark.py @@ -32,9 +32,20 @@ def _is_benchmark(module: str) -> bool: ) -def _imports_benchmark(tree: ast.AST) -> list[int]: +def _imports_benchmark(tree: ast.AST, *, lazy_ok: bool = False) -> list[int]: lines: list[int] = [] - for node in ast.walk(tree): + # An allowed module may import the harness only inside a function, where + # it runs on demand from a checkout. + nodes = ( + [ + n + for n in ast.iter_child_nodes(tree) + if isinstance(n, (ast.Import, ast.ImportFrom)) + ] + if lazy_ok + else ast.walk(tree) + ) + for node in nodes: if isinstance(node, ast.ImportFrom): module = node.module or "" if node.level == 0 and _is_benchmark(module): @@ -61,10 +72,10 @@ def test_product_modules_never_import_the_benchmark_package() -> None: offenders: list[str] = [] for path in sorted(ROOT.rglob("*.py")): rel = path.relative_to(ROOT).as_posix() - if rel in ALLOWED: - continue tree = ast.parse(path.read_text(encoding="utf-8")) - offenders += [f"{rel}:{n}" for n in _imports_benchmark(tree)] + offenders += [ + f"{rel}:{n}" for n in _imports_benchmark(tree, lazy_ok=rel in ALLOWED) + ] assert not offenders, ( "product code imports the benchmark harness (the wheel and a frozen " "build leave it out); move the shared name to " From 8a01ffe503960028859fddc84493cb5b69e6ad69 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 20:32:56 -0500 Subject: [PATCH 18/22] Undo an unrelated reformat of scripts/ --- .../build_showcase_encounter_loop_bundle.py | 4 +- packages/flow/scripts/demo_media.py | 128 ++++++------------ packages/flow/scripts/desktop/pn_db.py | 15 +- .../flow/scripts/desktop/session1_launch.py | 13 +- packages/flow/scripts/desktop/uia_arm.py | 28 ++-- packages/flow/scripts/frappe_lending_demo.py | 24 +++- packages/flow/scripts/make_demo_gif.py | 41 +++--- packages/flow/scripts/openemr_demo.py | 67 +++++---- packages/flow/scripts/openemr_param_depth.py | 52 +++---- .../flow/scripts/openimis_eligibility_demo.py | 1 - .../flow/scripts/qualify_macos_ax_identity.py | 24 ++-- 11 files changed, 164 insertions(+), 233 deletions(-) diff --git a/packages/flow/scripts/build_showcase_encounter_loop_bundle.py b/packages/flow/scripts/build_showcase_encounter_loop_bundle.py index 29e5372093..d829ac27d1 100644 --- a/packages/flow/scripts/build_showcase_encounter_loop_bundle.py +++ b/packages/flow/scripts/build_showcase_encounter_loop_bundle.py @@ -158,7 +158,9 @@ def main() -> int: _reset(LOOP_OUT) looped.save(LOOP_OUT) - print(f"Wrote {BODY_OUT} (program:false): {len(body.steps)} demonstrated steps.") + print( + f"Wrote {BODY_OUT} (program:false): {len(body.steps)} demonstrated steps." + ) print( f"Wrote {LOOP_OUT} (program:true): {len(records)} records over relation " f"'worklist', body = {len(body.steps)} demonstrated steps." diff --git a/packages/flow/scripts/demo_media.py b/packages/flow/scripts/demo_media.py index 72bd91f903..701167749c 100644 --- a/packages/flow/scripts/demo_media.py +++ b/packages/flow/scripts/demo_media.py @@ -247,8 +247,12 @@ def _build_master( for idx, ov in enumerate(overlays): inputs += ["-i", str(ov.png)] label = f"o{idx}" - enable = ":enable='lt(mod(t,1),0.55)'" if ov.blink else "" - parts.append(f"[{last}][{idx + 1}:v]overlay={ov.x}:{ov.y}{enable}[{label}]") + enable = ( + ":enable='lt(mod(t,1),0.55)'" if ov.blink else "" + ) + parts.append( + f"[{last}][{idx + 1}:v]overlay={ov.x}:{ov.y}{enable}[{label}]" + ) last = label filtergraph = ";".join(parts) _run( @@ -701,7 +705,9 @@ def _diff_card(patch: dict, heal: dict, base_frame: Path, card: Path) -> None: # Diff panel px, py, pw, ph = 80, 190, W - 160, 250 d.rounded_rectangle((px, py, px + pw, py + ph), radius=16, fill=CARD) - d.rounded_rectangle((px, py, px + pw, py + ph), radius=16, outline=LINE, width=1) + d.rounded_rectangle( + (px, py, px + pw, py + ph), radius=16, outline=LINE, width=1 + ) y = py + 26 changes = patch.get("changes", []) for ch in changes: @@ -727,9 +733,7 @@ def _diff_card(patch: dict, heal: dict, base_frame: Path, card: Path) -> None: continue d.text((px + 30, y), f"- {field_name}: {old!r}", font=f_mono, fill=DIFF_DEL) y += 42 - d.text( - (px + 30, y), f"+ {field_name}: {new!r}{ident}", font=f_mono, fill=DIFF_ADD - ) + d.text((px + 30, y), f"+ {field_name}: {new!r}{ident}", font=f_mono, fill=DIFF_ADD) y += 52 d.text( @@ -811,7 +815,8 @@ def build_heal(work: Path, out: Path, bundle: Path, *, headed: bool) -> Path: "-i", str(card), "-vf", - f"scale={OUT_W}:-2:flags=lanczos,fps={FPS},fade=t=in:st=0:d=0.3", + f"scale={OUT_W}:-2:flags=lanczos,fps={FPS}," + "fade=t=in:st=0:d=0.3", "-c:v", "libx264", "-profile:v", @@ -909,9 +914,7 @@ def _label( d.rounded_rectangle( (x, y, x + ww + 24, y + h), radius=10, outline=color + (255,), width=2 ) - d.rounded_rectangle( - (x + 10, y + 9, x + 10 + tw, y + h - 9), radius=6, fill=color + (255,) - ) + d.rounded_rectangle((x + 10, y + 9, x + 10 + tw, y + h - 9), radius=6, fill=color + (255,)) d.text((x + 20, y + 12), tag, font=f_tag, fill=(10, 13, 20, 255)) d.text((x + 22 + tw, y + 10), text, font=f_txt, fill=(232, 236, 244, 255)) @@ -922,12 +925,7 @@ def _anchor_box(size, region, color, alpha_line) -> Image.Image: x, y, w, h = region d.rectangle((x - 3, y - 3, x + w + 3, y + h + 3), outline=color + (255,), width=4) # corner ticks - for cx, cy in ( - (x - 3, y - 3), - (x + w + 3, y - 3), - (x - 3, y + h + 3), - (x + w + 3, y + h + 3), - ): + for cx, cy in ((x - 3, y - 3), (x + w + 3, y - 3), (x - 3, y + h + 3), (x + w + 3, y + h + 3)): d.rectangle((cx - 5, cy - 5, cx + 5, cy + 5), fill=color + (255,)) return layer @@ -941,7 +939,9 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None save_region = save["anchor"]["region"] note_region = note_click["anchor"]["region"] save_sel = save["anchor"]["structural"]["selector"] - assertion = next((e for e in save["expect"] if e["kind"] == "text_present"), None) + assertion = next( + (e for e in save["expect"] if e["kind"] == "text_present"), None + ) assert_text = assertion["text"] if assertion else "Encounter saved-" risk = save["risk"] @@ -954,41 +954,21 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None box_save = _anchor_box((1280, 800), save_region, ACCENT, 255) lbl_anchor = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) - _label( - lbl_anchor, - (save_region[0] + save_region[2] + 24, save_region[1] - 6), - "anchor", - save_sel, - ACCENT, - ) + _label(lbl_anchor, (save_region[0] + save_region[2] + 24, save_region[1] - 6), + "anchor", save_sel, ACCENT) lbl_assert = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) - _label( - lbl_assert, - (save_region[0] + save_region[2] + 24, save_region[1] + 54), - "assert", - f'text_present "{assert_text}"', - WARN, - ) + _label(lbl_assert, (save_region[0] + save_region[2] + 24, save_region[1] + 54), + "assert", f'text_present "{assert_text}"', WARN) # risk pill under the assert dr = ImageDraw.Draw(lbl_assert) rx, ry = save_region[0] + save_region[2] + 24, save_region[1] + 110 dr.rounded_rectangle((rx, ry, rx + 210, ry + 40), radius=20, fill=BAD + (255,)) - dr.text( - (rx + 18, ry + 8), - f"risk · {risk}", - font=_font(_FONT_SANS_B, 22), - fill=(10, 13, 20, 255), - ) + dr.text((rx + 18, ry + 8), f"risk · {risk}", font=_font(_FONT_SANS_B, 22), fill=(10, 13, 20, 255)) lbl_param = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) - _label( - lbl_param, - (note_region[0], note_region[1] - 58), - "param", - f'note = "{param_val}"', - GOOD, - ) + _label(lbl_param, (note_region[0], note_region[1] - 58), + "param", f'note = "{param_val}"', GOOD) # Code panel (real workflow.py step_010 excerpt), wrapped to fit. cp = tuple(save["anchor"]["click_point"]) @@ -1025,12 +1005,8 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None head = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) hd = ImageDraw.Draw(head) hd.rectangle((0, 0, 1280, 96), fill=(10, 13, 20, 210)) - hd.text( - (40, 24), - "The demonstration becomes an editable script", - font=title_f, - fill=(232, 236, 244, 255), - ) + hd.text((40, 24), "The demonstration becomes an editable script", + font=title_f, fill=(232, 236, 244, 255)) _paste_alpha(img, head, _ramp(t, 0.1, 0.5)) pulse = 0.55 + 0.45 * abs(((t * 1.6) % 1.0) - 0.5) * 2 @@ -1043,9 +1019,7 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None slide = _ramp(t, 4.6, 0.9) if slide > 0: shade = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) - ImageDraw.Draw(shade).rectangle( - (0, 96, 1280, 800), fill=(8, 11, 18, int(150 * slide)) - ) + ImageDraw.Draw(shade).rectangle((0, 96, 1280, 800), fill=(8, 11, 18, int(150 * slide))) _paste_alpha(img, shade, 1.0) px = int(1280 - (560 + 60) * slide) img.alpha_composite(panel, (px, 150)) @@ -1063,12 +1037,8 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None def _code_panel(lines: Sequence[str], width: int, height: int) -> Image.Image: panel = Image.new("RGBA", (width, height), (0, 0, 0, 0)) d = ImageDraw.Draw(panel) - d.rounded_rectangle( - (0, 0, width - 1, height - 1), radius=18, fill=(18, 22, 32, 255) - ) - d.rounded_rectangle( - (0, 0, width - 1, height - 1), radius=18, outline=LINE + (255,), width=1 - ) + d.rounded_rectangle((0, 0, width - 1, height - 1), radius=18, fill=(18, 22, 32, 255)) + d.rounded_rectangle((0, 0, width - 1, height - 1), radius=18, outline=LINE + (255,), width=1) # title bar dots for k, col in enumerate(((248, 113, 113), (251, 191, 36), (52, 211, 153))): d.ellipse((22 + k * 26, 20, 36 + k * 26, 34), fill=col + (255,)) @@ -1123,10 +1093,8 @@ def build_audit(work: Path, out: Path, run_dir: Path, *, headed: bool) -> None: with sync_playwright() as p: browser = p.chromium.launch(headless=not headed) context = browser.new_context( - viewport=view, - device_scale_factor=1, - record_video_dir=str(vid_dir), - record_video_size=view, + viewport=view, device_scale_factor=1, + record_video_dir=str(vid_dir), record_video_size=view, ) page = context.new_page() page.goto(file_url) @@ -1149,10 +1117,7 @@ def build_audit(work: Path, out: Path, run_dir: Path, *, headed: bool) -> None: _badge_png(badge, "illustrated run report", dot=ACCENT) master = work / "audit_master.mp4" _build_master( - raw, - master, - trim_start=0.5, - speed=1.0, + raw, master, trim_start=0.5, speed=1.0, overlays=[Overlay(badge, x="W-w-28", y="26")], ) # The audit clip is long and text-dense; bound the GIF fallback hard @@ -1166,19 +1131,9 @@ def build_audit(work: Path, out: Path, run_dir: Path, *, headed: bool) -> None: gif_max_s=3.0, ) # Prefer the crisp full-res top-of-report screenshot as the poster. - _run( - [ - "ffmpeg", - "-y", - "-i", - str(poster_png), - "-vf", - f"scale={OUT_W}:-2:flags=lanczos", - "-q:v", - "3", - str((out / "audit.jpg")), - ] - ) + _run(["ffmpeg", "-y", "-i", str(poster_png), "-vf", + f"scale={OUT_W}:-2:flags=lanczos", "-q:v", "3", + str((out / "audit.jpg"))]) sizes["jpg"] = (out / "audit.jpg").stat().st_size sizes["app"] = "mockmed" _update_manifest(out, "audit", sizes, "real") @@ -1271,7 +1226,9 @@ def main(argv: Optional[Sequence[str]] = None) -> int: # Everything else falls back to MockMed. No real PHI: the stock docker install # ships throwaway admin/pass creds and zero patient data. -OPENEMR_LOGIN = "https://localhost:9390/interface/login/login.php?site=default" +OPENEMR_LOGIN = ( + "https://localhost:9390/interface/login/login.php?site=default" +) def capture_record_openemr(work: Path, *, headed: bool) -> Optional[Path]: @@ -1333,7 +1290,9 @@ def center(getter) -> tuple[int, int]: page.wait_for_timeout(1200) # Open the Patient workflow via the real top nav. - rec.click(*center(lambda: page.get_by_text("Patient", exact=True).first)) + rec.click( + *center(lambda: page.get_by_text("Patient", exact=True).first) + ) page.wait_for_timeout(700) ns = page.get_by_text("New/Search", exact=False) if ns.count(): @@ -1453,10 +1412,7 @@ def build_run_openemr(work: Path, out: Path, *, headed: bool) -> bool: # would double up the indicator). See openadapt-web ReplayHero + PR #246. master = work / "run_openemr_master.mp4" _build_master( - raw, - master, - trim_start=0.6, - speed=1.4, + raw, master, trim_start=0.6, speed=1.4, overlays=[], ) sizes = _finalize(master, out / "run_openemr", poster_at=1.0) diff --git a/packages/flow/scripts/desktop/pn_db.py b/packages/flow/scripts/desktop/pn_db.py index 68c04e17be..434f61c000 100644 --- a/packages/flow/scripts/desktop/pn_db.py +++ b/packages/flow/scripts/desktop/pn_db.py @@ -86,13 +86,8 @@ def seed(drift: str = "none") -> None: def _patient_dict(r: sqlite3.Row) -> dict: - return { - "id": r["id"], - "first": r["first"], - "last": r["last"], - "dob": r["dob"], - "note": r["note"] or "", - } + return {"id": r["id"], "first": r["first"], "last": r["last"], + "dob": r["dob"], "note": r["note"] or ""} def list_patients(filt: str = "") -> None: @@ -130,10 +125,8 @@ def get(pid: int) -> None: def dump_all() -> None: con = _conn() - rows = [ - _patient_dict(r) - for r in con.execute("SELECT * FROM patients ORDER BY id").fetchall() - ] + rows = [_patient_dict(r) for r in + con.execute("SELECT * FROM patients ORDER BY id").fetchall()] con.close() print(json.dumps(rows)) diff --git a/packages/flow/scripts/desktop/session1_launch.py b/packages/flow/scripts/desktop/session1_launch.py index 5c6d33d52d..13934b794d 100644 --- a/packages/flow/scripts/desktop/session1_launch.py +++ b/packages/flow/scripts/desktop/session1_launch.py @@ -168,20 +168,15 @@ def main() -> None: import os if len(sys.argv) < 2: - print("usage: session1_launch.py [args...]", file=sys.stderr) + print("usage: session1_launch.py [args...]", + file=sys.stderr) sys.exit(2) script = os.path.abspath(sys.argv[1]) extra = list(sys.argv[2:]) if script.lower().endswith(".ps1"): # A WinForms/PowerShell UI target: run STA so WinForms works. - parts = [ - "powershell.exe", - "-STA", - "-ExecutionPolicy", - "Bypass", - "-File", - f'"{script}"', - ] + extra + parts = ["powershell.exe", "-STA", "-ExecutionPolicy", "Bypass", + "-File", f'"{script}"'] + extra else: pythonw = os.path.join(os.path.dirname(sys.executable), "pythonw.exe") if not os.path.exists(pythonw): diff --git a/packages/flow/scripts/desktop/uia_arm.py b/packages/flow/scripts/desktop/uia_arm.py index 52bd4f06c3..3123032d4e 100644 --- a/packages/flow/scripts/desktop/uia_arm.py +++ b/packages/flow/scripts/desktop/uia_arm.py @@ -35,13 +35,8 @@ import time from typing import Optional -WORKFLOW_TARGET_IDS = [ - "searchBox", - "searchButton", - "patientGrid", - "noteBox", - "saveButton", -] +WORKFLOW_TARGET_IDS = ["searchBox", "searchButton", "patientGrid", + "noteBox", "saveButton"] # The patient ROW is the identity-critical target and has NO usable id. IDENTITY_TARGET = "patient_row" @@ -154,18 +149,15 @@ def run(search: str, first: str, last: str, note: str, mode: str) -> dict: if rows: _select_row(rows[0].get("_cell")) selected_index = rows[0]["row_index"] - selected_name = ( - f"{rows[0].get('first', '')} {rows[0].get('last', '')}".strip() - ) + selected_name = f"{rows[0].get('first','')} {rows[0].get('last','')}".strip() else: # identity: exact match on rendered first AND last name text for row in rows: - if ( - row.get("first", "").strip().lower() == first.strip().lower() - and row.get("last", "").strip().lower() == last.strip().lower() - ): + if (row.get("first", "").strip().lower() == first.strip().lower() + and row.get("last", "").strip().lower() + == last.strip().lower()): _select_row(row.get("_cell")) selected_index = row["row_index"] - selected_name = f"{row.get('first', '')} {row.get('last', '')}".strip() + selected_name = f"{row.get('first','')} {row.get('last','')}".strip() break # 3. note + 4. save @@ -190,14 +182,16 @@ def main() -> None: ap.add_argument("--first", required=True) ap.add_argument("--last", required=True) ap.add_argument("--note", required=True) - ap.add_argument("--mode", choices=["positional", "identity"], default="identity") + ap.add_argument("--mode", choices=["positional", "identity"], + default="identity") args = ap.parse_args() try: out = run(args.search, args.first, args.last, args.note, args.mode) except Exception as e: # noqa: BLE001 import traceback - out = {"status": "error", "error": str(e), "trace": traceback.format_exc()} + out = {"status": "error", "error": str(e), + "trace": traceback.format_exc()} print(json.dumps(out)) diff --git a/packages/flow/scripts/frappe_lending_demo.py b/packages/flow/scripts/frappe_lending_demo.py index a174d83c43..13877e4925 100644 --- a/packages/flow/scripts/frappe_lending_demo.py +++ b/packages/flow/scripts/frappe_lending_demo.py @@ -347,7 +347,9 @@ def record(fixture: FrappeFixture, out_dir: Path, *, headed: bool) -> Path: _preauthenticate_browser(backend, fixture) page = backend.page before_counts = fixture.table_counts() - before_non_target_loan_sha256 = fixture.non_target_loan_applications_sha256() + before_non_target_loan_sha256 = ( + fixture.non_target_loan_applications_sha256() + ) recorder = Recorder( backend, out_dir, @@ -449,7 +451,9 @@ def link_field(field: str, value: str, *, param: str) -> None: "primary_outcome": verdict.primary_outcome.value, "rest_records_sha256": _safe_records_sha256(evidence.rest_records), "db_records_sha256": _safe_records_sha256(evidence.db_records), - "before_non_target_loan_sha256": (evidence.before_non_target_loan_sha256), + "before_non_target_loan_sha256": ( + evidence.before_non_target_loan_sha256 + ), "after_non_target_loan_sha256": evidence.after_non_target_loan_sha256, "all_table_deltas": evidence.all_table_deltas, "delta_violations": evidence.delta_violations, @@ -581,7 +585,9 @@ def _capture_post_evidence( errors.append("pre-write non-target Loan Application digest unavailable") else: try: - after_non_target_loan_sha256 = fixture.non_target_loan_applications_sha256() + after_non_target_loan_sha256 = ( + fixture.non_target_loan_applications_sha256() + ) if after_non_target_loan_sha256 != before_non_target_loan_sha256: violations.append("tabLoan Application:non-target-content-changed") except Exception as exc: # noqa: BLE001 - unreadable is not clean @@ -820,7 +826,9 @@ def _persist_evidence( "before_table_counts": evidence.before_table_counts, "after_table_counts": evidence.after_table_counts, "all_table_deltas": evidence.all_table_deltas, - "before_non_target_loan_sha256": (evidence.before_non_target_loan_sha256), + "before_non_target_loan_sha256": ( + evidence.before_non_target_loan_sha256 + ), "after_non_target_loan_sha256": evidence.after_non_target_loan_sha256, "delta_violations": evidence.delta_violations, "oracle_errors": evidence.errors, @@ -1056,7 +1064,9 @@ def run_compiled_trial( # Authentication/form navigation are declared unmeasured setup; take # the collateral-delta baseline only after they finish. before_counts = fixture.table_counts() - before_non_target_loan_sha256 = fixture.non_target_loan_applications_sha256() + before_non_target_loan_sha256 = ( + fixture.non_target_loan_applications_sha256() + ) start = time.monotonic() report = Replayer(backend, effect_verifier=oracle.verifier).run( Workflow.load(bundle_dir), @@ -1145,7 +1155,9 @@ def run_agent_trial( _preauthenticate_browser(backend, fixture) _apply_condition(backend, condition) before_counts = fixture.table_counts() - before_non_target_loan_sha256 = fixture.non_target_loan_applications_sha256() + before_non_target_loan_sha256 = ( + fixture.non_target_loan_applications_sha256() + ) start = time.monotonic() result = agent_baseline.run_agent( backend, diff --git a/packages/flow/scripts/make_demo_gif.py b/packages/flow/scripts/make_demo_gif.py index 92dbc7139c..337979edac 100644 --- a/packages/flow/scripts/make_demo_gif.py +++ b/packages/flow/scripts/make_demo_gif.py @@ -75,8 +75,12 @@ def _panel(draw, img, shot: Path, x: int, label: str, label_color): if shot.is_file(): panel = Image.open(shot).convert("RGB").resize((PANEL_W, PANEL_H)) img.paste(panel, (x, y)) - draw.rectangle([x - 1, y - 1, x + PANEL_W, y + PANEL_H], outline=(60, 60, 90)) - _text_center(draw, (x + PANEL_W / 2, y + PANEL_H + 10), label, F_BODY, label_color) + draw.rectangle( + [x - 1, y - 1, x + PANEL_W, y + PANEL_H], outline=(60, 60, 90) + ) + _text_center( + draw, (x + PANEL_W / 2, y + PANEL_H + 10), label, F_BODY, label_color + ) _PLAIN_RUNG = { @@ -130,43 +134,30 @@ def build( continue img, draw = _frame() _text_center( - draw, - (CANVAS_W / 2, 8), + draw, (CANVAS_W / 2, 8), "Left: the theme it was recorded on. Right: a dark theme it never saw.", - F_SMALL, - DIM, + F_SMALL, DIM, ) _text_center( - draw, - (CANVAS_W / 2, 32), + draw, (CANVAS_W / 2, 32), f"Step {number} of {len(results)}: {_plain_intent(res_b['intent'])}", - F_BODY, - FG, + F_BODY, FG, ) cap_b, col_b = _step_caption(res_b, healed=False) cap_d, col_d = _step_caption(res_d, healed=res_d.get("heal") is not None) _panel( - draw, - img, - baseline_dir / res_b["after_png"], - MARGIN, - f"Recorded theme: {cap_b}", - col_b, + draw, img, baseline_dir / res_b["after_png"], MARGIN, + f"Recorded theme: {cap_b}", col_b, ) _panel( - draw, - img, - drift_dir / res_d["after_png"], + draw, img, drift_dir / res_d["after_png"], MARGIN + PANEL_W + GUTTER, - f"Dark theme: {cap_d}", - col_d, + f"Dark theme: {cap_d}", col_d, ) _text_center( - draw, - (CANVAS_W / 2, CANVAS_H - FOOTER_H + 12), + draw, (CANVAS_W / 2, CANVAS_H - FOOTER_H + 12), "One recording, replayed on two themes. No model calls.", - F_SMALL, - DIM, + F_SMALL, DIM, ) frames.append(img) durations.append(1400 if res_d.get("heal") else 1000) diff --git a/packages/flow/scripts/openemr_demo.py b/packages/flow/scripts/openemr_demo.py index 2a6a49bbd8..b9a75aee21 100644 --- a/packages/flow/scripts/openemr_demo.py +++ b/packages/flow/scripts/openemr_demo.py @@ -49,7 +49,9 @@ DEMO_PASS = "pass" PATIENT_SEARCH = "Phil" # matches demo patient "Belford, Phil" only -RECORDED_NOTE = "Reviewed hypertension follow-up; BP within target range today." +RECORDED_NOTE = ( + "Reviewed hypertension follow-up; BP within target range today." +) # Distinct per-run values so replay success proves parameter substitution, # not replay of the baked-in literal. @@ -61,7 +63,8 @@ "Replay run 5: home BP log reviewed, readings stable.", ] -SETTLE = dict(settle_timeout_s=10.0, settle_stable_frames=3, settle_interval_s=0.3) +SETTLE = dict(settle_timeout_s=10.0, settle_stable_frames=3, + settle_interval_s=0.3) def _center(locator) -> tuple[int, int]: @@ -70,7 +73,8 @@ def _center(locator) -> tuple[int, int]: box = locator.bounding_box() if box is None: raise RuntimeError(f"no bounding box for {locator}") - return (int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2)) + return (int(box["x"] + box["width"] / 2), + int(box["y"] + box["height"] / 2)) def _frame(page, url_fragment: str): @@ -149,14 +153,14 @@ def record(headed: bool = False, recording_dir: Path = RECORDING_DIR) -> Path: recorder.click(*_center(pencil)) page.wait_for_timeout(3000) pnotes = _frame(page, "pnotes_full") - recorder.click( - *_center(pnotes.locator("button, a").filter(has_text="Add").first) - ) + recorder.click(*_center(pnotes.locator("button, a").filter( + has_text="Add").first)) page.wait_for_timeout(2000) # The add-note form opens as a second nested pnotes_full iframe. recorder.click(*_center(_locate_any_frame(page, "textarea#note"))) recorder.type_text(RECORDED_NOTE, param="note") - recorder.click(*_center(_locate_any_frame(page, "text=Save as new message"))) + recorder.click(*_center(_locate_any_frame( + page, "text=Save as new message"))) page.wait_for_timeout(3000) return recorder.finish() @@ -228,44 +232,35 @@ def replay(n: int = 5) -> dict: (run_dir / "final.png").write_bytes(final_png) note_visible = _note_visible(final_png, note) except Exception as exc: # noqa: BLE001 - record and continue - summary["runs"].append( - { - "run": i + 1, - "success": False, - "error": f"{type(exc).__name__}: {exc}", - } - ) + summary["runs"].append({ + "run": i + 1, "success": False, "error": f"{type(exc).__name__}: {exc}", + }) print(f"run {i + 1}: EXCEPTION {exc}") continue finally: close() failed = [r for r in report.results if not r.ok] - summary["runs"].append( - { - "run": i + 1, - "success": report.success, - "note_visible": note_visible, - "steps_ok": sum(1 for r in report.results if r.ok), - "steps_total": len(workflow.steps), - "rung_counts": report.rung_counts, - "heal_count": report.heal_count, - "total_s": round(elapsed, 1), - "first_failure": ( - {"step": failed[0].step_id, "error": failed[0].error} - if failed - else None - ), - } - ) + summary["runs"].append({ + "run": i + 1, + "success": report.success, + "note_visible": note_visible, + "steps_ok": sum(1 for r in report.results if r.ok), + "steps_total": len(workflow.steps), + "rung_counts": report.rung_counts, + "heal_count": report.heal_count, + "total_s": round(elapsed, 1), + "first_failure": ( + {"step": failed[0].step_id, "error": failed[0].error} + if failed else None + ), + }) try: render_run_report(run_dir) except Exception as exc: # noqa: BLE001 print(f"run {i + 1}: report render failed: {exc}") - print( - f"run {i + 1}: success={report.success} " - f"note_visible={note_visible} rungs={report.rung_counts} " - f"heals={report.heal_count} {elapsed:.0f}s" - ) + print(f"run {i + 1}: success={report.success} " + f"note_visible={note_visible} rungs={report.rung_counts} " + f"heals={report.heal_count} {elapsed:.0f}s") ok = sum(1 for r in summary["runs"] if r.get("success")) verified = sum( 1 for r in summary["runs"] if r.get("success") and r.get("note_visible") diff --git a/packages/flow/scripts/openemr_param_depth.py b/packages/flow/scripts/openemr_param_depth.py index bafc965f1f..5c77b4c57f 100644 --- a/packages/flow/scripts/openemr_param_depth.py +++ b/packages/flow/scripts/openemr_param_depth.py @@ -66,7 +66,9 @@ RUNS_DIR = WORK / "runs" RECORDED_PATIENT = "Phil" # demonstrated value; unique match "Belford, Phil" -RECORDED_NOTE = "Validation baseline: reviewed home readings, plan unchanged today." +RECORDED_NOTE = ( + "Validation baseline: reviewed home readings, plan unchanged today." +) PACING_S = 30.0 # public-demo courtesy between browser sessions @@ -97,7 +99,9 @@ def record() -> Path: page.wait_for_timeout(3000) finder = _frame(page, "dynamic_finder") - recorder.click(*_center(finder.locator("a", has_text=RECORDED_PATIENT).first)) + recorder.click( + *_center(finder.locator("a", has_text=RECORDED_PATIENT).first) + ) page.wait_for_timeout(4000) demographics = _frame(page, "demographics.php") @@ -114,13 +118,14 @@ def record() -> Path: recorder.click(*_center(pencil)) page.wait_for_timeout(3000) pnotes = _frame(page, "pnotes_full") - recorder.click( - *_center(pnotes.locator("button, a").filter(has_text="Add").first) - ) + recorder.click(*_center( + pnotes.locator("button, a").filter(has_text="Add").first + )) page.wait_for_timeout(2000) recorder.click(*_center(_locate_any_frame(page, "textarea#note"))) recorder.type_text(RECORDED_NOTE, param="note") - recorder.click(*_center(_locate_any_frame(page, "text=Save as new message"))) + recorder.click(*_center(_locate_any_frame( + page, "text=Save as new message"))) page.wait_for_timeout(3000) return recorder.finish() finally: @@ -132,7 +137,9 @@ def compile_bundle() -> None: if BUNDLE_DIR.exists(): shutil.rmtree(BUNDLE_DIR) - wf = compile_recording(RECORDING_DIR, BUNDLE_DIR, name="openemr-param-depth") + wf = compile_recording( + RECORDING_DIR, BUNDLE_DIR, name="openemr-param-depth" + ) print(f"compiled {len(wf.steps)} steps -> {BUNDLE_DIR}") for step in wf.steps: texts = [pc.text for pc in step.expect if pc.text] @@ -165,7 +172,9 @@ def _replay_one(label: str, params: dict[str, str], url: str) -> dict: note = params["note"] note_check = verify_note_saved(final_png, note) - final_text = normalize_text(" ".join(line.text for line in ocr(final_png))) + final_text = normalize_text( + " ".join(line.text for line in ocr(final_png)) + ) failed = [r for r in report.results if not r.ok] row = { "label": label, @@ -183,8 +192,7 @@ def _replay_one(label: str, params: dict[str, str], url: str) -> dict: "underwood_on_final": "underwood" in final_text, "first_failure": ( {"step": failed[0].step_id, "error": (failed[0].error or "")[:300]} - if failed - else None + if failed else None ), } print(json.dumps(row)) @@ -195,22 +203,14 @@ def replay() -> None: RUNS_DIR.mkdir(parents=True, exist_ok=True) rows = [] plan = [ - ( - "phil-control", - { - "patient": "Phil", - "note": "Param depth control: confirmed refill pickup scheduled.", - }, - DEMO_URL, - ), - ( - "susan-drift", - { - "patient": "Susan", - "note": "Param depth drift: dietary counseling summary delivered.", - }, - DEMO_URL, - ), + ("phil-control", { + "patient": "Phil", + "note": "Param depth control: confirmed refill pickup scheduled.", + }, DEMO_URL), + ("susan-drift", { + "patient": "Susan", + "note": "Param depth drift: dietary counseling summary delivered.", + }, DEMO_URL), ] for i, (label, params, url) in enumerate(plan): if i: diff --git a/packages/flow/scripts/openimis_eligibility_demo.py b/packages/flow/scripts/openimis_eligibility_demo.py index d2dbd15b77..d1241f57ca 100644 --- a/packages/flow/scripts/openimis_eligibility_demo.py +++ b/packages/flow/scripts/openimis_eligibility_demo.py @@ -212,7 +212,6 @@ def structured_text_at(self, x: int, y: int) -> str | None: context["insurance_no"] = dialogs[0][0] return json.dumps(context, separators=(",", ":")) - def eligibility_effects() -> list[Any]: """The eligibility outcome bound to the run's ``insurance_no``. diff --git a/packages/flow/scripts/qualify_macos_ax_identity.py b/packages/flow/scripts/qualify_macos_ax_identity.py index a575c9f8fe..1eb902da81 100644 --- a/packages/flow/scripts/qualify_macos_ax_identity.py +++ b/packages/flow/scripts/qualify_macos_ax_identity.py @@ -161,9 +161,7 @@ def _terminate(pid: int) -> dict[str, Any]: return receipt -def _blocked( - reason: str, missing: list[str], candidate: dict[str, Any] -) -> dict[str, Any]: +def _blocked(reason: str, missing: list[str], candidate: dict[str, Any]) -> dict[str, Any]: return { "task": "macos_ax_structured_identity", "status": "blocked", @@ -184,9 +182,7 @@ def qualify(*, client: Optional[MacWindowClient] = None) -> dict[str, Any]: if not client.input_trusted(): missing.append("accessibility") if missing: - return _blocked( - "required macOS permissions are not granted", missing, candidate - ) + return _blocked("required macOS permissions are not granted", missing, candidate) pids_before = _textedit_pids() root = Path(tempfile.mkdtemp(prefix="oa-axid-")) @@ -268,9 +264,7 @@ def qualify(*, client: Optional[MacWindowClient] = None) -> dict[str, Any]: # file's exact content (or a timeout, recording whatever was observed). observed: Optional[str] = None px = py = 0 - enum = ax.find_candidates( - window.pid, window.title, text_view_locator, limit=4000 - ) + enum = ax.find_candidates(window.pid, window.title, text_view_locator, limit=4000) deadline = time.monotonic() + 6.0 while time.monotonic() < deadline: enum = ax.find_candidates( @@ -309,7 +303,9 @@ def qualify(*, client: Optional[MacWindowClient] = None) -> dict[str, Any]: ) locator = backend.structural_locator_at(px, py) - report["structural_locator"] = None if locator is None else locator.model_dump() + report["structural_locator"] = ( + None if locator is None else locator.model_dump() + ) handle = backend.locate_structural(locator) if locator is not None else None report["structural_handle"] = ( @@ -368,8 +364,8 @@ def qualify(*, client: Optional[MacWindowClient] = None) -> dict[str, Any]: cleanup["terminate"] = _terminate(launched_pid) pids_after = _textedit_pids() harness = {launched_pid} if launched_pid is not None else set() - cleanup["unrelated_textedit_pids_preserved"] = (pids_before - harness) == ( - pids_after - harness + cleanup["unrelated_textedit_pids_preserved"] = ( + (pids_before - harness) == (pids_after - harness) ) cleanup["unrelated_textedit_pids_after"] = sorted(pids_after - harness) try: @@ -411,9 +407,7 @@ def _write_evidence(report: dict[str, Any]) -> dict[str, Any]: ), "refusal_contract_is_unit_covered_in": "tests/test_macos_structural.py", } - ADJUDICATION_PATH.write_text( - json.dumps(adjudication, indent=2, sort_keys=True) + "\n" - ) + ADJUDICATION_PATH.write_text(json.dumps(adjudication, indent=2, sort_keys=True) + "\n") return {"sha256": digest, "evidence": str(EVIDENCE_PATH)} From 23b46d65de63fedc8961296843c73b346ebeb471 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 20:34:13 -0500 Subject: [PATCH 19/22] Restore #1741's packaging excludes, lazy-import guard and sys.path.append lost in the main merge --- packages/flow/CHANGELOG.md | 4 ++-- packages/flow/docs/GENERALITY_AUDIT.md | 17 ++++++++------- packages/flow/openadapt_flow/__main__.py | 3 ++- packages/flow/pyproject.toml | 7 ++++--- .../test_runtime_never_imports_benchmark.py | 21 ++++++++++++++----- 5 files changed, 33 insertions(+), 19 deletions(-) diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md index 608ad9f850..b46fcc2b77 100644 --- a/packages/flow/CHANGELOG.md +++ b/packages/flow/CHANGELOG.md @@ -21,8 +21,6 @@ - The wheel no longer ships the benchmark harnesses. `openadapt_flow.benchmark` and `openadapt_flow.instrument` moved to `packages/flow/benchmark/harness/` in the source tree (import `benchmark.harness`); `openadapt-flow benchmark` runs from a source checkout and says so from a wheel. The engine audit for application-specific code is in `docs/GENERALITY_AUDIT.md`. Code that imported `openadapt_flow.benchmark` or `openadapt_flow.instrument` from an installed wheel imports `benchmark.harness` from a source checkout instead. -- The wheel no longer ships test fixtures and demo harnesses: the Harness² integration, the MockLoan app and its recorder, `guard_demo`, the MockMed teach-scope adapter, the synthetic drift stream (the `StructuralDiffInducer` it used now lives in `openadapt_flow.learning.induce`) and the task-ledger simulation moved to `benchmark/harness/` or beside their tests, the reward bundle helpers moved to `openadapt_flow.reward.bundle`, and the `harness2` extra is gone. - ### Features - Teach authorizations (`openadapt.teach-authorization/v1`). A person signs, with an approver key on the signed approver roster, what an agent may do when it teaches or repairs a program: the task, origins, inputs, the one write and forbidden actions and controls. `openadapt-flow teach-authorization create|sign|verify`; `--teach-authorization` on `compile-agent` and `openadapt-agent serve --teach`. Design and the upgrade notes: `docs/TEACH_AUTHORIZATION.md`. @@ -39,6 +37,8 @@ - A covered-band dismiss step takes its dialog box from the window the backend proved covers the band, before any pixel search, so a native message box that the accessibility tree doesn't label as a dialog can be dismissed. Without a proved window the step is refused as before. +- The wheel no longer ships test fixtures and demo harnesses: the Harness² integration, the MockLoan app and its recorder, `guard_demo`, the MockMed teach-scope adapter, the synthetic drift stream (the `StructuralDiffInducer` it used now lives in `openadapt_flow.learning.induce`) and the task-ledger simulation moved to `benchmark/harness/` or beside their tests, the reward bundle helpers moved to `openadapt_flow.reward.bundle`, and the `harness2` extra is gone. + - Self-repair on a covered identity band dismisses only the proved dialog: the covering window must span the band from where the read stops to its right edge at full height, the dismiss click must land in that same window, and only a dialog whose every word is a notice word (no question) may be dismissed ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). - Self-repair can take a stop where the app's own dialog covers the next step's identity band: when the backend proves the covering window belongs to the window the last verified step was in, and the part of the band read still matches the record, the dialog is dismissed after the previous click and the step re-checks its identity in full. Another record behind the dialog stays an identity halt. The runtime lets a backend remember a verified record's window (`record_verified_window`); macOS answers `owned_window_over` from the window's proved sheet ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md index ece04e3645..8f1982197e 100644 --- a/packages/flow/docs/GENERALITY_AUDIT.md +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -6,8 +6,9 @@ Audited 2026-11-03 on `main` at `213f3d9fe`. Scope: everything under The rule: the engine has no code written for one application. What one application needs lives in data (`app_profiles/*.yaml`, generated contracts, deployment configs) or in test fixtures and benchmark harnesses outside the -shipped package. A CI check (`scripts/check_engine_generality.py`) keeps it -that way; [CONTRIBUTING.md](../CONTRIBUTING.md) states the rule. +shipped package. The CI check that keeps it that way, +`scripts/check_engine_generality.py`, lands in the pull request that follows +this audit, together with the rule in [CONTRIBUTING.md](../CONTRIBUTING.md). ## Method @@ -37,7 +38,7 @@ that way; [CONTRIBUTING.md](../CONTRIBUTING.md) states the rule. | --- | --- | --- | | (a) comment or docstring example | 118 | — | | (b) data loaded generically | 4 | 2 (`app_profiles/openemr.yaml`, the `clinical-write` policy pack as a default) | -| (c) fixture, harness or demo in the package | 35 rows, plus 27 files of `benchmark/` | 19 | +| (c) fixture, harness or demo in the package | 35 rows, plus the 25 Python files of `benchmark/` and 2 of `instrument/` | 19 | | (d) special-casing in engine logic | 27 | 15 | Rows group consecutive lines of one hit. The full table is at the end. @@ -62,7 +63,7 @@ motivating application. | D10 | `__main__.py:2838-2934`, `:9305-9319`; `cli_variants.py:131,173-182` | `replay` and the variant probe fall back to the bundled MockMed when no URL is given; `--drift` turns off structural grounding | MockMed | Target URL required outside the demo commands | planned | | D11 | `qualification_oracle_gate.py` | `qualify accept` tests every workflow's oracle against MockMed (banner text, `/api/encounter`, `/api/db`) | MockMed | Build the fault fixture from the bundle's own effect binding | planned | | D12 | `execute/dispatch.py:111-188` | The Execute service's live runner always runs the MockMed tutorial workflow | MockMed | Replay through the admission's deployment config; MockMed only for synthetic admissions | planned | -| D13 | `compiler/compile.py:151`, `runtime/replayer.py:394`, `runtime/identity.py:108` | Thresholds measured on MockMed only | MockMed | Generic logic, one-app numbers. Recalibrate on two or more apps | follow-up issue | +| D13 | `compiler/compile.py:148`, `runtime/replayer.py:394`, `runtime/identity.py:108` | Thresholds measured on MockMed only | MockMed | Generic logic, one-app numbers. Recalibrate on two or more apps | follow-up issue | | D14 | `runtime/identity.py:275-302` | Hand-written list of non-identifying words drawn from EMR list screens | MockMed, EMRs | Derive from the recording (tokens shared by sibling rows) | follow-up issue | | D15 | `eligibility/client.py:37-41`, `:1091-1110` | One vendor's eligibility API (URL, key name, body) in the engine | Stedi | Move behind a connector interface | follow-up issue | @@ -70,7 +71,7 @@ motivating application. | Where | What | Action | Status | | --- | --- | --- | --- | -| `benchmark/` (27 Python files: `run_benchmark`, `openemr_benchmark`, `openemr_local`, `frappe_lending`, `hybrid_benchmark`, `dom_arm`, `silent_wrong_action`, `desktop_benchmark`, `verify`, `agent_baseline`, `reliability`, `chart_fonts`, `effectbench/`) | Benchmark harnesses shipped in the wheel | Moved to `packages/flow/benchmark/harness/` (import `benchmark.harness.*`). `openadapt-flow benchmark` loads it from a source checkout and says so from a wheel | done | +| `benchmark/` (25 Python files: `run_benchmark`, `openemr_benchmark`, `openemr_local`, `frappe_lending`, `hybrid_benchmark`, `dom_arm`, `silent_wrong_action`, `desktop_benchmark`, `verify`, `agent_baseline`, `reliability`, `chart_fonts`, `effectbench/`) | Benchmark harnesses shipped in the wheel | Moved to `packages/flow/benchmark/harness/` (import `benchmark.harness.*`). `openadapt-flow benchmark` loads it from a source checkout and says so from a wheel | done | | `instrument/` | Competitor-drift harness; imported the benchmark at module load, so it was already broken in a wheel | Moved to `packages/flow/benchmark/harness/instrument/` | done | | `integrations/harness2/` | Harness2 research harness on MockMed | Moved to `packages/flow/benchmark/harness/harness2/`; the `harness2` extra is gone (install `[browser]`) | done | | `mockloan/`, `demo_driver.record_disbursement_demo` | Lending demo app no product command uses | Moved to `packages/flow/benchmark/harness/mockloan/` (recorder in `mockloan/demo.py`) | done | @@ -175,7 +176,7 @@ Paths are relative to `packages/flow/openadapt_flow/` as of the audited commit. | `fleet/simulator.py:1-` | (c) | synthetic payer portal | in-package simulator fixture; move to benchmark/examples | | `guard_demo.py:1-296` | (c) | MockMed | `python -m openadapt_flow.guard_demo`: fake agent posts referral to MockMed with chart_pages READER creds; not a CLI subcommand. Move to packages/flow/examples or benchmark; importers: packages/agent/tests (1), packages/flow/tests (6). | | `instrument/ (__init__.py, competitor_drift.py)` | (c) | MockMed | MockMed competitor-drift harness. Moved to packages/flow/benchmark/harness/instrument. | -| `integrations/harness2/ (all: tasks.py, apps.py, agent.py, adapter.py, traces.py, proposers.py, scorer.py, llm.py, pixels.py, rollout.py, selection.py, __init__.py)` | (c) | MockMed | Whole subpackage is a research harness for Harness2 on MockMed: MOCKMED_PATIENT/PATIENTS (tasks.py:71-148), MockMedTarget (apps.py:50-105, /api/db at :70), regex for MockMed task text and 'triage'/'New Encounter'/'encountersaved' (agent.py:221-324), MockMed banner regex Name — MRN — DOB (traces.py:55-57,128-138), clinical prompts (adapter.py:93-120,429-445; proposers.py:50-63). Move to packages/flow/benchmark/harness2; importers: benchmark/harness2_scorer/run.py, tests/test_harness2_{adapter,pixels,traces}.py, pyproject extra 'harness2', .github/workflows/packages.yml:451. | +| `integrations/harness2/ (all: tasks.py, apps.py, agent.py, adapter.py, traces.py, proposers.py, scorer.py, llm.py, pixels.py, rollout.py, selection.py, __init__.py)` | (c) | MockMed | Whole subpackage is a research harness for Harness2 on MockMed: MOCKMED_PATIENT/PATIENTS (tasks.py:71-148), MockMedTarget (apps.py:50-105, /api/db at :70), regex for MockMed task text and 'triage'/'New Encounter'/'encountersaved' (agent.py:221-324), MockMed banner regex Name — MRN — DOB (traces.py:55-57,128-138), clinical prompts (adapter.py:93-120,429-445; proposers.py:50-63). Move to packages/flow/benchmark/harness/harness2; importers: benchmark/harness2_scorer/run.py, tests/test_harness2_{adapter,pixels,traces}.py, pyproject extra 'harness2', .github/workflows/packages.yml:451. | | `learning/synth_stream.py:1-245` | (c) | MockMed | synthetic MockMed add-patient-note stream (intents, 'Consent Required', 'Belford, Phil DOB'). Move to benchmark/ or tests/fixtures. | | `learning/synth_stream.py:246-525` | (c) | (generic) | StructuralDiffInducer is generic but lives in the MockMed harness module; imported by learning/teach.py:96, learning/branch_teach.py:76, benchmark/openemr_e2e/harness.py, 4 tests; pyproject mypy list (line 611). Move to learning/induce.py first, then move the stream out. | | `mockloan/ (server.py, fault_server.py, static/)` | (c) | MockLoan | Own synthetic lending app shipped in wheel but used by no product command; only demo_driver.record_disbursement_demo, benchmark/lending_fault_model, benchmark/undo_proposal, tests (test_mockloan, test_lending_swer, e2e/test_lending_fault_model), scripts (1). Move to packages/flow/benchmark/apps/mockloan. | @@ -304,8 +305,8 @@ Paths are relative to `packages/flow/openadapt_flow/` as of the audited commit. | `validation/dense_surface.py:1271` | (a) | OpenEMR | comment/docstring example | | `validation/dense_surface.py:5` | (a) | OpenEMR | comment/docstring example | | `verifier.py:2467` | (a) | OpenEMR | comment/docstring example | -| `vision/hashing.py:10` | (a) | MockMed | docstring for the MockMed-calibrated phash tolerance (see compile.py:151) | +| `vision/hashing.py:10` | (a) | MockMed | docstring for the MockMed-calibrated phash tolerance (see compile.py:148) | | `volatility.py:100` | (a) | OpenEMR | comment/docstring example | | `volatility.py:31` | (a) | OpenEMR | comment/docstring example | | `volatility.py:6` | (a) | OpenEMR | comment/docstring example | -| `benchmark/` (27 files) | (c) | OpenEMR, Frappe, MockMed, Open Dental | Moved to `packages/flow/benchmark/harness/` | +| `benchmark/` (25 files) | (c) | OpenEMR, Frappe, MockMed, Open Dental | Moved to `packages/flow/benchmark/harness/` | diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index 0a15c63874..2abfb44aa3 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -5458,7 +5458,8 @@ def _checkout_harness_on_path(command: str) -> bool: ) return False if str(checkout) not in sys.path: - sys.path.insert(0, str(checkout)) + # Appended, so nothing in the checkout shadows an installed module. + sys.path.append(str(checkout)) return True diff --git a/packages/flow/pyproject.toml b/packages/flow/pyproject.toml index a45b45596e..8787fd5bae 100644 --- a/packages/flow/pyproject.toml +++ b/packages/flow/pyproject.toml @@ -347,6 +347,7 @@ exclude = [ "/docs/validation/identity_roc.json", "/docs/validation/identity_roc.png", "/openadapt_flow/benchmark/reliability_corpus.py", + "/benchmark/harness/reliability_corpus.py", "/openadapt_flow/validation/adversary_corpus.py", "/openadapt_flow/validation/adversary_corpus_v2.py", "/openadapt_flow/validation/adversary_corpus_v3.py", @@ -410,6 +411,7 @@ exclude = [ "/docs/validation/identity_roc.json", "/docs/validation/identity_roc.png", "/openadapt_flow/benchmark/reliability_corpus.py", + "/benchmark/harness/reliability_corpus.py", "/openadapt_flow/validation/adversary_corpus.py", "/openadapt_flow/validation/adversary_corpus_v2.py", "/openadapt_flow/validation/adversary_corpus_v3.py", @@ -495,8 +497,8 @@ required-environments = [ target-version = "py310" line-length = 88 # Generated / non-package trees at the repo root are not part of the lint -# surface. Anchored with a leading slash so the package's own -# openadapt_flow/benchmark/ IS still linted (a bare "benchmark" would match it). +# surface. The benchmark harness (benchmark/harness/) is linted by naming it +# explicitly in CI (`ruff check openadapt_flow benchmark/harness`). extend-exclude = ["/benchmark", "/docs", "/runs", "/dist"] [tool.ruff.lint] @@ -599,7 +601,6 @@ module = [ "openadapt_flow.__main__", "openadapt_flow.backends.playwright_backend", "openadapt_flow.backends.rdp_backend", - "benchmark.harness.desktop_benchmark", "openadapt_flow.compiler.disambiguation", "openadapt_flow.interactive_recorder", "openadapt_flow.learning.induce", diff --git a/packages/flow/tests/test_runtime_never_imports_benchmark.py b/packages/flow/tests/test_runtime_never_imports_benchmark.py index d59f3083d7..7d052ea956 100644 --- a/packages/flow/tests/test_runtime_never_imports_benchmark.py +++ b/packages/flow/tests/test_runtime_never_imports_benchmark.py @@ -32,9 +32,20 @@ def _is_benchmark(module: str) -> bool: ) -def _imports_benchmark(tree: ast.AST) -> list[int]: +def _imports_benchmark(tree: ast.AST, *, lazy_ok: bool = False) -> list[int]: lines: list[int] = [] - for node in ast.walk(tree): + # An allowed module may import the harness only inside a function, where + # it runs on demand from a checkout. + nodes = ( + [ + n + for n in ast.iter_child_nodes(tree) + if isinstance(n, (ast.Import, ast.ImportFrom)) + ] + if lazy_ok + else ast.walk(tree) + ) + for node in nodes: if isinstance(node, ast.ImportFrom): module = node.module or "" if node.level == 0 and _is_benchmark(module): @@ -61,10 +72,10 @@ def test_product_modules_never_import_the_benchmark_package() -> None: offenders: list[str] = [] for path in sorted(ROOT.rglob("*.py")): rel = path.relative_to(ROOT).as_posix() - if rel in ALLOWED: - continue tree = ast.parse(path.read_text(encoding="utf-8")) - offenders += [f"{rel}:{n}" for n in _imports_benchmark(tree)] + offenders += [ + f"{rel}:{n}" for n in _imports_benchmark(tree, lazy_ok=rel in ALLOWED) + ] assert not offenders, ( "product code imports the benchmark harness (the wheel and a frozen " "build leave it out); move the shared name to " From 88eaf68cbf644dda6ceea6ebcad075d98fd9ece9 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 20:40:33 -0500 Subject: [PATCH 20/22] Guard: ordinary words like text_edit and mock_media pass; baseline 425 --- packages/flow/docs/GENERALITY_AUDIT.md | 2 +- packages/flow/scripts/check_engine_generality.py | 2 +- packages/flow/scripts/generality.toml | 6 +++--- packages/flow/scripts/generality_baseline.json | 3 +-- packages/flow/tests/test_engine_generality_check.py | 4 ++++ 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md index 24507e5974..deecfc90d6 100644 --- a/packages/flow/docs/GENERALITY_AUDIT.md +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -9,7 +9,7 @@ deployment configs) or in test fixtures and benchmark harnesses outside the shipped package. A CI check (`scripts/check_engine_generality.py`) keeps it that way; [CONTRIBUTING.md](../CONTRIBUTING.md) states the rule. It counts an app name inside an identifier too (`_openemr_base`, `isOpenEMR`); its baseline -started at 426 hits and only shrinks. +started at 425 hits and only shrinks. ## Method diff --git a/packages/flow/scripts/check_engine_generality.py b/packages/flow/scripts/check_engine_generality.py index d5b3976f2d..a37fc9c95f 100644 --- a/packages/flow/scripts/check_engine_generality.py +++ b/packages/flow/scripts/check_engine_generality.py @@ -99,7 +99,7 @@ def scan( continue split = _CAMEL_BOUNDARY.sub(" ", line) for term, (pattern, is_name) in terms.items(): - found = pattern.search(split if is_name else line) + found = pattern.search(line) or (is_name and pattern.search(split)) if found and not _allowed(rel, term, config): hits.setdefault(rel, {}).setdefault(term, []).append(number) return hits diff --git a/packages/flow/scripts/generality.toml b/packages/flow/scripts/generality.toml index 8c382c226d..edbc1e34ab 100644 --- a/packages/flow/scripts/generality.toml +++ b/packages/flow/scripts/generality.toml @@ -15,7 +15,7 @@ dolibarr = '(? None: "f.py": "row = 'tabLoanApplication'\n", # A name only stops at a letter or digit beside it. "g.py": "reopenemrs = 1\n", + "h.py": "TextEdit = 1\nMockMedServer = 1\n", + # Ordinary words that only look like a name. + "i.py": "text_edit = 1\nmock_media = 1\n", }, ) assert check.scan(engine) == { @@ -74,6 +77,7 @@ def test_a_name_inside_an_identifier_is_a_hit(check, tmp_path: Path) -> None: "d.py": {"stedi": [1]}, "e.py": {"epic": [1]}, "f.py": {"frappe-table": [1]}, + "h.py": {"mockmed": [2], "textedit": [1]}, } From cdf9c99c219a1044eb6e62a2850fa7258a080753 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 02:46:26 -0500 Subject: [PATCH 21/22] fix(flow): lower generality baseline after harness relocation Signed-off-by: abrichr --- .../flow/openadapt_flow/rl_env/environment.py | 2 +- packages/flow/openadapt_flow/rl_env/tasks.py | 2 +- .../flow/scripts/generality_baseline.json | 34 +++---------------- 3 files changed, 7 insertions(+), 31 deletions(-) diff --git a/packages/flow/openadapt_flow/rl_env/environment.py b/packages/flow/openadapt_flow/rl_env/environment.py index 0490ed6e9f..0e07633d56 100644 --- a/packages/flow/openadapt_flow/rl_env/environment.py +++ b/packages/flow/openadapt_flow/rl_env/environment.py @@ -206,7 +206,7 @@ def _norm(value: str) -> str: def _scalars(body: str) -> set[str]: """Every scalar value in a JSON or form body, as text. - A form field holding JSON (as Frappe sends ``doc=...``) is read through. + A form field holding JSON (such as ``doc=...``) is read through. """ from urllib.parse import parse_qsl diff --git a/packages/flow/openadapt_flow/rl_env/tasks.py b/packages/flow/openadapt_flow/rl_env/tasks.py index 2f557fbe2c..3bdd6a8358 100644 --- a/packages/flow/openadapt_flow/rl_env/tasks.py +++ b/packages/flow/openadapt_flow/rl_env/tasks.py @@ -136,7 +136,7 @@ class AgentSpec(_Strict): #: the oracle uses, reset routes, and anything else that would let the #: agent read or rewrite the answer. blocked_paths: list[str] = Field(default_factory=list) - #: The requests that carry the task's write (``"POST /api/encounter"``). + #: The requests that carry the task's write (``"POST /api/items"``). #: An infeasible task pays a failure report only if the agent's browser #: sent one whose body holds every value in the task's ``params``. write_requests: list[str] = Field(default_factory=list) diff --git a/packages/flow/scripts/generality_baseline.json b/packages/flow/scripts/generality_baseline.json index 46c2cab579..3fac4b495a 100644 --- a/packages/flow/scripts/generality_baseline.json +++ b/packages/flow/scripts/generality_baseline.json @@ -3,9 +3,6 @@ "mockmed": 43, "textedit": 1 }, - "agent/ledger_demo.py": { - "mockmed": 4 - }, "api.py": { "mockmed": 1 }, @@ -55,7 +52,7 @@ "mockmed": 1 }, "demo_driver.py": { - "mockloan": 5 + "mockloan": 2 }, "deployment.py": { "mockmed": 2, @@ -108,27 +105,6 @@ "espocrm": 2, "redmine": 2 }, - "guard_demo.py": { - "mockmed": 5, - "mockmed-records": 2, - "mockmed-write": 1 - }, - "integrations/harness2/__init__.py": { - "mockmed": 2 - }, - "integrations/harness2/adapter.py": { - "mockmed": 2 - }, - "integrations/harness2/agent.py": { - "mockmed": 1 - }, - "integrations/harness2/apps.py": { - "mockmed": 13, - "mockmed-records": 1 - }, - "integrations/harness2/tasks.py": { - "mockmed": 26 - }, "ir.py": { "token_main": 1 }, @@ -144,9 +120,6 @@ "jit/signature.py": { "mockmed": 1 }, - "learning/synth_stream.py": { - "mockmed": 5 - }, "mockloan/__init__.py": { "mockloan": 2, "mockmed": 2 @@ -214,11 +187,14 @@ "csrf_token_form": 2, "token_main": 1 }, + "reward/bundle.py": { + "mockmed": 3 + }, "reward/models.py": { "mockmed-records": 1 }, "reward/seed.py": { - "mockmed": 39 + "mockmed": 34 }, "risk.py": { "openemr": 1, From 1304436b1b9c5cc431026458f9017c7897d8192d Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 03:24:55 -0500 Subject: [PATCH 22/22] test(flow): declare read paths in synthetic verifier fixtures Signed-off-by: abrichr --- packages/flow/tests/test_at_rest_default.py | 4 +++- packages/flow/tests/test_auto_repair_cli.py | 1 + packages/flow/tests/test_create_form_wire.py | 8 ++++++-- packages/flow/tests/test_readback_volatile_fields.py | 8 ++++++++ packages/flow/tests/test_self_repair_cli.py | 1 + packages/flow/tests/test_windows_attempt_ledger.py | 2 ++ 6 files changed, 21 insertions(+), 3 deletions(-) diff --git a/packages/flow/tests/test_at_rest_default.py b/packages/flow/tests/test_at_rest_default.py index a758a3fa4a..d4b2f291ec 100644 --- a/packages/flow/tests/test_at_rest_default.py +++ b/packages/flow/tests/test_at_rest_default.py @@ -276,7 +276,9 @@ def _mockmed_run(tmp_path: Path, bundle: Path, run_dir: Path, **kw): return Replayer( MockMedBackend(url), vision=_vision(), - effect_verifier=RestRecordVerifier(url), + effect_verifier=RestRecordVerifier( + url, records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ).run(Workflow.load(bundle), bundle_dir=bundle, run_dir=run_dir, **kw) finally: diff --git a/packages/flow/tests/test_auto_repair_cli.py b/packages/flow/tests/test_auto_repair_cli.py index 2e34ecf672..43863dac73 100644 --- a/packages/flow/tests/test_auto_repair_cli.py +++ b/packages/flow/tests/test_auto_repair_cli.py @@ -41,6 +41,7 @@ def _config(tmp_path: Path, base_url: str, *, auto: bool = True) -> Path: config = tmp_path / "deploy.yaml" config.write_text( f"effects:\n kind: rest\n base_url: {base_url}\n" + " records_path: /api/db\n records_key: records\n" "policy:\n policy: clinical-write\n" f"{approvers_yaml()}" "repair:\n" diff --git a/packages/flow/tests/test_create_form_wire.py b/packages/flow/tests/test_create_form_wire.py index f0265811af..9f2fc25113 100644 --- a/packages/flow/tests/test_create_form_wire.py +++ b/packages/flow/tests/test_create_form_wire.py @@ -1166,7 +1166,9 @@ def test_an_api_span_after_a_late_refusal_halts_before_sending( report = Replayer( backend, web_form_actuator=actuator, - effect_verifier=RestRecordVerifier(sor_url.rstrip("/")), + effect_verifier=RestRecordVerifier( + sor_url.rstrip("/"), records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ).run( _span_workflow(), @@ -1269,7 +1271,9 @@ def test_a_late_refusal_adds_to_an_earlier_halt_instead_of_replacing_it( replayer = Replayer( backend, web_form_actuator=actuator, - effect_verifier=RestRecordVerifier(sor_url.rstrip("/")), + effect_verifier=RestRecordVerifier( + sor_url.rstrip("/"), records_path="/api/db", records_key="records" + ), poll_interval_s=0.01, ) report = replayer.run( diff --git a/packages/flow/tests/test_readback_volatile_fields.py b/packages/flow/tests/test_readback_volatile_fields.py index fb73ab01d8..769ecc4e38 100644 --- a/packages/flow/tests/test_readback_volatile_fields.py +++ b/packages/flow/tests/test_readback_volatile_fields.py @@ -98,6 +98,11 @@ def _run1_save(tmp_path: Path, base: str, store: _Store, **extra: Any) -> str: "kind": "fhir", "base_url": base, "resource_type": "Observation", + "field_paths": { + "id": "id", + "patient": "subject.reference", + "note": "valueString", + }, "timeout_s": 2.0, **extra, } @@ -171,6 +176,7 @@ def test_run1_refuses_mapped_server_metadata_before_recording( "verifier": { "kind": "fhir", "base_url": "https://example.test/fhir", + "resource_type": "Observation", "field_paths": {"revision": path}, } } @@ -201,6 +207,7 @@ def test_run1_allows_only_explicitly_dropped_metadata( section = { "kind": "fhir", "base_url": base, + "resource_type": "Observation", "field_paths": {"id": "id", "revision": path}, "records_drop_fields": ["revision"], } @@ -223,6 +230,7 @@ def test_run1_keeps_stable_metadata_mappings(tmp_path: Path) -> None: "verifier": { "kind": "fhir", "base_url": "https://example.test/fhir", + "resource_type": "Observation", "field_paths": {"profile": "meta.profile"}, } } diff --git a/packages/flow/tests/test_self_repair_cli.py b/packages/flow/tests/test_self_repair_cli.py index 143e7fe38c..8e2eda0c8c 100644 --- a/packages/flow/tests/test_self_repair_cli.py +++ b/packages/flow/tests/test_self_repair_cli.py @@ -57,6 +57,7 @@ def _config(tmp_path: Path, base_url: str) -> Path: config = tmp_path / "deploy.yaml" config.write_text( f"effects:\n kind: rest\n base_url: {base_url}\n" + " records_path: /api/db\n records_key: records\n" "policy:\n policy: clinical-write\n" + approvers_yaml() ) return config diff --git a/packages/flow/tests/test_windows_attempt_ledger.py b/packages/flow/tests/test_windows_attempt_ledger.py index ca01da449a..b217227ca6 100644 --- a/packages/flow/tests/test_windows_attempt_ledger.py +++ b/packages/flow/tests/test_windows_attempt_ledger.py @@ -134,6 +134,8 @@ def _run_case(main, bundle, inputs_path, run_dir): "rest", "--effects-base-url", "http://sor.local", + "--effects-records-path", + "/api/db", "--backend", "web", ]