From bc98a3af23f4b245a0afeb25a14ffa68eae7b3a7 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 12:39:45 -0500 Subject: [PATCH 01/16] Engine generality audit; move the benchmark harnesses out of the wheel Audit every place the engine (packages/flow/openadapt_flow) names a specific application, bucketed as example, data, harness in the package, or real special-casing: docs/GENERALITY_AUDIT.md. Move openadapt_flow/benchmark and openadapt_flow/instrument to packages/flow/benchmark/harness (import benchmark.harness). The wheel packages only openadapt_flow, so it no longer ships them. The benchmark command loads the harness from a source checkout and says so from a wheel; instrument already failed to import from a wheel. Reword two comments that named an app on a generic branch. --- packages/flow/CHANGELOG.md | 2 + .../benchmark/agent_arm_verticals/README.md | 2 +- .../code_as_action/measure_openemr.py | 4 +- packages/flow/benchmark/dom/BENCHMARK.md | 4 +- .../flow/benchmark/effect_e2e/EFFECT_E2E.md | 2 +- .../benchmark/effect_e2e/record_service.py | 2 +- packages/flow/benchmark/effect_e2e/run.py | 8 +- .../effectbench/reference_fault_model.py | 10 +- .../benchmark/effectbench/task_pack/README.md | 2 +- .../effectbench/task_pack/__init__.py | 4 +- .../effectbench/task_pack/_authoring.py | 10 +- .../benchmark/effectbench/task_pack/audit.py | 2 +- .../benchmark/effectbench/task_pack/driver.py | 8 +- .../effectbench/task_pack/frappe_tasks.py | 8 +- .../effectbench/task_pack/mockmed_tasks.py | 8 +- .../effectbench/task_pack/openemr_tasks.py | 8 +- .../benchmark/effectbench/task_pack/pack.py | 6 +- .../flow/benchmark/frappe_lending/README.md | 2 +- .../harness}/__init__.py | 6 +- .../harness}/agent_baseline.py | 0 .../harness}/chart_fonts.py | 0 .../harness}/desktop_benchmark.py | 4 +- .../harness}/dom_arm.py | 20 +- .../harness}/effectbench/README.md | 2 +- .../harness}/effectbench/__init__.py | 8 +- .../harness}/effectbench/metrics.py | 4 +- .../harness}/effectbench/oracle.py | 6 +- .../harness}/effectbench/runner/README.md | 6 +- .../harness}/effectbench/runner/__init__.py | 26 +- .../harness}/effectbench/runner/__main__.py | 18 +- .../harness}/effectbench/runner/arms.py | 12 +- .../harness}/effectbench/runner/baselines.py | 8 +- .../harness}/effectbench/runner/compound.py | 8 +- .../harness}/effectbench/runner/harness.py | 14 +- .../effectbench/runner/reference_tasks.py | 12 +- .../harness}/effectbench/runner/substrate.py | 4 +- .../harness}/effectbench/schema.py | 4 +- .../harness}/frappe_lending.py | 0 .../harness}/hybrid_benchmark.py | 14 +- .../harness}/instrument/__init__.py | 4 +- .../harness}/instrument/competitor_drift.py | 6 +- .../harness}/openemr_benchmark.py | 12 +- .../harness}/openemr_local.py | 2 +- .../harness}/reliability.py | 0 .../harness}/run_benchmark.py | 10 +- .../harness}/silent_wrong_action.py | 8 +- .../benchmark => benchmark/harness}/verify.py | 0 packages/flow/benchmark/hybrid/BENCHMARK.md | 2 +- .../benchmark/lending_fault_model/README.md | 2 +- .../benchmark/lending_fault_model/swer.py | 4 +- .../openemr_api_tier/approval_durable.py | 2 +- .../flow/benchmark/openemr_api_tier/common.py | 2 +- .../benchmark/openemr_api_tier/feature_e2e.py | 4 +- .../benchmark/openemr_api_tier/spike/bench.py | 2 +- .../openemr_api_tier/spike/capture_har.py | 2 +- packages/flow/benchmark/openemr_e2e/README.md | 2 +- .../flow/benchmark/openemr_e2e/harness.py | 2 +- .../flow/benchmark/openemr_local/README.md | 2 +- .../benchmark/openemr_throughput/harness.py | 4 +- .../flow/benchmark/openimis_claims/README.md | 2 +- .../benchmark/scoreboard/tasks/openemr.py | 2 +- .../flow/benchmark/silent_saves/metrics.py | 2 +- .../SILENT_WRONG_ACTION.md | 2 +- .../flow/benchmark/vision_hardening/README.md | 2 +- packages/flow/docs/EFFECT_KIT.md | 2 +- packages/flow/docs/GENERALITY_AUDIT.md | 311 ++++++++++++++++++ packages/flow/docs/design/EFFECT_VERIFIER.md | 2 +- packages/flow/docs/desktop/PHASE2.md | 4 +- packages/flow/openadapt_flow/__main__.py | 25 +- .../flow/openadapt_flow/anthropic_defaults.py | 2 +- packages/flow/openadapt_flow/authoring.py | 2 +- packages/flow/openadapt_flow/deployment.py | 2 +- packages/flow/paper/REVIEW_ADVERSARIAL.md | 4 +- packages/flow/pyproject.toml | 2 +- packages/flow/scripts/desktop/waa_shim.py | 2 +- packages/flow/scripts/frappe_lending_demo.py | 4 +- packages/flow/scripts/openemr_demo.py | 10 +- packages/flow/scripts/openemr_local_demo.py | 4 +- packages/flow/scripts/openemr_param_depth.py | 2 +- .../test_region_stable_theme_regression.py | 2 +- packages/flow/tests/test_benchmark.py | 14 +- packages/flow/tests/test_chart_fonts.py | 4 +- .../tests/test_competitor_drift_instrument.py | 4 +- packages/flow/tests/test_desktop_benchmark.py | 4 +- packages/flow/tests/test_dom_arm.py | 8 +- packages/flow/tests/test_effectbench.py | 12 +- .../flow/tests/test_effectbench_runner.py | 16 +- .../flow/tests/test_effectbench_standalone.py | 6 +- .../flow/tests/test_effectbench_task_pack.py | 4 +- .../tests/test_frappe_lending_benchmark.py | 2 +- packages/flow/tests/test_hybrid_benchmark.py | 8 +- packages/flow/tests/test_lending_swer.py | 2 +- packages/flow/tests/test_openemr_benchmark.py | 12 +- .../tests/test_openemr_local_benchmark.py | 4 +- .../tests/test_openemr_retained_finals.py | 2 +- .../flow/tests/test_reliability_harness.py | 2 +- .../test_runtime_never_imports_benchmark.py | 58 ++-- .../test_silent_wrong_action_benchmark.py | 2 +- .../tests/test_ui_dismiss_live_openemr.py | 2 +- 99 files changed, 622 insertions(+), 274 deletions(-) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/__init__.py (63%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/agent_baseline.py (100%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/chart_fonts.py (100%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/desktop_benchmark.py (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/dom_arm.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/README.md (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/__init__.py (93%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/metrics.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/oracle.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/README.md (95%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/__init__.py (72%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/__main__.py (90%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/arms.py (95%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/baselines.py (97%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/compound.py (94%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/harness.py (92%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/reference_tasks.py (96%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/substrate.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/schema.py (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/frappe_lending.py (100%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/hybrid_benchmark.py (99%) rename packages/flow/{openadapt_flow => benchmark/harness}/instrument/__init__.py (91%) rename packages/flow/{openadapt_flow => benchmark/harness}/instrument/competitor_drift.py (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/openemr_benchmark.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/openemr_local.py (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/reliability.py (100%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/run_benchmark.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/silent_wrong_action.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/verify.py (100%) create mode 100644 packages/flow/docs/GENERALITY_AUDIT.md diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md index 99ec48b05e..978a0b337a 100644 --- a/packages/flow/CHANGELOG.md +++ b/packages/flow/CHANGELOG.md @@ -25,6 +25,8 @@ ### Fixes +- The wheel no longer ships the benchmark harnesses. `openadapt_flow.benchmark` and `openadapt_flow.instrument` moved to `packages/flow/benchmark/harness/` in the source tree (import `benchmark.harness`); `openadapt-flow benchmark` runs from a source checkout and says so from a wheel. The engine audit for application-specific code is in `docs/GENERALITY_AUDIT.md`. + - Self-repair can take a stop where the app's own dialog covers the next step's identity band: when the backend proves the covering window belongs to the window the last verified step was in, and the part of the band read still matches the record, the dialog is dismissed after the previous click and the step re-checks its identity in full. Another record behind the dialog stays an identity halt. The runtime lets a backend remember a verified record's window (`record_verified_window`); macOS answers `owned_window_over` from the window's proved sheet ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). - macOS: a run can now act in the pinned window's own AppKit sheet (an `NSAlert` attached to it), which is a separate window. `MacOSBackend` accepts it only when AX lists exactly one `AXSheet` on the window and exactly one window of the same process has its bounds inside it; the capture composites exactly that sheet over the window, and focus, pointer and click-target checks accept it only inside its bounds. Any other window is still refused ([#1711](https://github.com/OpenAdaptAI/OpenAdapt/issues/1711)). diff --git a/packages/flow/benchmark/agent_arm_verticals/README.md b/packages/flow/benchmark/agent_arm_verticals/README.md index fbfe9544dc..54526bd8fa 100644 --- a/packages/flow/benchmark/agent_arm_verticals/README.md +++ b/packages/flow/benchmark/agent_arm_verticals/README.md @@ -26,7 +26,7 @@ budget without completing the write in every trial. ## Method and public/private boundary The public, generic agent-baseline mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. It provides explicit paid-run +`benchmark/harness/agent_baseline.py`. It provides explicit paid-run opt-in, action and cost caps, usage accounting, and an agent self-report that is never trusted for scoring. The public benchmark schemas and aggregate failure taxonomy also remain available for independently authored tasks. diff --git a/packages/flow/benchmark/code_as_action/measure_openemr.py b/packages/flow/benchmark/code_as_action/measure_openemr.py index 74a34443a6..9e1b5e70f0 100644 --- a/packages/flow/benchmark/code_as_action/measure_openemr.py +++ b/packages/flow/benchmark/code_as_action/measure_openemr.py @@ -184,7 +184,7 @@ def spec_for(key: str) -> Any: - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec return SyntheticPatientSpec(**REFERRALS[key], **COMMON_FIELDS) @@ -390,7 +390,7 @@ def main() -> int: keys = [k for k in args.referrals.split(",") if k] from benchmark.openemr_local import qualify_hooks - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec fixture = qualify_hooks.fixture() demo = _demo() diff --git a/packages/flow/benchmark/dom/BENCHMARK.md b/packages/flow/benchmark/dom/BENCHMARK.md index 7111596c12..2b40374899 100644 --- a/packages/flow/benchmark/dom/BENCHMARK.md +++ b/packages/flow/benchmark/dom/BENCHMARK.md @@ -23,7 +23,7 @@ row's Open button — the literal reading — and **DOM (name-filtered)** clicks the Open button in the row named `Jane Sample` — the identity reading, hardcoding the demonstrated patient exactly as the compiled arm's recorded identity band does. Every selector choice is -documented in `openadapt_flow/benchmark/dom_arm.py`. +documented in `benchmark/harness/dom_arm.py`. ## Verdict @@ -208,7 +208,7 @@ Unmeasured variants, for completeness: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.dom_arm --out benchmark/dom --n-per-perturbation 2 +.venv/bin/python -m benchmark.harness.dom_arm --out benchmark/dom --n-per-perturbation 2 ``` (`--n-per-perturbation 2` matches the committed results.) No API key diff --git a/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md b/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md index dfa26c1e61..58868ee444 100644 --- a/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md +++ b/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md @@ -2,7 +2,7 @@ Date: 2026-07-21. This is the genuinely independent, end-to-end measurement of the silent-wrong-effect (SWER) result. Unlike the definitional -`openadapt_flow.benchmark.silent_wrong_action` (raw `requests.post`; effect +`benchmark.harness.silent_wrong_action` (raw `requests.post`; effect verifier and ground truth read the SAME in-process object; the effect contract restates the ground-truth definition -- so its `0/90` is circular by construction), every write below goes through the ACTUAL governed replay path diff --git a/packages/flow/benchmark/effect_e2e/record_service.py b/packages/flow/benchmark/effect_e2e/record_service.py index 2bb71f2e90..0f8f6e0fa3 100644 --- a/packages/flow/benchmark/effect_e2e/record_service.py +++ b/packages/flow/benchmark/effect_e2e/record_service.py @@ -3,7 +3,7 @@ This is the system of record the end-to-end silent-wrong-effect harness (:mod:`benchmark.effect_e2e.run`) writes to THROUGH the governed replay path. It exists to break the circularity of the older definitional benchmark -(``openadapt_flow.benchmark.silent_wrong_action``), whose effect verifier and +(``benchmark.harness.silent_wrong_action``), whose effect verifier and ground-truth judge read the SAME in-process Python object and whose write bypassed the :class:`~openadapt_flow.runtime.replayer.Replayer`. diff --git a/packages/flow/benchmark/effect_e2e/run.py b/packages/flow/benchmark/effect_e2e/run.py index d08601d691..7860d35b89 100644 --- a/packages/flow/benchmark/effect_e2e/run.py +++ b/packages/flow/benchmark/effect_e2e/run.py @@ -1,7 +1,7 @@ """End-to-end silent-wrong-effect (SWER) harness through the REAL replayer. This is the genuinely independent, end-to-end version of the silent-wrong-action -result. The older ``openadapt_flow.benchmark.silent_wrong_action`` is +result. The older ``benchmark.harness.silent_wrong_action`` is *definitional*: it issues raw ``requests.post`` (bypassing the :class:`~openadapt_flow.runtime.replayer.Replayer`), and its effect verifier and ground-truth judge both read the SAME in-process ``FaultDB`` object while the @@ -629,7 +629,7 @@ def render_markdown(results: dict[str, Any]) -> str: Date: {date}. This is the genuinely independent, end-to-end measurement of the silent-wrong-effect (SWER) result. Unlike the definitional -`openadapt_flow.benchmark.silent_wrong_action` (raw `requests.post`; effect +`benchmark.harness.silent_wrong_action` (raw `requests.post`; effect verifier and ground truth read the SAME in-process object; the effect contract restates the ground-truth definition -- so its `0/90` is circular by construction), every write below goes through the ACTUAL governed replay path @@ -774,7 +774,7 @@ def render_markdown(results: dict[str, Any]) -> str: def render_chart(results: dict[str, Any], out_png: Path) -> Path: """Bar chart: silent-wrong-effect rate per arm (large -> small -> zero).""" - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() m = results["metrics"]["per_arm"] @@ -827,7 +827,7 @@ def write_outputs(results: dict[str, Any], out_dir: Path) -> None: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") (out_dir / "EFFECT_E2E.md").write_text(render_markdown(results)) - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "effect_e2e.png") diff --git a/packages/flow/benchmark/effectbench/reference_fault_model.py b/packages/flow/benchmark/effectbench/reference_fault_model.py index 5ba4904208..0488c51dff 100644 --- a/packages/flow/benchmark/effectbench/reference_fault_model.py +++ b/packages/flow/benchmark/effectbench/reference_fault_model.py @@ -31,16 +31,16 @@ How it re-expresses the study, faithfully and without hardcoding verdicts: - Each of the study's nine scenarios (``benchmark.fault_model.faults.FAULTS``) - becomes a :class:`~openadapt_flow.benchmark.effectbench.schema.TaskSpec` plus + becomes a :class:`~benchmark.harness.effectbench.schema.TaskSpec` plus the KNOWN post-action system-of-record state that fault produces (the same DB states the study's own taxonomy tests assert). The states live in :data:`SCENARIOS` — the record shapes are MockMed ``FaultDB`` rows. - The TRUE effect is read by the substrate-agnostic - :class:`~openadapt_flow.benchmark.effectbench.oracle.RecordSnapshotOracle` + :class:`~benchmark.harness.effectbench.oracle.RecordSnapshotOracle` running the SAME shared ``judge_records`` decision logic the runtime uses — once for the at-most-once/collateral ``record_written`` contract and once for the ``field_equals`` note read-back — reduced by - :func:`~openadapt_flow.benchmark.effectbench.oracle.combine_true_states`. + :func:`~benchmark.harness.effectbench.oracle.combine_true_states`. - Two arms are scored against the identical independent oracle: ``screen_only`` (the weak banner oracle — reports success whenever the screen looks saved) and ``effect_verify`` (the agent halts unless the oracle CONFIRMS). Only the @@ -56,7 +56,7 @@ from typing import Optional from benchmark.fault_model import faults as F -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, DivergenceCategory, Effect, @@ -72,7 +72,7 @@ oracle_verdict_of, summarize, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel # The consequential-save contract the study verifies: exactly one Triage # encounter for the target patient (at-most-once + no collateral loss) whose diff --git a/packages/flow/benchmark/effectbench/task_pack/README.md b/packages/flow/benchmark/effectbench/task_pack/README.md index 88fa4d6c25..8846166f4c 100644 --- a/packages/flow/benchmark/effectbench/task_pack/README.md +++ b/packages/flow/benchmark/effectbench/task_pack/README.md @@ -68,7 +68,7 @@ python -m benchmark.effectbench.task_pack.pack # Live-run the MockMed anchor through the oracle (prints per-task classification) python -c "from benchmark.effectbench.task_pack.driver import run_mockmed_pack; \ -from openadapt_flow.benchmark.effectbench import summarize; \ +from benchmark.harness.effectbench import summarize; \ eps=run_mockmed_pack(trials=3); \ print('screen SWER', summarize(eps, arm='screen_only').swer.rate); \ print('effect SWER', summarize(eps, arm='effect_verify').swer.rate)" diff --git a/packages/flow/benchmark/effectbench/task_pack/__init__.py b/packages/flow/benchmark/effectbench/task_pack/__init__.py index e3f3537b7b..c823a82252 100644 --- a/packages/flow/benchmark/effectbench/task_pack/__init__.py +++ b/packages/flow/benchmark/effectbench/task_pack/__init__.py @@ -6,9 +6,9 @@ ``benchmark.environments`` registry (PR #173) indexes, authored against the frozen EffectBench schema/oracle/metrics contract (PR #178). -Each task is a :class:`~openadapt_flow.benchmark.effectbench.TaskSpec` (goal = +Each task is a :class:`~benchmark.harness.effectbench.TaskSpec` (goal = INTENT only, never a step list) plus an -:class:`~openadapt_flow.benchmark.effectbench.OracleSpec` bound to the +:class:`~benchmark.harness.effectbench.OracleSpec` bound to the environment's system-of-record READ recipe, DESIGNED so a green screen (task-success) diverges from the correct business effect. The oracle reads pre/post SoR state independently of the screen; the non-gameability attestations diff --git a/packages/flow/benchmark/effectbench/task_pack/_authoring.py b/packages/flow/benchmark/effectbench/task_pack/_authoring.py index 00e02bc078..aa0ba26354 100644 --- a/packages/flow/benchmark/effectbench/task_pack/_authoring.py +++ b/packages/flow/benchmark/effectbench/task_pack/_authoring.py @@ -3,10 +3,10 @@ These build the two things every consequential-write task needs: - the DECLARED expected effect (a typed - :class:`~openadapt_flow.benchmark.effectbench.Effect`), parameterized by run - params via :class:`~openadapt_flow.benchmark.effectbench.ValueExpr` so the + :class:`~benchmark.harness.effectbench.Effect`), parameterized by run + params via :class:`~benchmark.harness.effectbench.ValueExpr` so the record CHECKED is the record this trial WROTE (trial-unique payload); and -- the :class:`~openadapt_flow.benchmark.effectbench.OracleSpec` that declares +- the :class:`~benchmark.harness.effectbench.OracleSpec` that declares the independent read channel and the non-gameability attestations the adversarial audit signs off on. @@ -21,13 +21,13 @@ from typing import Any, Optional -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, EffectKind, OracleSpec, ValueExpr, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel # Canonical run-param names a trial binds a trial-unique value to. The driver # derives each from the trial seed so the oracle checks THIS run's exact write. diff --git a/packages/flow/benchmark/effectbench/task_pack/audit.py b/packages/flow/benchmark/effectbench/task_pack/audit.py index c44706010e..fcba2b8c15 100644 --- a/packages/flow/benchmark/effectbench/task_pack/audit.py +++ b/packages/flow/benchmark/effectbench/task_pack/audit.py @@ -41,7 +41,7 @@ _note_effect, _record_effect, ) -from openadapt_flow.benchmark.effectbench import RestRecordVerifier, Verdict +from benchmark.harness.effectbench import RestRecordVerifier, Verdict _HTTP_TIMEOUT_S = 5.0 TRIAL_NOTE = "triage note trial-unique-AUDIT" diff --git a/packages/flow/benchmark/effectbench/task_pack/driver.py b/packages/flow/benchmark/effectbench/task_pack/driver.py index 4145e46ae7..035a025097 100644 --- a/packages/flow/benchmark/effectbench/task_pack/driver.py +++ b/packages/flow/benchmark/effectbench/task_pack/driver.py @@ -8,9 +8,9 @@ mocked verdicts. It depends ONLY on the public EffectBench surface -(:func:`~openadapt_flow.benchmark.effectbench.score_episode`, -:class:`~openadapt_flow.benchmark.effectbench.RestRecordVerifier`, -:class:`~openadapt_flow.benchmark.effectbench.AgentReport`) and the MockMed +(:func:`~benchmark.harness.effectbench.score_episode`, +:class:`~benchmark.harness.effectbench.RestRecordVerifier`, +:class:`~benchmark.harness.effectbench.AgentReport`) and the MockMed fault server. It does not reimplement any real baseline "arm"; the two arms here are the two SCORING conditions the benchmark contrasts: @@ -46,7 +46,7 @@ MockMedDrive, MockMedTask, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, EpisodeRecord, RestRecordVerifier, diff --git a/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py b/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py index e2c22a2ef5..9219f750ea 100644 --- a/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py +++ b/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py @@ -17,7 +17,7 @@ ``oracle.config["query"]`` + ``["query_params"]`` is what the multi-baseline runner materializes into a -:class:`~openadapt_flow.benchmark.effectbench.SqlRecordVerifier` (read-only role, +:class:`~benchmark.harness.effectbench.SqlRecordVerifier` (read-only role, DB-API-bound params). The loan-application docname has a space (`` `tabLoan Application` ``), so the table is back-tick quoted in the SELECT. """ @@ -33,15 +33,15 @@ record_written_effect, ) from benchmark.effectbench.task_pack.openemr_tasks import ContainerTask -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( DivergenceCategory as DC, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, Substrate, TaskSpec, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel ENVIRONMENT = "frappe_lending" diff --git a/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py b/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py index 898f90db42..9f9b679bc3 100644 --- a/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py +++ b/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py @@ -8,7 +8,7 @@ path is exercised in CI (no mocked verdicts). Each entry is a :class:`MockMedTask`: an agent-agnostic -:class:`~openadapt_flow.benchmark.effectbench.TaskSpec` (goal = intent only) +:class:`~benchmark.harness.effectbench.TaskSpec` (goal = intent only) plus a :class:`MockMedDrive` — the environment recipe the live driver replays to MANIFEST the divergence (which ``?fault=`` mode to inject, the write the agent performs, decoy/concurrent rows to seed, and the documented banner @@ -34,9 +34,9 @@ param, record_written_effect, ) -from openadapt_flow.benchmark.effectbench import DivergenceCategory as DC -from openadapt_flow.benchmark.effectbench import Effect, Substrate, TaskSpec -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench import DivergenceCategory as DC +from benchmark.harness.effectbench import Effect, Substrate, TaskSpec +from benchmark.harness.effectbench.schema import OracleChannel TARGET_PATIENT = "p1" TARGET_TYPE = "Triage" diff --git a/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py b/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py index 2f6d61974d..7581aa2520 100644 --- a/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py +++ b/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py @@ -18,7 +18,7 @@ The SQL each oracle runs (``oracle.config["query"]`` + ``["query_params"]``) is what the multi-baseline runner materializes into a -:class:`~openadapt_flow.benchmark.effectbench.SqlRecordVerifier` (read-only role; +:class:`~benchmark.harness.effectbench.SqlRecordVerifier` (read-only role; values bound through DB-API params, never interpolated). Column names become record keys, so the same typed ``Effect`` the MockMed anchor uses judges these rows unchanged. @@ -36,15 +36,15 @@ param, record_written_effect, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( DivergenceCategory as DC, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, Substrate, TaskSpec, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel ENVIRONMENT = "openemr_local" diff --git a/packages/flow/benchmark/effectbench/task_pack/pack.py b/packages/flow/benchmark/effectbench/task_pack/pack.py index 2be0450827..fa15417f72 100644 --- a/packages/flow/benchmark/effectbench/task_pack/pack.py +++ b/packages/flow/benchmark/effectbench/task_pack/pack.py @@ -4,7 +4,7 @@ the container-gated OpenEMR and Frappe families). It: - assembles the unified list of :class:`PackEntry` (one per authored task), - each carrying its :class:`~openadapt_flow.benchmark.effectbench.TaskSpec`, the + each carrying its :class:`~benchmark.harness.effectbench.TaskSpec`, the pinned environment it targets, and whether it runs LIVE here or needs a container; - validates every task against the ``benchmark.environments`` registry (the @@ -29,8 +29,8 @@ from benchmark.effectbench.task_pack.frappe_tasks import FRAPPE_TASKS from benchmark.effectbench.task_pack.mockmed_tasks import MOCKMED_TASKS from benchmark.effectbench.task_pack.openemr_tasks import OPENEMR_TASKS -from openadapt_flow.benchmark.effectbench import TaskSpec -from openadapt_flow.benchmark.effectbench.schema import DivergenceCategory, Substrate +from benchmark.harness.effectbench import TaskSpec +from benchmark.harness.effectbench.schema import DivergenceCategory, Substrate MANIFEST_SCHEMA_VERSION = 1 diff --git a/packages/flow/benchmark/frappe_lending/README.md b/packages/flow/benchmark/frappe_lending/README.md index 1504ddf329..89ae27fa5a 100644 --- a/packages/flow/benchmark/frappe_lending/README.md +++ b/packages/flow/benchmark/frappe_lending/README.md @@ -30,7 +30,7 @@ trials per cell remains the publication bar. Raw per-run rows, environment fingerprints, application-specific adapter/oracle wiring, and the detailed cost ledger are retained in the private `OpenAdaptAI/openadapt-corpus` repository. The public generic cost-capped agent mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. See the +`benchmark/harness/agent_baseline.py`. See the [aggregate report](../agent_arm_verticals/README.md). The accepted run used baseline SHA-256 diff --git a/packages/flow/openadapt_flow/benchmark/__init__.py b/packages/flow/benchmark/harness/__init__.py similarity index 63% rename from packages/flow/openadapt_flow/benchmark/__init__.py rename to packages/flow/benchmark/harness/__init__.py index b157c4fda1..2eba51f30b 100644 --- a/packages/flow/openadapt_flow/benchmark/__init__.py +++ b/packages/flow/benchmark/harness/__init__.py @@ -1,4 +1,8 @@ -"""Benchmark: compiled replay vs. a frontier computer-use agent. +"""Benchmark harnesses: compiled replay vs. a frontier computer-use agent. + +This package lives in the source checkout, outside ``openadapt_flow``, so the +wheel does not ship it. Import it as ``benchmark.harness`` from +``packages/flow``. Submodules (import them directly; nothing heavy is imported here): diff --git a/packages/flow/openadapt_flow/benchmark/agent_baseline.py b/packages/flow/benchmark/harness/agent_baseline.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/agent_baseline.py rename to packages/flow/benchmark/harness/agent_baseline.py diff --git a/packages/flow/openadapt_flow/benchmark/chart_fonts.py b/packages/flow/benchmark/harness/chart_fonts.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/chart_fonts.py rename to packages/flow/benchmark/harness/chart_fonts.py diff --git a/packages/flow/openadapt_flow/benchmark/desktop_benchmark.py b/packages/flow/benchmark/harness/desktop_benchmark.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/desktop_benchmark.py rename to packages/flow/benchmark/harness/desktop_benchmark.py index 270605e54b..c581ca306d 100644 --- a/packages/flow/openadapt_flow/benchmark/desktop_benchmark.py +++ b/packages/flow/benchmark/harness/desktop_benchmark.py @@ -745,7 +745,7 @@ def _identity_transfer_verdict(results: dict) -> str: def render_chart(results: dict, path: Path) -> None: """Stacked outcome bars per arm (success / wrong-action / halt).""" - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() arms = list(results["arms"]) @@ -776,7 +776,7 @@ def write_outputs(results: dict, out_dir: str | Path) -> None: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") (out_dir / "BENCHMARK.md").write_text(render_markdown(results)) - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "outcomes.png") diff --git a/packages/flow/openadapt_flow/benchmark/dom_arm.py b/packages/flow/benchmark/harness/dom_arm.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/dom_arm.py rename to packages/flow/benchmark/harness/dom_arm.py index a6c0e598dd..27a1e277ef 100644 --- a/packages/flow/openadapt_flow/benchmark/dom_arm.py +++ b/packages/flow/benchmark/harness/dom_arm.py @@ -10,7 +10,7 @@ Three arms, one frozen schedule, one arm-independent success check: - **compiled**: the existing vision-anchored compiled replay - (:func:`openadapt_flow.benchmark.run_benchmark._compiled_run`), healing + (:func:`benchmark.harness.run_benchmark._compiled_run`), healing enabled as always. - **dom** (positional) and **dom_named** (name-filtered): :func:`dom_script` — a Playwright script written the way a competent @@ -29,7 +29,7 @@ drift modes (:data:`PERTURBATIONS`: lookalike, missing, grow, sort, theme, rename, move, typelabel). Success is judged for BOTH arms by the hybrid benchmark's own arm-independent check -(:func:`~openadapt_flow.benchmark.hybrid_benchmark.verify_hybrid_final`): +(:func:`~benchmark.harness.hybrid_benchmark.verify_hybrid_final`): OCR of the final screenshot must show the saved-encounter evidence AND the right patient, with wrong-type writes flagged — so this head-to-head is judged by the identical bar. Wrong-action events are recorded per arm: @@ -51,7 +51,7 @@ from pathlib import Path from typing import Any, Callable, Optional -from openadapt_flow.benchmark.hybrid_benchmark import ( +from benchmark.harness.hybrid_benchmark import ( _NOTE_POOL, DRIFT_TYPES, SCHEDULE, @@ -465,7 +465,7 @@ def dom_arm_aggregate(rows: list[dict[str, Any]]) -> dict[str, Any]: Aggregate dict: base stats plus clean/drift splits, outcome counts, wrong-action count, and (DOM arm) maintenance events. """ - from openadapt_flow.benchmark.run_benchmark import _arm_aggregate + from benchmark.harness.run_benchmark import _arm_aggregate agg = _arm_aggregate(rows) agg["clean"] = _split_rate([r for r in rows if r.get("condition") == "clean"]) @@ -621,7 +621,7 @@ def render_dom_chart(results: dict[str, Any], out_png: Path) -> Path: Returns: The written PNG path. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() from matplotlib.patches import Rectangle @@ -1177,7 +1177,7 @@ def stat_row(label: str, fmt: Callable[[dict[str, Any]], str]) -> str: clicks the Open button in the row named `{TARGET_PATIENT}` — the identity reading, hardcoding the demonstrated patient exactly as the compiled arm's recorded identity band does. Every selector choice is -documented in `openadapt_flow/benchmark/dom_arm.py`. +documented in `benchmark/harness/dom_arm.py`. ## Verdict @@ -1333,7 +1333,7 @@ def stat_row(label: str, fmt: Callable[[dict[str, Any]], str]) -> str: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.dom_arm --out benchmark/dom --n-per-perturbation 2 +.venv/bin/python -m benchmark.harness.dom_arm --out benchmark/dom --n-per-perturbation 2 ``` (`--n-per-perturbation 2` matches the committed results.) No API key @@ -1350,7 +1350,7 @@ def write_dom_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_dom_chart, results, out_dir / "outcome_matrix.png") (out_dir / "BENCHMARK.md").write_text(render_dom_markdown(results)) @@ -1388,7 +1388,7 @@ def run_dom_benchmark( Returns: The results dict (also written to ``results.json``). """ - from openadapt_flow.benchmark.run_benchmark import _compiled_run + from benchmark.harness.run_benchmark import _compiled_run from openadapt_flow.compiler import compile_recording from openadapt_flow.demo_driver import record_triage_demo from openadapt_flow.mockmed.server import serve @@ -1499,7 +1499,7 @@ def one_run( def main(argv: Optional[list[str]] = None) -> int: - """CLI entry point: ``python -m openadapt_flow.benchmark.dom_arm``.""" + """CLI entry point: ``python -m benchmark.harness.dom_arm``.""" parser = argparse.ArgumentParser( description=( "DOM-selector benchmark arm vs. compiled vision replay on " diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/README.md b/packages/flow/benchmark/harness/effectbench/README.md similarity index 99% rename from packages/flow/openadapt_flow/benchmark/effectbench/README.md rename to packages/flow/benchmark/harness/effectbench/README.md index c42718959f..08b926e621 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/README.md +++ b/packages/flow/benchmark/harness/effectbench/README.md @@ -5,7 +5,7 @@ > system-of-record oracle* disagrees. Reported jointly with **over-halt rate**, > **task success**, and the **success–effect gap**. -This package (`openadapt_flow.benchmark.effectbench`) is the **stable contract** +This package (`benchmark.harness.effectbench`) is the **stable contract** that two downstream efforts build on: 1. the **multi-baseline runner adapter** — drives each agent arm through a common diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/__init__.py b/packages/flow/benchmark/harness/effectbench/__init__.py similarity index 93% rename from packages/flow/openadapt_flow/benchmark/effectbench/__init__.py rename to packages/flow/benchmark/harness/effectbench/__init__.py index 7291bdbf04..62562d9441 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/__init__.py +++ b/packages/flow/benchmark/harness/effectbench/__init__.py @@ -31,10 +31,10 @@ bootstrap confidence intervals, all decomposed by category x substrate. The design doc is ``.private/benchmark_design_2026_07_20.md``; the operator -contract is ``openadapt_flow/benchmark/effectbench/README.md``. +contract is ``benchmark/harness/effectbench/README.md``. """ -from openadapt_flow.benchmark.effectbench.metrics import ( # noqa: F401 +from benchmark.harness.effectbench.metrics import ( # noqa: F401 BenchmarkSummary, CellSummary, Interval, @@ -44,7 +44,7 @@ summarize, wilson_interval, ) -from openadapt_flow.benchmark.effectbench.oracle import ( # noqa: F401 +from benchmark.harness.effectbench.oracle import ( # noqa: F401 RecordSnapshotOracle, TrueEffectState, classify_outcome, @@ -53,7 +53,7 @@ oracle_verdict_of, score_episode, ) -from openadapt_flow.benchmark.effectbench.schema import ( # noqa: F401 +from benchmark.harness.effectbench.schema import ( # noqa: F401 AgentReport, DivergenceCategory, EpisodeRecord, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/metrics.py b/packages/flow/benchmark/harness/effectbench/metrics.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/effectbench/metrics.py rename to packages/flow/benchmark/harness/effectbench/metrics.py index b7ad870e60..356119fde2 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/metrics.py +++ b/packages/flow/benchmark/harness/effectbench/metrics.py @@ -1,7 +1,7 @@ """EffectBench metrics — SWER and its co-metrics, with confidence intervals. Every headline is a pure aggregation over a list of -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord` rows, so a +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord` rows, so a run's numbers can be recomputed from the published raw rows by anyone. No single mean is reported alone: :func:`summarize` always returns the per (category x substrate) decomposition alongside the overall rate (design doc @@ -39,7 +39,7 @@ from pydantic import BaseModel, ConfigDict, Field -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( EpisodeRecord, OutcomeLabel, SwerVariant, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/oracle.py b/packages/flow/benchmark/harness/effectbench/oracle.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/effectbench/oracle.py rename to packages/flow/benchmark/harness/effectbench/oracle.py index f7f5ca031a..e490479d8b 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/oracle.py +++ b/packages/flow/benchmark/harness/effectbench/oracle.py @@ -17,7 +17,7 @@ 2. **The classifier** — :func:`classify_outcome`, which crosses the oracle's reading of the true effect with the agent's (untrusted) self-report to - assign exactly one :class:`~openadapt_flow.benchmark.effectbench.schema.OutcomeLabel`. + assign exactly one :class:`~benchmark.harness.effectbench.schema.OutcomeLabel`. This is ``fault_model.classify`` generalized: it adds the OVER_HALT vs SAFE_HALT split (was the correct action available?) and makes ``SILENT_WRONG_EFFECT`` the single home of ``reported_success AND @@ -25,7 +25,7 @@ :func:`score_episode` ties the two together for a runner: capture pre-state, run the arm's action, verify, classify, and emit an -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord`. +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord`. """ from __future__ import annotations @@ -34,7 +34,7 @@ from enum import Enum from typing import Any, Callable, Mapping, Optional -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( AgentReport, DivergenceCategory, EpisodeRecord, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/README.md b/packages/flow/benchmark/harness/effectbench/runner/README.md similarity index 95% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/README.md rename to packages/flow/benchmark/harness/effectbench/runner/README.md index a29834a364..a00d55f331 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/README.md +++ b/packages/flow/benchmark/harness/effectbench/runner/README.md @@ -1,7 +1,7 @@ # EffectBench runner adapter — one interface, every baseline The **multi-baseline runner adapter** (downstream effort #2 on the -`openadapt_flow.benchmark.effectbench` contract). It drives every agent *arm* +`benchmark.harness.effectbench` contract). It drives every agent *arm* through one common interface against the **identical task** and **identical independent oracle**, records one `EpisodeRecord` per (task × arm × trial), and summarizes per arm with `summarize`. A difference between arms is then the @@ -10,8 +10,8 @@ AGENT, never the harness. Run the CI-fast dry-run (real in-process HTTP, no Docker, no paid API, no spend): ```bash -python -m openadapt_flow.benchmark.effectbench.runner # all live arms -python -m openadapt_flow.benchmark.effectbench.runner --list-arms +python -m benchmark.harness.effectbench.runner # all live arms +python -m benchmark.harness.effectbench.runner --list-arms ``` ## The arm interface diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__init__.py b/packages/flow/benchmark/harness/effectbench/runner/__init__.py similarity index 72% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/__init__.py rename to packages/flow/benchmark/harness/effectbench/runner/__init__.py index 759ebae7d1..27fe3e39ff 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__init__.py +++ b/packages/flow/benchmark/harness/effectbench/runner/__init__.py @@ -2,20 +2,20 @@ Drives every agent *arm* through one common interface against the IDENTICAL task and INDEPENDENT oracle, records one -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord` per +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord` per (task × arm × trial), and summarizes per arm with -:func:`~openadapt_flow.benchmark.effectbench.metrics.summarize`. Built on the -stable contract in :mod:`openadapt_flow.benchmark.effectbench` (schema / oracle / +:func:`~benchmark.harness.effectbench.metrics.summarize`. Built on the +stable contract in :mod:`benchmark.harness.effectbench` (schema / oracle / metrics) — this package adds only the runner, never re-implements the taxonomy. Layers: - :mod:`.arms` — the ``AgentArm`` interface + the concrete in-repo arms: - :class:`~openadapt_flow.benchmark.effectbench.runner.arms.CompilerArm` + :class:`~benchmark.harness.effectbench.runner.arms.CompilerArm` (record→compile→replay, effect-gated), - :class:`~openadapt_flow.benchmark.effectbench.runner.arms.ScreenOnlyArm` + :class:`~benchmark.harness.effectbench.runner.arms.ScreenOnlyArm` (the silent-wrong-effect ablation), and a substrate-free - :class:`~openadapt_flow.benchmark.effectbench.runner.arms.MockArm` for CI. + :class:`~benchmark.harness.effectbench.runner.arms.MockArm` for CI. - :mod:`.substrate` — the arm-facing action+perception channel (the reference MockMed session), isolated from the oracle read path. - :mod:`.compound` — the compound consequential-save verifier (record + note). @@ -25,19 +25,19 @@ - :mod:`.harness` — ``run_episode`` / ``run_matrix``: wire each arm through ``score_episode`` with the task's oracle, fairly and identically. -Run the dry-run: ``python -m openadapt_flow.benchmark.effectbench.runner``. +Run the dry-run: ``python -m benchmark.harness.effectbench.runner``. """ from __future__ import annotations -from openadapt_flow.benchmark.effectbench.runner.arms import ( +from benchmark.harness.effectbench.runner.arms import ( AgentArm, ArmResult, CompilerArm, MockArm, ScreenOnlyArm, ) -from openadapt_flow.benchmark.effectbench.runner.baselines import ( +from benchmark.harness.effectbench.runner.baselines import ( SCAFFOLDED_ARMS, BaselineRequirements, ClaudeComputerUseArm, @@ -46,21 +46,21 @@ SkyvernArm, UITarsArm, ) -from openadapt_flow.benchmark.effectbench.runner.compound import ( +from benchmark.harness.effectbench.runner.compound import ( CompoundEffectVerifier, ) -from openadapt_flow.benchmark.effectbench.runner.harness import ( +from benchmark.harness.effectbench.runner.harness import ( EnvFactory, EpisodeEnv, run_episode, run_matrix, ) -from openadapt_flow.benchmark.effectbench.runner.reference_tasks import ( +from benchmark.harness.effectbench.runner.reference_tasks import ( MockMedEnvProvider, mockmed_env_factory, reference_tasks, ) -from openadapt_flow.benchmark.effectbench.runner.substrate import ( +from benchmark.harness.effectbench.runner.substrate import ( MockMedFault, MockMedSession, ScreenObservation, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__main__.py b/packages/flow/benchmark/harness/effectbench/runner/__main__.py similarity index 90% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/__main__.py rename to packages/flow/benchmark/harness/effectbench/runner/__main__.py index 4ba0d142eb..bc6c79734f 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__main__.py +++ b/packages/flow/benchmark/harness/effectbench/runner/__main__.py @@ -1,6 +1,6 @@ """EffectBench runner CLI — the CI-fast dry-run over the MockMed reference pack. -``python -m openadapt_flow.benchmark.effectbench.runner`` drives the nine MockMed +``python -m benchmark.harness.effectbench.runner`` drives the nine MockMed reference tasks through the in-repo LIVE arms (compiler, screen-only ablation, mock) over a real in-process HTTP fault server — no Docker, no paid API, no spend — and prints the per-arm summary. It proves the pipeline end-to-end: the @@ -25,16 +25,16 @@ import sys from typing import Optional, Sequence -from openadapt_flow.benchmark.effectbench.metrics import BenchmarkSummary, summarize -from openadapt_flow.benchmark.effectbench.runner import LIVE_ARMS -from openadapt_flow.benchmark.effectbench.runner.arms import AgentArm -from openadapt_flow.benchmark.effectbench.runner.baselines import SCAFFOLDED_ARMS -from openadapt_flow.benchmark.effectbench.runner.harness import run_matrix -from openadapt_flow.benchmark.effectbench.runner.reference_tasks import ( +from benchmark.harness.effectbench.metrics import BenchmarkSummary, summarize +from benchmark.harness.effectbench.runner import LIVE_ARMS +from benchmark.harness.effectbench.runner.arms import AgentArm +from benchmark.harness.effectbench.runner.baselines import SCAFFOLDED_ARMS +from benchmark.harness.effectbench.runner.harness import run_matrix +from benchmark.harness.effectbench.runner.reference_tasks import ( MockMedEnvProvider, reference_tasks, ) -from openadapt_flow.benchmark.effectbench.schema import EpisodeRecord +from benchmark.harness.effectbench.schema import EpisodeRecord def _select_arms(names: Optional[str]) -> list[AgentArm]: @@ -128,7 +128,7 @@ def run_dry_run( def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser( - prog="python -m openadapt_flow.benchmark.effectbench.runner", + prog="python -m benchmark.harness.effectbench.runner", description="EffectBench multi-baseline runner — MockMed dry-run.", ) parser.add_argument("--trials", type=int, default=3) diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/arms.py b/packages/flow/benchmark/harness/effectbench/runner/arms.py similarity index 95% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/arms.py rename to packages/flow/benchmark/harness/effectbench/runner/arms.py index 063389bcef..a63a5985c3 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/arms.py +++ b/packages/flow/benchmark/harness/effectbench/runner/arms.py @@ -18,14 +18,14 @@ def run(self, task: TaskSpec, session: SubstrateSession, An arm is handed ONLY the task's natural-language ``goal`` (intent, never a step list — the fairness requirement) via ``task``, a -:class:`~openadapt_flow.benchmark.effectbench.runner.substrate.SubstrateSession` +:class:`~benchmark.harness.effectbench.runner.substrate.SubstrateSession` to drive, and the run's ``params`` (the trial-unique payload it must write). It returns an :class:`ArmResult` carrying the untrusted -:class:`~openadapt_flow.benchmark.effectbench.schema.AgentReport` plus any -recorded :class:`~openadapt_flow.benchmark.effectbench.schema.ModelCall` rows so +:class:`~benchmark.harness.effectbench.schema.AgentReport` plus any +recorded :class:`~benchmark.harness.effectbench.schema.ModelCall` rows so cost is auditable. The harness (:mod:`.harness`) adapts ``run`` into the ``run_action: Callable[[], AgentReport]`` that -:func:`~openadapt_flow.benchmark.effectbench.oracle.score_episode` calls, and +:func:`~benchmark.harness.effectbench.oracle.score_episode` calls, and supplies the independent oracle — which the arm can NEVER reach. Three concrete in-repo arms live here: @@ -50,11 +50,11 @@ def run(self, task: TaskSpec, session: SubstrateSession, from dataclasses import dataclass, field from typing import Any, Callable, Mapping, Optional, Protocol, runtime_checkable -from openadapt_flow.benchmark.effectbench.runner.substrate import ( +from benchmark.harness.effectbench.runner.substrate import ( ScreenObservation, SubstrateSession, ) -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( AgentReport, ModelCall, TaskSpec, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/baselines.py b/packages/flow/benchmark/harness/effectbench/runner/baselines.py similarity index 97% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/baselines.py rename to packages/flow/benchmark/harness/effectbench/runner/baselines.py index 3674a508ce..5a43fbf73d 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/baselines.py +++ b/packages/flow/benchmark/harness/effectbench/runner/baselines.py @@ -21,7 +21,7 @@ side channel to the system of record; the independent oracle, not the agent, judges the effect. 3. **Record every model call.** Append one - :class:`~openadapt_flow.benchmark.effectbench.schema.ModelCall` per API + :class:`~benchmark.harness.effectbench.schema.ModelCall` per API request with token counts and the pinned list price, so ``cost_usd`` is auditable ``sum(model_calls)``. An RPA tool with a flat run fee records a single ``ModelCall`` carrying that fee. @@ -38,9 +38,9 @@ from dataclasses import dataclass from typing import Mapping -from openadapt_flow.benchmark.effectbench.runner.arms import ArmResult -from openadapt_flow.benchmark.effectbench.runner.substrate import SubstrateSession -from openadapt_flow.benchmark.effectbench.schema import TaskSpec +from benchmark.harness.effectbench.runner.arms import ArmResult +from benchmark.harness.effectbench.runner.substrate import SubstrateSession +from benchmark.harness.effectbench.schema import TaskSpec class ScaffoldNotWired(NotImplementedError): diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/compound.py b/packages/flow/benchmark/harness/effectbench/runner/compound.py similarity index 94% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/compound.py rename to packages/flow/benchmark/harness/effectbench/runner/compound.py index 1c6236449a..3c15f795f5 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/compound.py +++ b/packages/flow/benchmark/harness/effectbench/runner/compound.py @@ -9,16 +9,16 @@ * ``field_equals`` — that encounter's note reads back the run's exact value; catches a *partial save* that persists the row but drops a field. -:class:`~openadapt_flow.benchmark.effectbench.oracle.score_episode` verifies a +:class:`~benchmark.harness.effectbench.oracle.score_episode` verifies a SINGLE :class:`Effect` through the oracle. This module supplies a thin :class:`~openadapt_flow.runtime.effects.effect.EffectVerifier` that wraps ANY concrete base verifier (the in-process -:class:`~openadapt_flow.benchmark.effectbench.oracle.RecordSnapshotOracle`, a +:class:`~benchmark.harness.effectbench.oracle.RecordSnapshotOracle`, a :class:`~openadapt_flow.runtime.effects.rest.RestRecordVerifier`, …) and, when ``score_episode`` calls :meth:`verify` with the primary ``record_written`` effect, ALSO checks the authored extra sub-effects against the SAME pre-state snapshot, reducing the per-effect true-effect states with -:func:`~openadapt_flow.benchmark.effectbench.oracle.combine_true_states`. +:func:`~benchmark.harness.effectbench.oracle.combine_true_states`. It returns the single **deciding** :class:`EffectVerdict` whose ``effect_state`` equals the combined state (the fault-model precedence: the @@ -33,7 +33,7 @@ from typing import Any, Sequence -from openadapt_flow.benchmark.effectbench.oracle import ( +from benchmark.harness.effectbench.oracle import ( TrueEffectState, combine_true_states, effect_state, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/harness.py b/packages/flow/benchmark/harness/effectbench/runner/harness.py similarity index 92% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/harness.py rename to packages/flow/benchmark/harness/effectbench/runner/harness.py index f21991fed0..f1bafeefbf 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/harness.py +++ b/packages/flow/benchmark/harness/effectbench/runner/harness.py @@ -1,11 +1,11 @@ """The runner harness — wire each arm through ``score_episode`` identically. This is the multi-baseline runner's core: given a set of -:class:`~openadapt_flow.benchmark.effectbench.schema.TaskSpec` and a set of arms, +:class:`~benchmark.harness.effectbench.schema.TaskSpec` and a set of arms, it produces one -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord` per +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord` per (task × arm × trial), each scored by the task's INDEPENDENT oracle through -:func:`~openadapt_flow.benchmark.effectbench.oracle.score_episode`. Every arm +:func:`~benchmark.harness.effectbench.oracle.score_episode`. Every arm runs the SAME task and is judged by the SAME oracle — the harness guarantees the comparison is fair by construction: @@ -30,10 +30,10 @@ from dataclasses import dataclass, field from typing import Any, Callable, List, Mapping, Optional, Sequence -from openadapt_flow.benchmark.effectbench.oracle import score_episode -from openadapt_flow.benchmark.effectbench.runner.arms import AgentArm -from openadapt_flow.benchmark.effectbench.runner.substrate import SubstrateSession -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.oracle import score_episode +from benchmark.harness.effectbench.runner.arms import AgentArm +from benchmark.harness.effectbench.runner.substrate import SubstrateSession +from benchmark.harness.effectbench.schema import ( AgentReport, EpisodeRecord, ModelCall, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/reference_tasks.py b/packages/flow/benchmark/harness/effectbench/runner/reference_tasks.py similarity index 96% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/reference_tasks.py rename to packages/flow/benchmark/harness/effectbench/runner/reference_tasks.py index 6afe49e2be..9c4c8de44e 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/reference_tasks.py +++ b/packages/flow/benchmark/harness/effectbench/runner/reference_tasks.py @@ -3,7 +3,7 @@ These are the reference tasks the multi-baseline runner exercises end-to-end without Docker or spend: the bundled MockMed transactional-fault surface (``openadapt_flow.mockmed.fault_server``) turned into -:class:`~openadapt_flow.benchmark.effectbench.schema.TaskSpec` s across the +:class:`~benchmark.harness.effectbench.schema.TaskSpec` s across the divergence categories, plus a :func:`mockmed_env_factory` that provisions one isolated fault server per episode with an INDEPENDENT in-process oracle. @@ -24,17 +24,17 @@ from dataclasses import dataclass from typing import Mapping, Optional -from openadapt_flow.benchmark.effectbench.oracle import RecordSnapshotOracle -from openadapt_flow.benchmark.effectbench.runner.compound import ( +from benchmark.harness.effectbench.oracle import RecordSnapshotOracle +from benchmark.harness.effectbench.runner.compound import ( CompoundEffectVerifier, ) -from openadapt_flow.benchmark.effectbench.runner.harness import EpisodeEnv -from openadapt_flow.benchmark.effectbench.runner.substrate import ( +from benchmark.harness.effectbench.runner.harness import EpisodeEnv +from benchmark.harness.effectbench.runner.substrate import ( MockMedFault, MockMedSession, mockmed_session, ) -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( DivergenceCategory, OracleChannel, OracleSpec, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/substrate.py b/packages/flow/benchmark/harness/effectbench/runner/substrate.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/substrate.py rename to packages/flow/benchmark/harness/effectbench/runner/substrate.py index 3d4e525528..1322d730f6 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/substrate.py +++ b/packages/flow/benchmark/harness/effectbench/runner/substrate.py @@ -34,10 +34,10 @@ from dataclasses import dataclass, field from typing import Any, Mapping, Optional, Protocol, Sequence, runtime_checkable -from openadapt_flow.benchmark.effectbench.runner.compound import ( +from benchmark.harness.effectbench.runner.compound import ( CompoundEffectVerifier, ) -from openadapt_flow.benchmark.effectbench.schema import Substrate +from benchmark.harness.effectbench.schema import Substrate from openadapt_flow.runtime.effects.effect import Effect, EffectVerifier diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/schema.py b/packages/flow/benchmark/harness/effectbench/schema.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/effectbench/schema.py rename to packages/flow/benchmark/harness/effectbench/schema.py index 4e529ae716..d92fb0af90 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/schema.py +++ b/packages/flow/benchmark/harness/effectbench/schema.py @@ -135,7 +135,7 @@ class OracleChannel(str, Enum): Maps 1:1 onto the ``substrate`` attribute of the concrete :class:`~openadapt_flow.runtime.effects.effect.EffectVerifier` - implementations re-exported by :mod:`openadapt_flow.benchmark.effectbench`. + implementations re-exported by :mod:`benchmark.harness.effectbench`. """ SQL = "sql" # SqlRecordVerifier — read-only SELECT under a read-only role @@ -311,7 +311,7 @@ class EpisodeRecord(BaseModel): """One trial of one task by one arm — the atomic benchmark result row. Every headline metric is a pure aggregation over a list of these (see - :func:`openadapt_flow.benchmark.effectbench.metrics.summarize`). The runner + :func:`benchmark.harness.effectbench.metrics.summarize`). The runner produces one per (task x arm x trial); ``summarize`` never needs anything the row does not carry, so raw rows can be published for external replication. """ diff --git a/packages/flow/openadapt_flow/benchmark/frappe_lending.py b/packages/flow/benchmark/harness/frappe_lending.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/frappe_lending.py rename to packages/flow/benchmark/harness/frappe_lending.py diff --git a/packages/flow/openadapt_flow/benchmark/hybrid_benchmark.py b/packages/flow/benchmark/harness/hybrid_benchmark.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/hybrid_benchmark.py rename to packages/flow/benchmark/harness/hybrid_benchmark.py index c343d13b45..282bb79cbb 100644 --- a/packages/flow/openadapt_flow/benchmark/hybrid_benchmark.py +++ b/packages/flow/benchmark/harness/hybrid_benchmark.py @@ -67,12 +67,12 @@ from pydantic import BaseModel -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.openemr_benchmark import ( +from benchmark.harness import agent_baseline +from benchmark.harness.openemr_benchmark import ( BILLING_ABORT_AFTER, _looks_like_billing_error, ) -from openadapt_flow.benchmark.run_benchmark import ( +from benchmark.harness.run_benchmark import ( _agent_run, _arm_aggregate, _compiled_run, @@ -863,7 +863,7 @@ def render_hybrid_chart(results: dict[str, Any], out_png: Path) -> Path: Returns: The written PNG path. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() @@ -1379,7 +1379,7 @@ def failures(arm: str) -> str: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.hybrid_benchmark --out benchmark/hybrid +.venv/bin/python -m benchmark.harness.hybrid_benchmark --out benchmark/hybrid ``` Requires `ANTHROPIC_API_KEY` (or `~/.anthropic/api_key`). The paid arms @@ -1398,7 +1398,7 @@ def write_hybrid_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_hybrid_chart, results, out_dir / "success_cost.png") (out_dir / "BENCHMARK.md").write_text(render_hybrid_markdown(results)) @@ -1649,7 +1649,7 @@ def _error_row(arm: str, exc: Exception) -> dict[str, Any]: def main(argv: Optional[list[str]] = None) -> int: - """CLI entry point: ``python -m openadapt_flow.benchmark.hybrid_benchmark``.""" + """CLI entry point: ``python -m benchmark.harness.hybrid_benchmark``.""" parser = argparse.ArgumentParser( description=( "Four-arm hybrid benchmark on MockMed (compiled / agent / " diff --git a/packages/flow/openadapt_flow/instrument/__init__.py b/packages/flow/benchmark/harness/instrument/__init__.py similarity index 91% rename from packages/flow/openadapt_flow/instrument/__init__.py rename to packages/flow/benchmark/harness/instrument/__init__.py index bf92686661..dbf7469cc1 100644 --- a/packages/flow/openadapt_flow/instrument/__init__.py +++ b/packages/flow/benchmark/harness/instrument/__init__.py @@ -1,7 +1,7 @@ """Competitor-drift instrument: point our EffectVerifier at ANY external agent. The self-directed silent-wrong-action benchmark -(``openadapt_flow.benchmark.silent_wrong_action``, #67) measures how often our +(``benchmark.harness.silent_wrong_action``, #67) measures how often our OWN runtime lets a wrong / absent / duplicate business effect land while reporting success. This package generalizes that measurement to an *arbitrary external computer-use agent*: given any agent behind the @@ -19,7 +19,7 @@ user-gated step. """ -from openadapt_flow.instrument.competitor_drift import ( # noqa: F401 +from benchmark.harness.instrument.competitor_drift import ( # noqa: F401 AgentRunResult, ArchitectureClassError, CostGuard, diff --git a/packages/flow/openadapt_flow/instrument/competitor_drift.py b/packages/flow/benchmark/harness/instrument/competitor_drift.py similarity index 99% rename from packages/flow/openadapt_flow/instrument/competitor_drift.py rename to packages/flow/benchmark/harness/instrument/competitor_drift.py index 3daa5076c6..2abf9fe980 100644 --- a/packages/flow/openadapt_flow/instrument/competitor_drift.py +++ b/packages/flow/benchmark/harness/instrument/competitor_drift.py @@ -2,7 +2,7 @@ runner, cost-capped. This EXTENDS the self-directed silent-wrong-action benchmark -(``openadapt_flow.benchmark.silent_wrong_action``, #67) from "our own runtime" +(``benchmark.harness.silent_wrong_action``, #67) from "our own runtime" to "any external computer-use agent." The measurement is identical — did a wrong / absent / duplicate business effect land in the system of record while the actor reported success (a *silent wrong-action*)? — but the actor is now an @@ -53,7 +53,7 @@ # judge, and the #63 effect contract. We REUSE these from the self-directed # benchmark rather than re-deriving them so the external-agent instrument and # the self-directed instrument grade with byte-identical semantics. -from openadapt_flow.benchmark.silent_wrong_action import ( +from benchmark.harness.silent_wrong_action import ( NOTE, SCENARIOS, Scenario, @@ -888,7 +888,7 @@ def write_outputs(report: InstrumentReport, out_dir: Path | str) -> None: def main(argv: Optional[list[str]] = None) -> int: - """CLI: ``python -m openadapt_flow.instrument.competitor_drift``. + """CLI: ``python -m benchmark.harness.instrument.competitor_drift``. Ships only the offline stub adapter and a dry-run estimator — a real external adapter is wired programmatically (see diff --git a/packages/flow/openadapt_flow/benchmark/openemr_benchmark.py b/packages/flow/benchmark/harness/openemr_benchmark.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/openemr_benchmark.py rename to packages/flow/benchmark/harness/openemr_benchmark.py index f2482f2307..5bb2396f11 100644 --- a/packages/flow/openadapt_flow/benchmark/openemr_benchmark.py +++ b/packages/flow/benchmark/harness/openemr_benchmark.py @@ -7,7 +7,7 @@ recorded by ``scripts/openemr_demo.py`` (fake demo patients only). Both arms are judged by the same criterion, implemented once -(:func:`openadapt_flow.benchmark.verify.verify_note_saved`): OCR of the +(:func:`benchmark.harness.verify.verify_note_saved`): OCR of the final screen must show the run's parameterized note text in the patient message list. Each run — in BOTH arms — uses a distinct note value, so a pass proves parameter substitution against live state, not replay of a @@ -60,16 +60,16 @@ from pathlib import Path from typing import Any, Callable -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.agent_baseline import _looks_like_billing_error -from openadapt_flow.benchmark.run_benchmark import ( +from benchmark.harness import agent_baseline +from benchmark.harness.agent_baseline import _looks_like_billing_error +from benchmark.harness.run_benchmark import ( _agent_run, _arm_aggregate, _compiled_run, identity_coverage_block, render_chart, ) -from openadapt_flow.benchmark.verify import verify_note_saved +from benchmark.harness.verify import verify_note_saved DEMO_URL = "https://demo.openemr.io/openemr/index.php" N_COMPILED = 20 @@ -481,7 +481,7 @@ def write_openemr_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "latency_cost.png") (out_dir / "BENCHMARK.md").write_text(render_openemr_markdown(results)) diff --git a/packages/flow/openadapt_flow/benchmark/openemr_local.py b/packages/flow/benchmark/harness/openemr_local.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/openemr_local.py rename to packages/flow/benchmark/harness/openemr_local.py index 125f6d9c24..1a4e337de8 100644 --- a/packages/flow/openadapt_flow/benchmark/openemr_local.py +++ b/packages/flow/benchmark/harness/openemr_local.py @@ -18,7 +18,7 @@ from typing import Any, Iterable, Mapping, Sequence from urllib.parse import urlencode -from openadapt_flow.benchmark.frappe_lending import ( +from benchmark.harness.frappe_lending import ( ARMS, CONDITIONS, INITIAL_TRIALS_PER_CELL, diff --git a/packages/flow/openadapt_flow/benchmark/reliability.py b/packages/flow/benchmark/harness/reliability.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/reliability.py rename to packages/flow/benchmark/harness/reliability.py diff --git a/packages/flow/openadapt_flow/benchmark/run_benchmark.py b/packages/flow/benchmark/harness/run_benchmark.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/run_benchmark.py rename to packages/flow/benchmark/harness/run_benchmark.py index cd2c4c2d1e..3aba486faf 100644 --- a/packages/flow/openadapt_flow/benchmark/run_benchmark.py +++ b/packages/flow/benchmark/harness/run_benchmark.py @@ -29,8 +29,8 @@ from typing import Any, Callable from openadapt_flow.bench import _percentile -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.verify import verify_encounter_saved +from benchmark.harness import agent_baseline +from benchmark.harness.verify import verify_encounter_saved N_COMPILED = 100 N_AGENT = 20 @@ -153,7 +153,7 @@ def _agent_run( A per-run row dict. API failures are recorded as failed rows with ``error`` set, never raised; a failed row still carries whatever usage/cost the run paid for before crashing (via - :class:`~openadapt_flow.benchmark.agent_baseline.UsageLedger`), so + :class:`~benchmark.harness.agent_baseline.UsageLedger`), so crashed runs' real spend counts in aggregates and cost ceilings. """ from openadapt_flow.backends.playwright_backend import PlaywrightBackend @@ -364,7 +364,7 @@ def render_chart(results: dict[str, Any], out_png: Path) -> Path: Returns: The written PNG path. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() @@ -578,7 +578,7 @@ def write_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "latency_cost.png") (out_dir / "BENCHMARK.md").write_text(render_markdown(results)) diff --git a/packages/flow/openadapt_flow/benchmark/silent_wrong_action.py b/packages/flow/benchmark/harness/silent_wrong_action.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/silent_wrong_action.py rename to packages/flow/benchmark/harness/silent_wrong_action.py index 009bdda54c..daeb0c7c9f 100644 --- a/packages/flow/openadapt_flow/benchmark/silent_wrong_action.py +++ b/packages/flow/benchmark/harness/silent_wrong_action.py @@ -566,7 +566,7 @@ def render_markdown(results: dict[str, Any]) -> str: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.silent_wrong_action \\ +.venv/bin/python -m benchmark.harness.silent_wrong_action \\ --out {DEFAULT_OUT_DIR} --n {results["n_per_scenario"]} ``` @@ -585,7 +585,7 @@ def render_chart(results: dict[str, Any], out_png: Path) -> Path: false-abort rate. The metric a record system cares about — a wrong write that reports success — collapses from a large bar to zero. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() @@ -658,7 +658,7 @@ def write_outputs(results: dict[str, Any], out_dir: Path) -> None: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") (out_dir / "SILENT_WRONG_ACTION.md").write_text(render_markdown(results)) - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "silent_wrong_action.png") @@ -689,7 +689,7 @@ def run_and_write( def main(argv: Optional[list[str]] = None) -> int: - """CLI: ``python -m openadapt_flow.benchmark.silent_wrong_action``.""" + """CLI: ``python -m benchmark.harness.silent_wrong_action``.""" parser = argparse.ArgumentParser( description=( "Silent-wrong-action-rate benchmark: measure how often " diff --git a/packages/flow/openadapt_flow/benchmark/verify.py b/packages/flow/benchmark/harness/verify.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/verify.py rename to packages/flow/benchmark/harness/verify.py diff --git a/packages/flow/benchmark/hybrid/BENCHMARK.md b/packages/flow/benchmark/hybrid/BENCHMARK.md index bc64083f0d..d85401a2a7 100644 --- a/packages/flow/benchmark/hybrid/BENCHMARK.md +++ b/packages/flow/benchmark/hybrid/BENCHMARK.md @@ -207,7 +207,7 @@ $0.0967), and `a` = the agent-only mean cost per run ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.hybrid_benchmark --out benchmark/hybrid +.venv/bin/python -m benchmark.harness.hybrid_benchmark --out benchmark/hybrid ``` Requires `ANTHROPIC_API_KEY` (or `~/.anthropic/api_key`). The paid arms diff --git a/packages/flow/benchmark/lending_fault_model/README.md b/packages/flow/benchmark/lending_fault_model/README.md index 740b9134ff..b6de3a698a 100644 --- a/packages/flow/benchmark/lending_fault_model/README.md +++ b/packages/flow/benchmark/lending_fault_model/README.md @@ -39,7 +39,7 @@ judged by an independent ledger read, with zero model calls. 2. **Silent Wrong-Effect Rate, three-arm verification ladder** - `swer.py` scores the same faults through the shared EffectBench contract - (`openadapt_flow.benchmark.effectbench.score_episode` + `summarize`) under + (`benchmark.harness.effectbench.score_episode` + `summarize`) under screen-only, single-surface, and complete-read-path arms. -> `swer_results.json`, `SWER.md`. Headline: screen-only SWER is 24/36, a disbursements-only verifier leaves the diff --git a/packages/flow/benchmark/lending_fault_model/swer.py b/packages/flow/benchmark/lending_fault_model/swer.py index 9471d68615..312cc57e4a 100644 --- a/packages/flow/benchmark/lending_fault_model/swer.py +++ b/packages/flow/benchmark/lending_fault_model/swer.py @@ -5,7 +5,7 @@ screen-only postcondition contract (and measures how often that silently mishandles a transactional fault), THIS module measures the same faults through the shared EffectBench scoring contract -(:func:`openadapt_flow.benchmark.effectbench.score_episode`) under a THREE-arm +(:func:`benchmark.harness.effectbench.score_episode`) under a THREE-arm ladder that mirrors the clinical ``effect_e2e`` study (screen / single-surface / complete-read-path), so the two domains are directly comparable: @@ -71,7 +71,7 @@ import requests -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, DivergenceCategory, Effect, diff --git a/packages/flow/benchmark/openemr_api_tier/approval_durable.py b/packages/flow/benchmark/openemr_api_tier/approval_durable.py index 7b3e57f929..47dbe33709 100644 --- a/packages/flow/benchmark/openemr_api_tier/approval_durable.py +++ b/packages/flow/benchmark/openemr_api_tier/approval_durable.py @@ -157,7 +157,7 @@ def run_one( ) -> dict[str, Any]: from openadapt_flow.approval_mode.gate import ApprovalGate from openadapt_flow.approval_mode.item_approval import ItemApprovalStore - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer from openadapt_flow.runtime.durable.checkpoint import CheckpointStore diff --git a/packages/flow/benchmark/openemr_api_tier/common.py b/packages/flow/benchmark/openemr_api_tier/common.py index 19bb064547..8764d08f22 100644 --- a/packages/flow/benchmark/openemr_api_tier/common.py +++ b/packages/flow/benchmark/openemr_api_tier/common.py @@ -74,7 +74,7 @@ def load_demo() -> Any: def spec_for(ref: Referral) -> Any: - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec return SyntheticPatientSpec(**ref.kwargs) diff --git a/packages/flow/benchmark/openemr_api_tier/feature_e2e.py b/packages/flow/benchmark/openemr_api_tier/feature_e2e.py index 35754d3a4d..eccb0e8d1f 100644 --- a/packages/flow/benchmark/openemr_api_tier/feature_e2e.py +++ b/packages/flow/benchmark/openemr_api_tier/feature_e2e.py @@ -51,7 +51,7 @@ def cmd_record(a: argparse.Namespace) -> None: - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec from openadapt_flow.network_capture import NetworkCapture from openadapt_flow.recorder import Recorder @@ -190,7 +190,7 @@ def wait_quiet(max_s: float = float(os.environ.get("GATE_MAX_S", "1200"))) -> di def run_one(env: dict[str, Any], arm: str, ref_idx: int, bundle: Path, out: Path) -> dict[str, Any]: - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer diff --git a/packages/flow/benchmark/openemr_api_tier/spike/bench.py b/packages/flow/benchmark/openemr_api_tier/spike/bench.py index d3a74c2f53..138570dcca 100644 --- a/packages/flow/benchmark/openemr_api_tier/spike/bench.py +++ b/packages/flow/benchmark/openemr_api_tier/spike/bench.py @@ -62,7 +62,7 @@ def _refuse(self, *_a: Any, **_k: Any) -> Any: class Env: def __init__(self, browser: bool) -> None: - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer diff --git a/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py b/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py index 12929d4e07..5e6297d2ca 100644 --- a/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py +++ b/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py @@ -39,7 +39,7 @@ def main() -> None: from playwright.sync_api import sync_playwright - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer diff --git a/packages/flow/benchmark/openemr_e2e/README.md b/packages/flow/benchmark/openemr_e2e/README.md index 39fdb23bbe..68471ac1f1 100644 --- a/packages/flow/benchmark/openemr_e2e/README.md +++ b/packages/flow/benchmark/openemr_e2e/README.md @@ -10,7 +10,7 @@ compile → replay → effect-verify (system of record) → catch a silent wrong ``` It does **not** duplicate the existing OpenEMR *benchmark* -(`openadapt_flow.benchmark.openemr_benchmark`, which measures the compiled arm +(`benchmark.harness.openemr_benchmark`, which measures the compiled arm vs. a paid agent arm on the **live** public demo and is explicitly *not* CI-reproducible). This harness *orchestrates* the runtime components that benchmark assumes work — the compiler, the `Replayer`, the `EffectVerifier`, the diff --git a/packages/flow/benchmark/openemr_e2e/harness.py b/packages/flow/benchmark/openemr_e2e/harness.py index 23a10fc498..5110b31c77 100644 --- a/packages/flow/benchmark/openemr_e2e/harness.py +++ b/packages/flow/benchmark/openemr_e2e/harness.py @@ -8,7 +8,7 @@ inject drift -> HALT -> teach the fix -> re-run clean Unlike the existing OpenEMR *benchmark* -(:mod:`openadapt_flow.benchmark.openemr_benchmark`), which measures the compiled +(:mod:`benchmark.harness.openemr_benchmark`), which measures the compiled arm vs. a paid agent arm on the LIVE public demo and is explicitly NOT CI-reproducible, this harness proves the pipeline *wiring* end to end on a deterministic, offline fixture (the in-process MockMed ``fault_server`` as the diff --git a/packages/flow/benchmark/openemr_local/README.md b/packages/flow/benchmark/openemr_local/README.md index 6a9441c40a..19efb71a28 100644 --- a/packages/flow/benchmark/openemr_local/README.md +++ b/packages/flow/benchmark/openemr_local/README.md @@ -33,7 +33,7 @@ publication bar. Raw per-run rows, environment fingerprints, application-specific adapter/oracle wiring, and the detailed cost ledger are retained in the private `OpenAdaptAI/openadapt-corpus` repository. The public generic cost-capped agent mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. See the +`benchmark/harness/agent_baseline.py`. See the [aggregate report](../agent_arm_verticals/README.md). The accepted run used baseline SHA-256 diff --git a/packages/flow/benchmark/openemr_throughput/harness.py b/packages/flow/benchmark/openemr_throughput/harness.py index 4bcd15079b..fcf9a2e817 100644 --- a/packages/flow/benchmark/openemr_throughput/harness.py +++ b/packages/flow/benchmark/openemr_throughput/harness.py @@ -237,7 +237,7 @@ def _demo_module() -> Any: def build_worker(user: str, opts: dict[str, Any]) -> Any: """One worker signed in as ``user``. Module-level so a child process can build it (``ProcessWorker``); ``opts`` holds only plain values.""" - from openadapt_flow.benchmark.openemr_local import ( + from benchmark.harness.openemr_local import ( OpenEMRPatientOracle, SyntheticPatientSpec, ) @@ -345,7 +345,7 @@ def run_item(page: Any, idx: int, row: dict[str, Any], item_dir: Path, pipeline: def run(a: argparse.Namespace) -> dict[str, Any]: from openadapt_flow._browser_setup import ensure_chromium_installed - from openadapt_flow.benchmark.openemr_local import ( + from benchmark.harness.openemr_local import ( SyntheticPatientSpec, ) from openadapt_flow.ir import Workflow diff --git a/packages/flow/benchmark/openimis_claims/README.md b/packages/flow/benchmark/openimis_claims/README.md index 94e94ad017..e01f263e40 100644 --- a/packages/flow/benchmark/openimis_claims/README.md +++ b/packages/flow/benchmark/openimis_claims/README.md @@ -28,7 +28,7 @@ Raw per-run rows, environment fingerprints, application-specific adapter/oracle wiring, and the detailed cost ledger are retained in the private `OpenAdaptAI/openadapt-corpus` repository. The public generic cost-capped agent mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. See the +`benchmark/harness/agent_baseline.py`. See the [aggregate report](../agent_arm_verticals/README.md). There is still no full timing matrix and no publication protocol here: a diff --git a/packages/flow/benchmark/scoreboard/tasks/openemr.py b/packages/flow/benchmark/scoreboard/tasks/openemr.py index c2ffaf02d6..e93b7eab6d 100644 --- a/packages/flow/benchmark/scoreboard/tasks/openemr.py +++ b/packages/flow/benchmark/scoreboard/tasks/openemr.py @@ -207,7 +207,7 @@ def check(values: dict[str, str], before_ids: set[str]) -> str: def oracle_verifier(values: dict[str, str]) -> Any: - from openadapt_flow.benchmark.openemr_local import ( + from benchmark.harness.openemr_local import ( OpenEMRPatientOracle, SyntheticPatientSpec, ) diff --git a/packages/flow/benchmark/silent_saves/metrics.py b/packages/flow/benchmark/silent_saves/metrics.py index cb7f3ee9f1..2a8fa76483 100644 --- a/packages/flow/benchmark/silent_saves/metrics.py +++ b/packages/flow/benchmark/silent_saves/metrics.py @@ -6,7 +6,7 @@ import statistics from typing import Any, Iterable, Optional -from openadapt_flow.benchmark.effectbench.metrics import wilson_interval +from benchmark.harness.effectbench.metrics import wilson_interval def rate(k: int, n: int) -> dict[str, Any]: diff --git a/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md b/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md index c09304b4b5..ce1dfe56c9 100644 --- a/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md +++ b/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md @@ -66,7 +66,7 @@ run-to-run disagreement); N proves reproducibility. ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.silent_wrong_action \ +.venv/bin/python -m benchmark.harness.silent_wrong_action \ --out benchmark/silent_wrong_action --n 10 ``` diff --git a/packages/flow/benchmark/vision_hardening/README.md b/packages/flow/benchmark/vision_hardening/README.md index 273c387138..46644fdc38 100644 --- a/packages/flow/benchmark/vision_hardening/README.md +++ b/packages/flow/benchmark/vision_hardening/README.md @@ -36,7 +36,7 @@ and CLASSIFIES what the **real** resolver does on each generated case: go DOWN. A hardening fix regenerates the corpus and lowers the ratchet. The headline metric is **SWER** (Silent Wrong-Effect/Resolution Rate), the same -taxonomy as EffectBench (`openadapt_flow/benchmark/effectbench/`) — this sweep +taxonomy as EffectBench (`benchmark/harness/effectbench/`) — this sweep is the *resolution-tier* feeder for that benchmark. ## The two tiers diff --git a/packages/flow/docs/EFFECT_KIT.md b/packages/flow/docs/EFFECT_KIT.md index 3739752c74..cb02250ed9 100644 --- a/packages/flow/docs/EFFECT_KIT.md +++ b/packages/flow/docs/EFFECT_KIT.md @@ -1420,7 +1420,7 @@ Runs against the pinned reference fixture README documents the pinned compose bring-up and fixture bootstrap). The bundle's effect contracts are exactly the ones the reference matrix ships -(`openadapt_flow/benchmark/frappe_lending.py::loan_application_effects`): +(`benchmark/harness/frappe_lending.py::loan_application_effects`): one `record_written` (at-most-once for the synthetic applicant) plus a `field_equals` read-back per entered field, each bound to `{param: ...}`. diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md new file mode 100644 index 0000000000..7fb257ecdb --- /dev/null +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -0,0 +1,311 @@ +# Generality audit of the engine + +Audited 2026-11-03 on `main` at `213f3d9fe`. Scope: everything under +`packages/flow/openadapt_flow/`. + +The rule: the engine has no code written for one application. What one +application needs lives in data (`app_profiles/*.yaml`, generated contracts, +deployment configs) or in test fixtures and benchmark harnesses outside the +shipped package. `scripts/check_engine_generality.py` enforces it in CI; see +[CONTRIBUTING.md](../CONTRIBUTING.md#app-knowledge-lives-in-data). + +## Method + +1. Searched for application names, case-insensitive: OpenEMR, Medplum, + Dolibarr, EspoCRM, Redmine, Snipe-IT, InvenTree, Kimai, OpenMRS, Gitea, + Firefly III, Vikunja, Zammad, Open Dental, Odoo, openIMIS, Frappe/ERPNext, + Patient Notes, Epic, Stedi, TextEdit, Notepad, and OpenAdapt's own demo apps + MockMed and MockLoan. +2. Searched for identifiers that only make sense for one application: table and + column names (`pnotes`, `patient_data`, `uuid_registry`, `user_settings`, + `tabLoan Application`), URL paths (`/api/db`, `/api/encounter`, + `/apis/default/fhir`), form and query parameters (`token_main`, + `csrf_token_form`, `clearpass`), selectors, button labels, dialog titles, + field names, and thresholds tuned on one application's screens. +3. Read the code around each hit and put it in one of four buckets: + - **(a)** a comment or docstring example. Fine; reworded where it read as a + rule. + - **(b)** application knowledge stored as data and loaded generically. + - **(c)** a test fixture, benchmark harness or demo application inside the + shipped package. Moved out, unless it backs a user-facing command that + must work from a wheel. + - **(d)** special-casing in engine logic. Generalized, or turned into data. + +## Counts + +| Bucket | Rows | Distinct findings | +| --- | --- | --- | +| (a) comment or docstring example | 118 | — | +| (b) data loaded generically | 4 | 2 (`app_profiles/openemr.yaml`, the `clinical-write` policy pack as a default) | +| (c) fixture, harness or demo in the package | 35 rows, plus 27 files of `benchmark/` | 19 | +| (d) special-casing in engine logic | 27 | 15 | + +Rows group consecutive lines of one hit. The full table is at the end. + +## (d) findings and actions + +Each finding names the application that motivated it. "Second app" is the +existing fixture that shows the generalized code still works beyond the +motivating application. + +| # | Where | What | Motivating app | Action | Status | +| --- | --- | --- | --- | --- | --- | +| D1 | `report_redaction.py:68-70`, `:131-133` | Exact secret names `token_main`, `csrf_token_form` | OpenEMR | Match secret names by pattern (`token`, `csrf`, `xsrf`, `session`, ...) instead of a per-app list | planned | +| D2 | `network_capture.py:78` | `clearpass` in the secret-name pattern | OpenEMR | Drop it; the `pass\b` alternative already matches | planned | +| D3 | `runtime/effects/rest.py:114-115`, `deployment.py:387-390`, `reward/models.py:226`, `fault_gate.py:722` | REST reader defaults `records_path="/api/db"`, `records_key="records"` | MockMed | Required when the effect kind is `rest`; refused at config load when missing | planned | +| D4 | `runtime/effects/compensation.py:364` | `delete_path_template="/api/encounter/{id}"` default | MockMed | Required | planned | +| D5 | `compiler/binding_admission.py:114-119`, `:71-73` | Admission write path defaults to `/api/encounter`; body skips MockMed's `source` field | MockMed | Write path and method from the observed `api_write` event; skip only server-assigned fields | planned | +| D6 | `compiler/effect_mining.py:100-102`, `runtime/effects/effect.py:900-901` | Idempotency key field defaults to `key` | MockMed | No default; from the contract or deployment config | planned | +| D7 | `runtime/effects/fhir.py:59-67`, `:221`, `deployment.py:448-449` | FHIR defaults: resource type `Observation`, `note -> valueString` | OpenEMR | Required in the contract (`generate/ensure_contract.py` writes them) | planned | +| D8 | `generate/ensure_contract.py:123-147` | Translates the Medplum probe script's output keys | Medplum | The probe emits the generic fact names; translation removed | planned | +| D9 | `runtime/effects/frame.py:1937`, `rollback_state.py:781` | `"0"` means "no record" | OpenEMR | The value moves into the profile rule (`no_record_values`) | planned | +| D10 | `__main__.py:2838-2934`, `:9305-9319`; `cli_variants.py:131,173-182` | `replay` and the variant probe fall back to the bundled MockMed when no URL is given; `--drift` turns off structural grounding | MockMed | Target URL required outside the demo commands | planned | +| D11 | `qualification_oracle_gate.py` | `qualify accept` tests every workflow's oracle against MockMed (banner text, `/api/encounter`, `/api/db`) | MockMed | Build the fault fixture from the bundle's own effect binding | planned | +| D12 | `execute/dispatch.py:111-188` | The Execute service's live runner always runs the MockMed tutorial workflow | MockMed | Replay through the admission's deployment config; MockMed only for synthetic admissions | planned | +| D13 | `compiler/compile.py:151`, `runtime/replayer.py:394`, `runtime/identity.py:108` | Thresholds measured on MockMed only | MockMed | Generic logic, one-app numbers. Recalibrate on two or more apps | follow-up issue | +| D14 | `runtime/identity.py:275-302` | Hand-written list of non-identifying words drawn from EMR list screens | MockMed, EMRs | Derive from the recording (tokens shared by sibling rows) | follow-up issue | +| D15 | `eligibility/client.py:37-41`, `:1091-1110` | One vendor's eligibility API (URL, key name, body) in the engine | Stedi | Move behind a connector interface | follow-up issue | + +## (c) findings and actions + +| Where | What | Action | Status | +| --- | --- | --- | --- | +| `benchmark/` (27 Python files: `run_benchmark`, `openemr_benchmark`, `openemr_local`, `frappe_lending`, `hybrid_benchmark`, `dom_arm`, `silent_wrong_action`, `desktop_benchmark`, `verify`, `agent_baseline`, `reliability`, `chart_fonts`, `effectbench/`) | Benchmark harnesses shipped in the wheel | Moved to `packages/flow/benchmark/harness/` (import `benchmark.harness.*`). `openadapt-flow benchmark` loads it from a source checkout and says so from a wheel | done | +| `instrument/` | Competitor-drift harness; imported the benchmark at module load, so it was already broken in a wheel | Moved to `packages/flow/benchmark/harness/instrument/` | done | +| `integrations/harness2/` | Harness2 research harness on MockMed | Move to `packages/flow/benchmark/harness/harness2/` | planned | +| `mockloan/`, `demo_driver.record_disbursement_demo` | Lending demo app no product command uses | Move to the benchmark tree | planned | +| `guard_demo.py`, `agent/ledger_demo.py` | `python -m` demos, no CLI command | Move to the benchmark tree | planned | +| `mockmed/teach_scope.py` | Used only by tests | Move to `tests/fixtures` | planned | +| `learning/synth_stream.py:1-245` | MockMed note stream | Split out the generic `StructuralDiffInducer`, then move the stream | planned | +| `reward/seed.py:1-236` | MockMed reward seed | Split out the generic bundle helpers (`:238-410`), then move the seed behind the demo flag | planned | +| `record_check_proof.py:60-124` | `MOCKMED` fixture and `if name == "mockmed"` | Fixtures registered by name from the demo app's own module | planned | +| `validation/dense_surface.py` | Study harness rendering a MockMed table | Move to `benchmark/dense_surface` | planned | +| `__main__.py:5246-5275` (`bench`) | Always replays against MockMed | Accept `--url`, or load from a checkout like `benchmark` | planned | +| `execute/registry.py`, `execute/service.py`, `execute/app.py`, `execute/dispatch.py:60-108`, `--seed-mockmed` | Demo seeding inside the Execute service | Generic `seed_admissions`; the CLI passes the demo seed | planned | +| `eligibility/payer_routes.yaml`, `fleet/simulator.py`, `repair/dialog_fixture.py` | Synthetic test data and simulators in the package | Move to the benchmark tree or `tests/fixtures` | planned | + +### Kept in the package on purpose + +These back user-facing commands that must work from `pip install openadapt` +(`openadapt quickstart`, `openadapt-flow tutorial`, `demo`, `demo-record`, +`openadapt-agent serve` with no flags, `openadapt-agent example-app`). They are +OpenAdapt's own synthetic demo applications, not customer applications, and the +engine does not branch on them: + +- `mockmed/` (server, fault server, chart pages, static UI) +- `tutorial.py`, `demo_proof.py`, `demo_driver.record_triage_demo` +- `agent/sandbox.py`, `agent/tutorial.py`, `agent/example_app.py` + +The generality check allows MockMed names only in these modules. + +## (b) data + +- `app_profiles/openemr.yaml`: OpenEMR's bookkeeping writes, loaded by + `app_profiles.load_profile(name)`, which has no per-app code. +- The `clinical-write` policy pack is the default policy in + `execute/models.py:74`, `cli_config.py:164`, `integrations/agents/skills.py:351` + and `agent/teach.py:121`. It is data, but a healthcare default; whether to make + the policy required is a product decision, noted here. + +## (a) rewording done + +- `deployment.py:548`: the example environment variable in a user-facing + error is now `APP_READER_PASSWORD`. +- `authoring.py:807`: the comment on a generic branch no longer names MockMed. + +The remaining (a) rows are examples in comments and docstrings. The code they +describe is generic and configured per contract or profile. Healthcare words +such as MRN, DOB and patient appear in generic heuristics (for example +`risk.py:370`); that is the product's domain, not one application. + +## Full table + +Paths are relative to `packages/flow/openadapt_flow/` as of the audited commit. + +| Location | Bucket | App | Note and action | +| --- | --- | --- | --- | +| `__main__.py:2838-2934` | (d) | MockMed | replay: general code path defaults to serving bundled MockMed when no --url/config URL, and --drift flips use_structural=False (grounding ladder changes on a demo flag). Make --url required for replay (or route no-URL to `tutorial`/`demo`), rename --drift behavior to a generic --force-visual-grounding, keep drift injection in the demo command. | +| `__main__.py:9305-9319` | (d) | MockMed | replay parser: --url default = bundled MockMed; --drift MockMed modes (see 2838-2934). | +| `cli_variants.py:131,173-182` | (d) | MockMed | first-screen probe serves bundled MockMed when no --url (same default as replay). Refuse instead (first_screen_unavailable). | +| `compiler/binding_admission.py:114` | (d) | MockMed | AdmissionFixture.write_path default '/api/encounter' (line 119); compile.py:3379 never overrides it | +| `compiler/binding_admission.py:71` | (d) | MockMed | _BODY_SKIP_FIELDS adds MockMed's 'source' field (line 73) | +| `compiler/compile.py:146` | (d) | MockMed | REGION_STABLE_TOLERANCE=16 (line 151) calibrated on MockMed only; recalibrate on >=2 apps | +| `compiler/effect_mining.py:100` | (d) | MockMed | IDEMPOTENCY_KEY_FIELD='key' (line 102) is MockMed's field; take from contract/profile | +| `deployment.py:387` | (d) | MockMed | records_path default '/api/db' + records_key 'records' (lines 389-390) are MockMed's shape; make required for kind: rest | +| `deployment.py:448` | (d) | OpenEMR | fhir section default resource_type='Observation' (line 449) mirrors OpenEMR note-as-Observation; make required | +| `eligibility/client.py:37-41` | (d) | Stedi | Stedi 270/271 vendor client in engine (hard-coded URL, STEDI_API_KEY env, to_stedi_body); move to integrations/connector | +| `execute/dispatch.py:111-188` | (d) | MockMed | live_replay (the only real Execute runner) always calls tutorial.run_tutorial_workflow: POSTs /api/reset to target_url, appends MockMed ?fault=ok&idempotency=demo#tasks, verifies via /api/db 'records', forces clinical-write policy and 'note' param; with no target_url serves MockMed. Ignores AdmittedBundle.policy. Generalize: replay via the standard replay path with the admission's deployment.yaml (effects/backend/policy); keep MockMed only behind synthetic/demo admissions. | +| `fault_gate.py:722` | (d) | MockMed | rest_memory fault case default records_path '/api/db' (test harness; low) | +| `generate/ensure_contract.py:123` | (d) | Medplum | facts_from_probe translates Medplum probe key names (133-147); make the probe emit generic FACTS | +| `network_capture.py:78` | (d) | OpenEMR | SECRET_NAME regex lists 'clearpass' (OpenEMR login field); redundant with pass\b alternative -> drop | +| `qualification_oracle_gate.py:5,36,70,225,325-455` | (d) | MockMed | evaluate_oracle_gate (called by qualification_proposal.py:221,372 for every `qualify accept`) runs ANY workflow's proposed oracle against MockMed: hard-coded banner 'Encounter saved — ', POST /api/encounter?fault=optimistic with {patient_id, type:'Triage', note}, /api/db verifier. Generalize: a generic lying-store fixture driven by the bundle's own effect binding (write path/body/records path from the effect), or fixtures registered as data. | +| `report_redaction.py:131-133` | (d) | OpenEMR | _SECRET_KEYS lists token_main/csrf_token_form; same fix as DEFAULT_PARAMS | +| `report_redaction.py:68` | (d) | OpenEMR | token_main/csrf_token_form in DEFAULT_PARAMS (with :131-133 _SECRET_KEYS): replace with generic token/csrf name regex (reuse network_capture.TOKEN_NAME) or per-app secret_params in app profile | +| `reward/models.py:226` | (d) | MockMed | RecordReadRecipe.records_path default '/api/db' (MockMed's endpoint) in a general model; make it required for kind=rest. | +| `rollback_state.py:781` | (d) | OpenEMR | same '0' no-record sentinel as frame.py:1937 | +| `runtime/effects/compensation.py:364` | (d) | MockMed | RestCompensator delete_path_template default '/api/encounter/{id}'; make required | +| `runtime/effects/effect.py:900` | (d) | MockMed | Effect.key_field default 'key' (line 901) is MockMed's field | +| `runtime/effects/fhir.py:59` | (d) | OpenEMR | DEFAULT_OBSERVATION_PATHS (61-67) maps note->valueString per OpenEMR; require field_paths from contract | +| `runtime/effects/frame.py:1937` | (d) | OpenEMR | ref in (None,'','0',0): '0' = OpenEMR's 'no chart' (patient_id 0); gated by profile insert_entry_names_no_record; move sentinel into profile | +| `runtime/effects/rest.py:114-115` | (d) | MockMed | RestRecordVerifier defaults records_path='/api/db', records_key='records'; make required | +| `runtime/identity.py:104` | (d) | MockMed | COVERAGE_THRESHOLD=0.8 (line 108) calibrated on MockMed rows only | +| `runtime/identity.py:275-302` | (d) | MockMed/EMR list | _GENERIC_IDENTITY_WORDS hand-tuned EMR list vocabulary (incl. 'demo','chart','review','results'); derive from recording or app profile | +| `runtime/replayer.py:392` | (d) | MockMed | PC_STRUCTURAL_TEMPLATE_THRESHOLD=0.8 (line 394) calibrated on MockMed only | +| `__main__.py:10030-10037` | (c) | MockMed | bench parser. | +| `__main__.py:10166` | (c) | MockMed | verify-scaffold --prove choices ('mockmed','effect-e2e'): fixture names hard-coded in CLI; see record_check_proof.py. | +| `__main__.py:11940-11989,12341-12415` | (c) | MockMed | serve-execute/serve-reward --seed-mockmed flags and banner paths (data_dir/mockmed/records.db). Opt-in demo seeding; keep as demo flag or rename --seed-demo. | +| `__main__.py:1787-1806` | (c) | MockMed | demo-record command: serves MockMed, runs demo_driver.record_triage_demo. User-facing CLI; demo data could be a recorded script in data. | +| `__main__.py:1808-1998` | (c) | MockMed | tutorial command (via tutorial.py). Deliberate product feature; must work from wheel. | +| `__main__.py:1999-2040` | (c) | MockMed | demo command (demo_proof + tutorial). Deliberate product feature; must work from wheel. | +| `__main__.py:326-330` | (c) | MockMed | _with_drift: builds MockMed ?drift= query; only meaningful for the bundled demo. Move with demo helpers. | +| `__main__.py:5246-5275` | (c) | MockMed | bench command: always replays against bundled MockMed. Not generic; move to benchmark/ (source-checkout command, like `benchmark`) or accept --url. | +| `__main__.py:8773,8782` | (c) | MockMed | demo-record parser help/--drift. | +| `agent/example_app.py:1-400` | (c) | (own demo: reports portal) | `openadapt-agent example-app` local sales-order portal (/login clerk/portal-demo, /api/orders). User-facing demo command; importers agent/cli.py:1250 + 4 flow tests. | +| `agent/ledger_demo.py:1-720` | (c) | MockMed | scripted 30-referral worklist simulation (python -m). Not a CLI command; move to benchmark or packages/agent/tests fixtures; importer packages/agent/tests/test_ledger.py. | +| `agent/sandbox.py:1-530` | (c) | MockMed | `openadapt-agent serve` zero-flag sandbox: add_triage_note on MockMed, CASES with MockMed fault/drift queries, clinical-write. User-facing; must ship. Case table could be data (sandbox YAML). | +| `agent/tutorial.py:1-140` | (c) | MockMed | prepare_tutorial_session: hosts MockMed, writes deployment YAML with /api/db, clinical-write. Used by sandbox and agent/cli.py:760; packages/agent/tests/test_tutorial.py. | +| `demo_driver.py:1-96` | (c) | MockMed | record_triage_demo: MockMed selectors and creds (nurse.demo / mockmed-demo-pass). Used by demo-record CLI + benchmark (fault_model, undo_proposal, ui_readback, harness, cosmetic_drift), tests (recorder, e2e), scripts (demo_media, openemr_demo). Split: keep record_triage_demo with mockmed package. | +| `demo_driver.py:97-171` | (c) | MockLoan | record_disbursement_demo (mockloan-demo-pass): no product command uses it. Move to benchmark/lending_fault_model; importers: benchmark/undo_proposal/run.py, benchmark/lending_fault_model/{run,resolution_ladder}.py, tests/e2e/test_lending_fault_model.py. | +| `demo_proof.py:1-60,975-1110` | (c) | MockMed | `demo` page renderer; CROP 640x320 tuned to MockMed layout, 'Encounter saved', /api/db wording. User-facing; must ship. Crop could come from the demo-app spec. | +| `eligibility/payer_routes.yaml:1-` | (c) | Stedi/Cigna | synthetic TEST payer route data shipped in package | +| `execute/app.py:29,39,141,151` | (c) | MockMed | seed_mockmed passthrough; same as service. | +| `execute/dispatch.py:60-108` | (c) | MockMed | default_runner/synthetic_mockmed: synthetic admissions project canned MockMed tutorial outcomes (evidence_tag mockmed-*). Rename to synthetic_runner and move the canned projection to tests/fixtures or keep behind --seed-mockmed only. | +| `execute/registry.py:10-16,81-106` | (c) | MockMed | MOCKMED_* constants + seed_mockmed_admissions in engine registry. Move to execute/demo_seed.py (or tests/fixtures); importers: __main__ serve-execute, tests test_execute_ref/test_execute_post_dispatch_faults/test_jit_draft_execution/test_backend_parity_refusals. | +| `execute/service.py:45,74,84-85` | (c) | MockMed | ExecuteService(seed_mockmed=) flag. Replace with generic seed_admissions: Iterable[AdmittedBundle] and let the CLI pass the demo seed. | +| `fleet/simulator.py:1-` | (c) | synthetic payer portal | in-package simulator fixture; move to benchmark/examples | +| `guard_demo.py:1-296` | (c) | MockMed | `python -m openadapt_flow.guard_demo`: fake agent posts referral to MockMed with chart_pages READER creds; not a CLI subcommand. Move to packages/flow/examples or benchmark; importers: packages/agent/tests (1), packages/flow/tests (6). | +| `instrument/ (__init__.py, competitor_drift.py)` | (c) | MockMed | MockMed competitor-drift harness. Moved to packages/flow/benchmark/harness/instrument. | +| `integrations/harness2/ (all: tasks.py, apps.py, agent.py, adapter.py, traces.py, proposers.py, scorer.py, llm.py, pixels.py, rollout.py, selection.py, __init__.py)` | (c) | MockMed | Whole subpackage is a research harness for Harness2 on MockMed: MOCKMED_PATIENT/PATIENTS (tasks.py:71-148), MockMedTarget (apps.py:50-105, /api/db at :70), regex for MockMed task text and 'triage'/'New Encounter'/'encountersaved' (agent.py:221-324), MockMed banner regex Name — MRN — DOB (traces.py:55-57,128-138), clinical prompts (adapter.py:93-120,429-445; proposers.py:50-63). Move to packages/flow/benchmark/harness2; importers: benchmark/harness2_scorer/run.py, tests/test_harness2_{adapter,pixels,traces}.py, pyproject extra 'harness2', .github/workflows/packages.yml:451. | +| `learning/synth_stream.py:1-245` | (c) | MockMed | synthetic MockMed add-patient-note stream (intents, 'Consent Required', 'Belford, Phil DOB'). Move to benchmark/ or tests/fixtures. | +| `learning/synth_stream.py:246-525` | (c) | (generic) | StructuralDiffInducer is generic but lives in the MockMed harness module; imported by learning/teach.py:96, learning/branch_teach.py:76, benchmark/openemr_e2e/harness.py, 4 tests; pyproject mypy list (line 611). Move to learning/induce.py first, then move the stream out. | +| `mockloan/ (server.py, fault_server.py, static/)` | (c) | MockLoan | Own synthetic lending app shipped in wheel but used by no product command; only demo_driver.record_disbursement_demo, benchmark/lending_fault_model, benchmark/undo_proposal, tests (test_mockloan, test_lending_swer, e2e/test_lending_fault_model), scripts (1). Move to packages/flow/benchmark/apps/mockloan. | +| `mockmed/ (server.py, fault_server.py, chart_pages.py, teach_scope.py, static/)` | (c) | MockMed | Own synthetic demo app shipped in wheel. Required by tutorial/demo/demo-record/replay-default/sandbox and root openadapt/quickstart (engine.py, workers.py) + root tests/test_partner_kit.py. Keep shipping (or split to a demo-apps package). mockmed/teach_scope.py (EncounterWrites) is used only by tests -> move to tests/fixtures. chart_pages used by guard_demo + benchmark (agent_guard, effect_e2e, ui_readback). | +| `record_check_proof.py:1-11,60-124` | (c) | MockMed | --prove fixture registry hard-codes MOCKMED Fixture (judge on patient_id/note, write_path /api/encounter, body type Triage) and `if name == 'mockmed'`. Opt-in proof against our fixture. Move MOCKMED Fixture to mockmed/proof_fixture.py and resolve fixtures by entry point / dotted path. Importers: scaffold_verifier.py:594, benchmark/effect_e2e, tests (2). | +| `repair/dialog_fixture.py:1-` | (c) | synthetic | rendered test app in product package (no real app) | +| `reward/seed.py:1-236` | (c) | MockMed | seed_mockmed, MockMed encounters table/SQL (MOCKMED_TABLE 'encounters', SELECT id, patient_id, type, status), patients, episodes. Move to reward/demo_seed or tests/fixtures; CLI serve-reward --seed-mockmed uses it. | +| `reward/seed.py:238-410` | (c) | (generic) | write_bundle/calibrate_bundle/self_signed_certificate are generic helpers living in the MockMed seed module; split into reward/bundle.py. Importers: integrations/harness2/scorer.py, benchmark/undo_proposal/run.py, tests/test_probe_induction.py, test_undo_proposal.py, test_reward_worker.py, test_reward_trust_boundary.py. | +| `tutorial.py:1-560,630-841` | (c) | MockMed | tutorial engine: MockMed selectors (.open-btn,#new-encounter,#type-triage,#note,#save-encounter,#saved-banner), /api/reset, /api/db, ?fault=ok entry, TUTORIAL_POLICY clinical-write. User-facing (tutorial, demo, quickstart in root openadapt/quickstart/engine.py imports _records, run_tutorial_workflow, record_tutorial; agent/tutorial.py; execute/dispatch.py). Must ship in wheel; could be data-driven (demo-app spec YAML: entry query, reset/records endpoints, recorded click script). | +| `validation/dense_surface.py:418` | (c) | MockMed | validation study renders MockMed-branded table inside product package; move to benchmark/dense_surface | +| `__main__.py:5422,9499,10873` | (b) | (domain) | clinical-write as default/suggested policy name (built-in policy pack, data). | +| `app_profiles/openemr.yaml:1-` | (b) | OpenEMR | per-app bookkeeping profile loaded by load_profile(name); only profile shipped | +| `cli_config.py:164; integrations/agents/skills.py:351; agent/teach.py:121` | (b) | (domain) | default policy 'clinical-write' (shipped policy pack data). | +| `execute/models.py:74` | (b) | (domain) | AdmittedBundle.policy default 'clinical-write' (healthcare policy pack); consider requiring it. | +| `__main__.py:10053` | (a) | MockMed | benchmark help text (benchmark already moved to source-checkout harness). | +| `__main__.py:10536,10577` | (a) | MockMed | --admit-local help mentions MockMed; code path is generic. | +| `__main__.py:1149,1257-1266,1288` | (a) | MockMed | docstrings / refusal text about --drift being MockMed-only. | +| `__main__.py:3819` | (a) | MockMed | docstring: --drift forced off. | +| `__main__.py:7994` | (a) | TextEdit | help example for --backend macos owner app. | +| `__main__.py:9,27,39` | (a) | MockMed | module docstring command list. | +| `agent/cards.py:9-14; agent/skill.py:145-147; agent/bridge.py:132-133,325-326; agent/ledger.py:3,378; agent/delegated.py:11,78; agent/mailbox.py:130; agent/teach.py:281-283` | (a) | (domain) | referral / EMR / MRN examples in docstrings and tool descriptions; fine. | +| `api.py:84` | (a) | MockMed | docstring: replay without url serves MockMed (changes if replay default changes). | +| `app_profiles/__init__.py:315` | (a) | OpenEMR | comment/docstring example | +| `app_profiles/__init__.py:58` | (a) | OpenEMR | comment/docstring example | +| `authoring.py:807` | (a) | MockMed | comment 'Non-secret MockMed note' on a generic branch; reword to 'non-secret field'. | +| `backends/factory.py:117` | (a) | TextEdit | comment/docstring example | +| `backends/parallels_vm.py:403` | (a) | OpenDental | comment/docstring example | +| `backends/playwright_backend.py:890` | (a) | OpenEMR | comment/docstring example | +| `backends/win_agent/server.py:1578` | (a) | Notepad | comment/docstring example | +| `cli_api.py:71` | (a) | OpenEMR | help example 'lname'/'mrn' column names. | +| `cli_undo.py:181` | (a) | MockMed | help says default target is MockMed but _cmd_undo_run refuses that case; reword help. | +| `cli_undo.py:538-541` | (a) | MockMed | comment explaining refusal. | +| `compile_agent/code_action.py:135` | (a) | OpenEMR | comment/docstring example | +| `compiler/binding_admission.py:19` | (a) | MockMed | comment/docstring example | +| `compiler/compile.py:193` | (a) | OpenEMR | comment/docstring example | +| `compiler/compile.py:2184` | (a) | MockMed | comment/docstring example | +| `compiler/compile.py:717` | (a) | MockMed | comment/docstring example | +| `compiler/fixup_passes.py:7` | (a) | OpenEMR | history note; passes are generic | +| `compiler/local_edit.py:321` | (a) | MockMed | comment/docstring example | +| `deployment.py:1035` | (a) | OpenEMR | comment/docstring example | +| `deployment.py:112` | (a) | MockMed | comment/docstring example | +| `deployment.py:144` | (a) | TextEdit | comment/docstring example | +| `deployment.py:548` | (a) | OpenEMR | user-facing error example OPENEMR_READER_PASSWORD; reword to neutral APP_READER_PASSWORD | +| `ensure_record.py:114` | (a) | Medplum | comment/docstring example | +| `ensure_record.py:162` | (a) | Medplum | comment/docstring example | +| `ensure_record.py:97` | (a) | Medplum | comment/docstring example | +| `execute/dispatch.py:1-6` | (a) | MockMed | module docstring; update after generalizing live_replay. | +| `execution_profiles.py:2595` | (a) | MockMed | comment/docstring example | +| `fault_gate.py:822` | (a) | OpenEMR | comment/docstring example | +| `generate/demo_analysis.py:781` | (a) | Redmine | comment/docstring example | +| `generate/ensure_contract.py:4` | (a) | Medplum | comment/docstring example | +| `generate/verify_contract.py:1234` | (a) | Redmine | comment/docstring example | +| `generate/verify_contract.py:1275` | (a) | EspoCRM | comment/docstring example | +| `generate/verify_contract.py:1276` | (a) | Dolibarr | comment/docstring example | +| `generate/verify_contract.py:225` | (a) | Redmine | comment/docstring example | +| `generate/verify_contract.py:566` | (a) | EspoCRM | comment/docstring example | +| `integrations/agents/__init__.py:79` | (a) | (domain) | 'a clinic, a tenant' example. | +| `jit/authority.py:463` | (a) | TextEdit | comment/docstring example | +| `jit/authority.py:623` | (a) | TextEdit | comment/docstring example | +| `jit/compile_agent.py:354` | (a) | TextEdit | comment/docstring example | +| `jit/recorded.py:6` | (a) | MockMed | docstring example task id | +| `jit/signature.py:72` | (a) | MockMed | comment/docstring example | +| `learning/__init__.py:20; learning/halt_loop.py:8; learning/trace.py:60; learning/branch_teach.py:6,108; learning/library.py` | (a) | MockMed | docstring references. | +| `operator_profile.py:79` | (a) | OpenEMR | comment/docstring example | +| `policy_packs.py:41` | (a) | MockMed | user-facing pack summary names MockMed; fine as the demo app | +| `qualification_dev_signer.py:3` | (a) | MockMed | comment/docstring example | +| `qualification_proposal.py:10` | (a) | MockMed | comment/docstring example | +| `qualification_web_session.py:3` | (a) | OpenEMR,openIMIS | comment/docstring example | +| `qualify/config.py:303` | (a) | OpenEMR | comment/docstring example | +| `record_check_binding.py:48` | (a) | openIMIS | comment/docstring example | +| `record_check_identity.py:4` | (a) | OpenEMR | comment/docstring example | +| `record_check_synthesis.py:25` | (a) | MockMed | comment/docstring example | +| `registry.py:3` | (a) | OpenEMR | comment/docstring example | +| `registry.py:813` | (a) | OpenEMR | comment/docstring example | +| `report_redaction.py:5` | (a) | OpenEMR | comment/docstring example | +| `reward/models.py:497; reward/calibration.py:13` | (a) | (domain) | 'triage rows' wording in docstrings. | +| `risk.py:476` | (a) | openIMIS | comment/docstring example | +| `risk.py:565` | (a) | OpenEMR | comment/docstring example | +| `rollback_state.py:77` | (a) | OpenEMR | comment/docstring example | +| `runtime/actuators/web_form.py:214` | (a) | OpenEMR | comment/docstring example | +| `runtime/actuators/web_form.py:324` | (a) | validate.js | var constraints recognizer is a JS library convention (OpenEMR among users), not app-specific | +| `runtime/actuators/web_form.py:331` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/__init__.py:21` | (a) | MockMed | comment/docstring example | +| `runtime/effects/__init__.py:23` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/__init__.py:38` | (a) | Frappe | comment/docstring example | +| `runtime/effects/_common.py:143` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/auth.py:33` | (a) | Frappe | comment/docstring example | +| `runtime/effects/db_snapshot.py:713` | (a) | Dolibarr | comment/docstring example | +| `runtime/effects/effect.py:1536` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/effect.py:1541` | (a) | OpenEMR,MockMed | comment/docstring example | +| `runtime/effects/effect.py:206` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/effect.py:208` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/effect.py:223` | (a) | Redmine | comment/docstring example | +| `runtime/effects/effect.py:903` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:1` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:12` | (a) | MockMed | comment/docstring example | +| `runtime/effects/fhir.py:14` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:191` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:195` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:3` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:38` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:386` | (a) | Epic | comment/docstring example | +| `runtime/effects/fhir.py:39` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:42` | (a) | MockMed | comment/docstring example | +| `runtime/effects/frame.py:135` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:144` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:1499` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:1630` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:1944` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:242` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:46` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fresh_session.py:24` | (a) | MockMed | comment/docstring example | +| `runtime/effects/rest.py:237` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/rest.py:5` | (a) | MockMed | comment/docstring example | +| `runtime/effects/sql.py:12` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:215` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:215` | (a) | Frappe | comment; _TABLE_NAME allows space generically | +| `runtime/effects/sql.py:3` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:365` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/sql.py:4` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:480` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:534` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:535` | (a) | Frappe | comment/docstring example | +| `runtime/grounder.py:136-155` | (a) | healthcare domain | _INTENT_STOPWORDS include patient/mrn; domain wording, intent-only; low | +| `submit_binding.py:8` | (a) | openIMIS | comment/docstring example | +| `validation/dense_surface.py:1271` | (a) | OpenEMR | comment/docstring example | +| `validation/dense_surface.py:5` | (a) | OpenEMR | comment/docstring example | +| `verifier.py:2467` | (a) | OpenEMR | comment/docstring example | +| `vision/hashing.py:10` | (a) | MockMed | docstring for the MockMed-calibrated phash tolerance (see compile.py:151) | +| `volatility.py:100` | (a) | OpenEMR | comment/docstring example | +| `volatility.py:31` | (a) | OpenEMR | comment/docstring example | +| `volatility.py:6` | (a) | OpenEMR | comment/docstring example | +| `benchmark/` (27 files) | (c) | OpenEMR, Frappe, MockMed, Open Dental | Moved to `packages/flow/benchmark/harness/` | diff --git a/packages/flow/docs/design/EFFECT_VERIFIER.md b/packages/flow/docs/design/EFFECT_VERIFIER.md index b7364ebbcf..b54ec7cb59 100644 --- a/packages/flow/docs/design/EFFECT_VERIFIER.md +++ b/packages/flow/docs/design/EFFECT_VERIFIER.md @@ -80,7 +80,7 @@ the note-as-Observation write the fake models cannot be created over FHIR on a stock OpenEMR; the property the live test establishes is the one the fake could not — the verdicts are correct against a **real FHIR server**, not a fake. CI still runs vision-only against the **public demo** -(`openadapt_flow/benchmark/openemr_benchmark.py`, `scripts/openemr_demo.py`); +(`benchmark/harness/openemr_benchmark.py`, `scripts/openemr_demo.py`); the live FHIR test is gated behind `OPENEMR_FHIR_BASE_URL` and skipped otherwise. See `benchmark/openemr_live/README.md` for the one-command bring-up. diff --git a/packages/flow/docs/desktop/PHASE2.md b/packages/flow/docs/desktop/PHASE2.md index 6eb67329e8..16eadb2505 100644 --- a/packages/flow/docs/desktop/PHASE2.md +++ b/packages/flow/docs/desktop/PHASE2.md @@ -58,7 +58,7 @@ so `WindowsBackend` sees exactly what it drives). | `scripts/desktop/patient_notes.ps1` | WinForms target app (list-select → edit note → save); drift knobs via `pn_env.json` | | `scripts/desktop/pn_db.py` | SQLite ground-truth CLI (seed/list/save/get/all) | | `scripts/desktop/uia_arm.py` | UIA-selector arm (pywinauto), identity + positional | -| `openadapt_flow/benchmark/desktop_benchmark.py` | Orchestrator: 3 arms × drift matrix, DB judge, results.json/BENCHMARK.md/chart | +| `benchmark/harness/desktop_benchmark.py` | Orchestrator: 3 arms × drift matrix, DB judge, results.json/BENCHMARK.md/chart | ## Snapshots (reversibility + warm boot) @@ -78,7 +78,7 @@ state without a full VM revert. ```bash # one-off full matrix (6 conditions × 3 arms × n): -python -m openadapt_flow.benchmark.desktop_benchmark --out benchmark/desktop --n 3 +python -m benchmark.harness.desktop_benchmark --out benchmark/desktop --n 3 ``` `DesktopHarness.connect()` starts the VM, launches the session-1 shim, deploys diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index bb52425ba8..9a1f71f1d6 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -5337,8 +5337,31 @@ def _cmd_silent_saves(args: argparse.Namespace) -> int: return int(silent_saves_main(list(args.silent_saves_argv))) +def _checkout_harness_on_path(command: str) -> bool: + """Put the checkout's ``benchmark`` harness on ``sys.path``. + + The harness lives in ``packages/flow/benchmark/harness``, outside the + package, so a wheel install has none and the command says so. + """ + checkout = Path(__file__).resolve().parents[1] + if not (checkout / "benchmark" / "harness" / "__init__.py").is_file(): + print( + f"{command} runs from a source checkout of OpenAdapt " + "(packages/flow/benchmark/harness); the wheel does not ship it.", + file=sys.stderr, + ) + return False + if str(checkout) not in sys.path: + sys.path.insert(0, str(checkout)) + return True + + def _cmd_benchmark(args: argparse.Namespace) -> int: - from openadapt_flow.benchmark.run_benchmark import run_benchmark + if not _checkout_harness_on_path("benchmark"): + return 2 + from benchmark.harness.run_benchmark import ( # type: ignore[import-not-found] + run_benchmark, + ) results = run_benchmark( Path(args.out), diff --git a/packages/flow/openadapt_flow/anthropic_defaults.py b/packages/flow/openadapt_flow/anthropic_defaults.py index 22ec8ce45b..fc4201c29e 100644 --- a/packages/flow/openadapt_flow/anthropic_defaults.py +++ b/packages/flow/openadapt_flow/anthropic_defaults.py @@ -1,6 +1,6 @@ """Anthropic defaults that product code shares with the benchmark harness. -These live here, not in ``openadapt_flow.benchmark``, because the product +These live here, not in ``benchmark.harness`` (the checkout's harness), because the product imports them and a frozen build leaves the benchmark package out (``tests/test_runtime_never_imports_benchmark.py`` enforces the rule). """ diff --git a/packages/flow/openadapt_flow/authoring.py b/packages/flow/openadapt_flow/authoring.py index a1c4e0c2a1..24a3df3402 100644 --- a/packages/flow/openadapt_flow/authoring.py +++ b/packages/flow/openadapt_flow/authoring.py @@ -804,7 +804,7 @@ def continue_input(self, *, operator_confirmed: bool = True) -> dict[str, Any]: "reason": "unconfirmed_masked_secret", } if not pause.secret and (text is None or not text): - # Non-secret MockMed note: do not persist an empty TYPE/param. + # Non-secret field left empty: do not persist an empty TYPE/param. return { "recorded": False, "param": pause.param, diff --git a/packages/flow/openadapt_flow/deployment.py b/packages/flow/openadapt_flow/deployment.py index e8b54023bd..a5f3d698e3 100644 --- a/packages/flow/openadapt_flow/deployment.py +++ b/packages/flow/openadapt_flow/deployment.py @@ -545,7 +545,7 @@ def _validate_secret_env(cls, v: dict[str, str]) -> dict[str, str]: if not isinstance(env_name, str) or not _ENV_VAR_NAME.fullmatch(env_name): raise ValueError( f"effects.secret_env[{slot!r}] must be the NAME of an " - "environment variable (for example OPENEMR_READER_PASSWORD), " + "environment variable (for example APP_READER_PASSWORD), " "never the credential itself" ) return v diff --git a/packages/flow/paper/REVIEW_ADVERSARIAL.md b/packages/flow/paper/REVIEW_ADVERSARIAL.md index 4ebe33f3c6..9e31c59679 100644 --- a/packages/flow/paper/REVIEW_ADVERSARIAL.md +++ b/packages/flow/paper/REVIEW_ADVERSARIAL.md @@ -35,7 +35,7 @@ Every quantitative result is at a sample size where inference is not possible, a This is my single most serious concern, and it is not disclosed in the paper. -- The headline "screen 50/90, effect 0/90" comes from `openadapt_flow/benchmark/silent_wrong_action.py`, **not** from the browser study. That harness does not drive the browser, does not run OCR, and does not invoke the `Replayer`. It issues raw `requests.post` calls that replicate what the app's JavaScript would send, encodes the screen oracle as a hardcoded rule over HTTP status codes, and calls `RestRecordVerifier.verify(...)` directly. +- The headline "screen 50/90, effect 0/90" comes from `benchmark/harness/silent_wrong_action.py`, **not** from the browser study. That harness does not drive the browser, does not run OCR, and does not invoke the `Replayer`. It issues raw `requests.post` calls that replicate what the app's JavaScript would send, encodes the screen oracle as a hardcoded rule over HTTP status codes, and calls `RestRecordVerifier.verify(...)` directly. - In that harness the **effect oracle and the ground-truth judge read the same in-process store**: the effect verifier reads `GET /api/db` and the ground-truth `_business_effect` reads `FaultDB.snapshot()`, which is the same `FaultDB`. Worse, the effect contract (`record_written` exactly once AND `field_equals` on the note) is essentially a restatement of the ground-truth definition of "correct." So "effect silent-wrong = 0 of 90" is guaranteed by construction: the oracle and the ground truth are computed from the same bytes with near-identical logic. The number is a property of the fixture, not an empirical finding. - The genuinely end-to-end harness (`benchmark/fault_model/run.py`, real Playwright + OCR + `Replayer`) measures **only the screen-only baseline**. Its own recommendations section describes effect verification as a *proposed* fix that the reference postcondition system does not yet implement ("The reference postcondition system is vision-only today, so on this corpus it provides no transactional-safety coverage"). The effect-verifier-through-the-replayer path for these fault classes exists only in unit tests (`tests/test_effect_fault_matrix.py`), and those tests also call `verifier.verify(...)` directly rather than driving the `Replayer`. - Consequently the Results sentence "those five classes refuted and halted **through the live replayer**" overstates what was measured. The live replayer never ran the effect verifier on these 90 runs. The suspiciously clean integers (exactly 50, exactly 0, exactly 10) are deterministic-fixture artifacts, not a measured rate. @@ -48,7 +48,7 @@ The seven fault classes are hand-picked transactional hazards (partial save, dup ### 3.4 Baseline fairness: same-backend, but a vanilla single-shot agent -The agent baseline (`openadapt_flow/benchmark/agent_baseline.py`) is not a strawman in the crude sense: it drives the same Playwright backend, uses a current computer-use model and tool, and is handed generous task information (credentials, exact patient, exact note text, a hint that the app is slow). But it is a **single-attempt, un-scaffolded** loop: no system prompt, no planning/reflection/ReAct, no whole-task retry, and hard caps (25 actions, $1.50, last-3-screenshots context). Modern agent results lean heavily on scaffolding and retries; comparing a compiled program against a vanilla single-shot agent inflates the latency and cost deltas. The paper's hedges are appropriate, but the abstract and introduction still lead with the comparison in a way that implies a capability gap the design does not license. At minimum the baseline's single-shot, no-scaffold nature belongs in the main text next to the comparison table, not only implicitly in the code. +The agent baseline (`benchmark/harness/agent_baseline.py`) is not a strawman in the crude sense: it drives the same Playwright backend, uses a current computer-use model and tool, and is handed generous task information (credentials, exact patient, exact note text, a hint that the app is slow). But it is a **single-attempt, un-scaffolded** loop: no system prompt, no planning/reflection/ReAct, no whole-task retry, and hard caps (25 actions, $1.50, last-3-screenshots context). Modern agent results lean heavily on scaffolding and retries; comparing a compiled program against a vanilla single-shot agent inflates the latency and cost deltas. The paper's hedges are appropriate, but the abstract and introduction still lead with the comparison in a way that implies a capability gap the design does not license. At minimum the baseline's single-shot, no-scaffold nature belongs in the main text next to the comparison table, not only implicitly in the code. ### 3.5 The strong-oracle claim is ambiguous exactly where it matters most (pixel-only substrates) diff --git a/packages/flow/pyproject.toml b/packages/flow/pyproject.toml index b63cfa7435..b4bedc3057 100644 --- a/packages/flow/pyproject.toml +++ b/packages/flow/pyproject.toml @@ -605,7 +605,7 @@ module = [ "openadapt_flow.__main__", "openadapt_flow.backends.playwright_backend", "openadapt_flow.backends.rdp_backend", - "openadapt_flow.benchmark.desktop_benchmark", + "benchmark.harness.desktop_benchmark", "openadapt_flow.compiler.disambiguation", "openadapt_flow.interactive_recorder", "openadapt_flow.learning.synth_stream", diff --git a/packages/flow/scripts/desktop/waa_shim.py b/packages/flow/scripts/desktop/waa_shim.py index a15e66b1ae..e0a877e4af 100644 --- a/packages/flow/scripts/desktop/waa_shim.py +++ b/packages/flow/scripts/desktop/waa_shim.py @@ -1,7 +1,7 @@ """In-guest WAA-contract HTTP shim (runs INSIDE the Windows VM). Serves the desktop benchmark harness -(``openadapt_flow.benchmark.desktop_benchmark``): +(``benchmark.harness.desktop_benchmark``): GET /screenshot -> raw PNG bytes of the desktop (Flask send_file) GET /health -> ``{"status": "ok", ...}`` liveness probe diff --git a/packages/flow/scripts/frappe_lending_demo.py b/packages/flow/scripts/frappe_lending_demo.py index 8681b128b4..13877e4925 100644 --- a/packages/flow/scripts/frappe_lending_demo.py +++ b/packages/flow/scripts/frappe_lending_demo.py @@ -42,8 +42,8 @@ from openadapt_flow.backends.playwright_backend import ( # noqa: E402 PlaywrightBackend as _BasePlaywrightBackend, ) -from openadapt_flow.benchmark import agent_baseline # noqa: E402 -from openadapt_flow.benchmark.frappe_lending import ( # noqa: E402 +from benchmark.harness import agent_baseline # noqa: E402 +from benchmark.harness.frappe_lending import ( # noqa: E402 CONDITIONS, INITIAL_TRIALS_PER_CELL, PUBLICATION_TRIALS_PER_CELL, diff --git a/packages/flow/scripts/openemr_demo.py b/packages/flow/scripts/openemr_demo.py index 6a91a699fe..b9a75aee21 100644 --- a/packages/flow/scripts/openemr_demo.py +++ b/packages/flow/scripts/openemr_demo.py @@ -19,7 +19,7 @@ Showcase artifacts land in docs/showcase-openemr/; ``benchmark`` records a fresh demonstration into a temporary directory and writes results to -benchmark/openemr/ (see openadapt_flow.benchmark.openemr_benchmark). +benchmark/openemr/ (see benchmark.harness.openemr_benchmark). """ from __future__ import annotations @@ -189,7 +189,7 @@ def _note_visible(final_png: bytes, note: str) -> bool: Delegates to the benchmark's shared arm-independent check (``verify_note_saved``) so the criterion is implemented exactly once. """ - from openadapt_flow.benchmark.verify import verify_note_saved + from benchmark.harness.verify import verify_note_saved return verify_note_saved(final_png, note).success @@ -280,15 +280,15 @@ def benchmark() -> None: (incremental, per run) plus results.json / BENCHMARK.md / latency_cost.png to benchmark/openemr/. - Hard cost guardrails (see openadapt_flow.benchmark.openemr_benchmark): + Hard cost guardrails (see benchmark.harness.openemr_benchmark): an API preflight before any spend, a $1.50 per-run cap, an $8.00 total agent-arm ceiling, and an abort on consecutive billing errors. All at list price — billed cost under the intro rate is lower. """ import tempfile - from openadapt_flow.benchmark import agent_baseline - from openadapt_flow.benchmark.openemr_benchmark import ( + from benchmark.harness import agent_baseline + from benchmark.harness.openemr_benchmark import ( MAX_TOTAL_COST_USD, run_openemr_benchmark, ) diff --git a/packages/flow/scripts/openemr_local_demo.py b/packages/flow/scripts/openemr_local_demo.py index d9d8438cd1..0a62f7a4d8 100644 --- a/packages/flow/scripts/openemr_local_demo.py +++ b/packages/flow/scripts/openemr_local_demo.py @@ -40,8 +40,8 @@ from openadapt_flow.backends.playwright_backend import ( # noqa: E402 PlaywrightBackend as _BasePlaywrightBackend, ) -from openadapt_flow.benchmark import agent_baseline # noqa: E402 -from openadapt_flow.benchmark.openemr_local import ( # noqa: E402 +from benchmark.harness import agent_baseline # noqa: E402 +from benchmark.harness.openemr_local import ( # noqa: E402 CONDITIONS, INITIAL_TRIALS_PER_CELL, PUBLICATION_TRIALS_PER_CELL, diff --git a/packages/flow/scripts/openemr_param_depth.py b/packages/flow/scripts/openemr_param_depth.py index 6afd55d641..5c77b4c57f 100644 --- a/packages/flow/scripts/openemr_param_depth.py +++ b/packages/flow/scripts/openemr_param_depth.py @@ -148,7 +148,7 @@ def compile_bundle() -> None: def _replay_one(label: str, params: dict[str, str], url: str) -> dict: """One fresh-browser compiled replay; returns a summary row.""" - from openadapt_flow.benchmark.verify import verify_note_saved + from benchmark.harness.verify import verify_note_saved from openadapt_flow.ir import Workflow from openadapt_flow.runtime.replayer import Replayer from openadapt_flow.vision.ocr import normalize_text, ocr diff --git a/packages/flow/tests/e2e/test_region_stable_theme_regression.py b/packages/flow/tests/e2e/test_region_stable_theme_regression.py index 4ce3990130..152450bc7a 100644 --- a/packages/flow/tests/e2e/test_region_stable_theme_regression.py +++ b/packages/flow/tests/e2e/test_region_stable_theme_regression.py @@ -13,7 +13,7 @@ import pytest from openadapt_flow.backends.playwright_backend import PlaywrightBackend -from openadapt_flow.benchmark.dom_arm import verify_final_state +from benchmark.harness.dom_arm import verify_final_state from openadapt_flow.compiler import compile_recording from openadapt_flow.demo_driver import record_triage_demo from openadapt_flow.ir import Workflow diff --git a/packages/flow/tests/test_benchmark.py b/packages/flow/tests/test_benchmark.py index 6f903b79c9..68dbac3e54 100644 --- a/packages/flow/tests/test_benchmark.py +++ b/packages/flow/tests/test_benchmark.py @@ -15,20 +15,20 @@ import numpy as np import pytest -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.agent_baseline import ( +from benchmark.harness import agent_baseline +from benchmark.harness.agent_baseline import ( _truncate_screenshots, compute_cost, load_api_key, run_agent, triage_task_prompt, ) -from openadapt_flow.benchmark.run_benchmark import ( +from benchmark.harness.run_benchmark import ( aggregate_results, render_markdown, write_outputs, ) -from openadapt_flow.benchmark.verify import verify_encounter_saved +from benchmark.harness.verify import verify_encounter_saved NOTE = "Follow-up in 2 weeks; BP recheck." @@ -521,7 +521,7 @@ def test_right_type_passes_at_every_note_length( # line. TestVerify above retains a real-OCR smoke for the benchmark's # actual 32-character note; these four cases pin the verifier logic. monkeypatch.setattr( - "openadapt_flow.benchmark.verify.ocr", + "benchmark.harness.verify.ocr", lambda _screen: [SimpleNamespace(text=f"Triage - {note[:60]}")], ) verdict = verify_encounter_saved(screen, note) @@ -542,7 +542,7 @@ def test_wrong_type_fails_at_every_note_length( f"Consult - {note[:60]}", ) monkeypatch.setattr( - "openadapt_flow.benchmark.verify.ocr", + "benchmark.harness.verify.ocr", lambda _screen: [SimpleNamespace(text=f"Consult - {note[:60]}")], ) verdict = verify_encounter_saved(screen, note) @@ -599,7 +599,7 @@ def test_bare_note_starting_with_known_type_is_not_a_row( # the note itself starts with a known categorical value. screen = self.make_screen(f"Encounter saved - {note[:40]}", note[:60]) monkeypatch.setattr( - "openadapt_flow.benchmark.verify.ocr", + "benchmark.harness.verify.ocr", lambda _screen: [SimpleNamespace(text=note[:60])], ) verdict = verify_encounter_saved(screen, note) diff --git a/packages/flow/tests/test_chart_fonts.py b/packages/flow/tests/test_chart_fonts.py index c54ceb2bc1..d40847a4da 100644 --- a/packages/flow/tests/test_chart_fonts.py +++ b/packages/flow/tests/test_chart_fonts.py @@ -2,7 +2,7 @@ The benchmark chart is cosmetic; a font-lookup failure must never fail the suite. These tests cover the two guarantees in -``openadapt_flow.benchmark.chart_fonts``: the bundled DejaVuSans font is +``benchmark.harness.chart_fonts``: the bundled DejaVuSans font is registered so ``findfont`` cannot miss, and ``safe_render`` swallows any rendering failure. """ @@ -12,7 +12,7 @@ from pathlib import Path from typing import Any -from openadapt_flow.benchmark.chart_fonts import ( +from benchmark.harness.chart_fonts import ( BUNDLED_FONT_NAME, configure_bundled_font, safe_render, diff --git a/packages/flow/tests/test_competitor_drift_instrument.py b/packages/flow/tests/test_competitor_drift_instrument.py index 984b79c792..ec52bd8589 100644 --- a/packages/flow/tests/test_competitor_drift_instrument.py +++ b/packages/flow/tests/test_competitor_drift_instrument.py @@ -22,7 +22,7 @@ import pytest -from openadapt_flow.instrument import ( +from benchmark.harness.instrument import ( AgentRunResult, ArchitectureClassError, CostGuard, @@ -35,7 +35,7 @@ ensure_architecture_class, run_instrument, ) -from openadapt_flow.instrument.competitor_drift import render_markdown, write_outputs +from benchmark.harness.instrument.competitor_drift import render_markdown, write_outputs # Fault classes a screen-only report is BLIND to: a wrong effect lands while # the "saved" banner still paints, so a screen-blind agent claims success. diff --git a/packages/flow/tests/test_desktop_benchmark.py b/packages/flow/tests/test_desktop_benchmark.py index 45d605ed10..f54425cae2 100644 --- a/packages/flow/tests/test_desktop_benchmark.py +++ b/packages/flow/tests/test_desktop_benchmark.py @@ -9,8 +9,8 @@ import json -from openadapt_flow.benchmark import desktop_benchmark as db -from openadapt_flow.benchmark.desktop_benchmark import ( +from benchmark.harness import desktop_benchmark as db +from benchmark.harness.desktop_benchmark import ( _classify, run_desktop_benchmark, ) diff --git a/packages/flow/tests/test_dom_arm.py b/packages/flow/tests/test_dom_arm.py index c19314c790..6dd3a6e4b4 100644 --- a/packages/flow/tests/test_dom_arm.py +++ b/packages/flow/tests/test_dom_arm.py @@ -15,7 +15,7 @@ import numpy as np import pytest -from openadapt_flow.benchmark.dom_arm import ( +from benchmark.harness.dom_arm import ( PERTURBATIONS, SCHEDULE, aggregate_dom_results, @@ -191,7 +191,7 @@ def test_perturbation_menu(self) -> None: ) def test_notes_distinct_per_arm_and_slot(self) -> None: - from openadapt_flow.benchmark.dom_arm import ARMS + from benchmark.harness.dom_arm import ARMS notes = { note_for_slot(arm, slot) @@ -262,7 +262,7 @@ def test_maintenance_is_dom_loud_break_on_drift_only(self) -> None: assert not needs_maintenance(disputed) def test_verification_dispute(self) -> None: - from openadapt_flow.benchmark.dom_arm import verification_dispute + from benchmark.harness.dom_arm import verification_dispute # DOM: every step ran, no error, judge said no -> dispute. assert verification_dispute( @@ -546,7 +546,7 @@ class TestVerifyFinalState: exists to measure.""" def test_dom_check_is_the_hybrid_check(self) -> None: - from openadapt_flow.benchmark.hybrid_benchmark import ( + from benchmark.harness.hybrid_benchmark import ( verify_hybrid_final, ) diff --git a/packages/flow/tests/test_effectbench.py b/packages/flow/tests/test_effectbench.py index 435364cdcd..19419163d9 100644 --- a/packages/flow/tests/test_effectbench.py +++ b/packages/flow/tests/test_effectbench.py @@ -26,7 +26,7 @@ build_reference_episodes, ) from benchmark.fault_model import faults as F -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, DivergenceCategory, Effect, @@ -44,7 +44,7 @@ summarize, wilson_interval, ) -from openadapt_flow.benchmark.effectbench.metrics import bootstrap_ci, rate +from benchmark.harness.effectbench.metrics import bootstrap_ci, rate # --------------------------------------------------------------------------- # 1. Reference re-expression regression (the headline result) @@ -334,7 +334,7 @@ def test_record_and_note_effects_are_reexported() -> None: def test_score_episode_end_to_end() -> None: """The runner orchestrator: capture pre-state, run the arm, verify, label.""" - from openadapt_flow.benchmark.effectbench import AgentReport, score_episode + from benchmark.harness.effectbench import AgentReport, score_episode box = {"acted": False} @@ -367,7 +367,7 @@ def run_action() -> AgentReport: def test_score_episode_raises_on_unreadable_oracle() -> None: - from openadapt_flow.benchmark.effectbench import AgentReport, score_episode + from benchmark.harness.effectbench import AgentReport, score_episode with pytest.raises(ValueError): score_episode( @@ -433,8 +433,8 @@ def test_summary_filters_by_arm(episodes: list[EpisodeRecord]) -> None: def test_episode_record_predicates_match_outcome() -> None: - from openadapt_flow.benchmark.effectbench import Verdict - from openadapt_flow.benchmark.effectbench.schema import OracleVerdict + from benchmark.harness.effectbench import Verdict + from benchmark.harness.effectbench.schema import OracleVerdict ep = EpisodeRecord( episode_id="e1", diff --git a/packages/flow/tests/test_effectbench_runner.py b/packages/flow/tests/test_effectbench_runner.py index 64cd934c61..8abe524670 100644 --- a/packages/flow/tests/test_effectbench_runner.py +++ b/packages/flow/tests/test_effectbench_runner.py @@ -20,7 +20,7 @@ import pytest -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, EffectKind, EpisodeRecord, @@ -30,7 +30,7 @@ SwerVariant, summarize, ) -from openadapt_flow.benchmark.effectbench.runner import ( +from benchmark.harness.effectbench.runner import ( LIVE_ARMS, CompilerArm, CompoundEffectVerifier, @@ -43,19 +43,19 @@ run_episode, run_matrix, ) -from openadapt_flow.benchmark.effectbench.runner.arms import AgentArm, ArmResult -from openadapt_flow.benchmark.effectbench.runner.baselines import ( +from benchmark.harness.effectbench.runner.arms import AgentArm, ArmResult +from benchmark.harness.effectbench.runner.baselines import ( SCAFFOLDED_ARMS, ClaudeComputerUseArm, ScaffoldNotWired, ) -from openadapt_flow.benchmark.effectbench.runner.harness import EpisodeEnv -from openadapt_flow.benchmark.effectbench.runner.reference_tasks import ( +from benchmark.harness.effectbench.runner.harness import EpisodeEnv +from benchmark.harness.effectbench.runner.reference_tasks import ( GOAL, TARGET, make_params, ) -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( DivergenceCategory, OracleChannel, OracleSpec, @@ -343,7 +343,7 @@ def test_run_matrix_skips_scaffolded_arms_by_default() -> None: def test_cli_dry_run_runs_green() -> None: - from openadapt_flow.benchmark.effectbench.runner.__main__ import run_dry_run + from benchmark.harness.effectbench.runner.__main__ import run_dry_run episodes, summaries = run_dry_run( trials=1, arm_names="compiler,screen_only", include_scaffolded=False diff --git a/packages/flow/tests/test_effectbench_standalone.py b/packages/flow/tests/test_effectbench_standalone.py index dc0fb92526..56ac997158 100644 --- a/packages/flow/tests/test_effectbench_standalone.py +++ b/packages/flow/tests/test_effectbench_standalone.py @@ -6,7 +6,7 @@ dependency). Because the standalone package vendors the effect MECHANISM (the typed ``Effect`` contract, the substrate-independent judge, the outcome classifier) that is otherwise defined in ``openadapt_flow.runtime.effects`` and -``openadapt_flow.benchmark.effectbench``, this test guards against the two copies +``benchmark.harness.effectbench``, this test guards against the two copies drifting: it asserts the standalone classifier and judge agree BYTE-FOR-BYTE with the engine on the reference inputs, and that the standalone reference reproduces the published headline. It also re-runs the standalone source-availability @@ -62,7 +62,7 @@ def test_classifier_parity_with_engine_over_the_full_truth_table(): """The standalone classifier equals the engine classifier on every input.""" import effectbench.oracle as sa - from openadapt_flow.benchmark.effectbench import oracle as eng + from benchmark.harness.effectbench import oracle as eng scoreable = ["CORRECT", "WRONG_PERSISTED", "ABSENT"] for reported, state_name, avail in itertools.product( @@ -86,7 +86,7 @@ def test_divergence_category_parity_with_engine(): """The public standalone taxonomy must not lag the engine copy.""" from effectbench.schema import DivergenceCategory as StandaloneCategory - from openadapt_flow.benchmark.effectbench.schema import ( + from benchmark.harness.effectbench.schema import ( DivergenceCategory as EngineCategory, ) diff --git a/packages/flow/tests/test_effectbench_task_pack.py b/packages/flow/tests/test_effectbench_task_pack.py index bed77414ca..ddb2ea85c0 100644 --- a/packages/flow/tests/test_effectbench_task_pack.py +++ b/packages/flow/tests/test_effectbench_task_pack.py @@ -36,8 +36,8 @@ from benchmark.effectbench.task_pack.audit import audit_mockmed_oracle from benchmark.effectbench.task_pack.driver import run_mockmed_pack from benchmark.effectbench.task_pack.mockmed_tasks import MOCKMED_TASKS -from openadapt_flow.benchmark.effectbench import OutcomeLabel, Substrate, summarize -from openadapt_flow.benchmark.effectbench.schema import DivergenceCategory +from benchmark.harness.effectbench import OutcomeLabel, Substrate, summarize +from benchmark.harness.effectbench.schema import DivergenceCategory from openadapt_flow.runtime.effects.sql import assert_read_only_sql MANIFEST_PATH = Path(__file__).resolve().parents[1] / ( diff --git a/packages/flow/tests/test_frappe_lending_benchmark.py b/packages/flow/tests/test_frappe_lending_benchmark.py index ee3f50c080..1737d2a80b 100644 --- a/packages/flow/tests/test_frappe_lending_benchmark.py +++ b/packages/flow/tests/test_frappe_lending_benchmark.py @@ -19,7 +19,7 @@ FrappeFixture, audit_table_deltas, ) -from openadapt_flow.benchmark.frappe_lending import ( +from benchmark.harness.frappe_lending import ( ARMS, CONDITIONS, FrappeLoanApplicationOracle, diff --git a/packages/flow/tests/test_hybrid_benchmark.py b/packages/flow/tests/test_hybrid_benchmark.py index 9779311f40..11ca0244ba 100644 --- a/packages/flow/tests/test_hybrid_benchmark.py +++ b/packages/flow/tests/test_hybrid_benchmark.py @@ -15,8 +15,8 @@ import numpy as np import pytest -from openadapt_flow.benchmark import agent_baseline, hybrid_benchmark -from openadapt_flow.benchmark.hybrid_benchmark import ( +from benchmark.harness import agent_baseline, hybrid_benchmark +from benchmark.harness.hybrid_benchmark import ( AGENT_SLOTS, DRIFT_TYPES, FALLBACK_MAX_ACTIONS, @@ -610,7 +610,7 @@ def fake_run_agent(backend, task, **kwargs): assert "no failing step recorded" in seen["task"] def test_failed_result_index_edge_cases(self) -> None: - from openadapt_flow.benchmark.hybrid_benchmark import ( + from benchmark.harness.hybrid_benchmark import ( _failed_result_index, ) @@ -978,7 +978,7 @@ def test_arms_see_identical_conditions_per_slot( the run helpers), not on the orchestrator-stamped condition labels — the labels come from the same schedule lookup the orchestrator uses to build the URLs, so asserting on them alone is circular.""" - from openadapt_flow.benchmark.hybrid_benchmark import ( + from benchmark.harness.hybrid_benchmark import ( AGENT_SLOTS, condition_url, ) diff --git a/packages/flow/tests/test_lending_swer.py b/packages/flow/tests/test_lending_swer.py index b09dabeeed..b787960357 100644 --- a/packages/flow/tests/test_lending_swer.py +++ b/packages/flow/tests/test_lending_swer.py @@ -22,7 +22,7 @@ from benchmark.lending_fault_model import faults as F from benchmark.lending_fault_model import ground_truth as G from benchmark.lending_fault_model import swer as S -from openadapt_flow.benchmark.effectbench import summarize +from benchmark.harness.effectbench import summarize from openadapt_flow.mockloan.fault_server import LedgerDB diff --git a/packages/flow/tests/test_openemr_benchmark.py b/packages/flow/tests/test_openemr_benchmark.py index b2f10ba067..f32a93db12 100644 --- a/packages/flow/tests/test_openemr_benchmark.py +++ b/packages/flow/tests/test_openemr_benchmark.py @@ -16,20 +16,20 @@ import cv2 import numpy as np -import openadapt_flow.benchmark.verify as benchmark_verify -from openadapt_flow.benchmark import agent_baseline, openemr_benchmark -from openadapt_flow.benchmark.agent_baseline import ( +import benchmark.harness.verify as benchmark_verify +from benchmark.harness import agent_baseline, openemr_benchmark +from benchmark.harness.agent_baseline import ( openemr_task_prompt, run_agent, ) -from openadapt_flow.benchmark.openemr_benchmark import ( +from benchmark.harness.openemr_benchmark import ( aggregate_openemr_results, note_for, render_openemr_markdown, run_openemr_benchmark, write_openemr_outputs, ) -from openadapt_flow.benchmark.verify import verify_note_saved +from benchmark.harness.verify import verify_note_saved from openadapt_flow.vision import OcrLine NOTE = "Insurance card copied and coverage verified by phone." @@ -848,7 +848,7 @@ def crashing_client(self) -> CrashingClient: ) def test_mid_run_crash_row_carries_partial_cost(self, monkeypatch: Any) -> None: - from openadapt_flow.benchmark.run_benchmark import _agent_run + from benchmark.harness.run_benchmark import _agent_run _fake_launch(monkeypatch, FakeBackend()) row = _agent_run("http://x", NOTE, client=self.crashing_client(), task="task") diff --git a/packages/flow/tests/test_openemr_local_benchmark.py b/packages/flow/tests/test_openemr_local_benchmark.py index e637aa076b..c67f0b6c79 100644 --- a/packages/flow/tests/test_openemr_local_benchmark.py +++ b/packages/flow/tests/test_openemr_local_benchmark.py @@ -22,7 +22,7 @@ audit_table_deltas, unexpected_table_deltas, ) -from openadapt_flow.benchmark.openemr_local import ( +from benchmark.harness.openemr_local import ( ARMS, CONDITIONS, OpenEMRPatientOracle, @@ -1011,7 +1011,7 @@ def __init__(self, messages: Messages) -> None: "max_tokens": 4096, } ledger = __import__( - "openadapt_flow.benchmark.agent_baseline", fromlist=["UsageLedger"] + "benchmark.harness.agent_baseline", fromlist=["UsageLedger"] ).UsageLedger() allowed_messages = Messages(10_000) diff --git a/packages/flow/tests/test_openemr_retained_finals.py b/packages/flow/tests/test_openemr_retained_finals.py index 8fbdc22904..c1b6709429 100644 --- a/packages/flow/tests/test_openemr_retained_finals.py +++ b/packages/flow/tests/test_openemr_retained_finals.py @@ -16,7 +16,7 @@ import pytest -from openadapt_flow.benchmark.verify import verify_note_saved +from benchmark.harness.verify import verify_note_saved REPO_ROOT = Path(__file__).resolve().parents[1] RUN_RETAINED_FINALS = os.environ.get("OPENADAPT_OPENEMR_FINALS_DIR") diff --git a/packages/flow/tests/test_reliability_harness.py b/packages/flow/tests/test_reliability_harness.py index 06be924f83..e3dade0d41 100644 --- a/packages/flow/tests/test_reliability_harness.py +++ b/packages/flow/tests/test_reliability_harness.py @@ -1,6 +1,6 @@ """Small synthetic checks for the public reliability aggregation mechanism.""" -from openadapt_flow.benchmark.reliability import _outcome, aggregate +from benchmark.harness.reliability import _outcome, aggregate def _result(outcome: str, *, category: str | None = None) -> dict: diff --git a/packages/flow/tests/test_runtime_never_imports_benchmark.py b/packages/flow/tests/test_runtime_never_imports_benchmark.py index dc4fa57ffb..d59f3083d7 100644 --- a/packages/flow/tests/test_runtime_never_imports_benchmark.py +++ b/packages/flow/tests/test_runtime_never_imports_benchmark.py @@ -1,9 +1,10 @@ -"""Product code never imports ``openadapt_flow.benchmark``. +"""Product code never imports the benchmark harness. -A frozen build (Desktop's engine) leaves the benchmark package out, so a -product module that imports it, even lazily, fails there. Shared names -(``MODEL``, ``load_api_key``, ``SCROLL_PX_PER_UNIT``) live in -``openadapt_flow.anthropic_defaults``. +The harness lives in the source checkout (``packages/flow/benchmark/harness``), +outside the ``openadapt_flow`` package, so neither the wheel nor a frozen build +(Desktop's engine) carries it. A product module that imports it, even lazily, +fails there. Shared names (``MODEL``, ``load_api_key``, ``SCROLL_PX_PER_UNIT``) +live in ``openadapt_flow.anthropic_defaults``. """ from __future__ import annotations @@ -17,57 +18,64 @@ #: Modules whose job is the benchmark itself. Nothing else may import it. ALLOWED = { - # The benchmark CLI command loads the harness on demand. + # The benchmark CLI command loads the harness on demand from a checkout. "__main__.py", - # The competitor-drift harness reuses the benchmark's fault scenarios. - "instrument/competitor_drift.py", + # `record-check prove` loads its fixture from a checkout on demand. + "record_check_proof.py", } +def _is_benchmark(module: str) -> bool: + return any( + module == root or module.startswith(root + ".") + for root in ("benchmark", "openadapt_flow.benchmark") + ) + + def _imports_benchmark(tree: ast.AST) -> list[int]: lines: list[int] = [] for node in ast.walk(tree): if isinstance(node, ast.ImportFrom): module = node.module or "" - if node.level == 0 and ( - module == "openadapt_flow.benchmark" - or module.startswith("openadapt_flow.benchmark.") - ): + if node.level == 0 and _is_benchmark(module): lines.append(node.lineno) elif node.level == 0 and module == "openadapt_flow": if any(a.name == "benchmark" for a in node.names): lines.append(node.lineno) - elif node.level > 0 and ( - module == "benchmark" or module.startswith("benchmark.") - ): + elif node.level > 0 and _is_benchmark(module): lines.append(node.lineno) elif isinstance(node, ast.Import): - for alias in node.names: - if alias.name == "openadapt_flow.benchmark" or alias.name.startswith( - "openadapt_flow.benchmark." - ): - lines.append(node.lineno) + if any(_is_benchmark(alias.name) for alias in node.names): + lines.append(node.lineno) return lines +def test_the_harness_is_not_inside_the_package() -> None: + assert not (ROOT / "benchmark").exists(), ( + "benchmark harnesses belong in packages/flow/benchmark/, outside the " + "shipped openadapt_flow package" + ) + + def test_product_modules_never_import_the_benchmark_package() -> None: offenders: list[str] = [] for path in sorted(ROOT.rglob("*.py")): rel = path.relative_to(ROOT).as_posix() - if rel.startswith("benchmark/") or rel in ALLOWED: + if rel in ALLOWED: continue tree = ast.parse(path.read_text(encoding="utf-8")) offenders += [f"{rel}:{n}" for n in _imports_benchmark(tree)] assert not offenders, ( - "product code imports openadapt_flow.benchmark (a frozen build leaves " - "it out); move the shared name to openadapt_flow.anthropic_defaults " - f"or a runtime module: {offenders}" + "product code imports the benchmark harness (the wheel and a frozen " + "build leave it out); move the shared name to " + "openadapt_flow.anthropic_defaults or a runtime module: " + f"{offenders}" ) def test_shared_anthropic_names_have_one_home() -> None: + from benchmark.harness import agent_baseline from openadapt_flow import anthropic_defaults - from openadapt_flow.benchmark import agent_baseline assert agent_baseline.MODEL is anthropic_defaults.MODEL assert agent_baseline.SCROLL_PX_PER_UNIT == anthropic_defaults.SCROLL_PX_PER_UNIT diff --git a/packages/flow/tests/test_silent_wrong_action_benchmark.py b/packages/flow/tests/test_silent_wrong_action_benchmark.py index a340eea1ae..cadfdbcacc 100644 --- a/packages/flow/tests/test_silent_wrong_action_benchmark.py +++ b/packages/flow/tests/test_silent_wrong_action_benchmark.py @@ -16,7 +16,7 @@ from __future__ import annotations -from openadapt_flow.benchmark.silent_wrong_action import ( +from benchmark.harness.silent_wrong_action import ( SCENARIOS, aggregate, render_markdown, diff --git a/packages/flow/tests/test_ui_dismiss_live_openemr.py b/packages/flow/tests/test_ui_dismiss_live_openemr.py index 1468c56f6d..3caceffee1 100644 --- a/packages/flow/tests/test_ui_dismiss_live_openemr.py +++ b/packages/flow/tests/test_ui_dismiss_live_openemr.py @@ -47,7 +47,7 @@ def _record(tmp_path: Path) -> tuple[Path, Any]: from benchmark.openemr_local.qualify_hooks import fixture - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec from openadapt_flow.recorder import Recorder from scripts.openemr_local_demo import ( FORM_FRAME_NAME, From 22de5f6cc62f22b1de455563614c6c9ed16b2120 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 13:04:32 -0500 Subject: [PATCH 02/16] Generality audit: link CONTRIBUTING without the guard PR's anchor --- packages/flow/docs/GENERALITY_AUDIT.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md index 7fb257ecdb..f8f4b51798 100644 --- a/packages/flow/docs/GENERALITY_AUDIT.md +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -6,8 +6,8 @@ Audited 2026-11-03 on `main` at `213f3d9fe`. Scope: everything under The rule: the engine has no code written for one application. What one application needs lives in data (`app_profiles/*.yaml`, generated contracts, deployment configs) or in test fixtures and benchmark harnesses outside the -shipped package. `scripts/check_engine_generality.py` enforces it in CI; see -[CONTRIBUTING.md](../CONTRIBUTING.md#app-knowledge-lives-in-data). +shipped package. A CI check (`scripts/check_engine_generality.py`) keeps it +that way; [CONTRIBUTING.md](../CONTRIBUTING.md) states the rule. ## Method From d10bcc30206f71db4ad548a910c1db5024dd1cb6 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 12:41:22 -0500 Subject: [PATCH 03/16] CI: fail when the engine names a specific application scripts/check_engine_generality.py scans openadapt_flow/ for the application names and app-specific identifiers in scripts/generality.toml. app_profiles/ is data and passes; OpenAdapt's demo modules may name the demo app; a comment line may give an example under '# example:'. Hits that predate the check are counted in generality_baseline.json, which only shrinks: a rise fails, and so does a fall the baseline doesn't record. Runs in the flow lint job. CONTRIBUTING documents the rule: app knowledge lives in profiles, contracts and fixtures. --- .github/workflows/packages.yml | 4 + packages/flow/CONTRIBUTING.md | 25 ++ .../flow/scripts/check_engine_generality.py | 178 +++++++++++ packages/flow/scripts/generality.toml | 65 ++++ .../flow/scripts/generality_baseline.json | 296 ++++++++++++++++++ .../tests/test_engine_generality_check.py | 97 ++++++ 6 files changed, 665 insertions(+) create mode 100644 packages/flow/scripts/check_engine_generality.py create mode 100644 packages/flow/scripts/generality.toml create mode 100644 packages/flow/scripts/generality_baseline.json create mode 100644 packages/flow/tests/test_engine_generality_check.py diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 9daecc8dc2..0c6328a009 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -348,6 +348,10 @@ jobs: run: python scripts/check_release_consistency.py - name: Ruff lint run: ruff check openadapt_flow + - name: No app-specific code in the engine + # App knowledge lives in profiles, contracts and fixtures (CONTRIBUTING: + # "App knowledge lives in data"); the denylist is scripts/generality.toml. + run: python scripts/check_engine_generality.py - name: Install uv (for the oldest-Python syntax check) uses: astral-sh/setup-uv@18d369b23da25a81e9458561248e1aafff6a70fa # v10.3.1 - name: Syntax at the oldest supported Python (3.10) diff --git a/packages/flow/CONTRIBUTING.md b/packages/flow/CONTRIBUTING.md index da76cb2625..8641a68f76 100644 --- a/packages/flow/CONTRIBUTING.md +++ b/packages/flow/CONTRIBUTING.md @@ -23,6 +23,7 @@ Python 3.10–3.12 are supported and exercised in CI. ruff check openadapt_flow # lint ruff format --check openadapt_flow # format (drop --check to auto-apply) mypy # type-check (config in pyproject.toml) +python scripts/check_engine_generality.py # no app names in the engine pytest -q # tests ``` @@ -174,6 +175,30 @@ surface from permissively licensed package artifacts. The release-consistency gate inspects the actual wheel and sdist. A source-tree notice alone is not sufficient. +## App knowledge lives in data + +The engine (`openadapt_flow/`) works the same way for every application. Never +write code, a default, a constant or a heuristic for one application. What one +application needs goes in data: + +- its bookkeeping writes: an app profile in `openadapt_flow/app_profiles/`, + loaded by name; +- its endpoints, tables, keys and field paths: the generated contract or the + deployment config (`deploy.yaml`), which the engine requires rather than + defaulting; +- tests and benchmarks for it: fixtures under `tests/` and harnesses under + `benchmark/`, outside the shipped package. + +`scripts/check_engine_generality.py` runs in CI and fails when engine source +names an application or one of its identifiers from the denylist in +`scripts/generality.toml`. Add new applications and identifiers to that list +when you meet them. A comment that only gives an example may name an +application on a line marked `# example:`. Hits that predate the check are +counted in `scripts/generality_baseline.json`, which only shrinks: when you +remove one, lower it with `--update-baseline`. OpenAdapt's own demo apps +(MockMed) may be named only in the demo modules listed in the denylist. The +audit behind the rule is `docs/GENERALITY_AUDIT.md`. + ## Safety-sensitive areas The identity gate, the resolution ladder, and the postcondition/halt logic are diff --git a/packages/flow/scripts/check_engine_generality.py b/packages/flow/scripts/check_engine_generality.py new file mode 100644 index 0000000000..1200d93154 --- /dev/null +++ b/packages/flow/scripts/check_engine_generality.py @@ -0,0 +1,178 @@ +"""Fail when the engine names a specific application. + +The engine (``openadapt_flow/``) has no code written for one application. What +one application needs lives in data: ``app_profiles/*.yaml``, generated +contracts, deployment configs, or test fixtures and benchmark harnesses outside +the package. ``generality.toml`` beside this script holds the denylist of +application names and app-specific identifiers, and the few modules allowed to +name OpenAdapt's own demo apps. + +A comment line may name an application as an example when it carries an +explicit ``# example:`` marker. Every other hit fails, unless it is in the +baseline: the hits that were already there when the check went live +(``docs/GENERALITY_AUDIT.md``). The baseline only shrinks. A file whose count +for a term rises fails; a file whose count falls must have its baseline entry +lowered in the same change (``--update-baseline`` writes it). + +Usage:: + + python scripts/check_engine_generality.py + python scripts/check_engine_generality.py --update-baseline +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from collections import Counter +from pathlib import Path + +import tomllib + +HERE = Path(__file__).resolve().parent +FLOW = HERE.parent +ENGINE = FLOW / "openadapt_flow" +CONFIG = HERE / "generality.toml" +BASELINE = HERE / "generality_baseline.json" + +#: Engine files that hold text: code and the data shipped with it. +SUFFIXES = {".py", ".js", ".html", ".css", ".json", ".yaml", ".yml", ".toml", ".txt"} + +EXAMPLE_MARKER = re.compile(r"#\s*example:", re.IGNORECASE) + + +def load_config(path: Path = CONFIG) -> dict: + return tomllib.loads(path.read_text(encoding="utf-8")) + + +def _term_patterns(config: dict) -> dict[str, re.Pattern[str]]: + terms: dict[str, re.Pattern[str]] = {} + for section in ("names", "identifiers"): + for term, pattern in config[section].items(): + terms[term] = re.compile(pattern, re.IGNORECASE) + return terms + + +def _allowed(rel: str, term: str, config: dict) -> bool: + for prefix in config["allow"]["paths"]: + if rel == prefix or rel.startswith(prefix.rstrip("/") + "/"): + return True + for prefix, terms in config["allow"].get("terms_in", {}).items(): + if ( + rel == prefix or rel.startswith(prefix.rstrip("/") + "/") + ) and term in terms: + return True + return False + + +def scan( + engine: Path = ENGINE, config: dict | None = None +) -> dict[str, dict[str, list[int]]]: + """Return ``{relative path: {term: [line numbers]}}`` for every hit.""" + config = config if config is not None else load_config() + terms = _term_patterns(config) + hits: dict[str, dict[str, list[int]]] = {} + for path in sorted(engine.rglob("*")): + if not path.is_file() or path.suffix not in SUFFIXES: + continue + if "__pycache__" in path.parts: + continue + rel = path.relative_to(engine).as_posix() + try: + text = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + for number, line in enumerate(text.splitlines(), start=1): + stripped = line.lstrip() + if stripped.startswith("#") and EXAMPLE_MARKER.search(stripped): + continue + for term, pattern in terms.items(): + if pattern.search(line) and not _allowed(rel, term, config): + hits.setdefault(rel, {}).setdefault(term, []).append(number) + return hits + + +def counts(hits: dict[str, dict[str, list[int]]]) -> dict[str, dict[str, int]]: + return { + rel: {term: len(lines) for term, lines in sorted(by_term.items())} + for rel, by_term in sorted(hits.items()) + } + + +def compare( + hits: dict[str, dict[str, list[int]]], baseline: dict[str, dict[str, int]] +) -> tuple[list[str], list[str]]: + """Return (new hits, stale baseline entries).""" + new: list[str] = [] + stale: list[str] = [] + seen = Counter() + for rel, by_term in hits.items(): + for term, lines in by_term.items(): + allowed = baseline.get(rel, {}).get(term, 0) + seen[(rel, term)] = len(lines) + if len(lines) > allowed: + where = ", ".join(str(n) for n in lines) + new.append( + f"openadapt_flow/{rel}: {term!r} on line(s) {where} " + f"({len(lines)} found, {allowed} in the baseline)" + ) + for rel, by_term in baseline.items(): + for term, allowed in by_term.items(): + if seen[(rel, term)] < allowed: + stale.append( + f"openadapt_flow/{rel}: {term!r} baseline {allowed}, " + f"now {seen[(rel, term)]}" + ) + return new, stale + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + parser.add_argument( + "--update-baseline", + action="store_true", + help="Lower the baseline to the current hits (it never grows)", + ) + args = parser.parse_args(argv) + + hits = scan() + baseline = json.loads(BASELINE.read_text(encoding="utf-8")) + new, stale = compare(hits, baseline) + + if args.update_baseline: + if new: + print("New hits cannot enter the baseline:", file=sys.stderr) + print("\n".join(f" {line}" for line in new), file=sys.stderr) + return 1 + BASELINE.write_text(json.dumps(counts(hits), indent=2) + "\n", encoding="utf-8") + print(f"Baseline lowered: {BASELINE.relative_to(FLOW)}") + return 0 + + if new: + print( + "The engine names a specific application. Put what one app needs " + "in data (app_profiles/*.yaml, a generated contract, a deployment " + "config) or in a fixture outside openadapt_flow/; see " + "CONTRIBUTING.md, 'App knowledge lives in data'. A comment that " + "only gives an example may carry '# example:'.", + file=sys.stderr, + ) + print("\n".join(f" {line}" for line in new), file=sys.stderr) + if stale: + print( + "The baseline is higher than the engine now; lower it with " + "`python scripts/check_engine_generality.py --update-baseline`:", + file=sys.stderr, + ) + print("\n".join(f" {line}" for line in stale), file=sys.stderr) + if new or stale: + return 1 + total = sum(n for by_term in baseline.values() for n in by_term.values()) + print(f"Engine generality OK ({total} baseline hits left)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/scripts/generality.toml b/packages/flow/scripts/generality.toml new file mode 100644 index 0000000000..7b5b261273 --- /dev/null +++ b/packages/flow/scripts/generality.toml @@ -0,0 +1,65 @@ +# Denylist for scripts/check_engine_generality.py. +# +# The engine (openadapt_flow/) must not name a specific application. Add a term +# here when a new application is qualified or a new app-specific identifier is +# found. Patterns are Python regular expressions, matched case-insensitively +# unless they turn that off with (?-i:...). + +[names] +openemr = '\bopen[ _-]?emr' +openimis = '\bopen[ _-]?imis' +medplum = '\bmedplum' +dolibarr = '\bdolibarr' +espocrm = '\bespo[ _-]?crm' +redmine = '\bredmine' +snipe-it = '\bsnipe[ _-]?it\b' +inventree = '\binventree' +kimai = '\bkimai' +openmrs = '\bopen[ _-]?mrs' +gitea = '\bgitea' +firefly-iii = '\bfirefly[ _-]?(iii|3)\b' +vikunja = '\bvikunja' +zammad = '\bzammad' +open-dental = '\bopen[ _-]?dental' +odoo = '\bodoo' +frappe = '\b(frappe|erpnext)' +patient-notes = '\bpatient[ _-]notes\b' +epic = '(?-i:\bEpic\b)' +cerner = '\bcerner' +stedi = '\bstedi\b' +textedit = '\btextedit' +notepad = '\bnotepad\b' +mockmed = '\bmock[ _-]?med' +mockloan = '\bmock[ _-]?loan' + +[identifiers] +# OpenEMR +token_main = '\btoken_main\b' +csrf_token_form = '\bcsrf_token_form\b' +clearpass = '\bclearpass\b' +pnotes = '\bpnotes\b' +patient_data = '\bpatient_data\b' +uuid_registry = '\buuid_registry\b' +openemr-fhir-path = '/apis/default/fhir' +# Frappe and Dolibarr table prefixes +frappe-table = '\btabLoan\b' +dolibarr-table = '\bllx_[a-z]' +# MockMed's REST shape +mockmed-records = '/api/db\b' +mockmed-write = '/api/encounter\b' + +[allow] +# Data, loaded generically: an app profile is where one app's bookkeeping lives. +paths = ["app_profiles/"] + +# OpenAdapt's own demo apps and the user-facing commands that serve them +# (tutorial, demo, demo-record, quickstart, `openadapt-agent serve` with no +# flags). They may name the demo app and its REST shape; nothing else may. +[allow.terms_in] +"mockmed/" = ["mockmed", "mockmed-records", "mockmed-write"] +"tutorial.py" = ["mockmed", "mockmed-records", "mockmed-write"] +"demo_proof.py" = ["mockmed", "mockmed-records", "mockmed-write"] +"demo_driver.py" = ["mockmed", "mockmed-records", "mockmed-write"] +"agent/sandbox.py" = ["mockmed", "mockmed-records", "mockmed-write"] +"agent/tutorial.py" = ["mockmed", "mockmed-records", "mockmed-write"] +"agent/example_app.py" = ["mockmed", "mockmed-records", "mockmed-write"] diff --git a/packages/flow/scripts/generality_baseline.json b/packages/flow/scripts/generality_baseline.json new file mode 100644 index 0000000000..63467d2bf5 --- /dev/null +++ b/packages/flow/scripts/generality_baseline.json @@ -0,0 +1,296 @@ +{ + "__main__.py": { + "mockmed": 38, + "textedit": 1 + }, + "agent/ledger_demo.py": { + "mockmed": 4 + }, + "api.py": { + "mockmed": 1 + }, + "backend.py": { + "mockmed-records": 1 + }, + "backends/factory.py": { + "textedit": 1 + }, + "backends/parallels_vm.py": { + "open-dental": 1 + }, + "backends/playwright_backend.py": { + "openemr": 1 + }, + "backends/win_agent/server.py": { + "notepad": 1 + }, + "cli_undo.py": { + "mockmed": 2 + }, + "cli_variants.py": { + "mockmed": 2 + }, + "compile_agent/code_action.py": { + "openemr": 1, + "token_main": 1 + }, + "compiler/binding_admission.py": { + "mockmed": 3, + "mockmed-records": 1, + "mockmed-write": 2 + }, + "compiler/compile.py": { + "mockmed": 3, + "mockmed-records": 1, + "openemr": 1 + }, + "compiler/effect_mining.py": { + "mockmed": 1, + "mockmed-records": 1 + }, + "compiler/fixup_passes.py": { + "openemr": 1 + }, + "compiler/local_edit.py": { + "mockmed": 1 + }, + "demo_driver.py": { + "mockloan": 5 + }, + "deployment.py": { + "mockmed": 2, + "mockmed-records": 2, + "openemr": 2, + "textedit": 1 + }, + "eligibility/client.py": { + "stedi": 16 + }, + "eligibility/payer_routes.yaml": { + "stedi": 3 + }, + "eligibility/waterfall.py": { + "stedi": 2 + }, + "ensure_record.py": { + "medplum": 3 + }, + "execute/dispatch.py": { + "mockmed": 7 + }, + "execute/registry.py": { + "mockmed": 18 + }, + "execution_profiles.py": { + "mockmed": 1 + }, + "fault_gate.py": { + "mockmed-records": 1, + "openemr": 1 + }, + "generate/demo_analysis.py": { + "redmine": 1 + }, + "generate/verify_contract.py": { + "dolibarr": 1, + "espocrm": 2, + "redmine": 2 + }, + "guard_demo.py": { + "mockmed": 5, + "mockmed-records": 2, + "mockmed-write": 1 + }, + "integrations/harness2/__init__.py": { + "mockmed": 2 + }, + "integrations/harness2/adapter.py": { + "mockmed": 2 + }, + "integrations/harness2/agent.py": { + "mockmed": 1 + }, + "integrations/harness2/apps.py": { + "mockmed": 13, + "mockmed-records": 1 + }, + "integrations/harness2/tasks.py": { + "mockmed": 24 + }, + "ir.py": { + "token_main": 1 + }, + "jit/authority.py": { + "textedit": 2 + }, + "jit/compile_agent.py": { + "textedit": 1 + }, + "jit/recorded.py": { + "mockmed": 1 + }, + "jit/signature.py": { + "mockmed": 1 + }, + "learning/synth_stream.py": { + "mockmed": 5 + }, + "mockloan/__init__.py": { + "mockloan": 2, + "mockmed": 2 + }, + "mockloan/fault_server.py": { + "mockloan": 6, + "mockmed-records": 2 + }, + "mockloan/server.py": { + "mockloan": 4 + }, + "mockloan/static/app.js": { + "mockloan": 3, + "mockmed": 1 + }, + "mockloan/static/index.html": { + "mockloan": 2 + }, + "mockloan/static/styles.css": { + "mockloan": 1, + "mockmed": 1 + }, + "network_capture.py": { + "clearpass": 1 + }, + "operator_profile.py": { + "openemr": 1 + }, + "policy_packs.py": { + "mockmed": 1 + }, + "qualification_dev_signer.py": { + "mockmed": 1 + }, + "qualification_oracle_gate.py": { + "mockmed": 6, + "mockmed-records": 2, + "mockmed-write": 1 + }, + "qualification_proposal.py": { + "mockmed": 1 + }, + "qualification_web_session.py": { + "openemr": 1, + "openimis": 1 + }, + "qualify/config.py": { + "openemr": 1 + }, + "record_check_binding.py": { + "openimis": 1 + }, + "record_check_identity.py": { + "openemr": 1 + }, + "record_check_proof.py": { + "mockmed": 8, + "mockmed-records": 1, + "mockmed-write": 1 + }, + "record_check_synthesis.py": { + "mockmed": 1 + }, + "registry.py": { + "openemr": 2 + }, + "report_redaction.py": { + "csrf_token_form": 3, + "openemr": 2, + "token_main": 5 + }, + "reward/models.py": { + "mockmed-records": 1 + }, + "reward/seed.py": { + "mockmed": 33 + }, + "risk.py": { + "openemr": 1, + "openimis": 1 + }, + "rollback_state.py": { + "openemr": 1 + }, + "runtime/actuators/web_form.py": { + "openemr": 2 + }, + "runtime/effects/__init__.py": { + "frappe": 1, + "mockmed": 1, + "mockmed-records": 1, + "openemr": 1 + }, + "runtime/effects/_common.py": { + "openemr": 1 + }, + "runtime/effects/auth.py": { + "frappe": 1 + }, + "runtime/effects/compensation.py": { + "mockmed-write": 2 + }, + "runtime/effects/db_snapshot.py": { + "dolibarr": 1, + "dolibarr-table": 1 + }, + "runtime/effects/effect.py": { + "mockmed": 2, + "openemr": 5, + "redmine": 1 + }, + "runtime/effects/fhir.py": { + "epic": 1, + "mockmed": 2, + "openemr": 8, + "openemr-fhir-path": 2 + }, + "runtime/effects/frame.py": { + "openemr": 7, + "patient_data": 1, + "pnotes": 1, + "uuid_registry": 1 + }, + "runtime/effects/fresh_session.py": { + "mockmed": 1 + }, + "runtime/effects/rest.py": { + "mockmed": 1, + "mockmed-records": 2, + "openemr": 1 + }, + "runtime/effects/sql.py": { + "frappe": 7, + "frappe-table": 1, + "openemr": 1 + }, + "runtime/identity.py": { + "mockmed": 1 + }, + "runtime/replayer.py": { + "mockmed": 1 + }, + "submit_binding.py": { + "openimis": 1 + }, + "validation/dense_surface.py": { + "mockmed": 1, + "openemr": 2 + }, + "verifier.py": { + "openemr": 1 + }, + "vision/hashing.py": { + "mockmed": 1 + }, + "volatility.py": { + "openemr": 3 + } +} diff --git a/packages/flow/tests/test_engine_generality_check.py b/packages/flow/tests/test_engine_generality_check.py new file mode 100644 index 0000000000..40f0ac17c5 --- /dev/null +++ b/packages/flow/tests/test_engine_generality_check.py @@ -0,0 +1,97 @@ +"""The engine-generality check: app names fail, examples and data pass.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "check_engine_generality.py" + + +@pytest.fixture(scope="module") +def check(): + spec = importlib.util.spec_from_file_location("check_engine_generality", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _engine(tmp_path: Path, files: dict[str, str]) -> Path: + for rel, text in files.items(): + path = tmp_path / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return tmp_path + + +def test_a_name_in_code_is_a_hit(check, tmp_path: Path) -> None: + engine = _engine(tmp_path, {"runtime/x.py": 'if app == "OpenEMR":\n pass\n'}) + assert check.scan(engine) == {"runtime/x.py": {"openemr": [1]}} + + +def test_an_identifier_without_a_name_is_a_hit(check, tmp_path: Path) -> None: + engine = _engine(tmp_path, {"report.py": 'SECRET = {"token_main"}\n'}) + assert check.scan(engine) == {"report.py": {"token_main": [1]}} + + +def test_a_marked_example_comment_passes(check, tmp_path: Path) -> None: + engine = _engine( + tmp_path, + { + "a.py": "# example: OpenEMR keys rows by uuid\nx = 1\n", + # Without the marker the same comment is a hit. + "b.py": "# OpenEMR keys rows by uuid\nx = 1\n", + # The marker only counts on a comment line, not after code. + "c.py": 'x = "OpenEMR" # example: no\n', + }, + ) + assert check.scan(engine) == {"b.py": {"openemr": [1]}, "c.py": {"openemr": [1]}} + + +def test_app_profiles_are_data(check, tmp_path: Path) -> None: + engine = _engine(tmp_path, {"app_profiles/openemr.yaml": "app: openemr\n"}) + assert check.scan(engine) == {} + + +def test_the_demo_app_is_named_only_in_demo_modules(check, tmp_path: Path) -> None: + engine = _engine( + tmp_path, + { + "tutorial.py": 'URL = "/api/db" # the MockMed demo\n', + "runtime/effects/rest.py": 'records_path = "/api/db"\n', + # The demo modules may name the demo app, not a customer app. + "demo_proof.py": "# OpenEMR\n", + }, + ) + assert check.scan(engine) == { + "runtime/effects/rest.py": {"mockmed-records": [1]}, + "demo_proof.py": {"openemr": [1]}, + } + + +def test_epic_matches_the_product_not_the_word(check, tmp_path: Path) -> None: + engine = _engine( + tmp_path, {"a.py": "# an epic refactor\n", "b.py": "# Epic sends\n"} + ) + assert check.scan(engine) == {"b.py": {"epic": [1]}} + + +def test_the_baseline_only_shrinks(check) -> None: + hits = {"a.py": {"openemr": [3, 9]}, "b.py": {"medplum": [1]}} + baseline = {"a.py": {"openemr": 2}, "c.py": {"redmine": 1}} + + new, stale = check.compare(hits, baseline) + + assert len(new) == 1 and "b.py" in new[0] and "medplum" in new[0] + assert len(stale) == 1 and "c.py" in stale[0] + + +def test_the_engine_passes_against_its_baseline(check) -> None: + baseline = json.loads(check.BASELINE.read_text(encoding="utf-8")) + new, stale = check.compare(check.scan(), baseline) + assert not new, "\n".join(new) + assert not stale, "\n".join(stale) From 83300a021c6b5c98655ba39e662dead4ddd50b0d Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 13:48:49 -0500 Subject: [PATCH 04/16] Register the generality denylist and baseline as public artifacts --- packages/flow/public-artifacts.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 17b27ecdb2..84e66c296b 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -571,6 +571,8 @@ "schemas/sanitization-ingest-v1.json", "schemas/sanitized-artifact-approval-v1.json", "schemas/sanitized-artifact-manifest-v1.json", + "scripts/generality.toml", + "scripts/generality_baseline.json", "source-policy.public.json", "tests/golden/tutorial_run.json" ] From 764339b136dc0c970b8f960ea5ee5fdc19e06163 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 12:39:45 -0500 Subject: [PATCH 05/16] Engine generality audit; move the benchmark harnesses out of the wheel Audit every place the engine (packages/flow/openadapt_flow) names a specific application, bucketed as example, data, harness in the package, or real special-casing: docs/GENERALITY_AUDIT.md. Move openadapt_flow/benchmark and openadapt_flow/instrument to packages/flow/benchmark/harness (import benchmark.harness). The wheel packages only openadapt_flow, so it no longer ships them. The benchmark command loads the harness from a source checkout and says so from a wheel; instrument already failed to import from a wheel. Reword two comments that named an app on a generic branch. --- packages/flow/CHANGELOG.md | 2 + .../benchmark/agent_arm_verticals/README.md | 2 +- .../code_as_action/measure_openemr.py | 4 +- packages/flow/benchmark/dom/BENCHMARK.md | 4 +- .../flow/benchmark/effect_e2e/EFFECT_E2E.md | 2 +- .../benchmark/effect_e2e/record_service.py | 2 +- packages/flow/benchmark/effect_e2e/run.py | 8 +- .../effectbench/reference_fault_model.py | 10 +- .../benchmark/effectbench/task_pack/README.md | 2 +- .../effectbench/task_pack/__init__.py | 4 +- .../effectbench/task_pack/_authoring.py | 10 +- .../benchmark/effectbench/task_pack/audit.py | 2 +- .../benchmark/effectbench/task_pack/driver.py | 8 +- .../effectbench/task_pack/frappe_tasks.py | 8 +- .../effectbench/task_pack/mockmed_tasks.py | 8 +- .../effectbench/task_pack/openemr_tasks.py | 8 +- .../benchmark/effectbench/task_pack/pack.py | 6 +- .../flow/benchmark/frappe_lending/README.md | 2 +- .../harness}/__init__.py | 6 +- .../harness}/agent_baseline.py | 0 .../harness}/chart_fonts.py | 0 .../harness}/desktop_benchmark.py | 4 +- .../harness}/dom_arm.py | 20 +- .../harness}/effectbench/README.md | 2 +- .../harness}/effectbench/__init__.py | 8 +- .../harness}/effectbench/metrics.py | 4 +- .../harness}/effectbench/oracle.py | 6 +- .../harness}/effectbench/runner/README.md | 6 +- .../harness}/effectbench/runner/__init__.py | 26 +- .../harness}/effectbench/runner/__main__.py | 18 +- .../harness}/effectbench/runner/arms.py | 12 +- .../harness}/effectbench/runner/baselines.py | 8 +- .../harness}/effectbench/runner/compound.py | 8 +- .../harness}/effectbench/runner/harness.py | 14 +- .../effectbench/runner/reference_tasks.py | 12 +- .../harness}/effectbench/runner/substrate.py | 4 +- .../harness}/effectbench/schema.py | 4 +- .../harness}/frappe_lending.py | 0 .../harness}/hybrid_benchmark.py | 14 +- .../harness}/instrument/__init__.py | 4 +- .../harness}/instrument/competitor_drift.py | 6 +- .../harness}/openemr_benchmark.py | 12 +- .../harness}/openemr_local.py | 2 +- .../harness}/reliability.py | 0 .../harness}/run_benchmark.py | 10 +- .../harness}/silent_wrong_action.py | 8 +- .../benchmark => benchmark/harness}/verify.py | 0 packages/flow/benchmark/hybrid/BENCHMARK.md | 2 +- .../benchmark/lending_fault_model/README.md | 2 +- .../benchmark/lending_fault_model/swer.py | 4 +- .../openemr_api_tier/approval_durable.py | 2 +- .../flow/benchmark/openemr_api_tier/common.py | 2 +- .../benchmark/openemr_api_tier/feature_e2e.py | 4 +- .../benchmark/openemr_api_tier/spike/bench.py | 2 +- .../openemr_api_tier/spike/capture_har.py | 2 +- packages/flow/benchmark/openemr_e2e/README.md | 2 +- .../flow/benchmark/openemr_e2e/harness.py | 2 +- .../flow/benchmark/openemr_local/README.md | 2 +- .../benchmark/openemr_throughput/harness.py | 4 +- .../flow/benchmark/openimis_claims/README.md | 2 +- .../benchmark/scoreboard/tasks/openemr.py | 2 +- .../flow/benchmark/silent_saves/metrics.py | 2 +- .../SILENT_WRONG_ACTION.md | 2 +- .../flow/benchmark/vision_hardening/README.md | 2 +- packages/flow/docs/EFFECT_KIT.md | 2 +- packages/flow/docs/GENERALITY_AUDIT.md | 311 ++++++++++++++++++ packages/flow/docs/design/EFFECT_VERIFIER.md | 2 +- packages/flow/docs/desktop/PHASE2.md | 4 +- packages/flow/openadapt_flow/__main__.py | 25 +- .../flow/openadapt_flow/anthropic_defaults.py | 2 +- packages/flow/openadapt_flow/authoring.py | 2 +- packages/flow/openadapt_flow/deployment.py | 2 +- packages/flow/paper/REVIEW_ADVERSARIAL.md | 4 +- packages/flow/pyproject.toml | 2 +- packages/flow/scripts/desktop/waa_shim.py | 2 +- packages/flow/scripts/frappe_lending_demo.py | 4 +- packages/flow/scripts/openemr_demo.py | 10 +- packages/flow/scripts/openemr_local_demo.py | 4 +- packages/flow/scripts/openemr_param_depth.py | 2 +- .../test_region_stable_theme_regression.py | 2 +- packages/flow/tests/test_benchmark.py | 14 +- packages/flow/tests/test_chart_fonts.py | 4 +- .../tests/test_competitor_drift_instrument.py | 4 +- packages/flow/tests/test_desktop_benchmark.py | 4 +- packages/flow/tests/test_dom_arm.py | 8 +- packages/flow/tests/test_effectbench.py | 12 +- .../flow/tests/test_effectbench_runner.py | 16 +- .../flow/tests/test_effectbench_standalone.py | 6 +- .../flow/tests/test_effectbench_task_pack.py | 4 +- .../tests/test_frappe_lending_benchmark.py | 2 +- packages/flow/tests/test_hybrid_benchmark.py | 8 +- packages/flow/tests/test_lending_swer.py | 2 +- packages/flow/tests/test_openemr_benchmark.py | 12 +- .../tests/test_openemr_local_benchmark.py | 4 +- .../tests/test_openemr_retained_finals.py | 2 +- .../flow/tests/test_reliability_harness.py | 2 +- .../test_runtime_never_imports_benchmark.py | 58 ++-- .../test_silent_wrong_action_benchmark.py | 2 +- .../tests/test_ui_dismiss_live_openemr.py | 2 +- 99 files changed, 622 insertions(+), 274 deletions(-) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/__init__.py (63%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/agent_baseline.py (100%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/chart_fonts.py (100%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/desktop_benchmark.py (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/dom_arm.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/README.md (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/__init__.py (93%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/metrics.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/oracle.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/README.md (95%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/__init__.py (72%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/__main__.py (90%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/arms.py (95%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/baselines.py (97%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/compound.py (94%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/harness.py (92%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/reference_tasks.py (96%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/runner/substrate.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/effectbench/schema.py (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/frappe_lending.py (100%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/hybrid_benchmark.py (99%) rename packages/flow/{openadapt_flow => benchmark/harness}/instrument/__init__.py (91%) rename packages/flow/{openadapt_flow => benchmark/harness}/instrument/competitor_drift.py (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/openemr_benchmark.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/openemr_local.py (99%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/reliability.py (100%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/run_benchmark.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/silent_wrong_action.py (98%) rename packages/flow/{openadapt_flow/benchmark => benchmark/harness}/verify.py (100%) create mode 100644 packages/flow/docs/GENERALITY_AUDIT.md diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md index 33467f5c83..37dd88ec85 100644 --- a/packages/flow/CHANGELOG.md +++ b/packages/flow/CHANGELOG.md @@ -25,6 +25,8 @@ ### Fixes +- The wheel no longer ships the benchmark harnesses. `openadapt_flow.benchmark` and `openadapt_flow.instrument` moved to `packages/flow/benchmark/harness/` in the source tree (import `benchmark.harness`); `openadapt-flow benchmark` runs from a source checkout and says so from a wheel. The engine audit for application-specific code is in `docs/GENERALITY_AUDIT.md`. + - Self-repair on a covered identity band dismisses only the proved dialog: the covering window must span the band from where the read stops to its right edge at full height, the dismiss click must land in that same window, and only a dialog whose every word is a notice word (no question) may be dismissed ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). - Self-repair can take a stop where the app's own dialog covers the next step's identity band: when the backend proves the covering window belongs to the window the last verified step was in, and the part of the band read still matches the record, the dialog is dismissed after the previous click and the step re-checks its identity in full. Another record behind the dialog stays an identity halt. The runtime lets a backend remember a verified record's window (`record_verified_window`); macOS answers `owned_window_over` from the window's proved sheet ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). diff --git a/packages/flow/benchmark/agent_arm_verticals/README.md b/packages/flow/benchmark/agent_arm_verticals/README.md index fbfe9544dc..54526bd8fa 100644 --- a/packages/flow/benchmark/agent_arm_verticals/README.md +++ b/packages/flow/benchmark/agent_arm_verticals/README.md @@ -26,7 +26,7 @@ budget without completing the write in every trial. ## Method and public/private boundary The public, generic agent-baseline mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. It provides explicit paid-run +`benchmark/harness/agent_baseline.py`. It provides explicit paid-run opt-in, action and cost caps, usage accounting, and an agent self-report that is never trusted for scoring. The public benchmark schemas and aggregate failure taxonomy also remain available for independently authored tasks. diff --git a/packages/flow/benchmark/code_as_action/measure_openemr.py b/packages/flow/benchmark/code_as_action/measure_openemr.py index 74a34443a6..9e1b5e70f0 100644 --- a/packages/flow/benchmark/code_as_action/measure_openemr.py +++ b/packages/flow/benchmark/code_as_action/measure_openemr.py @@ -184,7 +184,7 @@ def spec_for(key: str) -> Any: - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec return SyntheticPatientSpec(**REFERRALS[key], **COMMON_FIELDS) @@ -390,7 +390,7 @@ def main() -> int: keys = [k for k in args.referrals.split(",") if k] from benchmark.openemr_local import qualify_hooks - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec fixture = qualify_hooks.fixture() demo = _demo() diff --git a/packages/flow/benchmark/dom/BENCHMARK.md b/packages/flow/benchmark/dom/BENCHMARK.md index 7111596c12..2b40374899 100644 --- a/packages/flow/benchmark/dom/BENCHMARK.md +++ b/packages/flow/benchmark/dom/BENCHMARK.md @@ -23,7 +23,7 @@ row's Open button — the literal reading — and **DOM (name-filtered)** clicks the Open button in the row named `Jane Sample` — the identity reading, hardcoding the demonstrated patient exactly as the compiled arm's recorded identity band does. Every selector choice is -documented in `openadapt_flow/benchmark/dom_arm.py`. +documented in `benchmark/harness/dom_arm.py`. ## Verdict @@ -208,7 +208,7 @@ Unmeasured variants, for completeness: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.dom_arm --out benchmark/dom --n-per-perturbation 2 +.venv/bin/python -m benchmark.harness.dom_arm --out benchmark/dom --n-per-perturbation 2 ``` (`--n-per-perturbation 2` matches the committed results.) No API key diff --git a/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md b/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md index dfa26c1e61..58868ee444 100644 --- a/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md +++ b/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md @@ -2,7 +2,7 @@ Date: 2026-07-21. This is the genuinely independent, end-to-end measurement of the silent-wrong-effect (SWER) result. Unlike the definitional -`openadapt_flow.benchmark.silent_wrong_action` (raw `requests.post`; effect +`benchmark.harness.silent_wrong_action` (raw `requests.post`; effect verifier and ground truth read the SAME in-process object; the effect contract restates the ground-truth definition -- so its `0/90` is circular by construction), every write below goes through the ACTUAL governed replay path diff --git a/packages/flow/benchmark/effect_e2e/record_service.py b/packages/flow/benchmark/effect_e2e/record_service.py index 2bb71f2e90..0f8f6e0fa3 100644 --- a/packages/flow/benchmark/effect_e2e/record_service.py +++ b/packages/flow/benchmark/effect_e2e/record_service.py @@ -3,7 +3,7 @@ This is the system of record the end-to-end silent-wrong-effect harness (:mod:`benchmark.effect_e2e.run`) writes to THROUGH the governed replay path. It exists to break the circularity of the older definitional benchmark -(``openadapt_flow.benchmark.silent_wrong_action``), whose effect verifier and +(``benchmark.harness.silent_wrong_action``), whose effect verifier and ground-truth judge read the SAME in-process Python object and whose write bypassed the :class:`~openadapt_flow.runtime.replayer.Replayer`. diff --git a/packages/flow/benchmark/effect_e2e/run.py b/packages/flow/benchmark/effect_e2e/run.py index d08601d691..7860d35b89 100644 --- a/packages/flow/benchmark/effect_e2e/run.py +++ b/packages/flow/benchmark/effect_e2e/run.py @@ -1,7 +1,7 @@ """End-to-end silent-wrong-effect (SWER) harness through the REAL replayer. This is the genuinely independent, end-to-end version of the silent-wrong-action -result. The older ``openadapt_flow.benchmark.silent_wrong_action`` is +result. The older ``benchmark.harness.silent_wrong_action`` is *definitional*: it issues raw ``requests.post`` (bypassing the :class:`~openadapt_flow.runtime.replayer.Replayer`), and its effect verifier and ground-truth judge both read the SAME in-process ``FaultDB`` object while the @@ -629,7 +629,7 @@ def render_markdown(results: dict[str, Any]) -> str: Date: {date}. This is the genuinely independent, end-to-end measurement of the silent-wrong-effect (SWER) result. Unlike the definitional -`openadapt_flow.benchmark.silent_wrong_action` (raw `requests.post`; effect +`benchmark.harness.silent_wrong_action` (raw `requests.post`; effect verifier and ground truth read the SAME in-process object; the effect contract restates the ground-truth definition -- so its `0/90` is circular by construction), every write below goes through the ACTUAL governed replay path @@ -774,7 +774,7 @@ def render_markdown(results: dict[str, Any]) -> str: def render_chart(results: dict[str, Any], out_png: Path) -> Path: """Bar chart: silent-wrong-effect rate per arm (large -> small -> zero).""" - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() m = results["metrics"]["per_arm"] @@ -827,7 +827,7 @@ def write_outputs(results: dict[str, Any], out_dir: Path) -> None: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") (out_dir / "EFFECT_E2E.md").write_text(render_markdown(results)) - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "effect_e2e.png") diff --git a/packages/flow/benchmark/effectbench/reference_fault_model.py b/packages/flow/benchmark/effectbench/reference_fault_model.py index 5ba4904208..0488c51dff 100644 --- a/packages/flow/benchmark/effectbench/reference_fault_model.py +++ b/packages/flow/benchmark/effectbench/reference_fault_model.py @@ -31,16 +31,16 @@ How it re-expresses the study, faithfully and without hardcoding verdicts: - Each of the study's nine scenarios (``benchmark.fault_model.faults.FAULTS``) - becomes a :class:`~openadapt_flow.benchmark.effectbench.schema.TaskSpec` plus + becomes a :class:`~benchmark.harness.effectbench.schema.TaskSpec` plus the KNOWN post-action system-of-record state that fault produces (the same DB states the study's own taxonomy tests assert). The states live in :data:`SCENARIOS` — the record shapes are MockMed ``FaultDB`` rows. - The TRUE effect is read by the substrate-agnostic - :class:`~openadapt_flow.benchmark.effectbench.oracle.RecordSnapshotOracle` + :class:`~benchmark.harness.effectbench.oracle.RecordSnapshotOracle` running the SAME shared ``judge_records`` decision logic the runtime uses — once for the at-most-once/collateral ``record_written`` contract and once for the ``field_equals`` note read-back — reduced by - :func:`~openadapt_flow.benchmark.effectbench.oracle.combine_true_states`. + :func:`~benchmark.harness.effectbench.oracle.combine_true_states`. - Two arms are scored against the identical independent oracle: ``screen_only`` (the weak banner oracle — reports success whenever the screen looks saved) and ``effect_verify`` (the agent halts unless the oracle CONFIRMS). Only the @@ -56,7 +56,7 @@ from typing import Optional from benchmark.fault_model import faults as F -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, DivergenceCategory, Effect, @@ -72,7 +72,7 @@ oracle_verdict_of, summarize, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel # The consequential-save contract the study verifies: exactly one Triage # encounter for the target patient (at-most-once + no collateral loss) whose diff --git a/packages/flow/benchmark/effectbench/task_pack/README.md b/packages/flow/benchmark/effectbench/task_pack/README.md index 88fa4d6c25..8846166f4c 100644 --- a/packages/flow/benchmark/effectbench/task_pack/README.md +++ b/packages/flow/benchmark/effectbench/task_pack/README.md @@ -68,7 +68,7 @@ python -m benchmark.effectbench.task_pack.pack # Live-run the MockMed anchor through the oracle (prints per-task classification) python -c "from benchmark.effectbench.task_pack.driver import run_mockmed_pack; \ -from openadapt_flow.benchmark.effectbench import summarize; \ +from benchmark.harness.effectbench import summarize; \ eps=run_mockmed_pack(trials=3); \ print('screen SWER', summarize(eps, arm='screen_only').swer.rate); \ print('effect SWER', summarize(eps, arm='effect_verify').swer.rate)" diff --git a/packages/flow/benchmark/effectbench/task_pack/__init__.py b/packages/flow/benchmark/effectbench/task_pack/__init__.py index e3f3537b7b..c823a82252 100644 --- a/packages/flow/benchmark/effectbench/task_pack/__init__.py +++ b/packages/flow/benchmark/effectbench/task_pack/__init__.py @@ -6,9 +6,9 @@ ``benchmark.environments`` registry (PR #173) indexes, authored against the frozen EffectBench schema/oracle/metrics contract (PR #178). -Each task is a :class:`~openadapt_flow.benchmark.effectbench.TaskSpec` (goal = +Each task is a :class:`~benchmark.harness.effectbench.TaskSpec` (goal = INTENT only, never a step list) plus an -:class:`~openadapt_flow.benchmark.effectbench.OracleSpec` bound to the +:class:`~benchmark.harness.effectbench.OracleSpec` bound to the environment's system-of-record READ recipe, DESIGNED so a green screen (task-success) diverges from the correct business effect. The oracle reads pre/post SoR state independently of the screen; the non-gameability attestations diff --git a/packages/flow/benchmark/effectbench/task_pack/_authoring.py b/packages/flow/benchmark/effectbench/task_pack/_authoring.py index 00e02bc078..aa0ba26354 100644 --- a/packages/flow/benchmark/effectbench/task_pack/_authoring.py +++ b/packages/flow/benchmark/effectbench/task_pack/_authoring.py @@ -3,10 +3,10 @@ These build the two things every consequential-write task needs: - the DECLARED expected effect (a typed - :class:`~openadapt_flow.benchmark.effectbench.Effect`), parameterized by run - params via :class:`~openadapt_flow.benchmark.effectbench.ValueExpr` so the + :class:`~benchmark.harness.effectbench.Effect`), parameterized by run + params via :class:`~benchmark.harness.effectbench.ValueExpr` so the record CHECKED is the record this trial WROTE (trial-unique payload); and -- the :class:`~openadapt_flow.benchmark.effectbench.OracleSpec` that declares +- the :class:`~benchmark.harness.effectbench.OracleSpec` that declares the independent read channel and the non-gameability attestations the adversarial audit signs off on. @@ -21,13 +21,13 @@ from typing import Any, Optional -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, EffectKind, OracleSpec, ValueExpr, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel # Canonical run-param names a trial binds a trial-unique value to. The driver # derives each from the trial seed so the oracle checks THIS run's exact write. diff --git a/packages/flow/benchmark/effectbench/task_pack/audit.py b/packages/flow/benchmark/effectbench/task_pack/audit.py index c44706010e..fcba2b8c15 100644 --- a/packages/flow/benchmark/effectbench/task_pack/audit.py +++ b/packages/flow/benchmark/effectbench/task_pack/audit.py @@ -41,7 +41,7 @@ _note_effect, _record_effect, ) -from openadapt_flow.benchmark.effectbench import RestRecordVerifier, Verdict +from benchmark.harness.effectbench import RestRecordVerifier, Verdict _HTTP_TIMEOUT_S = 5.0 TRIAL_NOTE = "triage note trial-unique-AUDIT" diff --git a/packages/flow/benchmark/effectbench/task_pack/driver.py b/packages/flow/benchmark/effectbench/task_pack/driver.py index 4145e46ae7..035a025097 100644 --- a/packages/flow/benchmark/effectbench/task_pack/driver.py +++ b/packages/flow/benchmark/effectbench/task_pack/driver.py @@ -8,9 +8,9 @@ mocked verdicts. It depends ONLY on the public EffectBench surface -(:func:`~openadapt_flow.benchmark.effectbench.score_episode`, -:class:`~openadapt_flow.benchmark.effectbench.RestRecordVerifier`, -:class:`~openadapt_flow.benchmark.effectbench.AgentReport`) and the MockMed +(:func:`~benchmark.harness.effectbench.score_episode`, +:class:`~benchmark.harness.effectbench.RestRecordVerifier`, +:class:`~benchmark.harness.effectbench.AgentReport`) and the MockMed fault server. It does not reimplement any real baseline "arm"; the two arms here are the two SCORING conditions the benchmark contrasts: @@ -46,7 +46,7 @@ MockMedDrive, MockMedTask, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, EpisodeRecord, RestRecordVerifier, diff --git a/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py b/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py index e2c22a2ef5..9219f750ea 100644 --- a/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py +++ b/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py @@ -17,7 +17,7 @@ ``oracle.config["query"]`` + ``["query_params"]`` is what the multi-baseline runner materializes into a -:class:`~openadapt_flow.benchmark.effectbench.SqlRecordVerifier` (read-only role, +:class:`~benchmark.harness.effectbench.SqlRecordVerifier` (read-only role, DB-API-bound params). The loan-application docname has a space (`` `tabLoan Application` ``), so the table is back-tick quoted in the SELECT. """ @@ -33,15 +33,15 @@ record_written_effect, ) from benchmark.effectbench.task_pack.openemr_tasks import ContainerTask -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( DivergenceCategory as DC, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, Substrate, TaskSpec, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel ENVIRONMENT = "frappe_lending" diff --git a/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py b/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py index 898f90db42..9f9b679bc3 100644 --- a/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py +++ b/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py @@ -8,7 +8,7 @@ path is exercised in CI (no mocked verdicts). Each entry is a :class:`MockMedTask`: an agent-agnostic -:class:`~openadapt_flow.benchmark.effectbench.TaskSpec` (goal = intent only) +:class:`~benchmark.harness.effectbench.TaskSpec` (goal = intent only) plus a :class:`MockMedDrive` — the environment recipe the live driver replays to MANIFEST the divergence (which ``?fault=`` mode to inject, the write the agent performs, decoy/concurrent rows to seed, and the documented banner @@ -34,9 +34,9 @@ param, record_written_effect, ) -from openadapt_flow.benchmark.effectbench import DivergenceCategory as DC -from openadapt_flow.benchmark.effectbench import Effect, Substrate, TaskSpec -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench import DivergenceCategory as DC +from benchmark.harness.effectbench import Effect, Substrate, TaskSpec +from benchmark.harness.effectbench.schema import OracleChannel TARGET_PATIENT = "p1" TARGET_TYPE = "Triage" diff --git a/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py b/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py index 2f6d61974d..7581aa2520 100644 --- a/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py +++ b/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py @@ -18,7 +18,7 @@ The SQL each oracle runs (``oracle.config["query"]`` + ``["query_params"]``) is what the multi-baseline runner materializes into a -:class:`~openadapt_flow.benchmark.effectbench.SqlRecordVerifier` (read-only role; +:class:`~benchmark.harness.effectbench.SqlRecordVerifier` (read-only role; values bound through DB-API params, never interpolated). Column names become record keys, so the same typed ``Effect`` the MockMed anchor uses judges these rows unchanged. @@ -36,15 +36,15 @@ param, record_written_effect, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( DivergenceCategory as DC, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, Substrate, TaskSpec, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel ENVIRONMENT = "openemr_local" diff --git a/packages/flow/benchmark/effectbench/task_pack/pack.py b/packages/flow/benchmark/effectbench/task_pack/pack.py index 2be0450827..fa15417f72 100644 --- a/packages/flow/benchmark/effectbench/task_pack/pack.py +++ b/packages/flow/benchmark/effectbench/task_pack/pack.py @@ -4,7 +4,7 @@ the container-gated OpenEMR and Frappe families). It: - assembles the unified list of :class:`PackEntry` (one per authored task), - each carrying its :class:`~openadapt_flow.benchmark.effectbench.TaskSpec`, the + each carrying its :class:`~benchmark.harness.effectbench.TaskSpec`, the pinned environment it targets, and whether it runs LIVE here or needs a container; - validates every task against the ``benchmark.environments`` registry (the @@ -29,8 +29,8 @@ from benchmark.effectbench.task_pack.frappe_tasks import FRAPPE_TASKS from benchmark.effectbench.task_pack.mockmed_tasks import MOCKMED_TASKS from benchmark.effectbench.task_pack.openemr_tasks import OPENEMR_TASKS -from openadapt_flow.benchmark.effectbench import TaskSpec -from openadapt_flow.benchmark.effectbench.schema import DivergenceCategory, Substrate +from benchmark.harness.effectbench import TaskSpec +from benchmark.harness.effectbench.schema import DivergenceCategory, Substrate MANIFEST_SCHEMA_VERSION = 1 diff --git a/packages/flow/benchmark/frappe_lending/README.md b/packages/flow/benchmark/frappe_lending/README.md index 1504ddf329..89ae27fa5a 100644 --- a/packages/flow/benchmark/frappe_lending/README.md +++ b/packages/flow/benchmark/frappe_lending/README.md @@ -30,7 +30,7 @@ trials per cell remains the publication bar. Raw per-run rows, environment fingerprints, application-specific adapter/oracle wiring, and the detailed cost ledger are retained in the private `OpenAdaptAI/openadapt-corpus` repository. The public generic cost-capped agent mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. See the +`benchmark/harness/agent_baseline.py`. See the [aggregate report](../agent_arm_verticals/README.md). The accepted run used baseline SHA-256 diff --git a/packages/flow/openadapt_flow/benchmark/__init__.py b/packages/flow/benchmark/harness/__init__.py similarity index 63% rename from packages/flow/openadapt_flow/benchmark/__init__.py rename to packages/flow/benchmark/harness/__init__.py index b157c4fda1..2eba51f30b 100644 --- a/packages/flow/openadapt_flow/benchmark/__init__.py +++ b/packages/flow/benchmark/harness/__init__.py @@ -1,4 +1,8 @@ -"""Benchmark: compiled replay vs. a frontier computer-use agent. +"""Benchmark harnesses: compiled replay vs. a frontier computer-use agent. + +This package lives in the source checkout, outside ``openadapt_flow``, so the +wheel does not ship it. Import it as ``benchmark.harness`` from +``packages/flow``. Submodules (import them directly; nothing heavy is imported here): diff --git a/packages/flow/openadapt_flow/benchmark/agent_baseline.py b/packages/flow/benchmark/harness/agent_baseline.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/agent_baseline.py rename to packages/flow/benchmark/harness/agent_baseline.py diff --git a/packages/flow/openadapt_flow/benchmark/chart_fonts.py b/packages/flow/benchmark/harness/chart_fonts.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/chart_fonts.py rename to packages/flow/benchmark/harness/chart_fonts.py diff --git a/packages/flow/openadapt_flow/benchmark/desktop_benchmark.py b/packages/flow/benchmark/harness/desktop_benchmark.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/desktop_benchmark.py rename to packages/flow/benchmark/harness/desktop_benchmark.py index 270605e54b..c581ca306d 100644 --- a/packages/flow/openadapt_flow/benchmark/desktop_benchmark.py +++ b/packages/flow/benchmark/harness/desktop_benchmark.py @@ -745,7 +745,7 @@ def _identity_transfer_verdict(results: dict) -> str: def render_chart(results: dict, path: Path) -> None: """Stacked outcome bars per arm (success / wrong-action / halt).""" - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() arms = list(results["arms"]) @@ -776,7 +776,7 @@ def write_outputs(results: dict, out_dir: str | Path) -> None: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") (out_dir / "BENCHMARK.md").write_text(render_markdown(results)) - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "outcomes.png") diff --git a/packages/flow/openadapt_flow/benchmark/dom_arm.py b/packages/flow/benchmark/harness/dom_arm.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/dom_arm.py rename to packages/flow/benchmark/harness/dom_arm.py index a6c0e598dd..27a1e277ef 100644 --- a/packages/flow/openadapt_flow/benchmark/dom_arm.py +++ b/packages/flow/benchmark/harness/dom_arm.py @@ -10,7 +10,7 @@ Three arms, one frozen schedule, one arm-independent success check: - **compiled**: the existing vision-anchored compiled replay - (:func:`openadapt_flow.benchmark.run_benchmark._compiled_run`), healing + (:func:`benchmark.harness.run_benchmark._compiled_run`), healing enabled as always. - **dom** (positional) and **dom_named** (name-filtered): :func:`dom_script` — a Playwright script written the way a competent @@ -29,7 +29,7 @@ drift modes (:data:`PERTURBATIONS`: lookalike, missing, grow, sort, theme, rename, move, typelabel). Success is judged for BOTH arms by the hybrid benchmark's own arm-independent check -(:func:`~openadapt_flow.benchmark.hybrid_benchmark.verify_hybrid_final`): +(:func:`~benchmark.harness.hybrid_benchmark.verify_hybrid_final`): OCR of the final screenshot must show the saved-encounter evidence AND the right patient, with wrong-type writes flagged — so this head-to-head is judged by the identical bar. Wrong-action events are recorded per arm: @@ -51,7 +51,7 @@ from pathlib import Path from typing import Any, Callable, Optional -from openadapt_flow.benchmark.hybrid_benchmark import ( +from benchmark.harness.hybrid_benchmark import ( _NOTE_POOL, DRIFT_TYPES, SCHEDULE, @@ -465,7 +465,7 @@ def dom_arm_aggregate(rows: list[dict[str, Any]]) -> dict[str, Any]: Aggregate dict: base stats plus clean/drift splits, outcome counts, wrong-action count, and (DOM arm) maintenance events. """ - from openadapt_flow.benchmark.run_benchmark import _arm_aggregate + from benchmark.harness.run_benchmark import _arm_aggregate agg = _arm_aggregate(rows) agg["clean"] = _split_rate([r for r in rows if r.get("condition") == "clean"]) @@ -621,7 +621,7 @@ def render_dom_chart(results: dict[str, Any], out_png: Path) -> Path: Returns: The written PNG path. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() from matplotlib.patches import Rectangle @@ -1177,7 +1177,7 @@ def stat_row(label: str, fmt: Callable[[dict[str, Any]], str]) -> str: clicks the Open button in the row named `{TARGET_PATIENT}` — the identity reading, hardcoding the demonstrated patient exactly as the compiled arm's recorded identity band does. Every selector choice is -documented in `openadapt_flow/benchmark/dom_arm.py`. +documented in `benchmark/harness/dom_arm.py`. ## Verdict @@ -1333,7 +1333,7 @@ def stat_row(label: str, fmt: Callable[[dict[str, Any]], str]) -> str: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.dom_arm --out benchmark/dom --n-per-perturbation 2 +.venv/bin/python -m benchmark.harness.dom_arm --out benchmark/dom --n-per-perturbation 2 ``` (`--n-per-perturbation 2` matches the committed results.) No API key @@ -1350,7 +1350,7 @@ def write_dom_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_dom_chart, results, out_dir / "outcome_matrix.png") (out_dir / "BENCHMARK.md").write_text(render_dom_markdown(results)) @@ -1388,7 +1388,7 @@ def run_dom_benchmark( Returns: The results dict (also written to ``results.json``). """ - from openadapt_flow.benchmark.run_benchmark import _compiled_run + from benchmark.harness.run_benchmark import _compiled_run from openadapt_flow.compiler import compile_recording from openadapt_flow.demo_driver import record_triage_demo from openadapt_flow.mockmed.server import serve @@ -1499,7 +1499,7 @@ def one_run( def main(argv: Optional[list[str]] = None) -> int: - """CLI entry point: ``python -m openadapt_flow.benchmark.dom_arm``.""" + """CLI entry point: ``python -m benchmark.harness.dom_arm``.""" parser = argparse.ArgumentParser( description=( "DOM-selector benchmark arm vs. compiled vision replay on " diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/README.md b/packages/flow/benchmark/harness/effectbench/README.md similarity index 99% rename from packages/flow/openadapt_flow/benchmark/effectbench/README.md rename to packages/flow/benchmark/harness/effectbench/README.md index c42718959f..08b926e621 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/README.md +++ b/packages/flow/benchmark/harness/effectbench/README.md @@ -5,7 +5,7 @@ > system-of-record oracle* disagrees. Reported jointly with **over-halt rate**, > **task success**, and the **success–effect gap**. -This package (`openadapt_flow.benchmark.effectbench`) is the **stable contract** +This package (`benchmark.harness.effectbench`) is the **stable contract** that two downstream efforts build on: 1. the **multi-baseline runner adapter** — drives each agent arm through a common diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/__init__.py b/packages/flow/benchmark/harness/effectbench/__init__.py similarity index 93% rename from packages/flow/openadapt_flow/benchmark/effectbench/__init__.py rename to packages/flow/benchmark/harness/effectbench/__init__.py index 7291bdbf04..62562d9441 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/__init__.py +++ b/packages/flow/benchmark/harness/effectbench/__init__.py @@ -31,10 +31,10 @@ bootstrap confidence intervals, all decomposed by category x substrate. The design doc is ``.private/benchmark_design_2026_07_20.md``; the operator -contract is ``openadapt_flow/benchmark/effectbench/README.md``. +contract is ``benchmark/harness/effectbench/README.md``. """ -from openadapt_flow.benchmark.effectbench.metrics import ( # noqa: F401 +from benchmark.harness.effectbench.metrics import ( # noqa: F401 BenchmarkSummary, CellSummary, Interval, @@ -44,7 +44,7 @@ summarize, wilson_interval, ) -from openadapt_flow.benchmark.effectbench.oracle import ( # noqa: F401 +from benchmark.harness.effectbench.oracle import ( # noqa: F401 RecordSnapshotOracle, TrueEffectState, classify_outcome, @@ -53,7 +53,7 @@ oracle_verdict_of, score_episode, ) -from openadapt_flow.benchmark.effectbench.schema import ( # noqa: F401 +from benchmark.harness.effectbench.schema import ( # noqa: F401 AgentReport, DivergenceCategory, EpisodeRecord, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/metrics.py b/packages/flow/benchmark/harness/effectbench/metrics.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/effectbench/metrics.py rename to packages/flow/benchmark/harness/effectbench/metrics.py index b7ad870e60..356119fde2 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/metrics.py +++ b/packages/flow/benchmark/harness/effectbench/metrics.py @@ -1,7 +1,7 @@ """EffectBench metrics — SWER and its co-metrics, with confidence intervals. Every headline is a pure aggregation over a list of -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord` rows, so a +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord` rows, so a run's numbers can be recomputed from the published raw rows by anyone. No single mean is reported alone: :func:`summarize` always returns the per (category x substrate) decomposition alongside the overall rate (design doc @@ -39,7 +39,7 @@ from pydantic import BaseModel, ConfigDict, Field -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( EpisodeRecord, OutcomeLabel, SwerVariant, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/oracle.py b/packages/flow/benchmark/harness/effectbench/oracle.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/effectbench/oracle.py rename to packages/flow/benchmark/harness/effectbench/oracle.py index f7f5ca031a..e490479d8b 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/oracle.py +++ b/packages/flow/benchmark/harness/effectbench/oracle.py @@ -17,7 +17,7 @@ 2. **The classifier** — :func:`classify_outcome`, which crosses the oracle's reading of the true effect with the agent's (untrusted) self-report to - assign exactly one :class:`~openadapt_flow.benchmark.effectbench.schema.OutcomeLabel`. + assign exactly one :class:`~benchmark.harness.effectbench.schema.OutcomeLabel`. This is ``fault_model.classify`` generalized: it adds the OVER_HALT vs SAFE_HALT split (was the correct action available?) and makes ``SILENT_WRONG_EFFECT`` the single home of ``reported_success AND @@ -25,7 +25,7 @@ :func:`score_episode` ties the two together for a runner: capture pre-state, run the arm's action, verify, classify, and emit an -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord`. +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord`. """ from __future__ import annotations @@ -34,7 +34,7 @@ from enum import Enum from typing import Any, Callable, Mapping, Optional -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( AgentReport, DivergenceCategory, EpisodeRecord, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/README.md b/packages/flow/benchmark/harness/effectbench/runner/README.md similarity index 95% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/README.md rename to packages/flow/benchmark/harness/effectbench/runner/README.md index a29834a364..a00d55f331 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/README.md +++ b/packages/flow/benchmark/harness/effectbench/runner/README.md @@ -1,7 +1,7 @@ # EffectBench runner adapter — one interface, every baseline The **multi-baseline runner adapter** (downstream effort #2 on the -`openadapt_flow.benchmark.effectbench` contract). It drives every agent *arm* +`benchmark.harness.effectbench` contract). It drives every agent *arm* through one common interface against the **identical task** and **identical independent oracle**, records one `EpisodeRecord` per (task × arm × trial), and summarizes per arm with `summarize`. A difference between arms is then the @@ -10,8 +10,8 @@ AGENT, never the harness. Run the CI-fast dry-run (real in-process HTTP, no Docker, no paid API, no spend): ```bash -python -m openadapt_flow.benchmark.effectbench.runner # all live arms -python -m openadapt_flow.benchmark.effectbench.runner --list-arms +python -m benchmark.harness.effectbench.runner # all live arms +python -m benchmark.harness.effectbench.runner --list-arms ``` ## The arm interface diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__init__.py b/packages/flow/benchmark/harness/effectbench/runner/__init__.py similarity index 72% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/__init__.py rename to packages/flow/benchmark/harness/effectbench/runner/__init__.py index 759ebae7d1..27fe3e39ff 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__init__.py +++ b/packages/flow/benchmark/harness/effectbench/runner/__init__.py @@ -2,20 +2,20 @@ Drives every agent *arm* through one common interface against the IDENTICAL task and INDEPENDENT oracle, records one -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord` per +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord` per (task × arm × trial), and summarizes per arm with -:func:`~openadapt_flow.benchmark.effectbench.metrics.summarize`. Built on the -stable contract in :mod:`openadapt_flow.benchmark.effectbench` (schema / oracle / +:func:`~benchmark.harness.effectbench.metrics.summarize`. Built on the +stable contract in :mod:`benchmark.harness.effectbench` (schema / oracle / metrics) — this package adds only the runner, never re-implements the taxonomy. Layers: - :mod:`.arms` — the ``AgentArm`` interface + the concrete in-repo arms: - :class:`~openadapt_flow.benchmark.effectbench.runner.arms.CompilerArm` + :class:`~benchmark.harness.effectbench.runner.arms.CompilerArm` (record→compile→replay, effect-gated), - :class:`~openadapt_flow.benchmark.effectbench.runner.arms.ScreenOnlyArm` + :class:`~benchmark.harness.effectbench.runner.arms.ScreenOnlyArm` (the silent-wrong-effect ablation), and a substrate-free - :class:`~openadapt_flow.benchmark.effectbench.runner.arms.MockArm` for CI. + :class:`~benchmark.harness.effectbench.runner.arms.MockArm` for CI. - :mod:`.substrate` — the arm-facing action+perception channel (the reference MockMed session), isolated from the oracle read path. - :mod:`.compound` — the compound consequential-save verifier (record + note). @@ -25,19 +25,19 @@ - :mod:`.harness` — ``run_episode`` / ``run_matrix``: wire each arm through ``score_episode`` with the task's oracle, fairly and identically. -Run the dry-run: ``python -m openadapt_flow.benchmark.effectbench.runner``. +Run the dry-run: ``python -m benchmark.harness.effectbench.runner``. """ from __future__ import annotations -from openadapt_flow.benchmark.effectbench.runner.arms import ( +from benchmark.harness.effectbench.runner.arms import ( AgentArm, ArmResult, CompilerArm, MockArm, ScreenOnlyArm, ) -from openadapt_flow.benchmark.effectbench.runner.baselines import ( +from benchmark.harness.effectbench.runner.baselines import ( SCAFFOLDED_ARMS, BaselineRequirements, ClaudeComputerUseArm, @@ -46,21 +46,21 @@ SkyvernArm, UITarsArm, ) -from openadapt_flow.benchmark.effectbench.runner.compound import ( +from benchmark.harness.effectbench.runner.compound import ( CompoundEffectVerifier, ) -from openadapt_flow.benchmark.effectbench.runner.harness import ( +from benchmark.harness.effectbench.runner.harness import ( EnvFactory, EpisodeEnv, run_episode, run_matrix, ) -from openadapt_flow.benchmark.effectbench.runner.reference_tasks import ( +from benchmark.harness.effectbench.runner.reference_tasks import ( MockMedEnvProvider, mockmed_env_factory, reference_tasks, ) -from openadapt_flow.benchmark.effectbench.runner.substrate import ( +from benchmark.harness.effectbench.runner.substrate import ( MockMedFault, MockMedSession, ScreenObservation, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__main__.py b/packages/flow/benchmark/harness/effectbench/runner/__main__.py similarity index 90% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/__main__.py rename to packages/flow/benchmark/harness/effectbench/runner/__main__.py index 4ba0d142eb..bc6c79734f 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__main__.py +++ b/packages/flow/benchmark/harness/effectbench/runner/__main__.py @@ -1,6 +1,6 @@ """EffectBench runner CLI — the CI-fast dry-run over the MockMed reference pack. -``python -m openadapt_flow.benchmark.effectbench.runner`` drives the nine MockMed +``python -m benchmark.harness.effectbench.runner`` drives the nine MockMed reference tasks through the in-repo LIVE arms (compiler, screen-only ablation, mock) over a real in-process HTTP fault server — no Docker, no paid API, no spend — and prints the per-arm summary. It proves the pipeline end-to-end: the @@ -25,16 +25,16 @@ import sys from typing import Optional, Sequence -from openadapt_flow.benchmark.effectbench.metrics import BenchmarkSummary, summarize -from openadapt_flow.benchmark.effectbench.runner import LIVE_ARMS -from openadapt_flow.benchmark.effectbench.runner.arms import AgentArm -from openadapt_flow.benchmark.effectbench.runner.baselines import SCAFFOLDED_ARMS -from openadapt_flow.benchmark.effectbench.runner.harness import run_matrix -from openadapt_flow.benchmark.effectbench.runner.reference_tasks import ( +from benchmark.harness.effectbench.metrics import BenchmarkSummary, summarize +from benchmark.harness.effectbench.runner import LIVE_ARMS +from benchmark.harness.effectbench.runner.arms import AgentArm +from benchmark.harness.effectbench.runner.baselines import SCAFFOLDED_ARMS +from benchmark.harness.effectbench.runner.harness import run_matrix +from benchmark.harness.effectbench.runner.reference_tasks import ( MockMedEnvProvider, reference_tasks, ) -from openadapt_flow.benchmark.effectbench.schema import EpisodeRecord +from benchmark.harness.effectbench.schema import EpisodeRecord def _select_arms(names: Optional[str]) -> list[AgentArm]: @@ -128,7 +128,7 @@ def run_dry_run( def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser( - prog="python -m openadapt_flow.benchmark.effectbench.runner", + prog="python -m benchmark.harness.effectbench.runner", description="EffectBench multi-baseline runner — MockMed dry-run.", ) parser.add_argument("--trials", type=int, default=3) diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/arms.py b/packages/flow/benchmark/harness/effectbench/runner/arms.py similarity index 95% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/arms.py rename to packages/flow/benchmark/harness/effectbench/runner/arms.py index 063389bcef..a63a5985c3 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/arms.py +++ b/packages/flow/benchmark/harness/effectbench/runner/arms.py @@ -18,14 +18,14 @@ def run(self, task: TaskSpec, session: SubstrateSession, An arm is handed ONLY the task's natural-language ``goal`` (intent, never a step list — the fairness requirement) via ``task``, a -:class:`~openadapt_flow.benchmark.effectbench.runner.substrate.SubstrateSession` +:class:`~benchmark.harness.effectbench.runner.substrate.SubstrateSession` to drive, and the run's ``params`` (the trial-unique payload it must write). It returns an :class:`ArmResult` carrying the untrusted -:class:`~openadapt_flow.benchmark.effectbench.schema.AgentReport` plus any -recorded :class:`~openadapt_flow.benchmark.effectbench.schema.ModelCall` rows so +:class:`~benchmark.harness.effectbench.schema.AgentReport` plus any +recorded :class:`~benchmark.harness.effectbench.schema.ModelCall` rows so cost is auditable. The harness (:mod:`.harness`) adapts ``run`` into the ``run_action: Callable[[], AgentReport]`` that -:func:`~openadapt_flow.benchmark.effectbench.oracle.score_episode` calls, and +:func:`~benchmark.harness.effectbench.oracle.score_episode` calls, and supplies the independent oracle — which the arm can NEVER reach. Three concrete in-repo arms live here: @@ -50,11 +50,11 @@ def run(self, task: TaskSpec, session: SubstrateSession, from dataclasses import dataclass, field from typing import Any, Callable, Mapping, Optional, Protocol, runtime_checkable -from openadapt_flow.benchmark.effectbench.runner.substrate import ( +from benchmark.harness.effectbench.runner.substrate import ( ScreenObservation, SubstrateSession, ) -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( AgentReport, ModelCall, TaskSpec, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/baselines.py b/packages/flow/benchmark/harness/effectbench/runner/baselines.py similarity index 97% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/baselines.py rename to packages/flow/benchmark/harness/effectbench/runner/baselines.py index 3674a508ce..5a43fbf73d 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/baselines.py +++ b/packages/flow/benchmark/harness/effectbench/runner/baselines.py @@ -21,7 +21,7 @@ side channel to the system of record; the independent oracle, not the agent, judges the effect. 3. **Record every model call.** Append one - :class:`~openadapt_flow.benchmark.effectbench.schema.ModelCall` per API + :class:`~benchmark.harness.effectbench.schema.ModelCall` per API request with token counts and the pinned list price, so ``cost_usd`` is auditable ``sum(model_calls)``. An RPA tool with a flat run fee records a single ``ModelCall`` carrying that fee. @@ -38,9 +38,9 @@ from dataclasses import dataclass from typing import Mapping -from openadapt_flow.benchmark.effectbench.runner.arms import ArmResult -from openadapt_flow.benchmark.effectbench.runner.substrate import SubstrateSession -from openadapt_flow.benchmark.effectbench.schema import TaskSpec +from benchmark.harness.effectbench.runner.arms import ArmResult +from benchmark.harness.effectbench.runner.substrate import SubstrateSession +from benchmark.harness.effectbench.schema import TaskSpec class ScaffoldNotWired(NotImplementedError): diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/compound.py b/packages/flow/benchmark/harness/effectbench/runner/compound.py similarity index 94% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/compound.py rename to packages/flow/benchmark/harness/effectbench/runner/compound.py index 1c6236449a..3c15f795f5 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/compound.py +++ b/packages/flow/benchmark/harness/effectbench/runner/compound.py @@ -9,16 +9,16 @@ * ``field_equals`` — that encounter's note reads back the run's exact value; catches a *partial save* that persists the row but drops a field. -:class:`~openadapt_flow.benchmark.effectbench.oracle.score_episode` verifies a +:class:`~benchmark.harness.effectbench.oracle.score_episode` verifies a SINGLE :class:`Effect` through the oracle. This module supplies a thin :class:`~openadapt_flow.runtime.effects.effect.EffectVerifier` that wraps ANY concrete base verifier (the in-process -:class:`~openadapt_flow.benchmark.effectbench.oracle.RecordSnapshotOracle`, a +:class:`~benchmark.harness.effectbench.oracle.RecordSnapshotOracle`, a :class:`~openadapt_flow.runtime.effects.rest.RestRecordVerifier`, …) and, when ``score_episode`` calls :meth:`verify` with the primary ``record_written`` effect, ALSO checks the authored extra sub-effects against the SAME pre-state snapshot, reducing the per-effect true-effect states with -:func:`~openadapt_flow.benchmark.effectbench.oracle.combine_true_states`. +:func:`~benchmark.harness.effectbench.oracle.combine_true_states`. It returns the single **deciding** :class:`EffectVerdict` whose ``effect_state`` equals the combined state (the fault-model precedence: the @@ -33,7 +33,7 @@ from typing import Any, Sequence -from openadapt_flow.benchmark.effectbench.oracle import ( +from benchmark.harness.effectbench.oracle import ( TrueEffectState, combine_true_states, effect_state, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/harness.py b/packages/flow/benchmark/harness/effectbench/runner/harness.py similarity index 92% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/harness.py rename to packages/flow/benchmark/harness/effectbench/runner/harness.py index f21991fed0..f1bafeefbf 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/harness.py +++ b/packages/flow/benchmark/harness/effectbench/runner/harness.py @@ -1,11 +1,11 @@ """The runner harness — wire each arm through ``score_episode`` identically. This is the multi-baseline runner's core: given a set of -:class:`~openadapt_flow.benchmark.effectbench.schema.TaskSpec` and a set of arms, +:class:`~benchmark.harness.effectbench.schema.TaskSpec` and a set of arms, it produces one -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord` per +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord` per (task × arm × trial), each scored by the task's INDEPENDENT oracle through -:func:`~openadapt_flow.benchmark.effectbench.oracle.score_episode`. Every arm +:func:`~benchmark.harness.effectbench.oracle.score_episode`. Every arm runs the SAME task and is judged by the SAME oracle — the harness guarantees the comparison is fair by construction: @@ -30,10 +30,10 @@ from dataclasses import dataclass, field from typing import Any, Callable, List, Mapping, Optional, Sequence -from openadapt_flow.benchmark.effectbench.oracle import score_episode -from openadapt_flow.benchmark.effectbench.runner.arms import AgentArm -from openadapt_flow.benchmark.effectbench.runner.substrate import SubstrateSession -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.oracle import score_episode +from benchmark.harness.effectbench.runner.arms import AgentArm +from benchmark.harness.effectbench.runner.substrate import SubstrateSession +from benchmark.harness.effectbench.schema import ( AgentReport, EpisodeRecord, ModelCall, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/reference_tasks.py b/packages/flow/benchmark/harness/effectbench/runner/reference_tasks.py similarity index 96% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/reference_tasks.py rename to packages/flow/benchmark/harness/effectbench/runner/reference_tasks.py index 6afe49e2be..9c4c8de44e 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/reference_tasks.py +++ b/packages/flow/benchmark/harness/effectbench/runner/reference_tasks.py @@ -3,7 +3,7 @@ These are the reference tasks the multi-baseline runner exercises end-to-end without Docker or spend: the bundled MockMed transactional-fault surface (``openadapt_flow.mockmed.fault_server``) turned into -:class:`~openadapt_flow.benchmark.effectbench.schema.TaskSpec` s across the +:class:`~benchmark.harness.effectbench.schema.TaskSpec` s across the divergence categories, plus a :func:`mockmed_env_factory` that provisions one isolated fault server per episode with an INDEPENDENT in-process oracle. @@ -24,17 +24,17 @@ from dataclasses import dataclass from typing import Mapping, Optional -from openadapt_flow.benchmark.effectbench.oracle import RecordSnapshotOracle -from openadapt_flow.benchmark.effectbench.runner.compound import ( +from benchmark.harness.effectbench.oracle import RecordSnapshotOracle +from benchmark.harness.effectbench.runner.compound import ( CompoundEffectVerifier, ) -from openadapt_flow.benchmark.effectbench.runner.harness import EpisodeEnv -from openadapt_flow.benchmark.effectbench.runner.substrate import ( +from benchmark.harness.effectbench.runner.harness import EpisodeEnv +from benchmark.harness.effectbench.runner.substrate import ( MockMedFault, MockMedSession, mockmed_session, ) -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( DivergenceCategory, OracleChannel, OracleSpec, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/substrate.py b/packages/flow/benchmark/harness/effectbench/runner/substrate.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/substrate.py rename to packages/flow/benchmark/harness/effectbench/runner/substrate.py index 3d4e525528..1322d730f6 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/substrate.py +++ b/packages/flow/benchmark/harness/effectbench/runner/substrate.py @@ -34,10 +34,10 @@ from dataclasses import dataclass, field from typing import Any, Mapping, Optional, Protocol, Sequence, runtime_checkable -from openadapt_flow.benchmark.effectbench.runner.compound import ( +from benchmark.harness.effectbench.runner.compound import ( CompoundEffectVerifier, ) -from openadapt_flow.benchmark.effectbench.schema import Substrate +from benchmark.harness.effectbench.schema import Substrate from openadapt_flow.runtime.effects.effect import Effect, EffectVerifier diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/schema.py b/packages/flow/benchmark/harness/effectbench/schema.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/effectbench/schema.py rename to packages/flow/benchmark/harness/effectbench/schema.py index 4e529ae716..d92fb0af90 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/schema.py +++ b/packages/flow/benchmark/harness/effectbench/schema.py @@ -135,7 +135,7 @@ class OracleChannel(str, Enum): Maps 1:1 onto the ``substrate`` attribute of the concrete :class:`~openadapt_flow.runtime.effects.effect.EffectVerifier` - implementations re-exported by :mod:`openadapt_flow.benchmark.effectbench`. + implementations re-exported by :mod:`benchmark.harness.effectbench`. """ SQL = "sql" # SqlRecordVerifier — read-only SELECT under a read-only role @@ -311,7 +311,7 @@ class EpisodeRecord(BaseModel): """One trial of one task by one arm — the atomic benchmark result row. Every headline metric is a pure aggregation over a list of these (see - :func:`openadapt_flow.benchmark.effectbench.metrics.summarize`). The runner + :func:`benchmark.harness.effectbench.metrics.summarize`). The runner produces one per (task x arm x trial); ``summarize`` never needs anything the row does not carry, so raw rows can be published for external replication. """ diff --git a/packages/flow/openadapt_flow/benchmark/frappe_lending.py b/packages/flow/benchmark/harness/frappe_lending.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/frappe_lending.py rename to packages/flow/benchmark/harness/frappe_lending.py diff --git a/packages/flow/openadapt_flow/benchmark/hybrid_benchmark.py b/packages/flow/benchmark/harness/hybrid_benchmark.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/hybrid_benchmark.py rename to packages/flow/benchmark/harness/hybrid_benchmark.py index c343d13b45..282bb79cbb 100644 --- a/packages/flow/openadapt_flow/benchmark/hybrid_benchmark.py +++ b/packages/flow/benchmark/harness/hybrid_benchmark.py @@ -67,12 +67,12 @@ from pydantic import BaseModel -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.openemr_benchmark import ( +from benchmark.harness import agent_baseline +from benchmark.harness.openemr_benchmark import ( BILLING_ABORT_AFTER, _looks_like_billing_error, ) -from openadapt_flow.benchmark.run_benchmark import ( +from benchmark.harness.run_benchmark import ( _agent_run, _arm_aggregate, _compiled_run, @@ -863,7 +863,7 @@ def render_hybrid_chart(results: dict[str, Any], out_png: Path) -> Path: Returns: The written PNG path. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() @@ -1379,7 +1379,7 @@ def failures(arm: str) -> str: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.hybrid_benchmark --out benchmark/hybrid +.venv/bin/python -m benchmark.harness.hybrid_benchmark --out benchmark/hybrid ``` Requires `ANTHROPIC_API_KEY` (or `~/.anthropic/api_key`). The paid arms @@ -1398,7 +1398,7 @@ def write_hybrid_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_hybrid_chart, results, out_dir / "success_cost.png") (out_dir / "BENCHMARK.md").write_text(render_hybrid_markdown(results)) @@ -1649,7 +1649,7 @@ def _error_row(arm: str, exc: Exception) -> dict[str, Any]: def main(argv: Optional[list[str]] = None) -> int: - """CLI entry point: ``python -m openadapt_flow.benchmark.hybrid_benchmark``.""" + """CLI entry point: ``python -m benchmark.harness.hybrid_benchmark``.""" parser = argparse.ArgumentParser( description=( "Four-arm hybrid benchmark on MockMed (compiled / agent / " diff --git a/packages/flow/openadapt_flow/instrument/__init__.py b/packages/flow/benchmark/harness/instrument/__init__.py similarity index 91% rename from packages/flow/openadapt_flow/instrument/__init__.py rename to packages/flow/benchmark/harness/instrument/__init__.py index bf92686661..dbf7469cc1 100644 --- a/packages/flow/openadapt_flow/instrument/__init__.py +++ b/packages/flow/benchmark/harness/instrument/__init__.py @@ -1,7 +1,7 @@ """Competitor-drift instrument: point our EffectVerifier at ANY external agent. The self-directed silent-wrong-action benchmark -(``openadapt_flow.benchmark.silent_wrong_action``, #67) measures how often our +(``benchmark.harness.silent_wrong_action``, #67) measures how often our OWN runtime lets a wrong / absent / duplicate business effect land while reporting success. This package generalizes that measurement to an *arbitrary external computer-use agent*: given any agent behind the @@ -19,7 +19,7 @@ user-gated step. """ -from openadapt_flow.instrument.competitor_drift import ( # noqa: F401 +from benchmark.harness.instrument.competitor_drift import ( # noqa: F401 AgentRunResult, ArchitectureClassError, CostGuard, diff --git a/packages/flow/openadapt_flow/instrument/competitor_drift.py b/packages/flow/benchmark/harness/instrument/competitor_drift.py similarity index 99% rename from packages/flow/openadapt_flow/instrument/competitor_drift.py rename to packages/flow/benchmark/harness/instrument/competitor_drift.py index 3daa5076c6..2abf9fe980 100644 --- a/packages/flow/openadapt_flow/instrument/competitor_drift.py +++ b/packages/flow/benchmark/harness/instrument/competitor_drift.py @@ -2,7 +2,7 @@ runner, cost-capped. This EXTENDS the self-directed silent-wrong-action benchmark -(``openadapt_flow.benchmark.silent_wrong_action``, #67) from "our own runtime" +(``benchmark.harness.silent_wrong_action``, #67) from "our own runtime" to "any external computer-use agent." The measurement is identical — did a wrong / absent / duplicate business effect land in the system of record while the actor reported success (a *silent wrong-action*)? — but the actor is now an @@ -53,7 +53,7 @@ # judge, and the #63 effect contract. We REUSE these from the self-directed # benchmark rather than re-deriving them so the external-agent instrument and # the self-directed instrument grade with byte-identical semantics. -from openadapt_flow.benchmark.silent_wrong_action import ( +from benchmark.harness.silent_wrong_action import ( NOTE, SCENARIOS, Scenario, @@ -888,7 +888,7 @@ def write_outputs(report: InstrumentReport, out_dir: Path | str) -> None: def main(argv: Optional[list[str]] = None) -> int: - """CLI: ``python -m openadapt_flow.instrument.competitor_drift``. + """CLI: ``python -m benchmark.harness.instrument.competitor_drift``. Ships only the offline stub adapter and a dry-run estimator — a real external adapter is wired programmatically (see diff --git a/packages/flow/openadapt_flow/benchmark/openemr_benchmark.py b/packages/flow/benchmark/harness/openemr_benchmark.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/openemr_benchmark.py rename to packages/flow/benchmark/harness/openemr_benchmark.py index f2482f2307..5bb2396f11 100644 --- a/packages/flow/openadapt_flow/benchmark/openemr_benchmark.py +++ b/packages/flow/benchmark/harness/openemr_benchmark.py @@ -7,7 +7,7 @@ recorded by ``scripts/openemr_demo.py`` (fake demo patients only). Both arms are judged by the same criterion, implemented once -(:func:`openadapt_flow.benchmark.verify.verify_note_saved`): OCR of the +(:func:`benchmark.harness.verify.verify_note_saved`): OCR of the final screen must show the run's parameterized note text in the patient message list. Each run — in BOTH arms — uses a distinct note value, so a pass proves parameter substitution against live state, not replay of a @@ -60,16 +60,16 @@ from pathlib import Path from typing import Any, Callable -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.agent_baseline import _looks_like_billing_error -from openadapt_flow.benchmark.run_benchmark import ( +from benchmark.harness import agent_baseline +from benchmark.harness.agent_baseline import _looks_like_billing_error +from benchmark.harness.run_benchmark import ( _agent_run, _arm_aggregate, _compiled_run, identity_coverage_block, render_chart, ) -from openadapt_flow.benchmark.verify import verify_note_saved +from benchmark.harness.verify import verify_note_saved DEMO_URL = "https://demo.openemr.io/openemr/index.php" N_COMPILED = 20 @@ -481,7 +481,7 @@ def write_openemr_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "latency_cost.png") (out_dir / "BENCHMARK.md").write_text(render_openemr_markdown(results)) diff --git a/packages/flow/openadapt_flow/benchmark/openemr_local.py b/packages/flow/benchmark/harness/openemr_local.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/openemr_local.py rename to packages/flow/benchmark/harness/openemr_local.py index 125f6d9c24..1a4e337de8 100644 --- a/packages/flow/openadapt_flow/benchmark/openemr_local.py +++ b/packages/flow/benchmark/harness/openemr_local.py @@ -18,7 +18,7 @@ from typing import Any, Iterable, Mapping, Sequence from urllib.parse import urlencode -from openadapt_flow.benchmark.frappe_lending import ( +from benchmark.harness.frappe_lending import ( ARMS, CONDITIONS, INITIAL_TRIALS_PER_CELL, diff --git a/packages/flow/openadapt_flow/benchmark/reliability.py b/packages/flow/benchmark/harness/reliability.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/reliability.py rename to packages/flow/benchmark/harness/reliability.py diff --git a/packages/flow/openadapt_flow/benchmark/run_benchmark.py b/packages/flow/benchmark/harness/run_benchmark.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/run_benchmark.py rename to packages/flow/benchmark/harness/run_benchmark.py index cd2c4c2d1e..3aba486faf 100644 --- a/packages/flow/openadapt_flow/benchmark/run_benchmark.py +++ b/packages/flow/benchmark/harness/run_benchmark.py @@ -29,8 +29,8 @@ from typing import Any, Callable from openadapt_flow.bench import _percentile -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.verify import verify_encounter_saved +from benchmark.harness import agent_baseline +from benchmark.harness.verify import verify_encounter_saved N_COMPILED = 100 N_AGENT = 20 @@ -153,7 +153,7 @@ def _agent_run( A per-run row dict. API failures are recorded as failed rows with ``error`` set, never raised; a failed row still carries whatever usage/cost the run paid for before crashing (via - :class:`~openadapt_flow.benchmark.agent_baseline.UsageLedger`), so + :class:`~benchmark.harness.agent_baseline.UsageLedger`), so crashed runs' real spend counts in aggregates and cost ceilings. """ from openadapt_flow.backends.playwright_backend import PlaywrightBackend @@ -364,7 +364,7 @@ def render_chart(results: dict[str, Any], out_png: Path) -> Path: Returns: The written PNG path. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() @@ -578,7 +578,7 @@ def write_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "latency_cost.png") (out_dir / "BENCHMARK.md").write_text(render_markdown(results)) diff --git a/packages/flow/openadapt_flow/benchmark/silent_wrong_action.py b/packages/flow/benchmark/harness/silent_wrong_action.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/silent_wrong_action.py rename to packages/flow/benchmark/harness/silent_wrong_action.py index 009bdda54c..daeb0c7c9f 100644 --- a/packages/flow/openadapt_flow/benchmark/silent_wrong_action.py +++ b/packages/flow/benchmark/harness/silent_wrong_action.py @@ -566,7 +566,7 @@ def render_markdown(results: dict[str, Any]) -> str: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.silent_wrong_action \\ +.venv/bin/python -m benchmark.harness.silent_wrong_action \\ --out {DEFAULT_OUT_DIR} --n {results["n_per_scenario"]} ``` @@ -585,7 +585,7 @@ def render_chart(results: dict[str, Any], out_png: Path) -> Path: false-abort rate. The metric a record system cares about — a wrong write that reports success — collapses from a large bar to zero. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() @@ -658,7 +658,7 @@ def write_outputs(results: dict[str, Any], out_dir: Path) -> None: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") (out_dir / "SILENT_WRONG_ACTION.md").write_text(render_markdown(results)) - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "silent_wrong_action.png") @@ -689,7 +689,7 @@ def run_and_write( def main(argv: Optional[list[str]] = None) -> int: - """CLI: ``python -m openadapt_flow.benchmark.silent_wrong_action``.""" + """CLI: ``python -m benchmark.harness.silent_wrong_action``.""" parser = argparse.ArgumentParser( description=( "Silent-wrong-action-rate benchmark: measure how often " diff --git a/packages/flow/openadapt_flow/benchmark/verify.py b/packages/flow/benchmark/harness/verify.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/verify.py rename to packages/flow/benchmark/harness/verify.py diff --git a/packages/flow/benchmark/hybrid/BENCHMARK.md b/packages/flow/benchmark/hybrid/BENCHMARK.md index bc64083f0d..d85401a2a7 100644 --- a/packages/flow/benchmark/hybrid/BENCHMARK.md +++ b/packages/flow/benchmark/hybrid/BENCHMARK.md @@ -207,7 +207,7 @@ $0.0967), and `a` = the agent-only mean cost per run ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.hybrid_benchmark --out benchmark/hybrid +.venv/bin/python -m benchmark.harness.hybrid_benchmark --out benchmark/hybrid ``` Requires `ANTHROPIC_API_KEY` (or `~/.anthropic/api_key`). The paid arms diff --git a/packages/flow/benchmark/lending_fault_model/README.md b/packages/flow/benchmark/lending_fault_model/README.md index 740b9134ff..b6de3a698a 100644 --- a/packages/flow/benchmark/lending_fault_model/README.md +++ b/packages/flow/benchmark/lending_fault_model/README.md @@ -39,7 +39,7 @@ judged by an independent ledger read, with zero model calls. 2. **Silent Wrong-Effect Rate, three-arm verification ladder** - `swer.py` scores the same faults through the shared EffectBench contract - (`openadapt_flow.benchmark.effectbench.score_episode` + `summarize`) under + (`benchmark.harness.effectbench.score_episode` + `summarize`) under screen-only, single-surface, and complete-read-path arms. -> `swer_results.json`, `SWER.md`. Headline: screen-only SWER is 24/36, a disbursements-only verifier leaves the diff --git a/packages/flow/benchmark/lending_fault_model/swer.py b/packages/flow/benchmark/lending_fault_model/swer.py index 9471d68615..312cc57e4a 100644 --- a/packages/flow/benchmark/lending_fault_model/swer.py +++ b/packages/flow/benchmark/lending_fault_model/swer.py @@ -5,7 +5,7 @@ screen-only postcondition contract (and measures how often that silently mishandles a transactional fault), THIS module measures the same faults through the shared EffectBench scoring contract -(:func:`openadapt_flow.benchmark.effectbench.score_episode`) under a THREE-arm +(:func:`benchmark.harness.effectbench.score_episode`) under a THREE-arm ladder that mirrors the clinical ``effect_e2e`` study (screen / single-surface / complete-read-path), so the two domains are directly comparable: @@ -71,7 +71,7 @@ import requests -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, DivergenceCategory, Effect, diff --git a/packages/flow/benchmark/openemr_api_tier/approval_durable.py b/packages/flow/benchmark/openemr_api_tier/approval_durable.py index 7b3e57f929..47dbe33709 100644 --- a/packages/flow/benchmark/openemr_api_tier/approval_durable.py +++ b/packages/flow/benchmark/openemr_api_tier/approval_durable.py @@ -157,7 +157,7 @@ def run_one( ) -> dict[str, Any]: from openadapt_flow.approval_mode.gate import ApprovalGate from openadapt_flow.approval_mode.item_approval import ItemApprovalStore - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer from openadapt_flow.runtime.durable.checkpoint import CheckpointStore diff --git a/packages/flow/benchmark/openemr_api_tier/common.py b/packages/flow/benchmark/openemr_api_tier/common.py index 19bb064547..8764d08f22 100644 --- a/packages/flow/benchmark/openemr_api_tier/common.py +++ b/packages/flow/benchmark/openemr_api_tier/common.py @@ -74,7 +74,7 @@ def load_demo() -> Any: def spec_for(ref: Referral) -> Any: - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec return SyntheticPatientSpec(**ref.kwargs) diff --git a/packages/flow/benchmark/openemr_api_tier/feature_e2e.py b/packages/flow/benchmark/openemr_api_tier/feature_e2e.py index 35754d3a4d..eccb0e8d1f 100644 --- a/packages/flow/benchmark/openemr_api_tier/feature_e2e.py +++ b/packages/flow/benchmark/openemr_api_tier/feature_e2e.py @@ -51,7 +51,7 @@ def cmd_record(a: argparse.Namespace) -> None: - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec from openadapt_flow.network_capture import NetworkCapture from openadapt_flow.recorder import Recorder @@ -190,7 +190,7 @@ def wait_quiet(max_s: float = float(os.environ.get("GATE_MAX_S", "1200"))) -> di def run_one(env: dict[str, Any], arm: str, ref_idx: int, bundle: Path, out: Path) -> dict[str, Any]: - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer diff --git a/packages/flow/benchmark/openemr_api_tier/spike/bench.py b/packages/flow/benchmark/openemr_api_tier/spike/bench.py index d3a74c2f53..138570dcca 100644 --- a/packages/flow/benchmark/openemr_api_tier/spike/bench.py +++ b/packages/flow/benchmark/openemr_api_tier/spike/bench.py @@ -62,7 +62,7 @@ def _refuse(self, *_a: Any, **_k: Any) -> Any: class Env: def __init__(self, browser: bool) -> None: - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer diff --git a/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py b/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py index 12929d4e07..5e6297d2ca 100644 --- a/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py +++ b/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py @@ -39,7 +39,7 @@ def main() -> None: from playwright.sync_api import sync_playwright - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer diff --git a/packages/flow/benchmark/openemr_e2e/README.md b/packages/flow/benchmark/openemr_e2e/README.md index 39fdb23bbe..68471ac1f1 100644 --- a/packages/flow/benchmark/openemr_e2e/README.md +++ b/packages/flow/benchmark/openemr_e2e/README.md @@ -10,7 +10,7 @@ compile → replay → effect-verify (system of record) → catch a silent wrong ``` It does **not** duplicate the existing OpenEMR *benchmark* -(`openadapt_flow.benchmark.openemr_benchmark`, which measures the compiled arm +(`benchmark.harness.openemr_benchmark`, which measures the compiled arm vs. a paid agent arm on the **live** public demo and is explicitly *not* CI-reproducible). This harness *orchestrates* the runtime components that benchmark assumes work — the compiler, the `Replayer`, the `EffectVerifier`, the diff --git a/packages/flow/benchmark/openemr_e2e/harness.py b/packages/flow/benchmark/openemr_e2e/harness.py index 23a10fc498..5110b31c77 100644 --- a/packages/flow/benchmark/openemr_e2e/harness.py +++ b/packages/flow/benchmark/openemr_e2e/harness.py @@ -8,7 +8,7 @@ inject drift -> HALT -> teach the fix -> re-run clean Unlike the existing OpenEMR *benchmark* -(:mod:`openadapt_flow.benchmark.openemr_benchmark`), which measures the compiled +(:mod:`benchmark.harness.openemr_benchmark`), which measures the compiled arm vs. a paid agent arm on the LIVE public demo and is explicitly NOT CI-reproducible, this harness proves the pipeline *wiring* end to end on a deterministic, offline fixture (the in-process MockMed ``fault_server`` as the diff --git a/packages/flow/benchmark/openemr_local/README.md b/packages/flow/benchmark/openemr_local/README.md index 6a9441c40a..19efb71a28 100644 --- a/packages/flow/benchmark/openemr_local/README.md +++ b/packages/flow/benchmark/openemr_local/README.md @@ -33,7 +33,7 @@ publication bar. Raw per-run rows, environment fingerprints, application-specific adapter/oracle wiring, and the detailed cost ledger are retained in the private `OpenAdaptAI/openadapt-corpus` repository. The public generic cost-capped agent mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. See the +`benchmark/harness/agent_baseline.py`. See the [aggregate report](../agent_arm_verticals/README.md). The accepted run used baseline SHA-256 diff --git a/packages/flow/benchmark/openemr_throughput/harness.py b/packages/flow/benchmark/openemr_throughput/harness.py index 4bcd15079b..fcf9a2e817 100644 --- a/packages/flow/benchmark/openemr_throughput/harness.py +++ b/packages/flow/benchmark/openemr_throughput/harness.py @@ -237,7 +237,7 @@ def _demo_module() -> Any: def build_worker(user: str, opts: dict[str, Any]) -> Any: """One worker signed in as ``user``. Module-level so a child process can build it (``ProcessWorker``); ``opts`` holds only plain values.""" - from openadapt_flow.benchmark.openemr_local import ( + from benchmark.harness.openemr_local import ( OpenEMRPatientOracle, SyntheticPatientSpec, ) @@ -345,7 +345,7 @@ def run_item(page: Any, idx: int, row: dict[str, Any], item_dir: Path, pipeline: def run(a: argparse.Namespace) -> dict[str, Any]: from openadapt_flow._browser_setup import ensure_chromium_installed - from openadapt_flow.benchmark.openemr_local import ( + from benchmark.harness.openemr_local import ( SyntheticPatientSpec, ) from openadapt_flow.ir import Workflow diff --git a/packages/flow/benchmark/openimis_claims/README.md b/packages/flow/benchmark/openimis_claims/README.md index 94e94ad017..e01f263e40 100644 --- a/packages/flow/benchmark/openimis_claims/README.md +++ b/packages/flow/benchmark/openimis_claims/README.md @@ -28,7 +28,7 @@ Raw per-run rows, environment fingerprints, application-specific adapter/oracle wiring, and the detailed cost ledger are retained in the private `OpenAdaptAI/openadapt-corpus` repository. The public generic cost-capped agent mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. See the +`benchmark/harness/agent_baseline.py`. See the [aggregate report](../agent_arm_verticals/README.md). There is still no full timing matrix and no publication protocol here: a diff --git a/packages/flow/benchmark/scoreboard/tasks/openemr.py b/packages/flow/benchmark/scoreboard/tasks/openemr.py index c2ffaf02d6..e93b7eab6d 100644 --- a/packages/flow/benchmark/scoreboard/tasks/openemr.py +++ b/packages/flow/benchmark/scoreboard/tasks/openemr.py @@ -207,7 +207,7 @@ def check(values: dict[str, str], before_ids: set[str]) -> str: def oracle_verifier(values: dict[str, str]) -> Any: - from openadapt_flow.benchmark.openemr_local import ( + from benchmark.harness.openemr_local import ( OpenEMRPatientOracle, SyntheticPatientSpec, ) diff --git a/packages/flow/benchmark/silent_saves/metrics.py b/packages/flow/benchmark/silent_saves/metrics.py index cb7f3ee9f1..2a8fa76483 100644 --- a/packages/flow/benchmark/silent_saves/metrics.py +++ b/packages/flow/benchmark/silent_saves/metrics.py @@ -6,7 +6,7 @@ import statistics from typing import Any, Iterable, Optional -from openadapt_flow.benchmark.effectbench.metrics import wilson_interval +from benchmark.harness.effectbench.metrics import wilson_interval def rate(k: int, n: int) -> dict[str, Any]: diff --git a/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md b/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md index c09304b4b5..ce1dfe56c9 100644 --- a/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md +++ b/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md @@ -66,7 +66,7 @@ run-to-run disagreement); N proves reproducibility. ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.silent_wrong_action \ +.venv/bin/python -m benchmark.harness.silent_wrong_action \ --out benchmark/silent_wrong_action --n 10 ``` diff --git a/packages/flow/benchmark/vision_hardening/README.md b/packages/flow/benchmark/vision_hardening/README.md index 273c387138..46644fdc38 100644 --- a/packages/flow/benchmark/vision_hardening/README.md +++ b/packages/flow/benchmark/vision_hardening/README.md @@ -36,7 +36,7 @@ and CLASSIFIES what the **real** resolver does on each generated case: go DOWN. A hardening fix regenerates the corpus and lowers the ratchet. The headline metric is **SWER** (Silent Wrong-Effect/Resolution Rate), the same -taxonomy as EffectBench (`openadapt_flow/benchmark/effectbench/`) — this sweep +taxonomy as EffectBench (`benchmark/harness/effectbench/`) — this sweep is the *resolution-tier* feeder for that benchmark. ## The two tiers diff --git a/packages/flow/docs/EFFECT_KIT.md b/packages/flow/docs/EFFECT_KIT.md index 3739752c74..cb02250ed9 100644 --- a/packages/flow/docs/EFFECT_KIT.md +++ b/packages/flow/docs/EFFECT_KIT.md @@ -1420,7 +1420,7 @@ Runs against the pinned reference fixture README documents the pinned compose bring-up and fixture bootstrap). The bundle's effect contracts are exactly the ones the reference matrix ships -(`openadapt_flow/benchmark/frappe_lending.py::loan_application_effects`): +(`benchmark/harness/frappe_lending.py::loan_application_effects`): one `record_written` (at-most-once for the synthetic applicant) plus a `field_equals` read-back per entered field, each bound to `{param: ...}`. diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md new file mode 100644 index 0000000000..7fb257ecdb --- /dev/null +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -0,0 +1,311 @@ +# Generality audit of the engine + +Audited 2026-11-03 on `main` at `213f3d9fe`. Scope: everything under +`packages/flow/openadapt_flow/`. + +The rule: the engine has no code written for one application. What one +application needs lives in data (`app_profiles/*.yaml`, generated contracts, +deployment configs) or in test fixtures and benchmark harnesses outside the +shipped package. `scripts/check_engine_generality.py` enforces it in CI; see +[CONTRIBUTING.md](../CONTRIBUTING.md#app-knowledge-lives-in-data). + +## Method + +1. Searched for application names, case-insensitive: OpenEMR, Medplum, + Dolibarr, EspoCRM, Redmine, Snipe-IT, InvenTree, Kimai, OpenMRS, Gitea, + Firefly III, Vikunja, Zammad, Open Dental, Odoo, openIMIS, Frappe/ERPNext, + Patient Notes, Epic, Stedi, TextEdit, Notepad, and OpenAdapt's own demo apps + MockMed and MockLoan. +2. Searched for identifiers that only make sense for one application: table and + column names (`pnotes`, `patient_data`, `uuid_registry`, `user_settings`, + `tabLoan Application`), URL paths (`/api/db`, `/api/encounter`, + `/apis/default/fhir`), form and query parameters (`token_main`, + `csrf_token_form`, `clearpass`), selectors, button labels, dialog titles, + field names, and thresholds tuned on one application's screens. +3. Read the code around each hit and put it in one of four buckets: + - **(a)** a comment or docstring example. Fine; reworded where it read as a + rule. + - **(b)** application knowledge stored as data and loaded generically. + - **(c)** a test fixture, benchmark harness or demo application inside the + shipped package. Moved out, unless it backs a user-facing command that + must work from a wheel. + - **(d)** special-casing in engine logic. Generalized, or turned into data. + +## Counts + +| Bucket | Rows | Distinct findings | +| --- | --- | --- | +| (a) comment or docstring example | 118 | — | +| (b) data loaded generically | 4 | 2 (`app_profiles/openemr.yaml`, the `clinical-write` policy pack as a default) | +| (c) fixture, harness or demo in the package | 35 rows, plus 27 files of `benchmark/` | 19 | +| (d) special-casing in engine logic | 27 | 15 | + +Rows group consecutive lines of one hit. The full table is at the end. + +## (d) findings and actions + +Each finding names the application that motivated it. "Second app" is the +existing fixture that shows the generalized code still works beyond the +motivating application. + +| # | Where | What | Motivating app | Action | Status | +| --- | --- | --- | --- | --- | --- | +| D1 | `report_redaction.py:68-70`, `:131-133` | Exact secret names `token_main`, `csrf_token_form` | OpenEMR | Match secret names by pattern (`token`, `csrf`, `xsrf`, `session`, ...) instead of a per-app list | planned | +| D2 | `network_capture.py:78` | `clearpass` in the secret-name pattern | OpenEMR | Drop it; the `pass\b` alternative already matches | planned | +| D3 | `runtime/effects/rest.py:114-115`, `deployment.py:387-390`, `reward/models.py:226`, `fault_gate.py:722` | REST reader defaults `records_path="/api/db"`, `records_key="records"` | MockMed | Required when the effect kind is `rest`; refused at config load when missing | planned | +| D4 | `runtime/effects/compensation.py:364` | `delete_path_template="/api/encounter/{id}"` default | MockMed | Required | planned | +| D5 | `compiler/binding_admission.py:114-119`, `:71-73` | Admission write path defaults to `/api/encounter`; body skips MockMed's `source` field | MockMed | Write path and method from the observed `api_write` event; skip only server-assigned fields | planned | +| D6 | `compiler/effect_mining.py:100-102`, `runtime/effects/effect.py:900-901` | Idempotency key field defaults to `key` | MockMed | No default; from the contract or deployment config | planned | +| D7 | `runtime/effects/fhir.py:59-67`, `:221`, `deployment.py:448-449` | FHIR defaults: resource type `Observation`, `note -> valueString` | OpenEMR | Required in the contract (`generate/ensure_contract.py` writes them) | planned | +| D8 | `generate/ensure_contract.py:123-147` | Translates the Medplum probe script's output keys | Medplum | The probe emits the generic fact names; translation removed | planned | +| D9 | `runtime/effects/frame.py:1937`, `rollback_state.py:781` | `"0"` means "no record" | OpenEMR | The value moves into the profile rule (`no_record_values`) | planned | +| D10 | `__main__.py:2838-2934`, `:9305-9319`; `cli_variants.py:131,173-182` | `replay` and the variant probe fall back to the bundled MockMed when no URL is given; `--drift` turns off structural grounding | MockMed | Target URL required outside the demo commands | planned | +| D11 | `qualification_oracle_gate.py` | `qualify accept` tests every workflow's oracle against MockMed (banner text, `/api/encounter`, `/api/db`) | MockMed | Build the fault fixture from the bundle's own effect binding | planned | +| D12 | `execute/dispatch.py:111-188` | The Execute service's live runner always runs the MockMed tutorial workflow | MockMed | Replay through the admission's deployment config; MockMed only for synthetic admissions | planned | +| D13 | `compiler/compile.py:151`, `runtime/replayer.py:394`, `runtime/identity.py:108` | Thresholds measured on MockMed only | MockMed | Generic logic, one-app numbers. Recalibrate on two or more apps | follow-up issue | +| D14 | `runtime/identity.py:275-302` | Hand-written list of non-identifying words drawn from EMR list screens | MockMed, EMRs | Derive from the recording (tokens shared by sibling rows) | follow-up issue | +| D15 | `eligibility/client.py:37-41`, `:1091-1110` | One vendor's eligibility API (URL, key name, body) in the engine | Stedi | Move behind a connector interface | follow-up issue | + +## (c) findings and actions + +| Where | What | Action | Status | +| --- | --- | --- | --- | +| `benchmark/` (27 Python files: `run_benchmark`, `openemr_benchmark`, `openemr_local`, `frappe_lending`, `hybrid_benchmark`, `dom_arm`, `silent_wrong_action`, `desktop_benchmark`, `verify`, `agent_baseline`, `reliability`, `chart_fonts`, `effectbench/`) | Benchmark harnesses shipped in the wheel | Moved to `packages/flow/benchmark/harness/` (import `benchmark.harness.*`). `openadapt-flow benchmark` loads it from a source checkout and says so from a wheel | done | +| `instrument/` | Competitor-drift harness; imported the benchmark at module load, so it was already broken in a wheel | Moved to `packages/flow/benchmark/harness/instrument/` | done | +| `integrations/harness2/` | Harness2 research harness on MockMed | Move to `packages/flow/benchmark/harness/harness2/` | planned | +| `mockloan/`, `demo_driver.record_disbursement_demo` | Lending demo app no product command uses | Move to the benchmark tree | planned | +| `guard_demo.py`, `agent/ledger_demo.py` | `python -m` demos, no CLI command | Move to the benchmark tree | planned | +| `mockmed/teach_scope.py` | Used only by tests | Move to `tests/fixtures` | planned | +| `learning/synth_stream.py:1-245` | MockMed note stream | Split out the generic `StructuralDiffInducer`, then move the stream | planned | +| `reward/seed.py:1-236` | MockMed reward seed | Split out the generic bundle helpers (`:238-410`), then move the seed behind the demo flag | planned | +| `record_check_proof.py:60-124` | `MOCKMED` fixture and `if name == "mockmed"` | Fixtures registered by name from the demo app's own module | planned | +| `validation/dense_surface.py` | Study harness rendering a MockMed table | Move to `benchmark/dense_surface` | planned | +| `__main__.py:5246-5275` (`bench`) | Always replays against MockMed | Accept `--url`, or load from a checkout like `benchmark` | planned | +| `execute/registry.py`, `execute/service.py`, `execute/app.py`, `execute/dispatch.py:60-108`, `--seed-mockmed` | Demo seeding inside the Execute service | Generic `seed_admissions`; the CLI passes the demo seed | planned | +| `eligibility/payer_routes.yaml`, `fleet/simulator.py`, `repair/dialog_fixture.py` | Synthetic test data and simulators in the package | Move to the benchmark tree or `tests/fixtures` | planned | + +### Kept in the package on purpose + +These back user-facing commands that must work from `pip install openadapt` +(`openadapt quickstart`, `openadapt-flow tutorial`, `demo`, `demo-record`, +`openadapt-agent serve` with no flags, `openadapt-agent example-app`). They are +OpenAdapt's own synthetic demo applications, not customer applications, and the +engine does not branch on them: + +- `mockmed/` (server, fault server, chart pages, static UI) +- `tutorial.py`, `demo_proof.py`, `demo_driver.record_triage_demo` +- `agent/sandbox.py`, `agent/tutorial.py`, `agent/example_app.py` + +The generality check allows MockMed names only in these modules. + +## (b) data + +- `app_profiles/openemr.yaml`: OpenEMR's bookkeeping writes, loaded by + `app_profiles.load_profile(name)`, which has no per-app code. +- The `clinical-write` policy pack is the default policy in + `execute/models.py:74`, `cli_config.py:164`, `integrations/agents/skills.py:351` + and `agent/teach.py:121`. It is data, but a healthcare default; whether to make + the policy required is a product decision, noted here. + +## (a) rewording done + +- `deployment.py:548`: the example environment variable in a user-facing + error is now `APP_READER_PASSWORD`. +- `authoring.py:807`: the comment on a generic branch no longer names MockMed. + +The remaining (a) rows are examples in comments and docstrings. The code they +describe is generic and configured per contract or profile. Healthcare words +such as MRN, DOB and patient appear in generic heuristics (for example +`risk.py:370`); that is the product's domain, not one application. + +## Full table + +Paths are relative to `packages/flow/openadapt_flow/` as of the audited commit. + +| Location | Bucket | App | Note and action | +| --- | --- | --- | --- | +| `__main__.py:2838-2934` | (d) | MockMed | replay: general code path defaults to serving bundled MockMed when no --url/config URL, and --drift flips use_structural=False (grounding ladder changes on a demo flag). Make --url required for replay (or route no-URL to `tutorial`/`demo`), rename --drift behavior to a generic --force-visual-grounding, keep drift injection in the demo command. | +| `__main__.py:9305-9319` | (d) | MockMed | replay parser: --url default = bundled MockMed; --drift MockMed modes (see 2838-2934). | +| `cli_variants.py:131,173-182` | (d) | MockMed | first-screen probe serves bundled MockMed when no --url (same default as replay). Refuse instead (first_screen_unavailable). | +| `compiler/binding_admission.py:114` | (d) | MockMed | AdmissionFixture.write_path default '/api/encounter' (line 119); compile.py:3379 never overrides it | +| `compiler/binding_admission.py:71` | (d) | MockMed | _BODY_SKIP_FIELDS adds MockMed's 'source' field (line 73) | +| `compiler/compile.py:146` | (d) | MockMed | REGION_STABLE_TOLERANCE=16 (line 151) calibrated on MockMed only; recalibrate on >=2 apps | +| `compiler/effect_mining.py:100` | (d) | MockMed | IDEMPOTENCY_KEY_FIELD='key' (line 102) is MockMed's field; take from contract/profile | +| `deployment.py:387` | (d) | MockMed | records_path default '/api/db' + records_key 'records' (lines 389-390) are MockMed's shape; make required for kind: rest | +| `deployment.py:448` | (d) | OpenEMR | fhir section default resource_type='Observation' (line 449) mirrors OpenEMR note-as-Observation; make required | +| `eligibility/client.py:37-41` | (d) | Stedi | Stedi 270/271 vendor client in engine (hard-coded URL, STEDI_API_KEY env, to_stedi_body); move to integrations/connector | +| `execute/dispatch.py:111-188` | (d) | MockMed | live_replay (the only real Execute runner) always calls tutorial.run_tutorial_workflow: POSTs /api/reset to target_url, appends MockMed ?fault=ok&idempotency=demo#tasks, verifies via /api/db 'records', forces clinical-write policy and 'note' param; with no target_url serves MockMed. Ignores AdmittedBundle.policy. Generalize: replay via the standard replay path with the admission's deployment.yaml (effects/backend/policy); keep MockMed only behind synthetic/demo admissions. | +| `fault_gate.py:722` | (d) | MockMed | rest_memory fault case default records_path '/api/db' (test harness; low) | +| `generate/ensure_contract.py:123` | (d) | Medplum | facts_from_probe translates Medplum probe key names (133-147); make the probe emit generic FACTS | +| `network_capture.py:78` | (d) | OpenEMR | SECRET_NAME regex lists 'clearpass' (OpenEMR login field); redundant with pass\b alternative -> drop | +| `qualification_oracle_gate.py:5,36,70,225,325-455` | (d) | MockMed | evaluate_oracle_gate (called by qualification_proposal.py:221,372 for every `qualify accept`) runs ANY workflow's proposed oracle against MockMed: hard-coded banner 'Encounter saved — ', POST /api/encounter?fault=optimistic with {patient_id, type:'Triage', note}, /api/db verifier. Generalize: a generic lying-store fixture driven by the bundle's own effect binding (write path/body/records path from the effect), or fixtures registered as data. | +| `report_redaction.py:131-133` | (d) | OpenEMR | _SECRET_KEYS lists token_main/csrf_token_form; same fix as DEFAULT_PARAMS | +| `report_redaction.py:68` | (d) | OpenEMR | token_main/csrf_token_form in DEFAULT_PARAMS (with :131-133 _SECRET_KEYS): replace with generic token/csrf name regex (reuse network_capture.TOKEN_NAME) or per-app secret_params in app profile | +| `reward/models.py:226` | (d) | MockMed | RecordReadRecipe.records_path default '/api/db' (MockMed's endpoint) in a general model; make it required for kind=rest. | +| `rollback_state.py:781` | (d) | OpenEMR | same '0' no-record sentinel as frame.py:1937 | +| `runtime/effects/compensation.py:364` | (d) | MockMed | RestCompensator delete_path_template default '/api/encounter/{id}'; make required | +| `runtime/effects/effect.py:900` | (d) | MockMed | Effect.key_field default 'key' (line 901) is MockMed's field | +| `runtime/effects/fhir.py:59` | (d) | OpenEMR | DEFAULT_OBSERVATION_PATHS (61-67) maps note->valueString per OpenEMR; require field_paths from contract | +| `runtime/effects/frame.py:1937` | (d) | OpenEMR | ref in (None,'','0',0): '0' = OpenEMR's 'no chart' (patient_id 0); gated by profile insert_entry_names_no_record; move sentinel into profile | +| `runtime/effects/rest.py:114-115` | (d) | MockMed | RestRecordVerifier defaults records_path='/api/db', records_key='records'; make required | +| `runtime/identity.py:104` | (d) | MockMed | COVERAGE_THRESHOLD=0.8 (line 108) calibrated on MockMed rows only | +| `runtime/identity.py:275-302` | (d) | MockMed/EMR list | _GENERIC_IDENTITY_WORDS hand-tuned EMR list vocabulary (incl. 'demo','chart','review','results'); derive from recording or app profile | +| `runtime/replayer.py:392` | (d) | MockMed | PC_STRUCTURAL_TEMPLATE_THRESHOLD=0.8 (line 394) calibrated on MockMed only | +| `__main__.py:10030-10037` | (c) | MockMed | bench parser. | +| `__main__.py:10166` | (c) | MockMed | verify-scaffold --prove choices ('mockmed','effect-e2e'): fixture names hard-coded in CLI; see record_check_proof.py. | +| `__main__.py:11940-11989,12341-12415` | (c) | MockMed | serve-execute/serve-reward --seed-mockmed flags and banner paths (data_dir/mockmed/records.db). Opt-in demo seeding; keep as demo flag or rename --seed-demo. | +| `__main__.py:1787-1806` | (c) | MockMed | demo-record command: serves MockMed, runs demo_driver.record_triage_demo. User-facing CLI; demo data could be a recorded script in data. | +| `__main__.py:1808-1998` | (c) | MockMed | tutorial command (via tutorial.py). Deliberate product feature; must work from wheel. | +| `__main__.py:1999-2040` | (c) | MockMed | demo command (demo_proof + tutorial). Deliberate product feature; must work from wheel. | +| `__main__.py:326-330` | (c) | MockMed | _with_drift: builds MockMed ?drift= query; only meaningful for the bundled demo. Move with demo helpers. | +| `__main__.py:5246-5275` | (c) | MockMed | bench command: always replays against bundled MockMed. Not generic; move to benchmark/ (source-checkout command, like `benchmark`) or accept --url. | +| `__main__.py:8773,8782` | (c) | MockMed | demo-record parser help/--drift. | +| `agent/example_app.py:1-400` | (c) | (own demo: reports portal) | `openadapt-agent example-app` local sales-order portal (/login clerk/portal-demo, /api/orders). User-facing demo command; importers agent/cli.py:1250 + 4 flow tests. | +| `agent/ledger_demo.py:1-720` | (c) | MockMed | scripted 30-referral worklist simulation (python -m). Not a CLI command; move to benchmark or packages/agent/tests fixtures; importer packages/agent/tests/test_ledger.py. | +| `agent/sandbox.py:1-530` | (c) | MockMed | `openadapt-agent serve` zero-flag sandbox: add_triage_note on MockMed, CASES with MockMed fault/drift queries, clinical-write. User-facing; must ship. Case table could be data (sandbox YAML). | +| `agent/tutorial.py:1-140` | (c) | MockMed | prepare_tutorial_session: hosts MockMed, writes deployment YAML with /api/db, clinical-write. Used by sandbox and agent/cli.py:760; packages/agent/tests/test_tutorial.py. | +| `demo_driver.py:1-96` | (c) | MockMed | record_triage_demo: MockMed selectors and creds (nurse.demo / mockmed-demo-pass). Used by demo-record CLI + benchmark (fault_model, undo_proposal, ui_readback, harness, cosmetic_drift), tests (recorder, e2e), scripts (demo_media, openemr_demo). Split: keep record_triage_demo with mockmed package. | +| `demo_driver.py:97-171` | (c) | MockLoan | record_disbursement_demo (mockloan-demo-pass): no product command uses it. Move to benchmark/lending_fault_model; importers: benchmark/undo_proposal/run.py, benchmark/lending_fault_model/{run,resolution_ladder}.py, tests/e2e/test_lending_fault_model.py. | +| `demo_proof.py:1-60,975-1110` | (c) | MockMed | `demo` page renderer; CROP 640x320 tuned to MockMed layout, 'Encounter saved', /api/db wording. User-facing; must ship. Crop could come from the demo-app spec. | +| `eligibility/payer_routes.yaml:1-` | (c) | Stedi/Cigna | synthetic TEST payer route data shipped in package | +| `execute/app.py:29,39,141,151` | (c) | MockMed | seed_mockmed passthrough; same as service. | +| `execute/dispatch.py:60-108` | (c) | MockMed | default_runner/synthetic_mockmed: synthetic admissions project canned MockMed tutorial outcomes (evidence_tag mockmed-*). Rename to synthetic_runner and move the canned projection to tests/fixtures or keep behind --seed-mockmed only. | +| `execute/registry.py:10-16,81-106` | (c) | MockMed | MOCKMED_* constants + seed_mockmed_admissions in engine registry. Move to execute/demo_seed.py (or tests/fixtures); importers: __main__ serve-execute, tests test_execute_ref/test_execute_post_dispatch_faults/test_jit_draft_execution/test_backend_parity_refusals. | +| `execute/service.py:45,74,84-85` | (c) | MockMed | ExecuteService(seed_mockmed=) flag. Replace with generic seed_admissions: Iterable[AdmittedBundle] and let the CLI pass the demo seed. | +| `fleet/simulator.py:1-` | (c) | synthetic payer portal | in-package simulator fixture; move to benchmark/examples | +| `guard_demo.py:1-296` | (c) | MockMed | `python -m openadapt_flow.guard_demo`: fake agent posts referral to MockMed with chart_pages READER creds; not a CLI subcommand. Move to packages/flow/examples or benchmark; importers: packages/agent/tests (1), packages/flow/tests (6). | +| `instrument/ (__init__.py, competitor_drift.py)` | (c) | MockMed | MockMed competitor-drift harness. Moved to packages/flow/benchmark/harness/instrument. | +| `integrations/harness2/ (all: tasks.py, apps.py, agent.py, adapter.py, traces.py, proposers.py, scorer.py, llm.py, pixels.py, rollout.py, selection.py, __init__.py)` | (c) | MockMed | Whole subpackage is a research harness for Harness2 on MockMed: MOCKMED_PATIENT/PATIENTS (tasks.py:71-148), MockMedTarget (apps.py:50-105, /api/db at :70), regex for MockMed task text and 'triage'/'New Encounter'/'encountersaved' (agent.py:221-324), MockMed banner regex Name — MRN — DOB (traces.py:55-57,128-138), clinical prompts (adapter.py:93-120,429-445; proposers.py:50-63). Move to packages/flow/benchmark/harness2; importers: benchmark/harness2_scorer/run.py, tests/test_harness2_{adapter,pixels,traces}.py, pyproject extra 'harness2', .github/workflows/packages.yml:451. | +| `learning/synth_stream.py:1-245` | (c) | MockMed | synthetic MockMed add-patient-note stream (intents, 'Consent Required', 'Belford, Phil DOB'). Move to benchmark/ or tests/fixtures. | +| `learning/synth_stream.py:246-525` | (c) | (generic) | StructuralDiffInducer is generic but lives in the MockMed harness module; imported by learning/teach.py:96, learning/branch_teach.py:76, benchmark/openemr_e2e/harness.py, 4 tests; pyproject mypy list (line 611). Move to learning/induce.py first, then move the stream out. | +| `mockloan/ (server.py, fault_server.py, static/)` | (c) | MockLoan | Own synthetic lending app shipped in wheel but used by no product command; only demo_driver.record_disbursement_demo, benchmark/lending_fault_model, benchmark/undo_proposal, tests (test_mockloan, test_lending_swer, e2e/test_lending_fault_model), scripts (1). Move to packages/flow/benchmark/apps/mockloan. | +| `mockmed/ (server.py, fault_server.py, chart_pages.py, teach_scope.py, static/)` | (c) | MockMed | Own synthetic demo app shipped in wheel. Required by tutorial/demo/demo-record/replay-default/sandbox and root openadapt/quickstart (engine.py, workers.py) + root tests/test_partner_kit.py. Keep shipping (or split to a demo-apps package). mockmed/teach_scope.py (EncounterWrites) is used only by tests -> move to tests/fixtures. chart_pages used by guard_demo + benchmark (agent_guard, effect_e2e, ui_readback). | +| `record_check_proof.py:1-11,60-124` | (c) | MockMed | --prove fixture registry hard-codes MOCKMED Fixture (judge on patient_id/note, write_path /api/encounter, body type Triage) and `if name == 'mockmed'`. Opt-in proof against our fixture. Move MOCKMED Fixture to mockmed/proof_fixture.py and resolve fixtures by entry point / dotted path. Importers: scaffold_verifier.py:594, benchmark/effect_e2e, tests (2). | +| `repair/dialog_fixture.py:1-` | (c) | synthetic | rendered test app in product package (no real app) | +| `reward/seed.py:1-236` | (c) | MockMed | seed_mockmed, MockMed encounters table/SQL (MOCKMED_TABLE 'encounters', SELECT id, patient_id, type, status), patients, episodes. Move to reward/demo_seed or tests/fixtures; CLI serve-reward --seed-mockmed uses it. | +| `reward/seed.py:238-410` | (c) | (generic) | write_bundle/calibrate_bundle/self_signed_certificate are generic helpers living in the MockMed seed module; split into reward/bundle.py. Importers: integrations/harness2/scorer.py, benchmark/undo_proposal/run.py, tests/test_probe_induction.py, test_undo_proposal.py, test_reward_worker.py, test_reward_trust_boundary.py. | +| `tutorial.py:1-560,630-841` | (c) | MockMed | tutorial engine: MockMed selectors (.open-btn,#new-encounter,#type-triage,#note,#save-encounter,#saved-banner), /api/reset, /api/db, ?fault=ok entry, TUTORIAL_POLICY clinical-write. User-facing (tutorial, demo, quickstart in root openadapt/quickstart/engine.py imports _records, run_tutorial_workflow, record_tutorial; agent/tutorial.py; execute/dispatch.py). Must ship in wheel; could be data-driven (demo-app spec YAML: entry query, reset/records endpoints, recorded click script). | +| `validation/dense_surface.py:418` | (c) | MockMed | validation study renders MockMed-branded table inside product package; move to benchmark/dense_surface | +| `__main__.py:5422,9499,10873` | (b) | (domain) | clinical-write as default/suggested policy name (built-in policy pack, data). | +| `app_profiles/openemr.yaml:1-` | (b) | OpenEMR | per-app bookkeeping profile loaded by load_profile(name); only profile shipped | +| `cli_config.py:164; integrations/agents/skills.py:351; agent/teach.py:121` | (b) | (domain) | default policy 'clinical-write' (shipped policy pack data). | +| `execute/models.py:74` | (b) | (domain) | AdmittedBundle.policy default 'clinical-write' (healthcare policy pack); consider requiring it. | +| `__main__.py:10053` | (a) | MockMed | benchmark help text (benchmark already moved to source-checkout harness). | +| `__main__.py:10536,10577` | (a) | MockMed | --admit-local help mentions MockMed; code path is generic. | +| `__main__.py:1149,1257-1266,1288` | (a) | MockMed | docstrings / refusal text about --drift being MockMed-only. | +| `__main__.py:3819` | (a) | MockMed | docstring: --drift forced off. | +| `__main__.py:7994` | (a) | TextEdit | help example for --backend macos owner app. | +| `__main__.py:9,27,39` | (a) | MockMed | module docstring command list. | +| `agent/cards.py:9-14; agent/skill.py:145-147; agent/bridge.py:132-133,325-326; agent/ledger.py:3,378; agent/delegated.py:11,78; agent/mailbox.py:130; agent/teach.py:281-283` | (a) | (domain) | referral / EMR / MRN examples in docstrings and tool descriptions; fine. | +| `api.py:84` | (a) | MockMed | docstring: replay without url serves MockMed (changes if replay default changes). | +| `app_profiles/__init__.py:315` | (a) | OpenEMR | comment/docstring example | +| `app_profiles/__init__.py:58` | (a) | OpenEMR | comment/docstring example | +| `authoring.py:807` | (a) | MockMed | comment 'Non-secret MockMed note' on a generic branch; reword to 'non-secret field'. | +| `backends/factory.py:117` | (a) | TextEdit | comment/docstring example | +| `backends/parallels_vm.py:403` | (a) | OpenDental | comment/docstring example | +| `backends/playwright_backend.py:890` | (a) | OpenEMR | comment/docstring example | +| `backends/win_agent/server.py:1578` | (a) | Notepad | comment/docstring example | +| `cli_api.py:71` | (a) | OpenEMR | help example 'lname'/'mrn' column names. | +| `cli_undo.py:181` | (a) | MockMed | help says default target is MockMed but _cmd_undo_run refuses that case; reword help. | +| `cli_undo.py:538-541` | (a) | MockMed | comment explaining refusal. | +| `compile_agent/code_action.py:135` | (a) | OpenEMR | comment/docstring example | +| `compiler/binding_admission.py:19` | (a) | MockMed | comment/docstring example | +| `compiler/compile.py:193` | (a) | OpenEMR | comment/docstring example | +| `compiler/compile.py:2184` | (a) | MockMed | comment/docstring example | +| `compiler/compile.py:717` | (a) | MockMed | comment/docstring example | +| `compiler/fixup_passes.py:7` | (a) | OpenEMR | history note; passes are generic | +| `compiler/local_edit.py:321` | (a) | MockMed | comment/docstring example | +| `deployment.py:1035` | (a) | OpenEMR | comment/docstring example | +| `deployment.py:112` | (a) | MockMed | comment/docstring example | +| `deployment.py:144` | (a) | TextEdit | comment/docstring example | +| `deployment.py:548` | (a) | OpenEMR | user-facing error example OPENEMR_READER_PASSWORD; reword to neutral APP_READER_PASSWORD | +| `ensure_record.py:114` | (a) | Medplum | comment/docstring example | +| `ensure_record.py:162` | (a) | Medplum | comment/docstring example | +| `ensure_record.py:97` | (a) | Medplum | comment/docstring example | +| `execute/dispatch.py:1-6` | (a) | MockMed | module docstring; update after generalizing live_replay. | +| `execution_profiles.py:2595` | (a) | MockMed | comment/docstring example | +| `fault_gate.py:822` | (a) | OpenEMR | comment/docstring example | +| `generate/demo_analysis.py:781` | (a) | Redmine | comment/docstring example | +| `generate/ensure_contract.py:4` | (a) | Medplum | comment/docstring example | +| `generate/verify_contract.py:1234` | (a) | Redmine | comment/docstring example | +| `generate/verify_contract.py:1275` | (a) | EspoCRM | comment/docstring example | +| `generate/verify_contract.py:1276` | (a) | Dolibarr | comment/docstring example | +| `generate/verify_contract.py:225` | (a) | Redmine | comment/docstring example | +| `generate/verify_contract.py:566` | (a) | EspoCRM | comment/docstring example | +| `integrations/agents/__init__.py:79` | (a) | (domain) | 'a clinic, a tenant' example. | +| `jit/authority.py:463` | (a) | TextEdit | comment/docstring example | +| `jit/authority.py:623` | (a) | TextEdit | comment/docstring example | +| `jit/compile_agent.py:354` | (a) | TextEdit | comment/docstring example | +| `jit/recorded.py:6` | (a) | MockMed | docstring example task id | +| `jit/signature.py:72` | (a) | MockMed | comment/docstring example | +| `learning/__init__.py:20; learning/halt_loop.py:8; learning/trace.py:60; learning/branch_teach.py:6,108; learning/library.py` | (a) | MockMed | docstring references. | +| `operator_profile.py:79` | (a) | OpenEMR | comment/docstring example | +| `policy_packs.py:41` | (a) | MockMed | user-facing pack summary names MockMed; fine as the demo app | +| `qualification_dev_signer.py:3` | (a) | MockMed | comment/docstring example | +| `qualification_proposal.py:10` | (a) | MockMed | comment/docstring example | +| `qualification_web_session.py:3` | (a) | OpenEMR,openIMIS | comment/docstring example | +| `qualify/config.py:303` | (a) | OpenEMR | comment/docstring example | +| `record_check_binding.py:48` | (a) | openIMIS | comment/docstring example | +| `record_check_identity.py:4` | (a) | OpenEMR | comment/docstring example | +| `record_check_synthesis.py:25` | (a) | MockMed | comment/docstring example | +| `registry.py:3` | (a) | OpenEMR | comment/docstring example | +| `registry.py:813` | (a) | OpenEMR | comment/docstring example | +| `report_redaction.py:5` | (a) | OpenEMR | comment/docstring example | +| `reward/models.py:497; reward/calibration.py:13` | (a) | (domain) | 'triage rows' wording in docstrings. | +| `risk.py:476` | (a) | openIMIS | comment/docstring example | +| `risk.py:565` | (a) | OpenEMR | comment/docstring example | +| `rollback_state.py:77` | (a) | OpenEMR | comment/docstring example | +| `runtime/actuators/web_form.py:214` | (a) | OpenEMR | comment/docstring example | +| `runtime/actuators/web_form.py:324` | (a) | validate.js | var constraints recognizer is a JS library convention (OpenEMR among users), not app-specific | +| `runtime/actuators/web_form.py:331` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/__init__.py:21` | (a) | MockMed | comment/docstring example | +| `runtime/effects/__init__.py:23` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/__init__.py:38` | (a) | Frappe | comment/docstring example | +| `runtime/effects/_common.py:143` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/auth.py:33` | (a) | Frappe | comment/docstring example | +| `runtime/effects/db_snapshot.py:713` | (a) | Dolibarr | comment/docstring example | +| `runtime/effects/effect.py:1536` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/effect.py:1541` | (a) | OpenEMR,MockMed | comment/docstring example | +| `runtime/effects/effect.py:206` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/effect.py:208` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/effect.py:223` | (a) | Redmine | comment/docstring example | +| `runtime/effects/effect.py:903` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:1` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:12` | (a) | MockMed | comment/docstring example | +| `runtime/effects/fhir.py:14` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:191` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:195` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:3` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:38` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:386` | (a) | Epic | comment/docstring example | +| `runtime/effects/fhir.py:39` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:42` | (a) | MockMed | comment/docstring example | +| `runtime/effects/frame.py:135` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:144` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:1499` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:1630` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:1944` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:242` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:46` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fresh_session.py:24` | (a) | MockMed | comment/docstring example | +| `runtime/effects/rest.py:237` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/rest.py:5` | (a) | MockMed | comment/docstring example | +| `runtime/effects/sql.py:12` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:215` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:215` | (a) | Frappe | comment; _TABLE_NAME allows space generically | +| `runtime/effects/sql.py:3` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:365` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/sql.py:4` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:480` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:534` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:535` | (a) | Frappe | comment/docstring example | +| `runtime/grounder.py:136-155` | (a) | healthcare domain | _INTENT_STOPWORDS include patient/mrn; domain wording, intent-only; low | +| `submit_binding.py:8` | (a) | openIMIS | comment/docstring example | +| `validation/dense_surface.py:1271` | (a) | OpenEMR | comment/docstring example | +| `validation/dense_surface.py:5` | (a) | OpenEMR | comment/docstring example | +| `verifier.py:2467` | (a) | OpenEMR | comment/docstring example | +| `vision/hashing.py:10` | (a) | MockMed | docstring for the MockMed-calibrated phash tolerance (see compile.py:151) | +| `volatility.py:100` | (a) | OpenEMR | comment/docstring example | +| `volatility.py:31` | (a) | OpenEMR | comment/docstring example | +| `volatility.py:6` | (a) | OpenEMR | comment/docstring example | +| `benchmark/` (27 files) | (c) | OpenEMR, Frappe, MockMed, Open Dental | Moved to `packages/flow/benchmark/harness/` | diff --git a/packages/flow/docs/design/EFFECT_VERIFIER.md b/packages/flow/docs/design/EFFECT_VERIFIER.md index b7364ebbcf..b54ec7cb59 100644 --- a/packages/flow/docs/design/EFFECT_VERIFIER.md +++ b/packages/flow/docs/design/EFFECT_VERIFIER.md @@ -80,7 +80,7 @@ the note-as-Observation write the fake models cannot be created over FHIR on a stock OpenEMR; the property the live test establishes is the one the fake could not — the verdicts are correct against a **real FHIR server**, not a fake. CI still runs vision-only against the **public demo** -(`openadapt_flow/benchmark/openemr_benchmark.py`, `scripts/openemr_demo.py`); +(`benchmark/harness/openemr_benchmark.py`, `scripts/openemr_demo.py`); the live FHIR test is gated behind `OPENEMR_FHIR_BASE_URL` and skipped otherwise. See `benchmark/openemr_live/README.md` for the one-command bring-up. diff --git a/packages/flow/docs/desktop/PHASE2.md b/packages/flow/docs/desktop/PHASE2.md index 6eb67329e8..16eadb2505 100644 --- a/packages/flow/docs/desktop/PHASE2.md +++ b/packages/flow/docs/desktop/PHASE2.md @@ -58,7 +58,7 @@ so `WindowsBackend` sees exactly what it drives). | `scripts/desktop/patient_notes.ps1` | WinForms target app (list-select → edit note → save); drift knobs via `pn_env.json` | | `scripts/desktop/pn_db.py` | SQLite ground-truth CLI (seed/list/save/get/all) | | `scripts/desktop/uia_arm.py` | UIA-selector arm (pywinauto), identity + positional | -| `openadapt_flow/benchmark/desktop_benchmark.py` | Orchestrator: 3 arms × drift matrix, DB judge, results.json/BENCHMARK.md/chart | +| `benchmark/harness/desktop_benchmark.py` | Orchestrator: 3 arms × drift matrix, DB judge, results.json/BENCHMARK.md/chart | ## Snapshots (reversibility + warm boot) @@ -78,7 +78,7 @@ state without a full VM revert. ```bash # one-off full matrix (6 conditions × 3 arms × n): -python -m openadapt_flow.benchmark.desktop_benchmark --out benchmark/desktop --n 3 +python -m benchmark.harness.desktop_benchmark --out benchmark/desktop --n 3 ``` `DesktopHarness.connect()` starts the VM, launches the session-1 shim, deploys diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index bb52425ba8..9a1f71f1d6 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -5337,8 +5337,31 @@ def _cmd_silent_saves(args: argparse.Namespace) -> int: return int(silent_saves_main(list(args.silent_saves_argv))) +def _checkout_harness_on_path(command: str) -> bool: + """Put the checkout's ``benchmark`` harness on ``sys.path``. + + The harness lives in ``packages/flow/benchmark/harness``, outside the + package, so a wheel install has none and the command says so. + """ + checkout = Path(__file__).resolve().parents[1] + if not (checkout / "benchmark" / "harness" / "__init__.py").is_file(): + print( + f"{command} runs from a source checkout of OpenAdapt " + "(packages/flow/benchmark/harness); the wheel does not ship it.", + file=sys.stderr, + ) + return False + if str(checkout) not in sys.path: + sys.path.insert(0, str(checkout)) + return True + + def _cmd_benchmark(args: argparse.Namespace) -> int: - from openadapt_flow.benchmark.run_benchmark import run_benchmark + if not _checkout_harness_on_path("benchmark"): + return 2 + from benchmark.harness.run_benchmark import ( # type: ignore[import-not-found] + run_benchmark, + ) results = run_benchmark( Path(args.out), diff --git a/packages/flow/openadapt_flow/anthropic_defaults.py b/packages/flow/openadapt_flow/anthropic_defaults.py index 22ec8ce45b..fc4201c29e 100644 --- a/packages/flow/openadapt_flow/anthropic_defaults.py +++ b/packages/flow/openadapt_flow/anthropic_defaults.py @@ -1,6 +1,6 @@ """Anthropic defaults that product code shares with the benchmark harness. -These live here, not in ``openadapt_flow.benchmark``, because the product +These live here, not in ``benchmark.harness`` (the checkout's harness), because the product imports them and a frozen build leaves the benchmark package out (``tests/test_runtime_never_imports_benchmark.py`` enforces the rule). """ diff --git a/packages/flow/openadapt_flow/authoring.py b/packages/flow/openadapt_flow/authoring.py index a1c4e0c2a1..24a3df3402 100644 --- a/packages/flow/openadapt_flow/authoring.py +++ b/packages/flow/openadapt_flow/authoring.py @@ -804,7 +804,7 @@ def continue_input(self, *, operator_confirmed: bool = True) -> dict[str, Any]: "reason": "unconfirmed_masked_secret", } if not pause.secret and (text is None or not text): - # Non-secret MockMed note: do not persist an empty TYPE/param. + # Non-secret field left empty: do not persist an empty TYPE/param. return { "recorded": False, "param": pause.param, diff --git a/packages/flow/openadapt_flow/deployment.py b/packages/flow/openadapt_flow/deployment.py index e8b54023bd..a5f3d698e3 100644 --- a/packages/flow/openadapt_flow/deployment.py +++ b/packages/flow/openadapt_flow/deployment.py @@ -545,7 +545,7 @@ def _validate_secret_env(cls, v: dict[str, str]) -> dict[str, str]: if not isinstance(env_name, str) or not _ENV_VAR_NAME.fullmatch(env_name): raise ValueError( f"effects.secret_env[{slot!r}] must be the NAME of an " - "environment variable (for example OPENEMR_READER_PASSWORD), " + "environment variable (for example APP_READER_PASSWORD), " "never the credential itself" ) return v diff --git a/packages/flow/paper/REVIEW_ADVERSARIAL.md b/packages/flow/paper/REVIEW_ADVERSARIAL.md index 4ebe33f3c6..9e31c59679 100644 --- a/packages/flow/paper/REVIEW_ADVERSARIAL.md +++ b/packages/flow/paper/REVIEW_ADVERSARIAL.md @@ -35,7 +35,7 @@ Every quantitative result is at a sample size where inference is not possible, a This is my single most serious concern, and it is not disclosed in the paper. -- The headline "screen 50/90, effect 0/90" comes from `openadapt_flow/benchmark/silent_wrong_action.py`, **not** from the browser study. That harness does not drive the browser, does not run OCR, and does not invoke the `Replayer`. It issues raw `requests.post` calls that replicate what the app's JavaScript would send, encodes the screen oracle as a hardcoded rule over HTTP status codes, and calls `RestRecordVerifier.verify(...)` directly. +- The headline "screen 50/90, effect 0/90" comes from `benchmark/harness/silent_wrong_action.py`, **not** from the browser study. That harness does not drive the browser, does not run OCR, and does not invoke the `Replayer`. It issues raw `requests.post` calls that replicate what the app's JavaScript would send, encodes the screen oracle as a hardcoded rule over HTTP status codes, and calls `RestRecordVerifier.verify(...)` directly. - In that harness the **effect oracle and the ground-truth judge read the same in-process store**: the effect verifier reads `GET /api/db` and the ground-truth `_business_effect` reads `FaultDB.snapshot()`, which is the same `FaultDB`. Worse, the effect contract (`record_written` exactly once AND `field_equals` on the note) is essentially a restatement of the ground-truth definition of "correct." So "effect silent-wrong = 0 of 90" is guaranteed by construction: the oracle and the ground truth are computed from the same bytes with near-identical logic. The number is a property of the fixture, not an empirical finding. - The genuinely end-to-end harness (`benchmark/fault_model/run.py`, real Playwright + OCR + `Replayer`) measures **only the screen-only baseline**. Its own recommendations section describes effect verification as a *proposed* fix that the reference postcondition system does not yet implement ("The reference postcondition system is vision-only today, so on this corpus it provides no transactional-safety coverage"). The effect-verifier-through-the-replayer path for these fault classes exists only in unit tests (`tests/test_effect_fault_matrix.py`), and those tests also call `verifier.verify(...)` directly rather than driving the `Replayer`. - Consequently the Results sentence "those five classes refuted and halted **through the live replayer**" overstates what was measured. The live replayer never ran the effect verifier on these 90 runs. The suspiciously clean integers (exactly 50, exactly 0, exactly 10) are deterministic-fixture artifacts, not a measured rate. @@ -48,7 +48,7 @@ The seven fault classes are hand-picked transactional hazards (partial save, dup ### 3.4 Baseline fairness: same-backend, but a vanilla single-shot agent -The agent baseline (`openadapt_flow/benchmark/agent_baseline.py`) is not a strawman in the crude sense: it drives the same Playwright backend, uses a current computer-use model and tool, and is handed generous task information (credentials, exact patient, exact note text, a hint that the app is slow). But it is a **single-attempt, un-scaffolded** loop: no system prompt, no planning/reflection/ReAct, no whole-task retry, and hard caps (25 actions, $1.50, last-3-screenshots context). Modern agent results lean heavily on scaffolding and retries; comparing a compiled program against a vanilla single-shot agent inflates the latency and cost deltas. The paper's hedges are appropriate, but the abstract and introduction still lead with the comparison in a way that implies a capability gap the design does not license. At minimum the baseline's single-shot, no-scaffold nature belongs in the main text next to the comparison table, not only implicitly in the code. +The agent baseline (`benchmark/harness/agent_baseline.py`) is not a strawman in the crude sense: it drives the same Playwright backend, uses a current computer-use model and tool, and is handed generous task information (credentials, exact patient, exact note text, a hint that the app is slow). But it is a **single-attempt, un-scaffolded** loop: no system prompt, no planning/reflection/ReAct, no whole-task retry, and hard caps (25 actions, $1.50, last-3-screenshots context). Modern agent results lean heavily on scaffolding and retries; comparing a compiled program against a vanilla single-shot agent inflates the latency and cost deltas. The paper's hedges are appropriate, but the abstract and introduction still lead with the comparison in a way that implies a capability gap the design does not license. At minimum the baseline's single-shot, no-scaffold nature belongs in the main text next to the comparison table, not only implicitly in the code. ### 3.5 The strong-oracle claim is ambiguous exactly where it matters most (pixel-only substrates) diff --git a/packages/flow/pyproject.toml b/packages/flow/pyproject.toml index b63cfa7435..b4bedc3057 100644 --- a/packages/flow/pyproject.toml +++ b/packages/flow/pyproject.toml @@ -605,7 +605,7 @@ module = [ "openadapt_flow.__main__", "openadapt_flow.backends.playwright_backend", "openadapt_flow.backends.rdp_backend", - "openadapt_flow.benchmark.desktop_benchmark", + "benchmark.harness.desktop_benchmark", "openadapt_flow.compiler.disambiguation", "openadapt_flow.interactive_recorder", "openadapt_flow.learning.synth_stream", diff --git a/packages/flow/scripts/desktop/waa_shim.py b/packages/flow/scripts/desktop/waa_shim.py index a15e66b1ae..e0a877e4af 100644 --- a/packages/flow/scripts/desktop/waa_shim.py +++ b/packages/flow/scripts/desktop/waa_shim.py @@ -1,7 +1,7 @@ """In-guest WAA-contract HTTP shim (runs INSIDE the Windows VM). Serves the desktop benchmark harness -(``openadapt_flow.benchmark.desktop_benchmark``): +(``benchmark.harness.desktop_benchmark``): GET /screenshot -> raw PNG bytes of the desktop (Flask send_file) GET /health -> ``{"status": "ok", ...}`` liveness probe diff --git a/packages/flow/scripts/frappe_lending_demo.py b/packages/flow/scripts/frappe_lending_demo.py index 8681b128b4..13877e4925 100644 --- a/packages/flow/scripts/frappe_lending_demo.py +++ b/packages/flow/scripts/frappe_lending_demo.py @@ -42,8 +42,8 @@ from openadapt_flow.backends.playwright_backend import ( # noqa: E402 PlaywrightBackend as _BasePlaywrightBackend, ) -from openadapt_flow.benchmark import agent_baseline # noqa: E402 -from openadapt_flow.benchmark.frappe_lending import ( # noqa: E402 +from benchmark.harness import agent_baseline # noqa: E402 +from benchmark.harness.frappe_lending import ( # noqa: E402 CONDITIONS, INITIAL_TRIALS_PER_CELL, PUBLICATION_TRIALS_PER_CELL, diff --git a/packages/flow/scripts/openemr_demo.py b/packages/flow/scripts/openemr_demo.py index 6a91a699fe..b9a75aee21 100644 --- a/packages/flow/scripts/openemr_demo.py +++ b/packages/flow/scripts/openemr_demo.py @@ -19,7 +19,7 @@ Showcase artifacts land in docs/showcase-openemr/; ``benchmark`` records a fresh demonstration into a temporary directory and writes results to -benchmark/openemr/ (see openadapt_flow.benchmark.openemr_benchmark). +benchmark/openemr/ (see benchmark.harness.openemr_benchmark). """ from __future__ import annotations @@ -189,7 +189,7 @@ def _note_visible(final_png: bytes, note: str) -> bool: Delegates to the benchmark's shared arm-independent check (``verify_note_saved``) so the criterion is implemented exactly once. """ - from openadapt_flow.benchmark.verify import verify_note_saved + from benchmark.harness.verify import verify_note_saved return verify_note_saved(final_png, note).success @@ -280,15 +280,15 @@ def benchmark() -> None: (incremental, per run) plus results.json / BENCHMARK.md / latency_cost.png to benchmark/openemr/. - Hard cost guardrails (see openadapt_flow.benchmark.openemr_benchmark): + Hard cost guardrails (see benchmark.harness.openemr_benchmark): an API preflight before any spend, a $1.50 per-run cap, an $8.00 total agent-arm ceiling, and an abort on consecutive billing errors. All at list price — billed cost under the intro rate is lower. """ import tempfile - from openadapt_flow.benchmark import agent_baseline - from openadapt_flow.benchmark.openemr_benchmark import ( + from benchmark.harness import agent_baseline + from benchmark.harness.openemr_benchmark import ( MAX_TOTAL_COST_USD, run_openemr_benchmark, ) diff --git a/packages/flow/scripts/openemr_local_demo.py b/packages/flow/scripts/openemr_local_demo.py index d9d8438cd1..0a62f7a4d8 100644 --- a/packages/flow/scripts/openemr_local_demo.py +++ b/packages/flow/scripts/openemr_local_demo.py @@ -40,8 +40,8 @@ from openadapt_flow.backends.playwright_backend import ( # noqa: E402 PlaywrightBackend as _BasePlaywrightBackend, ) -from openadapt_flow.benchmark import agent_baseline # noqa: E402 -from openadapt_flow.benchmark.openemr_local import ( # noqa: E402 +from benchmark.harness import agent_baseline # noqa: E402 +from benchmark.harness.openemr_local import ( # noqa: E402 CONDITIONS, INITIAL_TRIALS_PER_CELL, PUBLICATION_TRIALS_PER_CELL, diff --git a/packages/flow/scripts/openemr_param_depth.py b/packages/flow/scripts/openemr_param_depth.py index 6afd55d641..5c77b4c57f 100644 --- a/packages/flow/scripts/openemr_param_depth.py +++ b/packages/flow/scripts/openemr_param_depth.py @@ -148,7 +148,7 @@ def compile_bundle() -> None: def _replay_one(label: str, params: dict[str, str], url: str) -> dict: """One fresh-browser compiled replay; returns a summary row.""" - from openadapt_flow.benchmark.verify import verify_note_saved + from benchmark.harness.verify import verify_note_saved from openadapt_flow.ir import Workflow from openadapt_flow.runtime.replayer import Replayer from openadapt_flow.vision.ocr import normalize_text, ocr diff --git a/packages/flow/tests/e2e/test_region_stable_theme_regression.py b/packages/flow/tests/e2e/test_region_stable_theme_regression.py index 4ce3990130..152450bc7a 100644 --- a/packages/flow/tests/e2e/test_region_stable_theme_regression.py +++ b/packages/flow/tests/e2e/test_region_stable_theme_regression.py @@ -13,7 +13,7 @@ import pytest from openadapt_flow.backends.playwright_backend import PlaywrightBackend -from openadapt_flow.benchmark.dom_arm import verify_final_state +from benchmark.harness.dom_arm import verify_final_state from openadapt_flow.compiler import compile_recording from openadapt_flow.demo_driver import record_triage_demo from openadapt_flow.ir import Workflow diff --git a/packages/flow/tests/test_benchmark.py b/packages/flow/tests/test_benchmark.py index 6f903b79c9..68dbac3e54 100644 --- a/packages/flow/tests/test_benchmark.py +++ b/packages/flow/tests/test_benchmark.py @@ -15,20 +15,20 @@ import numpy as np import pytest -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.agent_baseline import ( +from benchmark.harness import agent_baseline +from benchmark.harness.agent_baseline import ( _truncate_screenshots, compute_cost, load_api_key, run_agent, triage_task_prompt, ) -from openadapt_flow.benchmark.run_benchmark import ( +from benchmark.harness.run_benchmark import ( aggregate_results, render_markdown, write_outputs, ) -from openadapt_flow.benchmark.verify import verify_encounter_saved +from benchmark.harness.verify import verify_encounter_saved NOTE = "Follow-up in 2 weeks; BP recheck." @@ -521,7 +521,7 @@ def test_right_type_passes_at_every_note_length( # line. TestVerify above retains a real-OCR smoke for the benchmark's # actual 32-character note; these four cases pin the verifier logic. monkeypatch.setattr( - "openadapt_flow.benchmark.verify.ocr", + "benchmark.harness.verify.ocr", lambda _screen: [SimpleNamespace(text=f"Triage - {note[:60]}")], ) verdict = verify_encounter_saved(screen, note) @@ -542,7 +542,7 @@ def test_wrong_type_fails_at_every_note_length( f"Consult - {note[:60]}", ) monkeypatch.setattr( - "openadapt_flow.benchmark.verify.ocr", + "benchmark.harness.verify.ocr", lambda _screen: [SimpleNamespace(text=f"Consult - {note[:60]}")], ) verdict = verify_encounter_saved(screen, note) @@ -599,7 +599,7 @@ def test_bare_note_starting_with_known_type_is_not_a_row( # the note itself starts with a known categorical value. screen = self.make_screen(f"Encounter saved - {note[:40]}", note[:60]) monkeypatch.setattr( - "openadapt_flow.benchmark.verify.ocr", + "benchmark.harness.verify.ocr", lambda _screen: [SimpleNamespace(text=note[:60])], ) verdict = verify_encounter_saved(screen, note) diff --git a/packages/flow/tests/test_chart_fonts.py b/packages/flow/tests/test_chart_fonts.py index c54ceb2bc1..d40847a4da 100644 --- a/packages/flow/tests/test_chart_fonts.py +++ b/packages/flow/tests/test_chart_fonts.py @@ -2,7 +2,7 @@ The benchmark chart is cosmetic; a font-lookup failure must never fail the suite. These tests cover the two guarantees in -``openadapt_flow.benchmark.chart_fonts``: the bundled DejaVuSans font is +``benchmark.harness.chart_fonts``: the bundled DejaVuSans font is registered so ``findfont`` cannot miss, and ``safe_render`` swallows any rendering failure. """ @@ -12,7 +12,7 @@ from pathlib import Path from typing import Any -from openadapt_flow.benchmark.chart_fonts import ( +from benchmark.harness.chart_fonts import ( BUNDLED_FONT_NAME, configure_bundled_font, safe_render, diff --git a/packages/flow/tests/test_competitor_drift_instrument.py b/packages/flow/tests/test_competitor_drift_instrument.py index 984b79c792..ec52bd8589 100644 --- a/packages/flow/tests/test_competitor_drift_instrument.py +++ b/packages/flow/tests/test_competitor_drift_instrument.py @@ -22,7 +22,7 @@ import pytest -from openadapt_flow.instrument import ( +from benchmark.harness.instrument import ( AgentRunResult, ArchitectureClassError, CostGuard, @@ -35,7 +35,7 @@ ensure_architecture_class, run_instrument, ) -from openadapt_flow.instrument.competitor_drift import render_markdown, write_outputs +from benchmark.harness.instrument.competitor_drift import render_markdown, write_outputs # Fault classes a screen-only report is BLIND to: a wrong effect lands while # the "saved" banner still paints, so a screen-blind agent claims success. diff --git a/packages/flow/tests/test_desktop_benchmark.py b/packages/flow/tests/test_desktop_benchmark.py index 45d605ed10..f54425cae2 100644 --- a/packages/flow/tests/test_desktop_benchmark.py +++ b/packages/flow/tests/test_desktop_benchmark.py @@ -9,8 +9,8 @@ import json -from openadapt_flow.benchmark import desktop_benchmark as db -from openadapt_flow.benchmark.desktop_benchmark import ( +from benchmark.harness import desktop_benchmark as db +from benchmark.harness.desktop_benchmark import ( _classify, run_desktop_benchmark, ) diff --git a/packages/flow/tests/test_dom_arm.py b/packages/flow/tests/test_dom_arm.py index c19314c790..6dd3a6e4b4 100644 --- a/packages/flow/tests/test_dom_arm.py +++ b/packages/flow/tests/test_dom_arm.py @@ -15,7 +15,7 @@ import numpy as np import pytest -from openadapt_flow.benchmark.dom_arm import ( +from benchmark.harness.dom_arm import ( PERTURBATIONS, SCHEDULE, aggregate_dom_results, @@ -191,7 +191,7 @@ def test_perturbation_menu(self) -> None: ) def test_notes_distinct_per_arm_and_slot(self) -> None: - from openadapt_flow.benchmark.dom_arm import ARMS + from benchmark.harness.dom_arm import ARMS notes = { note_for_slot(arm, slot) @@ -262,7 +262,7 @@ def test_maintenance_is_dom_loud_break_on_drift_only(self) -> None: assert not needs_maintenance(disputed) def test_verification_dispute(self) -> None: - from openadapt_flow.benchmark.dom_arm import verification_dispute + from benchmark.harness.dom_arm import verification_dispute # DOM: every step ran, no error, judge said no -> dispute. assert verification_dispute( @@ -546,7 +546,7 @@ class TestVerifyFinalState: exists to measure.""" def test_dom_check_is_the_hybrid_check(self) -> None: - from openadapt_flow.benchmark.hybrid_benchmark import ( + from benchmark.harness.hybrid_benchmark import ( verify_hybrid_final, ) diff --git a/packages/flow/tests/test_effectbench.py b/packages/flow/tests/test_effectbench.py index 435364cdcd..19419163d9 100644 --- a/packages/flow/tests/test_effectbench.py +++ b/packages/flow/tests/test_effectbench.py @@ -26,7 +26,7 @@ build_reference_episodes, ) from benchmark.fault_model import faults as F -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, DivergenceCategory, Effect, @@ -44,7 +44,7 @@ summarize, wilson_interval, ) -from openadapt_flow.benchmark.effectbench.metrics import bootstrap_ci, rate +from benchmark.harness.effectbench.metrics import bootstrap_ci, rate # --------------------------------------------------------------------------- # 1. Reference re-expression regression (the headline result) @@ -334,7 +334,7 @@ def test_record_and_note_effects_are_reexported() -> None: def test_score_episode_end_to_end() -> None: """The runner orchestrator: capture pre-state, run the arm, verify, label.""" - from openadapt_flow.benchmark.effectbench import AgentReport, score_episode + from benchmark.harness.effectbench import AgentReport, score_episode box = {"acted": False} @@ -367,7 +367,7 @@ def run_action() -> AgentReport: def test_score_episode_raises_on_unreadable_oracle() -> None: - from openadapt_flow.benchmark.effectbench import AgentReport, score_episode + from benchmark.harness.effectbench import AgentReport, score_episode with pytest.raises(ValueError): score_episode( @@ -433,8 +433,8 @@ def test_summary_filters_by_arm(episodes: list[EpisodeRecord]) -> None: def test_episode_record_predicates_match_outcome() -> None: - from openadapt_flow.benchmark.effectbench import Verdict - from openadapt_flow.benchmark.effectbench.schema import OracleVerdict + from benchmark.harness.effectbench import Verdict + from benchmark.harness.effectbench.schema import OracleVerdict ep = EpisodeRecord( episode_id="e1", diff --git a/packages/flow/tests/test_effectbench_runner.py b/packages/flow/tests/test_effectbench_runner.py index 64cd934c61..8abe524670 100644 --- a/packages/flow/tests/test_effectbench_runner.py +++ b/packages/flow/tests/test_effectbench_runner.py @@ -20,7 +20,7 @@ import pytest -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, EffectKind, EpisodeRecord, @@ -30,7 +30,7 @@ SwerVariant, summarize, ) -from openadapt_flow.benchmark.effectbench.runner import ( +from benchmark.harness.effectbench.runner import ( LIVE_ARMS, CompilerArm, CompoundEffectVerifier, @@ -43,19 +43,19 @@ run_episode, run_matrix, ) -from openadapt_flow.benchmark.effectbench.runner.arms import AgentArm, ArmResult -from openadapt_flow.benchmark.effectbench.runner.baselines import ( +from benchmark.harness.effectbench.runner.arms import AgentArm, ArmResult +from benchmark.harness.effectbench.runner.baselines import ( SCAFFOLDED_ARMS, ClaudeComputerUseArm, ScaffoldNotWired, ) -from openadapt_flow.benchmark.effectbench.runner.harness import EpisodeEnv -from openadapt_flow.benchmark.effectbench.runner.reference_tasks import ( +from benchmark.harness.effectbench.runner.harness import EpisodeEnv +from benchmark.harness.effectbench.runner.reference_tasks import ( GOAL, TARGET, make_params, ) -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( DivergenceCategory, OracleChannel, OracleSpec, @@ -343,7 +343,7 @@ def test_run_matrix_skips_scaffolded_arms_by_default() -> None: def test_cli_dry_run_runs_green() -> None: - from openadapt_flow.benchmark.effectbench.runner.__main__ import run_dry_run + from benchmark.harness.effectbench.runner.__main__ import run_dry_run episodes, summaries = run_dry_run( trials=1, arm_names="compiler,screen_only", include_scaffolded=False diff --git a/packages/flow/tests/test_effectbench_standalone.py b/packages/flow/tests/test_effectbench_standalone.py index dc0fb92526..56ac997158 100644 --- a/packages/flow/tests/test_effectbench_standalone.py +++ b/packages/flow/tests/test_effectbench_standalone.py @@ -6,7 +6,7 @@ dependency). Because the standalone package vendors the effect MECHANISM (the typed ``Effect`` contract, the substrate-independent judge, the outcome classifier) that is otherwise defined in ``openadapt_flow.runtime.effects`` and -``openadapt_flow.benchmark.effectbench``, this test guards against the two copies +``benchmark.harness.effectbench``, this test guards against the two copies drifting: it asserts the standalone classifier and judge agree BYTE-FOR-BYTE with the engine on the reference inputs, and that the standalone reference reproduces the published headline. It also re-runs the standalone source-availability @@ -62,7 +62,7 @@ def test_classifier_parity_with_engine_over_the_full_truth_table(): """The standalone classifier equals the engine classifier on every input.""" import effectbench.oracle as sa - from openadapt_flow.benchmark.effectbench import oracle as eng + from benchmark.harness.effectbench import oracle as eng scoreable = ["CORRECT", "WRONG_PERSISTED", "ABSENT"] for reported, state_name, avail in itertools.product( @@ -86,7 +86,7 @@ def test_divergence_category_parity_with_engine(): """The public standalone taxonomy must not lag the engine copy.""" from effectbench.schema import DivergenceCategory as StandaloneCategory - from openadapt_flow.benchmark.effectbench.schema import ( + from benchmark.harness.effectbench.schema import ( DivergenceCategory as EngineCategory, ) diff --git a/packages/flow/tests/test_effectbench_task_pack.py b/packages/flow/tests/test_effectbench_task_pack.py index bed77414ca..ddb2ea85c0 100644 --- a/packages/flow/tests/test_effectbench_task_pack.py +++ b/packages/flow/tests/test_effectbench_task_pack.py @@ -36,8 +36,8 @@ from benchmark.effectbench.task_pack.audit import audit_mockmed_oracle from benchmark.effectbench.task_pack.driver import run_mockmed_pack from benchmark.effectbench.task_pack.mockmed_tasks import MOCKMED_TASKS -from openadapt_flow.benchmark.effectbench import OutcomeLabel, Substrate, summarize -from openadapt_flow.benchmark.effectbench.schema import DivergenceCategory +from benchmark.harness.effectbench import OutcomeLabel, Substrate, summarize +from benchmark.harness.effectbench.schema import DivergenceCategory from openadapt_flow.runtime.effects.sql import assert_read_only_sql MANIFEST_PATH = Path(__file__).resolve().parents[1] / ( diff --git a/packages/flow/tests/test_frappe_lending_benchmark.py b/packages/flow/tests/test_frappe_lending_benchmark.py index ee3f50c080..1737d2a80b 100644 --- a/packages/flow/tests/test_frappe_lending_benchmark.py +++ b/packages/flow/tests/test_frappe_lending_benchmark.py @@ -19,7 +19,7 @@ FrappeFixture, audit_table_deltas, ) -from openadapt_flow.benchmark.frappe_lending import ( +from benchmark.harness.frappe_lending import ( ARMS, CONDITIONS, FrappeLoanApplicationOracle, diff --git a/packages/flow/tests/test_hybrid_benchmark.py b/packages/flow/tests/test_hybrid_benchmark.py index 9779311f40..11ca0244ba 100644 --- a/packages/flow/tests/test_hybrid_benchmark.py +++ b/packages/flow/tests/test_hybrid_benchmark.py @@ -15,8 +15,8 @@ import numpy as np import pytest -from openadapt_flow.benchmark import agent_baseline, hybrid_benchmark -from openadapt_flow.benchmark.hybrid_benchmark import ( +from benchmark.harness import agent_baseline, hybrid_benchmark +from benchmark.harness.hybrid_benchmark import ( AGENT_SLOTS, DRIFT_TYPES, FALLBACK_MAX_ACTIONS, @@ -610,7 +610,7 @@ def fake_run_agent(backend, task, **kwargs): assert "no failing step recorded" in seen["task"] def test_failed_result_index_edge_cases(self) -> None: - from openadapt_flow.benchmark.hybrid_benchmark import ( + from benchmark.harness.hybrid_benchmark import ( _failed_result_index, ) @@ -978,7 +978,7 @@ def test_arms_see_identical_conditions_per_slot( the run helpers), not on the orchestrator-stamped condition labels — the labels come from the same schedule lookup the orchestrator uses to build the URLs, so asserting on them alone is circular.""" - from openadapt_flow.benchmark.hybrid_benchmark import ( + from benchmark.harness.hybrid_benchmark import ( AGENT_SLOTS, condition_url, ) diff --git a/packages/flow/tests/test_lending_swer.py b/packages/flow/tests/test_lending_swer.py index b09dabeeed..b787960357 100644 --- a/packages/flow/tests/test_lending_swer.py +++ b/packages/flow/tests/test_lending_swer.py @@ -22,7 +22,7 @@ from benchmark.lending_fault_model import faults as F from benchmark.lending_fault_model import ground_truth as G from benchmark.lending_fault_model import swer as S -from openadapt_flow.benchmark.effectbench import summarize +from benchmark.harness.effectbench import summarize from openadapt_flow.mockloan.fault_server import LedgerDB diff --git a/packages/flow/tests/test_openemr_benchmark.py b/packages/flow/tests/test_openemr_benchmark.py index b2f10ba067..f32a93db12 100644 --- a/packages/flow/tests/test_openemr_benchmark.py +++ b/packages/flow/tests/test_openemr_benchmark.py @@ -16,20 +16,20 @@ import cv2 import numpy as np -import openadapt_flow.benchmark.verify as benchmark_verify -from openadapt_flow.benchmark import agent_baseline, openemr_benchmark -from openadapt_flow.benchmark.agent_baseline import ( +import benchmark.harness.verify as benchmark_verify +from benchmark.harness import agent_baseline, openemr_benchmark +from benchmark.harness.agent_baseline import ( openemr_task_prompt, run_agent, ) -from openadapt_flow.benchmark.openemr_benchmark import ( +from benchmark.harness.openemr_benchmark import ( aggregate_openemr_results, note_for, render_openemr_markdown, run_openemr_benchmark, write_openemr_outputs, ) -from openadapt_flow.benchmark.verify import verify_note_saved +from benchmark.harness.verify import verify_note_saved from openadapt_flow.vision import OcrLine NOTE = "Insurance card copied and coverage verified by phone." @@ -848,7 +848,7 @@ def crashing_client(self) -> CrashingClient: ) def test_mid_run_crash_row_carries_partial_cost(self, monkeypatch: Any) -> None: - from openadapt_flow.benchmark.run_benchmark import _agent_run + from benchmark.harness.run_benchmark import _agent_run _fake_launch(monkeypatch, FakeBackend()) row = _agent_run("http://x", NOTE, client=self.crashing_client(), task="task") diff --git a/packages/flow/tests/test_openemr_local_benchmark.py b/packages/flow/tests/test_openemr_local_benchmark.py index e637aa076b..c67f0b6c79 100644 --- a/packages/flow/tests/test_openemr_local_benchmark.py +++ b/packages/flow/tests/test_openemr_local_benchmark.py @@ -22,7 +22,7 @@ audit_table_deltas, unexpected_table_deltas, ) -from openadapt_flow.benchmark.openemr_local import ( +from benchmark.harness.openemr_local import ( ARMS, CONDITIONS, OpenEMRPatientOracle, @@ -1011,7 +1011,7 @@ def __init__(self, messages: Messages) -> None: "max_tokens": 4096, } ledger = __import__( - "openadapt_flow.benchmark.agent_baseline", fromlist=["UsageLedger"] + "benchmark.harness.agent_baseline", fromlist=["UsageLedger"] ).UsageLedger() allowed_messages = Messages(10_000) diff --git a/packages/flow/tests/test_openemr_retained_finals.py b/packages/flow/tests/test_openemr_retained_finals.py index 8fbdc22904..c1b6709429 100644 --- a/packages/flow/tests/test_openemr_retained_finals.py +++ b/packages/flow/tests/test_openemr_retained_finals.py @@ -16,7 +16,7 @@ import pytest -from openadapt_flow.benchmark.verify import verify_note_saved +from benchmark.harness.verify import verify_note_saved REPO_ROOT = Path(__file__).resolve().parents[1] RUN_RETAINED_FINALS = os.environ.get("OPENADAPT_OPENEMR_FINALS_DIR") diff --git a/packages/flow/tests/test_reliability_harness.py b/packages/flow/tests/test_reliability_harness.py index 06be924f83..e3dade0d41 100644 --- a/packages/flow/tests/test_reliability_harness.py +++ b/packages/flow/tests/test_reliability_harness.py @@ -1,6 +1,6 @@ """Small synthetic checks for the public reliability aggregation mechanism.""" -from openadapt_flow.benchmark.reliability import _outcome, aggregate +from benchmark.harness.reliability import _outcome, aggregate def _result(outcome: str, *, category: str | None = None) -> dict: diff --git a/packages/flow/tests/test_runtime_never_imports_benchmark.py b/packages/flow/tests/test_runtime_never_imports_benchmark.py index dc4fa57ffb..d59f3083d7 100644 --- a/packages/flow/tests/test_runtime_never_imports_benchmark.py +++ b/packages/flow/tests/test_runtime_never_imports_benchmark.py @@ -1,9 +1,10 @@ -"""Product code never imports ``openadapt_flow.benchmark``. +"""Product code never imports the benchmark harness. -A frozen build (Desktop's engine) leaves the benchmark package out, so a -product module that imports it, even lazily, fails there. Shared names -(``MODEL``, ``load_api_key``, ``SCROLL_PX_PER_UNIT``) live in -``openadapt_flow.anthropic_defaults``. +The harness lives in the source checkout (``packages/flow/benchmark/harness``), +outside the ``openadapt_flow`` package, so neither the wheel nor a frozen build +(Desktop's engine) carries it. A product module that imports it, even lazily, +fails there. Shared names (``MODEL``, ``load_api_key``, ``SCROLL_PX_PER_UNIT``) +live in ``openadapt_flow.anthropic_defaults``. """ from __future__ import annotations @@ -17,57 +18,64 @@ #: Modules whose job is the benchmark itself. Nothing else may import it. ALLOWED = { - # The benchmark CLI command loads the harness on demand. + # The benchmark CLI command loads the harness on demand from a checkout. "__main__.py", - # The competitor-drift harness reuses the benchmark's fault scenarios. - "instrument/competitor_drift.py", + # `record-check prove` loads its fixture from a checkout on demand. + "record_check_proof.py", } +def _is_benchmark(module: str) -> bool: + return any( + module == root or module.startswith(root + ".") + for root in ("benchmark", "openadapt_flow.benchmark") + ) + + def _imports_benchmark(tree: ast.AST) -> list[int]: lines: list[int] = [] for node in ast.walk(tree): if isinstance(node, ast.ImportFrom): module = node.module or "" - if node.level == 0 and ( - module == "openadapt_flow.benchmark" - or module.startswith("openadapt_flow.benchmark.") - ): + if node.level == 0 and _is_benchmark(module): lines.append(node.lineno) elif node.level == 0 and module == "openadapt_flow": if any(a.name == "benchmark" for a in node.names): lines.append(node.lineno) - elif node.level > 0 and ( - module == "benchmark" or module.startswith("benchmark.") - ): + elif node.level > 0 and _is_benchmark(module): lines.append(node.lineno) elif isinstance(node, ast.Import): - for alias in node.names: - if alias.name == "openadapt_flow.benchmark" or alias.name.startswith( - "openadapt_flow.benchmark." - ): - lines.append(node.lineno) + if any(_is_benchmark(alias.name) for alias in node.names): + lines.append(node.lineno) return lines +def test_the_harness_is_not_inside_the_package() -> None: + assert not (ROOT / "benchmark").exists(), ( + "benchmark harnesses belong in packages/flow/benchmark/, outside the " + "shipped openadapt_flow package" + ) + + def test_product_modules_never_import_the_benchmark_package() -> None: offenders: list[str] = [] for path in sorted(ROOT.rglob("*.py")): rel = path.relative_to(ROOT).as_posix() - if rel.startswith("benchmark/") or rel in ALLOWED: + if rel in ALLOWED: continue tree = ast.parse(path.read_text(encoding="utf-8")) offenders += [f"{rel}:{n}" for n in _imports_benchmark(tree)] assert not offenders, ( - "product code imports openadapt_flow.benchmark (a frozen build leaves " - "it out); move the shared name to openadapt_flow.anthropic_defaults " - f"or a runtime module: {offenders}" + "product code imports the benchmark harness (the wheel and a frozen " + "build leave it out); move the shared name to " + "openadapt_flow.anthropic_defaults or a runtime module: " + f"{offenders}" ) def test_shared_anthropic_names_have_one_home() -> None: + from benchmark.harness import agent_baseline from openadapt_flow import anthropic_defaults - from openadapt_flow.benchmark import agent_baseline assert agent_baseline.MODEL is anthropic_defaults.MODEL assert agent_baseline.SCROLL_PX_PER_UNIT == anthropic_defaults.SCROLL_PX_PER_UNIT diff --git a/packages/flow/tests/test_silent_wrong_action_benchmark.py b/packages/flow/tests/test_silent_wrong_action_benchmark.py index a340eea1ae..cadfdbcacc 100644 --- a/packages/flow/tests/test_silent_wrong_action_benchmark.py +++ b/packages/flow/tests/test_silent_wrong_action_benchmark.py @@ -16,7 +16,7 @@ from __future__ import annotations -from openadapt_flow.benchmark.silent_wrong_action import ( +from benchmark.harness.silent_wrong_action import ( SCENARIOS, aggregate, render_markdown, diff --git a/packages/flow/tests/test_ui_dismiss_live_openemr.py b/packages/flow/tests/test_ui_dismiss_live_openemr.py index 1468c56f6d..3caceffee1 100644 --- a/packages/flow/tests/test_ui_dismiss_live_openemr.py +++ b/packages/flow/tests/test_ui_dismiss_live_openemr.py @@ -47,7 +47,7 @@ def _record(tmp_path: Path) -> tuple[Path, Any]: from benchmark.openemr_local.qualify_hooks import fixture - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec from openadapt_flow.recorder import Recorder from scripts.openemr_local_demo import ( FORM_FRAME_NAME, From 7d884c01b42462e6f087bdad4296814e74a89121 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 13:04:32 -0500 Subject: [PATCH 06/16] Generality audit: link CONTRIBUTING without the guard PR's anchor --- packages/flow/docs/GENERALITY_AUDIT.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md index 7fb257ecdb..f8f4b51798 100644 --- a/packages/flow/docs/GENERALITY_AUDIT.md +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -6,8 +6,8 @@ Audited 2026-11-03 on `main` at `213f3d9fe`. Scope: everything under The rule: the engine has no code written for one application. What one application needs lives in data (`app_profiles/*.yaml`, generated contracts, deployment configs) or in test fixtures and benchmark harnesses outside the -shipped package. `scripts/check_engine_generality.py` enforces it in CI; see -[CONTRIBUTING.md](../CONTRIBUTING.md#app-knowledge-lives-in-data). +shipped package. A CI check (`scripts/check_engine_generality.py`) keeps it +that way; [CONTRIBUTING.md](../CONTRIBUTING.md) states the rule. ## Method From cc145da332b549305f2c50b44c74256b791fe4f5 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 14:08:46 -0500 Subject: [PATCH 07/16] CONTRIBUTING: state the app-knowledge rule positively --- packages/flow/CONTRIBUTING.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/flow/CONTRIBUTING.md b/packages/flow/CONTRIBUTING.md index 8641a68f76..c91ee968f6 100644 --- a/packages/flow/CONTRIBUTING.md +++ b/packages/flow/CONTRIBUTING.md @@ -177,15 +177,13 @@ notice alone is not sufficient. ## App knowledge lives in data -The engine (`openadapt_flow/`) works the same way for every application. Never -write code, a default, a constant or a heuristic for one application. What one -application needs goes in data: +The engine (`openadapt_flow/`) works the same way for every application. App +knowledge lives in data: - its bookkeeping writes: an app profile in `openadapt_flow/app_profiles/`, loaded by name; - its endpoints, tables, keys and field paths: the generated contract or the - deployment config (`deploy.yaml`), which the engine requires rather than - defaulting; + deployment config (`deploy.yaml`); - tests and benchmarks for it: fixtures under `tests/` and harnesses under `benchmark/`, outside the shipped package. From 2bdcfa2e1721f6f3d89ceb206a316f53d46aa78e Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 14:10:32 -0500 Subject: [PATCH 08/16] Move test fixtures and demo harnesses out of the openadapt-flow wheel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wheel ships only the engine (see packages/flow/docs/GENERALITY_AUDIT.md). Moved out of openadapt_flow: - integrations/harness2 -> benchmark/harness/harness2 (the harness2 extra is dropped; install [browser] beside Harness² from a checkout) - mockloan/ and demo_driver.record_disbursement_demo -> benchmark/harness/mockloan/ (recorder in demo.py) - guard_demo.py -> benchmark/harness/guard_demo.py - agent/ledger_demo.py -> packages/agent/tests/ledger_demo.py - mockmed/teach_scope.py -> packages/flow/tests/_teach_scope.py - learning/synth_stream.py: StructuralDiffInducer -> learning/induce.py; the MockMed drift stream -> benchmark/harness/synth_stream.py reward/seed.py is split: the bundle helpers (write_bundle, calibrate_bundle, self_signed_certificate) are now reward/bundle.py. The MockMed seed stays, because serve-reward --seed-mockmed needs it from a wheel. --- .github/workflows/packages.yml | 2 +- CAPABILITIES.md | 2 +- llms-full.txt | 2 +- packages/agent/README.md | 2 +- .../agent => agent/tests}/ledger_demo.py | 5 +- packages/agent/tests/test_guard_tools.py | 12 +- packages/agent/tests/test_ledger.py | 3 +- packages/flow/CHANGELOG.md | 2 + .../harness}/guard_demo.py | 8 +- .../harness}/harness2/__init__.py | 0 .../harness}/harness2/adapter.py | 19 +- .../harness}/harness2/agent.py | 2 +- .../harness}/harness2/apps.py | 2 +- .../harness}/harness2/llm.py | 0 .../harness}/harness2/pixels.py | 2 +- .../harness}/harness2/proposers.py | 4 +- .../harness}/harness2/rollout.py | 8 +- .../harness}/harness2/scorer.py | 4 +- .../harness}/harness2/selection.py | 4 +- .../harness}/harness2/tasks.py | 0 .../harness}/harness2/traces.py | 8 +- .../harness}/mockloan/__init__.py | 2 +- .../flow/benchmark/harness/mockloan/demo.py | 97 +++++++ .../harness}/mockloan/fault_server.py | 0 .../harness}/mockloan/server.py | 2 +- .../harness}/mockloan/static/app.js | 2 +- .../harness}/mockloan/static/index.html | 0 .../harness}/mockloan/static/styles.css | 0 .../flow/benchmark/harness/synth_stream.py | 222 +++++++++++++++ .../flow/benchmark/harness2_scorer/RESULTS.md | 2 +- .../flow/benchmark/harness2_scorer/run.py | 14 +- .../LENDING_FAULT_MODEL.md | 2 +- .../benchmark/lending_fault_model/README.md | 6 +- .../benchmark/lending_fault_model/faults.py | 2 +- .../lending_fault_model/resolution_ladder.py | 4 +- .../flow/benchmark/lending_fault_model/run.py | 6 +- .../benchmark/lending_fault_model/swer.py | 2 +- .../flow/benchmark/openemr_e2e/harness.py | 2 +- packages/flow/benchmark/undo_proposal/run.py | 9 +- packages/flow/docs/GENERALITY_AUDIT.md | 14 +- packages/flow/docs/GUARDED_SAVE.md | 4 +- packages/flow/docs/GUARD_AN_AGENT.md | 2 +- packages/flow/docs/HARNESS2.md | 24 +- packages/flow/docs/JIT.md | 6 +- packages/flow/openadapt_flow/demo_driver.py | 86 +----- .../flow/openadapt_flow/learning/__init__.py | 5 +- .../openadapt_flow/learning/branch_teach.py | 2 +- .../flow/openadapt_flow/learning/halt_loop.py | 2 +- .../learning/{synth_stream.py => induce.py} | 199 ++------------ .../flow/openadapt_flow/learning/teach.py | 4 +- packages/flow/openadapt_flow/reward/bundle.py | 254 ++++++++++++++++++ packages/flow/openadapt_flow/reward/seed.py | 234 +--------------- packages/flow/public-artifacts.json | 6 +- packages/flow/pyproject.toml | 10 +- .../teach_scope.py => tests/_teach_scope.py} | 2 + .../flow/tests/e2e/test_injection_live.py | 2 +- .../tests/e2e/test_lending_fault_model.py | 4 +- packages/flow/tests/test_agent_guard.py | 4 +- .../flow/tests/test_continuous_learning.py | 12 +- packages/flow/tests/test_guarded_save.py | 20 +- packages/flow/tests/test_halt_learn_loop.py | 10 +- packages/flow/tests/test_harness2_adapter.py | 28 +- packages/flow/tests/test_harness2_pixels.py | 14 +- packages/flow/tests/test_harness2_traces.py | 10 +- packages/flow/tests/test_injection_paths.py | 4 +- packages/flow/tests/test_injection_suite.py | 2 +- .../tests/test_learning_gate_semantics.py | 12 +- packages/flow/tests/test_lending_swer.py | 4 +- packages/flow/tests/test_mockloan.py | 4 +- packages/flow/tests/test_probe_induction.py | 2 +- packages/flow/tests/test_provenance.py | 4 +- .../flow/tests/test_reward_trust_boundary.py | 6 +- packages/flow/tests/test_reward_worker.py | 8 +- packages/flow/tests/test_teach_placement.py | 2 +- packages/flow/tests/test_teach_scope.py | 8 +- packages/flow/tests/test_undo_proposal.py | 2 +- packages/flow/tests/test_verify_tools.py | 2 +- packages/flow/tests/test_verify_write.py | 2 +- packages/flow/tests/test_write_ledger.py | 4 +- packages/flow/uv.lock | 6 +- 80 files changed, 813 insertions(+), 682 deletions(-) rename packages/{flow/openadapt_flow/agent => agent/tests}/ledger_demo.py (99%) rename packages/flow/{openadapt_flow => benchmark/harness}/guard_demo.py (96%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/__init__.py (100%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/adapter.py (96%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/agent.py (99%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/apps.py (97%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/llm.py (100%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/pixels.py (98%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/proposers.py (98%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/rollout.py (96%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/scorer.py (98%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/selection.py (98%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/tasks.py (100%) rename packages/flow/{openadapt_flow/integrations => benchmark/harness}/harness2/traces.py (98%) rename packages/flow/{openadapt_flow => benchmark/harness}/mockloan/__init__.py (89%) create mode 100644 packages/flow/benchmark/harness/mockloan/demo.py rename packages/flow/{openadapt_flow => benchmark/harness}/mockloan/fault_server.py (100%) rename packages/flow/{openadapt_flow => benchmark/harness}/mockloan/server.py (95%) rename packages/flow/{openadapt_flow => benchmark/harness}/mockloan/static/app.js (99%) rename packages/flow/{openadapt_flow => benchmark/harness}/mockloan/static/index.html (100%) rename packages/flow/{openadapt_flow => benchmark/harness}/mockloan/static/styles.css (100%) create mode 100644 packages/flow/benchmark/harness/synth_stream.py rename packages/flow/openadapt_flow/learning/{synth_stream.py => induce.py} (64%) create mode 100644 packages/flow/openadapt_flow/reward/bundle.py rename packages/flow/{openadapt_flow/mockmed/teach_scope.py => tests/_teach_scope.py} (97%) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 9daecc8dc2..397dbd2c87 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -448,7 +448,7 @@ jobs: python ../../scripts/ci/locked_install.py --lock 'packages/flow[dev,capture,agent]' --lock packages/capture --editable packages/types --editable packages/capture --editable packages/flow - name: Install Harness² for the integration tests - # Not on PyPI. Pinned to the commit openadapt_flow.integrations.harness2 + # Not on PyPI. Pinned to the commit benchmark.harness.harness2 # is written against; it has no runtime dependencies. run: >- uv pip install --system --no-deps diff --git a/CAPABILITIES.md b/CAPABILITIES.md index 1f9d0d7c2d..ead5238109 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -285,7 +285,7 @@ improvising or retrying when something is off. | Task ledger per job: rules locked when the job opens and enforced on every run, item status set only from OpenAdapt's own results, and yes-or-no questions a person answers on the phone or in Desktop, including when a run stops | On main ([#1332](https://github.com/OpenAdaptAI/OpenAdapt/pull/1332)) | Synthetic (30-item MockMed worklist, scripted agent, no model calls) | MCP tools `open_job`, `get_job`, `add_items`, `record_evidence`, `ask_user`, `get_answer`, `change_constraint`, `close_job`; [Track a long job](packages/agent/README.md#track-a-long-job) | | Limit a teaching agent to the declared task on every teach path (`compile_agent`, `compile-agent`, teach over MCP, the airbag guard): typed text must be a declared value, links and browser requests to other origins are refused, writes are checked before they are sent, and the record is read back before a draft exists. Injected instructions in record notes, banners, tooltips, alt text and server responses are blocked or caught on MockMed through each path (scripted agent; a live-model mode is off by default). Self-repair refuses a control that leaves the app | On main ([#1359](https://github.com/OpenAdaptAI/OpenAdapt/pull/1359), [#1386](https://github.com/OpenAdaptAI/OpenAdapt/pull/1386)) | Synthetic | `openadapt_flow.teach_scope`, [THREAT_MODEL.md](packages/flow/docs/THREAT_MODEL.md) | | Compile an agent's first verified run from a prompt (`compile-agent`), and a code-as-action authoring agent | On main ([#1277](https://github.com/OpenAdaptAI/OpenAdapt/pull/1277), [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)); the agent harness behind it runs on every backend, on main ([#1283](https://github.com/OpenAdaptAI/OpenAdapt/pull/1283)) | Synthetic | `compile-agent` | -| Score harness optimizers (Harness², Meta-Harness) on GUI tasks with the record read-back as the reward, with full trace export | On main ([#1299](https://github.com/OpenAdaptAI/OpenAdapt/pull/1299), [#1300](https://github.com/OpenAdaptAI/OpenAdapt/pull/1300), [#1301](https://github.com/OpenAdaptAI/OpenAdapt/pull/1301)) | Synthetic (scripted proposer) | `integrations/harness2/` | +| Score harness optimizers (Harness², Meta-Harness) on GUI tasks with the record read-back as the reward, with full trace export | On main ([#1299](https://github.com/OpenAdaptAI/OpenAdapt/pull/1299), [#1300](https://github.com/OpenAdaptAI/OpenAdapt/pull/1300), [#1301](https://github.com/OpenAdaptAI/OpenAdapt/pull/1301)) | Synthetic (scripted proposer) | `benchmark/harness/harness2/` (source checkout) | | One scoreboard comparing the agent alone with OpenAdapt across tasks (correctness by read-back, cost, time, items per hour) | On main ([#1303](https://github.com/OpenAdaptAI/OpenAdapt/pull/1303)) | Synthetic and local OpenEMR and openIMIS test copies | `benchmark/scoreboard/` | | Pull a skill from the registry (`pull app/task@version`) and run it only after qualifying it on your own instance: `pull` marks it and expires any certification it carries, `skill bind` records your addresses and the names of the variables that hold credentials (never their values), `skill qualify` runs its bundled record check there, and `run`, `replay`, `emit-skill` and `emit-mcp` refuse it until then, and on any other instance. The index and its archives carry a keyless Sigstore attestation from the registry release workflow, checked offline against a pinned identity, and expire and can't be rolled back; a shipped record check runs no code and reads only your binding | On main | Unit tests | `openadapt-flow skill`, [SKILLS_REGISTRY.md](packages/flow/docs/design/SKILLS_REGISTRY.md) | diff --git a/llms-full.txt b/llms-full.txt index 46b17ec00a..affba2443a 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -682,7 +682,7 @@ improvising or retrying when something is off. | Task ledger per job: rules locked when the job opens and enforced on every run, item status set only from OpenAdapt's own results, and yes-or-no questions a person answers on the phone or in Desktop, including when a run stops | On main ([#1332](https://github.com/OpenAdaptAI/OpenAdapt/pull/1332)) | Synthetic (30-item MockMed worklist, scripted agent, no model calls) | MCP tools `open_job`, `get_job`, `add_items`, `record_evidence`, `ask_user`, `get_answer`, `change_constraint`, `close_job`; [Track a long job](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/agent/README.md#track-a-long-job) | | Limit a teaching agent to the declared task on every teach path (`compile_agent`, `compile-agent`, teach over MCP, the airbag guard): typed text must be a declared value, links and browser requests to other origins are refused, writes are checked before they are sent, and the record is read back before a draft exists. Injected instructions in record notes, banners, tooltips, alt text and server responses are blocked or caught on MockMed through each path (scripted agent; a live-model mode is off by default). Self-repair refuses a control that leaves the app | On main ([#1359](https://github.com/OpenAdaptAI/OpenAdapt/pull/1359), [#1386](https://github.com/OpenAdaptAI/OpenAdapt/pull/1386)) | Synthetic | `openadapt_flow.teach_scope`, [THREAT_MODEL.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/THREAT_MODEL.md) | | Compile an agent's first verified run from a prompt (`compile-agent`), and a code-as-action authoring agent | On main ([#1277](https://github.com/OpenAdaptAI/OpenAdapt/pull/1277), [#1305](https://github.com/OpenAdaptAI/OpenAdapt/pull/1305)); the agent harness behind it runs on every backend, on main ([#1283](https://github.com/OpenAdaptAI/OpenAdapt/pull/1283)) | Synthetic | `compile-agent` | -| Score harness optimizers (Harness², Meta-Harness) on GUI tasks with the record read-back as the reward, with full trace export | On main ([#1299](https://github.com/OpenAdaptAI/OpenAdapt/pull/1299), [#1300](https://github.com/OpenAdaptAI/OpenAdapt/pull/1300), [#1301](https://github.com/OpenAdaptAI/OpenAdapt/pull/1301)) | Synthetic (scripted proposer) | `integrations/harness2/` | +| Score harness optimizers (Harness², Meta-Harness) on GUI tasks with the record read-back as the reward, with full trace export | On main ([#1299](https://github.com/OpenAdaptAI/OpenAdapt/pull/1299), [#1300](https://github.com/OpenAdaptAI/OpenAdapt/pull/1300), [#1301](https://github.com/OpenAdaptAI/OpenAdapt/pull/1301)) | Synthetic (scripted proposer) | `benchmark/harness/harness2/` (source checkout) | | One scoreboard comparing the agent alone with OpenAdapt across tasks (correctness by read-back, cost, time, items per hour) | On main ([#1303](https://github.com/OpenAdaptAI/OpenAdapt/pull/1303)) | Synthetic and local OpenEMR and openIMIS test copies | `benchmark/scoreboard/` | | Pull a skill from the registry (`pull app/task@version`) and run it only after qualifying it on your own instance: `pull` marks it and expires any certification it carries, `skill bind` records your addresses and the names of the variables that hold credentials (never their values), `skill qualify` runs its bundled record check there, and `run`, `replay`, `emit-skill` and `emit-mcp` refuse it until then, and on any other instance. The index and its archives carry a keyless Sigstore attestation from the registry release workflow, checked offline against a pinned identity, and expire and can't be rolled back; a shipped record check runs no code and reads only your binding | On main | Unit tests | `openadapt-flow skill`, [SKILLS_REGISTRY.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/design/SKILLS_REGISTRY.md) | diff --git a/packages/agent/README.md b/packages/agent/README.md index 3b65776b02..11dd2ff83e 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -498,7 +498,7 @@ A scripted agent worked a 30-item synthetic MockMed worklist. The user changes a In the middle arm, the agent opens a job and uses each item's `request_id` but keeps its flaws. The ledger stops every rule break, but it can't stop an agent from guessing a value or misreporting its own tally. `close_job` still lists what was verified, so a caller can check the agent's report. The scripted agent stands for a policy, not a model, so these numbers test the mechanism. They aren't a model benchmark. To reproduce them (about 3 seconds): ```bash -python -m openadapt_flow.agent.ledger_demo +python packages/agent/tests/ledger_demo.py ``` Results: [docs/task-ledger-measurement.json](docs/task-ledger-measurement.json). diff --git a/packages/flow/openadapt_flow/agent/ledger_demo.py b/packages/agent/tests/ledger_demo.py similarity index 99% rename from packages/flow/openadapt_flow/agent/ledger_demo.py rename to packages/agent/tests/ledger_demo.py index 8ba2f1fe81..2f7c45c10a 100644 --- a/packages/flow/openadapt_flow/agent/ledger_demo.py +++ b/packages/agent/tests/ledger_demo.py @@ -1,8 +1,9 @@ """A scripted agent works a 30-item synthetic MockMed worklist, with and without the task ledger. -Run it:: +It lives beside the test that uses it (``test_ledger.py``). Run it from the +repository root:: - python -m openadapt_flow.agent.ledger_demo [--out results.json] + python packages/agent/tests/ledger_demo.py [--out results.json] Everything is synthetic and local: the referrals, the patients, the app (an in-memory record store behind the real ``run_workflow`` service) and the diff --git a/packages/agent/tests/test_guard_tools.py b/packages/agent/tests/test_guard_tools.py index 6d10a04748..2d7064cff9 100644 --- a/packages/agent/tests/test_guard_tools.py +++ b/packages/agent/tests/test_guard_tools.py @@ -4,6 +4,8 @@ import base64 import json +import sys +from pathlib import Path import anyio import pytest @@ -13,15 +15,19 @@ from openadapt_flow.approval_mode.demo_approver import demo_approver, demo_root from openadapt_flow.agent.mcp import build_server from openadapt_flow.guard import GUARD_TASK_SCHEMA, GuardedSave -from openadapt_flow.guard_demo import ( +from openadapt_flow.mockmed.fault_server import serve as fault_serve +from openadapt_flow.verify_run import parse_verify_config + +# The fake agent and its helpers are Flow's guarded-save demo, which lives in +# Flow's source checkout (packages/flow/benchmark/harness), outside the wheel. +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "flow")) +from benchmark.harness.guard_demo import ( # noqa: E402 NOTE, FakeAgent, person_approves, tiny_png, verify_yaml, ) -from openadapt_flow.mockmed.fault_server import serve as fault_serve -from openadapt_flow.verify_run import parse_verify_config @pytest.fixture diff --git a/packages/agent/tests/test_ledger.py b/packages/agent/tests/test_ledger.py index bfd588d543..c8b4260f1d 100644 --- a/packages/agent/tests/test_ledger.py +++ b/packages/agent/tests/test_ledger.py @@ -7,6 +7,8 @@ import anyio import pytest +# The scripted 30-referral simulation is test data; it lives beside this file. +from ledger_demo import WORKFLOW, SyntheticReferralApp, measure from openadapt_flow.agent.bridge import AgentBridge, BridgeError from openadapt_flow.agent.cli import main from openadapt_flow.agent.ledger import ( @@ -14,7 +16,6 @@ LEDGER_TOOLS, TaskLedger, ) -from openadapt_flow.agent.ledger_demo import WORKFLOW, SyntheticReferralApp, measure from openadapt_flow.agent.mcp import build_server from openadapt_flow.agent.runner import RunnerConfig from openadapt_flow.approval_mode.demo_approver import demo_approver, demo_root diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md index 978a0b337a..6a5b595ddc 100644 --- a/packages/flow/CHANGELOG.md +++ b/packages/flow/CHANGELOG.md @@ -27,6 +27,8 @@ - The wheel no longer ships the benchmark harnesses. `openadapt_flow.benchmark` and `openadapt_flow.instrument` moved to `packages/flow/benchmark/harness/` in the source tree (import `benchmark.harness`); `openadapt-flow benchmark` runs from a source checkout and says so from a wheel. The engine audit for application-specific code is in `docs/GENERALITY_AUDIT.md`. +- The wheel no longer ships test fixtures and demo harnesses: the Harness² integration, the MockLoan app and its recorder, `guard_demo`, the MockMed teach-scope adapter, the synthetic drift stream (the `StructuralDiffInducer` it used now lives in `openadapt_flow.learning.induce`) and the task-ledger simulation moved to `benchmark/harness/` or beside their tests, the reward bundle helpers moved to `openadapt_flow.reward.bundle`, and the `harness2` extra is gone. + - Self-repair can take a stop where the app's own dialog covers the next step's identity band: when the backend proves the covering window belongs to the window the last verified step was in, and the part of the band read still matches the record, the dialog is dismissed after the previous click and the step re-checks its identity in full. Another record behind the dialog stays an identity halt. The runtime lets a backend remember a verified record's window (`record_verified_window`); macOS answers `owned_window_over` from the window's proved sheet ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). - macOS: a run can now act in the pinned window's own AppKit sheet (an `NSAlert` attached to it), which is a separate window. `MacOSBackend` accepts it only when AX lists exactly one `AXSheet` on the window and exactly one window of the same process has its bounds inside it; the capture composites exactly that sheet over the window, and focus, pointer and click-target checks accept it only inside its bounds. Any other window is still refused ([#1711](https://github.com/OpenAdaptAI/OpenAdapt/issues/1711)). diff --git a/packages/flow/openadapt_flow/guard_demo.py b/packages/flow/benchmark/harness/guard_demo.py similarity index 96% rename from packages/flow/openadapt_flow/guard_demo.py rename to packages/flow/benchmark/harness/guard_demo.py index a7578d1bb3..4d1fb788d1 100644 --- a/packages/flow/openadapt_flow/guard_demo.py +++ b/packages/flow/benchmark/harness/guard_demo.py @@ -1,9 +1,9 @@ """Guarded-save demo: a fake agent saves a referral to MockMed through the guard. -Run it with ``python -m openadapt_flow.guard_demo``. It starts the MockMed -fault server on localhost, and for each case a fake agent opens a chart page, -asks the guard, and posts the referral to MockMed's API the way a -computer-use agent's click on Save would. The guard reads ``GET /api/db`` +Run it from ``packages/flow`` with ``python -m benchmark.harness.guard_demo``. +It starts the MockMed fault server on localhost, and for each case a fake +agent opens a chart page, asks the guard, and posts the referral to MockMed's +API the way a computer-use agent's click on Save would. The guard reads ``GET /api/db`` (the system of record) to judge the write. All data is synthetic, nothing leaves localhost, and no model is called. diff --git a/packages/flow/openadapt_flow/integrations/harness2/__init__.py b/packages/flow/benchmark/harness/harness2/__init__.py similarity index 100% rename from packages/flow/openadapt_flow/integrations/harness2/__init__.py rename to packages/flow/benchmark/harness/harness2/__init__.py diff --git a/packages/flow/openadapt_flow/integrations/harness2/adapter.py b/packages/flow/benchmark/harness/harness2/adapter.py similarity index 96% rename from packages/flow/openadapt_flow/integrations/harness2/adapter.py rename to packages/flow/benchmark/harness/harness2/adapter.py index ba92686434..a6d9c456a3 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/adapter.py +++ b/packages/flow/benchmark/harness/harness2/adapter.py @@ -7,8 +7,9 @@ only measured afterwards. This adapter gives it one: every rollout is scored by the reward worker reading the system of record of a fresh app copy. -This module imports Harness²; it copies none of it. Install it beside -``openadapt-flow[harness2]`` (it is not on PyPI):: +This module imports Harness²; it copies none of it. It lives in the source +checkout (run from ``packages/flow``). Install Harness² beside +``openadapt-flow[browser]`` (it is not on PyPI):: pip install 'harness2 @ git+https://github.com/google-research/harness2@1cfe254e8d25b15f37048c3730134686abe60335' @@ -19,7 +20,7 @@ This is Harness² as published, with no scorer. * ``readback``: each trajectory and report also carries what the system of record shows after the run (which records were added, whose they are, the - outcome). With :func:`~openadapt_flow.integrations.harness2.selection.reward_selection` + outcome). With :func:`~benchmark.harness.harness2.selection.reward_selection` the final is chosen by the read-back reward instead of a model. In both modes ``score()`` is the read-back. Harness² calls it only after @@ -34,17 +35,17 @@ from harness2.core.spec import BenchPrompts, BenchSpec, ChannelSpec from harness2.core.tree import SurfaceSpec, read_tree, write_tree -from openadapt_flow.integrations.harness2.agent import GuiAgent -from openadapt_flow.integrations.harness2.apps import AppTarget, default_targets -from openadapt_flow.integrations.harness2.rollout import ( +from benchmark.harness.harness2.agent import GuiAgent +from benchmark.harness.harness2.apps import AppTarget, default_targets +from benchmark.harness.harness2.rollout import ( REPORT_FILE, load_run, load_score, load_steps, run_rollout, ) -from openadapt_flow.integrations.harness2.scorer import EpisodeScore, Penalties -from openadapt_flow.integrations.harness2.tasks import MOCKMED_PATIENTS, GuiTask +from benchmark.harness.harness2.scorer import EpisodeScore, Penalties +from benchmark.harness.harness2.tasks import MOCKMED_PATIENTS, GuiTask EVIDENCE_MODES = ("screen", "readback") @@ -307,7 +308,7 @@ def trajectory(self, run_dir: Path, *, obs_trunc: int) -> str: if self.evidence == "readback" and run.get("task_id") in self._tasks: # OpenAdapt's identity check at each write: was the record on # screen the requested one? Part of the read-back evidence. - from openadapt_flow.integrations.harness2.traces import from_rollout + from benchmark.harness.harness2.traces import from_rollout trace = from_rollout(run_dir, self._tasks[run["task_id"]]) identity = {s.index: s.identity for s in trace.steps if s.identity} diff --git a/packages/flow/openadapt_flow/integrations/harness2/agent.py b/packages/flow/benchmark/harness/harness2/agent.py similarity index 99% rename from packages/flow/openadapt_flow/integrations/harness2/agent.py rename to packages/flow/benchmark/harness/harness2/agent.py index 3894249ba6..68fe348227 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/agent.py +++ b/packages/flow/benchmark/harness/harness2/agent.py @@ -27,7 +27,7 @@ from dataclasses import asdict, dataclass, field from typing import Any, Optional, Protocol, runtime_checkable -from openadapt_flow.integrations.harness2.llm import ChatModel +from benchmark.harness.harness2.llm import ChatModel #: The final report statuses and the runtime signal each maps to. STATUS_SIGNAL = {"saved": "completed", "stopped": "halted_before_effect"} diff --git a/packages/flow/openadapt_flow/integrations/harness2/apps.py b/packages/flow/benchmark/harness/harness2/apps.py similarity index 97% rename from packages/flow/openadapt_flow/integrations/harness2/apps.py rename to packages/flow/benchmark/harness/harness2/apps.py index 4f82c27e00..05a1867927 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/apps.py +++ b/packages/flow/benchmark/harness/harness2/apps.py @@ -15,7 +15,7 @@ from typing import Any, Callable, Optional, Protocol, runtime_checkable from urllib.parse import urlencode -from openadapt_flow.integrations.harness2.tasks import MOCKMED_PATIENTS, GuiTask +from benchmark.harness.harness2.tasks import MOCKMED_PATIENTS, GuiTask @runtime_checkable diff --git a/packages/flow/openadapt_flow/integrations/harness2/llm.py b/packages/flow/benchmark/harness/harness2/llm.py similarity index 100% rename from packages/flow/openadapt_flow/integrations/harness2/llm.py rename to packages/flow/benchmark/harness/harness2/llm.py diff --git a/packages/flow/openadapt_flow/integrations/harness2/pixels.py b/packages/flow/benchmark/harness/harness2/pixels.py similarity index 98% rename from packages/flow/openadapt_flow/integrations/harness2/pixels.py rename to packages/flow/benchmark/harness/harness2/pixels.py index af0ea96df2..0d6d9ea6c1 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/pixels.py +++ b/packages/flow/benchmark/harness/harness2/pixels.py @@ -26,7 +26,7 @@ import time from typing import Any, Callable, Optional -from openadapt_flow.integrations.harness2.agent import Element, Observation +from benchmark.harness.harness2.agent import Element, Observation #: Seconds between frames while waiting for the screen to stop changing. _FRAME_GAP_S = 0.25 diff --git a/packages/flow/openadapt_flow/integrations/harness2/proposers.py b/packages/flow/benchmark/harness/harness2/proposers.py similarity index 98% rename from packages/flow/openadapt_flow/integrations/harness2/proposers.py rename to packages/flow/benchmark/harness/harness2/proposers.py index 26840b7002..f08fae6352 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/proposers.py +++ b/packages/flow/benchmark/harness/harness2/proposers.py @@ -21,12 +21,12 @@ from harness2.core.proposer import Proposer -from openadapt_flow.integrations.harness2.agent import ( +from benchmark.harness.harness2.agent import ( RULE_IDENTITY, RULE_SAVE_ONCE, RULE_TYPE_LABEL, ) -from openadapt_flow.integrations.harness2.llm import ChatModel +from benchmark.harness.harness2.llm import ChatModel _CHOICE_HINT = "```choice" diff --git a/packages/flow/openadapt_flow/integrations/harness2/rollout.py b/packages/flow/benchmark/harness/harness2/rollout.py similarity index 96% rename from packages/flow/openadapt_flow/integrations/harness2/rollout.py rename to packages/flow/benchmark/harness/harness2/rollout.py index c404a95717..4b85761492 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/rollout.py +++ b/packages/flow/benchmark/harness/harness2/rollout.py @@ -27,14 +27,14 @@ from pathlib import Path from typing import Any, Callable, Optional -from openadapt_flow.integrations.harness2.agent import AgentResult, GuiAgent, PageDriver -from openadapt_flow.integrations.harness2.apps import AppTarget -from openadapt_flow.integrations.harness2.scorer import ( +from benchmark.harness.harness2.agent import AgentResult, GuiAgent, PageDriver +from benchmark.harness.harness2.apps import AppTarget +from benchmark.harness.harness2.scorer import ( EpisodeScore, Penalties, ReadBackScorer, ) -from openadapt_flow.integrations.harness2.tasks import GuiTask +from benchmark.harness.harness2.tasks import GuiTask RUN_FILE = "run.json" STEPS_FILE = "steps.jsonl" diff --git a/packages/flow/openadapt_flow/integrations/harness2/scorer.py b/packages/flow/benchmark/harness/harness2/scorer.py similarity index 98% rename from packages/flow/openadapt_flow/integrations/harness2/scorer.py rename to packages/flow/benchmark/harness/harness2/scorer.py index 7c08719030..ce86a38157 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/scorer.py +++ b/packages/flow/benchmark/harness/harness2/scorer.py @@ -32,8 +32,8 @@ from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey -from openadapt_flow.integrations.harness2.tasks import GuiTask -from openadapt_flow.reward.seed import write_bundle +from benchmark.harness.harness2.tasks import GuiTask +from openadapt_flow.reward.bundle import write_bundle from openadapt_flow.reward.worker import RewardWorker #: Runtime signals an agent's final report can map to. diff --git a/packages/flow/openadapt_flow/integrations/harness2/selection.py b/packages/flow/benchmark/harness/harness2/selection.py similarity index 98% rename from packages/flow/openadapt_flow/integrations/harness2/selection.py rename to packages/flow/benchmark/harness/harness2/selection.py index f98d0e5397..d2ee73597e 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/selection.py +++ b/packages/flow/benchmark/harness/harness2/selection.py @@ -24,8 +24,8 @@ from harness2.core.proposer import Proposer from harness2.core.spec import RunView -from openadapt_flow.integrations.harness2 import traces -from openadapt_flow.integrations.harness2.adapter import OpenAdaptGuiAdapter +from benchmark.harness.harness2 import traces +from benchmark.harness.harness2.adapter import OpenAdaptGuiAdapter #: Where an unscored candidate ranks: below a safe stop, above any penalty. _UNSCORED_RANK = -0.01 diff --git a/packages/flow/openadapt_flow/integrations/harness2/tasks.py b/packages/flow/benchmark/harness/harness2/tasks.py similarity index 100% rename from packages/flow/openadapt_flow/integrations/harness2/tasks.py rename to packages/flow/benchmark/harness/harness2/tasks.py diff --git a/packages/flow/openadapt_flow/integrations/harness2/traces.py b/packages/flow/benchmark/harness/harness2/traces.py similarity index 98% rename from packages/flow/openadapt_flow/integrations/harness2/traces.py rename to packages/flow/benchmark/harness/harness2/traces.py index 81eb053394..0d5fbb7b35 100644 --- a/packages/flow/openadapt_flow/integrations/harness2/traces.py +++ b/packages/flow/benchmark/harness/harness2/traces.py @@ -39,14 +39,14 @@ from pathlib import Path from typing import Any, Iterable, Optional, Sequence -from openadapt_flow import privacy -from openadapt_flow.integrations.harness2.rollout import ( +from benchmark.harness.harness2.rollout import ( load_run, load_score, load_steps, ) -from openadapt_flow.integrations.harness2.scorer import EpisodeScore -from openadapt_flow.integrations.harness2.tasks import GuiTask +from benchmark.harness.harness2.scorer import EpisodeScore +from benchmark.harness.harness2.tasks import GuiTask +from openadapt_flow import privacy SCHEMA = "openadapt.trace-export/v1" diff --git a/packages/flow/openadapt_flow/mockloan/__init__.py b/packages/flow/benchmark/harness/mockloan/__init__.py similarity index 89% rename from packages/flow/openadapt_flow/mockloan/__init__.py rename to packages/flow/benchmark/harness/mockloan/__init__.py index 1279fb3536..a5e02f767a 100644 --- a/packages/flow/openadapt_flow/mockloan/__init__.py +++ b/packages/flow/benchmark/harness/mockloan/__init__.py @@ -8,6 +8,6 @@ transactional fault injection via ``?fault=`` (see ``fault_server``). """ -from openadapt_flow.mockloan.server import serve # noqa: F401 +from benchmark.harness.mockloan.server import serve # noqa: F401 __all__ = ["serve"] diff --git a/packages/flow/benchmark/harness/mockloan/demo.py b/packages/flow/benchmark/harness/mockloan/demo.py new file mode 100644 index 0000000000..e56fc280dd --- /dev/null +++ b/packages/flow/benchmark/harness/mockloan/demo.py @@ -0,0 +1,97 @@ +"""Record the canonical MockLoan disbursement demonstration. + +Record time is allowed to cheat with Playwright locators (to find pixel +coordinates via ``bounding_box()``), but every action is performed through +`Recorder` so frames and events are captured exactly as a human demo would +be. Replay never uses selectors. + +The non-healthcare companion to +:func:`openadapt_flow.demo_driver.record_triage_demo`; both drive the same +governed record -> compile -> replay path. +""" + +from __future__ import annotations + +from pathlib import Path + +from openadapt_flow.backends.playwright_backend import PlaywrightBackend +from openadapt_flow.recorder import Recorder + +__all__ = ["record_disbursement_demo"] + + +def record_disbursement_demo( + url: str, + out_dir: Path | str, + *, + memo_text: str, + param_name: str = "memo", + headed: bool = False, + record_video_dir: Path | str | None = None, +) -> Path: + """Record the canonical MockLoan disbursement demo against a running app. + + The non-healthcare companion to :func:`record_triage_demo`. Flow: + login -> pipeline -> Open first loan -> New Disbursement -> Personal product + -> click Funding memo field -> type memo (as a parameter) -> Authorize + Disbursement. The consequential final step authorizes moving money to a + borrower, the lending analog of clinically saving an encounter. + + Args: + url: MockLoan base URL (from `benchmark.harness.mockloan.server.serve`). + out_dir: Directory to write the recording into. + memo_text: The funding memo typed during the demo (recorded as a param). + param_name: Parameter name for the typed memo (default ``"memo"``). + headed: Run the browser headed (visible) instead of headless. + record_video_dir: OPT-IN (default ``None`` = off). When set, a WebM + video of the recording session is captured into this directory; it + does not change what is recorded to ``out_dir``. + + Returns: + The recording directory (contains meta.json, events.jsonl, frames/). + """ + backend, close = PlaywrightBackend.launch( + url, + headless=not headed, + record_video_dir=( + str(record_video_dir) if record_video_dir is not None else None + ), + ) + try: + page = backend.page + recorder = Recorder(backend, out_dir, app_url=url) + + def center(selector: str) -> tuple[int, int]: + """Center pixel coordinates of the first element matching CSS.""" + locator = page.locator(selector).first + locator.wait_for(state="visible") + box = locator.bounding_box() + if box is None: # pragma: no cover - visible => box exists + raise RuntimeError(f"no bounding box for {selector!r}") + return ( + int(box["x"] + box["width"] / 2), + int(box["y"] + box["height"] / 2), + ) + + # Login + recorder.click(*center("#username")) + recorder.type_text("officer.demo") + recorder.click(*center("#password")) + recorder.type_text("mockloan-demo-pass") + recorder.click(*center("#signin")) + + # Pipeline -> open first loan + recorder.click(*center(".open-btn")) + + # Loan -> new disbursement + recorder.click(*center("#new-disbursement")) + + # Disbursement: product chooser, memo (parameterized), authorize + recorder.click(*center("#product-personal")) + recorder.click(*center("#memo")) + recorder.type_text(memo_text, param=param_name) + recorder.click(*center("#authorize")) + + return recorder.finish() + finally: + close() diff --git a/packages/flow/openadapt_flow/mockloan/fault_server.py b/packages/flow/benchmark/harness/mockloan/fault_server.py similarity index 100% rename from packages/flow/openadapt_flow/mockloan/fault_server.py rename to packages/flow/benchmark/harness/mockloan/fault_server.py diff --git a/packages/flow/openadapt_flow/mockloan/server.py b/packages/flow/benchmark/harness/mockloan/server.py similarity index 95% rename from packages/flow/openadapt_flow/mockloan/server.py rename to packages/flow/benchmark/harness/mockloan/server.py index 0bc09c6603..f20c68e5b4 100644 --- a/packages/flow/openadapt_flow/mockloan/server.py +++ b/packages/flow/benchmark/harness/mockloan/server.py @@ -1,6 +1,6 @@ """Threaded static-file server for the MockLoan demo app. -Serves ``openadapt_flow/mockloan/static`` on localhost. No external resources +Serves ``benchmark/harness/mockloan/static`` on localhost. No external resources are referenced by the app, so tests never touch the network beyond localhost. """ diff --git a/packages/flow/openadapt_flow/mockloan/static/app.js b/packages/flow/benchmark/harness/mockloan/static/app.js similarity index 99% rename from packages/flow/openadapt_flow/mockloan/static/app.js rename to packages/flow/benchmark/harness/mockloan/static/app.js index ed305c1714..7f5bce22c9 100644 --- a/packages/flow/openadapt_flow/mockloan/static/app.js +++ b/packages/flow/benchmark/harness/mockloan/static/app.js @@ -29,7 +29,7 @@ var LABEL_OPEN = DRIFT.has('rename') ? 'View' : 'Open'; // Transactional fault-injection hook (flag-gated, exactly like ?drift=). // When the page is loaded with ?fault= the Authorize handler routes the -// write through a REAL backend API (openadapt_flow/mockloan/fault_server.py) +// write through a REAL backend API (benchmark/harness/mockloan/fault_server.py) // so the fault-model study can judge a replay against DB ground truth rather // than the on-screen banner. With no ?fault query this is inert and Authorize // behaves byte-for-byte as before (the normal benchmark is unaffected). diff --git a/packages/flow/openadapt_flow/mockloan/static/index.html b/packages/flow/benchmark/harness/mockloan/static/index.html similarity index 100% rename from packages/flow/openadapt_flow/mockloan/static/index.html rename to packages/flow/benchmark/harness/mockloan/static/index.html diff --git a/packages/flow/openadapt_flow/mockloan/static/styles.css b/packages/flow/benchmark/harness/mockloan/static/styles.css similarity index 100% rename from packages/flow/openadapt_flow/mockloan/static/styles.css rename to packages/flow/benchmark/harness/mockloan/static/styles.css diff --git a/packages/flow/benchmark/harness/synth_stream.py b/packages/flow/benchmark/harness/synth_stream.py new file mode 100644 index 0000000000..a516971965 --- /dev/null +++ b/packages/flow/benchmark/harness/synth_stream.py @@ -0,0 +1,222 @@ +"""Synthetic trace-stream harness: a MockMed skill drifting over "time". + +To exercise the learn/promote loop end-to-end without a live app, this module + +1. builds a base MockMed skill program (``add-patient-note`` from + ``docs/design/WORKFLOW_PROGRAM_IR.md`` section 2.5: sign in -> open the + patient record -> new encounter -> type note -> SAVE), and +2. emits a STREAM of :class:`~openadapt_flow.learning.trace.ExecutionTrace`s over + a logical time axis with INJECTED DRIFT at a chosen trace index -- e.g. a new + consent dialog starts appearing mid-stream, so the active program stops + reproducing the new successful traces and the loop must detect the novelty, + induce a revision, validate it, and promote it. + +The reference inducer that revises the program, +:class:`~openadapt_flow.learning.induce.StructuralDiffInducer`, ships in the +engine; this stream is a test and benchmark fixture and lives in the source +checkout (``packages/flow/benchmark/harness``), outside the wheel. + +All deterministic, ``$0``, no model calls. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Optional + +from openadapt_flow.ir import ( + ActionKind, + Guard, + ProgramGraph, + State, + StateKind, +) +from openadapt_flow.learning.induce import StructuralDiffInducer +from openadapt_flow.learning.induce import _action_state as _induce_action_state +from openadapt_flow.learning.trace import ExecutionTrace, TraceStep +from openadapt_flow.runtime.effects.effect import Effect, EffectKind + +__all__ = [ + "Drift", + "StructuralDiffInducer", + "generate_stream", + "mockmed_base_program", +] + +# -- the MockMed skill's canonical intents ------------------------------------ + +INTENT_LOGIN = "Sign in" +INTENT_OPEN = "Open patient record" +INTENT_ENCOUNTER = "Start new encounter" +INTENT_NOTE = "Enter note" +INTENT_SAVE = "Save encounter" +INTENT_CONSENT = "Acknowledge consent notice" + +#: The screen fact that co-occurs with the injected consent dialog -- the text a +#: ``TEXT_PRESENT`` guard keys on. +FACT_CONSENT = "Consent Required" + +#: A representative identity band for the patient row (name + DOB) -- the +#: evidence the pre-click identity gate keys on; makes the "open patient" step +#: identity-ARMED so an identity regression is meaningful. +PATIENT_BAND = "Belford, Phil DOB 1990-01-01" + + +def _action_state( + sid: str, + intent: str, + target: str, + *, + action: ActionKind = ActionKind.CLICK, + armed: bool = False, + risk: str = "reversible", + effects: Optional[list[Effect]] = None, + guard: Optional[Guard] = None, + text: Optional[str] = None, +) -> State: + return _induce_action_state( + sid, + intent, + target, + action=action, + armed=armed, + risk=risk, + effects=effects, + guard=guard, + text=text, + context_text=PATIENT_BAND, + ) + + +def mockmed_base_program() -> ProgramGraph: + """The base ``add-patient-note`` program (no consent dialog). + + A straight-line state machine: sign in -> open patient (identity-armed) -> + new encounter -> type note -> SAVE (irreversible, with a ``record_written`` + system-of-record effect) -> success. The identity band + effect + risk on + the sensitive steps are what a rigged revision could weaken, and what the + regression gate protects. + """ + save_effect = Effect( + kind=EffectKind.RECORD_WRITTEN, + match={"type": "encounter", "patient": "p1"}, + expected_count=1, + risk="irreversible", + probe="encounter record written for patient", + ) + states = { + "s_login": _action_state("s_login", INTENT_LOGIN, "s_open"), + "s_open": _action_state("s_open", INTENT_OPEN, "s_encounter", armed=True), + "s_encounter": _action_state("s_encounter", INTENT_ENCOUNTER, "s_note"), + "s_note": _action_state( + "s_note", INTENT_NOTE, "s_save", action=ActionKind.TYPE, text="Follow-up" + ), + "s_save": _action_state( + "s_save", + INTENT_SAVE, + "__end__", + risk="irreversible", + effects=[save_effect], + ), + "__end__": State(id="__end__", kind=StateKind.TERMINAL, outcome="success"), + } + return ProgramGraph(entry="s_login", states=states) + + +# -- the drift stream --------------------------------------------------------- + + +class Drift(str, Enum): + """The injected drift/variant a stream can carry from trace K onward.""" + + NONE = "none" + #: A new consent dialog appears after sign-in (a new OPTIONAL step + branch). + CONSENT_DIALOG = "consent_dialog" + #: The note field is renamed (the observed intent changes) -- a structural + #: change the additive reference inducer CANNOT cover (used to show the loop + #: correctly refuses an inadequate revision). + FIELD_RENAME = "field_rename" + + +def _baseline_steps() -> list[TraceStep]: + return [ + TraceStep(intent=INTENT_LOGIN), + TraceStep(intent=INTENT_OPEN, identity=PATIENT_BAND), + TraceStep(intent=INTENT_ENCOUNTER), + TraceStep(intent=INTENT_NOTE, action=ActionKind.TYPE), + TraceStep(intent=INTENT_SAVE), + ] + + +def _drifted_steps(drift: Drift) -> tuple[list[TraceStep], dict[str, bool]]: + steps = _baseline_steps() + facts: dict[str, bool] = {} + if drift is Drift.CONSENT_DIALOG: + # The consent dialog appears right after sign-in. + steps.insert(1, TraceStep(intent=INTENT_CONSENT)) + facts[FACT_CONSENT] = True + elif drift is Drift.FIELD_RENAME: + for s in steps: + if s.intent == INTENT_NOTE: + s.intent = "Enter clinical note" + return steps, facts + + +def generate_stream( + *, + n_baseline: int = 6, + n_drift: int = 6, + drift: Drift = Drift.CONSENT_DIALOG, + n_failures: int = 0, + prefix: str = "mockmed", +) -> list[ExecutionTrace]: + """Emit a deterministic stream of traces with drift injected at trace + ``n_baseline``. + + The first ``n_baseline`` traces are clean successful runs of the base skill; + from index ``n_baseline`` the injected ``drift`` appears in every successful + run. ``n_failures`` failure traces (noise) are appended -- runs that halted + partway with no novel successful structure, so a batch of them alone must NOT + trigger a revision. + """ + traces: list[ExecutionTrace] = [] + t = 0 + for _ in range(n_baseline): + traces.append( + ExecutionTrace( + trace_id=f"{prefix}-{t:04d}", + t=t, + outcome="success", + steps=_baseline_steps(), + params={"encounter_type": "Triage"}, + ) + ) + t += 1 + for _ in range(n_drift): + steps, facts = _drifted_steps(drift) + traces.append( + ExecutionTrace( + trace_id=f"{prefix}-{t:04d}", + t=t, + outcome="success", + steps=steps, + facts=facts, + params={"encounter_type": "Triage"}, + ) + ) + t += 1 + for _ in range(n_failures): + # A run that halted after opening the patient (e.g. a transient + # resolution failure): truncated, no novel successful structure. + traces.append( + ExecutionTrace( + trace_id=f"{prefix}-{t:04d}", + t=t, + outcome="failure", + steps=_baseline_steps()[:2], + failure_reason="transient resolution failure on 'Start new encounter'", + params={"encounter_type": "Triage"}, + ) + ) + t += 1 + return traces diff --git a/packages/flow/benchmark/harness2_scorer/RESULTS.md b/packages/flow/benchmark/harness2_scorer/RESULTS.md index eff54f7ebb..975843b034 100644 --- a/packages/flow/benchmark/harness2_scorer/RESULTS.md +++ b/packages/flow/benchmark/harness2_scorer/RESULTS.md @@ -16,7 +16,7 @@ reasons under [What went wrong](#what-went-wrong). ## Setup - **Tasks:** the seven MockMed variants in - `openadapt_flow.integrations.harness2.tasks`: `plain`, `lookalike`, `sort`, + `benchmark.harness.harness2.tasks`: `plain`, `lookalike`, `sort`, `typelabel`, `timeout`, `missing`, `slow`. Every task asks for a Triage encounter for Jane Sample (born 1980-01-01) with a fixed note. - **Agent:** `ModelGuiAgent` on `gpt-4.1-nano`, reading the DOM text and diff --git a/packages/flow/benchmark/harness2_scorer/run.py b/packages/flow/benchmark/harness2_scorer/run.py index fd7d17cd6d..e96f0dff64 100644 --- a/packages/flow/benchmark/harness2_scorer/run.py +++ b/packages/flow/benchmark/harness2_scorer/run.py @@ -34,21 +34,21 @@ from harness2.core.spec import ChannelSpec from harness2.core.tree import write_tree -from openadapt_flow import privacy -from openadapt_flow.integrations.harness2.adapter import ( +from benchmark.harness.harness2.adapter import ( GUI_SURFACE, OpenAdaptGuiAdapter, ) -from openadapt_flow.integrations.harness2.agent import ModelGuiAgent -from openadapt_flow.integrations.harness2.llm import ( +from benchmark.harness.harness2.agent import ModelGuiAgent +from benchmark.harness.harness2.llm import ( BudgetExceeded, CostLedger, HttpChatModel, Price, ) -from openadapt_flow.integrations.harness2.proposers import ModelProposer -from openadapt_flow.integrations.harness2.selection import run_task -from openadapt_flow.integrations.harness2.tasks import mockmed_tasks +from benchmark.harness.harness2.proposers import ModelProposer +from benchmark.harness.harness2.selection import run_task +from benchmark.harness.harness2.tasks import mockmed_tasks +from openadapt_flow import privacy HERE = Path(__file__).resolve().parent diff --git a/packages/flow/benchmark/lending_fault_model/LENDING_FAULT_MODEL.md b/packages/flow/benchmark/lending_fault_model/LENDING_FAULT_MODEL.md index 8ec43c4fe9..9aa741b969 100644 --- a/packages/flow/benchmark/lending_fault_model/LENDING_FAULT_MODEL.md +++ b/packages/flow/benchmark/lending_fault_model/LENDING_FAULT_MODEL.md @@ -10,7 +10,7 @@ Model calls: **0** (compiled replay; `ANTHROPIC_API_KEY` unset, grounder rung ne ## Method -The bundled MockLoan app is a client-side SPA with no backend, so an "authorize" only mutates in-page state - the UI *is* the source of truth. This study adds a real persistence boundary (`openadapt_flow/mockloan/fault_server.py`): a flag-gated `?fault=` hook in the app (mirroring the `?drift=` hooks) routes the Authorize write through a backend API with an isolated SQLite ledger. **With no `?fault` query the app never calls the API and the normal benchmark is byte-for-byte unaffected** (pinned by a test). Each fault class is injected at that boundary; the SAME recorded bundle is replayed through the REAL `Replayer` against it, and the outcome is judged by `GET /api/db` (ground truth) plus whether the replay halted - never by the replay's self-report. `/api/db` is a read path the SPA itself never calls, so the oracle cannot be gamed by the screen. +The bundled MockLoan app is a client-side SPA with no backend, so an "authorize" only mutates in-page state - the UI *is* the source of truth. This study adds a real persistence boundary (`benchmark/harness/mockloan/fault_server.py`): a flag-gated `?fault=` hook in the app (mirroring the `?drift=` hooks) routes the Authorize write through a backend API with an isolated SQLite ledger. **With no `?fault` query the app never calls the API and the normal benchmark is byte-for-byte unaffected** (pinned by a test). Each fault class is injected at that boundary; the SAME recorded bundle is replayed through the REAL `Replayer` against it, and the outcome is judged by `GET /api/db` (ground truth) plus whether the replay halted - never by the replay's self-report. `/api/db` is a read path the SPA itself never calls, so the oracle cannot be gamed by the screen. ### Outcome taxonomy diff --git a/packages/flow/benchmark/lending_fault_model/README.md b/packages/flow/benchmark/lending_fault_model/README.md index b6de3a698a..f2b801ca09 100644 --- a/packages/flow/benchmark/lending_fault_model/README.md +++ b/packages/flow/benchmark/lending_fault_model/README.md @@ -6,7 +6,7 @@ All prior transactional evidence (`benchmark/fault_model`, `benchmark/effect_readback`, `benchmark/silent_wrong_action`) ran against **MockMed**, a clinical fixture. This study replicates the same rigorous method on a **second, non-healthcare system of record with a distinct UI**: **MockLoan** -(`openadapt_flow/mockloan/`), a loan-origination console whose consequential +(`benchmark/harness/mockloan/`), a loan-origination console whose consequential write **authorizes a disbursement of funds to a borrower** - an irreversible money-movement write. @@ -16,11 +16,11 @@ judged by an independent ledger read, with zero model calls. ## The fixture -- `openadapt_flow/mockloan/` - a self-contained, deterministic static SPA +- `benchmark/harness/mockloan/` - a self-contained, deterministic static SPA (login -> funding pipeline -> loan -> new disbursement -> **Authorize Disbursement**), a distinct finance UI from MockMed. All data is fake/synthetic (no real PII). -- `openadapt_flow/mockloan/fault_server.py` - adds a real persistence boundary +- `benchmark/harness/mockloan/fault_server.py` - adds a real persistence boundary (an in-process **ledger**) and a flag-gated `?fault=` hook that mirrors MockMed's. **With no `?fault` query the app never calls the API** and the normal benchmark is byte-for-byte unaffected (pinned by a test). The ledger is diff --git a/packages/flow/benchmark/lending_fault_model/faults.py b/packages/flow/benchmark/lending_fault_model/faults.py index b549bab918..b9a1b726ea 100644 --- a/packages/flow/benchmark/lending_fault_model/faults.py +++ b/packages/flow/benchmark/lending_fault_model/faults.py @@ -6,7 +6,7 @@ (``tests/e2e/test_lending_fault_model.py``). It defines: - the transactional fault classes injected at the persistence boundary - (realized by ``openadapt_flow/mockloan/fault_server.py`` + the flag-gated + (realized by ``benchmark/harness/mockloan/fault_server.py`` + the flag-gated ``?fault=`` hook in ``mockloan/static/app.js``), and - ``classify()``, which assigns each replay an outcome judged ONLY by the backend ledger (ground truth) and whether the replay halted - never by the diff --git a/packages/flow/benchmark/lending_fault_model/resolution_ladder.py b/packages/flow/benchmark/lending_fault_model/resolution_ladder.py index 2a998611f7..fc4b120d08 100644 --- a/packages/flow/benchmark/lending_fault_model/resolution_ladder.py +++ b/packages/flow/benchmark/lending_fault_model/resolution_ladder.py @@ -38,12 +38,12 @@ import requests +from benchmark.harness.mockloan.demo import record_disbursement_demo +from benchmark.harness.mockloan.fault_server import serve as fault_serve from benchmark.lending_fault_model import faults as F from openadapt_flow.backends.playwright_backend import PlaywrightBackend from openadapt_flow.compiler import compile_recording -from openadapt_flow.demo_driver import record_disbursement_demo from openadapt_flow.ir import Workflow -from openadapt_flow.mockloan.fault_server import serve as fault_serve from openadapt_flow.runtime import Replayer HERE = Path(__file__).resolve().parent diff --git a/packages/flow/benchmark/lending_fault_model/run.py b/packages/flow/benchmark/lending_fault_model/run.py index 94fd661270..6603f5afc8 100644 --- a/packages/flow/benchmark/lending_fault_model/run.py +++ b/packages/flow/benchmark/lending_fault_model/run.py @@ -44,12 +44,12 @@ import requests +from benchmark.harness.mockloan.demo import record_disbursement_demo +from benchmark.harness.mockloan.fault_server import serve as fault_serve from benchmark.lending_fault_model import faults as F from openadapt_flow.backends.playwright_backend import PlaywrightBackend from openadapt_flow.compiler import compile_recording -from openadapt_flow.demo_driver import record_disbursement_demo from openadapt_flow.ir import RunReport, Workflow -from openadapt_flow.mockloan.fault_server import serve as fault_serve from openadapt_flow.runtime import Replayer HERE = Path(__file__).resolve().parent @@ -261,7 +261,7 @@ def render_markdown(results: list[ClassResult], meta: dict) -> str: "The bundled MockLoan app is a client-side SPA with no backend, so an " '"authorize" only mutates in-page state - the UI *is* the source of ' "truth. This study adds a real persistence boundary " - "(`openadapt_flow/mockloan/fault_server.py`): a flag-gated `?fault=` " + "(`benchmark/harness/mockloan/fault_server.py`): a flag-gated `?fault=` " "hook in the app (mirroring the `?drift=` hooks) routes the Authorize " "write through a backend API with an isolated SQLite ledger. **With no " "`?fault` query the app never calls the API and the normal benchmark " diff --git a/packages/flow/benchmark/lending_fault_model/swer.py b/packages/flow/benchmark/lending_fault_model/swer.py index 312cc57e4a..716bce000f 100644 --- a/packages/flow/benchmark/lending_fault_model/swer.py +++ b/packages/flow/benchmark/lending_fault_model/swer.py @@ -83,7 +83,7 @@ score_episode, summarize, ) -from openadapt_flow.mockloan.fault_server import LedgerDB, serve +from benchmark.harness.mockloan.fault_server import LedgerDB, serve from .ground_truth import SQLiteGroundTruthVerifier diff --git a/packages/flow/benchmark/openemr_e2e/harness.py b/packages/flow/benchmark/openemr_e2e/harness.py index 5110b31c77..114e17d82d 100644 --- a/packages/flow/benchmark/openemr_e2e/harness.py +++ b/packages/flow/benchmark/openemr_e2e/harness.py @@ -57,7 +57,7 @@ promoted_workflow, resolution_demonstration, ) -from openadapt_flow.learning.synth_stream import StructuralDiffInducer +from openadapt_flow.learning.induce import StructuralDiffInducer from openadapt_flow.mockmed.fault_server import serve as fault_serve from openadapt_flow.runtime.effects import RestRecordVerifier from openadapt_flow.runtime.replayer import Replayer diff --git a/packages/flow/benchmark/undo_proposal/run.py b/packages/flow/benchmark/undo_proposal/run.py index 2dd6127ddd..169941bafe 100644 --- a/packages/flow/benchmark/undo_proposal/run.py +++ b/packages/flow/benchmark/undo_proposal/run.py @@ -55,8 +55,9 @@ class Fixture: def _fixtures() -> dict[str, Fixture]: + from benchmark.harness.mockloan.demo import record_disbursement_demo + from benchmark.harness.mockloan.fault_server import serve as serve_loan from openadapt_flow import demo_driver - from openadapt_flow.mockloan.fault_server import serve as serve_loan from openadapt_flow.mockmed.fault_server import serve as serve_med return { @@ -74,7 +75,7 @@ def _fixtures() -> dict[str, Fixture]: "mockloan": Fixture( name="mockloan", serve=serve_loan, - record=demo_driver.record_disbursement_demo, + record=record_disbursement_demo, record_kwargs={"memo_text": "Synthetic funding memo"}, query="?fault=ok", match_fields=("loan_id", "product"), @@ -86,7 +87,7 @@ def _fixtures() -> dict[str, Fixture]: "mockloan-fee-row": Fixture( name="mockloan-fee-row", serve=serve_loan, - record=demo_driver.record_disbursement_demo, + record=record_disbursement_demo, record_kwargs={"memo_text": "Synthetic funding memo"}, query="?fault=collateral", match_fields=("loan_id", "product"), @@ -185,7 +186,7 @@ def _bind_save_effect( def _contract(directory: Path, base_url: str, fixture: Fixture, body: dict) -> Path: from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey - from openadapt_flow.reward.seed import write_bundle + from openadapt_flow.reward.bundle import write_bundle match: dict[str, Any] = {k: str(body[k]) for k in fixture.match_fields} match.update(dict(fixture.extra_match)) diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md index f8f4b51798..ece04e3645 100644 --- a/packages/flow/docs/GENERALITY_AUDIT.md +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -72,14 +72,14 @@ motivating application. | --- | --- | --- | --- | | `benchmark/` (27 Python files: `run_benchmark`, `openemr_benchmark`, `openemr_local`, `frappe_lending`, `hybrid_benchmark`, `dom_arm`, `silent_wrong_action`, `desktop_benchmark`, `verify`, `agent_baseline`, `reliability`, `chart_fonts`, `effectbench/`) | Benchmark harnesses shipped in the wheel | Moved to `packages/flow/benchmark/harness/` (import `benchmark.harness.*`). `openadapt-flow benchmark` loads it from a source checkout and says so from a wheel | done | | `instrument/` | Competitor-drift harness; imported the benchmark at module load, so it was already broken in a wheel | Moved to `packages/flow/benchmark/harness/instrument/` | done | -| `integrations/harness2/` | Harness2 research harness on MockMed | Move to `packages/flow/benchmark/harness/harness2/` | planned | -| `mockloan/`, `demo_driver.record_disbursement_demo` | Lending demo app no product command uses | Move to the benchmark tree | planned | -| `guard_demo.py`, `agent/ledger_demo.py` | `python -m` demos, no CLI command | Move to the benchmark tree | planned | -| `mockmed/teach_scope.py` | Used only by tests | Move to `tests/fixtures` | planned | -| `learning/synth_stream.py:1-245` | MockMed note stream | Split out the generic `StructuralDiffInducer`, then move the stream | planned | -| `reward/seed.py:1-236` | MockMed reward seed | Split out the generic bundle helpers (`:238-410`), then move the seed behind the demo flag | planned | +| `integrations/harness2/` | Harness2 research harness on MockMed | Moved to `packages/flow/benchmark/harness/harness2/`; the `harness2` extra is gone (install `[browser]`) | done | +| `mockloan/`, `demo_driver.record_disbursement_demo` | Lending demo app no product command uses | Moved to `packages/flow/benchmark/harness/mockloan/` (recorder in `mockloan/demo.py`) | done | +| `guard_demo.py`, `agent/ledger_demo.py` | `python -m` demos, no CLI command | Moved to `packages/flow/benchmark/harness/guard_demo.py` and `packages/agent/tests/ledger_demo.py` | done | +| `mockmed/teach_scope.py` | Used only by tests | Moved to `packages/flow/tests/_teach_scope.py` | done | +| `learning/synth_stream.py:1-245` | MockMed note stream | `StructuralDiffInducer` moved to `openadapt_flow/learning/induce.py`; the stream moved to `packages/flow/benchmark/harness/synth_stream.py` | done | +| `reward/seed.py:1-236` | MockMed reward seed | Bundle helpers split out to `openadapt_flow/reward/bundle.py`. The seed stays: `serve-reward --seed-mockmed` needs it from a wheel | done | | `record_check_proof.py:60-124` | `MOCKMED` fixture and `if name == "mockmed"` | Fixtures registered by name from the demo app's own module | planned | -| `validation/dense_surface.py` | Study harness rendering a MockMed table | Move to `benchmark/dense_surface` | planned | +| `validation/dense_surface.py` | Study harness rendering a MockMed table | Move to the benchmark tree together with the identity studies that import it (`identity_ladder`, `identity_looks_dense`, `pixel_identity_probe`, `known_ids_dense`) | planned | | `__main__.py:5246-5275` (`bench`) | Always replays against MockMed | Accept `--url`, or load from a checkout like `benchmark` | planned | | `execute/registry.py`, `execute/service.py`, `execute/app.py`, `execute/dispatch.py:60-108`, `--seed-mockmed` | Demo seeding inside the Execute service | Generic `seed_admissions`; the CLI passes the demo seed | planned | | `eligibility/payer_routes.yaml`, `fleet/simulator.py`, `repair/dialog_fixture.py` | Synthetic test data and simulators in the package | Move to the benchmark tree or `tests/fixtures` | planned | diff --git a/packages/flow/docs/GUARDED_SAVE.md b/packages/flow/docs/GUARDED_SAVE.md index 9ede4e7e2e..7b6a56b104 100644 --- a/packages/flow/docs/GUARDED_SAVE.md +++ b/packages/flow/docs/GUARDED_SAVE.md @@ -208,8 +208,10 @@ second commit. ## Try it +From `packages/flow` in a source checkout: + ```bash -python -m openadapt_flow.guard_demo +python -m benchmark.harness.guard_demo ``` The demo starts MockMed on localhost. A fake agent signs in to the chart diff --git a/packages/flow/docs/GUARD_AN_AGENT.md b/packages/flow/docs/GUARD_AN_AGENT.md index b46adf5752..77b78b0d7d 100644 --- a/packages/flow/docs/GUARD_AN_AGENT.md +++ b/packages/flow/docs/GUARD_AN_AGENT.md @@ -44,7 +44,7 @@ Nothing leaves your computer, and the guard makes no model calls. - {method: POST, path: /chart/signin} approval: not_gated # each_item: a person approves each save allow_origins: [] # optional: other origins the tab may reach - write_adapter: openadapt_flow.mockmed.teach_scope:EncounterWrites # optional + write_adapter: myapp.writes:EncounterWrites # optional: module:Class on your path allow_text: [Triage] # optional: values a save may carry besides the task's teach: # optional: make run 2 a program dir: .openadapt-teach diff --git a/packages/flow/docs/HARNESS2.md b/packages/flow/docs/HARNESS2.md index 73e37febf4..526dbb5cfe 100644 --- a/packages/flow/docs/HARNESS2.md +++ b/packages/flow/docs/HARNESS2.md @@ -9,7 +9,7 @@ is another model's judgement. GUI tasks on a system of record do have a scorer. After the agent stops, the store either holds the requested record on the requested patient, exactly -once, or it does not. `openadapt_flow.integrations.harness2` connects the two: +once, or it does not. `benchmark.harness.harness2` connects the two: Harness² supplies the search, and OpenAdapt supplies the task environment, the score, and the safety envelope. @@ -100,7 +100,7 @@ read-back of the chosen run. Meta-Harness ([arXiv:2603.28052](https://arxiv.org/abs/2603.28052)) gives its proposer a filesystem with every earlier candidate's harness, full execution traces and scores. In its ablation, full traces beat summaries, -56.7 to 38.7. `openadapt_flow.integrations.harness2.traces` writes that +56.7 to 38.7. `benchmark.harness.harness2.traces` writes that filesystem for GUI work. `run_task` writes one for every task under `/traces`. @@ -143,12 +143,14 @@ rollout. The raw data is committed beside it. ## Install -Harness² needs Python 3.12 and is not on PyPI. Install it from its -repository at the commit this integration is written against, beside the -extra: +Harness² needs Python 3.12 and is not on PyPI. The integration lives in the +source checkout, in `packages/flow/benchmark/harness/harness2/`, and is not in +the `openadapt-flow` wheel. From `packages/flow`, install Flow with the browser +extra, then Harness² from its repository at the commit this integration is +written against: ```bash -pip install 'openadapt-flow[harness2]' +pip install -e '.[browser]' ``` ```bash @@ -170,11 +172,11 @@ commit is pinned. from pathlib import Path from harness2.core.spec import ChannelSpec -from openadapt_flow.integrations.harness2.adapter import OpenAdaptGuiAdapter -from openadapt_flow.integrations.harness2.agent import ScriptedGuiAgent -from openadapt_flow.integrations.harness2.proposers import ScriptedProposer -from openadapt_flow.integrations.harness2.selection import run_task -from openadapt_flow.integrations.harness2.tasks import mockmed_tasks +from benchmark.harness.harness2.adapter import OpenAdaptGuiAdapter +from benchmark.harness.harness2.agent import ScriptedGuiAgent +from benchmark.harness.harness2.proposers import ScriptedProposer +from benchmark.harness.harness2.selection import run_task +from benchmark.harness.harness2.tasks import mockmed_tasks adapter = OpenAdaptGuiAdapter( mockmed_tasks(("lookalike",)), ScriptedGuiAgent(), Path("runs"), evidence="readback" diff --git a/packages/flow/docs/JIT.md b/packages/flow/docs/JIT.md index 6cc6119127..070cb7fc3b 100644 --- a/packages/flow/docs/JIT.md +++ b/packages/flow/docs/JIT.md @@ -307,8 +307,12 @@ it. Pass `teach_scope=` to `compile_agent` to fix what the recording run may do before the agent reads a page: the app origins, the record, the fields and exact values the save may write, and any other text it may type. +The example uses MockMed's write adapter, a test fixture in the source +checkout (`packages/flow/tests/_teach_scope.py`); write the same few lines for +your app. + ```python -from openadapt_flow.mockmed.teach_scope import encounter_scope +from tests._teach_scope import encounter_scope jit = compile_agent( agent, diff --git a/packages/flow/openadapt_flow/demo_driver.py b/packages/flow/openadapt_flow/demo_driver.py index 9ee4d3586f..5ca0e8be37 100644 --- a/packages/flow/openadapt_flow/demo_driver.py +++ b/packages/flow/openadapt_flow/demo_driver.py @@ -1,14 +1,13 @@ -"""Demo driver: records the canonical MockMed and MockLoan demonstrations. +"""Demo driver: records the canonical MockMed demonstration. Record time is allowed to cheat with Playwright locators (to find pixel coordinates via ``bounding_box()``), but every action is performed through `Recorder` so frames and events are captured exactly as a human demo would be. Replay never uses selectors. -Two self-contained demonstration targets are supported: the healthcare -MockMed triage-save (``record_triage_demo``) and the non-healthcare MockLoan -loan-disbursement (``record_disbursement_demo``). Both drive the identical -governed record -> compile -> replay path. +``record_triage_demo`` records the MockMed triage save that ``openadapt-flow +demo-record`` uses. The MockLoan disbursement recorder is a benchmark fixture +in the source checkout (``benchmark/harness/mockloan/demo.py``). """ from __future__ import annotations @@ -92,80 +91,3 @@ def center(selector: str) -> tuple[int, int]: return recorder.finish() finally: close() - - -def record_disbursement_demo( - url: str, - out_dir: Path | str, - *, - memo_text: str, - param_name: str = "memo", - headed: bool = False, - record_video_dir: Path | str | None = None, -) -> Path: - """Record the canonical MockLoan disbursement demo against a running app. - - The non-healthcare companion to :func:`record_triage_demo`. Flow: - login -> pipeline -> Open first loan -> New Disbursement -> Personal product - -> click Funding memo field -> type memo (as a parameter) -> Authorize - Disbursement. The consequential final step authorizes moving money to a - borrower, the lending analog of clinically saving an encounter. - - Args: - url: MockLoan base URL (from `openadapt_flow.mockloan.server.serve`). - out_dir: Directory to write the recording into. - memo_text: The funding memo typed during the demo (recorded as a param). - param_name: Parameter name for the typed memo (default ``"memo"``). - headed: Run the browser headed (visible) instead of headless. - record_video_dir: OPT-IN (default ``None`` = off). When set, a WebM - video of the recording session is captured into this directory; it - does not change what is recorded to ``out_dir``. - - Returns: - The recording directory (contains meta.json, events.jsonl, frames/). - """ - backend, close = PlaywrightBackend.launch( - url, - headless=not headed, - record_video_dir=( - str(record_video_dir) if record_video_dir is not None else None - ), - ) - try: - page = backend.page - recorder = Recorder(backend, out_dir, app_url=url) - - def center(selector: str) -> tuple[int, int]: - """Center pixel coordinates of the first element matching CSS.""" - locator = page.locator(selector).first - locator.wait_for(state="visible") - box = locator.bounding_box() - if box is None: # pragma: no cover - visible => box exists - raise RuntimeError(f"no bounding box for {selector!r}") - return ( - int(box["x"] + box["width"] / 2), - int(box["y"] + box["height"] / 2), - ) - - # Login - recorder.click(*center("#username")) - recorder.type_text("officer.demo") - recorder.click(*center("#password")) - recorder.type_text("mockloan-demo-pass") - recorder.click(*center("#signin")) - - # Pipeline -> open first loan - recorder.click(*center(".open-btn")) - - # Loan -> new disbursement - recorder.click(*center("#new-disbursement")) - - # Disbursement: product chooser, memo (parameterized), authorize - recorder.click(*center("#product-personal")) - recorder.click(*center("#memo")) - recorder.type_text(memo_text, param=param_name) - recorder.click(*center("#authorize")) - - return recorder.finish() - finally: - close() diff --git a/packages/flow/openadapt_flow/learning/__init__.py b/packages/flow/openadapt_flow/learning/__init__.py index 963729c2c8..610a3b0bf2 100644 --- a/packages/flow/openadapt_flow/learning/__init__.py +++ b/packages/flow/openadapt_flow/learning/__init__.py @@ -17,8 +17,9 @@ Multi-trace INDUCTION (turning traces into a revised graph) is a sibling PR, depended on only through the :class:`~openadapt_flow.learning.loop.Inducer` -Protocol; :mod:`openadapt_flow.learning.synth_stream` provides a synthetic -drift-stream harness and a deterministic reference inducer to exercise the loop. +Protocol; :mod:`openadapt_flow.learning.induce` provides a deterministic +reference inducer. The synthetic drift stream that exercises the loop is a +fixture in the source checkout (``benchmark/harness/synth_stream.py``). No model calls anywhere on the runtime path. """ diff --git a/packages/flow/openadapt_flow/learning/branch_teach.py b/packages/flow/openadapt_flow/learning/branch_teach.py index 7301b8abeb..811ac469b5 100644 --- a/packages/flow/openadapt_flow/learning/branch_teach.py +++ b/packages/flow/openadapt_flow/learning/branch_teach.py @@ -70,10 +70,10 @@ execution_trace_from_halt, promoted_workflow, ) +from openadapt_flow.learning.induce import StructuralDiffInducer from openadapt_flow.learning.interpreter import program_reproduces from openadapt_flow.learning.library import SkillLibrary from openadapt_flow.learning.loop import LearnOutcome, RegressionGate, learn_from_traces -from openadapt_flow.learning.synth_stream import StructuralDiffInducer from openadapt_flow.learning.teach import ( CorrectionSpec, TeachError, diff --git a/packages/flow/openadapt_flow/learning/halt_loop.py b/packages/flow/openadapt_flow/learning/halt_loop.py index 6599706958..fd17b308b9 100644 --- a/packages/flow/openadapt_flow/learning/halt_loop.py +++ b/packages/flow/openadapt_flow/learning/halt_loop.py @@ -5,7 +5,7 @@ The continuous-learning scaffold (:mod:`openadapt_flow.learning.loop`, :mod:`~openadapt_flow.learning.gate`, :mod:`~openadapt_flow.learning.library`) and the multi-trace inducer (:mod:`openadapt_flow.compiler.induction`, -:mod:`openadapt_flow.learning.synth_stream`) already EXIST but no real run feeds +:mod:`openadapt_flow.learning.induce`) already EXIST but no real run feeds them. This module is the thin BRIDGE that makes the loop real: 1. **A halt emits a learnable trace.** ``Replayer.run`` now populates diff --git a/packages/flow/openadapt_flow/learning/synth_stream.py b/packages/flow/openadapt_flow/learning/induce.py similarity index 64% rename from packages/flow/openadapt_flow/learning/synth_stream.py rename to packages/flow/openadapt_flow/learning/induce.py index 323194500c..67f6f1b58c 100644 --- a/packages/flow/openadapt_flow/learning/synth_stream.py +++ b/packages/flow/openadapt_flow/learning/induce.py @@ -1,26 +1,11 @@ -"""Synthetic trace-stream harness: a MockMed skill drifting over "time". - -To exercise the learn/promote loop end-to-end without a live app, this module - -1. builds a base MockMed skill program (``add-patient-note`` from - ``docs/design/WORKFLOW_PROGRAM_IR.md`` section 2.5: sign in -> open the - patient record -> new encounter -> type note -> SAVE), and -2. emits a STREAM of :class:`~openadapt_flow.learning.trace.ExecutionTrace`s over - a logical time axis with INJECTED DRIFT at a chosen trace index -- e.g. a new - consent dialog starts appearing mid-stream, so the active program stops - reproducing the new successful traces and the loop must detect the novelty, - induce a revision, validate it, and promote it. - -It also provides :class:`StructuralDiffInducer`, a deterministic, model-free -REFERENCE inducer used by the harness/tests. It is a stand-in for the real -multi-trace-induction sibling PR (wired in at merge via the -:class:`~openadapt_flow.learning.loop.Inducer` Protocol): it induces exactly the -"a new optional step appeared" revision -- detect an intent present in some -successful traces but absent from the base program, and splice it in as a -GUARDED optional step (a Phase-1 ``skip`` guard keyed on the screen fact that -co-occurs with it). It does NOT invent identity/effect/risk -- a dialog -dismissal is reversible and unarmed -- so the regression gate has nothing to -refuse on the happy path, exactly as intended. +"""A deterministic, model-free inducer: splice a newly observed step in. + +:class:`StructuralDiffInducer` diffs successful traces against a base program +and splices in the one step they show that the program lacks, guarded on the +screen fact that co-occurs with it. ``openadapt-flow teach`` and the branch +teach use it; it plugs into the learn/promote loop through the +:class:`~openadapt_flow.learning.loop.Inducer` Protocol. It does not invent +identity, effects or risk, so the regression gate still decides promotion. All deterministic, ``$0``, no model calls. """ @@ -28,7 +13,6 @@ from __future__ import annotations from collections.abc import Mapping -from enum import Enum from typing import Optional from openadapt_flow.ir import ( @@ -44,34 +28,20 @@ Step, Transition, ) -from openadapt_flow.learning.trace import ExecutionTrace, TraceStep -from openadapt_flow.runtime.effects.effect import Effect, EffectKind - -# -- the MockMed skill's canonical intents ------------------------------------ - -INTENT_LOGIN = "Sign in" -INTENT_OPEN = "Open patient record" -INTENT_ENCOUNTER = "Start new encounter" -INTENT_NOTE = "Enter note" -INTENT_SAVE = "Save encounter" -INTENT_CONSENT = "Acknowledge consent notice" - -#: The screen fact that co-occurs with the injected consent dialog -- the text a -#: ``TEXT_PRESENT`` guard keys on. -FACT_CONSENT = "Consent Required" +from openadapt_flow.learning.trace import ExecutionTrace +from openadapt_flow.runtime.effects.effect import Effect -#: A representative identity band for the patient row (name + DOB) -- the -#: evidence the pre-click identity gate keys on; makes the "open patient" step -#: identity-ARMED so an identity regression is meaningful. -PATIENT_BAND = "Belford, Phil DOB 1990-01-01" +__all__ = ["StructuralDiffInducer"] -def _anchor(*, armed: bool = False, cp: tuple[int, int] = (100, 100)) -> Anchor: +def _anchor( + *, context_text: Optional[str] = None, cp: tuple[int, int] = (100, 100) +) -> Anchor: return Anchor( template="templates/x.png", region=(cp[0] - 10, cp[1] - 10, 20, 20), click_point=cp, - context_text=PATIENT_BAND if armed else None, + context_text=context_text, ) @@ -86,14 +56,17 @@ def _action_state( effects: Optional[list[Effect]] = None, guard: Optional[Guard] = None, text: Optional[str] = None, + context_text: Optional[str] = None, ) -> State: + """An action state with a placeholder anchor; ``context_text`` is the + identity band an armed step's anchor carries.""" step = Step( id=sid, intent=intent, action=action, anchor=None if action in (ActionKind.KEY, ActionKind.WAIT) - else _anchor(armed=armed), + else _anchor(context_text=context_text if armed else None), text=text, risk=risk, # type: ignore[arg-type] effects=effects or [], @@ -108,140 +81,6 @@ def _action_state( ) -def mockmed_base_program() -> ProgramGraph: - """The base ``add-patient-note`` program (no consent dialog). - - A straight-line state machine: sign in -> open patient (identity-armed) -> - new encounter -> type note -> SAVE (irreversible, with a ``record_written`` - system-of-record effect) -> success. The identity band + effect + risk on - the sensitive steps are what a rigged revision could weaken, and what the - regression gate protects. - """ - save_effect = Effect( - kind=EffectKind.RECORD_WRITTEN, - match={"type": "encounter", "patient": "p1"}, - expected_count=1, - risk="irreversible", - probe="encounter record written for patient", - ) - states = { - "s_login": _action_state("s_login", INTENT_LOGIN, "s_open"), - "s_open": _action_state("s_open", INTENT_OPEN, "s_encounter", armed=True), - "s_encounter": _action_state("s_encounter", INTENT_ENCOUNTER, "s_note"), - "s_note": _action_state( - "s_note", INTENT_NOTE, "s_save", action=ActionKind.TYPE, text="Follow-up" - ), - "s_save": _action_state( - "s_save", - INTENT_SAVE, - "__end__", - risk="irreversible", - effects=[save_effect], - ), - "__end__": State(id="__end__", kind=StateKind.TERMINAL, outcome="success"), - } - return ProgramGraph(entry="s_login", states=states) - - -# -- the drift stream --------------------------------------------------------- - - -class Drift(str, Enum): - """The injected drift/variant a stream can carry from trace K onward.""" - - NONE = "none" - #: A new consent dialog appears after sign-in (a new OPTIONAL step + branch). - CONSENT_DIALOG = "consent_dialog" - #: The note field is renamed (the observed intent changes) -- a structural - #: change the additive reference inducer CANNOT cover (used to show the loop - #: correctly refuses an inadequate revision). - FIELD_RENAME = "field_rename" - - -def _baseline_steps() -> list[TraceStep]: - return [ - TraceStep(intent=INTENT_LOGIN), - TraceStep(intent=INTENT_OPEN, identity=PATIENT_BAND), - TraceStep(intent=INTENT_ENCOUNTER), - TraceStep(intent=INTENT_NOTE, action=ActionKind.TYPE), - TraceStep(intent=INTENT_SAVE), - ] - - -def _drifted_steps(drift: Drift) -> tuple[list[TraceStep], dict[str, bool]]: - steps = _baseline_steps() - facts: dict[str, bool] = {} - if drift is Drift.CONSENT_DIALOG: - # The consent dialog appears right after sign-in. - steps.insert(1, TraceStep(intent=INTENT_CONSENT)) - facts[FACT_CONSENT] = True - elif drift is Drift.FIELD_RENAME: - for s in steps: - if s.intent == INTENT_NOTE: - s.intent = "Enter clinical note" - return steps, facts - - -def generate_stream( - *, - n_baseline: int = 6, - n_drift: int = 6, - drift: Drift = Drift.CONSENT_DIALOG, - n_failures: int = 0, - prefix: str = "mockmed", -) -> list[ExecutionTrace]: - """Emit a deterministic stream of traces with drift injected at trace - ``n_baseline``. - - The first ``n_baseline`` traces are clean successful runs of the base skill; - from index ``n_baseline`` the injected ``drift`` appears in every successful - run. ``n_failures`` failure traces (noise) are appended -- runs that halted - partway with no novel successful structure, so a batch of them alone must NOT - trigger a revision. - """ - traces: list[ExecutionTrace] = [] - t = 0 - for _ in range(n_baseline): - traces.append( - ExecutionTrace( - trace_id=f"{prefix}-{t:04d}", - t=t, - outcome="success", - steps=_baseline_steps(), - params={"encounter_type": "Triage"}, - ) - ) - t += 1 - for _ in range(n_drift): - steps, facts = _drifted_steps(drift) - traces.append( - ExecutionTrace( - trace_id=f"{prefix}-{t:04d}", - t=t, - outcome="success", - steps=steps, - facts=facts, - params={"encounter_type": "Triage"}, - ) - ) - t += 1 - for _ in range(n_failures): - # A run that halted after opening the patient (e.g. a transient - # resolution failure): truncated, no novel successful structure. - traces.append( - ExecutionTrace( - trace_id=f"{prefix}-{t:04d}", - t=t, - outcome="failure", - steps=_baseline_steps()[:2], - failure_reason="transient resolution failure on 'Start new encounter'", - params={"encounter_type": "Triage"}, - ) - ) - t += 1 - return traces - - # -- the reference (stand-in) inducer ----------------------------------------- #: Postconditions about what the screen shows, which stay meaningful when diff --git a/packages/flow/openadapt_flow/learning/teach.py b/packages/flow/openadapt_flow/learning/teach.py index fdca14a554..7f11d6eecc 100644 --- a/packages/flow/openadapt_flow/learning/teach.py +++ b/packages/flow/openadapt_flow/learning/teach.py @@ -31,7 +31,7 @@ Deterministic and ``$0`` on the shipped path: the resolution is induced by the model-free reference inducer -(:class:`~openadapt_flow.learning.synth_stream.StructuralDiffInducer`), which +(:class:`~openadapt_flow.learning.induce.StructuralDiffInducer`), which handles the "an unexpected dialog intercepted the workflow" resolution class the halt->learn loop was built for. Three things make that work on a real bundle, not only on the synthetic stream: @@ -86,6 +86,7 @@ promoted_workflow, resolution_demonstration, ) +from openadapt_flow.learning.induce import StructuralDiffInducer from openadapt_flow.learning.library import SkillLibrary from openadapt_flow.learning.loop import ( CanaryContext, @@ -93,7 +94,6 @@ LearnOutcome, ProgramCanaryFn, ) -from openadapt_flow.learning.synth_stream import StructuralDiffInducer from openadapt_flow.learning.trace import ExecutionTrace, TraceStep from openadapt_flow.runtime.authorization import runtime_params_for_gui diff --git a/packages/flow/openadapt_flow/reward/bundle.py b/packages/flow/openadapt_flow/reward/bundle.py new file mode 100644 index 0000000000..1f79155b28 --- /dev/null +++ b/packages/flow/openadapt_flow/reward/bundle.py @@ -0,0 +1,254 @@ +"""Write, calibrate and self-sign a synthetic reward bundle. + +A reward bundle is the directory ``openadapt_flow.reward.models.RewardBundle`` +loads: the contract, its required and forbidden effects, the oracle recipe, +and (when certified) a self-signed certificate with the calibration counts +beside it. :func:`write_bundle` writes one from the effects and oracle a +caller passes; tests, the benchmark harnesses and the ``serve-reward`` demo +seed (:mod:`openadapt_flow.reward.seed`) all build their bundles with it. + +Nothing here is a production recipe. A production-scope certificate is +calibrated on a held-out corpus by the OpenAdapt control service. +""" + +from __future__ import annotations + +import base64 +import json +from pathlib import Path +from typing import Any + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from openadapt_types.process_capability import ( + _digest_payload, + canonical_json_bytes, +) +from openadapt_types.reward import RewardCertificateV1, RewardContractV1 + +from openadapt_flow.reward.calibration import ( + CalibrationResult, + confidence_delta, + corpus_digest_for, + corpus_from_effects, + extradup_trials, +) +from openadapt_flow.reward.models import ( + CERTIFICATE_FILE, + CONTRACT_FILE, + FORBIDDEN_EFFECTS_FILE, + ORACLE_FILE, + REQUIRED_EFFECTS_FILE, + RewardBundle, +) +from openadapt_flow.runtime.effects.effect import Effect + +#: The task and environment a synthetic bundle's contract names. The values +#: are the ones the ``serve-reward`` demo seed has always used, so a bundle's +#: digests do not change with the module it was written from. +SYNTHETIC_TASK_ID = "task_mockmed_encounter_note" +SYNTHETIC_ENVIRONMENT_ID = "environment_mockmed_synthetic" +CERTIFICATE_EXPIRY_UPDATES = 1000 +CALIBRATION_FILE = "calibration.json" +#: Trials run before the synthetic certificate is signed. 300 trials with +#: zero false accepts bound the rate at 0.0099 (95%). +CALIBRATION_TRIALS = 300 +CALIBRATION_SEED = 20260901 +CALIBRATION_CONFIDENCE = 0.95 + +#: Effects a bundle requires and forbids when the caller passes none. +DEFAULT_REQUIRED_EFFECTS: list[dict[str, Any]] = [ + { + "kind": "record_written", + "match": {"patient_id": {"param": "patient_id"}, "type": "Triage"}, + "expected_count": 1, + # The claim is about what this episode added, not about what the + # store happens to hold. Without it a rollout that did nothing + # scores the full reward whenever the subject already had a row. + "count_new_only": True, + } +] + +DEFAULT_FORBIDDEN_EFFECTS: list[dict[str, Any]] = [ + { + "kind": "record_written", + "match": {"patient_id": {"param": "patient_id"}, "type": "Discharge"}, + "expected_count": 1, + } +] + + +def write_bundle( + directory: Path, + *, + contract_id: str, + oracle: dict[str, Any], + channel: str, + key: Ed25519PrivateKey, + issuer_key_id: str, + certify: bool, + required_effects: list[dict[str, Any]] | None = None, + forbidden_effects: list[dict[str, Any]] | None = None, + identity_keys: list[str] | None = None, +) -> None: + """Write one synthetic bundle. Tests pass their own effects and keys.""" + + directory.mkdir(parents=True, exist_ok=True) + required = _canonical( + DEFAULT_REQUIRED_EFFECTS if required_effects is None else required_effects + ) + forbidden = _canonical( + DEFAULT_FORBIDDEN_EFFECTS if forbidden_effects is None else forbidden_effects + ) + oracle_doc = _canonical(oracle) + keys = ["patient_id"] if identity_keys is None else list(identity_keys) + # The corpus a certificate is calibrated on comes from the contract's own + # effects, so its digest is computed here and not chosen. + corpus_digest = corpus_digest_for( + [Effect.model_validate(item) for item in required], + [Effect.model_validate(item) for item in forbidden], + keys, + ) + contract = RewardContractV1.model_validate( + { + "contract_id": contract_id, + "contract_version": "version_0001", + "task_id": SYNTHETIC_TASK_ID, + "task_digest": _digest_payload({"task": SYNTHETIC_TASK_ID}), + "environment_id": SYNTHETIC_ENVIRONMENT_ID, + "environment_digest": _digest_payload( + {"environment": SYNTHETIC_ENVIRONMENT_ID} + ), + "required_effect_contract_digest": _digest_payload(required), + "forbidden_effect_contract_digest": _digest_payload(forbidden), + "oracle": { + "channel": channel, + "identity_keys": keys, + "oracle_contract_digest": _digest_payload(oracle_doc), + }, + "components": [{"name": "terminal_effect", "weight": 1.0}], + # The synthetic banner lie yields 0, not the default -1 penalty: + # this fixture demonstrates "no reward", not a tuned penalty. + "scoring": {"wrong_effect_reward": 0.0}, + "certificate_policy": { + "epsilon": 0.05, + "delta": 0.05, + "threshold": 0.5, + "calibration_corpus_digest": corpus_digest, + "expiry_policy_updates": CERTIFICATE_EXPIRY_UPDATES, + }, + } + ) + _write(directory / CONTRACT_FILE, contract.model_dump(mode="json")) + _write(directory / REQUIRED_EFFECTS_FILE, required) + _write(directory / FORBIDDEN_EFFECTS_FILE, forbidden) + _write(directory / ORACLE_FILE, oracle_doc) + if certify: + calibration = calibrate_bundle(directory) + certificate = self_signed_certificate( + contract, + key=key, + issuer_key_id=issuer_key_id, + issued_at_policy_update=0, + expiry_policy_updates=CERTIFICATE_EXPIRY_UPDATES, + epsilon=calibration.epsilon, + delta=confidence_delta(calibration.confidence), + ) + _write(directory / CERTIFICATE_FILE, certificate.model_dump(mode="json")) + _write(directory / CALIBRATION_FILE, calibration.as_metadata()) + + +def calibrate_bundle(directory: Path) -> CalibrationResult: + """Run the seeded trials through this bundle's judge. + + The corpus is read off the bundle's own required and forbidden effects, + so the faults it plants are faults in the records this contract talks + about. The certificate's ``epsilon`` is the exact one-sided + Clopper-Pearson bound from these counts, and ``calibration.json`` beside + the certificate records the counts, the corpus digest, and which fault + classes applied, so a reader can recompute the bound and see what it + covers. + """ + + from openadapt_types.oracle import OracleChannel + + from openadapt_flow.reward.oracles import observation + from openadapt_flow.reward.worker import judge_episode + from openadapt_flow.runtime.effects.effect import EffectState + + bundle = RewardBundle.load(directory) + channel = OracleChannel(bundle.contract.oracle.channel) + corpus = corpus_from_effects( + bundle.required_effects, + bundle.forbidden_effects, + bundle.identity_keys, + ) + + def checker(before: Any, current: Any, identity: dict[str, str]) -> Any: + pre = EffectState(substrate=channel.value, reachable=True, records=list(before)) + observed = observation(channel, identity, list(current)) + return judge_episode(bundle, identity, "completed", pre, observed).outcome + + return extradup_trials( + checker, + corpus, + trials=CALIBRATION_TRIALS, + generator_seed=CALIBRATION_SEED, + confidence=CALIBRATION_CONFIDENCE, + corpus_digest=bundle.contract.certificate_policy.calibration_corpus_digest, + ) + + +def self_signed_certificate( + contract: RewardContractV1, + *, + key: Ed25519PrivateKey, + issuer_key_id: str, + issued_at_policy_update: int, + expiry_policy_updates: int, + epsilon: float, + delta: float, + certificate_id: str = "reward_certificate_mockmed", +) -> RewardCertificateV1: + """A certificate signed by the local key, synthetic scope. + + ``epsilon`` comes from :func:`calibrate_bundle`, never from a constant. + A production-scope certificate is calibrated on a held-out corpus by + the OpenAdapt control service and is not published. This one exists so + the seeded run can show a certified receipt next to an uncertified one. + """ + + from openadapt_flow.reward.worker import _now + + unsigned = { + "schema_version": "openadapt.reward-certificate/v1", + "certificate_id": certificate_id, + "reward_contract_digest": contract.digest, + "checker_configuration_digest": _digest_payload({"checker": "synthetic"}), + "epsilon": epsilon, + "delta": delta, + "threshold": contract.certificate_policy.threshold, + "calibration_corpus_digest": contract.certificate_policy.calibration_corpus_digest, + "calibration_scope": "synthetic", + "issued_at_policy_update": issued_at_policy_update, + "expiry_policy_updates": expiry_policy_updates, + "issued_at": _now(), + "issuer": "self_signed", + "issuer_key_id": issuer_key_id, + } + signature = base64.b64encode(key.sign(canonical_json_bytes(unsigned))).decode( + "ascii" + ) + return RewardCertificateV1.model_validate( + {**unsigned, "signature_algorithm": "ed25519", "signature": signature} + ) + + +def _canonical(value: Any) -> Any: + return json.loads(json.dumps(value, sort_keys=True)) + + +def _write(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) diff --git a/packages/flow/openadapt_flow/reward/seed.py b/packages/flow/openadapt_flow/reward/seed.py index d0c3856835..eeec87e987 100644 --- a/packages/flow/openadapt_flow/reward/seed.py +++ b/packages/flow/openadapt_flow/reward/seed.py @@ -27,51 +27,28 @@ from __future__ import annotations -import base64 import json import sqlite3 from pathlib import Path from typing import Any from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey -from openadapt_types.process_capability import ( - _digest_payload, - canonical_json_bytes, -) -from openadapt_types.reward import RewardCertificateV1, RewardContractV1 -from openadapt_flow.reward.calibration import ( - CalibrationResult, - confidence_delta, - corpus_digest_for, - corpus_from_effects, - extradup_trials, -) -from openadapt_flow.reward.models import ( - CERTIFICATE_FILE, - CONTRACT_FILE, - FORBIDDEN_EFFECTS_FILE, - ORACLE_FILE, - REQUIRED_EFFECTS_FILE, - RewardBundle, +from openadapt_flow.reward.bundle import ( + SYNTHETIC_ENVIRONMENT_ID, + SYNTHETIC_TASK_ID, + _write, + write_bundle, ) -from openadapt_flow.runtime.effects.effect import Effect -MOCKMED_TASK_ID = "task_mockmed_encounter_note" -MOCKMED_ENVIRONMENT_ID = "environment_mockmed_synthetic" +MOCKMED_TASK_ID = SYNTHETIC_TASK_ID +MOCKMED_ENVIRONMENT_ID = SYNTHETIC_ENVIRONMENT_ID MOCKMED_CONTRACT_ID = "reward_contract_mockmed" MOCKMED_TIER0_CONTRACT_ID = "reward_contract_mockmed_tier0" MOCKMED_HONEST_PATIENT = "patient-honest-0001" MOCKMED_LIE_PATIENT = "patient-lie-0002" MOCKMED_DUPLICATE_PATIENT = "patient-dup-0003" MOCKMED_CHECKPOINT = "policy_checkpoint_mockmed_0" -CERTIFICATE_EXPIRY_UPDATES = 1000 -CALIBRATION_FILE = "calibration.json" -#: Trials the seed runs before it signs the synthetic certificate. 300 trials -#: with zero false accepts bound the rate at 0.0099 (95%). -CALIBRATION_TRIALS = 300 -CALIBRATION_SEED = 20260901 -CALIBRATION_CONFIDENCE = 0.95 #: The tier-2 store's one table and the read-only SELECT the oracle runs. MOCKMED_TABLE = "encounters" @@ -90,26 +67,6 @@ {"id": 2, "patient_id": MOCKMED_DUPLICATE_PATIENT, "type": "Triage"}, ] -_REQUIRED_EFFECTS: list[dict[str, Any]] = [ - { - "kind": "record_written", - "match": {"patient_id": {"param": "patient_id"}, "type": "Triage"}, - "expected_count": 1, - # The claim is about what this episode added, not about what the - # store happens to hold. Without it a rollout that did nothing - # scores the full reward whenever the subject already had a row. - "count_new_only": True, - } -] - -_FORBIDDEN_EFFECTS: list[dict[str, Any]] = [ - { - "kind": "record_written", - "match": {"patient_id": {"param": "patient_id"}, "type": "Discharge"}, - "expected_count": 1, - } -] - def seed_mockmed( data_dir: Path, key: Ed25519PrivateKey, issuer_key_id: str @@ -233,180 +190,3 @@ def mockmed_episode( "runtime_signal": runtime_signal, }, } - - -def write_bundle( - directory: Path, - *, - contract_id: str, - oracle: dict[str, Any], - channel: str, - key: Ed25519PrivateKey, - issuer_key_id: str, - certify: bool, - required_effects: list[dict[str, Any]] | None = None, - forbidden_effects: list[dict[str, Any]] | None = None, - identity_keys: list[str] | None = None, -) -> None: - """Write one synthetic bundle. Tests pass their own effects and keys.""" - - directory.mkdir(parents=True, exist_ok=True) - required = _canonical( - _REQUIRED_EFFECTS if required_effects is None else required_effects - ) - forbidden = _canonical( - _FORBIDDEN_EFFECTS if forbidden_effects is None else forbidden_effects - ) - oracle_doc = _canonical(oracle) - keys = ["patient_id"] if identity_keys is None else list(identity_keys) - # The corpus a certificate is calibrated on comes from the contract's own - # effects, so its digest is computed here and not chosen. - corpus_digest = corpus_digest_for( - [Effect.model_validate(item) for item in required], - [Effect.model_validate(item) for item in forbidden], - keys, - ) - contract = RewardContractV1.model_validate( - { - "contract_id": contract_id, - "contract_version": "version_0001", - "task_id": MOCKMED_TASK_ID, - "task_digest": _digest_payload({"task": MOCKMED_TASK_ID}), - "environment_id": MOCKMED_ENVIRONMENT_ID, - "environment_digest": _digest_payload( - {"environment": MOCKMED_ENVIRONMENT_ID} - ), - "required_effect_contract_digest": _digest_payload(required), - "forbidden_effect_contract_digest": _digest_payload(forbidden), - "oracle": { - "channel": channel, - "identity_keys": keys, - "oracle_contract_digest": _digest_payload(oracle_doc), - }, - "components": [{"name": "terminal_effect", "weight": 1.0}], - # The synthetic banner lie yields 0, not the default -1 penalty: - # this fixture demonstrates "no reward", not a tuned penalty. - "scoring": {"wrong_effect_reward": 0.0}, - "certificate_policy": { - "epsilon": 0.05, - "delta": 0.05, - "threshold": 0.5, - "calibration_corpus_digest": corpus_digest, - "expiry_policy_updates": CERTIFICATE_EXPIRY_UPDATES, - }, - } - ) - _write(directory / CONTRACT_FILE, contract.model_dump(mode="json")) - _write(directory / REQUIRED_EFFECTS_FILE, required) - _write(directory / FORBIDDEN_EFFECTS_FILE, forbidden) - _write(directory / ORACLE_FILE, oracle_doc) - if certify: - calibration = calibrate_bundle(directory) - certificate = self_signed_certificate( - contract, - key=key, - issuer_key_id=issuer_key_id, - issued_at_policy_update=0, - expiry_policy_updates=CERTIFICATE_EXPIRY_UPDATES, - epsilon=calibration.epsilon, - delta=confidence_delta(calibration.confidence), - ) - _write(directory / CERTIFICATE_FILE, certificate.model_dump(mode="json")) - _write(directory / CALIBRATION_FILE, calibration.as_metadata()) - - -def calibrate_bundle(directory: Path) -> CalibrationResult: - """Run the seeded trials through this bundle's judge. - - The corpus is read off the bundle's own required and forbidden effects, - so the faults it plants are faults in the records this contract talks - about. The certificate's ``epsilon`` is the exact one-sided - Clopper-Pearson bound from these counts, and ``calibration.json`` beside - the certificate records the counts, the corpus digest, and which fault - classes applied, so a reader can recompute the bound and see what it - covers. - """ - - from openadapt_types.oracle import OracleChannel - - from openadapt_flow.reward.oracles import observation - from openadapt_flow.reward.worker import judge_episode - from openadapt_flow.runtime.effects.effect import EffectState - - bundle = RewardBundle.load(directory) - channel = OracleChannel(bundle.contract.oracle.channel) - corpus = corpus_from_effects( - bundle.required_effects, - bundle.forbidden_effects, - bundle.identity_keys, - ) - - def checker(before: Any, current: Any, identity: dict[str, str]) -> Any: - pre = EffectState(substrate=channel.value, reachable=True, records=list(before)) - observed = observation(channel, identity, list(current)) - return judge_episode(bundle, identity, "completed", pre, observed).outcome - - return extradup_trials( - checker, - corpus, - trials=CALIBRATION_TRIALS, - generator_seed=CALIBRATION_SEED, - confidence=CALIBRATION_CONFIDENCE, - corpus_digest=bundle.contract.certificate_policy.calibration_corpus_digest, - ) - - -def self_signed_certificate( - contract: RewardContractV1, - *, - key: Ed25519PrivateKey, - issuer_key_id: str, - issued_at_policy_update: int, - expiry_policy_updates: int, - epsilon: float, - delta: float, - certificate_id: str = "reward_certificate_mockmed", -) -> RewardCertificateV1: - """A certificate signed by the local key, synthetic scope. - - ``epsilon`` comes from :func:`calibrate_bundle`, never from a constant. - A production-scope certificate is calibrated on a held-out corpus by - the OpenAdapt control service and is not published. This one exists so - the seeded run can show a certified receipt next to an uncertified one. - """ - - from openadapt_flow.reward.worker import _now - - unsigned = { - "schema_version": "openadapt.reward-certificate/v1", - "certificate_id": certificate_id, - "reward_contract_digest": contract.digest, - "checker_configuration_digest": _digest_payload({"checker": "synthetic"}), - "epsilon": epsilon, - "delta": delta, - "threshold": contract.certificate_policy.threshold, - "calibration_corpus_digest": contract.certificate_policy.calibration_corpus_digest, - "calibration_scope": "synthetic", - "issued_at_policy_update": issued_at_policy_update, - "expiry_policy_updates": expiry_policy_updates, - "issued_at": _now(), - "issuer": "self_signed", - "issuer_key_id": issuer_key_id, - } - signature = base64.b64encode(key.sign(canonical_json_bytes(unsigned))).decode( - "ascii" - ) - return RewardCertificateV1.model_validate( - {**unsigned, "signature_algorithm": "ed25519", "signature": signature} - ) - - -def _canonical(value: Any) -> Any: - return json.loads(json.dumps(value, sort_keys=True)) - - -def _write(path: Path, payload: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text( - json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 17b27ecdb2..49131b61ee 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -286,6 +286,9 @@ "benchmark/grounding_eval/results.json", "benchmark/grounding_eval/results_sweep.json", "benchmark/halt_cleanup/results/openemr.json", + "benchmark/harness/mockloan/static/app.js", + "benchmark/harness/mockloan/static/index.html", + "benchmark/harness/mockloan/static/styles.css", "benchmark/harness2_scorer/evidence/base/results.json", "benchmark/harness2_scorer/evidence/harness2/results.json", "benchmark/harness2_scorer/evidence/harness2_readback/results.json", @@ -542,9 +545,6 @@ "openadapt_flow/console/static/console.js", "openadapt_flow/console/static/index.html", "openadapt_flow/eligibility/payer_routes.yaml", - "openadapt_flow/mockloan/static/app.js", - "openadapt_flow/mockloan/static/index.html", - "openadapt_flow/mockloan/static/styles.css", "openadapt_flow/mockmed/static/app.js", "openadapt_flow/mockmed/static/index.html", "openadapt_flow/mockmed/static/styles.css", diff --git a/packages/flow/pyproject.toml b/packages/flow/pyproject.toml index b4bedc3057..a45b45596e 100644 --- a/packages/flow/pyproject.toml +++ b/packages/flow/pyproject.toml @@ -197,12 +197,6 @@ reward = [ "fastapi>=0.110", "uvicorn>=0.29", ] -# Harness² integration (openadapt_flow.integrations.harness2): GUI tasks on -# resettable apps, scored by reading the system of record, for -# google-research/harness2's harness search. Harness² (Apache-2.0, Python -# 3.12+) is not on PyPI, and a published wheel cannot name a git URL, so it is -# installed beside this extra; docs/HARNESS2.md gives the pinned command. -harness2 = ["playwright>=1.44"] # Native macOS window capture/input. Imported lazily; other platforms never # install or import these framework bindings. macos = [ @@ -592,7 +586,7 @@ follow_imports_for_stubs = true # - alternate backends: backends.playwright_backend, backends.rdp_backend # - benchmark & interactive tooling: benchmark.desktop_benchmark, # interactive_recorder -# - learning-loop synthesis: learning.synth_stream +# - learning-loop synthesis: learning.induce # - disambiguation heuristics: compiler.disambiguation # - runtime helpers still on untyped deps: runtime.effects._common, # runtime.grounder, runtime.healing.perturbation @@ -608,7 +602,7 @@ module = [ "benchmark.harness.desktop_benchmark", "openadapt_flow.compiler.disambiguation", "openadapt_flow.interactive_recorder", - "openadapt_flow.learning.synth_stream", + "openadapt_flow.learning.induce", "openadapt_flow.report", "openadapt_flow.runtime.effects._common", "openadapt_flow.runtime.grounder", diff --git a/packages/flow/openadapt_flow/mockmed/teach_scope.py b/packages/flow/tests/_teach_scope.py similarity index 97% rename from packages/flow/openadapt_flow/mockmed/teach_scope.py rename to packages/flow/tests/_teach_scope.py index 170bc0b60c..811cac5052 100644 --- a/packages/flow/openadapt_flow/mockmed/teach_scope.py +++ b/packages/flow/tests/_teach_scope.py @@ -1,5 +1,7 @@ """MockMed's write adapter for a teach scope (:mod:`openadapt_flow.teach_scope`). +A test fixture: it lives in the source checkout, outside the wheel. + MockMed's fault server (:mod:`openadapt_flow.mockmed.fault_server`) takes one kind of record write: ``POST /api/encounter`` with a JSON body naming the patient, the encounter type and the note. :class:`EncounterWrites` reads that diff --git a/packages/flow/tests/e2e/test_injection_live.py b/packages/flow/tests/e2e/test_injection_live.py index 59fe7e9986..d9ff0ae339 100644 --- a/packages/flow/tests/e2e/test_injection_live.py +++ b/packages/flow/tests/e2e/test_injection_live.py @@ -42,9 +42,9 @@ from openadapt_flow.jit import TaskSignature, WebApp, compile_agent from openadapt_flow.jit.adapters import CaptureComputer, UnsupportedAction from openadapt_flow.mockmed.fault_server import serve -from openadapt_flow.mockmed.teach_scope import encounter_scope from openadapt_flow.runtime.effects import RestRecordVerifier from openadapt_flow.tutorial import _http_json, _records +from tests._teach_scope import encounter_scope LIVE = os.environ.get("OPENADAPT_INJECTION_LIVE") == "1" APPROVED_BY = os.environ.get("OPENADAPT_INJECTION_LIVE_APPROVED_BY", "").strip() diff --git a/packages/flow/tests/e2e/test_lending_fault_model.py b/packages/flow/tests/e2e/test_lending_fault_model.py index 1923442d22..374f3f6bd1 100644 --- a/packages/flow/tests/e2e/test_lending_fault_model.py +++ b/packages/flow/tests/e2e/test_lending_fault_model.py @@ -28,12 +28,12 @@ import pytest import requests +from benchmark.harness.mockloan.demo import record_disbursement_demo +from benchmark.harness.mockloan.fault_server import serve as fault_serve from benchmark.lending_fault_model import faults as F from openadapt_flow.backends.playwright_backend import PlaywrightBackend from openadapt_flow.compiler import compile_recording -from openadapt_flow.demo_driver import record_disbursement_demo from openadapt_flow.ir import Workflow -from openadapt_flow.mockloan.fault_server import serve as fault_serve from openadapt_flow.runtime import Replayer from .conftest import VIEWPORT diff --git a/packages/flow/tests/test_agent_guard.py b/packages/flow/tests/test_agent_guard.py index b57057b97e..21696760e9 100644 --- a/packages/flow/tests/test_agent_guard.py +++ b/packages/flow/tests/test_agent_guard.py @@ -14,6 +14,7 @@ import pytest +from benchmark.harness.guard_demo import NOTE, FakeAgent, person_approves, verify_yaml from openadapt_flow.agent_guard.cdp import CdpGuardPage from openadapt_flow.agent_guard.commit_guard import BLOCK_STATUS, CommitGuard from openadapt_flow.agent_guard.config import ConfigError, parse_config @@ -24,7 +25,6 @@ origin_of, ) from openadapt_flow.guard import GuardedSave, GuardTask -from openadapt_flow.guard_demo import NOTE, FakeAgent, person_approves, verify_yaml from openadapt_flow.mockmed.fault_server import serve as fault_serve from openadapt_flow.verify_run import parse_verify_config @@ -155,7 +155,7 @@ def screen_text(self) -> Optional[str]: return self.agent.open_chart(self.patient) def screenshot(self) -> Optional[bytes]: - from openadapt_flow.guard_demo import tiny_png + from benchmark.harness.guard_demo import tiny_png return tiny_png() diff --git a/packages/flow/tests/test_continuous_learning.py b/packages/flow/tests/test_continuous_learning.py index 74606077e3..ef2bf78ded 100644 --- a/packages/flow/tests/test_continuous_learning.py +++ b/packages/flow/tests/test_continuous_learning.py @@ -25,6 +25,11 @@ import pytest +from benchmark.harness.synth_stream import ( + Drift, + generate_stream, + mockmed_base_program, +) from openadapt_flow.ir import ( ProgramGraph, StateKind, @@ -38,13 +43,8 @@ program_regression_gate, program_reproduces, ) +from openadapt_flow.learning.induce import StructuralDiffInducer from openadapt_flow.learning.loop import Inducer -from openadapt_flow.learning.synth_stream import ( - Drift, - StructuralDiffInducer, - generate_stream, - mockmed_base_program, -) from openadapt_flow.runtime.identity_template import build_identity_template # -- stub inducers ------------------------------------------------------------ diff --git a/packages/flow/tests/test_guarded_save.py b/packages/flow/tests/test_guarded_save.py index 3349372137..439054f52e 100644 --- a/packages/flow/tests/test_guarded_save.py +++ b/packages/flow/tests/test_guarded_save.py @@ -1,6 +1,6 @@ """Guarded save: a fake agent writes a referral to MockMed through the guard. -The agent here is test code (``openadapt_flow.guard_demo.FakeAgent``) that +The agent here is test code (``benchmark.harness.guard_demo.FakeAgent``) that opens MockMed's server-rendered chart page and posts to its encounter API, the way a computer-use agent's click on Save would. The guard judges the write from ``GET /api/db`` alone. No model calls, nothing beyond localhost. @@ -14,15 +14,7 @@ import pytest -from openadapt_flow.agent.guard_tools import GuardBridge -from openadapt_flow.guard import ( - GUARD_TASK_SCHEMA, - GuardedSave, - GuardRefused, - GuardVerdict, - check_identity, -) -from openadapt_flow.guard_demo import ( +from benchmark.harness.guard_demo import ( NOTE, FakeAgent, person_approves, @@ -31,6 +23,14 @@ tiny_png, verify_yaml, ) +from openadapt_flow.agent.guard_tools import GuardBridge +from openadapt_flow.guard import ( + GUARD_TASK_SCHEMA, + GuardedSave, + GuardRefused, + GuardVerdict, + check_identity, +) from openadapt_flow.mockmed.fault_server import serve as fault_serve from openadapt_flow.verify_run import ( evidence_problem, diff --git a/packages/flow/tests/test_halt_learn_loop.py b/packages/flow/tests/test_halt_learn_loop.py index 458c9f81aa..5175c57a0a 100644 --- a/packages/flow/tests/test_halt_learn_loop.py +++ b/packages/flow/tests/test_halt_learn_loop.py @@ -29,6 +29,10 @@ import pytest +from benchmark.harness.synth_stream import ( + INTENT_OPEN, + mockmed_base_program, +) from openadapt_flow.ir import ( ActionKind, Anchor, @@ -51,11 +55,7 @@ promoted_workflow, resolution_demonstration, ) -from openadapt_flow.learning.synth_stream import ( - INTENT_OPEN, - StructuralDiffInducer, - mockmed_base_program, -) +from openadapt_flow.learning.induce import StructuralDiffInducer from openadapt_flow.runtime.replayer import Replayer from openadapt_flow.vision.ocr import OcrLine from tests.test_replayer import FakeBackend, FakeVision, Match, make_png diff --git a/packages/flow/tests/test_harness2_adapter.py b/packages/flow/tests/test_harness2_adapter.py index 09f0b1fed4..9f5bb149dd 100644 --- a/packages/flow/tests/test_harness2_adapter.py +++ b/packages/flow/tests/test_harness2_adapter.py @@ -13,25 +13,25 @@ import pytest -from openadapt_flow.integrations.harness2.agent import ( +from benchmark.harness.harness2.agent import ( RULE_IDENTITY, RULE_SAVE_ONCE, RULE_TYPE_LABEL, ScriptedGuiAgent, ) -from openadapt_flow.integrations.harness2.apps import MockMedTarget -from openadapt_flow.integrations.harness2.llm import ( +from benchmark.harness.harness2.apps import MockMedTarget +from benchmark.harness.harness2.llm import ( BudgetExceeded, Completion, CostLedger, ) -from openadapt_flow.integrations.harness2.rollout import ( +from benchmark.harness.harness2.rollout import ( load_run, load_score, run_rollout, ) -from openadapt_flow.integrations.harness2.scorer import ReadBackScorer -from openadapt_flow.integrations.harness2.tasks import mockmed_tasks +from benchmark.harness.harness2.scorer import ReadBackScorer +from benchmark.harness.harness2.tasks import mockmed_tasks CAREFUL = ( f"Check the {RULE_IDENTITY}. Choose the type {RULE_TYPE_LABEL}. {RULE_SAVE_ONCE}." @@ -192,7 +192,7 @@ def test_a_rollout_is_redone_when_the_harness_changed(tmp_path): def test_reward_selection_ranks_the_right_outcome_first(tmp_path, h2): from harness2.core.spec import RunView - from openadapt_flow.integrations.harness2.selection import reward_selection + from benchmark.harness.harness2.selection import reward_selection scores = { # A single save a person reconciles: right on the timeout task, unscored. @@ -214,8 +214,8 @@ def score(self, run_dir, task_id): def test_model_requests_pass_the_privacy_gate(monkeypatch, tmp_path): + from benchmark.harness.harness2.llm import HttpChatModel, Price from openadapt_flow import privacy - from openadapt_flow.integrations.harness2.llm import HttpChatModel, Price monkeypatch.setenv("OPENADAPT_HOME", str(tmp_path)) monkeypatch.delenv(privacy.OUTBOUND_OPT_OUT_ENV, raising=False) @@ -252,7 +252,7 @@ def h2(): def _adapter(tmp_path, variants, evidence="readback"): - from openadapt_flow.integrations.harness2.adapter import OpenAdaptGuiAdapter + from benchmark.harness.harness2.adapter import OpenAdaptGuiAdapter return OpenAdaptGuiAdapter( mockmed_tasks(variants), @@ -279,8 +279,8 @@ def test_adapter_is_a_harness2_benchmark_adapter(tmp_path, h2): def test_reward_selection_finds_the_safe_harness(tmp_path, h2): pytest.importorskip("playwright.sync_api") - from openadapt_flow.integrations.harness2.proposers import ScriptedProposer - from openadapt_flow.integrations.harness2.selection import run_task + from benchmark.harness.harness2.proposers import ScriptedProposer + from benchmark.harness.harness2.selection import run_task adapter = _adapter(tmp_path, ("lookalike",)) prop = ScriptedProposer( @@ -331,8 +331,8 @@ def test_screen_evidence_hides_the_readback(tmp_path, h2): def test_model_selector_path_runs_harness2_selection(tmp_path, h2): pytest.importorskip("playwright.sync_api") - from openadapt_flow.integrations.harness2.proposers import ScriptedProposer - from openadapt_flow.integrations.harness2.selection import run_task + from benchmark.harness.harness2.proposers import ScriptedProposer + from benchmark.harness.harness2.selection import run_task adapter = _adapter(tmp_path, ("plain",), evidence="screen") prop = ScriptedProposer( @@ -365,7 +365,7 @@ def complete(self, system: str, user: str, *, max_tokens: int = 800) -> Completi def test_model_proposer_writes_only_harness_files(tmp_path, h2): - from openadapt_flow.integrations.harness2.proposers import ModelProposer + from benchmark.harness.harness2.proposers import ModelProposer reply = ( '\n# New\n- check the date of birth\n\n' diff --git a/packages/flow/tests/test_harness2_pixels.py b/packages/flow/tests/test_harness2_pixels.py index 40d24b216a..774e2f1917 100644 --- a/packages/flow/tests/test_harness2_pixels.py +++ b/packages/flow/tests/test_harness2_pixels.py @@ -11,19 +11,19 @@ import pytest -from openadapt_flow.integrations.harness2.agent import ( +from benchmark.harness.harness2.agent import ( RULE_IDENTITY, RULE_TYPE_LABEL, ScriptedGuiAgent, ) -from openadapt_flow.integrations.harness2.apps import MockMedTarget -from openadapt_flow.integrations.harness2.pixels import PixelDriver, web_pixels -from openadapt_flow.integrations.harness2.rollout import ( +from benchmark.harness.harness2.apps import MockMedTarget +from benchmark.harness.harness2.pixels import PixelDriver, web_pixels +from benchmark.harness.harness2.rollout import ( load_run, load_score, run_rollout, ) -from openadapt_flow.integrations.harness2.tasks import mockmed_tasks +from benchmark.harness.harness2.tasks import mockmed_tasks CAREFUL = f"Check the {RULE_IDENTITY}. Choose the type {RULE_TYPE_LABEL}." @@ -138,8 +138,8 @@ def rdp(page: Any) -> PixelDriver: def test_model_agent_is_told_typing_appends_on_pixels(): - from openadapt_flow.integrations.harness2.agent import ModelGuiAgent, Observation - from openadapt_flow.integrations.harness2.llm import Completion + from benchmark.harness.harness2.agent import ModelGuiAgent, Observation + from benchmark.harness.harness2.llm import Completion class Model: name = "fake/model" diff --git a/packages/flow/tests/test_harness2_traces.py b/packages/flow/tests/test_harness2_traces.py index abdcec1daf..5d4ca9d60a 100644 --- a/packages/flow/tests/test_harness2_traces.py +++ b/packages/flow/tests/test_harness2_traces.py @@ -6,12 +6,12 @@ import pytest +from benchmark.harness.harness2 import traces +from benchmark.harness.harness2.agent import RULE_IDENTITY, ScriptedGuiAgent +from benchmark.harness.harness2.apps import MockMedTarget +from benchmark.harness.harness2.rollout import run_rollout +from benchmark.harness.harness2.tasks import mockmed_tasks from openadapt_flow import privacy -from openadapt_flow.integrations.harness2 import traces -from openadapt_flow.integrations.harness2.agent import RULE_IDENTITY, ScriptedGuiAgent -from openadapt_flow.integrations.harness2.apps import MockMedTarget -from openadapt_flow.integrations.harness2.rollout import run_rollout -from openadapt_flow.integrations.harness2.tasks import mockmed_tasks @pytest.fixture(autouse=True) diff --git a/packages/flow/tests/test_injection_paths.py b/packages/flow/tests/test_injection_paths.py index 21fa304d03..3fb5ffaf6d 100644 --- a/packages/flow/tests/test_injection_paths.py +++ b/packages/flow/tests/test_injection_paths.py @@ -32,8 +32,8 @@ from openadapt_flow.compile_agent import FirstRunCompiler # noqa: E402 from openadapt_flow.compile_agent.agents import AgentLedger # noqa: E402 from openadapt_flow.mockmed.fault_server import serve # noqa: E402 -from openadapt_flow.mockmed.teach_scope import EncounterWrites # noqa: E402 from openadapt_flow.tutorial import _http_json # noqa: E402 +from tests._teach_scope import EncounterWrites # noqa: E402 from tests.test_injection_suite import _Attacker, _intent, _untrusted_text # noqa: E402 NOTE = "Synthetic follow-up in two weeks" @@ -470,7 +470,7 @@ def _guarded_run( "verify": str(_guard_verify(tmp_path, base)), "allow_text": ["Triage"], **( - {"write_adapter": "openadapt_flow.mockmed.teach_scope:EncounterWrites"} + {"write_adapter": "tests._teach_scope:EncounterWrites"} if write_adapter else {} ), diff --git a/packages/flow/tests/test_injection_suite.py b/packages/flow/tests/test_injection_suite.py index 3561f02e4f..3fa9f63eb5 100644 --- a/packages/flow/tests/test_injection_suite.py +++ b/packages/flow/tests/test_injection_suite.py @@ -40,7 +40,6 @@ from openadapt_flow.jit import TaskSignature, WebApp, compile_agent # noqa: E402 from openadapt_flow.jit.adapters import CaptureComputer, UnsupportedAction # noqa: E402 from openadapt_flow.mockmed.fault_server import serve # noqa: E402 -from openadapt_flow.mockmed.teach_scope import encounter_scope # noqa: E402 from openadapt_flow.runtime.effects import RestRecordVerifier # noqa: E402 from openadapt_flow.teach_scope import ( # noqa: E402 ScopeViolation, @@ -48,6 +47,7 @@ instruction_like, ) from openadapt_flow.tutorial import _http_json, _records # noqa: E402 +from tests._teach_scope import encounter_scope # noqa: E402 NOTE = "Synthetic follow-up in two weeks" SURFACES = ("note", "banner", "tooltip", "alt", "network") diff --git a/packages/flow/tests/test_learning_gate_semantics.py b/packages/flow/tests/test_learning_gate_semantics.py index 5fa506c9b0..47d3432ab2 100644 --- a/packages/flow/tests/test_learning_gate_semantics.py +++ b/packages/flow/tests/test_learning_gate_semantics.py @@ -26,6 +26,11 @@ import pytest +from benchmark.harness.synth_stream import ( + Drift, + generate_stream, + mockmed_base_program, +) from openadapt_flow.ir import ( ActionKind, Anchor, @@ -43,12 +48,7 @@ learn_from_traces, program_regression_gate, ) -from openadapt_flow.learning.synth_stream import ( - Drift, - StructuralDiffInducer, - generate_stream, - mockmed_base_program, -) +from openadapt_flow.learning.induce import StructuralDiffInducer # -- program builders --------------------------------------------------------- diff --git a/packages/flow/tests/test_lending_swer.py b/packages/flow/tests/test_lending_swer.py index b787960357..2b62a724f9 100644 --- a/packages/flow/tests/test_lending_swer.py +++ b/packages/flow/tests/test_lending_swer.py @@ -19,11 +19,11 @@ import inspect import sqlite3 +from benchmark.harness.effectbench import summarize +from benchmark.harness.mockloan.fault_server import LedgerDB from benchmark.lending_fault_model import faults as F from benchmark.lending_fault_model import ground_truth as G from benchmark.lending_fault_model import swer as S -from benchmark.harness.effectbench import summarize -from openadapt_flow.mockloan.fault_server import LedgerDB def _row(memo: str = F.MEMO_TEXT, source: str = "replay", **kw) -> dict: diff --git a/packages/flow/tests/test_mockloan.py b/packages/flow/tests/test_mockloan.py index a401562850..629e3169c4 100644 --- a/packages/flow/tests/test_mockloan.py +++ b/packages/flow/tests/test_mockloan.py @@ -14,8 +14,8 @@ import requests from playwright.sync_api import Browser, Page, sync_playwright -from openadapt_flow.mockloan.fault_server import serve as fault_serve -from openadapt_flow.mockloan.server import serve +from benchmark.harness.mockloan.fault_server import serve as fault_serve +from benchmark.harness.mockloan.server import serve MEMO = "Funding released per approval, first advance" diff --git a/packages/flow/tests/test_probe_induction.py b/packages/flow/tests/test_probe_induction.py index 40939eb2fe..9457f96f31 100644 --- a/packages/flow/tests/test_probe_induction.py +++ b/packages/flow/tests/test_probe_induction.py @@ -41,7 +41,7 @@ from openadapt_flow.deployment import DeploymentConfig from openadapt_flow.ir import ActionKind, Anchor, StateKind, Step, Workflow from openadapt_flow.mockmed.fault_server import serve as serve_fault -from openadapt_flow.reward.seed import write_bundle +from openadapt_flow.reward.bundle import write_bundle from openadapt_flow.runtime.replayer import Replayer from tests.test_replayer import FakeBackend, FakeVision, Match, make_png diff --git a/packages/flow/tests/test_provenance.py b/packages/flow/tests/test_provenance.py index e7886cf68d..41cae68e9a 100644 --- a/packages/flow/tests/test_provenance.py +++ b/packages/flow/tests/test_provenance.py @@ -225,7 +225,7 @@ def _note_provenance(sha: str, value: str) -> ProvenanceFile: def test_verify_run_refutes_a_faithfully_saved_misread(sor, tmp_path: Path) -> None: import requests - from openadapt_flow.guard_demo import verify_yaml + from benchmark.harness.guard_demo import verify_yaml from openadapt_flow.provenance import read_sources from openadapt_flow.verify_run import ( capture_baseline, @@ -288,8 +288,8 @@ def _task_file(tmp_path: Path, note: str) -> Path: def test_the_guard_sends_a_disagreement_to_a_person(sor, tmp_path: Path) -> None: + from benchmark.harness.guard_demo import FakeAgent, verify_yaml from openadapt_flow.guard import GuardedSave, GuardRefused, load_task - from openadapt_flow.guard_demo import FakeAgent, verify_yaml from openadapt_flow.verify_run import parse_verify_config base, _db = sor diff --git a/packages/flow/tests/test_reward_trust_boundary.py b/packages/flow/tests/test_reward_trust_boundary.py index d92a0ad65a..328887651d 100644 --- a/packages/flow/tests/test_reward_trust_boundary.py +++ b/packages/flow/tests/test_reward_trust_boundary.py @@ -21,6 +21,10 @@ from openadapt_types.oracle import OracleChannel # noqa: E402 from openadapt_types.reward import RewardOutcomeV1 # noqa: E402 +from openadapt_flow.reward.bundle import ( # noqa: E402 + calibrate_bundle, + write_bundle, +) from openadapt_flow.reward.calibration import ( # noqa: E402 FAULT_CLASSES, CalibrationRefused, @@ -44,10 +48,8 @@ MOCKMED_HONEST_PATIENT, MOCKMED_LIE_PATIENT, MOCKMED_QUERY, - calibrate_bundle, mockmed_episode, seed_mockmed, - write_bundle, write_mockmed_encounter, ) from openadapt_flow.reward.worker import RewardWorker, RewardWorkerError # noqa: E402 diff --git a/packages/flow/tests/test_reward_worker.py b/packages/flow/tests/test_reward_worker.py index 08b020c6aa..67f912df70 100644 --- a/packages/flow/tests/test_reward_worker.py +++ b/packages/flow/tests/test_reward_worker.py @@ -24,6 +24,11 @@ ) import openadapt_flow.reward.callables as callables # noqa: E402 +from openadapt_flow.reward.bundle import ( # noqa: E402 + CALIBRATION_FILE, + CALIBRATION_TRIALS, + write_bundle, +) from openadapt_flow.reward.calibration import ( # noqa: E402 CorpusRecipe, clopper_pearson_upper, @@ -46,15 +51,12 @@ observation, ) from openadapt_flow.reward.seed import ( # noqa: E402 - CALIBRATION_FILE, - CALIBRATION_TRIALS, MOCKMED_DUPLICATE_PATIENT, MOCKMED_HONEST_PATIENT, MOCKMED_LIE_PATIENT, MOCKMED_QUERY, mockmed_episode, seed_mockmed, - write_bundle, write_mockmed_banner, write_mockmed_encounter, ) diff --git a/packages/flow/tests/test_teach_placement.py b/packages/flow/tests/test_teach_placement.py index 3367fc1cd7..03014ed541 100644 --- a/packages/flow/tests/test_teach_placement.py +++ b/packages/flow/tests/test_teach_placement.py @@ -45,8 +45,8 @@ Workflow, ) from openadapt_flow.learning.gate import program_regression_gate +from openadapt_flow.learning.induce import StructuralDiffInducer from openadapt_flow.learning.interpreter import program_reproduces -from openadapt_flow.learning.synth_stream import StructuralDiffInducer from openadapt_flow.learning.teach import _choose_guard_fact from openadapt_flow.learning.trace import ExecutionTrace, TraceStep from openadapt_flow.runtime.effects.effect import Effect, EffectKind diff --git a/packages/flow/tests/test_teach_scope.py b/packages/flow/tests/test_teach_scope.py index 588cf3bc92..a631e9ade1 100644 --- a/packages/flow/tests/test_teach_scope.py +++ b/packages/flow/tests/test_teach_scope.py @@ -8,7 +8,6 @@ from openadapt_flow.guard import GuardTask from openadapt_flow.jit.adapters import Op -from openadapt_flow.mockmed.teach_scope import EncounterWrites, encounter_scope from openadapt_flow.teach_scope import ( ScopeViolation, TeachGuard, @@ -17,6 +16,7 @@ origin_of, provenance_for, ) +from tests._teach_scope import EncounterWrites, encounter_scope BASE = "http://127.0.0.1:8765" @@ -349,9 +349,9 @@ def test_a_scope_with_no_origin_lets_a_browser_reach_nothing() -> None: def test_write_adapters_load_by_name() -> None: from openadapt_flow.teach_scope import load_write_adapter - adapter = load_write_adapter("openadapt_flow.mockmed.teach_scope:EncounterWrites") + adapter = load_write_adapter("tests._teach_scope:EncounterWrites") assert isinstance(adapter, EncounterWrites) with pytest.raises(ValueError, match="module:attr"): - load_write_adapter("openadapt_flow.mockmed.teach_scope") + load_write_adapter("tests._teach_scope") with pytest.raises(ValueError, match="not a write adapter"): - load_write_adapter("openadapt_flow.mockmed.teach_scope:encounter_scope") + load_write_adapter("tests._teach_scope:encounter_scope") diff --git a/packages/flow/tests/test_undo_proposal.py b/packages/flow/tests/test_undo_proposal.py index 30c493e9a8..a7f4f01a40 100644 --- a/packages/flow/tests/test_undo_proposal.py +++ b/packages/flow/tests/test_undo_proposal.py @@ -891,7 +891,7 @@ def test_deployment_test_copy_resets_and_reads_every_record(tmp_path, mockmed): from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey from openadapt_flow.deployment import load_deployment - from openadapt_flow.reward.seed import write_bundle + from openadapt_flow.reward.bundle import write_bundle from openadapt_flow.undo_proposal import DeploymentUndoTestCopy write_bundle( diff --git a/packages/flow/tests/test_verify_tools.py b/packages/flow/tests/test_verify_tools.py index a681cb68b7..5d81e31009 100644 --- a/packages/flow/tests/test_verify_tools.py +++ b/packages/flow/tests/test_verify_tools.py @@ -21,12 +21,12 @@ from mcp import ClientSession # noqa: E402 from mcp.shared.memory import create_client_server_memory_streams # noqa: E402 +from benchmark.harness.guard_demo import FakeAgent, verify_yaml # noqa: E402 from openadapt_flow import verifier # noqa: E402 from openadapt_flow.agent.bridge import BridgeError # noqa: E402 from openadapt_flow.agent.cli import main as agent_main # noqa: E402 from openadapt_flow.agent.mcp import build_server # noqa: E402 from openadapt_flow.agent.verify_tools import VerifyBridge # noqa: E402 -from openadapt_flow.guard_demo import FakeAgent, verify_yaml # noqa: E402 from openadapt_flow.mockmed.fault_server import serve as fault_serve # noqa: E402 NOTE = "Follow-up in two weeks" diff --git a/packages/flow/tests/test_verify_write.py b/packages/flow/tests/test_verify_write.py index 62ab78fc3b..cb12b3a7ee 100644 --- a/packages/flow/tests/test_verify_write.py +++ b/packages/flow/tests/test_verify_write.py @@ -14,7 +14,7 @@ import pytest -from openadapt_flow.guard_demo import FakeAgent, verify_yaml +from benchmark.harness.guard_demo import FakeAgent, verify_yaml from openadapt_flow.mockmed.fault_server import serve as fault_serve from openadapt_flow.verify import ( BaselineUnreadable, diff --git a/packages/flow/tests/test_write_ledger.py b/packages/flow/tests/test_write_ledger.py index b2cd21b90c..d2639de45f 100644 --- a/packages/flow/tests/test_write_ledger.py +++ b/packages/flow/tests/test_write_ledger.py @@ -18,7 +18,7 @@ import pytest -from openadapt_flow.guard_demo import FakeAgent, verify_yaml +from benchmark.harness.guard_demo import FakeAgent, verify_yaml from openadapt_flow.mockmed.fault_server import serve as fault_serve from openadapt_flow.verifier import ( Actor, @@ -73,7 +73,7 @@ def _landed(db) -> int: """ import json, os, signal, sys, time from pathlib import Path - from openadapt_flow.guard_demo import FakeAgent, verify_yaml + from benchmark.harness.guard_demo import FakeAgent, verify_yaml from openadapt_flow.verifier import ( Actor, AlreadyWritten, Contract, WriteHeld, begin_write, verify_write, ) diff --git a/packages/flow/uv.lock b/packages/flow/uv.lock index 349fa70602..930fb9713b 100644 --- a/packages/flow/uv.lock +++ b/packages/flow/uv.lock @@ -2372,9 +2372,6 @@ grounder = [ grounding = [ { name = "openadapt-grounding" }, ] -harness2 = [ - { name = "playwright" }, -] hosted = [ { name = "keyring" }, ] @@ -2495,7 +2492,6 @@ requires-dist = [ { name = "pillow", specifier = ">=10.0" }, { name = "playwright", marker = "extra == 'browser'", specifier = ">=1.44" }, { name = "playwright", marker = "extra == 'dev'", specifier = ">=1.44" }, - { name = "playwright", marker = "extra == 'harness2'", specifier = ">=1.44" }, { name = "pydantic", specifier = ">=2.5" }, { name = "pygobject", marker = "sys_platform == 'linux' and extra == 'linux'", specifier = ">=3.46" }, { name = "pyobjc-core", marker = "sys_platform == 'darwin' and extra == 'native-ocr'", specifier = ">=10.0" }, @@ -2534,7 +2530,7 @@ requires-dist = [ { name = "winrt-windows-media-ocr", marker = "sys_platform == 'win32' and extra == 'native-ocr'", specifier = ">=2.0" }, { name = "winrt-windows-storage-streams", marker = "sys_platform == 'win32' and extra == 'native-ocr'", specifier = ">=2.0" }, ] -provides-extras = ["native-ocr", "browser", "dev", "grounder", "grounding", "local-grounding", "console", "countersign", "receipts", "keychain", "pkcs11", "execute", "windows", "reward", "harness2", "macos", "linux", "rdp", "registry", "privacy", "hosted", "service", "service-mlx", "capture", "interop", "agent", "claude-agent-sdk", "openai-agents", "langgraph"] +provides-extras = ["native-ocr", "browser", "dev", "grounder", "grounding", "local-grounding", "console", "countersign", "receipts", "keychain", "pkcs11", "execute", "windows", "reward", "macos", "linux", "rdp", "registry", "privacy", "hosted", "service", "service-mlx", "capture", "interop", "agent", "claude-agent-sdk", "openai-agents", "langgraph"] [[package]] name = "openadapt-grounding" From 3ccd99c59410531bd58ab4faf94b1a08aeb21101 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 20:18:33 -0500 Subject: [PATCH 09/16] Guard: reword new app-name examples from main; lower the baseline to 365 --- packages/flow/openadapt_flow/ir.py | 6 +++--- packages/flow/openadapt_flow/report_redaction.py | 2 +- .../flow/openadapt_flow/runtime/actuators/form_eval.py | 6 +++--- packages/flow/scripts/generality_baseline.json | 8 ++------ 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/packages/flow/openadapt_flow/ir.py b/packages/flow/openadapt_flow/ir.py index 6366891200..5922d72681 100644 --- a/packages/flow/openadapt_flow/ir.py +++ b/packages/flow/openadapt_flow/ir.py @@ -835,7 +835,7 @@ class WebFormWrite(BaseModel): - ``ambient``: the page sent it on its own. It came from a script of another document than the form, one that was loaded before the form - opened (for example OpenEMR's tab shell, which posts + opened (for example an app's tab shell that posts ``execute_background_services.php`` on a timer), and the form page's source doesn't name it. No replaced step caused it, so the API step neither sends nor waits for it; the live page goes on as it does under @@ -886,7 +886,7 @@ class WebFormSpec(BaseModel): #: Page script the browser runs on this save that the API step can't #: evaluate by reading the form: the form's ``on*`` handlers, the ``on*`` #: handlers of the fields it sends, a page script listening for submit, - #: and validate.js rule kinds such as OpenEMR's ``pastDate``. Before each + #: and validate.js rule kinds such as a ``pastDate`` rule. Before each #: save the runtime runs that script itself, in a scratch page of the live #: session with every write blocked, on the run's values, and sends only #: when the page would submit exactly the body the API step built. When @@ -901,7 +901,7 @@ class WebFormSpec(BaseModel): submitter: Optional[str] = None submitter_requests: list[str] = Field(default_factory=list) #: The control the person clicked to confirm the save when it isn't the - #: submitter (for example OpenEMR's ``#confirmCreate`` in the duplicate + #: submitter (for example a ``#confirmCreate`` button in a duplicate #: check dialog). Running the page's script clicks it too, in whichever #: frame has it, and the page must then submit the form. confirm: Optional[str] = None diff --git a/packages/flow/openadapt_flow/report_redaction.py b/packages/flow/openadapt_flow/report_redaction.py index 1770cb75b7..5755138fc5 100644 --- a/packages/flow/openadapt_flow/report_redaction.py +++ b/packages/flow/openadapt_flow/report_redaction.py @@ -234,7 +234,7 @@ def is_credential_name(name: str, *, header: bool = False) -> bool: The name is split into parts on anything but letters and digits; a part that is a token, CSRF/XSRF token or session id makes it one - (``csrf_token_form``, ``X-Frappe-CSRF-Token``, ``app_sessid``). For a + (``csrf_token_form``, ``X-CSRF-Token``, ``app_sessid``). For a header (``header=True``) an ``authorization`` part does too (``Espo-Authorization``); elsewhere that word names an approval, not a credential. A parameter or header named so is redacted in every app and diff --git a/packages/flow/openadapt_flow/runtime/actuators/form_eval.py b/packages/flow/openadapt_flow/runtime/actuators/form_eval.py index dd12e6c5c0..2c3b776732 100644 --- a/packages/flow/openadapt_flow/runtime/actuators/form_eval.py +++ b/packages/flow/openadapt_flow/runtime/actuators/form_eval.py @@ -4,7 +4,7 @@ and submit a form. Most forms run script on the way: key and change handlers on the fields, an ``onsubmit`` validator, a click handler on the save button, a page listener for ``submit``, custom validate.js rules such -as OpenEMR's ``pastDate``. The API step can't read what that script does, +as a ``pastDate`` rule. The API step can't read what that script does, so it runs it, before each save, the way the person's steps would: 1. open the form page in a scratch copy of the live session: a separate @@ -14,8 +14,8 @@ 2. while the page loads, let only same-origin GET requests through (no other origin at all). Once it has loaded, block every request except the exact scripts and stylesheets the page loaded, the reads the person's - click made in the recording (``submitter_requests``, such as OpenEMR's - duplicate search) and the static ``.js``/``.css`` files of a recorded + click made in the recording (``submitter_requests``, such as a + duplicate-record search) and the static ``.js``/``.css`` files of a recorded read's own frame, so nothing the page does can write; a form submit or a blocked channel in any frame counts as the page sending; 3. type the run's values into the fields the person typed or picked, with diff --git a/packages/flow/scripts/generality_baseline.json b/packages/flow/scripts/generality_baseline.json index 63467d2bf5..d7e0e8672c 100644 --- a/packages/flow/scripts/generality_baseline.json +++ b/packages/flow/scripts/generality_baseline.json @@ -157,9 +157,6 @@ "mockloan": 1, "mockmed": 1 }, - "network_capture.py": { - "clearpass": 1 - }, "operator_profile.py": { "openemr": 1 }, @@ -202,9 +199,8 @@ "openemr": 2 }, "report_redaction.py": { - "csrf_token_form": 3, - "openemr": 2, - "token_main": 5 + "csrf_token_form": 2, + "token_main": 1 }, "reward/models.py": { "mockmed-records": 1 From e47cdf11598eecfa74507bbcdb5b06b31cca90db Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 20:20:36 -0500 Subject: [PATCH 10/16] Point the new teach-authorization test at tests._teach_scope --- packages/flow/tests/_teach_scope.py | 2 +- packages/flow/tests/test_teach_authorization.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/flow/tests/_teach_scope.py b/packages/flow/tests/_teach_scope.py index d34ef23de1..ad66529a15 100644 --- a/packages/flow/tests/_teach_scope.py +++ b/packages/flow/tests/_teach_scope.py @@ -109,7 +109,7 @@ def encounter_authorization_spec( "identity": {"patient_id": {"value": patient_id}}, "fields": {"type": {"value": encounter_type}, "note": {"input": "note"}}, "expected_rows": 1, - "write_adapter": "openadapt_flow.mockmed.teach_scope:EncounterWrites", + "write_adapter": "tests._teach_scope:EncounterWrites", }, "forbidden": { "actions": [ diff --git a/packages/flow/tests/test_teach_authorization.py b/packages/flow/tests/test_teach_authorization.py index 2536cfe5c5..112e0c597e 100644 --- a/packages/flow/tests/test_teach_authorization.py +++ b/packages/flow/tests/test_teach_authorization.py @@ -16,13 +16,13 @@ from openadapt_flow import teach_authorization as ta from openadapt_flow.jit.adapters import Op -from openadapt_flow.mockmed.teach_scope import ( +from openadapt_flow.teach_authorization_fixture import LocalAuthorizer +from openadapt_flow.teach_scope import ScopeViolation, TeachGuard, TeachScope +from tests._teach_scope import ( EncounterWrites, encounter_authorization_spec, encounter_scope, ) -from openadapt_flow.teach_authorization_fixture import LocalAuthorizer -from openadapt_flow.teach_scope import ScopeViolation, TeachGuard, TeachScope BASE = "http://127.0.0.1:8765" NOTE = "Synthetic follow-up in two weeks" From 10e3734b83541b5304740d6829a8f0d0e8efa006 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 20:21:28 -0500 Subject: [PATCH 11/16] Guard: take main's harness import order; reword a new MockMed example --- packages/flow/benchmark/harness/run_benchmark.py | 2 +- packages/flow/openadapt_flow/teach_authorization_fixture.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/flow/benchmark/harness/run_benchmark.py b/packages/flow/benchmark/harness/run_benchmark.py index 3aba486faf..cdfc68a791 100644 --- a/packages/flow/benchmark/harness/run_benchmark.py +++ b/packages/flow/benchmark/harness/run_benchmark.py @@ -28,9 +28,9 @@ from pathlib import Path from typing import Any, Callable -from openadapt_flow.bench import _percentile from benchmark.harness import agent_baseline from benchmark.harness.verify import verify_encounter_saved +from openadapt_flow.bench import _percentile N_COMPILED = 100 N_AGENT = 20 diff --git a/packages/flow/openadapt_flow/teach_authorization_fixture.py b/packages/flow/openadapt_flow/teach_authorization_fixture.py index 6ef5c2d289..2aa82a9c0d 100644 --- a/packages/flow/openadapt_flow/teach_authorization_fixture.py +++ b/packages/flow/openadapt_flow/teach_authorization_fixture.py @@ -1,7 +1,7 @@ """A throwaway approver for tests, demos and evaluations. A real teach authorization is signed by a person with their approver key -(:mod:`openadapt_flow.teach_authorization`). Tests, the MockMed demos and the +(:mod:`openadapt_flow.teach_authorization`). Tests, the demos and the injection evaluation need one without a person at the keyboard, so :class:`LocalAuthorizer` makes an Ed25519 approver key in memory, starts (or joins) a real signed approver roster in a key directory, and signs statements From 5e3742e08ff5b11d1c3e31d491e82bcebff14011 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 20:32:35 -0500 Subject: [PATCH 12/16] Guard: count app names inside identifiers; restore main's lint, packaging and import-guard lines --- .github/workflows/packages.yml | 5 +- packages/flow/docs/GENERALITY_AUDIT.md | 16 ++- packages/flow/openadapt_flow/__main__.py | 3 +- packages/flow/pyproject.toml | 7 +- .../build_showcase_encounter_loop_bundle.py | 4 +- .../flow/scripts/check_engine_generality.py | 25 +++- packages/flow/scripts/demo_media.py | 128 ++++++++++++------ packages/flow/scripts/desktop/pn_db.py | 15 +- .../flow/scripts/desktop/session1_launch.py | 13 +- packages/flow/scripts/desktop/uia_arm.py | 28 ++-- packages/flow/scripts/frappe_lending_demo.py | 24 +--- packages/flow/scripts/generality.toml | 56 ++++---- .../flow/scripts/generality_baseline.json | 35 +++-- packages/flow/scripts/make_demo_gif.py | 41 +++--- packages/flow/scripts/openemr_demo.py | 67 ++++----- packages/flow/scripts/openemr_param_depth.py | 52 +++---- .../flow/scripts/openimis_eligibility_demo.py | 1 + .../flow/scripts/qualify_macos_ax_identity.py | 24 ++-- .../tests/test_engine_generality_check.py | 29 ++++ .../test_runtime_never_imports_benchmark.py | 21 ++- 20 files changed, 367 insertions(+), 227 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 2eec7df276..72880b1d77 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -357,7 +357,7 @@ jobs: # every public data/config asset listed in public-artifacts.json. run: python scripts/check_release_consistency.py - name: Ruff lint - run: ruff check openadapt_flow + run: ruff check openadapt_flow benchmark/harness - name: No app-specific code in the engine # App knowledge lives in profiles, contracts and fixtures (CONTRIBUTING: # "App knowledge lives in data"); the denylist is scripts/generality.toml. @@ -376,7 +376,7 @@ jobs: ruff check --isolated --select E9 --target-version py310 . uv run --no-project --python 3.10 python -m compileall -q -x '(^|/)(\.venv|node_modules|build|dist)/' . - name: Ruff format check - run: ruff format --check openadapt_flow tests + run: ruff format --check openadapt_flow tests benchmark/harness - name: Mypy run: mypy - name: Mypy (strict, safety path only) @@ -580,6 +580,7 @@ jobs: tests/e2e/test_structural_action.py \ tests/e2e/test_compile_agent_e2e.py \ tests/e2e/test_code_action_e2e.py \ + tests/e2e/test_code_agent_default_e2e.py \ tests/e2e/test_mcp_teach_e2e.py \ tests/e2e/test_browser_teach_e2e.py \ tests/e2e/test_browser_teach_write_e2e.py \ diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md index f8f4b51798..24507e5974 100644 --- a/packages/flow/docs/GENERALITY_AUDIT.md +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -7,7 +7,9 @@ The rule: the engine has no code written for one application. What one application needs lives in data (`app_profiles/*.yaml`, generated contracts, deployment configs) or in test fixtures and benchmark harnesses outside the shipped package. A CI check (`scripts/check_engine_generality.py`) keeps it -that way; [CONTRIBUTING.md](../CONTRIBUTING.md) states the rule. +that way; [CONTRIBUTING.md](../CONTRIBUTING.md) states the rule. It counts an +app name inside an identifier too (`_openemr_base`, `isOpenEMR`); its baseline +started at 426 hits and only shrinks. ## Method @@ -37,7 +39,7 @@ that way; [CONTRIBUTING.md](../CONTRIBUTING.md) states the rule. | --- | --- | --- | | (a) comment or docstring example | 118 | — | | (b) data loaded generically | 4 | 2 (`app_profiles/openemr.yaml`, the `clinical-write` policy pack as a default) | -| (c) fixture, harness or demo in the package | 35 rows, plus 27 files of `benchmark/` | 19 | +| (c) fixture, harness or demo in the package | 35 rows, plus the 25 Python files of `benchmark/` and 2 of `instrument/` | 19 | | (d) special-casing in engine logic | 27 | 15 | Rows group consecutive lines of one hit. The full table is at the end. @@ -62,7 +64,7 @@ motivating application. | D10 | `__main__.py:2838-2934`, `:9305-9319`; `cli_variants.py:131,173-182` | `replay` and the variant probe fall back to the bundled MockMed when no URL is given; `--drift` turns off structural grounding | MockMed | Target URL required outside the demo commands | planned | | D11 | `qualification_oracle_gate.py` | `qualify accept` tests every workflow's oracle against MockMed (banner text, `/api/encounter`, `/api/db`) | MockMed | Build the fault fixture from the bundle's own effect binding | planned | | D12 | `execute/dispatch.py:111-188` | The Execute service's live runner always runs the MockMed tutorial workflow | MockMed | Replay through the admission's deployment config; MockMed only for synthetic admissions | planned | -| D13 | `compiler/compile.py:151`, `runtime/replayer.py:394`, `runtime/identity.py:108` | Thresholds measured on MockMed only | MockMed | Generic logic, one-app numbers. Recalibrate on two or more apps | follow-up issue | +| D13 | `compiler/compile.py:148`, `runtime/replayer.py:394`, `runtime/identity.py:108` | Thresholds measured on MockMed only | MockMed | Generic logic, one-app numbers. Recalibrate on two or more apps | follow-up issue | | D14 | `runtime/identity.py:275-302` | Hand-written list of non-identifying words drawn from EMR list screens | MockMed, EMRs | Derive from the recording (tokens shared by sibling rows) | follow-up issue | | D15 | `eligibility/client.py:37-41`, `:1091-1110` | One vendor's eligibility API (URL, key name, body) in the engine | Stedi | Move behind a connector interface | follow-up issue | @@ -70,7 +72,7 @@ motivating application. | Where | What | Action | Status | | --- | --- | --- | --- | -| `benchmark/` (27 Python files: `run_benchmark`, `openemr_benchmark`, `openemr_local`, `frappe_lending`, `hybrid_benchmark`, `dom_arm`, `silent_wrong_action`, `desktop_benchmark`, `verify`, `agent_baseline`, `reliability`, `chart_fonts`, `effectbench/`) | Benchmark harnesses shipped in the wheel | Moved to `packages/flow/benchmark/harness/` (import `benchmark.harness.*`). `openadapt-flow benchmark` loads it from a source checkout and says so from a wheel | done | +| `benchmark/` (25 Python files: `run_benchmark`, `openemr_benchmark`, `openemr_local`, `frappe_lending`, `hybrid_benchmark`, `dom_arm`, `silent_wrong_action`, `desktop_benchmark`, `verify`, `agent_baseline`, `reliability`, `chart_fonts`, `effectbench/`) | Benchmark harnesses shipped in the wheel | Moved to `packages/flow/benchmark/harness/` (import `benchmark.harness.*`). `openadapt-flow benchmark` loads it from a source checkout and says so from a wheel | done | | `instrument/` | Competitor-drift harness; imported the benchmark at module load, so it was already broken in a wheel | Moved to `packages/flow/benchmark/harness/instrument/` | done | | `integrations/harness2/` | Harness2 research harness on MockMed | Move to `packages/flow/benchmark/harness/harness2/` | planned | | `mockloan/`, `demo_driver.record_disbursement_demo` | Lending demo app no product command uses | Move to the benchmark tree | planned | @@ -175,7 +177,7 @@ Paths are relative to `packages/flow/openadapt_flow/` as of the audited commit. | `fleet/simulator.py:1-` | (c) | synthetic payer portal | in-package simulator fixture; move to benchmark/examples | | `guard_demo.py:1-296` | (c) | MockMed | `python -m openadapt_flow.guard_demo`: fake agent posts referral to MockMed with chart_pages READER creds; not a CLI subcommand. Move to packages/flow/examples or benchmark; importers: packages/agent/tests (1), packages/flow/tests (6). | | `instrument/ (__init__.py, competitor_drift.py)` | (c) | MockMed | MockMed competitor-drift harness. Moved to packages/flow/benchmark/harness/instrument. | -| `integrations/harness2/ (all: tasks.py, apps.py, agent.py, adapter.py, traces.py, proposers.py, scorer.py, llm.py, pixels.py, rollout.py, selection.py, __init__.py)` | (c) | MockMed | Whole subpackage is a research harness for Harness2 on MockMed: MOCKMED_PATIENT/PATIENTS (tasks.py:71-148), MockMedTarget (apps.py:50-105, /api/db at :70), regex for MockMed task text and 'triage'/'New Encounter'/'encountersaved' (agent.py:221-324), MockMed banner regex Name — MRN — DOB (traces.py:55-57,128-138), clinical prompts (adapter.py:93-120,429-445; proposers.py:50-63). Move to packages/flow/benchmark/harness2; importers: benchmark/harness2_scorer/run.py, tests/test_harness2_{adapter,pixels,traces}.py, pyproject extra 'harness2', .github/workflows/packages.yml:451. | +| `integrations/harness2/ (all: tasks.py, apps.py, agent.py, adapter.py, traces.py, proposers.py, scorer.py, llm.py, pixels.py, rollout.py, selection.py, __init__.py)` | (c) | MockMed | Whole subpackage is a research harness for Harness2 on MockMed: MOCKMED_PATIENT/PATIENTS (tasks.py:71-148), MockMedTarget (apps.py:50-105, /api/db at :70), regex for MockMed task text and 'triage'/'New Encounter'/'encountersaved' (agent.py:221-324), MockMed banner regex Name — MRN — DOB (traces.py:55-57,128-138), clinical prompts (adapter.py:93-120,429-445; proposers.py:50-63). Move to packages/flow/benchmark/harness/harness2; importers: benchmark/harness2_scorer/run.py, tests/test_harness2_{adapter,pixels,traces}.py, pyproject extra 'harness2', .github/workflows/packages.yml:451. | | `learning/synth_stream.py:1-245` | (c) | MockMed | synthetic MockMed add-patient-note stream (intents, 'Consent Required', 'Belford, Phil DOB'). Move to benchmark/ or tests/fixtures. | | `learning/synth_stream.py:246-525` | (c) | (generic) | StructuralDiffInducer is generic but lives in the MockMed harness module; imported by learning/teach.py:96, learning/branch_teach.py:76, benchmark/openemr_e2e/harness.py, 4 tests; pyproject mypy list (line 611). Move to learning/induce.py first, then move the stream out. | | `mockloan/ (server.py, fault_server.py, static/)` | (c) | MockLoan | Own synthetic lending app shipped in wheel but used by no product command; only demo_driver.record_disbursement_demo, benchmark/lending_fault_model, benchmark/undo_proposal, tests (test_mockloan, test_lending_swer, e2e/test_lending_fault_model), scripts (1). Move to packages/flow/benchmark/apps/mockloan. | @@ -304,8 +306,8 @@ Paths are relative to `packages/flow/openadapt_flow/` as of the audited commit. | `validation/dense_surface.py:1271` | (a) | OpenEMR | comment/docstring example | | `validation/dense_surface.py:5` | (a) | OpenEMR | comment/docstring example | | `verifier.py:2467` | (a) | OpenEMR | comment/docstring example | -| `vision/hashing.py:10` | (a) | MockMed | docstring for the MockMed-calibrated phash tolerance (see compile.py:151) | +| `vision/hashing.py:10` | (a) | MockMed | docstring for the MockMed-calibrated phash tolerance (see compile.py:148) | | `volatility.py:100` | (a) | OpenEMR | comment/docstring example | | `volatility.py:31` | (a) | OpenEMR | comment/docstring example | | `volatility.py:6` | (a) | OpenEMR | comment/docstring example | -| `benchmark/` (27 files) | (c) | OpenEMR, Frappe, MockMed, Open Dental | Moved to `packages/flow/benchmark/harness/` | +| `benchmark/` (25 files) | (c) | OpenEMR, Frappe, MockMed, Open Dental | Moved to `packages/flow/benchmark/harness/` | diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index 0a15c63874..2abfb44aa3 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -5458,7 +5458,8 @@ def _checkout_harness_on_path(command: str) -> bool: ) return False if str(checkout) not in sys.path: - sys.path.insert(0, str(checkout)) + # Appended, so nothing in the checkout shadows an installed module. + sys.path.append(str(checkout)) return True diff --git a/packages/flow/pyproject.toml b/packages/flow/pyproject.toml index b4bedc3057..f8723b8307 100644 --- a/packages/flow/pyproject.toml +++ b/packages/flow/pyproject.toml @@ -353,6 +353,7 @@ exclude = [ "/docs/validation/identity_roc.json", "/docs/validation/identity_roc.png", "/openadapt_flow/benchmark/reliability_corpus.py", + "/benchmark/harness/reliability_corpus.py", "/openadapt_flow/validation/adversary_corpus.py", "/openadapt_flow/validation/adversary_corpus_v2.py", "/openadapt_flow/validation/adversary_corpus_v3.py", @@ -416,6 +417,7 @@ exclude = [ "/docs/validation/identity_roc.json", "/docs/validation/identity_roc.png", "/openadapt_flow/benchmark/reliability_corpus.py", + "/benchmark/harness/reliability_corpus.py", "/openadapt_flow/validation/adversary_corpus.py", "/openadapt_flow/validation/adversary_corpus_v2.py", "/openadapt_flow/validation/adversary_corpus_v3.py", @@ -501,8 +503,8 @@ required-environments = [ target-version = "py310" line-length = 88 # Generated / non-package trees at the repo root are not part of the lint -# surface. Anchored with a leading slash so the package's own -# openadapt_flow/benchmark/ IS still linted (a bare "benchmark" would match it). +# surface. The benchmark harness (benchmark/harness/) is linted by naming it +# explicitly in CI (`ruff check openadapt_flow benchmark/harness`). extend-exclude = ["/benchmark", "/docs", "/runs", "/dist"] [tool.ruff.lint] @@ -605,7 +607,6 @@ module = [ "openadapt_flow.__main__", "openadapt_flow.backends.playwright_backend", "openadapt_flow.backends.rdp_backend", - "benchmark.harness.desktop_benchmark", "openadapt_flow.compiler.disambiguation", "openadapt_flow.interactive_recorder", "openadapt_flow.learning.synth_stream", diff --git a/packages/flow/scripts/build_showcase_encounter_loop_bundle.py b/packages/flow/scripts/build_showcase_encounter_loop_bundle.py index d829ac27d1..29e5372093 100644 --- a/packages/flow/scripts/build_showcase_encounter_loop_bundle.py +++ b/packages/flow/scripts/build_showcase_encounter_loop_bundle.py @@ -158,9 +158,7 @@ def main() -> int: _reset(LOOP_OUT) looped.save(LOOP_OUT) - print( - f"Wrote {BODY_OUT} (program:false): {len(body.steps)} demonstrated steps." - ) + print(f"Wrote {BODY_OUT} (program:false): {len(body.steps)} demonstrated steps.") print( f"Wrote {LOOP_OUT} (program:true): {len(records)} records over relation " f"'worklist', body = {len(body.steps)} demonstrated steps." diff --git a/packages/flow/scripts/check_engine_generality.py b/packages/flow/scripts/check_engine_generality.py index 536a2ee77a..d5b3976f2d 100644 --- a/packages/flow/scripts/check_engine_generality.py +++ b/packages/flow/scripts/check_engine_generality.py @@ -29,7 +29,10 @@ from collections import Counter from pathlib import Path -import tomllib +try: + import tomllib +except ModuleNotFoundError: # Python 3.10 + import tomli as tomllib HERE = Path(__file__).resolve().parent FLOW = HERE.parent @@ -40,18 +43,24 @@ #: Engine files that hold text: code and the data shipped with it. SUFFIXES = {".py", ".js", ".html", ".css", ".json", ".yaml", ".yml", ".toml", ".txt"} +#: The marker opens the comment: ``# example: ...`` or ``#: example: ...``. EXAMPLE_MARKER = re.compile(r"#:?\s*example:", re.IGNORECASE) +#: A lower-case letter or digit followed by a capital: where a camelCase word +#: splits, so ``isOpenEMR`` reads as ``is Open EMR`` for the name patterns. +_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") + def load_config(path: Path = CONFIG) -> dict: return tomllib.loads(path.read_text(encoding="utf-8")) -def _term_patterns(config: dict) -> dict[str, re.Pattern[str]]: - terms: dict[str, re.Pattern[str]] = {} +def _term_patterns(config: dict) -> dict[str, tuple[re.Pattern[str], bool]]: + """``{term: (pattern, is_name)}``; names also match inside camelCase words.""" + terms: dict[str, tuple[re.Pattern[str], bool]] = {} for section in ("names", "identifiers"): for term, pattern in config[section].items(): - terms[term] = re.compile(pattern, re.IGNORECASE) + terms[term] = (re.compile(pattern, re.IGNORECASE), section == "names") return terms @@ -86,10 +95,12 @@ def scan( continue for number, line in enumerate(text.splitlines(), start=1): stripped = line.lstrip() - if stripped.startswith("#") and EXAMPLE_MARKER.search(stripped): + if EXAMPLE_MARKER.match(stripped): continue - for term, pattern in terms.items(): - if pattern.search(line) and not _allowed(rel, term, config): + split = _CAMEL_BOUNDARY.sub(" ", line) + for term, (pattern, is_name) in terms.items(): + found = pattern.search(split if is_name else line) + if found and not _allowed(rel, term, config): hits.setdefault(rel, {}).setdefault(term, []).append(number) return hits diff --git a/packages/flow/scripts/demo_media.py b/packages/flow/scripts/demo_media.py index 701167749c..72bd91f903 100644 --- a/packages/flow/scripts/demo_media.py +++ b/packages/flow/scripts/demo_media.py @@ -247,12 +247,8 @@ def _build_master( for idx, ov in enumerate(overlays): inputs += ["-i", str(ov.png)] label = f"o{idx}" - enable = ( - ":enable='lt(mod(t,1),0.55)'" if ov.blink else "" - ) - parts.append( - f"[{last}][{idx + 1}:v]overlay={ov.x}:{ov.y}{enable}[{label}]" - ) + enable = ":enable='lt(mod(t,1),0.55)'" if ov.blink else "" + parts.append(f"[{last}][{idx + 1}:v]overlay={ov.x}:{ov.y}{enable}[{label}]") last = label filtergraph = ";".join(parts) _run( @@ -705,9 +701,7 @@ def _diff_card(patch: dict, heal: dict, base_frame: Path, card: Path) -> None: # Diff panel px, py, pw, ph = 80, 190, W - 160, 250 d.rounded_rectangle((px, py, px + pw, py + ph), radius=16, fill=CARD) - d.rounded_rectangle( - (px, py, px + pw, py + ph), radius=16, outline=LINE, width=1 - ) + d.rounded_rectangle((px, py, px + pw, py + ph), radius=16, outline=LINE, width=1) y = py + 26 changes = patch.get("changes", []) for ch in changes: @@ -733,7 +727,9 @@ def _diff_card(patch: dict, heal: dict, base_frame: Path, card: Path) -> None: continue d.text((px + 30, y), f"- {field_name}: {old!r}", font=f_mono, fill=DIFF_DEL) y += 42 - d.text((px + 30, y), f"+ {field_name}: {new!r}{ident}", font=f_mono, fill=DIFF_ADD) + d.text( + (px + 30, y), f"+ {field_name}: {new!r}{ident}", font=f_mono, fill=DIFF_ADD + ) y += 52 d.text( @@ -815,8 +811,7 @@ def build_heal(work: Path, out: Path, bundle: Path, *, headed: bool) -> Path: "-i", str(card), "-vf", - f"scale={OUT_W}:-2:flags=lanczos,fps={FPS}," - "fade=t=in:st=0:d=0.3", + f"scale={OUT_W}:-2:flags=lanczos,fps={FPS},fade=t=in:st=0:d=0.3", "-c:v", "libx264", "-profile:v", @@ -914,7 +909,9 @@ def _label( d.rounded_rectangle( (x, y, x + ww + 24, y + h), radius=10, outline=color + (255,), width=2 ) - d.rounded_rectangle((x + 10, y + 9, x + 10 + tw, y + h - 9), radius=6, fill=color + (255,)) + d.rounded_rectangle( + (x + 10, y + 9, x + 10 + tw, y + h - 9), radius=6, fill=color + (255,) + ) d.text((x + 20, y + 12), tag, font=f_tag, fill=(10, 13, 20, 255)) d.text((x + 22 + tw, y + 10), text, font=f_txt, fill=(232, 236, 244, 255)) @@ -925,7 +922,12 @@ def _anchor_box(size, region, color, alpha_line) -> Image.Image: x, y, w, h = region d.rectangle((x - 3, y - 3, x + w + 3, y + h + 3), outline=color + (255,), width=4) # corner ticks - for cx, cy in ((x - 3, y - 3), (x + w + 3, y - 3), (x - 3, y + h + 3), (x + w + 3, y + h + 3)): + for cx, cy in ( + (x - 3, y - 3), + (x + w + 3, y - 3), + (x - 3, y + h + 3), + (x + w + 3, y + h + 3), + ): d.rectangle((cx - 5, cy - 5, cx + 5, cy + 5), fill=color + (255,)) return layer @@ -939,9 +941,7 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None save_region = save["anchor"]["region"] note_region = note_click["anchor"]["region"] save_sel = save["anchor"]["structural"]["selector"] - assertion = next( - (e for e in save["expect"] if e["kind"] == "text_present"), None - ) + assertion = next((e for e in save["expect"] if e["kind"] == "text_present"), None) assert_text = assertion["text"] if assertion else "Encounter saved-" risk = save["risk"] @@ -954,21 +954,41 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None box_save = _anchor_box((1280, 800), save_region, ACCENT, 255) lbl_anchor = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) - _label(lbl_anchor, (save_region[0] + save_region[2] + 24, save_region[1] - 6), - "anchor", save_sel, ACCENT) + _label( + lbl_anchor, + (save_region[0] + save_region[2] + 24, save_region[1] - 6), + "anchor", + save_sel, + ACCENT, + ) lbl_assert = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) - _label(lbl_assert, (save_region[0] + save_region[2] + 24, save_region[1] + 54), - "assert", f'text_present "{assert_text}"', WARN) + _label( + lbl_assert, + (save_region[0] + save_region[2] + 24, save_region[1] + 54), + "assert", + f'text_present "{assert_text}"', + WARN, + ) # risk pill under the assert dr = ImageDraw.Draw(lbl_assert) rx, ry = save_region[0] + save_region[2] + 24, save_region[1] + 110 dr.rounded_rectangle((rx, ry, rx + 210, ry + 40), radius=20, fill=BAD + (255,)) - dr.text((rx + 18, ry + 8), f"risk · {risk}", font=_font(_FONT_SANS_B, 22), fill=(10, 13, 20, 255)) + dr.text( + (rx + 18, ry + 8), + f"risk · {risk}", + font=_font(_FONT_SANS_B, 22), + fill=(10, 13, 20, 255), + ) lbl_param = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) - _label(lbl_param, (note_region[0], note_region[1] - 58), - "param", f'note = "{param_val}"', GOOD) + _label( + lbl_param, + (note_region[0], note_region[1] - 58), + "param", + f'note = "{param_val}"', + GOOD, + ) # Code panel (real workflow.py step_010 excerpt), wrapped to fit. cp = tuple(save["anchor"]["click_point"]) @@ -1005,8 +1025,12 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None head = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) hd = ImageDraw.Draw(head) hd.rectangle((0, 0, 1280, 96), fill=(10, 13, 20, 210)) - hd.text((40, 24), "The demonstration becomes an editable script", - font=title_f, fill=(232, 236, 244, 255)) + hd.text( + (40, 24), + "The demonstration becomes an editable script", + font=title_f, + fill=(232, 236, 244, 255), + ) _paste_alpha(img, head, _ramp(t, 0.1, 0.5)) pulse = 0.55 + 0.45 * abs(((t * 1.6) % 1.0) - 0.5) * 2 @@ -1019,7 +1043,9 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None slide = _ramp(t, 4.6, 0.9) if slide > 0: shade = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) - ImageDraw.Draw(shade).rectangle((0, 96, 1280, 800), fill=(8, 11, 18, int(150 * slide))) + ImageDraw.Draw(shade).rectangle( + (0, 96, 1280, 800), fill=(8, 11, 18, int(150 * slide)) + ) _paste_alpha(img, shade, 1.0) px = int(1280 - (560 + 60) * slide) img.alpha_composite(panel, (px, 150)) @@ -1037,8 +1063,12 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None def _code_panel(lines: Sequence[str], width: int, height: int) -> Image.Image: panel = Image.new("RGBA", (width, height), (0, 0, 0, 0)) d = ImageDraw.Draw(panel) - d.rounded_rectangle((0, 0, width - 1, height - 1), radius=18, fill=(18, 22, 32, 255)) - d.rounded_rectangle((0, 0, width - 1, height - 1), radius=18, outline=LINE + (255,), width=1) + d.rounded_rectangle( + (0, 0, width - 1, height - 1), radius=18, fill=(18, 22, 32, 255) + ) + d.rounded_rectangle( + (0, 0, width - 1, height - 1), radius=18, outline=LINE + (255,), width=1 + ) # title bar dots for k, col in enumerate(((248, 113, 113), (251, 191, 36), (52, 211, 153))): d.ellipse((22 + k * 26, 20, 36 + k * 26, 34), fill=col + (255,)) @@ -1093,8 +1123,10 @@ def build_audit(work: Path, out: Path, run_dir: Path, *, headed: bool) -> None: with sync_playwright() as p: browser = p.chromium.launch(headless=not headed) context = browser.new_context( - viewport=view, device_scale_factor=1, - record_video_dir=str(vid_dir), record_video_size=view, + viewport=view, + device_scale_factor=1, + record_video_dir=str(vid_dir), + record_video_size=view, ) page = context.new_page() page.goto(file_url) @@ -1117,7 +1149,10 @@ def build_audit(work: Path, out: Path, run_dir: Path, *, headed: bool) -> None: _badge_png(badge, "illustrated run report", dot=ACCENT) master = work / "audit_master.mp4" _build_master( - raw, master, trim_start=0.5, speed=1.0, + raw, + master, + trim_start=0.5, + speed=1.0, overlays=[Overlay(badge, x="W-w-28", y="26")], ) # The audit clip is long and text-dense; bound the GIF fallback hard @@ -1131,9 +1166,19 @@ def build_audit(work: Path, out: Path, run_dir: Path, *, headed: bool) -> None: gif_max_s=3.0, ) # Prefer the crisp full-res top-of-report screenshot as the poster. - _run(["ffmpeg", "-y", "-i", str(poster_png), "-vf", - f"scale={OUT_W}:-2:flags=lanczos", "-q:v", "3", - str((out / "audit.jpg"))]) + _run( + [ + "ffmpeg", + "-y", + "-i", + str(poster_png), + "-vf", + f"scale={OUT_W}:-2:flags=lanczos", + "-q:v", + "3", + str((out / "audit.jpg")), + ] + ) sizes["jpg"] = (out / "audit.jpg").stat().st_size sizes["app"] = "mockmed" _update_manifest(out, "audit", sizes, "real") @@ -1226,9 +1271,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: # Everything else falls back to MockMed. No real PHI: the stock docker install # ships throwaway admin/pass creds and zero patient data. -OPENEMR_LOGIN = ( - "https://localhost:9390/interface/login/login.php?site=default" -) +OPENEMR_LOGIN = "https://localhost:9390/interface/login/login.php?site=default" def capture_record_openemr(work: Path, *, headed: bool) -> Optional[Path]: @@ -1290,9 +1333,7 @@ def center(getter) -> tuple[int, int]: page.wait_for_timeout(1200) # Open the Patient workflow via the real top nav. - rec.click( - *center(lambda: page.get_by_text("Patient", exact=True).first) - ) + rec.click(*center(lambda: page.get_by_text("Patient", exact=True).first)) page.wait_for_timeout(700) ns = page.get_by_text("New/Search", exact=False) if ns.count(): @@ -1412,7 +1453,10 @@ def build_run_openemr(work: Path, out: Path, *, headed: bool) -> bool: # would double up the indicator). See openadapt-web ReplayHero + PR #246. master = work / "run_openemr_master.mp4" _build_master( - raw, master, trim_start=0.6, speed=1.4, + raw, + master, + trim_start=0.6, + speed=1.4, overlays=[], ) sizes = _finalize(master, out / "run_openemr", poster_at=1.0) diff --git a/packages/flow/scripts/desktop/pn_db.py b/packages/flow/scripts/desktop/pn_db.py index 434f61c000..68c04e17be 100644 --- a/packages/flow/scripts/desktop/pn_db.py +++ b/packages/flow/scripts/desktop/pn_db.py @@ -86,8 +86,13 @@ def seed(drift: str = "none") -> None: def _patient_dict(r: sqlite3.Row) -> dict: - return {"id": r["id"], "first": r["first"], "last": r["last"], - "dob": r["dob"], "note": r["note"] or ""} + return { + "id": r["id"], + "first": r["first"], + "last": r["last"], + "dob": r["dob"], + "note": r["note"] or "", + } def list_patients(filt: str = "") -> None: @@ -125,8 +130,10 @@ def get(pid: int) -> None: def dump_all() -> None: con = _conn() - rows = [_patient_dict(r) for r in - con.execute("SELECT * FROM patients ORDER BY id").fetchall()] + rows = [ + _patient_dict(r) + for r in con.execute("SELECT * FROM patients ORDER BY id").fetchall() + ] con.close() print(json.dumps(rows)) diff --git a/packages/flow/scripts/desktop/session1_launch.py b/packages/flow/scripts/desktop/session1_launch.py index 13934b794d..5c6d33d52d 100644 --- a/packages/flow/scripts/desktop/session1_launch.py +++ b/packages/flow/scripts/desktop/session1_launch.py @@ -168,15 +168,20 @@ def main() -> None: import os if len(sys.argv) < 2: - print("usage: session1_launch.py [args...]", - file=sys.stderr) + print("usage: session1_launch.py [args...]", file=sys.stderr) sys.exit(2) script = os.path.abspath(sys.argv[1]) extra = list(sys.argv[2:]) if script.lower().endswith(".ps1"): # A WinForms/PowerShell UI target: run STA so WinForms works. - parts = ["powershell.exe", "-STA", "-ExecutionPolicy", "Bypass", - "-File", f'"{script}"'] + extra + parts = [ + "powershell.exe", + "-STA", + "-ExecutionPolicy", + "Bypass", + "-File", + f'"{script}"', + ] + extra else: pythonw = os.path.join(os.path.dirname(sys.executable), "pythonw.exe") if not os.path.exists(pythonw): diff --git a/packages/flow/scripts/desktop/uia_arm.py b/packages/flow/scripts/desktop/uia_arm.py index 3123032d4e..52bd4f06c3 100644 --- a/packages/flow/scripts/desktop/uia_arm.py +++ b/packages/flow/scripts/desktop/uia_arm.py @@ -35,8 +35,13 @@ import time from typing import Optional -WORKFLOW_TARGET_IDS = ["searchBox", "searchButton", "patientGrid", - "noteBox", "saveButton"] +WORKFLOW_TARGET_IDS = [ + "searchBox", + "searchButton", + "patientGrid", + "noteBox", + "saveButton", +] # The patient ROW is the identity-critical target and has NO usable id. IDENTITY_TARGET = "patient_row" @@ -149,15 +154,18 @@ def run(search: str, first: str, last: str, note: str, mode: str) -> dict: if rows: _select_row(rows[0].get("_cell")) selected_index = rows[0]["row_index"] - selected_name = f"{rows[0].get('first','')} {rows[0].get('last','')}".strip() + selected_name = ( + f"{rows[0].get('first', '')} {rows[0].get('last', '')}".strip() + ) else: # identity: exact match on rendered first AND last name text for row in rows: - if (row.get("first", "").strip().lower() == first.strip().lower() - and row.get("last", "").strip().lower() - == last.strip().lower()): + if ( + row.get("first", "").strip().lower() == first.strip().lower() + and row.get("last", "").strip().lower() == last.strip().lower() + ): _select_row(row.get("_cell")) selected_index = row["row_index"] - selected_name = f"{row.get('first','')} {row.get('last','')}".strip() + selected_name = f"{row.get('first', '')} {row.get('last', '')}".strip() break # 3. note + 4. save @@ -182,16 +190,14 @@ def main() -> None: ap.add_argument("--first", required=True) ap.add_argument("--last", required=True) ap.add_argument("--note", required=True) - ap.add_argument("--mode", choices=["positional", "identity"], - default="identity") + ap.add_argument("--mode", choices=["positional", "identity"], default="identity") args = ap.parse_args() try: out = run(args.search, args.first, args.last, args.note, args.mode) except Exception as e: # noqa: BLE001 import traceback - out = {"status": "error", "error": str(e), - "trace": traceback.format_exc()} + out = {"status": "error", "error": str(e), "trace": traceback.format_exc()} print(json.dumps(out)) diff --git a/packages/flow/scripts/frappe_lending_demo.py b/packages/flow/scripts/frappe_lending_demo.py index 13877e4925..a174d83c43 100644 --- a/packages/flow/scripts/frappe_lending_demo.py +++ b/packages/flow/scripts/frappe_lending_demo.py @@ -347,9 +347,7 @@ def record(fixture: FrappeFixture, out_dir: Path, *, headed: bool) -> Path: _preauthenticate_browser(backend, fixture) page = backend.page before_counts = fixture.table_counts() - before_non_target_loan_sha256 = ( - fixture.non_target_loan_applications_sha256() - ) + before_non_target_loan_sha256 = fixture.non_target_loan_applications_sha256() recorder = Recorder( backend, out_dir, @@ -451,9 +449,7 @@ def link_field(field: str, value: str, *, param: str) -> None: "primary_outcome": verdict.primary_outcome.value, "rest_records_sha256": _safe_records_sha256(evidence.rest_records), "db_records_sha256": _safe_records_sha256(evidence.db_records), - "before_non_target_loan_sha256": ( - evidence.before_non_target_loan_sha256 - ), + "before_non_target_loan_sha256": (evidence.before_non_target_loan_sha256), "after_non_target_loan_sha256": evidence.after_non_target_loan_sha256, "all_table_deltas": evidence.all_table_deltas, "delta_violations": evidence.delta_violations, @@ -585,9 +581,7 @@ def _capture_post_evidence( errors.append("pre-write non-target Loan Application digest unavailable") else: try: - after_non_target_loan_sha256 = ( - fixture.non_target_loan_applications_sha256() - ) + after_non_target_loan_sha256 = fixture.non_target_loan_applications_sha256() if after_non_target_loan_sha256 != before_non_target_loan_sha256: violations.append("tabLoan Application:non-target-content-changed") except Exception as exc: # noqa: BLE001 - unreadable is not clean @@ -826,9 +820,7 @@ def _persist_evidence( "before_table_counts": evidence.before_table_counts, "after_table_counts": evidence.after_table_counts, "all_table_deltas": evidence.all_table_deltas, - "before_non_target_loan_sha256": ( - evidence.before_non_target_loan_sha256 - ), + "before_non_target_loan_sha256": (evidence.before_non_target_loan_sha256), "after_non_target_loan_sha256": evidence.after_non_target_loan_sha256, "delta_violations": evidence.delta_violations, "oracle_errors": evidence.errors, @@ -1064,9 +1056,7 @@ def run_compiled_trial( # Authentication/form navigation are declared unmeasured setup; take # the collateral-delta baseline only after they finish. before_counts = fixture.table_counts() - before_non_target_loan_sha256 = ( - fixture.non_target_loan_applications_sha256() - ) + before_non_target_loan_sha256 = fixture.non_target_loan_applications_sha256() start = time.monotonic() report = Replayer(backend, effect_verifier=oracle.verifier).run( Workflow.load(bundle_dir), @@ -1155,9 +1145,7 @@ def run_agent_trial( _preauthenticate_browser(backend, fixture) _apply_condition(backend, condition) before_counts = fixture.table_counts() - before_non_target_loan_sha256 = ( - fixture.non_target_loan_applications_sha256() - ) + before_non_target_loan_sha256 = fixture.non_target_loan_applications_sha256() start = time.monotonic() result = agent_baseline.run_agent( backend, diff --git a/packages/flow/scripts/generality.toml b/packages/flow/scripts/generality.toml index 7b5b261273..8c382c226d 100644 --- a/packages/flow/scripts/generality.toml +++ b/packages/flow/scripts/generality.toml @@ -3,34 +3,36 @@ # The engine (openadapt_flow/) must not name a specific application. Add a term # here when a new application is qualified or a new app-specific identifier is # found. Patterns are Python regular expressions, matched case-insensitively -# unless they turn that off with (?-i:...). +# unless they turn that off with (?-i:...). A name matches inside an +# identifier too (`_openemr_base`, `isOpenEMR`, `to_stedi_body`): only a letter +# or digit next to it stops a match. [names] -openemr = '\bopen[ _-]?emr' -openimis = '\bopen[ _-]?imis' -medplum = '\bmedplum' -dolibarr = '\bdolibarr' -espocrm = '\bespo[ _-]?crm' -redmine = '\bredmine' -snipe-it = '\bsnipe[ _-]?it\b' -inventree = '\binventree' -kimai = '\bkimai' -openmrs = '\bopen[ _-]?mrs' -gitea = '\bgitea' -firefly-iii = '\bfirefly[ _-]?(iii|3)\b' -vikunja = '\bvikunja' -zammad = '\bzammad' -open-dental = '\bopen[ _-]?dental' -odoo = '\bodoo' -frappe = '\b(frappe|erpnext)' -patient-notes = '\bpatient[ _-]notes\b' -epic = '(?-i:\bEpic\b)' -cerner = '\bcerner' -stedi = '\bstedi\b' -textedit = '\btextedit' -notepad = '\bnotepad\b' -mockmed = '\bmock[ _-]?med' -mockloan = '\bmock[ _-]?loan' +openemr = '(? tuple[int, int]: @@ -73,8 +70,7 @@ def _center(locator) -> tuple[int, int]: box = locator.bounding_box() if box is None: raise RuntimeError(f"no bounding box for {locator}") - return (int(box["x"] + box["width"] / 2), - int(box["y"] + box["height"] / 2)) + return (int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2)) def _frame(page, url_fragment: str): @@ -153,14 +149,14 @@ def record(headed: bool = False, recording_dir: Path = RECORDING_DIR) -> Path: recorder.click(*_center(pencil)) page.wait_for_timeout(3000) pnotes = _frame(page, "pnotes_full") - recorder.click(*_center(pnotes.locator("button, a").filter( - has_text="Add").first)) + recorder.click( + *_center(pnotes.locator("button, a").filter(has_text="Add").first) + ) page.wait_for_timeout(2000) # The add-note form opens as a second nested pnotes_full iframe. recorder.click(*_center(_locate_any_frame(page, "textarea#note"))) recorder.type_text(RECORDED_NOTE, param="note") - recorder.click(*_center(_locate_any_frame( - page, "text=Save as new message"))) + recorder.click(*_center(_locate_any_frame(page, "text=Save as new message"))) page.wait_for_timeout(3000) return recorder.finish() @@ -232,35 +228,44 @@ def replay(n: int = 5) -> dict: (run_dir / "final.png").write_bytes(final_png) note_visible = _note_visible(final_png, note) except Exception as exc: # noqa: BLE001 - record and continue - summary["runs"].append({ - "run": i + 1, "success": False, "error": f"{type(exc).__name__}: {exc}", - }) + summary["runs"].append( + { + "run": i + 1, + "success": False, + "error": f"{type(exc).__name__}: {exc}", + } + ) print(f"run {i + 1}: EXCEPTION {exc}") continue finally: close() failed = [r for r in report.results if not r.ok] - summary["runs"].append({ - "run": i + 1, - "success": report.success, - "note_visible": note_visible, - "steps_ok": sum(1 for r in report.results if r.ok), - "steps_total": len(workflow.steps), - "rung_counts": report.rung_counts, - "heal_count": report.heal_count, - "total_s": round(elapsed, 1), - "first_failure": ( - {"step": failed[0].step_id, "error": failed[0].error} - if failed else None - ), - }) + summary["runs"].append( + { + "run": i + 1, + "success": report.success, + "note_visible": note_visible, + "steps_ok": sum(1 for r in report.results if r.ok), + "steps_total": len(workflow.steps), + "rung_counts": report.rung_counts, + "heal_count": report.heal_count, + "total_s": round(elapsed, 1), + "first_failure": ( + {"step": failed[0].step_id, "error": failed[0].error} + if failed + else None + ), + } + ) try: render_run_report(run_dir) except Exception as exc: # noqa: BLE001 print(f"run {i + 1}: report render failed: {exc}") - print(f"run {i + 1}: success={report.success} " - f"note_visible={note_visible} rungs={report.rung_counts} " - f"heals={report.heal_count} {elapsed:.0f}s") + print( + f"run {i + 1}: success={report.success} " + f"note_visible={note_visible} rungs={report.rung_counts} " + f"heals={report.heal_count} {elapsed:.0f}s" + ) ok = sum(1 for r in summary["runs"] if r.get("success")) verified = sum( 1 for r in summary["runs"] if r.get("success") and r.get("note_visible") diff --git a/packages/flow/scripts/openemr_param_depth.py b/packages/flow/scripts/openemr_param_depth.py index 5c77b4c57f..bafc965f1f 100644 --- a/packages/flow/scripts/openemr_param_depth.py +++ b/packages/flow/scripts/openemr_param_depth.py @@ -66,9 +66,7 @@ RUNS_DIR = WORK / "runs" RECORDED_PATIENT = "Phil" # demonstrated value; unique match "Belford, Phil" -RECORDED_NOTE = ( - "Validation baseline: reviewed home readings, plan unchanged today." -) +RECORDED_NOTE = "Validation baseline: reviewed home readings, plan unchanged today." PACING_S = 30.0 # public-demo courtesy between browser sessions @@ -99,9 +97,7 @@ def record() -> Path: page.wait_for_timeout(3000) finder = _frame(page, "dynamic_finder") - recorder.click( - *_center(finder.locator("a", has_text=RECORDED_PATIENT).first) - ) + recorder.click(*_center(finder.locator("a", has_text=RECORDED_PATIENT).first)) page.wait_for_timeout(4000) demographics = _frame(page, "demographics.php") @@ -118,14 +114,13 @@ def record() -> Path: recorder.click(*_center(pencil)) page.wait_for_timeout(3000) pnotes = _frame(page, "pnotes_full") - recorder.click(*_center( - pnotes.locator("button, a").filter(has_text="Add").first - )) + recorder.click( + *_center(pnotes.locator("button, a").filter(has_text="Add").first) + ) page.wait_for_timeout(2000) recorder.click(*_center(_locate_any_frame(page, "textarea#note"))) recorder.type_text(RECORDED_NOTE, param="note") - recorder.click(*_center(_locate_any_frame( - page, "text=Save as new message"))) + recorder.click(*_center(_locate_any_frame(page, "text=Save as new message"))) page.wait_for_timeout(3000) return recorder.finish() finally: @@ -137,9 +132,7 @@ def compile_bundle() -> None: if BUNDLE_DIR.exists(): shutil.rmtree(BUNDLE_DIR) - wf = compile_recording( - RECORDING_DIR, BUNDLE_DIR, name="openemr-param-depth" - ) + wf = compile_recording(RECORDING_DIR, BUNDLE_DIR, name="openemr-param-depth") print(f"compiled {len(wf.steps)} steps -> {BUNDLE_DIR}") for step in wf.steps: texts = [pc.text for pc in step.expect if pc.text] @@ -172,9 +165,7 @@ def _replay_one(label: str, params: dict[str, str], url: str) -> dict: note = params["note"] note_check = verify_note_saved(final_png, note) - final_text = normalize_text( - " ".join(line.text for line in ocr(final_png)) - ) + final_text = normalize_text(" ".join(line.text for line in ocr(final_png))) failed = [r for r in report.results if not r.ok] row = { "label": label, @@ -192,7 +183,8 @@ def _replay_one(label: str, params: dict[str, str], url: str) -> dict: "underwood_on_final": "underwood" in final_text, "first_failure": ( {"step": failed[0].step_id, "error": (failed[0].error or "")[:300]} - if failed else None + if failed + else None ), } print(json.dumps(row)) @@ -203,14 +195,22 @@ def replay() -> None: RUNS_DIR.mkdir(parents=True, exist_ok=True) rows = [] plan = [ - ("phil-control", { - "patient": "Phil", - "note": "Param depth control: confirmed refill pickup scheduled.", - }, DEMO_URL), - ("susan-drift", { - "patient": "Susan", - "note": "Param depth drift: dietary counseling summary delivered.", - }, DEMO_URL), + ( + "phil-control", + { + "patient": "Phil", + "note": "Param depth control: confirmed refill pickup scheduled.", + }, + DEMO_URL, + ), + ( + "susan-drift", + { + "patient": "Susan", + "note": "Param depth drift: dietary counseling summary delivered.", + }, + DEMO_URL, + ), ] for i, (label, params, url) in enumerate(plan): if i: diff --git a/packages/flow/scripts/openimis_eligibility_demo.py b/packages/flow/scripts/openimis_eligibility_demo.py index d1241f57ca..d2dbd15b77 100644 --- a/packages/flow/scripts/openimis_eligibility_demo.py +++ b/packages/flow/scripts/openimis_eligibility_demo.py @@ -212,6 +212,7 @@ def structured_text_at(self, x: int, y: int) -> str | None: context["insurance_no"] = dialogs[0][0] return json.dumps(context, separators=(",", ":")) + def eligibility_effects() -> list[Any]: """The eligibility outcome bound to the run's ``insurance_no``. diff --git a/packages/flow/scripts/qualify_macos_ax_identity.py b/packages/flow/scripts/qualify_macos_ax_identity.py index 1eb902da81..a575c9f8fe 100644 --- a/packages/flow/scripts/qualify_macos_ax_identity.py +++ b/packages/flow/scripts/qualify_macos_ax_identity.py @@ -161,7 +161,9 @@ def _terminate(pid: int) -> dict[str, Any]: return receipt -def _blocked(reason: str, missing: list[str], candidate: dict[str, Any]) -> dict[str, Any]: +def _blocked( + reason: str, missing: list[str], candidate: dict[str, Any] +) -> dict[str, Any]: return { "task": "macos_ax_structured_identity", "status": "blocked", @@ -182,7 +184,9 @@ def qualify(*, client: Optional[MacWindowClient] = None) -> dict[str, Any]: if not client.input_trusted(): missing.append("accessibility") if missing: - return _blocked("required macOS permissions are not granted", missing, candidate) + return _blocked( + "required macOS permissions are not granted", missing, candidate + ) pids_before = _textedit_pids() root = Path(tempfile.mkdtemp(prefix="oa-axid-")) @@ -264,7 +268,9 @@ def qualify(*, client: Optional[MacWindowClient] = None) -> dict[str, Any]: # file's exact content (or a timeout, recording whatever was observed). observed: Optional[str] = None px = py = 0 - enum = ax.find_candidates(window.pid, window.title, text_view_locator, limit=4000) + enum = ax.find_candidates( + window.pid, window.title, text_view_locator, limit=4000 + ) deadline = time.monotonic() + 6.0 while time.monotonic() < deadline: enum = ax.find_candidates( @@ -303,9 +309,7 @@ def qualify(*, client: Optional[MacWindowClient] = None) -> dict[str, Any]: ) locator = backend.structural_locator_at(px, py) - report["structural_locator"] = ( - None if locator is None else locator.model_dump() - ) + report["structural_locator"] = None if locator is None else locator.model_dump() handle = backend.locate_structural(locator) if locator is not None else None report["structural_handle"] = ( @@ -364,8 +368,8 @@ def qualify(*, client: Optional[MacWindowClient] = None) -> dict[str, Any]: cleanup["terminate"] = _terminate(launched_pid) pids_after = _textedit_pids() harness = {launched_pid} if launched_pid is not None else set() - cleanup["unrelated_textedit_pids_preserved"] = ( - (pids_before - harness) == (pids_after - harness) + cleanup["unrelated_textedit_pids_preserved"] = (pids_before - harness) == ( + pids_after - harness ) cleanup["unrelated_textedit_pids_after"] = sorted(pids_after - harness) try: @@ -407,7 +411,9 @@ def _write_evidence(report: dict[str, Any]) -> dict[str, Any]: ), "refusal_contract_is_unit_covered_in": "tests/test_macos_structural.py", } - ADJUDICATION_PATH.write_text(json.dumps(adjudication, indent=2, sort_keys=True) + "\n") + ADJUDICATION_PATH.write_text( + json.dumps(adjudication, indent=2, sort_keys=True) + "\n" + ) return {"sha256": digest, "evidence": str(EVIDENCE_PATH)} diff --git a/packages/flow/tests/test_engine_generality_check.py b/packages/flow/tests/test_engine_generality_check.py index 9bf2f5a22d..b794d132fd 100644 --- a/packages/flow/tests/test_engine_generality_check.py +++ b/packages/flow/tests/test_engine_generality_check.py @@ -53,6 +53,35 @@ def test_a_marked_example_comment_passes(check, tmp_path: Path) -> None: assert check.scan(engine) == {"b.py": {"openemr": [1]}, "c.py": {"openemr": [1]}} +def test_a_name_inside_an_identifier_is_a_hit(check, tmp_path: Path) -> None: + engine = _engine( + tmp_path, + { + "a.py": "def _openemr_fixup():\n pass\n", + "b.py": "APP_OPENEMR = 1\n", + "c.py": "isOpenEMR = True\n", + "d.py": "def to_stedi_body():\n pass\n", + "e.py": "EPIC_BASE = 1\n", + "f.py": "row = 'tabLoanApplication'\n", + # A name only stops at a letter or digit beside it. + "g.py": "reopenemrs = 1\n", + }, + ) + assert check.scan(engine) == { + "a.py": {"openemr": [1]}, + "b.py": {"openemr": [1]}, + "c.py": {"openemr": [1]}, + "d.py": {"stedi": [1]}, + "e.py": {"epic": [1]}, + "f.py": {"frappe-table": [1]}, + } + + +def test_the_marker_must_open_the_comment(check, tmp_path: Path) -> None: + engine = _engine(tmp_path, {"a.py": "# OpenEMR-only hack # example: no\n"}) + assert check.scan(engine) == {"a.py": {"openemr": [1]}} + + def test_app_profiles_are_data(check, tmp_path: Path) -> None: engine = _engine(tmp_path, {"app_profiles/openemr.yaml": "app: openemr\n"}) assert check.scan(engine) == {} diff --git a/packages/flow/tests/test_runtime_never_imports_benchmark.py b/packages/flow/tests/test_runtime_never_imports_benchmark.py index d59f3083d7..7d052ea956 100644 --- a/packages/flow/tests/test_runtime_never_imports_benchmark.py +++ b/packages/flow/tests/test_runtime_never_imports_benchmark.py @@ -32,9 +32,20 @@ def _is_benchmark(module: str) -> bool: ) -def _imports_benchmark(tree: ast.AST) -> list[int]: +def _imports_benchmark(tree: ast.AST, *, lazy_ok: bool = False) -> list[int]: lines: list[int] = [] - for node in ast.walk(tree): + # An allowed module may import the harness only inside a function, where + # it runs on demand from a checkout. + nodes = ( + [ + n + for n in ast.iter_child_nodes(tree) + if isinstance(n, (ast.Import, ast.ImportFrom)) + ] + if lazy_ok + else ast.walk(tree) + ) + for node in nodes: if isinstance(node, ast.ImportFrom): module = node.module or "" if node.level == 0 and _is_benchmark(module): @@ -61,10 +72,10 @@ def test_product_modules_never_import_the_benchmark_package() -> None: offenders: list[str] = [] for path in sorted(ROOT.rglob("*.py")): rel = path.relative_to(ROOT).as_posix() - if rel in ALLOWED: - continue tree = ast.parse(path.read_text(encoding="utf-8")) - offenders += [f"{rel}:{n}" for n in _imports_benchmark(tree)] + offenders += [ + f"{rel}:{n}" for n in _imports_benchmark(tree, lazy_ok=rel in ALLOWED) + ] assert not offenders, ( "product code imports the benchmark harness (the wheel and a frozen " "build leave it out); move the shared name to " From 8a01ffe503960028859fddc84493cb5b69e6ad69 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 20:32:56 -0500 Subject: [PATCH 13/16] Undo an unrelated reformat of scripts/ --- .../build_showcase_encounter_loop_bundle.py | 4 +- packages/flow/scripts/demo_media.py | 128 ++++++------------ packages/flow/scripts/desktop/pn_db.py | 15 +- .../flow/scripts/desktop/session1_launch.py | 13 +- packages/flow/scripts/desktop/uia_arm.py | 28 ++-- packages/flow/scripts/frappe_lending_demo.py | 24 +++- packages/flow/scripts/make_demo_gif.py | 41 +++--- packages/flow/scripts/openemr_demo.py | 67 +++++---- packages/flow/scripts/openemr_param_depth.py | 52 +++---- .../flow/scripts/openimis_eligibility_demo.py | 1 - .../flow/scripts/qualify_macos_ax_identity.py | 24 ++-- 11 files changed, 164 insertions(+), 233 deletions(-) diff --git a/packages/flow/scripts/build_showcase_encounter_loop_bundle.py b/packages/flow/scripts/build_showcase_encounter_loop_bundle.py index 29e5372093..d829ac27d1 100644 --- a/packages/flow/scripts/build_showcase_encounter_loop_bundle.py +++ b/packages/flow/scripts/build_showcase_encounter_loop_bundle.py @@ -158,7 +158,9 @@ def main() -> int: _reset(LOOP_OUT) looped.save(LOOP_OUT) - print(f"Wrote {BODY_OUT} (program:false): {len(body.steps)} demonstrated steps.") + print( + f"Wrote {BODY_OUT} (program:false): {len(body.steps)} demonstrated steps." + ) print( f"Wrote {LOOP_OUT} (program:true): {len(records)} records over relation " f"'worklist', body = {len(body.steps)} demonstrated steps." diff --git a/packages/flow/scripts/demo_media.py b/packages/flow/scripts/demo_media.py index 72bd91f903..701167749c 100644 --- a/packages/flow/scripts/demo_media.py +++ b/packages/flow/scripts/demo_media.py @@ -247,8 +247,12 @@ def _build_master( for idx, ov in enumerate(overlays): inputs += ["-i", str(ov.png)] label = f"o{idx}" - enable = ":enable='lt(mod(t,1),0.55)'" if ov.blink else "" - parts.append(f"[{last}][{idx + 1}:v]overlay={ov.x}:{ov.y}{enable}[{label}]") + enable = ( + ":enable='lt(mod(t,1),0.55)'" if ov.blink else "" + ) + parts.append( + f"[{last}][{idx + 1}:v]overlay={ov.x}:{ov.y}{enable}[{label}]" + ) last = label filtergraph = ";".join(parts) _run( @@ -701,7 +705,9 @@ def _diff_card(patch: dict, heal: dict, base_frame: Path, card: Path) -> None: # Diff panel px, py, pw, ph = 80, 190, W - 160, 250 d.rounded_rectangle((px, py, px + pw, py + ph), radius=16, fill=CARD) - d.rounded_rectangle((px, py, px + pw, py + ph), radius=16, outline=LINE, width=1) + d.rounded_rectangle( + (px, py, px + pw, py + ph), radius=16, outline=LINE, width=1 + ) y = py + 26 changes = patch.get("changes", []) for ch in changes: @@ -727,9 +733,7 @@ def _diff_card(patch: dict, heal: dict, base_frame: Path, card: Path) -> None: continue d.text((px + 30, y), f"- {field_name}: {old!r}", font=f_mono, fill=DIFF_DEL) y += 42 - d.text( - (px + 30, y), f"+ {field_name}: {new!r}{ident}", font=f_mono, fill=DIFF_ADD - ) + d.text((px + 30, y), f"+ {field_name}: {new!r}{ident}", font=f_mono, fill=DIFF_ADD) y += 52 d.text( @@ -811,7 +815,8 @@ def build_heal(work: Path, out: Path, bundle: Path, *, headed: bool) -> Path: "-i", str(card), "-vf", - f"scale={OUT_W}:-2:flags=lanczos,fps={FPS},fade=t=in:st=0:d=0.3", + f"scale={OUT_W}:-2:flags=lanczos,fps={FPS}," + "fade=t=in:st=0:d=0.3", "-c:v", "libx264", "-profile:v", @@ -909,9 +914,7 @@ def _label( d.rounded_rectangle( (x, y, x + ww + 24, y + h), radius=10, outline=color + (255,), width=2 ) - d.rounded_rectangle( - (x + 10, y + 9, x + 10 + tw, y + h - 9), radius=6, fill=color + (255,) - ) + d.rounded_rectangle((x + 10, y + 9, x + 10 + tw, y + h - 9), radius=6, fill=color + (255,)) d.text((x + 20, y + 12), tag, font=f_tag, fill=(10, 13, 20, 255)) d.text((x + 22 + tw, y + 10), text, font=f_txt, fill=(232, 236, 244, 255)) @@ -922,12 +925,7 @@ def _anchor_box(size, region, color, alpha_line) -> Image.Image: x, y, w, h = region d.rectangle((x - 3, y - 3, x + w + 3, y + h + 3), outline=color + (255,), width=4) # corner ticks - for cx, cy in ( - (x - 3, y - 3), - (x + w + 3, y - 3), - (x - 3, y + h + 3), - (x + w + 3, y + h + 3), - ): + for cx, cy in ((x - 3, y - 3), (x + w + 3, y - 3), (x - 3, y + h + 3), (x + w + 3, y + h + 3)): d.rectangle((cx - 5, cy - 5, cx + 5, cy + 5), fill=color + (255,)) return layer @@ -941,7 +939,9 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None save_region = save["anchor"]["region"] note_region = note_click["anchor"]["region"] save_sel = save["anchor"]["structural"]["selector"] - assertion = next((e for e in save["expect"] if e["kind"] == "text_present"), None) + assertion = next( + (e for e in save["expect"] if e["kind"] == "text_present"), None + ) assert_text = assertion["text"] if assertion else "Encounter saved-" risk = save["risk"] @@ -954,41 +954,21 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None box_save = _anchor_box((1280, 800), save_region, ACCENT, 255) lbl_anchor = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) - _label( - lbl_anchor, - (save_region[0] + save_region[2] + 24, save_region[1] - 6), - "anchor", - save_sel, - ACCENT, - ) + _label(lbl_anchor, (save_region[0] + save_region[2] + 24, save_region[1] - 6), + "anchor", save_sel, ACCENT) lbl_assert = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) - _label( - lbl_assert, - (save_region[0] + save_region[2] + 24, save_region[1] + 54), - "assert", - f'text_present "{assert_text}"', - WARN, - ) + _label(lbl_assert, (save_region[0] + save_region[2] + 24, save_region[1] + 54), + "assert", f'text_present "{assert_text}"', WARN) # risk pill under the assert dr = ImageDraw.Draw(lbl_assert) rx, ry = save_region[0] + save_region[2] + 24, save_region[1] + 110 dr.rounded_rectangle((rx, ry, rx + 210, ry + 40), radius=20, fill=BAD + (255,)) - dr.text( - (rx + 18, ry + 8), - f"risk · {risk}", - font=_font(_FONT_SANS_B, 22), - fill=(10, 13, 20, 255), - ) + dr.text((rx + 18, ry + 8), f"risk · {risk}", font=_font(_FONT_SANS_B, 22), fill=(10, 13, 20, 255)) lbl_param = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) - _label( - lbl_param, - (note_region[0], note_region[1] - 58), - "param", - f'note = "{param_val}"', - GOOD, - ) + _label(lbl_param, (note_region[0], note_region[1] - 58), + "param", f'note = "{param_val}"', GOOD) # Code panel (real workflow.py step_010 excerpt), wrapped to fit. cp = tuple(save["anchor"]["click_point"]) @@ -1025,12 +1005,8 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None head = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) hd = ImageDraw.Draw(head) hd.rectangle((0, 0, 1280, 96), fill=(10, 13, 20, 210)) - hd.text( - (40, 24), - "The demonstration becomes an editable script", - font=title_f, - fill=(232, 236, 244, 255), - ) + hd.text((40, 24), "The demonstration becomes an editable script", + font=title_f, fill=(232, 236, 244, 255)) _paste_alpha(img, head, _ramp(t, 0.1, 0.5)) pulse = 0.55 + 0.45 * abs(((t * 1.6) % 1.0) - 0.5) * 2 @@ -1043,9 +1019,7 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None slide = _ramp(t, 4.6, 0.9) if slide > 0: shade = Image.new("RGBA", (1280, 800), (0, 0, 0, 0)) - ImageDraw.Draw(shade).rectangle( - (0, 96, 1280, 800), fill=(8, 11, 18, int(150 * slide)) - ) + ImageDraw.Draw(shade).rectangle((0, 96, 1280, 800), fill=(8, 11, 18, int(150 * slide))) _paste_alpha(img, shade, 1.0) px = int(1280 - (560 + 60) * slide) img.alpha_composite(panel, (px, 150)) @@ -1063,12 +1037,8 @@ def build_compile(work: Path, out: Path, bundle: Path, base_frame: Path) -> None def _code_panel(lines: Sequence[str], width: int, height: int) -> Image.Image: panel = Image.new("RGBA", (width, height), (0, 0, 0, 0)) d = ImageDraw.Draw(panel) - d.rounded_rectangle( - (0, 0, width - 1, height - 1), radius=18, fill=(18, 22, 32, 255) - ) - d.rounded_rectangle( - (0, 0, width - 1, height - 1), radius=18, outline=LINE + (255,), width=1 - ) + d.rounded_rectangle((0, 0, width - 1, height - 1), radius=18, fill=(18, 22, 32, 255)) + d.rounded_rectangle((0, 0, width - 1, height - 1), radius=18, outline=LINE + (255,), width=1) # title bar dots for k, col in enumerate(((248, 113, 113), (251, 191, 36), (52, 211, 153))): d.ellipse((22 + k * 26, 20, 36 + k * 26, 34), fill=col + (255,)) @@ -1123,10 +1093,8 @@ def build_audit(work: Path, out: Path, run_dir: Path, *, headed: bool) -> None: with sync_playwright() as p: browser = p.chromium.launch(headless=not headed) context = browser.new_context( - viewport=view, - device_scale_factor=1, - record_video_dir=str(vid_dir), - record_video_size=view, + viewport=view, device_scale_factor=1, + record_video_dir=str(vid_dir), record_video_size=view, ) page = context.new_page() page.goto(file_url) @@ -1149,10 +1117,7 @@ def build_audit(work: Path, out: Path, run_dir: Path, *, headed: bool) -> None: _badge_png(badge, "illustrated run report", dot=ACCENT) master = work / "audit_master.mp4" _build_master( - raw, - master, - trim_start=0.5, - speed=1.0, + raw, master, trim_start=0.5, speed=1.0, overlays=[Overlay(badge, x="W-w-28", y="26")], ) # The audit clip is long and text-dense; bound the GIF fallback hard @@ -1166,19 +1131,9 @@ def build_audit(work: Path, out: Path, run_dir: Path, *, headed: bool) -> None: gif_max_s=3.0, ) # Prefer the crisp full-res top-of-report screenshot as the poster. - _run( - [ - "ffmpeg", - "-y", - "-i", - str(poster_png), - "-vf", - f"scale={OUT_W}:-2:flags=lanczos", - "-q:v", - "3", - str((out / "audit.jpg")), - ] - ) + _run(["ffmpeg", "-y", "-i", str(poster_png), "-vf", + f"scale={OUT_W}:-2:flags=lanczos", "-q:v", "3", + str((out / "audit.jpg"))]) sizes["jpg"] = (out / "audit.jpg").stat().st_size sizes["app"] = "mockmed" _update_manifest(out, "audit", sizes, "real") @@ -1271,7 +1226,9 @@ def main(argv: Optional[Sequence[str]] = None) -> int: # Everything else falls back to MockMed. No real PHI: the stock docker install # ships throwaway admin/pass creds and zero patient data. -OPENEMR_LOGIN = "https://localhost:9390/interface/login/login.php?site=default" +OPENEMR_LOGIN = ( + "https://localhost:9390/interface/login/login.php?site=default" +) def capture_record_openemr(work: Path, *, headed: bool) -> Optional[Path]: @@ -1333,7 +1290,9 @@ def center(getter) -> tuple[int, int]: page.wait_for_timeout(1200) # Open the Patient workflow via the real top nav. - rec.click(*center(lambda: page.get_by_text("Patient", exact=True).first)) + rec.click( + *center(lambda: page.get_by_text("Patient", exact=True).first) + ) page.wait_for_timeout(700) ns = page.get_by_text("New/Search", exact=False) if ns.count(): @@ -1453,10 +1412,7 @@ def build_run_openemr(work: Path, out: Path, *, headed: bool) -> bool: # would double up the indicator). See openadapt-web ReplayHero + PR #246. master = work / "run_openemr_master.mp4" _build_master( - raw, - master, - trim_start=0.6, - speed=1.4, + raw, master, trim_start=0.6, speed=1.4, overlays=[], ) sizes = _finalize(master, out / "run_openemr", poster_at=1.0) diff --git a/packages/flow/scripts/desktop/pn_db.py b/packages/flow/scripts/desktop/pn_db.py index 68c04e17be..434f61c000 100644 --- a/packages/flow/scripts/desktop/pn_db.py +++ b/packages/flow/scripts/desktop/pn_db.py @@ -86,13 +86,8 @@ def seed(drift: str = "none") -> None: def _patient_dict(r: sqlite3.Row) -> dict: - return { - "id": r["id"], - "first": r["first"], - "last": r["last"], - "dob": r["dob"], - "note": r["note"] or "", - } + return {"id": r["id"], "first": r["first"], "last": r["last"], + "dob": r["dob"], "note": r["note"] or ""} def list_patients(filt: str = "") -> None: @@ -130,10 +125,8 @@ def get(pid: int) -> None: def dump_all() -> None: con = _conn() - rows = [ - _patient_dict(r) - for r in con.execute("SELECT * FROM patients ORDER BY id").fetchall() - ] + rows = [_patient_dict(r) for r in + con.execute("SELECT * FROM patients ORDER BY id").fetchall()] con.close() print(json.dumps(rows)) diff --git a/packages/flow/scripts/desktop/session1_launch.py b/packages/flow/scripts/desktop/session1_launch.py index 5c6d33d52d..13934b794d 100644 --- a/packages/flow/scripts/desktop/session1_launch.py +++ b/packages/flow/scripts/desktop/session1_launch.py @@ -168,20 +168,15 @@ def main() -> None: import os if len(sys.argv) < 2: - print("usage: session1_launch.py [args...]", file=sys.stderr) + print("usage: session1_launch.py [args...]", + file=sys.stderr) sys.exit(2) script = os.path.abspath(sys.argv[1]) extra = list(sys.argv[2:]) if script.lower().endswith(".ps1"): # A WinForms/PowerShell UI target: run STA so WinForms works. - parts = [ - "powershell.exe", - "-STA", - "-ExecutionPolicy", - "Bypass", - "-File", - f'"{script}"', - ] + extra + parts = ["powershell.exe", "-STA", "-ExecutionPolicy", "Bypass", + "-File", f'"{script}"'] + extra else: pythonw = os.path.join(os.path.dirname(sys.executable), "pythonw.exe") if not os.path.exists(pythonw): diff --git a/packages/flow/scripts/desktop/uia_arm.py b/packages/flow/scripts/desktop/uia_arm.py index 52bd4f06c3..3123032d4e 100644 --- a/packages/flow/scripts/desktop/uia_arm.py +++ b/packages/flow/scripts/desktop/uia_arm.py @@ -35,13 +35,8 @@ import time from typing import Optional -WORKFLOW_TARGET_IDS = [ - "searchBox", - "searchButton", - "patientGrid", - "noteBox", - "saveButton", -] +WORKFLOW_TARGET_IDS = ["searchBox", "searchButton", "patientGrid", + "noteBox", "saveButton"] # The patient ROW is the identity-critical target and has NO usable id. IDENTITY_TARGET = "patient_row" @@ -154,18 +149,15 @@ def run(search: str, first: str, last: str, note: str, mode: str) -> dict: if rows: _select_row(rows[0].get("_cell")) selected_index = rows[0]["row_index"] - selected_name = ( - f"{rows[0].get('first', '')} {rows[0].get('last', '')}".strip() - ) + selected_name = f"{rows[0].get('first','')} {rows[0].get('last','')}".strip() else: # identity: exact match on rendered first AND last name text for row in rows: - if ( - row.get("first", "").strip().lower() == first.strip().lower() - and row.get("last", "").strip().lower() == last.strip().lower() - ): + if (row.get("first", "").strip().lower() == first.strip().lower() + and row.get("last", "").strip().lower() + == last.strip().lower()): _select_row(row.get("_cell")) selected_index = row["row_index"] - selected_name = f"{row.get('first', '')} {row.get('last', '')}".strip() + selected_name = f"{row.get('first','')} {row.get('last','')}".strip() break # 3. note + 4. save @@ -190,14 +182,16 @@ def main() -> None: ap.add_argument("--first", required=True) ap.add_argument("--last", required=True) ap.add_argument("--note", required=True) - ap.add_argument("--mode", choices=["positional", "identity"], default="identity") + ap.add_argument("--mode", choices=["positional", "identity"], + default="identity") args = ap.parse_args() try: out = run(args.search, args.first, args.last, args.note, args.mode) except Exception as e: # noqa: BLE001 import traceback - out = {"status": "error", "error": str(e), "trace": traceback.format_exc()} + out = {"status": "error", "error": str(e), + "trace": traceback.format_exc()} print(json.dumps(out)) diff --git a/packages/flow/scripts/frappe_lending_demo.py b/packages/flow/scripts/frappe_lending_demo.py index a174d83c43..13877e4925 100644 --- a/packages/flow/scripts/frappe_lending_demo.py +++ b/packages/flow/scripts/frappe_lending_demo.py @@ -347,7 +347,9 @@ def record(fixture: FrappeFixture, out_dir: Path, *, headed: bool) -> Path: _preauthenticate_browser(backend, fixture) page = backend.page before_counts = fixture.table_counts() - before_non_target_loan_sha256 = fixture.non_target_loan_applications_sha256() + before_non_target_loan_sha256 = ( + fixture.non_target_loan_applications_sha256() + ) recorder = Recorder( backend, out_dir, @@ -449,7 +451,9 @@ def link_field(field: str, value: str, *, param: str) -> None: "primary_outcome": verdict.primary_outcome.value, "rest_records_sha256": _safe_records_sha256(evidence.rest_records), "db_records_sha256": _safe_records_sha256(evidence.db_records), - "before_non_target_loan_sha256": (evidence.before_non_target_loan_sha256), + "before_non_target_loan_sha256": ( + evidence.before_non_target_loan_sha256 + ), "after_non_target_loan_sha256": evidence.after_non_target_loan_sha256, "all_table_deltas": evidence.all_table_deltas, "delta_violations": evidence.delta_violations, @@ -581,7 +585,9 @@ def _capture_post_evidence( errors.append("pre-write non-target Loan Application digest unavailable") else: try: - after_non_target_loan_sha256 = fixture.non_target_loan_applications_sha256() + after_non_target_loan_sha256 = ( + fixture.non_target_loan_applications_sha256() + ) if after_non_target_loan_sha256 != before_non_target_loan_sha256: violations.append("tabLoan Application:non-target-content-changed") except Exception as exc: # noqa: BLE001 - unreadable is not clean @@ -820,7 +826,9 @@ def _persist_evidence( "before_table_counts": evidence.before_table_counts, "after_table_counts": evidence.after_table_counts, "all_table_deltas": evidence.all_table_deltas, - "before_non_target_loan_sha256": (evidence.before_non_target_loan_sha256), + "before_non_target_loan_sha256": ( + evidence.before_non_target_loan_sha256 + ), "after_non_target_loan_sha256": evidence.after_non_target_loan_sha256, "delta_violations": evidence.delta_violations, "oracle_errors": evidence.errors, @@ -1056,7 +1064,9 @@ def run_compiled_trial( # Authentication/form navigation are declared unmeasured setup; take # the collateral-delta baseline only after they finish. before_counts = fixture.table_counts() - before_non_target_loan_sha256 = fixture.non_target_loan_applications_sha256() + before_non_target_loan_sha256 = ( + fixture.non_target_loan_applications_sha256() + ) start = time.monotonic() report = Replayer(backend, effect_verifier=oracle.verifier).run( Workflow.load(bundle_dir), @@ -1145,7 +1155,9 @@ def run_agent_trial( _preauthenticate_browser(backend, fixture) _apply_condition(backend, condition) before_counts = fixture.table_counts() - before_non_target_loan_sha256 = fixture.non_target_loan_applications_sha256() + before_non_target_loan_sha256 = ( + fixture.non_target_loan_applications_sha256() + ) start = time.monotonic() result = agent_baseline.run_agent( backend, diff --git a/packages/flow/scripts/make_demo_gif.py b/packages/flow/scripts/make_demo_gif.py index 92dbc7139c..337979edac 100644 --- a/packages/flow/scripts/make_demo_gif.py +++ b/packages/flow/scripts/make_demo_gif.py @@ -75,8 +75,12 @@ def _panel(draw, img, shot: Path, x: int, label: str, label_color): if shot.is_file(): panel = Image.open(shot).convert("RGB").resize((PANEL_W, PANEL_H)) img.paste(panel, (x, y)) - draw.rectangle([x - 1, y - 1, x + PANEL_W, y + PANEL_H], outline=(60, 60, 90)) - _text_center(draw, (x + PANEL_W / 2, y + PANEL_H + 10), label, F_BODY, label_color) + draw.rectangle( + [x - 1, y - 1, x + PANEL_W, y + PANEL_H], outline=(60, 60, 90) + ) + _text_center( + draw, (x + PANEL_W / 2, y + PANEL_H + 10), label, F_BODY, label_color + ) _PLAIN_RUNG = { @@ -130,43 +134,30 @@ def build( continue img, draw = _frame() _text_center( - draw, - (CANVAS_W / 2, 8), + draw, (CANVAS_W / 2, 8), "Left: the theme it was recorded on. Right: a dark theme it never saw.", - F_SMALL, - DIM, + F_SMALL, DIM, ) _text_center( - draw, - (CANVAS_W / 2, 32), + draw, (CANVAS_W / 2, 32), f"Step {number} of {len(results)}: {_plain_intent(res_b['intent'])}", - F_BODY, - FG, + F_BODY, FG, ) cap_b, col_b = _step_caption(res_b, healed=False) cap_d, col_d = _step_caption(res_d, healed=res_d.get("heal") is not None) _panel( - draw, - img, - baseline_dir / res_b["after_png"], - MARGIN, - f"Recorded theme: {cap_b}", - col_b, + draw, img, baseline_dir / res_b["after_png"], MARGIN, + f"Recorded theme: {cap_b}", col_b, ) _panel( - draw, - img, - drift_dir / res_d["after_png"], + draw, img, drift_dir / res_d["after_png"], MARGIN + PANEL_W + GUTTER, - f"Dark theme: {cap_d}", - col_d, + f"Dark theme: {cap_d}", col_d, ) _text_center( - draw, - (CANVAS_W / 2, CANVAS_H - FOOTER_H + 12), + draw, (CANVAS_W / 2, CANVAS_H - FOOTER_H + 12), "One recording, replayed on two themes. No model calls.", - F_SMALL, - DIM, + F_SMALL, DIM, ) frames.append(img) durations.append(1400 if res_d.get("heal") else 1000) diff --git a/packages/flow/scripts/openemr_demo.py b/packages/flow/scripts/openemr_demo.py index 2a6a49bbd8..b9a75aee21 100644 --- a/packages/flow/scripts/openemr_demo.py +++ b/packages/flow/scripts/openemr_demo.py @@ -49,7 +49,9 @@ DEMO_PASS = "pass" PATIENT_SEARCH = "Phil" # matches demo patient "Belford, Phil" only -RECORDED_NOTE = "Reviewed hypertension follow-up; BP within target range today." +RECORDED_NOTE = ( + "Reviewed hypertension follow-up; BP within target range today." +) # Distinct per-run values so replay success proves parameter substitution, # not replay of the baked-in literal. @@ -61,7 +63,8 @@ "Replay run 5: home BP log reviewed, readings stable.", ] -SETTLE = dict(settle_timeout_s=10.0, settle_stable_frames=3, settle_interval_s=0.3) +SETTLE = dict(settle_timeout_s=10.0, settle_stable_frames=3, + settle_interval_s=0.3) def _center(locator) -> tuple[int, int]: @@ -70,7 +73,8 @@ def _center(locator) -> tuple[int, int]: box = locator.bounding_box() if box is None: raise RuntimeError(f"no bounding box for {locator}") - return (int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2)) + return (int(box["x"] + box["width"] / 2), + int(box["y"] + box["height"] / 2)) def _frame(page, url_fragment: str): @@ -149,14 +153,14 @@ def record(headed: bool = False, recording_dir: Path = RECORDING_DIR) -> Path: recorder.click(*_center(pencil)) page.wait_for_timeout(3000) pnotes = _frame(page, "pnotes_full") - recorder.click( - *_center(pnotes.locator("button, a").filter(has_text="Add").first) - ) + recorder.click(*_center(pnotes.locator("button, a").filter( + has_text="Add").first)) page.wait_for_timeout(2000) # The add-note form opens as a second nested pnotes_full iframe. recorder.click(*_center(_locate_any_frame(page, "textarea#note"))) recorder.type_text(RECORDED_NOTE, param="note") - recorder.click(*_center(_locate_any_frame(page, "text=Save as new message"))) + recorder.click(*_center(_locate_any_frame( + page, "text=Save as new message"))) page.wait_for_timeout(3000) return recorder.finish() @@ -228,44 +232,35 @@ def replay(n: int = 5) -> dict: (run_dir / "final.png").write_bytes(final_png) note_visible = _note_visible(final_png, note) except Exception as exc: # noqa: BLE001 - record and continue - summary["runs"].append( - { - "run": i + 1, - "success": False, - "error": f"{type(exc).__name__}: {exc}", - } - ) + summary["runs"].append({ + "run": i + 1, "success": False, "error": f"{type(exc).__name__}: {exc}", + }) print(f"run {i + 1}: EXCEPTION {exc}") continue finally: close() failed = [r for r in report.results if not r.ok] - summary["runs"].append( - { - "run": i + 1, - "success": report.success, - "note_visible": note_visible, - "steps_ok": sum(1 for r in report.results if r.ok), - "steps_total": len(workflow.steps), - "rung_counts": report.rung_counts, - "heal_count": report.heal_count, - "total_s": round(elapsed, 1), - "first_failure": ( - {"step": failed[0].step_id, "error": failed[0].error} - if failed - else None - ), - } - ) + summary["runs"].append({ + "run": i + 1, + "success": report.success, + "note_visible": note_visible, + "steps_ok": sum(1 for r in report.results if r.ok), + "steps_total": len(workflow.steps), + "rung_counts": report.rung_counts, + "heal_count": report.heal_count, + "total_s": round(elapsed, 1), + "first_failure": ( + {"step": failed[0].step_id, "error": failed[0].error} + if failed else None + ), + }) try: render_run_report(run_dir) except Exception as exc: # noqa: BLE001 print(f"run {i + 1}: report render failed: {exc}") - print( - f"run {i + 1}: success={report.success} " - f"note_visible={note_visible} rungs={report.rung_counts} " - f"heals={report.heal_count} {elapsed:.0f}s" - ) + print(f"run {i + 1}: success={report.success} " + f"note_visible={note_visible} rungs={report.rung_counts} " + f"heals={report.heal_count} {elapsed:.0f}s") ok = sum(1 for r in summary["runs"] if r.get("success")) verified = sum( 1 for r in summary["runs"] if r.get("success") and r.get("note_visible") diff --git a/packages/flow/scripts/openemr_param_depth.py b/packages/flow/scripts/openemr_param_depth.py index bafc965f1f..5c77b4c57f 100644 --- a/packages/flow/scripts/openemr_param_depth.py +++ b/packages/flow/scripts/openemr_param_depth.py @@ -66,7 +66,9 @@ RUNS_DIR = WORK / "runs" RECORDED_PATIENT = "Phil" # demonstrated value; unique match "Belford, Phil" -RECORDED_NOTE = "Validation baseline: reviewed home readings, plan unchanged today." +RECORDED_NOTE = ( + "Validation baseline: reviewed home readings, plan unchanged today." +) PACING_S = 30.0 # public-demo courtesy between browser sessions @@ -97,7 +99,9 @@ def record() -> Path: page.wait_for_timeout(3000) finder = _frame(page, "dynamic_finder") - recorder.click(*_center(finder.locator("a", has_text=RECORDED_PATIENT).first)) + recorder.click( + *_center(finder.locator("a", has_text=RECORDED_PATIENT).first) + ) page.wait_for_timeout(4000) demographics = _frame(page, "demographics.php") @@ -114,13 +118,14 @@ def record() -> Path: recorder.click(*_center(pencil)) page.wait_for_timeout(3000) pnotes = _frame(page, "pnotes_full") - recorder.click( - *_center(pnotes.locator("button, a").filter(has_text="Add").first) - ) + recorder.click(*_center( + pnotes.locator("button, a").filter(has_text="Add").first + )) page.wait_for_timeout(2000) recorder.click(*_center(_locate_any_frame(page, "textarea#note"))) recorder.type_text(RECORDED_NOTE, param="note") - recorder.click(*_center(_locate_any_frame(page, "text=Save as new message"))) + recorder.click(*_center(_locate_any_frame( + page, "text=Save as new message"))) page.wait_for_timeout(3000) return recorder.finish() finally: @@ -132,7 +137,9 @@ def compile_bundle() -> None: if BUNDLE_DIR.exists(): shutil.rmtree(BUNDLE_DIR) - wf = compile_recording(RECORDING_DIR, BUNDLE_DIR, name="openemr-param-depth") + wf = compile_recording( + RECORDING_DIR, BUNDLE_DIR, name="openemr-param-depth" + ) print(f"compiled {len(wf.steps)} steps -> {BUNDLE_DIR}") for step in wf.steps: texts = [pc.text for pc in step.expect if pc.text] @@ -165,7 +172,9 @@ def _replay_one(label: str, params: dict[str, str], url: str) -> dict: note = params["note"] note_check = verify_note_saved(final_png, note) - final_text = normalize_text(" ".join(line.text for line in ocr(final_png))) + final_text = normalize_text( + " ".join(line.text for line in ocr(final_png)) + ) failed = [r for r in report.results if not r.ok] row = { "label": label, @@ -183,8 +192,7 @@ def _replay_one(label: str, params: dict[str, str], url: str) -> dict: "underwood_on_final": "underwood" in final_text, "first_failure": ( {"step": failed[0].step_id, "error": (failed[0].error or "")[:300]} - if failed - else None + if failed else None ), } print(json.dumps(row)) @@ -195,22 +203,14 @@ def replay() -> None: RUNS_DIR.mkdir(parents=True, exist_ok=True) rows = [] plan = [ - ( - "phil-control", - { - "patient": "Phil", - "note": "Param depth control: confirmed refill pickup scheduled.", - }, - DEMO_URL, - ), - ( - "susan-drift", - { - "patient": "Susan", - "note": "Param depth drift: dietary counseling summary delivered.", - }, - DEMO_URL, - ), + ("phil-control", { + "patient": "Phil", + "note": "Param depth control: confirmed refill pickup scheduled.", + }, DEMO_URL), + ("susan-drift", { + "patient": "Susan", + "note": "Param depth drift: dietary counseling summary delivered.", + }, DEMO_URL), ] for i, (label, params, url) in enumerate(plan): if i: diff --git a/packages/flow/scripts/openimis_eligibility_demo.py b/packages/flow/scripts/openimis_eligibility_demo.py index d2dbd15b77..d1241f57ca 100644 --- a/packages/flow/scripts/openimis_eligibility_demo.py +++ b/packages/flow/scripts/openimis_eligibility_demo.py @@ -212,7 +212,6 @@ def structured_text_at(self, x: int, y: int) -> str | None: context["insurance_no"] = dialogs[0][0] return json.dumps(context, separators=(",", ":")) - def eligibility_effects() -> list[Any]: """The eligibility outcome bound to the run's ``insurance_no``. diff --git a/packages/flow/scripts/qualify_macos_ax_identity.py b/packages/flow/scripts/qualify_macos_ax_identity.py index a575c9f8fe..1eb902da81 100644 --- a/packages/flow/scripts/qualify_macos_ax_identity.py +++ b/packages/flow/scripts/qualify_macos_ax_identity.py @@ -161,9 +161,7 @@ def _terminate(pid: int) -> dict[str, Any]: return receipt -def _blocked( - reason: str, missing: list[str], candidate: dict[str, Any] -) -> dict[str, Any]: +def _blocked(reason: str, missing: list[str], candidate: dict[str, Any]) -> dict[str, Any]: return { "task": "macos_ax_structured_identity", "status": "blocked", @@ -184,9 +182,7 @@ def qualify(*, client: Optional[MacWindowClient] = None) -> dict[str, Any]: if not client.input_trusted(): missing.append("accessibility") if missing: - return _blocked( - "required macOS permissions are not granted", missing, candidate - ) + return _blocked("required macOS permissions are not granted", missing, candidate) pids_before = _textedit_pids() root = Path(tempfile.mkdtemp(prefix="oa-axid-")) @@ -268,9 +264,7 @@ def qualify(*, client: Optional[MacWindowClient] = None) -> dict[str, Any]: # file's exact content (or a timeout, recording whatever was observed). observed: Optional[str] = None px = py = 0 - enum = ax.find_candidates( - window.pid, window.title, text_view_locator, limit=4000 - ) + enum = ax.find_candidates(window.pid, window.title, text_view_locator, limit=4000) deadline = time.monotonic() + 6.0 while time.monotonic() < deadline: enum = ax.find_candidates( @@ -309,7 +303,9 @@ def qualify(*, client: Optional[MacWindowClient] = None) -> dict[str, Any]: ) locator = backend.structural_locator_at(px, py) - report["structural_locator"] = None if locator is None else locator.model_dump() + report["structural_locator"] = ( + None if locator is None else locator.model_dump() + ) handle = backend.locate_structural(locator) if locator is not None else None report["structural_handle"] = ( @@ -368,8 +364,8 @@ def qualify(*, client: Optional[MacWindowClient] = None) -> dict[str, Any]: cleanup["terminate"] = _terminate(launched_pid) pids_after = _textedit_pids() harness = {launched_pid} if launched_pid is not None else set() - cleanup["unrelated_textedit_pids_preserved"] = (pids_before - harness) == ( - pids_after - harness + cleanup["unrelated_textedit_pids_preserved"] = ( + (pids_before - harness) == (pids_after - harness) ) cleanup["unrelated_textedit_pids_after"] = sorted(pids_after - harness) try: @@ -411,9 +407,7 @@ def _write_evidence(report: dict[str, Any]) -> dict[str, Any]: ), "refusal_contract_is_unit_covered_in": "tests/test_macos_structural.py", } - ADJUDICATION_PATH.write_text( - json.dumps(adjudication, indent=2, sort_keys=True) + "\n" - ) + ADJUDICATION_PATH.write_text(json.dumps(adjudication, indent=2, sort_keys=True) + "\n") return {"sha256": digest, "evidence": str(EVIDENCE_PATH)} From 23b46d65de63fedc8961296843c73b346ebeb471 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 20:34:13 -0500 Subject: [PATCH 14/16] Restore #1741's packaging excludes, lazy-import guard and sys.path.append lost in the main merge --- packages/flow/CHANGELOG.md | 4 ++-- packages/flow/docs/GENERALITY_AUDIT.md | 17 ++++++++------- packages/flow/openadapt_flow/__main__.py | 3 ++- packages/flow/pyproject.toml | 7 ++++--- .../test_runtime_never_imports_benchmark.py | 21 ++++++++++++++----- 5 files changed, 33 insertions(+), 19 deletions(-) diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md index 608ad9f850..b46fcc2b77 100644 --- a/packages/flow/CHANGELOG.md +++ b/packages/flow/CHANGELOG.md @@ -21,8 +21,6 @@ - The wheel no longer ships the benchmark harnesses. `openadapt_flow.benchmark` and `openadapt_flow.instrument` moved to `packages/flow/benchmark/harness/` in the source tree (import `benchmark.harness`); `openadapt-flow benchmark` runs from a source checkout and says so from a wheel. The engine audit for application-specific code is in `docs/GENERALITY_AUDIT.md`. Code that imported `openadapt_flow.benchmark` or `openadapt_flow.instrument` from an installed wheel imports `benchmark.harness` from a source checkout instead. -- The wheel no longer ships test fixtures and demo harnesses: the Harness² integration, the MockLoan app and its recorder, `guard_demo`, the MockMed teach-scope adapter, the synthetic drift stream (the `StructuralDiffInducer` it used now lives in `openadapt_flow.learning.induce`) and the task-ledger simulation moved to `benchmark/harness/` or beside their tests, the reward bundle helpers moved to `openadapt_flow.reward.bundle`, and the `harness2` extra is gone. - ### Features - Teach authorizations (`openadapt.teach-authorization/v1`). A person signs, with an approver key on the signed approver roster, what an agent may do when it teaches or repairs a program: the task, origins, inputs, the one write and forbidden actions and controls. `openadapt-flow teach-authorization create|sign|verify`; `--teach-authorization` on `compile-agent` and `openadapt-agent serve --teach`. Design and the upgrade notes: `docs/TEACH_AUTHORIZATION.md`. @@ -39,6 +37,8 @@ - A covered-band dismiss step takes its dialog box from the window the backend proved covers the band, before any pixel search, so a native message box that the accessibility tree doesn't label as a dialog can be dismissed. Without a proved window the step is refused as before. +- The wheel no longer ships test fixtures and demo harnesses: the Harness² integration, the MockLoan app and its recorder, `guard_demo`, the MockMed teach-scope adapter, the synthetic drift stream (the `StructuralDiffInducer` it used now lives in `openadapt_flow.learning.induce`) and the task-ledger simulation moved to `benchmark/harness/` or beside their tests, the reward bundle helpers moved to `openadapt_flow.reward.bundle`, and the `harness2` extra is gone. + - Self-repair on a covered identity band dismisses only the proved dialog: the covering window must span the band from where the read stops to its right edge at full height, the dismiss click must land in that same window, and only a dialog whose every word is a notice word (no question) may be dismissed ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). - Self-repair can take a stop where the app's own dialog covers the next step's identity band: when the backend proves the covering window belongs to the window the last verified step was in, and the part of the band read still matches the record, the dialog is dismissed after the previous click and the step re-checks its identity in full. Another record behind the dialog stays an identity halt. The runtime lets a backend remember a verified record's window (`record_verified_window`); macOS answers `owned_window_over` from the window's proved sheet ([#1724](https://github.com/OpenAdaptAI/OpenAdapt/issues/1724)). diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md index ece04e3645..8f1982197e 100644 --- a/packages/flow/docs/GENERALITY_AUDIT.md +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -6,8 +6,9 @@ Audited 2026-11-03 on `main` at `213f3d9fe`. Scope: everything under The rule: the engine has no code written for one application. What one application needs lives in data (`app_profiles/*.yaml`, generated contracts, deployment configs) or in test fixtures and benchmark harnesses outside the -shipped package. A CI check (`scripts/check_engine_generality.py`) keeps it -that way; [CONTRIBUTING.md](../CONTRIBUTING.md) states the rule. +shipped package. The CI check that keeps it that way, +`scripts/check_engine_generality.py`, lands in the pull request that follows +this audit, together with the rule in [CONTRIBUTING.md](../CONTRIBUTING.md). ## Method @@ -37,7 +38,7 @@ that way; [CONTRIBUTING.md](../CONTRIBUTING.md) states the rule. | --- | --- | --- | | (a) comment or docstring example | 118 | — | | (b) data loaded generically | 4 | 2 (`app_profiles/openemr.yaml`, the `clinical-write` policy pack as a default) | -| (c) fixture, harness or demo in the package | 35 rows, plus 27 files of `benchmark/` | 19 | +| (c) fixture, harness or demo in the package | 35 rows, plus the 25 Python files of `benchmark/` and 2 of `instrument/` | 19 | | (d) special-casing in engine logic | 27 | 15 | Rows group consecutive lines of one hit. The full table is at the end. @@ -62,7 +63,7 @@ motivating application. | D10 | `__main__.py:2838-2934`, `:9305-9319`; `cli_variants.py:131,173-182` | `replay` and the variant probe fall back to the bundled MockMed when no URL is given; `--drift` turns off structural grounding | MockMed | Target URL required outside the demo commands | planned | | D11 | `qualification_oracle_gate.py` | `qualify accept` tests every workflow's oracle against MockMed (banner text, `/api/encounter`, `/api/db`) | MockMed | Build the fault fixture from the bundle's own effect binding | planned | | D12 | `execute/dispatch.py:111-188` | The Execute service's live runner always runs the MockMed tutorial workflow | MockMed | Replay through the admission's deployment config; MockMed only for synthetic admissions | planned | -| D13 | `compiler/compile.py:151`, `runtime/replayer.py:394`, `runtime/identity.py:108` | Thresholds measured on MockMed only | MockMed | Generic logic, one-app numbers. Recalibrate on two or more apps | follow-up issue | +| D13 | `compiler/compile.py:148`, `runtime/replayer.py:394`, `runtime/identity.py:108` | Thresholds measured on MockMed only | MockMed | Generic logic, one-app numbers. Recalibrate on two or more apps | follow-up issue | | D14 | `runtime/identity.py:275-302` | Hand-written list of non-identifying words drawn from EMR list screens | MockMed, EMRs | Derive from the recording (tokens shared by sibling rows) | follow-up issue | | D15 | `eligibility/client.py:37-41`, `:1091-1110` | One vendor's eligibility API (URL, key name, body) in the engine | Stedi | Move behind a connector interface | follow-up issue | @@ -70,7 +71,7 @@ motivating application. | Where | What | Action | Status | | --- | --- | --- | --- | -| `benchmark/` (27 Python files: `run_benchmark`, `openemr_benchmark`, `openemr_local`, `frappe_lending`, `hybrid_benchmark`, `dom_arm`, `silent_wrong_action`, `desktop_benchmark`, `verify`, `agent_baseline`, `reliability`, `chart_fonts`, `effectbench/`) | Benchmark harnesses shipped in the wheel | Moved to `packages/flow/benchmark/harness/` (import `benchmark.harness.*`). `openadapt-flow benchmark` loads it from a source checkout and says so from a wheel | done | +| `benchmark/` (25 Python files: `run_benchmark`, `openemr_benchmark`, `openemr_local`, `frappe_lending`, `hybrid_benchmark`, `dom_arm`, `silent_wrong_action`, `desktop_benchmark`, `verify`, `agent_baseline`, `reliability`, `chart_fonts`, `effectbench/`) | Benchmark harnesses shipped in the wheel | Moved to `packages/flow/benchmark/harness/` (import `benchmark.harness.*`). `openadapt-flow benchmark` loads it from a source checkout and says so from a wheel | done | | `instrument/` | Competitor-drift harness; imported the benchmark at module load, so it was already broken in a wheel | Moved to `packages/flow/benchmark/harness/instrument/` | done | | `integrations/harness2/` | Harness2 research harness on MockMed | Moved to `packages/flow/benchmark/harness/harness2/`; the `harness2` extra is gone (install `[browser]`) | done | | `mockloan/`, `demo_driver.record_disbursement_demo` | Lending demo app no product command uses | Moved to `packages/flow/benchmark/harness/mockloan/` (recorder in `mockloan/demo.py`) | done | @@ -175,7 +176,7 @@ Paths are relative to `packages/flow/openadapt_flow/` as of the audited commit. | `fleet/simulator.py:1-` | (c) | synthetic payer portal | in-package simulator fixture; move to benchmark/examples | | `guard_demo.py:1-296` | (c) | MockMed | `python -m openadapt_flow.guard_demo`: fake agent posts referral to MockMed with chart_pages READER creds; not a CLI subcommand. Move to packages/flow/examples or benchmark; importers: packages/agent/tests (1), packages/flow/tests (6). | | `instrument/ (__init__.py, competitor_drift.py)` | (c) | MockMed | MockMed competitor-drift harness. Moved to packages/flow/benchmark/harness/instrument. | -| `integrations/harness2/ (all: tasks.py, apps.py, agent.py, adapter.py, traces.py, proposers.py, scorer.py, llm.py, pixels.py, rollout.py, selection.py, __init__.py)` | (c) | MockMed | Whole subpackage is a research harness for Harness2 on MockMed: MOCKMED_PATIENT/PATIENTS (tasks.py:71-148), MockMedTarget (apps.py:50-105, /api/db at :70), regex for MockMed task text and 'triage'/'New Encounter'/'encountersaved' (agent.py:221-324), MockMed banner regex Name — MRN — DOB (traces.py:55-57,128-138), clinical prompts (adapter.py:93-120,429-445; proposers.py:50-63). Move to packages/flow/benchmark/harness2; importers: benchmark/harness2_scorer/run.py, tests/test_harness2_{adapter,pixels,traces}.py, pyproject extra 'harness2', .github/workflows/packages.yml:451. | +| `integrations/harness2/ (all: tasks.py, apps.py, agent.py, adapter.py, traces.py, proposers.py, scorer.py, llm.py, pixels.py, rollout.py, selection.py, __init__.py)` | (c) | MockMed | Whole subpackage is a research harness for Harness2 on MockMed: MOCKMED_PATIENT/PATIENTS (tasks.py:71-148), MockMedTarget (apps.py:50-105, /api/db at :70), regex for MockMed task text and 'triage'/'New Encounter'/'encountersaved' (agent.py:221-324), MockMed banner regex Name — MRN — DOB (traces.py:55-57,128-138), clinical prompts (adapter.py:93-120,429-445; proposers.py:50-63). Move to packages/flow/benchmark/harness/harness2; importers: benchmark/harness2_scorer/run.py, tests/test_harness2_{adapter,pixels,traces}.py, pyproject extra 'harness2', .github/workflows/packages.yml:451. | | `learning/synth_stream.py:1-245` | (c) | MockMed | synthetic MockMed add-patient-note stream (intents, 'Consent Required', 'Belford, Phil DOB'). Move to benchmark/ or tests/fixtures. | | `learning/synth_stream.py:246-525` | (c) | (generic) | StructuralDiffInducer is generic but lives in the MockMed harness module; imported by learning/teach.py:96, learning/branch_teach.py:76, benchmark/openemr_e2e/harness.py, 4 tests; pyproject mypy list (line 611). Move to learning/induce.py first, then move the stream out. | | `mockloan/ (server.py, fault_server.py, static/)` | (c) | MockLoan | Own synthetic lending app shipped in wheel but used by no product command; only demo_driver.record_disbursement_demo, benchmark/lending_fault_model, benchmark/undo_proposal, tests (test_mockloan, test_lending_swer, e2e/test_lending_fault_model), scripts (1). Move to packages/flow/benchmark/apps/mockloan. | @@ -304,8 +305,8 @@ Paths are relative to `packages/flow/openadapt_flow/` as of the audited commit. | `validation/dense_surface.py:1271` | (a) | OpenEMR | comment/docstring example | | `validation/dense_surface.py:5` | (a) | OpenEMR | comment/docstring example | | `verifier.py:2467` | (a) | OpenEMR | comment/docstring example | -| `vision/hashing.py:10` | (a) | MockMed | docstring for the MockMed-calibrated phash tolerance (see compile.py:151) | +| `vision/hashing.py:10` | (a) | MockMed | docstring for the MockMed-calibrated phash tolerance (see compile.py:148) | | `volatility.py:100` | (a) | OpenEMR | comment/docstring example | | `volatility.py:31` | (a) | OpenEMR | comment/docstring example | | `volatility.py:6` | (a) | OpenEMR | comment/docstring example | -| `benchmark/` (27 files) | (c) | OpenEMR, Frappe, MockMed, Open Dental | Moved to `packages/flow/benchmark/harness/` | +| `benchmark/` (25 files) | (c) | OpenEMR, Frappe, MockMed, Open Dental | Moved to `packages/flow/benchmark/harness/` | diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index 0a15c63874..2abfb44aa3 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -5458,7 +5458,8 @@ def _checkout_harness_on_path(command: str) -> bool: ) return False if str(checkout) not in sys.path: - sys.path.insert(0, str(checkout)) + # Appended, so nothing in the checkout shadows an installed module. + sys.path.append(str(checkout)) return True diff --git a/packages/flow/pyproject.toml b/packages/flow/pyproject.toml index a45b45596e..8787fd5bae 100644 --- a/packages/flow/pyproject.toml +++ b/packages/flow/pyproject.toml @@ -347,6 +347,7 @@ exclude = [ "/docs/validation/identity_roc.json", "/docs/validation/identity_roc.png", "/openadapt_flow/benchmark/reliability_corpus.py", + "/benchmark/harness/reliability_corpus.py", "/openadapt_flow/validation/adversary_corpus.py", "/openadapt_flow/validation/adversary_corpus_v2.py", "/openadapt_flow/validation/adversary_corpus_v3.py", @@ -410,6 +411,7 @@ exclude = [ "/docs/validation/identity_roc.json", "/docs/validation/identity_roc.png", "/openadapt_flow/benchmark/reliability_corpus.py", + "/benchmark/harness/reliability_corpus.py", "/openadapt_flow/validation/adversary_corpus.py", "/openadapt_flow/validation/adversary_corpus_v2.py", "/openadapt_flow/validation/adversary_corpus_v3.py", @@ -495,8 +497,8 @@ required-environments = [ target-version = "py310" line-length = 88 # Generated / non-package trees at the repo root are not part of the lint -# surface. Anchored with a leading slash so the package's own -# openadapt_flow/benchmark/ IS still linted (a bare "benchmark" would match it). +# surface. The benchmark harness (benchmark/harness/) is linted by naming it +# explicitly in CI (`ruff check openadapt_flow benchmark/harness`). extend-exclude = ["/benchmark", "/docs", "/runs", "/dist"] [tool.ruff.lint] @@ -599,7 +601,6 @@ module = [ "openadapt_flow.__main__", "openadapt_flow.backends.playwright_backend", "openadapt_flow.backends.rdp_backend", - "benchmark.harness.desktop_benchmark", "openadapt_flow.compiler.disambiguation", "openadapt_flow.interactive_recorder", "openadapt_flow.learning.induce", diff --git a/packages/flow/tests/test_runtime_never_imports_benchmark.py b/packages/flow/tests/test_runtime_never_imports_benchmark.py index d59f3083d7..7d052ea956 100644 --- a/packages/flow/tests/test_runtime_never_imports_benchmark.py +++ b/packages/flow/tests/test_runtime_never_imports_benchmark.py @@ -32,9 +32,20 @@ def _is_benchmark(module: str) -> bool: ) -def _imports_benchmark(tree: ast.AST) -> list[int]: +def _imports_benchmark(tree: ast.AST, *, lazy_ok: bool = False) -> list[int]: lines: list[int] = [] - for node in ast.walk(tree): + # An allowed module may import the harness only inside a function, where + # it runs on demand from a checkout. + nodes = ( + [ + n + for n in ast.iter_child_nodes(tree) + if isinstance(n, (ast.Import, ast.ImportFrom)) + ] + if lazy_ok + else ast.walk(tree) + ) + for node in nodes: if isinstance(node, ast.ImportFrom): module = node.module or "" if node.level == 0 and _is_benchmark(module): @@ -61,10 +72,10 @@ def test_product_modules_never_import_the_benchmark_package() -> None: offenders: list[str] = [] for path in sorted(ROOT.rglob("*.py")): rel = path.relative_to(ROOT).as_posix() - if rel in ALLOWED: - continue tree = ast.parse(path.read_text(encoding="utf-8")) - offenders += [f"{rel}:{n}" for n in _imports_benchmark(tree)] + offenders += [ + f"{rel}:{n}" for n in _imports_benchmark(tree, lazy_ok=rel in ALLOWED) + ] assert not offenders, ( "product code imports the benchmark harness (the wheel and a frozen " "build leave it out); move the shared name to " From 88eaf68cbf644dda6ceea6ebcad075d98fd9ece9 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 20:40:33 -0500 Subject: [PATCH 15/16] Guard: ordinary words like text_edit and mock_media pass; baseline 425 --- packages/flow/docs/GENERALITY_AUDIT.md | 2 +- packages/flow/scripts/check_engine_generality.py | 2 +- packages/flow/scripts/generality.toml | 6 +++--- packages/flow/scripts/generality_baseline.json | 3 +-- packages/flow/tests/test_engine_generality_check.py | 4 ++++ 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md index 24507e5974..deecfc90d6 100644 --- a/packages/flow/docs/GENERALITY_AUDIT.md +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -9,7 +9,7 @@ deployment configs) or in test fixtures and benchmark harnesses outside the shipped package. A CI check (`scripts/check_engine_generality.py`) keeps it that way; [CONTRIBUTING.md](../CONTRIBUTING.md) states the rule. It counts an app name inside an identifier too (`_openemr_base`, `isOpenEMR`); its baseline -started at 426 hits and only shrinks. +started at 425 hits and only shrinks. ## Method diff --git a/packages/flow/scripts/check_engine_generality.py b/packages/flow/scripts/check_engine_generality.py index d5b3976f2d..a37fc9c95f 100644 --- a/packages/flow/scripts/check_engine_generality.py +++ b/packages/flow/scripts/check_engine_generality.py @@ -99,7 +99,7 @@ def scan( continue split = _CAMEL_BOUNDARY.sub(" ", line) for term, (pattern, is_name) in terms.items(): - found = pattern.search(split if is_name else line) + found = pattern.search(line) or (is_name and pattern.search(split)) if found and not _allowed(rel, term, config): hits.setdefault(rel, {}).setdefault(term, []).append(number) return hits diff --git a/packages/flow/scripts/generality.toml b/packages/flow/scripts/generality.toml index 8c382c226d..edbc1e34ab 100644 --- a/packages/flow/scripts/generality.toml +++ b/packages/flow/scripts/generality.toml @@ -15,7 +15,7 @@ dolibarr = '(? None: "f.py": "row = 'tabLoanApplication'\n", # A name only stops at a letter or digit beside it. "g.py": "reopenemrs = 1\n", + "h.py": "TextEdit = 1\nMockMedServer = 1\n", + # Ordinary words that only look like a name. + "i.py": "text_edit = 1\nmock_media = 1\n", }, ) assert check.scan(engine) == { @@ -74,6 +77,7 @@ def test_a_name_inside_an_identifier_is_a_hit(check, tmp_path: Path) -> None: "d.py": {"stedi": [1]}, "e.py": {"epic": [1]}, "f.py": {"frappe-table": [1]}, + "h.py": {"mockmed": [2], "textedit": [1]}, } From cdf9c99c219a1044eb6e62a2850fa7258a080753 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 02:46:26 -0500 Subject: [PATCH 16/16] fix(flow): lower generality baseline after harness relocation Signed-off-by: abrichr --- .../flow/openadapt_flow/rl_env/environment.py | 2 +- packages/flow/openadapt_flow/rl_env/tasks.py | 2 +- .../flow/scripts/generality_baseline.json | 34 +++---------------- 3 files changed, 7 insertions(+), 31 deletions(-) diff --git a/packages/flow/openadapt_flow/rl_env/environment.py b/packages/flow/openadapt_flow/rl_env/environment.py index 0490ed6e9f..0e07633d56 100644 --- a/packages/flow/openadapt_flow/rl_env/environment.py +++ b/packages/flow/openadapt_flow/rl_env/environment.py @@ -206,7 +206,7 @@ def _norm(value: str) -> str: def _scalars(body: str) -> set[str]: """Every scalar value in a JSON or form body, as text. - A form field holding JSON (as Frappe sends ``doc=...``) is read through. + A form field holding JSON (such as ``doc=...``) is read through. """ from urllib.parse import parse_qsl diff --git a/packages/flow/openadapt_flow/rl_env/tasks.py b/packages/flow/openadapt_flow/rl_env/tasks.py index 2f557fbe2c..3bdd6a8358 100644 --- a/packages/flow/openadapt_flow/rl_env/tasks.py +++ b/packages/flow/openadapt_flow/rl_env/tasks.py @@ -136,7 +136,7 @@ class AgentSpec(_Strict): #: the oracle uses, reset routes, and anything else that would let the #: agent read or rewrite the answer. blocked_paths: list[str] = Field(default_factory=list) - #: The requests that carry the task's write (``"POST /api/encounter"``). + #: The requests that carry the task's write (``"POST /api/items"``). #: An infeasible task pays a failure report only if the agent's browser #: sent one whose body holds every value in the task's ``params``. write_requests: list[str] = Field(default_factory=list) diff --git a/packages/flow/scripts/generality_baseline.json b/packages/flow/scripts/generality_baseline.json index 46c2cab579..3fac4b495a 100644 --- a/packages/flow/scripts/generality_baseline.json +++ b/packages/flow/scripts/generality_baseline.json @@ -3,9 +3,6 @@ "mockmed": 43, "textedit": 1 }, - "agent/ledger_demo.py": { - "mockmed": 4 - }, "api.py": { "mockmed": 1 }, @@ -55,7 +52,7 @@ "mockmed": 1 }, "demo_driver.py": { - "mockloan": 5 + "mockloan": 2 }, "deployment.py": { "mockmed": 2, @@ -108,27 +105,6 @@ "espocrm": 2, "redmine": 2 }, - "guard_demo.py": { - "mockmed": 5, - "mockmed-records": 2, - "mockmed-write": 1 - }, - "integrations/harness2/__init__.py": { - "mockmed": 2 - }, - "integrations/harness2/adapter.py": { - "mockmed": 2 - }, - "integrations/harness2/agent.py": { - "mockmed": 1 - }, - "integrations/harness2/apps.py": { - "mockmed": 13, - "mockmed-records": 1 - }, - "integrations/harness2/tasks.py": { - "mockmed": 26 - }, "ir.py": { "token_main": 1 }, @@ -144,9 +120,6 @@ "jit/signature.py": { "mockmed": 1 }, - "learning/synth_stream.py": { - "mockmed": 5 - }, "mockloan/__init__.py": { "mockloan": 2, "mockmed": 2 @@ -214,11 +187,14 @@ "csrf_token_form": 2, "token_main": 1 }, + "reward/bundle.py": { + "mockmed": 3 + }, "reward/models.py": { "mockmed-records": 1 }, "reward/seed.py": { - "mockmed": 39 + "mockmed": 34 }, "risk.py": { "openemr": 1,