diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index bba496d4fc..59425e18d1 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -347,7 +347,7 @@ jobs: # every public data/config asset listed in public-artifacts.json. run: python scripts/check_release_consistency.py - name: Ruff lint - run: ruff check openadapt_flow + run: ruff check openadapt_flow benchmark/harness - name: Install uv (for the oldest-Python syntax check) uses: astral-sh/setup-uv@18d369b23da25a81e9458561248e1aafff6a70fa # v10.3.1 - name: Syntax at the oldest supported Python (3.10) @@ -362,7 +362,7 @@ jobs: ruff check --isolated --select E9 --target-version py310 . uv run --no-project --python 3.10 python -m compileall -q -x '(^|/)(\.venv|node_modules|build|dist)/' . - name: Ruff format check - run: ruff format --check openadapt_flow tests + run: ruff format --check openadapt_flow tests benchmark/harness - name: Mypy run: mypy - name: Mypy (strict, safety path only) diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md index dd3ae68cf5..3dadc9c2d6 100644 --- a/packages/flow/CHANGELOG.md +++ b/packages/flow/CHANGELOG.md @@ -17,6 +17,10 @@ ## Unreleased +### Changed + +- The wheel no longer ships the benchmark harnesses. `openadapt_flow.benchmark` and `openadapt_flow.instrument` moved to `packages/flow/benchmark/harness/` in the source tree (import `benchmark.harness`); `openadapt-flow benchmark` runs from a source checkout and says so from a wheel. The engine audit for application-specific code is in `docs/GENERALITY_AUDIT.md`. Code that imported `openadapt_flow.benchmark` or `openadapt_flow.instrument` from an installed wheel imports `benchmark.harness` from a source checkout instead. + ### Features - `run --self-repair` (and repair by default) now works outside a browser. `repair.session` in `deploy.yaml` says how to start the app for each probe, fault probe and resumed run (an argv, no shell, kept running while the app is up; `fault_commands`, `close_command`, `ready_s`). The run's configured backend must bind to a window owned by that command's process group, so a session never binds to the halted session or a copy already open; Flow ends the group afterwards, and closes the halted session before each probe. This works where the bound window is on Flow's machine (macOS, and the local RDP or Citrix client window); Windows over its agent and FreeRDP are refused for now. Without `repair.session`, a desktop run refuses repair with that reason, as before. diff --git a/packages/flow/benchmark/agent_arm_verticals/README.md b/packages/flow/benchmark/agent_arm_verticals/README.md index fbfe9544dc..54526bd8fa 100644 --- a/packages/flow/benchmark/agent_arm_verticals/README.md +++ b/packages/flow/benchmark/agent_arm_verticals/README.md @@ -26,7 +26,7 @@ budget without completing the write in every trial. ## Method and public/private boundary The public, generic agent-baseline mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. It provides explicit paid-run +`benchmark/harness/agent_baseline.py`. It provides explicit paid-run opt-in, action and cost caps, usage accounting, and an agent self-report that is never trusted for scoring. The public benchmark schemas and aggregate failure taxonomy also remain available for independently authored tasks. diff --git a/packages/flow/benchmark/code_as_action/measure_openemr.py b/packages/flow/benchmark/code_as_action/measure_openemr.py index 74a34443a6..9e1b5e70f0 100644 --- a/packages/flow/benchmark/code_as_action/measure_openemr.py +++ b/packages/flow/benchmark/code_as_action/measure_openemr.py @@ -184,7 +184,7 @@ def spec_for(key: str) -> Any: - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec return SyntheticPatientSpec(**REFERRALS[key], **COMMON_FIELDS) @@ -390,7 +390,7 @@ def main() -> int: keys = [k for k in args.referrals.split(",") if k] from benchmark.openemr_local import qualify_hooks - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec fixture = qualify_hooks.fixture() demo = _demo() diff --git a/packages/flow/benchmark/dom/BENCHMARK.md b/packages/flow/benchmark/dom/BENCHMARK.md index 7111596c12..2b40374899 100644 --- a/packages/flow/benchmark/dom/BENCHMARK.md +++ b/packages/flow/benchmark/dom/BENCHMARK.md @@ -23,7 +23,7 @@ row's Open button — the literal reading — and **DOM (name-filtered)** clicks the Open button in the row named `Jane Sample` — the identity reading, hardcoding the demonstrated patient exactly as the compiled arm's recorded identity band does. Every selector choice is -documented in `openadapt_flow/benchmark/dom_arm.py`. +documented in `benchmark/harness/dom_arm.py`. ## Verdict @@ -208,7 +208,7 @@ Unmeasured variants, for completeness: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.dom_arm --out benchmark/dom --n-per-perturbation 2 +.venv/bin/python -m benchmark.harness.dom_arm --out benchmark/dom --n-per-perturbation 2 ``` (`--n-per-perturbation 2` matches the committed results.) No API key diff --git a/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md b/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md index dfa26c1e61..58868ee444 100644 --- a/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md +++ b/packages/flow/benchmark/effect_e2e/EFFECT_E2E.md @@ -2,7 +2,7 @@ Date: 2026-07-21. This is the genuinely independent, end-to-end measurement of the silent-wrong-effect (SWER) result. Unlike the definitional -`openadapt_flow.benchmark.silent_wrong_action` (raw `requests.post`; effect +`benchmark.harness.silent_wrong_action` (raw `requests.post`; effect verifier and ground truth read the SAME in-process object; the effect contract restates the ground-truth definition -- so its `0/90` is circular by construction), every write below goes through the ACTUAL governed replay path diff --git a/packages/flow/benchmark/effect_e2e/record_service.py b/packages/flow/benchmark/effect_e2e/record_service.py index 2bb71f2e90..0f8f6e0fa3 100644 --- a/packages/flow/benchmark/effect_e2e/record_service.py +++ b/packages/flow/benchmark/effect_e2e/record_service.py @@ -3,7 +3,7 @@ This is the system of record the end-to-end silent-wrong-effect harness (:mod:`benchmark.effect_e2e.run`) writes to THROUGH the governed replay path. It exists to break the circularity of the older definitional benchmark -(``openadapt_flow.benchmark.silent_wrong_action``), whose effect verifier and +(``benchmark.harness.silent_wrong_action``), whose effect verifier and ground-truth judge read the SAME in-process Python object and whose write bypassed the :class:`~openadapt_flow.runtime.replayer.Replayer`. diff --git a/packages/flow/benchmark/effect_e2e/run.py b/packages/flow/benchmark/effect_e2e/run.py index d08601d691..7860d35b89 100644 --- a/packages/flow/benchmark/effect_e2e/run.py +++ b/packages/flow/benchmark/effect_e2e/run.py @@ -1,7 +1,7 @@ """End-to-end silent-wrong-effect (SWER) harness through the REAL replayer. This is the genuinely independent, end-to-end version of the silent-wrong-action -result. The older ``openadapt_flow.benchmark.silent_wrong_action`` is +result. The older ``benchmark.harness.silent_wrong_action`` is *definitional*: it issues raw ``requests.post`` (bypassing the :class:`~openadapt_flow.runtime.replayer.Replayer`), and its effect verifier and ground-truth judge both read the SAME in-process ``FaultDB`` object while the @@ -629,7 +629,7 @@ def render_markdown(results: dict[str, Any]) -> str: Date: {date}. This is the genuinely independent, end-to-end measurement of the silent-wrong-effect (SWER) result. Unlike the definitional -`openadapt_flow.benchmark.silent_wrong_action` (raw `requests.post`; effect +`benchmark.harness.silent_wrong_action` (raw `requests.post`; effect verifier and ground truth read the SAME in-process object; the effect contract restates the ground-truth definition -- so its `0/90` is circular by construction), every write below goes through the ACTUAL governed replay path @@ -774,7 +774,7 @@ def render_markdown(results: dict[str, Any]) -> str: def render_chart(results: dict[str, Any], out_png: Path) -> Path: """Bar chart: silent-wrong-effect rate per arm (large -> small -> zero).""" - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() m = results["metrics"]["per_arm"] @@ -827,7 +827,7 @@ def write_outputs(results: dict[str, Any], out_dir: Path) -> None: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") (out_dir / "EFFECT_E2E.md").write_text(render_markdown(results)) - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "effect_e2e.png") diff --git a/packages/flow/benchmark/effectbench/reference_fault_model.py b/packages/flow/benchmark/effectbench/reference_fault_model.py index 5ba4904208..0488c51dff 100644 --- a/packages/flow/benchmark/effectbench/reference_fault_model.py +++ b/packages/flow/benchmark/effectbench/reference_fault_model.py @@ -31,16 +31,16 @@ How it re-expresses the study, faithfully and without hardcoding verdicts: - Each of the study's nine scenarios (``benchmark.fault_model.faults.FAULTS``) - becomes a :class:`~openadapt_flow.benchmark.effectbench.schema.TaskSpec` plus + becomes a :class:`~benchmark.harness.effectbench.schema.TaskSpec` plus the KNOWN post-action system-of-record state that fault produces (the same DB states the study's own taxonomy tests assert). The states live in :data:`SCENARIOS` — the record shapes are MockMed ``FaultDB`` rows. - The TRUE effect is read by the substrate-agnostic - :class:`~openadapt_flow.benchmark.effectbench.oracle.RecordSnapshotOracle` + :class:`~benchmark.harness.effectbench.oracle.RecordSnapshotOracle` running the SAME shared ``judge_records`` decision logic the runtime uses — once for the at-most-once/collateral ``record_written`` contract and once for the ``field_equals`` note read-back — reduced by - :func:`~openadapt_flow.benchmark.effectbench.oracle.combine_true_states`. + :func:`~benchmark.harness.effectbench.oracle.combine_true_states`. - Two arms are scored against the identical independent oracle: ``screen_only`` (the weak banner oracle — reports success whenever the screen looks saved) and ``effect_verify`` (the agent halts unless the oracle CONFIRMS). Only the @@ -56,7 +56,7 @@ from typing import Optional from benchmark.fault_model import faults as F -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, DivergenceCategory, Effect, @@ -72,7 +72,7 @@ oracle_verdict_of, summarize, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel # The consequential-save contract the study verifies: exactly one Triage # encounter for the target patient (at-most-once + no collateral loss) whose diff --git a/packages/flow/benchmark/effectbench/task_pack/README.md b/packages/flow/benchmark/effectbench/task_pack/README.md index 88fa4d6c25..8846166f4c 100644 --- a/packages/flow/benchmark/effectbench/task_pack/README.md +++ b/packages/flow/benchmark/effectbench/task_pack/README.md @@ -68,7 +68,7 @@ python -m benchmark.effectbench.task_pack.pack # Live-run the MockMed anchor through the oracle (prints per-task classification) python -c "from benchmark.effectbench.task_pack.driver import run_mockmed_pack; \ -from openadapt_flow.benchmark.effectbench import summarize; \ +from benchmark.harness.effectbench import summarize; \ eps=run_mockmed_pack(trials=3); \ print('screen SWER', summarize(eps, arm='screen_only').swer.rate); \ print('effect SWER', summarize(eps, arm='effect_verify').swer.rate)" diff --git a/packages/flow/benchmark/effectbench/task_pack/__init__.py b/packages/flow/benchmark/effectbench/task_pack/__init__.py index e3f3537b7b..c823a82252 100644 --- a/packages/flow/benchmark/effectbench/task_pack/__init__.py +++ b/packages/flow/benchmark/effectbench/task_pack/__init__.py @@ -6,9 +6,9 @@ ``benchmark.environments`` registry (PR #173) indexes, authored against the frozen EffectBench schema/oracle/metrics contract (PR #178). -Each task is a :class:`~openadapt_flow.benchmark.effectbench.TaskSpec` (goal = +Each task is a :class:`~benchmark.harness.effectbench.TaskSpec` (goal = INTENT only, never a step list) plus an -:class:`~openadapt_flow.benchmark.effectbench.OracleSpec` bound to the +:class:`~benchmark.harness.effectbench.OracleSpec` bound to the environment's system-of-record READ recipe, DESIGNED so a green screen (task-success) diverges from the correct business effect. The oracle reads pre/post SoR state independently of the screen; the non-gameability attestations diff --git a/packages/flow/benchmark/effectbench/task_pack/_authoring.py b/packages/flow/benchmark/effectbench/task_pack/_authoring.py index 00e02bc078..aa0ba26354 100644 --- a/packages/flow/benchmark/effectbench/task_pack/_authoring.py +++ b/packages/flow/benchmark/effectbench/task_pack/_authoring.py @@ -3,10 +3,10 @@ These build the two things every consequential-write task needs: - the DECLARED expected effect (a typed - :class:`~openadapt_flow.benchmark.effectbench.Effect`), parameterized by run - params via :class:`~openadapt_flow.benchmark.effectbench.ValueExpr` so the + :class:`~benchmark.harness.effectbench.Effect`), parameterized by run + params via :class:`~benchmark.harness.effectbench.ValueExpr` so the record CHECKED is the record this trial WROTE (trial-unique payload); and -- the :class:`~openadapt_flow.benchmark.effectbench.OracleSpec` that declares +- the :class:`~benchmark.harness.effectbench.OracleSpec` that declares the independent read channel and the non-gameability attestations the adversarial audit signs off on. @@ -21,13 +21,13 @@ from typing import Any, Optional -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, EffectKind, OracleSpec, ValueExpr, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel # Canonical run-param names a trial binds a trial-unique value to. The driver # derives each from the trial seed so the oracle checks THIS run's exact write. diff --git a/packages/flow/benchmark/effectbench/task_pack/audit.py b/packages/flow/benchmark/effectbench/task_pack/audit.py index c44706010e..fcba2b8c15 100644 --- a/packages/flow/benchmark/effectbench/task_pack/audit.py +++ b/packages/flow/benchmark/effectbench/task_pack/audit.py @@ -41,7 +41,7 @@ _note_effect, _record_effect, ) -from openadapt_flow.benchmark.effectbench import RestRecordVerifier, Verdict +from benchmark.harness.effectbench import RestRecordVerifier, Verdict _HTTP_TIMEOUT_S = 5.0 TRIAL_NOTE = "triage note trial-unique-AUDIT" diff --git a/packages/flow/benchmark/effectbench/task_pack/driver.py b/packages/flow/benchmark/effectbench/task_pack/driver.py index 4145e46ae7..035a025097 100644 --- a/packages/flow/benchmark/effectbench/task_pack/driver.py +++ b/packages/flow/benchmark/effectbench/task_pack/driver.py @@ -8,9 +8,9 @@ mocked verdicts. It depends ONLY on the public EffectBench surface -(:func:`~openadapt_flow.benchmark.effectbench.score_episode`, -:class:`~openadapt_flow.benchmark.effectbench.RestRecordVerifier`, -:class:`~openadapt_flow.benchmark.effectbench.AgentReport`) and the MockMed +(:func:`~benchmark.harness.effectbench.score_episode`, +:class:`~benchmark.harness.effectbench.RestRecordVerifier`, +:class:`~benchmark.harness.effectbench.AgentReport`) and the MockMed fault server. It does not reimplement any real baseline "arm"; the two arms here are the two SCORING conditions the benchmark contrasts: @@ -46,7 +46,7 @@ MockMedDrive, MockMedTask, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, EpisodeRecord, RestRecordVerifier, diff --git a/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py b/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py index e2c22a2ef5..9219f750ea 100644 --- a/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py +++ b/packages/flow/benchmark/effectbench/task_pack/frappe_tasks.py @@ -17,7 +17,7 @@ ``oracle.config["query"]`` + ``["query_params"]`` is what the multi-baseline runner materializes into a -:class:`~openadapt_flow.benchmark.effectbench.SqlRecordVerifier` (read-only role, +:class:`~benchmark.harness.effectbench.SqlRecordVerifier` (read-only role, DB-API-bound params). The loan-application docname has a space (`` `tabLoan Application` ``), so the table is back-tick quoted in the SELECT. """ @@ -33,15 +33,15 @@ record_written_effect, ) from benchmark.effectbench.task_pack.openemr_tasks import ContainerTask -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( DivergenceCategory as DC, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, Substrate, TaskSpec, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel ENVIRONMENT = "frappe_lending" diff --git a/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py b/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py index 898f90db42..9f9b679bc3 100644 --- a/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py +++ b/packages/flow/benchmark/effectbench/task_pack/mockmed_tasks.py @@ -8,7 +8,7 @@ path is exercised in CI (no mocked verdicts). Each entry is a :class:`MockMedTask`: an agent-agnostic -:class:`~openadapt_flow.benchmark.effectbench.TaskSpec` (goal = intent only) +:class:`~benchmark.harness.effectbench.TaskSpec` (goal = intent only) plus a :class:`MockMedDrive` — the environment recipe the live driver replays to MANIFEST the divergence (which ``?fault=`` mode to inject, the write the agent performs, decoy/concurrent rows to seed, and the documented banner @@ -34,9 +34,9 @@ param, record_written_effect, ) -from openadapt_flow.benchmark.effectbench import DivergenceCategory as DC -from openadapt_flow.benchmark.effectbench import Effect, Substrate, TaskSpec -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench import DivergenceCategory as DC +from benchmark.harness.effectbench import Effect, Substrate, TaskSpec +from benchmark.harness.effectbench.schema import OracleChannel TARGET_PATIENT = "p1" TARGET_TYPE = "Triage" diff --git a/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py b/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py index 2f6d61974d..7581aa2520 100644 --- a/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py +++ b/packages/flow/benchmark/effectbench/task_pack/openemr_tasks.py @@ -18,7 +18,7 @@ The SQL each oracle runs (``oracle.config["query"]`` + ``["query_params"]``) is what the multi-baseline runner materializes into a -:class:`~openadapt_flow.benchmark.effectbench.SqlRecordVerifier` (read-only role; +:class:`~benchmark.harness.effectbench.SqlRecordVerifier` (read-only role; values bound through DB-API params, never interpolated). Column names become record keys, so the same typed ``Effect`` the MockMed anchor uses judges these rows unchanged. @@ -36,15 +36,15 @@ param, record_written_effect, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( DivergenceCategory as DC, ) -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, Substrate, TaskSpec, ) -from openadapt_flow.benchmark.effectbench.schema import OracleChannel +from benchmark.harness.effectbench.schema import OracleChannel ENVIRONMENT = "openemr_local" diff --git a/packages/flow/benchmark/effectbench/task_pack/pack.py b/packages/flow/benchmark/effectbench/task_pack/pack.py index 2be0450827..fa15417f72 100644 --- a/packages/flow/benchmark/effectbench/task_pack/pack.py +++ b/packages/flow/benchmark/effectbench/task_pack/pack.py @@ -4,7 +4,7 @@ the container-gated OpenEMR and Frappe families). It: - assembles the unified list of :class:`PackEntry` (one per authored task), - each carrying its :class:`~openadapt_flow.benchmark.effectbench.TaskSpec`, the + each carrying its :class:`~benchmark.harness.effectbench.TaskSpec`, the pinned environment it targets, and whether it runs LIVE here or needs a container; - validates every task against the ``benchmark.environments`` registry (the @@ -29,8 +29,8 @@ from benchmark.effectbench.task_pack.frappe_tasks import FRAPPE_TASKS from benchmark.effectbench.task_pack.mockmed_tasks import MOCKMED_TASKS from benchmark.effectbench.task_pack.openemr_tasks import OPENEMR_TASKS -from openadapt_flow.benchmark.effectbench import TaskSpec -from openadapt_flow.benchmark.effectbench.schema import DivergenceCategory, Substrate +from benchmark.harness.effectbench import TaskSpec +from benchmark.harness.effectbench.schema import DivergenceCategory, Substrate MANIFEST_SCHEMA_VERSION = 1 diff --git a/packages/flow/benchmark/frappe_lending/README.md b/packages/flow/benchmark/frappe_lending/README.md index 1504ddf329..89ae27fa5a 100644 --- a/packages/flow/benchmark/frappe_lending/README.md +++ b/packages/flow/benchmark/frappe_lending/README.md @@ -30,7 +30,7 @@ trials per cell remains the publication bar. Raw per-run rows, environment fingerprints, application-specific adapter/oracle wiring, and the detailed cost ledger are retained in the private `OpenAdaptAI/openadapt-corpus` repository. The public generic cost-capped agent mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. See the +`benchmark/harness/agent_baseline.py`. See the [aggregate report](../agent_arm_verticals/README.md). The accepted run used baseline SHA-256 diff --git a/packages/flow/openadapt_flow/benchmark/__init__.py b/packages/flow/benchmark/harness/__init__.py similarity index 63% rename from packages/flow/openadapt_flow/benchmark/__init__.py rename to packages/flow/benchmark/harness/__init__.py index b157c4fda1..2eba51f30b 100644 --- a/packages/flow/openadapt_flow/benchmark/__init__.py +++ b/packages/flow/benchmark/harness/__init__.py @@ -1,4 +1,8 @@ -"""Benchmark: compiled replay vs. a frontier computer-use agent. +"""Benchmark harnesses: compiled replay vs. a frontier computer-use agent. + +This package lives in the source checkout, outside ``openadapt_flow``, so the +wheel does not ship it. Import it as ``benchmark.harness`` from +``packages/flow``. Submodules (import them directly; nothing heavy is imported here): diff --git a/packages/flow/openadapt_flow/benchmark/agent_baseline.py b/packages/flow/benchmark/harness/agent_baseline.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/agent_baseline.py rename to packages/flow/benchmark/harness/agent_baseline.py diff --git a/packages/flow/openadapt_flow/benchmark/chart_fonts.py b/packages/flow/benchmark/harness/chart_fonts.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/chart_fonts.py rename to packages/flow/benchmark/harness/chart_fonts.py diff --git a/packages/flow/openadapt_flow/benchmark/desktop_benchmark.py b/packages/flow/benchmark/harness/desktop_benchmark.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/desktop_benchmark.py rename to packages/flow/benchmark/harness/desktop_benchmark.py index 270605e54b..c581ca306d 100644 --- a/packages/flow/openadapt_flow/benchmark/desktop_benchmark.py +++ b/packages/flow/benchmark/harness/desktop_benchmark.py @@ -745,7 +745,7 @@ def _identity_transfer_verdict(results: dict) -> str: def render_chart(results: dict, path: Path) -> None: """Stacked outcome bars per arm (success / wrong-action / halt).""" - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() arms = list(results["arms"]) @@ -776,7 +776,7 @@ def write_outputs(results: dict, out_dir: str | Path) -> None: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") (out_dir / "BENCHMARK.md").write_text(render_markdown(results)) - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "outcomes.png") diff --git a/packages/flow/openadapt_flow/benchmark/dom_arm.py b/packages/flow/benchmark/harness/dom_arm.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/dom_arm.py rename to packages/flow/benchmark/harness/dom_arm.py index a6c0e598dd..27a1e277ef 100644 --- a/packages/flow/openadapt_flow/benchmark/dom_arm.py +++ b/packages/flow/benchmark/harness/dom_arm.py @@ -10,7 +10,7 @@ Three arms, one frozen schedule, one arm-independent success check: - **compiled**: the existing vision-anchored compiled replay - (:func:`openadapt_flow.benchmark.run_benchmark._compiled_run`), healing + (:func:`benchmark.harness.run_benchmark._compiled_run`), healing enabled as always. - **dom** (positional) and **dom_named** (name-filtered): :func:`dom_script` — a Playwright script written the way a competent @@ -29,7 +29,7 @@ drift modes (:data:`PERTURBATIONS`: lookalike, missing, grow, sort, theme, rename, move, typelabel). Success is judged for BOTH arms by the hybrid benchmark's own arm-independent check -(:func:`~openadapt_flow.benchmark.hybrid_benchmark.verify_hybrid_final`): +(:func:`~benchmark.harness.hybrid_benchmark.verify_hybrid_final`): OCR of the final screenshot must show the saved-encounter evidence AND the right patient, with wrong-type writes flagged — so this head-to-head is judged by the identical bar. Wrong-action events are recorded per arm: @@ -51,7 +51,7 @@ from pathlib import Path from typing import Any, Callable, Optional -from openadapt_flow.benchmark.hybrid_benchmark import ( +from benchmark.harness.hybrid_benchmark import ( _NOTE_POOL, DRIFT_TYPES, SCHEDULE, @@ -465,7 +465,7 @@ def dom_arm_aggregate(rows: list[dict[str, Any]]) -> dict[str, Any]: Aggregate dict: base stats plus clean/drift splits, outcome counts, wrong-action count, and (DOM arm) maintenance events. """ - from openadapt_flow.benchmark.run_benchmark import _arm_aggregate + from benchmark.harness.run_benchmark import _arm_aggregate agg = _arm_aggregate(rows) agg["clean"] = _split_rate([r for r in rows if r.get("condition") == "clean"]) @@ -621,7 +621,7 @@ def render_dom_chart(results: dict[str, Any], out_png: Path) -> Path: Returns: The written PNG path. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() from matplotlib.patches import Rectangle @@ -1177,7 +1177,7 @@ def stat_row(label: str, fmt: Callable[[dict[str, Any]], str]) -> str: clicks the Open button in the row named `{TARGET_PATIENT}` — the identity reading, hardcoding the demonstrated patient exactly as the compiled arm's recorded identity band does. Every selector choice is -documented in `openadapt_flow/benchmark/dom_arm.py`. +documented in `benchmark/harness/dom_arm.py`. ## Verdict @@ -1333,7 +1333,7 @@ def stat_row(label: str, fmt: Callable[[dict[str, Any]], str]) -> str: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.dom_arm --out benchmark/dom --n-per-perturbation 2 +.venv/bin/python -m benchmark.harness.dom_arm --out benchmark/dom --n-per-perturbation 2 ``` (`--n-per-perturbation 2` matches the committed results.) No API key @@ -1350,7 +1350,7 @@ def write_dom_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_dom_chart, results, out_dir / "outcome_matrix.png") (out_dir / "BENCHMARK.md").write_text(render_dom_markdown(results)) @@ -1388,7 +1388,7 @@ def run_dom_benchmark( Returns: The results dict (also written to ``results.json``). """ - from openadapt_flow.benchmark.run_benchmark import _compiled_run + from benchmark.harness.run_benchmark import _compiled_run from openadapt_flow.compiler import compile_recording from openadapt_flow.demo_driver import record_triage_demo from openadapt_flow.mockmed.server import serve @@ -1499,7 +1499,7 @@ def one_run( def main(argv: Optional[list[str]] = None) -> int: - """CLI entry point: ``python -m openadapt_flow.benchmark.dom_arm``.""" + """CLI entry point: ``python -m benchmark.harness.dom_arm``.""" parser = argparse.ArgumentParser( description=( "DOM-selector benchmark arm vs. compiled vision replay on " diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/README.md b/packages/flow/benchmark/harness/effectbench/README.md similarity index 99% rename from packages/flow/openadapt_flow/benchmark/effectbench/README.md rename to packages/flow/benchmark/harness/effectbench/README.md index c42718959f..08b926e621 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/README.md +++ b/packages/flow/benchmark/harness/effectbench/README.md @@ -5,7 +5,7 @@ > system-of-record oracle* disagrees. Reported jointly with **over-halt rate**, > **task success**, and the **success–effect gap**. -This package (`openadapt_flow.benchmark.effectbench`) is the **stable contract** +This package (`benchmark.harness.effectbench`) is the **stable contract** that two downstream efforts build on: 1. the **multi-baseline runner adapter** — drives each agent arm through a common diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/__init__.py b/packages/flow/benchmark/harness/effectbench/__init__.py similarity index 93% rename from packages/flow/openadapt_flow/benchmark/effectbench/__init__.py rename to packages/flow/benchmark/harness/effectbench/__init__.py index 7291bdbf04..62562d9441 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/__init__.py +++ b/packages/flow/benchmark/harness/effectbench/__init__.py @@ -31,10 +31,10 @@ bootstrap confidence intervals, all decomposed by category x substrate. The design doc is ``.private/benchmark_design_2026_07_20.md``; the operator -contract is ``openadapt_flow/benchmark/effectbench/README.md``. +contract is ``benchmark/harness/effectbench/README.md``. """ -from openadapt_flow.benchmark.effectbench.metrics import ( # noqa: F401 +from benchmark.harness.effectbench.metrics import ( # noqa: F401 BenchmarkSummary, CellSummary, Interval, @@ -44,7 +44,7 @@ summarize, wilson_interval, ) -from openadapt_flow.benchmark.effectbench.oracle import ( # noqa: F401 +from benchmark.harness.effectbench.oracle import ( # noqa: F401 RecordSnapshotOracle, TrueEffectState, classify_outcome, @@ -53,7 +53,7 @@ oracle_verdict_of, score_episode, ) -from openadapt_flow.benchmark.effectbench.schema import ( # noqa: F401 +from benchmark.harness.effectbench.schema import ( # noqa: F401 AgentReport, DivergenceCategory, EpisodeRecord, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/metrics.py b/packages/flow/benchmark/harness/effectbench/metrics.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/effectbench/metrics.py rename to packages/flow/benchmark/harness/effectbench/metrics.py index b7ad870e60..356119fde2 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/metrics.py +++ b/packages/flow/benchmark/harness/effectbench/metrics.py @@ -1,7 +1,7 @@ """EffectBench metrics — SWER and its co-metrics, with confidence intervals. Every headline is a pure aggregation over a list of -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord` rows, so a +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord` rows, so a run's numbers can be recomputed from the published raw rows by anyone. No single mean is reported alone: :func:`summarize` always returns the per (category x substrate) decomposition alongside the overall rate (design doc @@ -39,7 +39,7 @@ from pydantic import BaseModel, ConfigDict, Field -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( EpisodeRecord, OutcomeLabel, SwerVariant, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/oracle.py b/packages/flow/benchmark/harness/effectbench/oracle.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/effectbench/oracle.py rename to packages/flow/benchmark/harness/effectbench/oracle.py index f7f5ca031a..e490479d8b 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/oracle.py +++ b/packages/flow/benchmark/harness/effectbench/oracle.py @@ -17,7 +17,7 @@ 2. **The classifier** — :func:`classify_outcome`, which crosses the oracle's reading of the true effect with the agent's (untrusted) self-report to - assign exactly one :class:`~openadapt_flow.benchmark.effectbench.schema.OutcomeLabel`. + assign exactly one :class:`~benchmark.harness.effectbench.schema.OutcomeLabel`. This is ``fault_model.classify`` generalized: it adds the OVER_HALT vs SAFE_HALT split (was the correct action available?) and makes ``SILENT_WRONG_EFFECT`` the single home of ``reported_success AND @@ -25,7 +25,7 @@ :func:`score_episode` ties the two together for a runner: capture pre-state, run the arm's action, verify, classify, and emit an -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord`. +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord`. """ from __future__ import annotations @@ -34,7 +34,7 @@ from enum import Enum from typing import Any, Callable, Mapping, Optional -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( AgentReport, DivergenceCategory, EpisodeRecord, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/README.md b/packages/flow/benchmark/harness/effectbench/runner/README.md similarity index 95% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/README.md rename to packages/flow/benchmark/harness/effectbench/runner/README.md index a29834a364..a00d55f331 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/README.md +++ b/packages/flow/benchmark/harness/effectbench/runner/README.md @@ -1,7 +1,7 @@ # EffectBench runner adapter — one interface, every baseline The **multi-baseline runner adapter** (downstream effort #2 on the -`openadapt_flow.benchmark.effectbench` contract). It drives every agent *arm* +`benchmark.harness.effectbench` contract). It drives every agent *arm* through one common interface against the **identical task** and **identical independent oracle**, records one `EpisodeRecord` per (task × arm × trial), and summarizes per arm with `summarize`. A difference between arms is then the @@ -10,8 +10,8 @@ AGENT, never the harness. Run the CI-fast dry-run (real in-process HTTP, no Docker, no paid API, no spend): ```bash -python -m openadapt_flow.benchmark.effectbench.runner # all live arms -python -m openadapt_flow.benchmark.effectbench.runner --list-arms +python -m benchmark.harness.effectbench.runner # all live arms +python -m benchmark.harness.effectbench.runner --list-arms ``` ## The arm interface diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__init__.py b/packages/flow/benchmark/harness/effectbench/runner/__init__.py similarity index 72% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/__init__.py rename to packages/flow/benchmark/harness/effectbench/runner/__init__.py index 759ebae7d1..27fe3e39ff 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__init__.py +++ b/packages/flow/benchmark/harness/effectbench/runner/__init__.py @@ -2,20 +2,20 @@ Drives every agent *arm* through one common interface against the IDENTICAL task and INDEPENDENT oracle, records one -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord` per +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord` per (task × arm × trial), and summarizes per arm with -:func:`~openadapt_flow.benchmark.effectbench.metrics.summarize`. Built on the -stable contract in :mod:`openadapt_flow.benchmark.effectbench` (schema / oracle / +:func:`~benchmark.harness.effectbench.metrics.summarize`. Built on the +stable contract in :mod:`benchmark.harness.effectbench` (schema / oracle / metrics) — this package adds only the runner, never re-implements the taxonomy. Layers: - :mod:`.arms` — the ``AgentArm`` interface + the concrete in-repo arms: - :class:`~openadapt_flow.benchmark.effectbench.runner.arms.CompilerArm` + :class:`~benchmark.harness.effectbench.runner.arms.CompilerArm` (record→compile→replay, effect-gated), - :class:`~openadapt_flow.benchmark.effectbench.runner.arms.ScreenOnlyArm` + :class:`~benchmark.harness.effectbench.runner.arms.ScreenOnlyArm` (the silent-wrong-effect ablation), and a substrate-free - :class:`~openadapt_flow.benchmark.effectbench.runner.arms.MockArm` for CI. + :class:`~benchmark.harness.effectbench.runner.arms.MockArm` for CI. - :mod:`.substrate` — the arm-facing action+perception channel (the reference MockMed session), isolated from the oracle read path. - :mod:`.compound` — the compound consequential-save verifier (record + note). @@ -25,19 +25,19 @@ - :mod:`.harness` — ``run_episode`` / ``run_matrix``: wire each arm through ``score_episode`` with the task's oracle, fairly and identically. -Run the dry-run: ``python -m openadapt_flow.benchmark.effectbench.runner``. +Run the dry-run: ``python -m benchmark.harness.effectbench.runner``. """ from __future__ import annotations -from openadapt_flow.benchmark.effectbench.runner.arms import ( +from benchmark.harness.effectbench.runner.arms import ( AgentArm, ArmResult, CompilerArm, MockArm, ScreenOnlyArm, ) -from openadapt_flow.benchmark.effectbench.runner.baselines import ( +from benchmark.harness.effectbench.runner.baselines import ( SCAFFOLDED_ARMS, BaselineRequirements, ClaudeComputerUseArm, @@ -46,21 +46,21 @@ SkyvernArm, UITarsArm, ) -from openadapt_flow.benchmark.effectbench.runner.compound import ( +from benchmark.harness.effectbench.runner.compound import ( CompoundEffectVerifier, ) -from openadapt_flow.benchmark.effectbench.runner.harness import ( +from benchmark.harness.effectbench.runner.harness import ( EnvFactory, EpisodeEnv, run_episode, run_matrix, ) -from openadapt_flow.benchmark.effectbench.runner.reference_tasks import ( +from benchmark.harness.effectbench.runner.reference_tasks import ( MockMedEnvProvider, mockmed_env_factory, reference_tasks, ) -from openadapt_flow.benchmark.effectbench.runner.substrate import ( +from benchmark.harness.effectbench.runner.substrate import ( MockMedFault, MockMedSession, ScreenObservation, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__main__.py b/packages/flow/benchmark/harness/effectbench/runner/__main__.py similarity index 90% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/__main__.py rename to packages/flow/benchmark/harness/effectbench/runner/__main__.py index 4ba0d142eb..bc6c79734f 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/__main__.py +++ b/packages/flow/benchmark/harness/effectbench/runner/__main__.py @@ -1,6 +1,6 @@ """EffectBench runner CLI — the CI-fast dry-run over the MockMed reference pack. -``python -m openadapt_flow.benchmark.effectbench.runner`` drives the nine MockMed +``python -m benchmark.harness.effectbench.runner`` drives the nine MockMed reference tasks through the in-repo LIVE arms (compiler, screen-only ablation, mock) over a real in-process HTTP fault server — no Docker, no paid API, no spend — and prints the per-arm summary. It proves the pipeline end-to-end: the @@ -25,16 +25,16 @@ import sys from typing import Optional, Sequence -from openadapt_flow.benchmark.effectbench.metrics import BenchmarkSummary, summarize -from openadapt_flow.benchmark.effectbench.runner import LIVE_ARMS -from openadapt_flow.benchmark.effectbench.runner.arms import AgentArm -from openadapt_flow.benchmark.effectbench.runner.baselines import SCAFFOLDED_ARMS -from openadapt_flow.benchmark.effectbench.runner.harness import run_matrix -from openadapt_flow.benchmark.effectbench.runner.reference_tasks import ( +from benchmark.harness.effectbench.metrics import BenchmarkSummary, summarize +from benchmark.harness.effectbench.runner import LIVE_ARMS +from benchmark.harness.effectbench.runner.arms import AgentArm +from benchmark.harness.effectbench.runner.baselines import SCAFFOLDED_ARMS +from benchmark.harness.effectbench.runner.harness import run_matrix +from benchmark.harness.effectbench.runner.reference_tasks import ( MockMedEnvProvider, reference_tasks, ) -from openadapt_flow.benchmark.effectbench.schema import EpisodeRecord +from benchmark.harness.effectbench.schema import EpisodeRecord def _select_arms(names: Optional[str]) -> list[AgentArm]: @@ -128,7 +128,7 @@ def run_dry_run( def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser( - prog="python -m openadapt_flow.benchmark.effectbench.runner", + prog="python -m benchmark.harness.effectbench.runner", description="EffectBench multi-baseline runner — MockMed dry-run.", ) parser.add_argument("--trials", type=int, default=3) diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/arms.py b/packages/flow/benchmark/harness/effectbench/runner/arms.py similarity index 95% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/arms.py rename to packages/flow/benchmark/harness/effectbench/runner/arms.py index 063389bcef..a63a5985c3 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/arms.py +++ b/packages/flow/benchmark/harness/effectbench/runner/arms.py @@ -18,14 +18,14 @@ def run(self, task: TaskSpec, session: SubstrateSession, An arm is handed ONLY the task's natural-language ``goal`` (intent, never a step list — the fairness requirement) via ``task``, a -:class:`~openadapt_flow.benchmark.effectbench.runner.substrate.SubstrateSession` +:class:`~benchmark.harness.effectbench.runner.substrate.SubstrateSession` to drive, and the run's ``params`` (the trial-unique payload it must write). It returns an :class:`ArmResult` carrying the untrusted -:class:`~openadapt_flow.benchmark.effectbench.schema.AgentReport` plus any -recorded :class:`~openadapt_flow.benchmark.effectbench.schema.ModelCall` rows so +:class:`~benchmark.harness.effectbench.schema.AgentReport` plus any +recorded :class:`~benchmark.harness.effectbench.schema.ModelCall` rows so cost is auditable. The harness (:mod:`.harness`) adapts ``run`` into the ``run_action: Callable[[], AgentReport]`` that -:func:`~openadapt_flow.benchmark.effectbench.oracle.score_episode` calls, and +:func:`~benchmark.harness.effectbench.oracle.score_episode` calls, and supplies the independent oracle — which the arm can NEVER reach. Three concrete in-repo arms live here: @@ -50,11 +50,11 @@ def run(self, task: TaskSpec, session: SubstrateSession, from dataclasses import dataclass, field from typing import Any, Callable, Mapping, Optional, Protocol, runtime_checkable -from openadapt_flow.benchmark.effectbench.runner.substrate import ( +from benchmark.harness.effectbench.runner.substrate import ( ScreenObservation, SubstrateSession, ) -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( AgentReport, ModelCall, TaskSpec, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/baselines.py b/packages/flow/benchmark/harness/effectbench/runner/baselines.py similarity index 97% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/baselines.py rename to packages/flow/benchmark/harness/effectbench/runner/baselines.py index 3674a508ce..5a43fbf73d 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/baselines.py +++ b/packages/flow/benchmark/harness/effectbench/runner/baselines.py @@ -21,7 +21,7 @@ side channel to the system of record; the independent oracle, not the agent, judges the effect. 3. **Record every model call.** Append one - :class:`~openadapt_flow.benchmark.effectbench.schema.ModelCall` per API + :class:`~benchmark.harness.effectbench.schema.ModelCall` per API request with token counts and the pinned list price, so ``cost_usd`` is auditable ``sum(model_calls)``. An RPA tool with a flat run fee records a single ``ModelCall`` carrying that fee. @@ -38,9 +38,9 @@ from dataclasses import dataclass from typing import Mapping -from openadapt_flow.benchmark.effectbench.runner.arms import ArmResult -from openadapt_flow.benchmark.effectbench.runner.substrate import SubstrateSession -from openadapt_flow.benchmark.effectbench.schema import TaskSpec +from benchmark.harness.effectbench.runner.arms import ArmResult +from benchmark.harness.effectbench.runner.substrate import SubstrateSession +from benchmark.harness.effectbench.schema import TaskSpec class ScaffoldNotWired(NotImplementedError): diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/compound.py b/packages/flow/benchmark/harness/effectbench/runner/compound.py similarity index 94% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/compound.py rename to packages/flow/benchmark/harness/effectbench/runner/compound.py index 1c6236449a..3c15f795f5 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/compound.py +++ b/packages/flow/benchmark/harness/effectbench/runner/compound.py @@ -9,16 +9,16 @@ * ``field_equals`` — that encounter's note reads back the run's exact value; catches a *partial save* that persists the row but drops a field. -:class:`~openadapt_flow.benchmark.effectbench.oracle.score_episode` verifies a +:class:`~benchmark.harness.effectbench.oracle.score_episode` verifies a SINGLE :class:`Effect` through the oracle. This module supplies a thin :class:`~openadapt_flow.runtime.effects.effect.EffectVerifier` that wraps ANY concrete base verifier (the in-process -:class:`~openadapt_flow.benchmark.effectbench.oracle.RecordSnapshotOracle`, a +:class:`~benchmark.harness.effectbench.oracle.RecordSnapshotOracle`, a :class:`~openadapt_flow.runtime.effects.rest.RestRecordVerifier`, …) and, when ``score_episode`` calls :meth:`verify` with the primary ``record_written`` effect, ALSO checks the authored extra sub-effects against the SAME pre-state snapshot, reducing the per-effect true-effect states with -:func:`~openadapt_flow.benchmark.effectbench.oracle.combine_true_states`. +:func:`~benchmark.harness.effectbench.oracle.combine_true_states`. It returns the single **deciding** :class:`EffectVerdict` whose ``effect_state`` equals the combined state (the fault-model precedence: the @@ -33,7 +33,7 @@ from typing import Any, Sequence -from openadapt_flow.benchmark.effectbench.oracle import ( +from benchmark.harness.effectbench.oracle import ( TrueEffectState, combine_true_states, effect_state, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/harness.py b/packages/flow/benchmark/harness/effectbench/runner/harness.py similarity index 92% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/harness.py rename to packages/flow/benchmark/harness/effectbench/runner/harness.py index f21991fed0..f1bafeefbf 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/harness.py +++ b/packages/flow/benchmark/harness/effectbench/runner/harness.py @@ -1,11 +1,11 @@ """The runner harness — wire each arm through ``score_episode`` identically. This is the multi-baseline runner's core: given a set of -:class:`~openadapt_flow.benchmark.effectbench.schema.TaskSpec` and a set of arms, +:class:`~benchmark.harness.effectbench.schema.TaskSpec` and a set of arms, it produces one -:class:`~openadapt_flow.benchmark.effectbench.schema.EpisodeRecord` per +:class:`~benchmark.harness.effectbench.schema.EpisodeRecord` per (task × arm × trial), each scored by the task's INDEPENDENT oracle through -:func:`~openadapt_flow.benchmark.effectbench.oracle.score_episode`. Every arm +:func:`~benchmark.harness.effectbench.oracle.score_episode`. Every arm runs the SAME task and is judged by the SAME oracle — the harness guarantees the comparison is fair by construction: @@ -30,10 +30,10 @@ from dataclasses import dataclass, field from typing import Any, Callable, List, Mapping, Optional, Sequence -from openadapt_flow.benchmark.effectbench.oracle import score_episode -from openadapt_flow.benchmark.effectbench.runner.arms import AgentArm -from openadapt_flow.benchmark.effectbench.runner.substrate import SubstrateSession -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.oracle import score_episode +from benchmark.harness.effectbench.runner.arms import AgentArm +from benchmark.harness.effectbench.runner.substrate import SubstrateSession +from benchmark.harness.effectbench.schema import ( AgentReport, EpisodeRecord, ModelCall, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/reference_tasks.py b/packages/flow/benchmark/harness/effectbench/runner/reference_tasks.py similarity index 96% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/reference_tasks.py rename to packages/flow/benchmark/harness/effectbench/runner/reference_tasks.py index 6afe49e2be..9c4c8de44e 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/reference_tasks.py +++ b/packages/flow/benchmark/harness/effectbench/runner/reference_tasks.py @@ -3,7 +3,7 @@ These are the reference tasks the multi-baseline runner exercises end-to-end without Docker or spend: the bundled MockMed transactional-fault surface (``openadapt_flow.mockmed.fault_server``) turned into -:class:`~openadapt_flow.benchmark.effectbench.schema.TaskSpec` s across the +:class:`~benchmark.harness.effectbench.schema.TaskSpec` s across the divergence categories, plus a :func:`mockmed_env_factory` that provisions one isolated fault server per episode with an INDEPENDENT in-process oracle. @@ -24,17 +24,17 @@ from dataclasses import dataclass from typing import Mapping, Optional -from openadapt_flow.benchmark.effectbench.oracle import RecordSnapshotOracle -from openadapt_flow.benchmark.effectbench.runner.compound import ( +from benchmark.harness.effectbench.oracle import RecordSnapshotOracle +from benchmark.harness.effectbench.runner.compound import ( CompoundEffectVerifier, ) -from openadapt_flow.benchmark.effectbench.runner.harness import EpisodeEnv -from openadapt_flow.benchmark.effectbench.runner.substrate import ( +from benchmark.harness.effectbench.runner.harness import EpisodeEnv +from benchmark.harness.effectbench.runner.substrate import ( MockMedFault, MockMedSession, mockmed_session, ) -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( DivergenceCategory, OracleChannel, OracleSpec, diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/runner/substrate.py b/packages/flow/benchmark/harness/effectbench/runner/substrate.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/effectbench/runner/substrate.py rename to packages/flow/benchmark/harness/effectbench/runner/substrate.py index 3d4e525528..1322d730f6 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/runner/substrate.py +++ b/packages/flow/benchmark/harness/effectbench/runner/substrate.py @@ -34,10 +34,10 @@ from dataclasses import dataclass, field from typing import Any, Mapping, Optional, Protocol, Sequence, runtime_checkable -from openadapt_flow.benchmark.effectbench.runner.compound import ( +from benchmark.harness.effectbench.runner.compound import ( CompoundEffectVerifier, ) -from openadapt_flow.benchmark.effectbench.schema import Substrate +from benchmark.harness.effectbench.schema import Substrate from openadapt_flow.runtime.effects.effect import Effect, EffectVerifier diff --git a/packages/flow/openadapt_flow/benchmark/effectbench/schema.py b/packages/flow/benchmark/harness/effectbench/schema.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/effectbench/schema.py rename to packages/flow/benchmark/harness/effectbench/schema.py index 4e529ae716..d92fb0af90 100644 --- a/packages/flow/openadapt_flow/benchmark/effectbench/schema.py +++ b/packages/flow/benchmark/harness/effectbench/schema.py @@ -135,7 +135,7 @@ class OracleChannel(str, Enum): Maps 1:1 onto the ``substrate`` attribute of the concrete :class:`~openadapt_flow.runtime.effects.effect.EffectVerifier` - implementations re-exported by :mod:`openadapt_flow.benchmark.effectbench`. + implementations re-exported by :mod:`benchmark.harness.effectbench`. """ SQL = "sql" # SqlRecordVerifier — read-only SELECT under a read-only role @@ -311,7 +311,7 @@ class EpisodeRecord(BaseModel): """One trial of one task by one arm — the atomic benchmark result row. Every headline metric is a pure aggregation over a list of these (see - :func:`openadapt_flow.benchmark.effectbench.metrics.summarize`). The runner + :func:`benchmark.harness.effectbench.metrics.summarize`). The runner produces one per (task x arm x trial); ``summarize`` never needs anything the row does not carry, so raw rows can be published for external replication. """ diff --git a/packages/flow/openadapt_flow/benchmark/frappe_lending.py b/packages/flow/benchmark/harness/frappe_lending.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/frappe_lending.py rename to packages/flow/benchmark/harness/frappe_lending.py diff --git a/packages/flow/openadapt_flow/benchmark/hybrid_benchmark.py b/packages/flow/benchmark/harness/hybrid_benchmark.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/hybrid_benchmark.py rename to packages/flow/benchmark/harness/hybrid_benchmark.py index c343d13b45..282bb79cbb 100644 --- a/packages/flow/openadapt_flow/benchmark/hybrid_benchmark.py +++ b/packages/flow/benchmark/harness/hybrid_benchmark.py @@ -67,12 +67,12 @@ from pydantic import BaseModel -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.openemr_benchmark import ( +from benchmark.harness import agent_baseline +from benchmark.harness.openemr_benchmark import ( BILLING_ABORT_AFTER, _looks_like_billing_error, ) -from openadapt_flow.benchmark.run_benchmark import ( +from benchmark.harness.run_benchmark import ( _agent_run, _arm_aggregate, _compiled_run, @@ -863,7 +863,7 @@ def render_hybrid_chart(results: dict[str, Any], out_png: Path) -> Path: Returns: The written PNG path. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() @@ -1379,7 +1379,7 @@ def failures(arm: str) -> str: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.hybrid_benchmark --out benchmark/hybrid +.venv/bin/python -m benchmark.harness.hybrid_benchmark --out benchmark/hybrid ``` Requires `ANTHROPIC_API_KEY` (or `~/.anthropic/api_key`). The paid arms @@ -1398,7 +1398,7 @@ def write_hybrid_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_hybrid_chart, results, out_dir / "success_cost.png") (out_dir / "BENCHMARK.md").write_text(render_hybrid_markdown(results)) @@ -1649,7 +1649,7 @@ def _error_row(arm: str, exc: Exception) -> dict[str, Any]: def main(argv: Optional[list[str]] = None) -> int: - """CLI entry point: ``python -m openadapt_flow.benchmark.hybrid_benchmark``.""" + """CLI entry point: ``python -m benchmark.harness.hybrid_benchmark``.""" parser = argparse.ArgumentParser( description=( "Four-arm hybrid benchmark on MockMed (compiled / agent / " diff --git a/packages/flow/openadapt_flow/instrument/__init__.py b/packages/flow/benchmark/harness/instrument/__init__.py similarity index 91% rename from packages/flow/openadapt_flow/instrument/__init__.py rename to packages/flow/benchmark/harness/instrument/__init__.py index bf92686661..dbf7469cc1 100644 --- a/packages/flow/openadapt_flow/instrument/__init__.py +++ b/packages/flow/benchmark/harness/instrument/__init__.py @@ -1,7 +1,7 @@ """Competitor-drift instrument: point our EffectVerifier at ANY external agent. The self-directed silent-wrong-action benchmark -(``openadapt_flow.benchmark.silent_wrong_action``, #67) measures how often our +(``benchmark.harness.silent_wrong_action``, #67) measures how often our OWN runtime lets a wrong / absent / duplicate business effect land while reporting success. This package generalizes that measurement to an *arbitrary external computer-use agent*: given any agent behind the @@ -19,7 +19,7 @@ user-gated step. """ -from openadapt_flow.instrument.competitor_drift import ( # noqa: F401 +from benchmark.harness.instrument.competitor_drift import ( # noqa: F401 AgentRunResult, ArchitectureClassError, CostGuard, diff --git a/packages/flow/openadapt_flow/instrument/competitor_drift.py b/packages/flow/benchmark/harness/instrument/competitor_drift.py similarity index 99% rename from packages/flow/openadapt_flow/instrument/competitor_drift.py rename to packages/flow/benchmark/harness/instrument/competitor_drift.py index 3daa5076c6..2abf9fe980 100644 --- a/packages/flow/openadapt_flow/instrument/competitor_drift.py +++ b/packages/flow/benchmark/harness/instrument/competitor_drift.py @@ -2,7 +2,7 @@ runner, cost-capped. This EXTENDS the self-directed silent-wrong-action benchmark -(``openadapt_flow.benchmark.silent_wrong_action``, #67) from "our own runtime" +(``benchmark.harness.silent_wrong_action``, #67) from "our own runtime" to "any external computer-use agent." The measurement is identical — did a wrong / absent / duplicate business effect land in the system of record while the actor reported success (a *silent wrong-action*)? — but the actor is now an @@ -53,7 +53,7 @@ # judge, and the #63 effect contract. We REUSE these from the self-directed # benchmark rather than re-deriving them so the external-agent instrument and # the self-directed instrument grade with byte-identical semantics. -from openadapt_flow.benchmark.silent_wrong_action import ( +from benchmark.harness.silent_wrong_action import ( NOTE, SCENARIOS, Scenario, @@ -888,7 +888,7 @@ def write_outputs(report: InstrumentReport, out_dir: Path | str) -> None: def main(argv: Optional[list[str]] = None) -> int: - """CLI: ``python -m openadapt_flow.instrument.competitor_drift``. + """CLI: ``python -m benchmark.harness.instrument.competitor_drift``. Ships only the offline stub adapter and a dry-run estimator — a real external adapter is wired programmatically (see diff --git a/packages/flow/openadapt_flow/benchmark/openemr_benchmark.py b/packages/flow/benchmark/harness/openemr_benchmark.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/openemr_benchmark.py rename to packages/flow/benchmark/harness/openemr_benchmark.py index f2482f2307..5bb2396f11 100644 --- a/packages/flow/openadapt_flow/benchmark/openemr_benchmark.py +++ b/packages/flow/benchmark/harness/openemr_benchmark.py @@ -7,7 +7,7 @@ recorded by ``scripts/openemr_demo.py`` (fake demo patients only). Both arms are judged by the same criterion, implemented once -(:func:`openadapt_flow.benchmark.verify.verify_note_saved`): OCR of the +(:func:`benchmark.harness.verify.verify_note_saved`): OCR of the final screen must show the run's parameterized note text in the patient message list. Each run — in BOTH arms — uses a distinct note value, so a pass proves parameter substitution against live state, not replay of a @@ -60,16 +60,16 @@ from pathlib import Path from typing import Any, Callable -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.agent_baseline import _looks_like_billing_error -from openadapt_flow.benchmark.run_benchmark import ( +from benchmark.harness import agent_baseline +from benchmark.harness.agent_baseline import _looks_like_billing_error +from benchmark.harness.run_benchmark import ( _agent_run, _arm_aggregate, _compiled_run, identity_coverage_block, render_chart, ) -from openadapt_flow.benchmark.verify import verify_note_saved +from benchmark.harness.verify import verify_note_saved DEMO_URL = "https://demo.openemr.io/openemr/index.php" N_COMPILED = 20 @@ -481,7 +481,7 @@ def write_openemr_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "latency_cost.png") (out_dir / "BENCHMARK.md").write_text(render_openemr_markdown(results)) diff --git a/packages/flow/openadapt_flow/benchmark/openemr_local.py b/packages/flow/benchmark/harness/openemr_local.py similarity index 99% rename from packages/flow/openadapt_flow/benchmark/openemr_local.py rename to packages/flow/benchmark/harness/openemr_local.py index 125f6d9c24..1a4e337de8 100644 --- a/packages/flow/openadapt_flow/benchmark/openemr_local.py +++ b/packages/flow/benchmark/harness/openemr_local.py @@ -18,7 +18,7 @@ from typing import Any, Iterable, Mapping, Sequence from urllib.parse import urlencode -from openadapt_flow.benchmark.frappe_lending import ( +from benchmark.harness.frappe_lending import ( ARMS, CONDITIONS, INITIAL_TRIALS_PER_CELL, diff --git a/packages/flow/openadapt_flow/benchmark/reliability.py b/packages/flow/benchmark/harness/reliability.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/reliability.py rename to packages/flow/benchmark/harness/reliability.py diff --git a/packages/flow/openadapt_flow/benchmark/run_benchmark.py b/packages/flow/benchmark/harness/run_benchmark.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/run_benchmark.py rename to packages/flow/benchmark/harness/run_benchmark.py index cd2c4c2d1e..cdfc68a791 100644 --- a/packages/flow/openadapt_flow/benchmark/run_benchmark.py +++ b/packages/flow/benchmark/harness/run_benchmark.py @@ -28,9 +28,9 @@ from pathlib import Path from typing import Any, Callable +from benchmark.harness import agent_baseline +from benchmark.harness.verify import verify_encounter_saved from openadapt_flow.bench import _percentile -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.verify import verify_encounter_saved N_COMPILED = 100 N_AGENT = 20 @@ -153,7 +153,7 @@ def _agent_run( A per-run row dict. API failures are recorded as failed rows with ``error`` set, never raised; a failed row still carries whatever usage/cost the run paid for before crashing (via - :class:`~openadapt_flow.benchmark.agent_baseline.UsageLedger`), so + :class:`~benchmark.harness.agent_baseline.UsageLedger`), so crashed runs' real spend counts in aggregates and cost ceilings. """ from openadapt_flow.backends.playwright_backend import PlaywrightBackend @@ -364,7 +364,7 @@ def render_chart(results: dict[str, Any], out_png: Path) -> Path: Returns: The written PNG path. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() @@ -578,7 +578,7 @@ def write_outputs(results: dict[str, Any], out_dir: Path) -> None: """ out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "latency_cost.png") (out_dir / "BENCHMARK.md").write_text(render_markdown(results)) diff --git a/packages/flow/openadapt_flow/benchmark/silent_wrong_action.py b/packages/flow/benchmark/harness/silent_wrong_action.py similarity index 98% rename from packages/flow/openadapt_flow/benchmark/silent_wrong_action.py rename to packages/flow/benchmark/harness/silent_wrong_action.py index 009bdda54c..daeb0c7c9f 100644 --- a/packages/flow/openadapt_flow/benchmark/silent_wrong_action.py +++ b/packages/flow/benchmark/harness/silent_wrong_action.py @@ -566,7 +566,7 @@ def render_markdown(results: dict[str, Any]) -> str: ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.silent_wrong_action \\ +.venv/bin/python -m benchmark.harness.silent_wrong_action \\ --out {DEFAULT_OUT_DIR} --n {results["n_per_scenario"]} ``` @@ -585,7 +585,7 @@ def render_chart(results: dict[str, Any], out_png: Path) -> Path: false-abort rate. The metric a record system cares about — a wrong write that reports success — collapses from a large bar to zero. """ - from openadapt_flow.benchmark.chart_fonts import configure_bundled_font + from benchmark.harness.chart_fonts import configure_bundled_font plt = configure_bundled_font() @@ -658,7 +658,7 @@ def write_outputs(results: dict[str, Any], out_dir: Path) -> None: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") (out_dir / "SILENT_WRONG_ACTION.md").write_text(render_markdown(results)) - from openadapt_flow.benchmark.chart_fonts import safe_render + from benchmark.harness.chart_fonts import safe_render safe_render(render_chart, results, out_dir / "silent_wrong_action.png") @@ -689,7 +689,7 @@ def run_and_write( def main(argv: Optional[list[str]] = None) -> int: - """CLI: ``python -m openadapt_flow.benchmark.silent_wrong_action``.""" + """CLI: ``python -m benchmark.harness.silent_wrong_action``.""" parser = argparse.ArgumentParser( description=( "Silent-wrong-action-rate benchmark: measure how often " diff --git a/packages/flow/openadapt_flow/benchmark/verify.py b/packages/flow/benchmark/harness/verify.py similarity index 100% rename from packages/flow/openadapt_flow/benchmark/verify.py rename to packages/flow/benchmark/harness/verify.py diff --git a/packages/flow/benchmark/hybrid/BENCHMARK.md b/packages/flow/benchmark/hybrid/BENCHMARK.md index bc64083f0d..d85401a2a7 100644 --- a/packages/flow/benchmark/hybrid/BENCHMARK.md +++ b/packages/flow/benchmark/hybrid/BENCHMARK.md @@ -207,7 +207,7 @@ $0.0967), and `a` = the agent-only mean cost per run ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.hybrid_benchmark --out benchmark/hybrid +.venv/bin/python -m benchmark.harness.hybrid_benchmark --out benchmark/hybrid ``` Requires `ANTHROPIC_API_KEY` (or `~/.anthropic/api_key`). The paid arms diff --git a/packages/flow/benchmark/lending_fault_model/README.md b/packages/flow/benchmark/lending_fault_model/README.md index 740b9134ff..b6de3a698a 100644 --- a/packages/flow/benchmark/lending_fault_model/README.md +++ b/packages/flow/benchmark/lending_fault_model/README.md @@ -39,7 +39,7 @@ judged by an independent ledger read, with zero model calls. 2. **Silent Wrong-Effect Rate, three-arm verification ladder** - `swer.py` scores the same faults through the shared EffectBench contract - (`openadapt_flow.benchmark.effectbench.score_episode` + `summarize`) under + (`benchmark.harness.effectbench.score_episode` + `summarize`) under screen-only, single-surface, and complete-read-path arms. -> `swer_results.json`, `SWER.md`. Headline: screen-only SWER is 24/36, a disbursements-only verifier leaves the diff --git a/packages/flow/benchmark/lending_fault_model/swer.py b/packages/flow/benchmark/lending_fault_model/swer.py index 9471d68615..312cc57e4a 100644 --- a/packages/flow/benchmark/lending_fault_model/swer.py +++ b/packages/flow/benchmark/lending_fault_model/swer.py @@ -5,7 +5,7 @@ screen-only postcondition contract (and measures how often that silently mishandles a transactional fault), THIS module measures the same faults through the shared EffectBench scoring contract -(:func:`openadapt_flow.benchmark.effectbench.score_episode`) under a THREE-arm +(:func:`benchmark.harness.effectbench.score_episode`) under a THREE-arm ladder that mirrors the clinical ``effect_e2e`` study (screen / single-surface / complete-read-path), so the two domains are directly comparable: @@ -71,7 +71,7 @@ import requests -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, DivergenceCategory, Effect, diff --git a/packages/flow/benchmark/openemr_api_tier/approval_durable.py b/packages/flow/benchmark/openemr_api_tier/approval_durable.py index 7b3e57f929..47dbe33709 100644 --- a/packages/flow/benchmark/openemr_api_tier/approval_durable.py +++ b/packages/flow/benchmark/openemr_api_tier/approval_durable.py @@ -157,7 +157,7 @@ def run_one( ) -> dict[str, Any]: from openadapt_flow.approval_mode.gate import ApprovalGate from openadapt_flow.approval_mode.item_approval import ItemApprovalStore - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer from openadapt_flow.runtime.durable.checkpoint import CheckpointStore diff --git a/packages/flow/benchmark/openemr_api_tier/common.py b/packages/flow/benchmark/openemr_api_tier/common.py index 19bb064547..8764d08f22 100644 --- a/packages/flow/benchmark/openemr_api_tier/common.py +++ b/packages/flow/benchmark/openemr_api_tier/common.py @@ -74,7 +74,7 @@ def load_demo() -> Any: def spec_for(ref: Referral) -> Any: - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec return SyntheticPatientSpec(**ref.kwargs) diff --git a/packages/flow/benchmark/openemr_api_tier/feature_e2e.py b/packages/flow/benchmark/openemr_api_tier/feature_e2e.py index de1744eb7a..1681add2ff 100644 --- a/packages/flow/benchmark/openemr_api_tier/feature_e2e.py +++ b/packages/flow/benchmark/openemr_api_tier/feature_e2e.py @@ -51,7 +51,7 @@ def cmd_record(a: argparse.Namespace) -> None: - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec from openadapt_flow.network_capture import NetworkCapture from openadapt_flow.recorder import Recorder @@ -191,7 +191,7 @@ def wait_quiet(max_s: float = float(os.environ.get("GATE_MAX_S", "1200"))) -> di def run_one(env: dict[str, Any], arm: str, ref_idx: int, bundle: Path, out: Path) -> dict[str, Any]: - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer diff --git a/packages/flow/benchmark/openemr_api_tier/spike/bench.py b/packages/flow/benchmark/openemr_api_tier/spike/bench.py index d3a74c2f53..138570dcca 100644 --- a/packages/flow/benchmark/openemr_api_tier/spike/bench.py +++ b/packages/flow/benchmark/openemr_api_tier/spike/bench.py @@ -62,7 +62,7 @@ def _refuse(self, *_a: Any, **_k: Any) -> Any: class Env: def __init__(self, browser: bool) -> None: - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer diff --git a/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py b/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py index 12929d4e07..5e6297d2ca 100644 --- a/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py +++ b/packages/flow/benchmark/openemr_api_tier/spike/capture_har.py @@ -39,7 +39,7 @@ def main() -> None: from playwright.sync_api import sync_playwright - from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from benchmark.harness.openemr_local import OpenEMRPatientOracle from openadapt_flow.ir import Workflow from openadapt_flow.runtime import Replayer diff --git a/packages/flow/benchmark/openemr_e2e/README.md b/packages/flow/benchmark/openemr_e2e/README.md index 39fdb23bbe..68471ac1f1 100644 --- a/packages/flow/benchmark/openemr_e2e/README.md +++ b/packages/flow/benchmark/openemr_e2e/README.md @@ -10,7 +10,7 @@ compile → replay → effect-verify (system of record) → catch a silent wrong ``` It does **not** duplicate the existing OpenEMR *benchmark* -(`openadapt_flow.benchmark.openemr_benchmark`, which measures the compiled arm +(`benchmark.harness.openemr_benchmark`, which measures the compiled arm vs. a paid agent arm on the **live** public demo and is explicitly *not* CI-reproducible). This harness *orchestrates* the runtime components that benchmark assumes work — the compiler, the `Replayer`, the `EffectVerifier`, the diff --git a/packages/flow/benchmark/openemr_e2e/harness.py b/packages/flow/benchmark/openemr_e2e/harness.py index 23a10fc498..5110b31c77 100644 --- a/packages/flow/benchmark/openemr_e2e/harness.py +++ b/packages/flow/benchmark/openemr_e2e/harness.py @@ -8,7 +8,7 @@ inject drift -> HALT -> teach the fix -> re-run clean Unlike the existing OpenEMR *benchmark* -(:mod:`openadapt_flow.benchmark.openemr_benchmark`), which measures the compiled +(:mod:`benchmark.harness.openemr_benchmark`), which measures the compiled arm vs. a paid agent arm on the LIVE public demo and is explicitly NOT CI-reproducible, this harness proves the pipeline *wiring* end to end on a deterministic, offline fixture (the in-process MockMed ``fault_server`` as the diff --git a/packages/flow/benchmark/openemr_local/README.md b/packages/flow/benchmark/openemr_local/README.md index 6a9441c40a..19efb71a28 100644 --- a/packages/flow/benchmark/openemr_local/README.md +++ b/packages/flow/benchmark/openemr_local/README.md @@ -33,7 +33,7 @@ publication bar. Raw per-run rows, environment fingerprints, application-specific adapter/oracle wiring, and the detailed cost ledger are retained in the private `OpenAdaptAI/openadapt-corpus` repository. The public generic cost-capped agent mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. See the +`benchmark/harness/agent_baseline.py`. See the [aggregate report](../agent_arm_verticals/README.md). The accepted run used baseline SHA-256 diff --git a/packages/flow/benchmark/openemr_throughput/harness.py b/packages/flow/benchmark/openemr_throughput/harness.py index 4bcd15079b..fcf9a2e817 100644 --- a/packages/flow/benchmark/openemr_throughput/harness.py +++ b/packages/flow/benchmark/openemr_throughput/harness.py @@ -237,7 +237,7 @@ def _demo_module() -> Any: def build_worker(user: str, opts: dict[str, Any]) -> Any: """One worker signed in as ``user``. Module-level so a child process can build it (``ProcessWorker``); ``opts`` holds only plain values.""" - from openadapt_flow.benchmark.openemr_local import ( + from benchmark.harness.openemr_local import ( OpenEMRPatientOracle, SyntheticPatientSpec, ) @@ -345,7 +345,7 @@ def run_item(page: Any, idx: int, row: dict[str, Any], item_dir: Path, pipeline: def run(a: argparse.Namespace) -> dict[str, Any]: from openadapt_flow._browser_setup import ensure_chromium_installed - from openadapt_flow.benchmark.openemr_local import ( + from benchmark.harness.openemr_local import ( SyntheticPatientSpec, ) from openadapt_flow.ir import Workflow diff --git a/packages/flow/benchmark/openimis_claims/README.md b/packages/flow/benchmark/openimis_claims/README.md index 94e94ad017..e01f263e40 100644 --- a/packages/flow/benchmark/openimis_claims/README.md +++ b/packages/flow/benchmark/openimis_claims/README.md @@ -28,7 +28,7 @@ Raw per-run rows, environment fingerprints, application-specific adapter/oracle wiring, and the detailed cost ledger are retained in the private `OpenAdaptAI/openadapt-corpus` repository. The public generic cost-capped agent mechanism remains in -`openadapt_flow/benchmark/agent_baseline.py`. See the +`benchmark/harness/agent_baseline.py`. See the [aggregate report](../agent_arm_verticals/README.md). There is still no full timing matrix and no publication protocol here: a diff --git a/packages/flow/benchmark/scoreboard/tasks/openemr.py b/packages/flow/benchmark/scoreboard/tasks/openemr.py index c2ffaf02d6..e93b7eab6d 100644 --- a/packages/flow/benchmark/scoreboard/tasks/openemr.py +++ b/packages/flow/benchmark/scoreboard/tasks/openemr.py @@ -207,7 +207,7 @@ def check(values: dict[str, str], before_ids: set[str]) -> str: def oracle_verifier(values: dict[str, str]) -> Any: - from openadapt_flow.benchmark.openemr_local import ( + from benchmark.harness.openemr_local import ( OpenEMRPatientOracle, SyntheticPatientSpec, ) diff --git a/packages/flow/benchmark/silent_saves/metrics.py b/packages/flow/benchmark/silent_saves/metrics.py index cb7f3ee9f1..2a8fa76483 100644 --- a/packages/flow/benchmark/silent_saves/metrics.py +++ b/packages/flow/benchmark/silent_saves/metrics.py @@ -6,7 +6,7 @@ import statistics from typing import Any, Iterable, Optional -from openadapt_flow.benchmark.effectbench.metrics import wilson_interval +from benchmark.harness.effectbench.metrics import wilson_interval def rate(k: int, n: int) -> dict[str, Any]: diff --git a/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md b/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md index c09304b4b5..ce1dfe56c9 100644 --- a/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md +++ b/packages/flow/benchmark/silent_wrong_action/SILENT_WRONG_ACTION.md @@ -66,7 +66,7 @@ run-to-run disagreement); N proves reproducibility. ## Reproduce ``` -.venv/bin/python -m openadapt_flow.benchmark.silent_wrong_action \ +.venv/bin/python -m benchmark.harness.silent_wrong_action \ --out benchmark/silent_wrong_action --n 10 ``` diff --git a/packages/flow/benchmark/vision_hardening/README.md b/packages/flow/benchmark/vision_hardening/README.md index 273c387138..46644fdc38 100644 --- a/packages/flow/benchmark/vision_hardening/README.md +++ b/packages/flow/benchmark/vision_hardening/README.md @@ -36,7 +36,7 @@ and CLASSIFIES what the **real** resolver does on each generated case: go DOWN. A hardening fix regenerates the corpus and lowers the ratchet. The headline metric is **SWER** (Silent Wrong-Effect/Resolution Rate), the same -taxonomy as EffectBench (`openadapt_flow/benchmark/effectbench/`) — this sweep +taxonomy as EffectBench (`benchmark/harness/effectbench/`) — this sweep is the *resolution-tier* feeder for that benchmark. ## The two tiers diff --git a/packages/flow/docs/EFFECT_KIT.md b/packages/flow/docs/EFFECT_KIT.md index 3739752c74..cb02250ed9 100644 --- a/packages/flow/docs/EFFECT_KIT.md +++ b/packages/flow/docs/EFFECT_KIT.md @@ -1420,7 +1420,7 @@ Runs against the pinned reference fixture README documents the pinned compose bring-up and fixture bootstrap). The bundle's effect contracts are exactly the ones the reference matrix ships -(`openadapt_flow/benchmark/frappe_lending.py::loan_application_effects`): +(`benchmark/harness/frappe_lending.py::loan_application_effects`): one `record_written` (at-most-once for the synthetic applicant) plus a `field_equals` read-back per entered field, each bound to `{param: ...}`. diff --git a/packages/flow/docs/GENERALITY_AUDIT.md b/packages/flow/docs/GENERALITY_AUDIT.md new file mode 100644 index 0000000000..0bd8ec4a80 --- /dev/null +++ b/packages/flow/docs/GENERALITY_AUDIT.md @@ -0,0 +1,312 @@ +# Generality audit of the engine + +Audited 2026-11-03 on `main` at `213f3d9fe`. Scope: everything under +`packages/flow/openadapt_flow/`. + +The rule: the engine has no code written for one application. What one +application needs lives in data (`app_profiles/*.yaml`, generated contracts, +deployment configs) or in test fixtures and benchmark harnesses outside the +shipped package. The CI check that keeps it that way, +`scripts/check_engine_generality.py`, lands in the pull request that follows +this audit, together with the rule in [CONTRIBUTING.md](../CONTRIBUTING.md). + +## Method + +1. Searched for application names, case-insensitive: OpenEMR, Medplum, + Dolibarr, EspoCRM, Redmine, Snipe-IT, InvenTree, Kimai, OpenMRS, Gitea, + Firefly III, Vikunja, Zammad, Open Dental, Odoo, openIMIS, Frappe/ERPNext, + Patient Notes, Epic, Stedi, TextEdit, Notepad, and OpenAdapt's own demo apps + MockMed and MockLoan. +2. Searched for identifiers that only make sense for one application: table and + column names (`pnotes`, `patient_data`, `uuid_registry`, `user_settings`, + `tabLoan Application`), URL paths (`/api/db`, `/api/encounter`, + `/apis/default/fhir`), form and query parameters (`token_main`, + `csrf_token_form`, `clearpass`), selectors, button labels, dialog titles, + field names, and thresholds tuned on one application's screens. +3. Read the code around each hit and put it in one of four buckets: + - **(a)** a comment or docstring example. Fine; reworded where it read as a + rule. + - **(b)** application knowledge stored as data and loaded generically. + - **(c)** a test fixture, benchmark harness or demo application inside the + shipped package. Moved out, unless it backs a user-facing command that + must work from a wheel. + - **(d)** special-casing in engine logic. Generalized, or turned into data. + +## Counts + +| Bucket | Rows | Distinct findings | +| --- | --- | --- | +| (a) comment or docstring example | 118 | — | +| (b) data loaded generically | 4 | 2 (`app_profiles/openemr.yaml`, the `clinical-write` policy pack as a default) | +| (c) fixture, harness or demo in the package | 35 rows, plus the 25 Python files of `benchmark/` and 2 of `instrument/` | 19 | +| (d) special-casing in engine logic | 27 | 15 | + +Rows group consecutive lines of one hit. The full table is at the end. + +## (d) findings and actions + +Each finding names the application that motivated it. "Second app" is the +existing fixture that shows the generalized code still works beyond the +motivating application. + +| # | Where | What | Motivating app | Action | Status | +| --- | --- | --- | --- | --- | --- | +| D1 | `report_redaction.py:68-70`, `:131-133` | Exact secret names `token_main`, `csrf_token_form` | OpenEMR | Match secret names by pattern (`token`, `csrf`, `xsrf`, `session`, ...) instead of a per-app list | planned | +| D2 | `network_capture.py:78` | `clearpass` in the secret-name pattern | OpenEMR | Drop it; the `pass\b` alternative already matches | planned | +| D3 | `runtime/effects/rest.py:114-115`, `deployment.py:387-390`, `reward/models.py:226`, `fault_gate.py:722` | REST reader defaults `records_path="/api/db"`, `records_key="records"` | MockMed | Required when the effect kind is `rest`; refused at config load when missing | planned | +| D4 | `runtime/effects/compensation.py:364` | `delete_path_template="/api/encounter/{id}"` default | MockMed | Required | planned | +| D5 | `compiler/binding_admission.py:114-119`, `:71-73` | Admission write path defaults to `/api/encounter`; body skips MockMed's `source` field | MockMed | Write path and method from the observed `api_write` event; skip only server-assigned fields | planned | +| D6 | `compiler/effect_mining.py:100-102`, `runtime/effects/effect.py:900-901` | Idempotency key field defaults to `key` | MockMed | No default; from the contract or deployment config | planned | +| D7 | `runtime/effects/fhir.py:59-67`, `:221`, `deployment.py:448-449` | FHIR defaults: resource type `Observation`, `note -> valueString` | OpenEMR | Required in the contract (`generate/ensure_contract.py` writes them) | planned | +| D8 | `generate/ensure_contract.py:123-147` | Translates the Medplum probe script's output keys | Medplum | The probe emits the generic fact names; translation removed | planned | +| D9 | `runtime/effects/frame.py:1937`, `rollback_state.py:781` | `"0"` means "no record" | OpenEMR | The value moves into the profile rule (`no_record_values`) | planned | +| D10 | `__main__.py:2838-2934`, `:9305-9319`; `cli_variants.py:131,173-182` | `replay` and the variant probe fall back to the bundled MockMed when no URL is given; `--drift` turns off structural grounding | MockMed | Target URL required outside the demo commands | planned | +| D11 | `qualification_oracle_gate.py` | `qualify accept` tests every workflow's oracle against MockMed (banner text, `/api/encounter`, `/api/db`) | MockMed | Build the fault fixture from the bundle's own effect binding | planned | +| D12 | `execute/dispatch.py:111-188` | The Execute service's live runner always runs the MockMed tutorial workflow | MockMed | Replay through the admission's deployment config; MockMed only for synthetic admissions | planned | +| D13 | `compiler/compile.py:148`, `runtime/replayer.py:394`, `runtime/identity.py:108` | Thresholds measured on MockMed only | MockMed | Generic logic, one-app numbers. Recalibrate on two or more apps | follow-up issue | +| D14 | `runtime/identity.py:275-302` | Hand-written list of non-identifying words drawn from EMR list screens | MockMed, EMRs | Derive from the recording (tokens shared by sibling rows) | follow-up issue | +| D15 | `eligibility/client.py:37-41`, `:1091-1110` | One vendor's eligibility API (URL, key name, body) in the engine | Stedi | Move behind a connector interface | follow-up issue | + +## (c) findings and actions + +| Where | What | Action | Status | +| --- | --- | --- | --- | +| `benchmark/` (25 Python files: `run_benchmark`, `openemr_benchmark`, `openemr_local`, `frappe_lending`, `hybrid_benchmark`, `dom_arm`, `silent_wrong_action`, `desktop_benchmark`, `verify`, `agent_baseline`, `reliability`, `chart_fonts`, `effectbench/`) | Benchmark harnesses shipped in the wheel | Moved to `packages/flow/benchmark/harness/` (import `benchmark.harness.*`). `openadapt-flow benchmark` loads it from a source checkout and says so from a wheel | done | +| `instrument/` | Competitor-drift harness; imported the benchmark at module load, so it was already broken in a wheel | Moved to `packages/flow/benchmark/harness/instrument/` | done | +| `integrations/harness2/` | Harness2 research harness on MockMed | Move to `packages/flow/benchmark/harness/harness2/` | planned | +| `mockloan/`, `demo_driver.record_disbursement_demo` | Lending demo app no product command uses | Move to the benchmark tree | planned | +| `guard_demo.py`, `agent/ledger_demo.py` | `python -m` demos, no CLI command | Move to the benchmark tree | planned | +| `mockmed/teach_scope.py` | Used only by tests | Move to `tests/fixtures` | planned | +| `learning/synth_stream.py:1-245` | MockMed note stream | Split out the generic `StructuralDiffInducer`, then move the stream | planned | +| `reward/seed.py:1-236` | MockMed reward seed | Split out the generic bundle helpers (`:238-410`), then move the seed behind the demo flag | planned | +| `record_check_proof.py:60-124` | `MOCKMED` fixture and `if name == "mockmed"` | Fixtures registered by name from the demo app's own module | planned | +| `validation/dense_surface.py` | Study harness rendering a MockMed table | Move to `benchmark/dense_surface` | planned | +| `__main__.py:5246-5275` (`bench`) | Always replays against MockMed | Accept `--url`, or load from a checkout like `benchmark` | planned | +| `execute/registry.py`, `execute/service.py`, `execute/app.py`, `execute/dispatch.py:60-108`, `--seed-mockmed` | Demo seeding inside the Execute service | Generic `seed_admissions`; the CLI passes the demo seed | planned | +| `eligibility/payer_routes.yaml`, `fleet/simulator.py`, `repair/dialog_fixture.py` | Synthetic test data and simulators in the package | Move to the benchmark tree or `tests/fixtures` | planned | + +### Kept in the package on purpose + +These back user-facing commands that must work from `pip install openadapt` +(`openadapt quickstart`, `openadapt-flow tutorial`, `demo`, `demo-record`, +`openadapt-agent serve` with no flags, `openadapt-agent example-app`). They are +OpenAdapt's own synthetic demo applications, not customer applications, and the +engine does not branch on them: + +- `mockmed/` (server, fault server, chart pages, static UI) +- `tutorial.py`, `demo_proof.py`, `demo_driver.record_triage_demo` +- `agent/sandbox.py`, `agent/tutorial.py`, `agent/example_app.py` + +The generality check allows MockMed names only in these modules. + +## (b) data + +- `app_profiles/openemr.yaml`: OpenEMR's bookkeeping writes, loaded by + `app_profiles.load_profile(name)`, which has no per-app code. +- The `clinical-write` policy pack is the default policy in + `execute/models.py:74`, `cli_config.py:164`, `integrations/agents/skills.py:351` + and `agent/teach.py:121`. It is data, but a healthcare default; whether to make + the policy required is a product decision, noted here. + +## (a) rewording done + +- `deployment.py:548`: the example environment variable in a user-facing + error is now `APP_READER_PASSWORD`. +- `authoring.py:807`: the comment on a generic branch no longer names MockMed. + +The remaining (a) rows are examples in comments and docstrings. The code they +describe is generic and configured per contract or profile. Healthcare words +such as MRN, DOB and patient appear in generic heuristics (for example +`risk.py:370`); that is the product's domain, not one application. + +## Full table + +Paths are relative to `packages/flow/openadapt_flow/` as of the audited commit. + +| Location | Bucket | App | Note and action | +| --- | --- | --- | --- | +| `__main__.py:2838-2934` | (d) | MockMed | replay: general code path defaults to serving bundled MockMed when no --url/config URL, and --drift flips use_structural=False (grounding ladder changes on a demo flag). Make --url required for replay (or route no-URL to `tutorial`/`demo`), rename --drift behavior to a generic --force-visual-grounding, keep drift injection in the demo command. | +| `__main__.py:9305-9319` | (d) | MockMed | replay parser: --url default = bundled MockMed; --drift MockMed modes (see 2838-2934). | +| `cli_variants.py:131,173-182` | (d) | MockMed | first-screen probe serves bundled MockMed when no --url (same default as replay). Refuse instead (first_screen_unavailable). | +| `compiler/binding_admission.py:114` | (d) | MockMed | AdmissionFixture.write_path default '/api/encounter' (line 119); compile.py:3379 never overrides it | +| `compiler/binding_admission.py:71` | (d) | MockMed | _BODY_SKIP_FIELDS adds MockMed's 'source' field (line 73) | +| `compiler/compile.py:146` | (d) | MockMed | REGION_STABLE_TOLERANCE=16 (line 151) calibrated on MockMed only; recalibrate on >=2 apps | +| `compiler/effect_mining.py:100` | (d) | MockMed | IDEMPOTENCY_KEY_FIELD='key' (line 102) is MockMed's field; take from contract/profile | +| `deployment.py:387` | (d) | MockMed | records_path default '/api/db' + records_key 'records' (lines 389-390) are MockMed's shape; make required for kind: rest | +| `deployment.py:448` | (d) | OpenEMR | fhir section default resource_type='Observation' (line 449) mirrors OpenEMR note-as-Observation; make required | +| `eligibility/client.py:37-41` | (d) | Stedi | Stedi 270/271 vendor client in engine (hard-coded URL, STEDI_API_KEY env, to_stedi_body); move to integrations/connector | +| `execute/dispatch.py:111-188` | (d) | MockMed | live_replay (the only real Execute runner) always calls tutorial.run_tutorial_workflow: POSTs /api/reset to target_url, appends MockMed ?fault=ok&idempotency=demo#tasks, verifies via /api/db 'records', forces clinical-write policy and 'note' param; with no target_url serves MockMed. Ignores AdmittedBundle.policy. Generalize: replay via the standard replay path with the admission's deployment.yaml (effects/backend/policy); keep MockMed only behind synthetic/demo admissions. | +| `fault_gate.py:722` | (d) | MockMed | rest_memory fault case default records_path '/api/db' (test harness; low) | +| `generate/ensure_contract.py:123` | (d) | Medplum | facts_from_probe translates Medplum probe key names (133-147); make the probe emit generic FACTS | +| `network_capture.py:78` | (d) | OpenEMR | SECRET_NAME regex lists 'clearpass' (OpenEMR login field); redundant with pass\b alternative -> drop | +| `qualification_oracle_gate.py:5,36,70,225,325-455` | (d) | MockMed | evaluate_oracle_gate (called by qualification_proposal.py:221,372 for every `qualify accept`) runs ANY workflow's proposed oracle against MockMed: hard-coded banner 'Encounter saved — ', POST /api/encounter?fault=optimistic with {patient_id, type:'Triage', note}, /api/db verifier. Generalize: a generic lying-store fixture driven by the bundle's own effect binding (write path/body/records path from the effect), or fixtures registered as data. | +| `report_redaction.py:131-133` | (d) | OpenEMR | _SECRET_KEYS lists token_main/csrf_token_form; same fix as DEFAULT_PARAMS | +| `report_redaction.py:68` | (d) | OpenEMR | token_main/csrf_token_form in DEFAULT_PARAMS (with :131-133 _SECRET_KEYS): replace with generic token/csrf name regex (reuse network_capture.TOKEN_NAME) or per-app secret_params in app profile | +| `reward/models.py:226` | (d) | MockMed | RecordReadRecipe.records_path default '/api/db' (MockMed's endpoint) in a general model; make it required for kind=rest. | +| `rollback_state.py:781` | (d) | OpenEMR | same '0' no-record sentinel as frame.py:1937 | +| `runtime/effects/compensation.py:364` | (d) | MockMed | RestCompensator delete_path_template default '/api/encounter/{id}'; make required | +| `runtime/effects/effect.py:900` | (d) | MockMed | Effect.key_field default 'key' (line 901) is MockMed's field | +| `runtime/effects/fhir.py:59` | (d) | OpenEMR | DEFAULT_OBSERVATION_PATHS (61-67) maps note->valueString per OpenEMR; require field_paths from contract | +| `runtime/effects/frame.py:1937` | (d) | OpenEMR | ref in (None,'','0',0): '0' = OpenEMR's 'no chart' (patient_id 0); gated by profile insert_entry_names_no_record; move sentinel into profile | +| `runtime/effects/rest.py:114-115` | (d) | MockMed | RestRecordVerifier defaults records_path='/api/db', records_key='records'; make required | +| `runtime/identity.py:104` | (d) | MockMed | COVERAGE_THRESHOLD=0.8 (line 108) calibrated on MockMed rows only | +| `runtime/identity.py:275-302` | (d) | MockMed/EMR list | _GENERIC_IDENTITY_WORDS hand-tuned EMR list vocabulary (incl. 'demo','chart','review','results'); derive from recording or app profile | +| `runtime/replayer.py:392` | (d) | MockMed | PC_STRUCTURAL_TEMPLATE_THRESHOLD=0.8 (line 394) calibrated on MockMed only | +| `__main__.py:10030-10037` | (c) | MockMed | bench parser. | +| `__main__.py:10166` | (c) | MockMed | verify-scaffold --prove choices ('mockmed','effect-e2e'): fixture names hard-coded in CLI; see record_check_proof.py. | +| `__main__.py:11940-11989,12341-12415` | (c) | MockMed | serve-execute/serve-reward --seed-mockmed flags and banner paths (data_dir/mockmed/records.db). Opt-in demo seeding; keep as demo flag or rename --seed-demo. | +| `__main__.py:1787-1806` | (c) | MockMed | demo-record command: serves MockMed, runs demo_driver.record_triage_demo. User-facing CLI; demo data could be a recorded script in data. | +| `__main__.py:1808-1998` | (c) | MockMed | tutorial command (via tutorial.py). Deliberate product feature; must work from wheel. | +| `__main__.py:1999-2040` | (c) | MockMed | demo command (demo_proof + tutorial). Deliberate product feature; must work from wheel. | +| `__main__.py:326-330` | (c) | MockMed | _with_drift: builds MockMed ?drift= query; only meaningful for the bundled demo. Move with demo helpers. | +| `__main__.py:5246-5275` | (c) | MockMed | bench command: always replays against bundled MockMed. Not generic; move to benchmark/ (source-checkout command, like `benchmark`) or accept --url. | +| `__main__.py:8773,8782` | (c) | MockMed | demo-record parser help/--drift. | +| `agent/example_app.py:1-400` | (c) | (own demo: reports portal) | `openadapt-agent example-app` local sales-order portal (/login clerk/portal-demo, /api/orders). User-facing demo command; importers agent/cli.py:1250 + 4 flow tests. | +| `agent/ledger_demo.py:1-720` | (c) | MockMed | scripted 30-referral worklist simulation (python -m). Not a CLI command; move to benchmark or packages/agent/tests fixtures; importer packages/agent/tests/test_ledger.py. | +| `agent/sandbox.py:1-530` | (c) | MockMed | `openadapt-agent serve` zero-flag sandbox: add_triage_note on MockMed, CASES with MockMed fault/drift queries, clinical-write. User-facing; must ship. Case table could be data (sandbox YAML). | +| `agent/tutorial.py:1-140` | (c) | MockMed | prepare_tutorial_session: hosts MockMed, writes deployment YAML with /api/db, clinical-write. Used by sandbox and agent/cli.py:760; packages/agent/tests/test_tutorial.py. | +| `demo_driver.py:1-96` | (c) | MockMed | record_triage_demo: MockMed selectors and creds (nurse.demo / mockmed-demo-pass). Used by demo-record CLI + benchmark (fault_model, undo_proposal, ui_readback, harness, cosmetic_drift), tests (recorder, e2e), scripts (demo_media, openemr_demo). Split: keep record_triage_demo with mockmed package. | +| `demo_driver.py:97-171` | (c) | MockLoan | record_disbursement_demo (mockloan-demo-pass): no product command uses it. Move to benchmark/lending_fault_model; importers: benchmark/undo_proposal/run.py, benchmark/lending_fault_model/{run,resolution_ladder}.py, tests/e2e/test_lending_fault_model.py. | +| `demo_proof.py:1-60,975-1110` | (c) | MockMed | `demo` page renderer; CROP 640x320 tuned to MockMed layout, 'Encounter saved', /api/db wording. User-facing; must ship. Crop could come from the demo-app spec. | +| `eligibility/payer_routes.yaml:1-` | (c) | Stedi/Cigna | synthetic TEST payer route data shipped in package | +| `execute/app.py:29,39,141,151` | (c) | MockMed | seed_mockmed passthrough; same as service. | +| `execute/dispatch.py:60-108` | (c) | MockMed | default_runner/synthetic_mockmed: synthetic admissions project canned MockMed tutorial outcomes (evidence_tag mockmed-*). Rename to synthetic_runner and move the canned projection to tests/fixtures or keep behind --seed-mockmed only. | +| `execute/registry.py:10-16,81-106` | (c) | MockMed | MOCKMED_* constants + seed_mockmed_admissions in engine registry. Move to execute/demo_seed.py (or tests/fixtures); importers: __main__ serve-execute, tests test_execute_ref/test_execute_post_dispatch_faults/test_jit_draft_execution/test_backend_parity_refusals. | +| `execute/service.py:45,74,84-85` | (c) | MockMed | ExecuteService(seed_mockmed=) flag. Replace with generic seed_admissions: Iterable[AdmittedBundle] and let the CLI pass the demo seed. | +| `fleet/simulator.py:1-` | (c) | synthetic payer portal | in-package simulator fixture; move to benchmark/examples | +| `guard_demo.py:1-296` | (c) | MockMed | `python -m openadapt_flow.guard_demo`: fake agent posts referral to MockMed with chart_pages READER creds; not a CLI subcommand. Move to packages/flow/examples or benchmark; importers: packages/agent/tests (1), packages/flow/tests (6). | +| `instrument/ (__init__.py, competitor_drift.py)` | (c) | MockMed | MockMed competitor-drift harness. Moved to packages/flow/benchmark/harness/instrument. | +| `integrations/harness2/ (all: tasks.py, apps.py, agent.py, adapter.py, traces.py, proposers.py, scorer.py, llm.py, pixels.py, rollout.py, selection.py, __init__.py)` | (c) | MockMed | Whole subpackage is a research harness for Harness2 on MockMed: MOCKMED_PATIENT/PATIENTS (tasks.py:71-148), MockMedTarget (apps.py:50-105, /api/db at :70), regex for MockMed task text and 'triage'/'New Encounter'/'encountersaved' (agent.py:221-324), MockMed banner regex Name — MRN — DOB (traces.py:55-57,128-138), clinical prompts (adapter.py:93-120,429-445; proposers.py:50-63). Move to packages/flow/benchmark/harness/harness2; importers: benchmark/harness2_scorer/run.py, tests/test_harness2_{adapter,pixels,traces}.py, pyproject extra 'harness2', .github/workflows/packages.yml:451. | +| `learning/synth_stream.py:1-245` | (c) | MockMed | synthetic MockMed add-patient-note stream (intents, 'Consent Required', 'Belford, Phil DOB'). Move to benchmark/ or tests/fixtures. | +| `learning/synth_stream.py:246-525` | (c) | (generic) | StructuralDiffInducer is generic but lives in the MockMed harness module; imported by learning/teach.py:96, learning/branch_teach.py:76, benchmark/openemr_e2e/harness.py, 4 tests; pyproject mypy list (line 611). Move to learning/induce.py first, then move the stream out. | +| `mockloan/ (server.py, fault_server.py, static/)` | (c) | MockLoan | Own synthetic lending app shipped in wheel but used by no product command; only demo_driver.record_disbursement_demo, benchmark/lending_fault_model, benchmark/undo_proposal, tests (test_mockloan, test_lending_swer, e2e/test_lending_fault_model), scripts (1). Move to packages/flow/benchmark/apps/mockloan. | +| `mockmed/ (server.py, fault_server.py, chart_pages.py, teach_scope.py, static/)` | (c) | MockMed | Own synthetic demo app shipped in wheel. Required by tutorial/demo/demo-record/replay-default/sandbox and root openadapt/quickstart (engine.py, workers.py) + root tests/test_partner_kit.py. Keep shipping (or split to a demo-apps package). mockmed/teach_scope.py (EncounterWrites) is used only by tests -> move to tests/fixtures. chart_pages used by guard_demo + benchmark (agent_guard, effect_e2e, ui_readback). | +| `record_check_proof.py:1-11,60-124` | (c) | MockMed | --prove fixture registry hard-codes MOCKMED Fixture (judge on patient_id/note, write_path /api/encounter, body type Triage) and `if name == 'mockmed'`. Opt-in proof against our fixture. Move MOCKMED Fixture to mockmed/proof_fixture.py and resolve fixtures by entry point / dotted path. Importers: scaffold_verifier.py:594, benchmark/effect_e2e, tests (2). | +| `repair/dialog_fixture.py:1-` | (c) | synthetic | rendered test app in product package (no real app) | +| `reward/seed.py:1-236` | (c) | MockMed | seed_mockmed, MockMed encounters table/SQL (MOCKMED_TABLE 'encounters', SELECT id, patient_id, type, status), patients, episodes. Move to reward/demo_seed or tests/fixtures; CLI serve-reward --seed-mockmed uses it. | +| `reward/seed.py:238-410` | (c) | (generic) | write_bundle/calibrate_bundle/self_signed_certificate are generic helpers living in the MockMed seed module; split into reward/bundle.py. Importers: integrations/harness2/scorer.py, benchmark/undo_proposal/run.py, tests/test_probe_induction.py, test_undo_proposal.py, test_reward_worker.py, test_reward_trust_boundary.py. | +| `tutorial.py:1-560,630-841` | (c) | MockMed | tutorial engine: MockMed selectors (.open-btn,#new-encounter,#type-triage,#note,#save-encounter,#saved-banner), /api/reset, /api/db, ?fault=ok entry, TUTORIAL_POLICY clinical-write. User-facing (tutorial, demo, quickstart in root openadapt/quickstart/engine.py imports _records, run_tutorial_workflow, record_tutorial; agent/tutorial.py; execute/dispatch.py). Must ship in wheel; could be data-driven (demo-app spec YAML: entry query, reset/records endpoints, recorded click script). | +| `validation/dense_surface.py:418` | (c) | MockMed | validation study renders MockMed-branded table inside product package; move to benchmark/dense_surface | +| `__main__.py:5422,9499,10873` | (b) | (domain) | clinical-write as default/suggested policy name (built-in policy pack, data). | +| `app_profiles/openemr.yaml:1-` | (b) | OpenEMR | per-app bookkeeping profile loaded by load_profile(name); only profile shipped | +| `cli_config.py:164; integrations/agents/skills.py:351; agent/teach.py:121` | (b) | (domain) | default policy 'clinical-write' (shipped policy pack data). | +| `execute/models.py:74` | (b) | (domain) | AdmittedBundle.policy default 'clinical-write' (healthcare policy pack); consider requiring it. | +| `__main__.py:10053` | (a) | MockMed | benchmark help text (benchmark already moved to source-checkout harness). | +| `__main__.py:10536,10577` | (a) | MockMed | --admit-local help mentions MockMed; code path is generic. | +| `__main__.py:1149,1257-1266,1288` | (a) | MockMed | docstrings / refusal text about --drift being MockMed-only. | +| `__main__.py:3819` | (a) | MockMed | docstring: --drift forced off. | +| `__main__.py:7994` | (a) | TextEdit | help example for --backend macos owner app. | +| `__main__.py:9,27,39` | (a) | MockMed | module docstring command list. | +| `agent/cards.py:9-14; agent/skill.py:145-147; agent/bridge.py:132-133,325-326; agent/ledger.py:3,378; agent/delegated.py:11,78; agent/mailbox.py:130; agent/teach.py:281-283` | (a) | (domain) | referral / EMR / MRN examples in docstrings and tool descriptions; fine. | +| `api.py:84` | (a) | MockMed | docstring: replay without url serves MockMed (changes if replay default changes). | +| `app_profiles/__init__.py:315` | (a) | OpenEMR | comment/docstring example | +| `app_profiles/__init__.py:58` | (a) | OpenEMR | comment/docstring example | +| `authoring.py:807` | (a) | MockMed | comment 'Non-secret MockMed note' on a generic branch; reword to 'non-secret field'. | +| `backends/factory.py:117` | (a) | TextEdit | comment/docstring example | +| `backends/parallels_vm.py:403` | (a) | OpenDental | comment/docstring example | +| `backends/playwright_backend.py:890` | (a) | OpenEMR | comment/docstring example | +| `backends/win_agent/server.py:1578` | (a) | Notepad | comment/docstring example | +| `cli_api.py:71` | (a) | OpenEMR | help example 'lname'/'mrn' column names. | +| `cli_undo.py:181` | (a) | MockMed | help says default target is MockMed but _cmd_undo_run refuses that case; reword help. | +| `cli_undo.py:538-541` | (a) | MockMed | comment explaining refusal. | +| `compile_agent/code_action.py:135` | (a) | OpenEMR | comment/docstring example | +| `compiler/binding_admission.py:19` | (a) | MockMed | comment/docstring example | +| `compiler/compile.py:193` | (a) | OpenEMR | comment/docstring example | +| `compiler/compile.py:2184` | (a) | MockMed | comment/docstring example | +| `compiler/compile.py:717` | (a) | MockMed | comment/docstring example | +| `compiler/fixup_passes.py:7` | (a) | OpenEMR | history note; passes are generic | +| `compiler/local_edit.py:321` | (a) | MockMed | comment/docstring example | +| `deployment.py:1035` | (a) | OpenEMR | comment/docstring example | +| `deployment.py:112` | (a) | MockMed | comment/docstring example | +| `deployment.py:144` | (a) | TextEdit | comment/docstring example | +| `deployment.py:548` | (a) | OpenEMR | user-facing error example OPENEMR_READER_PASSWORD; reword to neutral APP_READER_PASSWORD | +| `ensure_record.py:114` | (a) | Medplum | comment/docstring example | +| `ensure_record.py:162` | (a) | Medplum | comment/docstring example | +| `ensure_record.py:97` | (a) | Medplum | comment/docstring example | +| `execute/dispatch.py:1-6` | (a) | MockMed | module docstring; update after generalizing live_replay. | +| `execution_profiles.py:2595` | (a) | MockMed | comment/docstring example | +| `fault_gate.py:822` | (a) | OpenEMR | comment/docstring example | +| `generate/demo_analysis.py:781` | (a) | Redmine | comment/docstring example | +| `generate/ensure_contract.py:4` | (a) | Medplum | comment/docstring example | +| `generate/verify_contract.py:1234` | (a) | Redmine | comment/docstring example | +| `generate/verify_contract.py:1275` | (a) | EspoCRM | comment/docstring example | +| `generate/verify_contract.py:1276` | (a) | Dolibarr | comment/docstring example | +| `generate/verify_contract.py:225` | (a) | Redmine | comment/docstring example | +| `generate/verify_contract.py:566` | (a) | EspoCRM | comment/docstring example | +| `integrations/agents/__init__.py:79` | (a) | (domain) | 'a clinic, a tenant' example. | +| `jit/authority.py:463` | (a) | TextEdit | comment/docstring example | +| `jit/authority.py:623` | (a) | TextEdit | comment/docstring example | +| `jit/compile_agent.py:354` | (a) | TextEdit | comment/docstring example | +| `jit/recorded.py:6` | (a) | MockMed | docstring example task id | +| `jit/signature.py:72` | (a) | MockMed | comment/docstring example | +| `learning/__init__.py:20; learning/halt_loop.py:8; learning/trace.py:60; learning/branch_teach.py:6,108; learning/library.py` | (a) | MockMed | docstring references. | +| `operator_profile.py:79` | (a) | OpenEMR | comment/docstring example | +| `policy_packs.py:41` | (a) | MockMed | user-facing pack summary names MockMed; fine as the demo app | +| `qualification_dev_signer.py:3` | (a) | MockMed | comment/docstring example | +| `qualification_proposal.py:10` | (a) | MockMed | comment/docstring example | +| `qualification_web_session.py:3` | (a) | OpenEMR,openIMIS | comment/docstring example | +| `qualify/config.py:303` | (a) | OpenEMR | comment/docstring example | +| `record_check_binding.py:48` | (a) | openIMIS | comment/docstring example | +| `record_check_identity.py:4` | (a) | OpenEMR | comment/docstring example | +| `record_check_synthesis.py:25` | (a) | MockMed | comment/docstring example | +| `registry.py:3` | (a) | OpenEMR | comment/docstring example | +| `registry.py:813` | (a) | OpenEMR | comment/docstring example | +| `report_redaction.py:5` | (a) | OpenEMR | comment/docstring example | +| `reward/models.py:497; reward/calibration.py:13` | (a) | (domain) | 'triage rows' wording in docstrings. | +| `risk.py:476` | (a) | openIMIS | comment/docstring example | +| `risk.py:565` | (a) | OpenEMR | comment/docstring example | +| `rollback_state.py:77` | (a) | OpenEMR | comment/docstring example | +| `runtime/actuators/web_form.py:214` | (a) | OpenEMR | comment/docstring example | +| `runtime/actuators/web_form.py:324` | (a) | validate.js | var constraints recognizer is a JS library convention (OpenEMR among users), not app-specific | +| `runtime/actuators/web_form.py:331` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/__init__.py:21` | (a) | MockMed | comment/docstring example | +| `runtime/effects/__init__.py:23` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/__init__.py:38` | (a) | Frappe | comment/docstring example | +| `runtime/effects/_common.py:143` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/auth.py:33` | (a) | Frappe | comment/docstring example | +| `runtime/effects/db_snapshot.py:713` | (a) | Dolibarr | comment/docstring example | +| `runtime/effects/effect.py:1536` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/effect.py:1541` | (a) | OpenEMR,MockMed | comment/docstring example | +| `runtime/effects/effect.py:206` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/effect.py:208` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/effect.py:223` | (a) | Redmine | comment/docstring example | +| `runtime/effects/effect.py:903` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:1` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:12` | (a) | MockMed | comment/docstring example | +| `runtime/effects/fhir.py:14` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:191` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:195` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:3` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:38` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:386` | (a) | Epic | comment/docstring example | +| `runtime/effects/fhir.py:39` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fhir.py:42` | (a) | MockMed | comment/docstring example | +| `runtime/effects/frame.py:135` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:144` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:1499` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:1630` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:1944` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:242` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/frame.py:46` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/fresh_session.py:24` | (a) | MockMed | comment/docstring example | +| `runtime/effects/rest.py:237` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/rest.py:5` | (a) | MockMed | comment/docstring example | +| `runtime/effects/sql.py:12` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:215` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:215` | (a) | Frappe | comment; _TABLE_NAME allows space generically | +| `runtime/effects/sql.py:3` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:365` | (a) | OpenEMR | comment/docstring example | +| `runtime/effects/sql.py:4` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:480` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:534` | (a) | Frappe | comment/docstring example | +| `runtime/effects/sql.py:535` | (a) | Frappe | comment/docstring example | +| `runtime/grounder.py:136-155` | (a) | healthcare domain | _INTENT_STOPWORDS include patient/mrn; domain wording, intent-only; low | +| `submit_binding.py:8` | (a) | openIMIS | comment/docstring example | +| `validation/dense_surface.py:1271` | (a) | OpenEMR | comment/docstring example | +| `validation/dense_surface.py:5` | (a) | OpenEMR | comment/docstring example | +| `verifier.py:2467` | (a) | OpenEMR | comment/docstring example | +| `vision/hashing.py:10` | (a) | MockMed | docstring for the MockMed-calibrated phash tolerance (see compile.py:148) | +| `volatility.py:100` | (a) | OpenEMR | comment/docstring example | +| `volatility.py:31` | (a) | OpenEMR | comment/docstring example | +| `volatility.py:6` | (a) | OpenEMR | comment/docstring example | +| `benchmark/` (25 files) | (c) | OpenEMR, Frappe, MockMed, Open Dental | Moved to `packages/flow/benchmark/harness/` | diff --git a/packages/flow/docs/design/EFFECT_VERIFIER.md b/packages/flow/docs/design/EFFECT_VERIFIER.md index b7364ebbcf..b54ec7cb59 100644 --- a/packages/flow/docs/design/EFFECT_VERIFIER.md +++ b/packages/flow/docs/design/EFFECT_VERIFIER.md @@ -80,7 +80,7 @@ the note-as-Observation write the fake models cannot be created over FHIR on a stock OpenEMR; the property the live test establishes is the one the fake could not — the verdicts are correct against a **real FHIR server**, not a fake. CI still runs vision-only against the **public demo** -(`openadapt_flow/benchmark/openemr_benchmark.py`, `scripts/openemr_demo.py`); +(`benchmark/harness/openemr_benchmark.py`, `scripts/openemr_demo.py`); the live FHIR test is gated behind `OPENEMR_FHIR_BASE_URL` and skipped otherwise. See `benchmark/openemr_live/README.md` for the one-command bring-up. diff --git a/packages/flow/docs/desktop/PHASE2.md b/packages/flow/docs/desktop/PHASE2.md index 6eb67329e8..16eadb2505 100644 --- a/packages/flow/docs/desktop/PHASE2.md +++ b/packages/flow/docs/desktop/PHASE2.md @@ -58,7 +58,7 @@ so `WindowsBackend` sees exactly what it drives). | `scripts/desktop/patient_notes.ps1` | WinForms target app (list-select → edit note → save); drift knobs via `pn_env.json` | | `scripts/desktop/pn_db.py` | SQLite ground-truth CLI (seed/list/save/get/all) | | `scripts/desktop/uia_arm.py` | UIA-selector arm (pywinauto), identity + positional | -| `openadapt_flow/benchmark/desktop_benchmark.py` | Orchestrator: 3 arms × drift matrix, DB judge, results.json/BENCHMARK.md/chart | +| `benchmark/harness/desktop_benchmark.py` | Orchestrator: 3 arms × drift matrix, DB judge, results.json/BENCHMARK.md/chart | ## Snapshots (reversibility + warm boot) @@ -78,7 +78,7 @@ state without a full VM revert. ```bash # one-off full matrix (6 conditions × 3 arms × n): -python -m openadapt_flow.benchmark.desktop_benchmark --out benchmark/desktop --n 3 +python -m benchmark.harness.desktop_benchmark --out benchmark/desktop --n 3 ``` `DesktopHarness.connect()` starts the VM, launches the session-1 shim, deploys diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index bb52425ba8..db183ff837 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -5337,8 +5337,32 @@ def _cmd_silent_saves(args: argparse.Namespace) -> int: return int(silent_saves_main(list(args.silent_saves_argv))) +def _checkout_harness_on_path(command: str) -> bool: + """Put the checkout's ``benchmark`` harness on ``sys.path``. + + The harness lives in ``packages/flow/benchmark/harness``, outside the + package, so a wheel install has none and the command says so. + """ + checkout = Path(__file__).resolve().parents[1] + if not (checkout / "benchmark" / "harness" / "__init__.py").is_file(): + print( + f"{command} runs from a source checkout of OpenAdapt " + "(packages/flow/benchmark/harness); the wheel does not ship it.", + file=sys.stderr, + ) + return False + if str(checkout) not in sys.path: + # Appended, so nothing in the checkout shadows an installed module. + sys.path.append(str(checkout)) + return True + + def _cmd_benchmark(args: argparse.Namespace) -> int: - from openadapt_flow.benchmark.run_benchmark import run_benchmark + if not _checkout_harness_on_path("benchmark"): + return 2 + from benchmark.harness.run_benchmark import ( # type: ignore[import-not-found] + run_benchmark, + ) results = run_benchmark( Path(args.out), diff --git a/packages/flow/openadapt_flow/anthropic_defaults.py b/packages/flow/openadapt_flow/anthropic_defaults.py index 22ec8ce45b..fc4201c29e 100644 --- a/packages/flow/openadapt_flow/anthropic_defaults.py +++ b/packages/flow/openadapt_flow/anthropic_defaults.py @@ -1,6 +1,6 @@ """Anthropic defaults that product code shares with the benchmark harness. -These live here, not in ``openadapt_flow.benchmark``, because the product +These live here, not in ``benchmark.harness`` (the checkout's harness), because the product imports them and a frozen build leaves the benchmark package out (``tests/test_runtime_never_imports_benchmark.py`` enforces the rule). """ diff --git a/packages/flow/openadapt_flow/authoring.py b/packages/flow/openadapt_flow/authoring.py index a1c4e0c2a1..24a3df3402 100644 --- a/packages/flow/openadapt_flow/authoring.py +++ b/packages/flow/openadapt_flow/authoring.py @@ -804,7 +804,7 @@ def continue_input(self, *, operator_confirmed: bool = True) -> dict[str, Any]: "reason": "unconfirmed_masked_secret", } if not pause.secret and (text is None or not text): - # Non-secret MockMed note: do not persist an empty TYPE/param. + # Non-secret field left empty: do not persist an empty TYPE/param. return { "recorded": False, "param": pause.param, diff --git a/packages/flow/openadapt_flow/deployment.py b/packages/flow/openadapt_flow/deployment.py index e8b54023bd..a5f3d698e3 100644 --- a/packages/flow/openadapt_flow/deployment.py +++ b/packages/flow/openadapt_flow/deployment.py @@ -545,7 +545,7 @@ def _validate_secret_env(cls, v: dict[str, str]) -> dict[str, str]: if not isinstance(env_name, str) or not _ENV_VAR_NAME.fullmatch(env_name): raise ValueError( f"effects.secret_env[{slot!r}] must be the NAME of an " - "environment variable (for example OPENEMR_READER_PASSWORD), " + "environment variable (for example APP_READER_PASSWORD), " "never the credential itself" ) return v diff --git a/packages/flow/paper/REVIEW_ADVERSARIAL.md b/packages/flow/paper/REVIEW_ADVERSARIAL.md index 4ebe33f3c6..9e31c59679 100644 --- a/packages/flow/paper/REVIEW_ADVERSARIAL.md +++ b/packages/flow/paper/REVIEW_ADVERSARIAL.md @@ -35,7 +35,7 @@ Every quantitative result is at a sample size where inference is not possible, a This is my single most serious concern, and it is not disclosed in the paper. -- The headline "screen 50/90, effect 0/90" comes from `openadapt_flow/benchmark/silent_wrong_action.py`, **not** from the browser study. That harness does not drive the browser, does not run OCR, and does not invoke the `Replayer`. It issues raw `requests.post` calls that replicate what the app's JavaScript would send, encodes the screen oracle as a hardcoded rule over HTTP status codes, and calls `RestRecordVerifier.verify(...)` directly. +- The headline "screen 50/90, effect 0/90" comes from `benchmark/harness/silent_wrong_action.py`, **not** from the browser study. That harness does not drive the browser, does not run OCR, and does not invoke the `Replayer`. It issues raw `requests.post` calls that replicate what the app's JavaScript would send, encodes the screen oracle as a hardcoded rule over HTTP status codes, and calls `RestRecordVerifier.verify(...)` directly. - In that harness the **effect oracle and the ground-truth judge read the same in-process store**: the effect verifier reads `GET /api/db` and the ground-truth `_business_effect` reads `FaultDB.snapshot()`, which is the same `FaultDB`. Worse, the effect contract (`record_written` exactly once AND `field_equals` on the note) is essentially a restatement of the ground-truth definition of "correct." So "effect silent-wrong = 0 of 90" is guaranteed by construction: the oracle and the ground truth are computed from the same bytes with near-identical logic. The number is a property of the fixture, not an empirical finding. - The genuinely end-to-end harness (`benchmark/fault_model/run.py`, real Playwright + OCR + `Replayer`) measures **only the screen-only baseline**. Its own recommendations section describes effect verification as a *proposed* fix that the reference postcondition system does not yet implement ("The reference postcondition system is vision-only today, so on this corpus it provides no transactional-safety coverage"). The effect-verifier-through-the-replayer path for these fault classes exists only in unit tests (`tests/test_effect_fault_matrix.py`), and those tests also call `verifier.verify(...)` directly rather than driving the `Replayer`. - Consequently the Results sentence "those five classes refuted and halted **through the live replayer**" overstates what was measured. The live replayer never ran the effect verifier on these 90 runs. The suspiciously clean integers (exactly 50, exactly 0, exactly 10) are deterministic-fixture artifacts, not a measured rate. @@ -48,7 +48,7 @@ The seven fault classes are hand-picked transactional hazards (partial save, dup ### 3.4 Baseline fairness: same-backend, but a vanilla single-shot agent -The agent baseline (`openadapt_flow/benchmark/agent_baseline.py`) is not a strawman in the crude sense: it drives the same Playwright backend, uses a current computer-use model and tool, and is handed generous task information (credentials, exact patient, exact note text, a hint that the app is slow). But it is a **single-attempt, un-scaffolded** loop: no system prompt, no planning/reflection/ReAct, no whole-task retry, and hard caps (25 actions, $1.50, last-3-screenshots context). Modern agent results lean heavily on scaffolding and retries; comparing a compiled program against a vanilla single-shot agent inflates the latency and cost deltas. The paper's hedges are appropriate, but the abstract and introduction still lead with the comparison in a way that implies a capability gap the design does not license. At minimum the baseline's single-shot, no-scaffold nature belongs in the main text next to the comparison table, not only implicitly in the code. +The agent baseline (`benchmark/harness/agent_baseline.py`) is not a strawman in the crude sense: it drives the same Playwright backend, uses a current computer-use model and tool, and is handed generous task information (credentials, exact patient, exact note text, a hint that the app is slow). But it is a **single-attempt, un-scaffolded** loop: no system prompt, no planning/reflection/ReAct, no whole-task retry, and hard caps (25 actions, $1.50, last-3-screenshots context). Modern agent results lean heavily on scaffolding and retries; comparing a compiled program against a vanilla single-shot agent inflates the latency and cost deltas. The paper's hedges are appropriate, but the abstract and introduction still lead with the comparison in a way that implies a capability gap the design does not license. At minimum the baseline's single-shot, no-scaffold nature belongs in the main text next to the comparison table, not only implicitly in the code. ### 3.5 The strong-oracle claim is ambiguous exactly where it matters most (pixel-only substrates) diff --git a/packages/flow/pyproject.toml b/packages/flow/pyproject.toml index b63cfa7435..f8723b8307 100644 --- a/packages/flow/pyproject.toml +++ b/packages/flow/pyproject.toml @@ -353,6 +353,7 @@ exclude = [ "/docs/validation/identity_roc.json", "/docs/validation/identity_roc.png", "/openadapt_flow/benchmark/reliability_corpus.py", + "/benchmark/harness/reliability_corpus.py", "/openadapt_flow/validation/adversary_corpus.py", "/openadapt_flow/validation/adversary_corpus_v2.py", "/openadapt_flow/validation/adversary_corpus_v3.py", @@ -416,6 +417,7 @@ exclude = [ "/docs/validation/identity_roc.json", "/docs/validation/identity_roc.png", "/openadapt_flow/benchmark/reliability_corpus.py", + "/benchmark/harness/reliability_corpus.py", "/openadapt_flow/validation/adversary_corpus.py", "/openadapt_flow/validation/adversary_corpus_v2.py", "/openadapt_flow/validation/adversary_corpus_v3.py", @@ -501,8 +503,8 @@ required-environments = [ target-version = "py310" line-length = 88 # Generated / non-package trees at the repo root are not part of the lint -# surface. Anchored with a leading slash so the package's own -# openadapt_flow/benchmark/ IS still linted (a bare "benchmark" would match it). +# surface. The benchmark harness (benchmark/harness/) is linted by naming it +# explicitly in CI (`ruff check openadapt_flow benchmark/harness`). extend-exclude = ["/benchmark", "/docs", "/runs", "/dist"] [tool.ruff.lint] @@ -605,7 +607,6 @@ module = [ "openadapt_flow.__main__", "openadapt_flow.backends.playwright_backend", "openadapt_flow.backends.rdp_backend", - "openadapt_flow.benchmark.desktop_benchmark", "openadapt_flow.compiler.disambiguation", "openadapt_flow.interactive_recorder", "openadapt_flow.learning.synth_stream", diff --git a/packages/flow/scripts/desktop/waa_shim.py b/packages/flow/scripts/desktop/waa_shim.py index a15e66b1ae..e0a877e4af 100644 --- a/packages/flow/scripts/desktop/waa_shim.py +++ b/packages/flow/scripts/desktop/waa_shim.py @@ -1,7 +1,7 @@ """In-guest WAA-contract HTTP shim (runs INSIDE the Windows VM). Serves the desktop benchmark harness -(``openadapt_flow.benchmark.desktop_benchmark``): +(``benchmark.harness.desktop_benchmark``): GET /screenshot -> raw PNG bytes of the desktop (Flask send_file) GET /health -> ``{"status": "ok", ...}`` liveness probe diff --git a/packages/flow/scripts/frappe_lending_demo.py b/packages/flow/scripts/frappe_lending_demo.py index 8681b128b4..13877e4925 100644 --- a/packages/flow/scripts/frappe_lending_demo.py +++ b/packages/flow/scripts/frappe_lending_demo.py @@ -42,8 +42,8 @@ from openadapt_flow.backends.playwright_backend import ( # noqa: E402 PlaywrightBackend as _BasePlaywrightBackend, ) -from openadapt_flow.benchmark import agent_baseline # noqa: E402 -from openadapt_flow.benchmark.frappe_lending import ( # noqa: E402 +from benchmark.harness import agent_baseline # noqa: E402 +from benchmark.harness.frappe_lending import ( # noqa: E402 CONDITIONS, INITIAL_TRIALS_PER_CELL, PUBLICATION_TRIALS_PER_CELL, diff --git a/packages/flow/scripts/openemr_demo.py b/packages/flow/scripts/openemr_demo.py index 6a91a699fe..b9a75aee21 100644 --- a/packages/flow/scripts/openemr_demo.py +++ b/packages/flow/scripts/openemr_demo.py @@ -19,7 +19,7 @@ Showcase artifacts land in docs/showcase-openemr/; ``benchmark`` records a fresh demonstration into a temporary directory and writes results to -benchmark/openemr/ (see openadapt_flow.benchmark.openemr_benchmark). +benchmark/openemr/ (see benchmark.harness.openemr_benchmark). """ from __future__ import annotations @@ -189,7 +189,7 @@ def _note_visible(final_png: bytes, note: str) -> bool: Delegates to the benchmark's shared arm-independent check (``verify_note_saved``) so the criterion is implemented exactly once. """ - from openadapt_flow.benchmark.verify import verify_note_saved + from benchmark.harness.verify import verify_note_saved return verify_note_saved(final_png, note).success @@ -280,15 +280,15 @@ def benchmark() -> None: (incremental, per run) plus results.json / BENCHMARK.md / latency_cost.png to benchmark/openemr/. - Hard cost guardrails (see openadapt_flow.benchmark.openemr_benchmark): + Hard cost guardrails (see benchmark.harness.openemr_benchmark): an API preflight before any spend, a $1.50 per-run cap, an $8.00 total agent-arm ceiling, and an abort on consecutive billing errors. All at list price — billed cost under the intro rate is lower. """ import tempfile - from openadapt_flow.benchmark import agent_baseline - from openadapt_flow.benchmark.openemr_benchmark import ( + from benchmark.harness import agent_baseline + from benchmark.harness.openemr_benchmark import ( MAX_TOTAL_COST_USD, run_openemr_benchmark, ) diff --git a/packages/flow/scripts/openemr_local_demo.py b/packages/flow/scripts/openemr_local_demo.py index 4a30b188b2..a0f71c203d 100644 --- a/packages/flow/scripts/openemr_local_demo.py +++ b/packages/flow/scripts/openemr_local_demo.py @@ -40,8 +40,8 @@ from openadapt_flow.backends.playwright_backend import ( # noqa: E402 PlaywrightBackend as _BasePlaywrightBackend, ) -from openadapt_flow.benchmark import agent_baseline # noqa: E402 -from openadapt_flow.benchmark.openemr_local import ( # noqa: E402 +from benchmark.harness import agent_baseline # noqa: E402 +from benchmark.harness.openemr_local import ( # noqa: E402 CONDITIONS, INITIAL_TRIALS_PER_CELL, PUBLICATION_TRIALS_PER_CELL, diff --git a/packages/flow/scripts/openemr_param_depth.py b/packages/flow/scripts/openemr_param_depth.py index 6afd55d641..5c77b4c57f 100644 --- a/packages/flow/scripts/openemr_param_depth.py +++ b/packages/flow/scripts/openemr_param_depth.py @@ -148,7 +148,7 @@ def compile_bundle() -> None: def _replay_one(label: str, params: dict[str, str], url: str) -> dict: """One fresh-browser compiled replay; returns a summary row.""" - from openadapt_flow.benchmark.verify import verify_note_saved + from benchmark.harness.verify import verify_note_saved from openadapt_flow.ir import Workflow from openadapt_flow.runtime.replayer import Replayer from openadapt_flow.vision.ocr import normalize_text, ocr diff --git a/packages/flow/tests/e2e/test_region_stable_theme_regression.py b/packages/flow/tests/e2e/test_region_stable_theme_regression.py index 4ce3990130..152450bc7a 100644 --- a/packages/flow/tests/e2e/test_region_stable_theme_regression.py +++ b/packages/flow/tests/e2e/test_region_stable_theme_regression.py @@ -13,7 +13,7 @@ import pytest from openadapt_flow.backends.playwright_backend import PlaywrightBackend -from openadapt_flow.benchmark.dom_arm import verify_final_state +from benchmark.harness.dom_arm import verify_final_state from openadapt_flow.compiler import compile_recording from openadapt_flow.demo_driver import record_triage_demo from openadapt_flow.ir import Workflow diff --git a/packages/flow/tests/test_benchmark.py b/packages/flow/tests/test_benchmark.py index 6f903b79c9..68dbac3e54 100644 --- a/packages/flow/tests/test_benchmark.py +++ b/packages/flow/tests/test_benchmark.py @@ -15,20 +15,20 @@ import numpy as np import pytest -from openadapt_flow.benchmark import agent_baseline -from openadapt_flow.benchmark.agent_baseline import ( +from benchmark.harness import agent_baseline +from benchmark.harness.agent_baseline import ( _truncate_screenshots, compute_cost, load_api_key, run_agent, triage_task_prompt, ) -from openadapt_flow.benchmark.run_benchmark import ( +from benchmark.harness.run_benchmark import ( aggregate_results, render_markdown, write_outputs, ) -from openadapt_flow.benchmark.verify import verify_encounter_saved +from benchmark.harness.verify import verify_encounter_saved NOTE = "Follow-up in 2 weeks; BP recheck." @@ -521,7 +521,7 @@ def test_right_type_passes_at_every_note_length( # line. TestVerify above retains a real-OCR smoke for the benchmark's # actual 32-character note; these four cases pin the verifier logic. monkeypatch.setattr( - "openadapt_flow.benchmark.verify.ocr", + "benchmark.harness.verify.ocr", lambda _screen: [SimpleNamespace(text=f"Triage - {note[:60]}")], ) verdict = verify_encounter_saved(screen, note) @@ -542,7 +542,7 @@ def test_wrong_type_fails_at_every_note_length( f"Consult - {note[:60]}", ) monkeypatch.setattr( - "openadapt_flow.benchmark.verify.ocr", + "benchmark.harness.verify.ocr", lambda _screen: [SimpleNamespace(text=f"Consult - {note[:60]}")], ) verdict = verify_encounter_saved(screen, note) @@ -599,7 +599,7 @@ def test_bare_note_starting_with_known_type_is_not_a_row( # the note itself starts with a known categorical value. screen = self.make_screen(f"Encounter saved - {note[:40]}", note[:60]) monkeypatch.setattr( - "openadapt_flow.benchmark.verify.ocr", + "benchmark.harness.verify.ocr", lambda _screen: [SimpleNamespace(text=note[:60])], ) verdict = verify_encounter_saved(screen, note) diff --git a/packages/flow/tests/test_chart_fonts.py b/packages/flow/tests/test_chart_fonts.py index c54ceb2bc1..d40847a4da 100644 --- a/packages/flow/tests/test_chart_fonts.py +++ b/packages/flow/tests/test_chart_fonts.py @@ -2,7 +2,7 @@ The benchmark chart is cosmetic; a font-lookup failure must never fail the suite. These tests cover the two guarantees in -``openadapt_flow.benchmark.chart_fonts``: the bundled DejaVuSans font is +``benchmark.harness.chart_fonts``: the bundled DejaVuSans font is registered so ``findfont`` cannot miss, and ``safe_render`` swallows any rendering failure. """ @@ -12,7 +12,7 @@ from pathlib import Path from typing import Any -from openadapt_flow.benchmark.chart_fonts import ( +from benchmark.harness.chart_fonts import ( BUNDLED_FONT_NAME, configure_bundled_font, safe_render, diff --git a/packages/flow/tests/test_competitor_drift_instrument.py b/packages/flow/tests/test_competitor_drift_instrument.py index 984b79c792..ec52bd8589 100644 --- a/packages/flow/tests/test_competitor_drift_instrument.py +++ b/packages/flow/tests/test_competitor_drift_instrument.py @@ -22,7 +22,7 @@ import pytest -from openadapt_flow.instrument import ( +from benchmark.harness.instrument import ( AgentRunResult, ArchitectureClassError, CostGuard, @@ -35,7 +35,7 @@ ensure_architecture_class, run_instrument, ) -from openadapt_flow.instrument.competitor_drift import render_markdown, write_outputs +from benchmark.harness.instrument.competitor_drift import render_markdown, write_outputs # Fault classes a screen-only report is BLIND to: a wrong effect lands while # the "saved" banner still paints, so a screen-blind agent claims success. diff --git a/packages/flow/tests/test_desktop_benchmark.py b/packages/flow/tests/test_desktop_benchmark.py index 45d605ed10..f54425cae2 100644 --- a/packages/flow/tests/test_desktop_benchmark.py +++ b/packages/flow/tests/test_desktop_benchmark.py @@ -9,8 +9,8 @@ import json -from openadapt_flow.benchmark import desktop_benchmark as db -from openadapt_flow.benchmark.desktop_benchmark import ( +from benchmark.harness import desktop_benchmark as db +from benchmark.harness.desktop_benchmark import ( _classify, run_desktop_benchmark, ) diff --git a/packages/flow/tests/test_dom_arm.py b/packages/flow/tests/test_dom_arm.py index c19314c790..6dd3a6e4b4 100644 --- a/packages/flow/tests/test_dom_arm.py +++ b/packages/flow/tests/test_dom_arm.py @@ -15,7 +15,7 @@ import numpy as np import pytest -from openadapt_flow.benchmark.dom_arm import ( +from benchmark.harness.dom_arm import ( PERTURBATIONS, SCHEDULE, aggregate_dom_results, @@ -191,7 +191,7 @@ def test_perturbation_menu(self) -> None: ) def test_notes_distinct_per_arm_and_slot(self) -> None: - from openadapt_flow.benchmark.dom_arm import ARMS + from benchmark.harness.dom_arm import ARMS notes = { note_for_slot(arm, slot) @@ -262,7 +262,7 @@ def test_maintenance_is_dom_loud_break_on_drift_only(self) -> None: assert not needs_maintenance(disputed) def test_verification_dispute(self) -> None: - from openadapt_flow.benchmark.dom_arm import verification_dispute + from benchmark.harness.dom_arm import verification_dispute # DOM: every step ran, no error, judge said no -> dispute. assert verification_dispute( @@ -546,7 +546,7 @@ class TestVerifyFinalState: exists to measure.""" def test_dom_check_is_the_hybrid_check(self) -> None: - from openadapt_flow.benchmark.hybrid_benchmark import ( + from benchmark.harness.hybrid_benchmark import ( verify_hybrid_final, ) diff --git a/packages/flow/tests/test_duplicate_guard_live_openemr.py b/packages/flow/tests/test_duplicate_guard_live_openemr.py index e7395a2000..0036361244 100644 --- a/packages/flow/tests/test_duplicate_guard_live_openemr.py +++ b/packages/flow/tests/test_duplicate_guard_live_openemr.py @@ -137,7 +137,7 @@ def test_a_possible_duplicate_halts_before_the_save( open_session, seed_existing_chart, ) - from openadapt_flow.benchmark.openemr_local import ( + from benchmark.harness.openemr_local import ( OpenEMRPatientOracle, SyntheticPatientSpec, ) diff --git a/packages/flow/tests/test_effectbench.py b/packages/flow/tests/test_effectbench.py index 435364cdcd..19419163d9 100644 --- a/packages/flow/tests/test_effectbench.py +++ b/packages/flow/tests/test_effectbench.py @@ -26,7 +26,7 @@ build_reference_episodes, ) from benchmark.fault_model import faults as F -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( AgentReport, DivergenceCategory, Effect, @@ -44,7 +44,7 @@ summarize, wilson_interval, ) -from openadapt_flow.benchmark.effectbench.metrics import bootstrap_ci, rate +from benchmark.harness.effectbench.metrics import bootstrap_ci, rate # --------------------------------------------------------------------------- # 1. Reference re-expression regression (the headline result) @@ -334,7 +334,7 @@ def test_record_and_note_effects_are_reexported() -> None: def test_score_episode_end_to_end() -> None: """The runner orchestrator: capture pre-state, run the arm, verify, label.""" - from openadapt_flow.benchmark.effectbench import AgentReport, score_episode + from benchmark.harness.effectbench import AgentReport, score_episode box = {"acted": False} @@ -367,7 +367,7 @@ def run_action() -> AgentReport: def test_score_episode_raises_on_unreadable_oracle() -> None: - from openadapt_flow.benchmark.effectbench import AgentReport, score_episode + from benchmark.harness.effectbench import AgentReport, score_episode with pytest.raises(ValueError): score_episode( @@ -433,8 +433,8 @@ def test_summary_filters_by_arm(episodes: list[EpisodeRecord]) -> None: def test_episode_record_predicates_match_outcome() -> None: - from openadapt_flow.benchmark.effectbench import Verdict - from openadapt_flow.benchmark.effectbench.schema import OracleVerdict + from benchmark.harness.effectbench import Verdict + from benchmark.harness.effectbench.schema import OracleVerdict ep = EpisodeRecord( episode_id="e1", diff --git a/packages/flow/tests/test_effectbench_runner.py b/packages/flow/tests/test_effectbench_runner.py index 64cd934c61..8abe524670 100644 --- a/packages/flow/tests/test_effectbench_runner.py +++ b/packages/flow/tests/test_effectbench_runner.py @@ -20,7 +20,7 @@ import pytest -from openadapt_flow.benchmark.effectbench import ( +from benchmark.harness.effectbench import ( Effect, EffectKind, EpisodeRecord, @@ -30,7 +30,7 @@ SwerVariant, summarize, ) -from openadapt_flow.benchmark.effectbench.runner import ( +from benchmark.harness.effectbench.runner import ( LIVE_ARMS, CompilerArm, CompoundEffectVerifier, @@ -43,19 +43,19 @@ run_episode, run_matrix, ) -from openadapt_flow.benchmark.effectbench.runner.arms import AgentArm, ArmResult -from openadapt_flow.benchmark.effectbench.runner.baselines import ( +from benchmark.harness.effectbench.runner.arms import AgentArm, ArmResult +from benchmark.harness.effectbench.runner.baselines import ( SCAFFOLDED_ARMS, ClaudeComputerUseArm, ScaffoldNotWired, ) -from openadapt_flow.benchmark.effectbench.runner.harness import EpisodeEnv -from openadapt_flow.benchmark.effectbench.runner.reference_tasks import ( +from benchmark.harness.effectbench.runner.harness import EpisodeEnv +from benchmark.harness.effectbench.runner.reference_tasks import ( GOAL, TARGET, make_params, ) -from openadapt_flow.benchmark.effectbench.schema import ( +from benchmark.harness.effectbench.schema import ( DivergenceCategory, OracleChannel, OracleSpec, @@ -343,7 +343,7 @@ def test_run_matrix_skips_scaffolded_arms_by_default() -> None: def test_cli_dry_run_runs_green() -> None: - from openadapt_flow.benchmark.effectbench.runner.__main__ import run_dry_run + from benchmark.harness.effectbench.runner.__main__ import run_dry_run episodes, summaries = run_dry_run( trials=1, arm_names="compiler,screen_only", include_scaffolded=False diff --git a/packages/flow/tests/test_effectbench_standalone.py b/packages/flow/tests/test_effectbench_standalone.py index dc0fb92526..56ac997158 100644 --- a/packages/flow/tests/test_effectbench_standalone.py +++ b/packages/flow/tests/test_effectbench_standalone.py @@ -6,7 +6,7 @@ dependency). Because the standalone package vendors the effect MECHANISM (the typed ``Effect`` contract, the substrate-independent judge, the outcome classifier) that is otherwise defined in ``openadapt_flow.runtime.effects`` and -``openadapt_flow.benchmark.effectbench``, this test guards against the two copies +``benchmark.harness.effectbench``, this test guards against the two copies drifting: it asserts the standalone classifier and judge agree BYTE-FOR-BYTE with the engine on the reference inputs, and that the standalone reference reproduces the published headline. It also re-runs the standalone source-availability @@ -62,7 +62,7 @@ def test_classifier_parity_with_engine_over_the_full_truth_table(): """The standalone classifier equals the engine classifier on every input.""" import effectbench.oracle as sa - from openadapt_flow.benchmark.effectbench import oracle as eng + from benchmark.harness.effectbench import oracle as eng scoreable = ["CORRECT", "WRONG_PERSISTED", "ABSENT"] for reported, state_name, avail in itertools.product( @@ -86,7 +86,7 @@ def test_divergence_category_parity_with_engine(): """The public standalone taxonomy must not lag the engine copy.""" from effectbench.schema import DivergenceCategory as StandaloneCategory - from openadapt_flow.benchmark.effectbench.schema import ( + from benchmark.harness.effectbench.schema import ( DivergenceCategory as EngineCategory, ) diff --git a/packages/flow/tests/test_effectbench_task_pack.py b/packages/flow/tests/test_effectbench_task_pack.py index bed77414ca..ddb2ea85c0 100644 --- a/packages/flow/tests/test_effectbench_task_pack.py +++ b/packages/flow/tests/test_effectbench_task_pack.py @@ -36,8 +36,8 @@ from benchmark.effectbench.task_pack.audit import audit_mockmed_oracle from benchmark.effectbench.task_pack.driver import run_mockmed_pack from benchmark.effectbench.task_pack.mockmed_tasks import MOCKMED_TASKS -from openadapt_flow.benchmark.effectbench import OutcomeLabel, Substrate, summarize -from openadapt_flow.benchmark.effectbench.schema import DivergenceCategory +from benchmark.harness.effectbench import OutcomeLabel, Substrate, summarize +from benchmark.harness.effectbench.schema import DivergenceCategory from openadapt_flow.runtime.effects.sql import assert_read_only_sql MANIFEST_PATH = Path(__file__).resolve().parents[1] / ( diff --git a/packages/flow/tests/test_frappe_lending_benchmark.py b/packages/flow/tests/test_frappe_lending_benchmark.py index ee3f50c080..1737d2a80b 100644 --- a/packages/flow/tests/test_frappe_lending_benchmark.py +++ b/packages/flow/tests/test_frappe_lending_benchmark.py @@ -19,7 +19,7 @@ FrappeFixture, audit_table_deltas, ) -from openadapt_flow.benchmark.frappe_lending import ( +from benchmark.harness.frappe_lending import ( ARMS, CONDITIONS, FrappeLoanApplicationOracle, diff --git a/packages/flow/tests/test_hybrid_benchmark.py b/packages/flow/tests/test_hybrid_benchmark.py index 9779311f40..11ca0244ba 100644 --- a/packages/flow/tests/test_hybrid_benchmark.py +++ b/packages/flow/tests/test_hybrid_benchmark.py @@ -15,8 +15,8 @@ import numpy as np import pytest -from openadapt_flow.benchmark import agent_baseline, hybrid_benchmark -from openadapt_flow.benchmark.hybrid_benchmark import ( +from benchmark.harness import agent_baseline, hybrid_benchmark +from benchmark.harness.hybrid_benchmark import ( AGENT_SLOTS, DRIFT_TYPES, FALLBACK_MAX_ACTIONS, @@ -610,7 +610,7 @@ def fake_run_agent(backend, task, **kwargs): assert "no failing step recorded" in seen["task"] def test_failed_result_index_edge_cases(self) -> None: - from openadapt_flow.benchmark.hybrid_benchmark import ( + from benchmark.harness.hybrid_benchmark import ( _failed_result_index, ) @@ -978,7 +978,7 @@ def test_arms_see_identical_conditions_per_slot( the run helpers), not on the orchestrator-stamped condition labels — the labels come from the same schedule lookup the orchestrator uses to build the URLs, so asserting on them alone is circular.""" - from openadapt_flow.benchmark.hybrid_benchmark import ( + from benchmark.harness.hybrid_benchmark import ( AGENT_SLOTS, condition_url, ) diff --git a/packages/flow/tests/test_lending_swer.py b/packages/flow/tests/test_lending_swer.py index b09dabeeed..b787960357 100644 --- a/packages/flow/tests/test_lending_swer.py +++ b/packages/flow/tests/test_lending_swer.py @@ -22,7 +22,7 @@ from benchmark.lending_fault_model import faults as F from benchmark.lending_fault_model import ground_truth as G from benchmark.lending_fault_model import swer as S -from openadapt_flow.benchmark.effectbench import summarize +from benchmark.harness.effectbench import summarize from openadapt_flow.mockloan.fault_server import LedgerDB diff --git a/packages/flow/tests/test_openemr_benchmark.py b/packages/flow/tests/test_openemr_benchmark.py index b2f10ba067..f32a93db12 100644 --- a/packages/flow/tests/test_openemr_benchmark.py +++ b/packages/flow/tests/test_openemr_benchmark.py @@ -16,20 +16,20 @@ import cv2 import numpy as np -import openadapt_flow.benchmark.verify as benchmark_verify -from openadapt_flow.benchmark import agent_baseline, openemr_benchmark -from openadapt_flow.benchmark.agent_baseline import ( +import benchmark.harness.verify as benchmark_verify +from benchmark.harness import agent_baseline, openemr_benchmark +from benchmark.harness.agent_baseline import ( openemr_task_prompt, run_agent, ) -from openadapt_flow.benchmark.openemr_benchmark import ( +from benchmark.harness.openemr_benchmark import ( aggregate_openemr_results, note_for, render_openemr_markdown, run_openemr_benchmark, write_openemr_outputs, ) -from openadapt_flow.benchmark.verify import verify_note_saved +from benchmark.harness.verify import verify_note_saved from openadapt_flow.vision import OcrLine NOTE = "Insurance card copied and coverage verified by phone." @@ -848,7 +848,7 @@ def crashing_client(self) -> CrashingClient: ) def test_mid_run_crash_row_carries_partial_cost(self, monkeypatch: Any) -> None: - from openadapt_flow.benchmark.run_benchmark import _agent_run + from benchmark.harness.run_benchmark import _agent_run _fake_launch(monkeypatch, FakeBackend()) row = _agent_run("http://x", NOTE, client=self.crashing_client(), task="task") diff --git a/packages/flow/tests/test_openemr_local_benchmark.py b/packages/flow/tests/test_openemr_local_benchmark.py index 41052e881a..74835d52d8 100644 --- a/packages/flow/tests/test_openemr_local_benchmark.py +++ b/packages/flow/tests/test_openemr_local_benchmark.py @@ -23,7 +23,7 @@ audit_table_deltas, unexpected_table_deltas, ) -from openadapt_flow.benchmark.openemr_local import ( +from benchmark.harness.openemr_local import ( ARMS, CONDITIONS, OpenEMRPatientOracle, @@ -1062,7 +1062,7 @@ def __init__(self, messages: Messages) -> None: "max_tokens": 4096, } ledger = __import__( - "openadapt_flow.benchmark.agent_baseline", fromlist=["UsageLedger"] + "benchmark.harness.agent_baseline", fromlist=["UsageLedger"] ).UsageLedger() allowed_messages = Messages(10_000) diff --git a/packages/flow/tests/test_openemr_retained_finals.py b/packages/flow/tests/test_openemr_retained_finals.py index 8fbdc22904..c1b6709429 100644 --- a/packages/flow/tests/test_openemr_retained_finals.py +++ b/packages/flow/tests/test_openemr_retained_finals.py @@ -16,7 +16,7 @@ import pytest -from openadapt_flow.benchmark.verify import verify_note_saved +from benchmark.harness.verify import verify_note_saved REPO_ROOT = Path(__file__).resolve().parents[1] RUN_RETAINED_FINALS = os.environ.get("OPENADAPT_OPENEMR_FINALS_DIR") diff --git a/packages/flow/tests/test_qualify_business_rule.py b/packages/flow/tests/test_qualify_business_rule.py index ec61d1b9ee..7478971d08 100644 --- a/packages/flow/tests/test_qualify_business_rule.py +++ b/packages/flow/tests/test_qualify_business_rule.py @@ -90,7 +90,7 @@ def test_rule_ids_are_unique_and_cases_must_name_a_rule() -> None: def test_the_openemr_verify_yaml_declares_the_existing_duplicate_rule() -> None: - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec from openadapt_flow.qualify import load_verify_config config = load_verify_config(FLOW / "benchmark/openemr_local/verify.yaml", env={}) diff --git a/packages/flow/tests/test_release_contract.py b/packages/flow/tests/test_release_contract.py index c185df3248..ae13081400 100644 --- a/packages/flow/tests/test_release_contract.py +++ b/packages/flow/tests/test_release_contract.py @@ -113,6 +113,7 @@ def _approved_artifact_paths() -> set[str]: "/docs/validation/identity_roc.json", "/docs/validation/identity_roc.png", "/openadapt_flow/benchmark/reliability_corpus.py", + "/benchmark/harness/reliability_corpus.py", "/openadapt_flow/validation/adversary_corpus.py", "/openadapt_flow/validation/adversary_corpus_v2.py", "/openadapt_flow/validation/adversary_corpus_v3.py", diff --git a/packages/flow/tests/test_reliability_harness.py b/packages/flow/tests/test_reliability_harness.py index 06be924f83..e3dade0d41 100644 --- a/packages/flow/tests/test_reliability_harness.py +++ b/packages/flow/tests/test_reliability_harness.py @@ -1,6 +1,6 @@ """Small synthetic checks for the public reliability aggregation mechanism.""" -from openadapt_flow.benchmark.reliability import _outcome, aggregate +from benchmark.harness.reliability import _outcome, aggregate def _result(outcome: str, *, category: str | None = None) -> dict: diff --git a/packages/flow/tests/test_runtime_never_imports_benchmark.py b/packages/flow/tests/test_runtime_never_imports_benchmark.py index dc4fa57ffb..7d052ea956 100644 --- a/packages/flow/tests/test_runtime_never_imports_benchmark.py +++ b/packages/flow/tests/test_runtime_never_imports_benchmark.py @@ -1,9 +1,10 @@ -"""Product code never imports ``openadapt_flow.benchmark``. +"""Product code never imports the benchmark harness. -A frozen build (Desktop's engine) leaves the benchmark package out, so a -product module that imports it, even lazily, fails there. Shared names -(``MODEL``, ``load_api_key``, ``SCROLL_PX_PER_UNIT``) live in -``openadapt_flow.anthropic_defaults``. +The harness lives in the source checkout (``packages/flow/benchmark/harness``), +outside the ``openadapt_flow`` package, so neither the wheel nor a frozen build +(Desktop's engine) carries it. A product module that imports it, even lazily, +fails there. Shared names (``MODEL``, ``load_api_key``, ``SCROLL_PX_PER_UNIT``) +live in ``openadapt_flow.anthropic_defaults``. """ from __future__ import annotations @@ -17,57 +18,75 @@ #: Modules whose job is the benchmark itself. Nothing else may import it. ALLOWED = { - # The benchmark CLI command loads the harness on demand. + # The benchmark CLI command loads the harness on demand from a checkout. "__main__.py", - # The competitor-drift harness reuses the benchmark's fault scenarios. - "instrument/competitor_drift.py", + # `record-check prove` loads its fixture from a checkout on demand. + "record_check_proof.py", } -def _imports_benchmark(tree: ast.AST) -> list[int]: +def _is_benchmark(module: str) -> bool: + return any( + module == root or module.startswith(root + ".") + for root in ("benchmark", "openadapt_flow.benchmark") + ) + + +def _imports_benchmark(tree: ast.AST, *, lazy_ok: bool = False) -> list[int]: lines: list[int] = [] - for node in ast.walk(tree): + # An allowed module may import the harness only inside a function, where + # it runs on demand from a checkout. + nodes = ( + [ + n + for n in ast.iter_child_nodes(tree) + if isinstance(n, (ast.Import, ast.ImportFrom)) + ] + if lazy_ok + else ast.walk(tree) + ) + for node in nodes: if isinstance(node, ast.ImportFrom): module = node.module or "" - if node.level == 0 and ( - module == "openadapt_flow.benchmark" - or module.startswith("openadapt_flow.benchmark.") - ): + if node.level == 0 and _is_benchmark(module): lines.append(node.lineno) elif node.level == 0 and module == "openadapt_flow": if any(a.name == "benchmark" for a in node.names): lines.append(node.lineno) - elif node.level > 0 and ( - module == "benchmark" or module.startswith("benchmark.") - ): + elif node.level > 0 and _is_benchmark(module): lines.append(node.lineno) elif isinstance(node, ast.Import): - for alias in node.names: - if alias.name == "openadapt_flow.benchmark" or alias.name.startswith( - "openadapt_flow.benchmark." - ): - lines.append(node.lineno) + if any(_is_benchmark(alias.name) for alias in node.names): + lines.append(node.lineno) return lines +def test_the_harness_is_not_inside_the_package() -> None: + assert not (ROOT / "benchmark").exists(), ( + "benchmark harnesses belong in packages/flow/benchmark/, outside the " + "shipped openadapt_flow package" + ) + + def test_product_modules_never_import_the_benchmark_package() -> None: offenders: list[str] = [] for path in sorted(ROOT.rglob("*.py")): rel = path.relative_to(ROOT).as_posix() - if rel.startswith("benchmark/") or rel in ALLOWED: - continue tree = ast.parse(path.read_text(encoding="utf-8")) - offenders += [f"{rel}:{n}" for n in _imports_benchmark(tree)] + offenders += [ + f"{rel}:{n}" for n in _imports_benchmark(tree, lazy_ok=rel in ALLOWED) + ] assert not offenders, ( - "product code imports openadapt_flow.benchmark (a frozen build leaves " - "it out); move the shared name to openadapt_flow.anthropic_defaults " - f"or a runtime module: {offenders}" + "product code imports the benchmark harness (the wheel and a frozen " + "build leave it out); move the shared name to " + "openadapt_flow.anthropic_defaults or a runtime module: " + f"{offenders}" ) def test_shared_anthropic_names_have_one_home() -> None: + from benchmark.harness import agent_baseline from openadapt_flow import anthropic_defaults - from openadapt_flow.benchmark import agent_baseline assert agent_baseline.MODEL is anthropic_defaults.MODEL assert agent_baseline.SCROLL_PX_PER_UNIT == anthropic_defaults.SCROLL_PX_PER_UNIT diff --git a/packages/flow/tests/test_silent_wrong_action_benchmark.py b/packages/flow/tests/test_silent_wrong_action_benchmark.py index a340eea1ae..cadfdbcacc 100644 --- a/packages/flow/tests/test_silent_wrong_action_benchmark.py +++ b/packages/flow/tests/test_silent_wrong_action_benchmark.py @@ -16,7 +16,7 @@ from __future__ import annotations -from openadapt_flow.benchmark.silent_wrong_action import ( +from benchmark.harness.silent_wrong_action import ( SCENARIOS, aggregate, render_markdown, diff --git a/packages/flow/tests/test_ui_dismiss_live_openemr.py b/packages/flow/tests/test_ui_dismiss_live_openemr.py index 1468c56f6d..3caceffee1 100644 --- a/packages/flow/tests/test_ui_dismiss_live_openemr.py +++ b/packages/flow/tests/test_ui_dismiss_live_openemr.py @@ -47,7 +47,7 @@ def _record(tmp_path: Path) -> tuple[Path, Any]: from benchmark.openemr_local.qualify_hooks import fixture - from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from benchmark.harness.openemr_local import SyntheticPatientSpec from openadapt_flow.recorder import Recorder from scripts.openemr_local_demo import ( FORM_FRAME_NAME,