From 56056b46149d3d891852010a082d9e84f3b5983a Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Tue, 3 Nov 2026 14:32:25 -0500 Subject: [PATCH 1/3] feat(extension): export the model helpers an asset domain uses usersim.extension exports to_model_configs, to_data_designer_kwargs and require_aliases, which turn the models an asset domain's generate and evaluate receive into Data Designer model configs and provider arguments, and check that the models declare the aliases the domain needs. The reference lists them, importing them leaves Data Designer and the conversation loop unloaded, and the financial_services generator imports them from usersim.extension. Signed-off-by: Yev Meyer --- docs/extension_api.md | 3 +++ src/usersim/asset_gen/financial_services/evaluate.py | 4 ++-- src/usersim/asset_gen/financial_services/pipeline.py | 10 +++++----- src/usersim/extension.py | 5 +++++ tests/test_extension_api.py | 1 + 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/extension_api.md b/docs/extension_api.md index 385407ea..9bee91f8 100644 --- a/docs/extension_api.md +++ b/docs/extension_api.md @@ -121,6 +121,9 @@ does not load the conversation loop. | `CapabilityDefinition` | A row of the capability report and the evidence it is scored from | | `EvidenceSource` | One probe, scorer and set of axes a capability reads | | `AssetDomain` | A domain `usersim gen-assets` can build banks for | +| `to_model_configs` | The Data Designer model configs for the models an asset domain receives | +| `to_data_designer_kwargs` | The `DataDesigner` keyword arguments for those models' providers | +| `require_aliases` | Raises a configuration error naming the aliases a domain needs that the models lack | | `ExecutionBackend` | Somewhere a run can be submitted, for `--backend` | | `JobSpec` | What to run, independent of where | | `JobHandle` | A backend's reference to work it accepted | diff --git a/src/usersim/asset_gen/financial_services/evaluate.py b/src/usersim/asset_gen/financial_services/evaluate.py index ba0a4785..f577ed52 100644 --- a/src/usersim/asset_gen/financial_services/evaluate.py +++ b/src/usersim/asset_gen/financial_services/evaluate.py @@ -257,7 +257,7 @@ def _num(v: Any) -> float | None: def validate_eval_models(models: Any) -> None: - from usersim.cli._models import require_aliases + from usersim.extension import require_aliases require_aliases(models, required=(EVAL_JUDGE_MODEL_ALIAS,), context="evaluate-assets --domain financial_services") @@ -309,7 +309,7 @@ def evaluate_bank( import data_designer.config as dd from usersim.asset_gen.financial_services.pipeline import _make_engine - from usersim.cli._models import to_model_configs + from usersim.extension import to_model_configs seed = build_eval_seed(bank_dir, region_spec) if seed.empty: diff --git a/src/usersim/asset_gen/financial_services/pipeline.py b/src/usersim/asset_gen/financial_services/pipeline.py index c9e73b16..efcea108 100644 --- a/src/usersim/asset_gen/financial_services/pipeline.py +++ b/src/usersim/asset_gen/financial_services/pipeline.py @@ -669,7 +669,7 @@ def build_brief_config(models: Any, inst_seed) -> Any: """Assemble the Phase-1 (engine, builder). One place, explicit add_column.""" import data_designer.config as dd - from usersim.cli._models import to_model_configs + from usersim.extension import to_model_configs engine = _make_engine(models) builder = dd.DataDesignerConfigBuilder(model_configs=to_model_configs(models)) @@ -873,7 +873,7 @@ def build_docset_config(models: Any, cluster_seed) -> Any: """Assemble the Phase-2 (engine, builder). One place, explicit add_column.""" import data_designer.config as dd - from usersim.cli._models import to_model_configs + from usersim.extension import to_model_configs engine = _make_engine(models) builder = dd.DataDesignerConfigBuilder(model_configs=to_model_configs(models)) @@ -1122,7 +1122,7 @@ def serialize_bank( def validate_generation_models(models: Any) -> None: """Fail fast if the doc-gen / judge / embedding aliases aren't configured.""" - from usersim.cli._models import require_aliases + from usersim.extension import require_aliases require_aliases( models, @@ -1136,7 +1136,7 @@ def _make_engine(models: Any) -> Any: import data_designer.config as dd from data_designer.interface import DataDesigner - from usersim.cli._models import to_data_designer_kwargs + from usersim.extension import to_data_designer_kwargs engine = DataDesigner(**to_data_designer_kwargs(models)) engine.set_run_config(dd.RunConfig(jinja_rendering_engine=dd.JinjaRenderingEngine.NATIVE)) @@ -1514,7 +1514,7 @@ def _embed_documents( return edf = pd.DataFrame(rows) - from usersim.cli._models import to_model_configs + from usersim.extension import to_model_configs result = None for attempt in range(1, max_attempts + 1): diff --git a/src/usersim/extension.py b/src/usersim/extension.py index d823c318..38c0812a 100644 --- a/src/usersim/extension.py +++ b/src/usersim/extension.py @@ -23,6 +23,7 @@ if TYPE_CHECKING: from usersim.asset_gen.registry import AssetDomain + from usersim.cli._models import require_aliases, to_data_designer_kwargs, to_model_configs from usersim.engine.config import MODEL_ASSISTANT, MODEL_JUDGE, MODEL_USER, ConversationSimulatorConfig from usersim.engine.core.backends import ExecutionBackend, JobHandle, JobSpec, JobStatus from usersim.engine.core.behavioral import ( @@ -146,11 +147,15 @@ "JobHandle", "JobSpec", "JobStatus", + "require_aliases", + "to_data_designer_kwargs", + "to_model_configs", ] #: The module each public name is defined in, by module. _EXPORTS: dict[str, tuple[str, ...]] = { "usersim.asset_gen.registry": ("AssetDomain",), + "usersim.cli._models": ("require_aliases", "to_data_designer_kwargs", "to_model_configs"), "usersim.engine.config": ("MODEL_ASSISTANT", "MODEL_JUDGE", "MODEL_USER", "ConversationSimulatorConfig"), "usersim.engine.core.backends": ("ExecutionBackend", "JobHandle", "JobSpec", "JobStatus"), "usersim.engine.core.behavioral": ( diff --git a/tests/test_extension_api.py b/tests/test_extension_api.py index 077ede88..c67404fb 100644 --- a/tests/test_extension_api.py +++ b/tests/test_extension_api.py @@ -127,6 +127,7 @@ def test_an_asset_domain_or_backend_import_leaves_the_conversation_loop_unloaded code = ( "import sys\n" "from usersim.extension import AssetDomain, CapabilityDefinition, ExecutionBackend, JobSpec\n" + "from usersim.extension import require_aliases, to_data_designer_kwargs, to_model_configs\n" "print(sorted(m for m in ('usersim.engine.core.simulation', 'data_designer') if m in sys.modules))\n" ) result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) From 880bc150863a4316827018eac2ec916e91f13d56 Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Tue, 3 Nov 2026 14:32:40 -0500 Subject: [PATCH 2/3] refactor(engine): keep financial_services in one folder The probe module is probes/financial_services/probe.py, beside its bank loader bank.py, its task builder tasks.py, its query embedder embeddings.py, its scorer scorer.py and its banks in assets/financial_services/, the folder the class names as its assets_root. Its tests are in tests/probes/financial_services/. The label, the scorer name, the banks, the stored columns and the caches are unchanged. gen-assets, validate-assets and evaluate-assets default to the shipped bank of the domain's probe, so they write and read the probe's own folder, and the package-data exclusions and ignore rules for generated embeddings and scorecards follow the banks. DEFAULT_SCORER_MODULES, the capability source references, the docs and the asset notebook name the new modules and paths, the authoring guide lists the probe among those kept in one folder, the architecture overview describes the engine folders as they are, and the shared-code label allowances list neither the bank loader nor the scorer. Signed-off-by: Yev Meyer --- .gitignore | 6 +- architecture/overview.md | 9 +-- docs/engine/AUTHORING_A_PROBE.md | 9 +-- docs/engine/README.md | 7 +- pyproject.toml | 6 +- .../asset_gen/financial_services/pipeline.py | 6 +- .../asset_gen/financial_services/validate.py | 6 +- .../asset_gen/notebooks/finance_assets.ipynb | 8 +-- src/usersim/cli/evaluate_assets.py | 4 +- src/usersim/cli/gen_assets.py | 4 +- src/usersim/cli/validate_assets.py | 4 +- src/usersim/engine/core/probes.py | 2 +- .../engine/evaluator/scorers/__init__.py | 2 +- .../probes/financial_services/README.md | 12 ++-- .../assets/financial_services/SCHEMA.md | 4 +- .../de_DE/_generation_report.json | 0 .../bank/bausparkasse_heimstatt/corpus.yaml | 0 .../institution_meta.yaml | 0 .../bank/bausparkasse_heimstatt/tasks.yaml | 0 .../bank/bausparkasse_heimstatt/tools.yaml | 0 .../bank/sparkasse_lindenhain/corpus.yaml | 0 .../institution_meta.yaml | 0 .../bank/sparkasse_lindenhain/tasks.yaml | 0 .../bank/sparkasse_lindenhain/tools.yaml | 0 .../brokerage/kranich_invest/corpus.yaml | 0 .../kranich_invest/institution_meta.yaml | 0 .../de_DE/brokerage/kranich_invest/tasks.yaml | 0 .../de_DE/brokerage/kranich_invest/tools.yaml | 0 .../financial_services/de_DE/dynamic.yaml | 0 .../hanseatica_versicherung/corpus.yaml | 0 .../institution_meta.yaml | 0 .../hanseatica_versicherung/tasks.yaml | 0 .../hanseatica_versicherung/tools.yaml | 0 .../neobank/kompass_direktbank/corpus.yaml | 0 .../kompass_direktbank/institution_meta.yaml | 0 .../neobank/kompass_direktbank/tasks.yaml | 0 .../neobank/kompass_direktbank/tools.yaml | 0 .../financial_services/de_DE/region_meta.yaml | 0 .../en_GB/_generation_report.json | 0 .../fernbridge_building_society/corpus.yaml | 0 .../institution_meta.yaml | 0 .../fernbridge_building_society/tasks.yaml | 0 .../fernbridge_building_society/tools.yaml | 0 .../en_GB/bank/wrenfield_bank/corpus.yaml | 0 .../bank/wrenfield_bank/institution_meta.yaml | 0 .../en_GB/bank/wrenfield_bank/tasks.yaml | 0 .../en_GB/bank/wrenfield_bank/tools.yaml | 0 .../brokerage/copperleaf_invest/corpus.yaml | 0 .../copperleaf_invest/institution_meta.yaml | 0 .../brokerage/copperleaf_invest/tasks.yaml | 0 .../brokerage/copperleaf_invest/tools.yaml | 0 .../financial_services/en_GB/dynamic.yaml | 0 .../insurer/kingsmere_insurance/corpus.yaml | 0 .../kingsmere_insurance/institution_meta.yaml | 0 .../insurer/kingsmere_insurance/tasks.yaml | 0 .../insurer/kingsmere_insurance/tools.yaml | 0 .../en_GB/neobank/quill/corpus.yaml | 0 .../en_GB/neobank/quill/institution_meta.yaml | 0 .../en_GB/neobank/quill/tasks.yaml | 0 .../en_GB/neobank/quill/tools.yaml | 0 .../financial_services/en_GB/region_meta.yaml | 0 .../oakhaven_pensions/corpus.yaml | 0 .../oakhaven_pensions/institution_meta.yaml | 0 .../oakhaven_pensions/tasks.yaml | 0 .../oakhaven_pensions/tools.yaml | 0 .../en_IN/_generation_report.json | 0 .../en_IN/bank/chandrika_bank/corpus.yaml | 0 .../bank/chandrika_bank/institution_meta.yaml | 0 .../en_IN/bank/chandrika_bank/tasks.yaml | 0 .../en_IN/bank/chandrika_bank/tools.yaml | 0 .../en_IN/brokerage/tarang_invest/corpus.yaml | 0 .../tarang_invest/institution_meta.yaml | 0 .../en_IN/brokerage/tarang_invest/tasks.yaml | 0 .../en_IN/brokerage/tarang_invest/tools.yaml | 0 .../financial_services/en_IN/dynamic.yaml | 0 .../insurer/abhaya_insurance/corpus.yaml | 0 .../abhaya_insurance/institution_meta.yaml | 0 .../en_IN/insurer/abhaya_insurance/tasks.yaml | 0 .../en_IN/insurer/abhaya_insurance/tools.yaml | 0 .../en_IN/nbfc/kalash_finserv/corpus.yaml | 0 .../nbfc/kalash_finserv/institution_meta.yaml | 0 .../en_IN/nbfc/kalash_finserv/tasks.yaml | 0 .../en_IN/nbfc/kalash_finserv/tools.yaml | 0 .../en_IN/neobank/paysetu/corpus.yaml | 0 .../neobank/paysetu/institution_meta.yaml | 0 .../en_IN/neobank/paysetu/tasks.yaml | 0 .../en_IN/neobank/paysetu/tools.yaml | 0 .../financial_services/en_IN/region_meta.yaml | 0 .../en_US/_generation_report.json | 0 .../en_US/bank/northwind_bank/corpus.yaml | 0 .../bank/northwind_bank/institution_meta.yaml | 0 .../en_US/bank/northwind_bank/tasks.yaml | 0 .../en_US/bank/northwind_bank/tools.yaml | 0 .../brokerage/solstice_invest/corpus.yaml | 0 .../solstice_invest/institution_meta.yaml | 0 .../brokerage/solstice_invest/tasks.yaml | 0 .../brokerage/solstice_invest/tools.yaml | 0 .../financial_services/en_US/dynamic.yaml | 0 .../financial_services/en_US/region_meta.yaml | 0 .../evergreen_retirement/corpus.yaml | 0 .../institution_meta.yaml | 0 .../evergreen_retirement/tasks.yaml | 0 .../evergreen_retirement/tools.yaml | 0 .../meridian_wealth/corpus.yaml | 0 .../meridian_wealth/institution_meta.yaml | 0 .../wealth_manager/meridian_wealth/tasks.yaml | 0 .../wealth_manager/meridian_wealth/tools.yaml | 0 .../fr_FR/_generation_report.json | 0 .../fr_FR/bank/credit_mirabelle/corpus.yaml | 0 .../credit_mirabelle/institution_meta.yaml | 0 .../fr_FR/bank/credit_mirabelle/tasks.yaml | 0 .../fr_FR/bank/credit_mirabelle/tools.yaml | 0 .../brokerage/valbreuil_bourse/corpus.yaml | 0 .../valbreuil_bourse/institution_meta.yaml | 0 .../brokerage/valbreuil_bourse/tasks.yaml | 0 .../brokerage/valbreuil_bourse/tools.yaml | 0 .../financial_services/fr_FR/dynamic.yaml | 0 .../insurer/egide_assurances/corpus.yaml | 0 .../egide_assurances/institution_meta.yaml | 0 .../fr_FR/insurer/egide_assurances/tasks.yaml | 0 .../fr_FR/insurer/egide_assurances/tools.yaml | 0 .../fr_FR/nbfc/ondine_credit/corpus.yaml | 0 .../nbfc/ondine_credit/institution_meta.yaml | 0 .../fr_FR/nbfc/ondine_credit/tasks.yaml | 0 .../fr_FR/nbfc/ondine_credit/tools.yaml | 0 .../fr_FR/neobank/kiosque/corpus.yaml | 0 .../neobank/kiosque/institution_meta.yaml | 0 .../fr_FR/neobank/kiosque/tasks.yaml | 0 .../fr_FR/neobank/kiosque/tools.yaml | 0 .../financial_services/fr_FR/region_meta.yaml | 0 .../hi_Deva_IN/_generation_report.json | 0 .../bank/chandrika_bank/corpus.yaml | 0 .../bank/chandrika_bank/institution_meta.yaml | 0 .../hi_Deva_IN/bank/chandrika_bank/tools.yaml | 0 .../brokerage/tarang_invest/corpus.yaml | 0 .../tarang_invest/institution_meta.yaml | 0 .../brokerage/tarang_invest/tools.yaml | 0 .../insurer/abhaya_insurance/corpus.yaml | 0 .../abhaya_insurance/institution_meta.yaml | 0 .../insurer/abhaya_insurance/tools.yaml | 0 .../nbfc/kalash_finserv/corpus.yaml | 0 .../nbfc/kalash_finserv/institution_meta.yaml | 0 .../hi_Deva_IN/nbfc/kalash_finserv/tools.yaml | 0 .../hi_Deva_IN/neobank/paysetu/corpus.yaml | 0 .../neobank/paysetu/institution_meta.yaml | 0 .../hi_Deva_IN/neobank/paysetu/tools.yaml | 0 .../hi_Deva_IN/region_meta.yaml | 0 .../ja_JP/_generation_report.json | 0 .../ja_JP/bank/asanagi_bank/corpus.yaml | 0 .../bank/asanagi_bank/institution_meta.yaml | 0 .../ja_JP/bank/asanagi_bank/tasks.yaml | 0 .../ja_JP/bank/asanagi_bank/tools.yaml | 0 .../brokerage/hoshimi_securities/corpus.yaml | 0 .../hoshimi_securities/institution_meta.yaml | 0 .../brokerage/hoshimi_securities/tasks.yaml | 0 .../brokerage/hoshimi_securities/tools.yaml | 0 .../financial_services/ja_JP/dynamic.yaml | 0 .../ja_JP/insurer/komorebi_life/corpus.yaml | 0 .../komorebi_life/institution_meta.yaml | 0 .../ja_JP/insurer/komorebi_life/tasks.yaml | 0 .../ja_JP/insurer/komorebi_life/tools.yaml | 0 .../ja_JP/nbfc/tsumugi_card/corpus.yaml | 0 .../nbfc/tsumugi_card/institution_meta.yaml | 0 .../ja_JP/nbfc/tsumugi_card/tasks.yaml | 0 .../ja_JP/nbfc/tsumugi_card/tools.yaml | 0 .../ja_JP/neobank/sorairo_bank/corpus.yaml | 0 .../sorairo_bank/institution_meta.yaml | 0 .../ja_JP/neobank/sorairo_bank/tasks.yaml | 0 .../ja_JP/neobank/sorairo_bank/tools.yaml | 0 .../financial_services/ja_JP/region_meta.yaml | 0 .../ko_KR/_generation_report.json | 0 .../ko_KR/bank/yunseul_bank/corpus.yaml | 0 .../bank/yunseul_bank/institution_meta.yaml | 0 .../ko_KR/bank/yunseul_bank/tasks.yaml | 0 .../ko_KR/bank/yunseul_bank/tools.yaml | 0 .../brokerage/garam_securities/corpus.yaml | 0 .../garam_securities/institution_meta.yaml | 0 .../brokerage/garam_securities/tasks.yaml | 0 .../brokerage/garam_securities/tools.yaml | 0 .../financial_services/ko_KR/dynamic.yaml | 0 .../ko_KR/insurer/dodam_life/corpus.yaml | 0 .../insurer/dodam_life/institution_meta.yaml | 0 .../ko_KR/insurer/dodam_life/tasks.yaml | 0 .../ko_KR/insurer/dodam_life/tools.yaml | 0 .../ko_KR/nbfc/yeoul_card/corpus.yaml | 0 .../nbfc/yeoul_card/institution_meta.yaml | 0 .../ko_KR/nbfc/yeoul_card/tasks.yaml | 0 .../ko_KR/nbfc/yeoul_card/tools.yaml | 0 .../ko_KR/neobank/mirinae_bank/corpus.yaml | 0 .../mirinae_bank/institution_meta.yaml | 0 .../ko_KR/neobank/mirinae_bank/tasks.yaml | 0 .../ko_KR/neobank/mirinae_bank/tools.yaml | 0 .../financial_services/ko_KR/region_meta.yaml | 0 .../financial_services/bank.py} | 0 .../financial_services}/embeddings.py | 0 .../{generator.py => probe.py} | 29 ++++---- .../probes/financial_services/retrieval.py | 2 +- .../financial_services/scorer.py} | 12 ++-- .../financial_services/task_derivation.py | 6 +- .../financial_services/tasks.py} | 2 +- .../engine/probes/financial_services/tools.py | 4 +- src/usersim/selection/schemas.py | 2 +- src/usersim/taxonomy/capabilities.py | 10 +-- tests/conftest.py | 4 +- tests/engine/core/test_assets_discovery.py | 4 +- tests/probes/financial_services/__init__.py | 2 + .../financial_services/test_e2e.py} | 8 +-- .../financial_services/test_probe.py} | 72 ++++++++++--------- tests/test_asset_gen_pipeline.py | 10 +-- tests/test_asset_gen_validate.py | 4 +- tests/test_cli.py | 2 +- tests/test_probe_names_in_shared_code.py | 2 - tests/test_selection.py | 2 +- 213 files changed, 137 insertions(+), 129 deletions(-) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/SCHEMA.md (99%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/_generation_report.json (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/bank/sparkasse_lindenhain/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/bank/sparkasse_lindenhain/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/bank/sparkasse_lindenhain/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/bank/sparkasse_lindenhain/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/brokerage/kranich_invest/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/brokerage/kranich_invest/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/brokerage/kranich_invest/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/brokerage/kranich_invest/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/dynamic.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/insurer/hanseatica_versicherung/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/insurer/hanseatica_versicherung/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/insurer/hanseatica_versicherung/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/insurer/hanseatica_versicherung/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/neobank/kompass_direktbank/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/neobank/kompass_direktbank/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/neobank/kompass_direktbank/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/neobank/kompass_direktbank/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/de_DE/region_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/_generation_report.json (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/bank/fernbridge_building_society/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/bank/fernbridge_building_society/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/bank/fernbridge_building_society/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/bank/fernbridge_building_society/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/bank/wrenfield_bank/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/bank/wrenfield_bank/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/bank/wrenfield_bank/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/bank/wrenfield_bank/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/brokerage/copperleaf_invest/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/brokerage/copperleaf_invest/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/brokerage/copperleaf_invest/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/brokerage/copperleaf_invest/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/dynamic.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/insurer/kingsmere_insurance/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/insurer/kingsmere_insurance/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/insurer/kingsmere_insurance/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/insurer/kingsmere_insurance/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/neobank/quill/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/neobank/quill/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/neobank/quill/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/neobank/quill/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/region_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/_generation_report.json (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/bank/chandrika_bank/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/bank/chandrika_bank/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/bank/chandrika_bank/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/bank/chandrika_bank/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/brokerage/tarang_invest/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/brokerage/tarang_invest/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/brokerage/tarang_invest/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/brokerage/tarang_invest/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/dynamic.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/insurer/abhaya_insurance/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/insurer/abhaya_insurance/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/insurer/abhaya_insurance/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/insurer/abhaya_insurance/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/nbfc/kalash_finserv/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/nbfc/kalash_finserv/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/nbfc/kalash_finserv/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/nbfc/kalash_finserv/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/neobank/paysetu/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/neobank/paysetu/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/neobank/paysetu/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/neobank/paysetu/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_IN/region_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/_generation_report.json (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/bank/northwind_bank/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/bank/northwind_bank/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/bank/northwind_bank/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/bank/northwind_bank/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/brokerage/solstice_invest/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/brokerage/solstice_invest/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/brokerage/solstice_invest/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/brokerage/solstice_invest/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/dynamic.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/region_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/retirement_provider/evergreen_retirement/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/retirement_provider/evergreen_retirement/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/wealth_manager/meridian_wealth/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/wealth_manager/meridian_wealth/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/wealth_manager/meridian_wealth/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/en_US/wealth_manager/meridian_wealth/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/_generation_report.json (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/bank/credit_mirabelle/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/bank/credit_mirabelle/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/bank/credit_mirabelle/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/bank/credit_mirabelle/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/dynamic.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/insurer/egide_assurances/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/insurer/egide_assurances/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/insurer/egide_assurances/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/insurer/egide_assurances/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/nbfc/ondine_credit/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/nbfc/ondine_credit/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/nbfc/ondine_credit/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/nbfc/ondine_credit/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/neobank/kiosque/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/neobank/kiosque/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/neobank/kiosque/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/neobank/kiosque/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/fr_FR/region_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/_generation_report.json (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/neobank/paysetu/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/neobank/paysetu/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/neobank/paysetu/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/hi_Deva_IN/region_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/_generation_report.json (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/bank/asanagi_bank/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/bank/asanagi_bank/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/bank/asanagi_bank/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/bank/asanagi_bank/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/brokerage/hoshimi_securities/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/brokerage/hoshimi_securities/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/brokerage/hoshimi_securities/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/brokerage/hoshimi_securities/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/dynamic.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/insurer/komorebi_life/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/insurer/komorebi_life/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/insurer/komorebi_life/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/insurer/komorebi_life/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/nbfc/tsumugi_card/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/nbfc/tsumugi_card/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/nbfc/tsumugi_card/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/nbfc/tsumugi_card/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/neobank/sorairo_bank/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/neobank/sorairo_bank/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/neobank/sorairo_bank/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/neobank/sorairo_bank/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ja_JP/region_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/_generation_report.json (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/bank/yunseul_bank/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/bank/yunseul_bank/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/bank/yunseul_bank/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/bank/yunseul_bank/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/brokerage/garam_securities/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/brokerage/garam_securities/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/brokerage/garam_securities/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/brokerage/garam_securities/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/dynamic.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/insurer/dodam_life/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/insurer/dodam_life/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/insurer/dodam_life/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/insurer/dodam_life/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/nbfc/yeoul_card/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/nbfc/yeoul_card/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/nbfc/yeoul_card/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/nbfc/yeoul_card/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/neobank/mirinae_bank/corpus.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/neobank/mirinae_bank/institution_meta.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/neobank/mirinae_bank/tasks.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/neobank/mirinae_bank/tools.yaml (100%) rename src/usersim/engine/{ => probes/financial_services}/assets/financial_services/ko_KR/region_meta.yaml (100%) rename src/usersim/engine/{core/finance_bank.py => probes/financial_services/bank.py} (100%) rename src/usersim/engine/{core => probes/financial_services}/embeddings.py (100%) rename src/usersim/engine/probes/financial_services/{generator.py => probe.py} (99%) rename src/usersim/engine/{evaluator/scorers/financial_services.py => probes/financial_services/scorer.py} (98%) rename src/usersim/engine/{core/finance_tasks.py => probes/financial_services/tasks.py} (99%) create mode 100644 tests/probes/financial_services/__init__.py rename tests/{test_financial_services_e2e.py => probes/financial_services/test_e2e.py} (98%) rename tests/{engine/probes/test_financial_services.py => probes/financial_services/test_probe.py} (98%) diff --git a/.gitignore b/.gitignore index a8328023..9d839e8d 100644 --- a/.gitignore +++ b/.gitignore @@ -43,10 +43,10 @@ data/ artifacts/ output/ # Dense-retrieval vectors `gen-assets` writes beside a bank stay local. -src/usersim/engine/assets/financial_services/**/embeddings.parquet +src/usersim/engine/probes/financial_services/assets/financial_services/**/embeddings.parquet # So do the judge scorecards `evaluate-assets` writes there. -src/usersim/engine/assets/financial_services/**/_eval_scorecard.parquet -src/usersim/engine/assets/financial_services/**/_scorecards/ +src/usersim/engine/probes/financial_services/assets/financial_services/**/_eval_scorecard.parquet +src/usersim/engine/probes/financial_services/assets/financial_services/**/_scorecards/ # IDE .vscode/ diff --git a/architecture/overview.md b/architecture/overview.md index 80abdbb0..890dd997 100644 --- a/architecture/overview.md +++ b/architecture/overview.md @@ -84,11 +84,12 @@ throughout, so a name tells you which one you are reading about. src/usersim/ ├── cli/ console driver; models_*.toml catalogs ├── engine/ -│ ├── core/ the conversation loop, probe base classes, banks, -│ │ identity, outcomes, storage +│ ├── core/ the conversation loop, probe base classes, asset lookup, +│ │ shared bank loaders, identity, outcomes, storage │ ├── evaluator/ judges, axes, and the scorer registry -│ ├── probes/ the shipped probes -│ ├── assets/ per-probe YAML and parquet banks +│ ├── probes/ the shipped probes, most with their banks, loaders and +│ │ scorer in their own folder +│ ├── assets/ the banks of the probes that keep none of their own │ └── plugin.py the two Data Designer entry points ├── reporting/ capability dashboard and model comparison ├── selection/ trajectory curation and export diff --git a/docs/engine/AUTHORING_A_PROBE.md b/docs/engine/AUTHORING_A_PROBE.md index bb6d88e0..fa76a9de 100644 --- a/docs/engine/AUTHORING_A_PROBE.md +++ b/docs/engine/AUTHORING_A_PROBE.md @@ -258,10 +258,11 @@ It must be absolute: `@register_probe` refuses a relative path, which would resolve against the working directory. In this repository the banks are then at `src/usersim/engine/probes/