Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions docs/engine/AUTHORING_A_PROBE.md
Original file line number Diff line number Diff line change
Expand Up @@ -563,7 +563,8 @@ each name it takes from another `usersim` module is an import to change if it
ever leaves this repository.
`tests/test_probe_private_imports.py` counts the names each built-in package
imports from other `usersim` modules and fails when the count grows; a new
package adds its count to that test's `_ALLOWED`.
package adds its count to that test's `_ALLOWED`, which is 0 for a package that
imports everything from `usersim.extension`.

### Declare every column your probe writes

Expand Down Expand Up @@ -823,7 +824,6 @@ import random
from pathlib import Path
from typing import Any

from usersim.engine.core.probes import dispatch_episode
from usersim.engine.probes.bookings.bank import LOCALES, Booking, draw_booking, read_booking_bank
from usersim.engine.probes.bookings.prompts import BOOKINGS_USER_PROMPTS
from usersim.extension import (
Expand All @@ -832,6 +832,7 @@ from usersim.extension import (
Provenance,
ToolExecutionMixin,
WarningKind,
dispatch_episode,
format_persona_for_prompt,
persona_uuid,
register_probe,
Expand Down
14 changes: 14 additions & 0 deletions docs/extension_api.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ does not load the conversation loop.
| Name | What it is |
|---|---|
| `ConversationState` | The conversation so far, as the hooks receive it, with its messages, metadata and `OutcomeBuilder` |
| `dispatch_episode` | Runs one episode through a probe's `run_dispatch` with the run's per-episode settings installed, as `usersim simulate` runs it |
| `make_result` | Builds the row a conversation produces, for a probe that writes its own `run_dispatch` |
| `make_failed` | Builds the row for a conversation that failed |
| `finalize_completed` | Finishes a conversation that ran to its end, failing it when the probe did not complete its protocol |
Expand Down Expand Up @@ -88,11 +89,23 @@ does not load the conversation loop.
| `load_prompt_file` | A probe's `prompts.yaml` as a mapping, read from the folder its banks resolve to; empty when there is none |
| `get_prompt` | One prompt from a probe's `prompts.yaml`, or the default it is given when the file or the key is missing |

### Banks and assets

| Name | What it is |
|---|---|
| `probe_assets_dir` | The folder a probe's assets resolve to: the first override root that holds them, else the root its class declares |
| `load_seeds` | A probe's seeds of one kind for a locale, such as its topics, from `<probe>/<locale>/<kind>.json`, or the `en_US` file when the locale has none |
| `LocalizedText` | Bank text with a rendering per locale, read with `for_locale`, which falls back to `en_US` |
| `LocalizedTextError` | Raised when a bank value cannot be read as `LocalizedText` |
| `persona_to_tags` | The tags a persona matches bank entries on, such as its region, age and occupation, for a locale |

### Locales and languages

| Name | What it is |
|---|---|
| `SHIPPED_LOCALES` | The locales User Sim ships |
| `asset_locale` | The locale whose banks a conversation locale reads: an India language variant reads its base locale's unless it has its own |
| `india_variant` | The India language variant a locale is, with its base locale, language and script, or None for any other locale |
| `get_conversation_language` | The language a locale's conversations are in |
| `expected_language_display` | A locale's language as a label for prompts |
| `language_instruction` | The prompt block telling a model which language, and for some locales which script, to write in |
Expand All @@ -113,6 +126,7 @@ does not load the conversation loop.
| `ScorerRegistrationError` | Raised when `register_scorer` refuses a name |
| `is_missing` | Whether a table cell is empty: None, NaN, `pd.NA` or `pd.NaT` |
| `none_for_missing` | A copy of a row with every empty cell as None |
| `recorded_value` | A value a probe recorded for a row, from the row or from its `conversation_metadata` |

### Capabilities, asset domains and backends

Expand Down
4 changes: 2 additions & 2 deletions src/usersim/engine/probes/financial_services/bank.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
Sequence,
)

from usersim.engine.core.locale import india_variant
from usersim.extension import india_variant

logger = logging.getLogger("usersim.engine")

Expand Down Expand Up @@ -707,7 +707,7 @@ def _load_embeddings(path: Path) -> dict[str, tuple[float, ...]]:


def default_finance_bank_dir(locale: str) -> Path:
from usersim.engine.core._assets import probe_assets_dir
from usersim.extension import probe_assets_dir

return probe_assets_dir("financial_services") / locale

Expand Down
63 changes: 27 additions & 36 deletions src/usersim/engine/probes/financial_services/probe.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,42 +37,8 @@
from pathlib import Path
from typing import Any, Sequence

from usersim.engine.core.behavioral import (
format_behavioral_profile_for_prompt,
format_disclosure_instructions,
format_interaction_style_instructions,
)
from usersim.engine.core.identity import persona_uuid as compute_persona_uuid
from usersim.engine.core.locale import (
expected_language_display,
persona_dataset_locale,
)
from usersim.engine.core.outcomes import (
FailureAttribution,
FailureClass,
OutcomeBuilder,
OutcomeStatus,
Provenance,
SimulationTrace,
TraceKind,
WarningKind,
)
from usersim.engine.core.persona import format_persona_for_prompt, persona_full_name
from usersim.engine.core.probes import (
BankBackedProbe,
BankLoadError,
ToolExecutionMixin,
dispatch_episode,
register_probe,
)
from usersim.engine.core.locale import persona_dataset_locale
from usersim.engine.core.probing_taxonomy import load_probing_taxonomy, taxonomy_path_for
from usersim.engine.core.simulation import (
ConversationState,
make_failed,
)
from usersim.engine.core.simulation import (
language_instruction as _language_instruction,
)
from usersim.engine.probes.financial_services import retrieval as _retrieval
from usersim.engine.probes.financial_services import tools as _tools
from usersim.engine.probes.financial_services.bank import (
Expand All @@ -97,6 +63,31 @@
derive_instance,
)
from usersim.engine.probes.financial_services.tasks import FinanceInstance
from usersim.extension import (
BankBackedProbe,
BankLoadError,
ConversationState,
FailureAttribution,
FailureClass,
OutcomeBuilder,
OutcomeStatus,
Provenance,
SimulationTrace,
ToolExecutionMixin,
TraceKind,
WarningKind,
dispatch_episode,
expected_language_display,
format_behavioral_profile_for_prompt,
format_disclosure_instructions,
format_interaction_style_instructions,
format_persona_for_prompt,
make_failed,
persona_full_name,
register_probe,
)
from usersim.extension import language_instruction as _language_instruction
from usersim.extension import persona_uuid as compute_persona_uuid

logger = logging.getLogger("usersim.engine")

Expand Down Expand Up @@ -1158,7 +1149,7 @@ async def simulate_financial_services(
**kwargs: Any,
) -> dict[str, Any]:
"""Thin shim: catch construction errors -> structured SCENARIO_ABORTED."""
from usersim.engine.core.probes import BankLoadError
from usersim.extension import BankLoadError

provenance = kwargs.get("provenance") or Provenance()
outcome_builder = kwargs.get("outcome_builder") or OutcomeBuilder(
Expand Down
4 changes: 1 addition & 3 deletions src/usersim/engine/probes/financial_services/scorer.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,7 @@
create_judge_structured_output_model,
)

from usersim.engine.core.llm import acall_llm
from usersim.engine.core.messages import format_conversation_history_for_prompt
from usersim.engine.evaluator.scorers import register_scorer
from usersim.extension import acall_llm, format_conversation_history_for_prompt, register_scorer

logger = logging.getLogger("usersim.engine")

Expand Down
26 changes: 15 additions & 11 deletions src/usersim/engine/probes/general_educational/probe.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,27 @@
from pathlib import Path
from typing import Any

from usersim.engine.core.behavioral import (
format_behavioral_profile_for_prompt,
format_disclosure_instructions,
format_interaction_style_instructions,
)
from usersim.engine.core.locale import SHIPPED_LOCALES
from usersim.engine.core.persona import format_persona_for_prompt
from usersim.engine.core.probes import BaseProbe, PreviewField, dispatch_episode, register_probe
from usersim.engine.core.prompt_loader import load_prompt_file, render_prompt
from usersim.engine.core.seeds import load_seeds
from usersim.engine.core.simulation import language_instruction
from usersim.engine.probes.general_educational.prompts import (
_SCENARIO,
assistant_system_prompt,
user_agent_system_prompt,
user_judge_turn_prompt,
)
from usersim.extension import (
SHIPPED_LOCALES,
BaseProbe,
PreviewField,
dispatch_episode,
format_behavioral_profile_for_prompt,
format_disclosure_instructions,
format_interaction_style_instructions,
format_persona_for_prompt,
language_instruction,
load_prompt_file,
load_seeds,
register_probe,
render_prompt,
)

# ── Probe metadata ──────────────────────────────────────────────────
# Bumping ``PROMPT_VERSION`` invalidates prior evaluator rows on
Expand Down
2 changes: 1 addition & 1 deletion src/usersim/engine/probes/general_educational/prompts.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
falling back to the compiled-in defaults below.
"""

from usersim.engine.core.prompt_loader import get_prompt
from usersim.extension import get_prompt

_SCENARIO = "general_educational"

Expand Down
26 changes: 15 additions & 11 deletions src/usersim/engine/probes/general_open_ended/probe.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,27 @@
from pathlib import Path
from typing import Any

from usersim.engine.core.behavioral import (
format_behavioral_profile_for_prompt,
format_disclosure_instructions,
format_interaction_style_instructions,
)
from usersim.engine.core.locale import SHIPPED_LOCALES
from usersim.engine.core.persona import format_persona_for_prompt
from usersim.engine.core.probes import BaseProbe, PreviewField, dispatch_episode, register_probe
from usersim.engine.core.prompt_loader import load_prompt_file, render_prompt
from usersim.engine.core.seeds import load_seeds
from usersim.engine.core.simulation import language_instruction
from usersim.engine.probes.general_open_ended.prompts import (
_SCENARIO,
assistant_system_prompt,
user_agent_system_prompt,
user_judge_turn_prompt,
)
from usersim.extension import (
SHIPPED_LOCALES,
BaseProbe,
PreviewField,
dispatch_episode,
format_behavioral_profile_for_prompt,
format_disclosure_instructions,
format_interaction_style_instructions,
format_persona_for_prompt,
language_instruction,
load_prompt_file,
load_seeds,
register_probe,
render_prompt,
)

# ── Probe metadata ──────────────────────────────────────────────────
# Bumping ``PROMPT_VERSION`` invalidates prior evaluator rows on
Expand Down
2 changes: 1 addition & 1 deletion src/usersim/engine/probes/general_open_ended/prompts.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
falling back to the compiled-in defaults below.
"""

from usersim.engine.core.prompt_loader import get_prompt
from usersim.extension import get_prompt

_SCENARIO = "general_open_ended"

Expand Down
2 changes: 1 addition & 1 deletion src/usersim/engine/probes/health_disclosure/bank.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ def load_clinical_profile_bank(path: str | Path) -> ClinicalProfileBank:
# ---------------------------------------------------------------------------
def default_clinical_profile_bank_path(client: str) -> Path:
"""Bundled bank path: ``assets/<client>/sample.yaml`` (locale-independent)."""
from usersim.engine.core._assets import probe_assets_dir
from usersim.extension import probe_assets_dir

if client not in CLIENT_PROBE_LABELS:
raise ClinicalProfileBankError(f"unknown client probe label: {client!r}")
Expand Down
18 changes: 9 additions & 9 deletions src/usersim/engine/probes/health_disclosure/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,6 @@
from pathlib import Path
from typing import Any, Mapping

from usersim.engine.core.behavioral import (
format_behavioral_profile_for_prompt,
format_disclosure_instructions,
format_interaction_style_instructions,
)
from usersim.engine.core.outcomes import WarningKind
from usersim.engine.core.persona import format_persona_for_prompt
from usersim.engine.core.probes import BankBackedProbe
from usersim.engine.core.simulation import language_instruction
from usersim.engine.probes.health_disclosure.bank import (
DEFAULT_PROFILE_ID,
ClinicalProfile,
Expand All @@ -39,6 +30,15 @@
GuardedMoveMixin,
env_override,
)
from usersim.extension import (
BankBackedProbe,
WarningKind,
format_behavioral_profile_for_prompt,
format_disclosure_instructions,
format_interaction_style_instructions,
format_persona_for_prompt,
language_instruction,
)

# Local-dev override for the guarded move-space (primary switch is the ``guarded``
# probe_variant; see GuardedMoveMixin). Off/unset → driven by the variant.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@

from __future__ import annotations

from usersim.engine.core.probes import register_probe
from usersim.engine.probes.health_disclosure.bank import (
load_clinical_profile_bank_for_client,
)
from usersim.engine.probes.health_disclosure.base import HealthDisclosureProbe
from usersim.engine.probes.health_disclosure.clients import CLIENTS
from usersim.extension import register_probe

_CFG = CLIENTS["health_decision_support_disclosure"]
_LABEL = "health_decision_support_disclosure"
Expand Down
2 changes: 1 addition & 1 deletion src/usersim/engine/probes/health_disclosure/general.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@

from __future__ import annotations

from usersim.engine.core.probes import register_probe
from usersim.engine.probes.health_disclosure.bank import (
load_clinical_profile_bank_for_client,
)
from usersim.engine.probes.health_disclosure.base import HealthDisclosureProbe
from usersim.engine.probes.health_disclosure.clients import CLIENTS
from usersim.extension import register_probe

_CFG = CLIENTS["health_general_disclosure"]
_LABEL = "health_general_disclosure"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,6 @@
import os
from typing import Any, Callable

from usersim.engine.core.llm import acall_llm
from usersim.engine.core.tool_calls import recover_tool_call
from usersim.engine.probes.health_disclosure import clients as C
from usersim.engine.probes.health_disclosure.moves import (
AFFECTS,
Expand All @@ -45,6 +43,7 @@
PatientEnv,
)
from usersim.engine.probes.health_disclosure.prompts import PROPOSAL_SYSTEM_PACK
from usersim.extension import acall_llm, recover_tool_call

MODEL_USER = "user_model"

Expand Down
3 changes: 1 addition & 2 deletions src/usersim/engine/probes/health_disclosure/prompts.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,7 @@
completeness.
"""

from usersim.engine.core.locale import SHIPPED_LOCALES
from usersim.engine.core.probes import LocalePromptPack
from usersim.extension import SHIPPED_LOCALES, LocalePromptPack


def _english_scaffold_pack(label: str, template: str) -> LocalePromptPack:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,7 @@
import os
from typing import Any

from usersim.engine.core.llm import acall_llm
from usersim.engine.core.locale import SHIPPED_LOCALES
from usersim.engine.core.probes import LocalePromptPack
from usersim.engine.core.tool_calls import recover_tool_call
from usersim.extension import SHIPPED_LOCALES, LocalePromptPack, acall_llm, recover_tool_call

logger = logging.getLogger("usersim.engine")

Expand Down
3 changes: 1 addition & 2 deletions src/usersim/engine/probes/health_disclosure/scorer.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,14 +88,13 @@
import logging
from typing import Any

from usersim.engine.core.missing import is_missing
from usersim.engine.evaluator.scorers import register_scorer
from usersim.engine.probes.health_disclosure.realized_audit import (
DISCLOSURE_CREDIT,
pair_realized_turns,
reconcile_realized,
verify_realized_transcript,
)
from usersim.extension import is_missing, register_scorer

logger = logging.getLogger("usersim.engine")

Expand Down
Loading
Loading