diff --git a/docs/engine/AUTHORING_A_PROBE.md b/docs/engine/AUTHORING_A_PROBE.md index 767a46c..1327fac 100644 --- a/docs/engine/AUTHORING_A_PROBE.md +++ b/docs/engine/AUTHORING_A_PROBE.md @@ -559,6 +559,13 @@ After this, add your module's import path to `BUILTIN_PROBE_MODULES` in The registry imports every listed module the first time a probe is looked up, and that import is what runs the `@register_probe` decorator. +A built-in imports what it can from `usersim.extension`, as an extension does: +each name it takes from another `usersim` module is an import to change if it +ever leaves this repository. +`tests/test_probe_private_imports.py` counts the names each built-in package +imports from other `usersim` modules and fails when the count grows; a new +package adds its count to that test's `_ALLOWED`. + ### Declare every column your probe writes If `build_result_extras` adds a key beyond the core columns diff --git a/docs/engine/README.md b/docs/engine/README.md index 63d8ad2..1860fd6 100644 --- a/docs/engine/README.md +++ b/docs/engine/README.md @@ -174,7 +174,7 @@ fields each scorer reads: `ground_truth` / `false_premises` for `safety_chat_pressure`, and `expected_behavior` plus the action taxonomy for `safety_agentic`. -The three sovereign-AI probes and the two safety probes share one persona-tag vocabulary: `sov_ai_facts.task_derivation.persona_to_tags(...)` is the source of truth and is re-exported by `sov_ai_dynamic.task_derivation` (as `persona_to_interest_tags(...)` for clarity), `sov_ai_multilingual_parity.task_derivation`, `safety_chat_pressure.task_derivation`, and `safety_agentic.task_derivation`. A persona feature such as `age:25-44` or `education:tertiary` therefore matches bank entries the same way in all five probes. +The three sovereign-AI probes and the two safety probes share one persona-tag vocabulary: `core/persona_tags.py`'s `persona_to_tags(...)` is the source of truth, and `sov_ai_facts.task_derivation`, `sov_ai_dynamic.task_derivation` (as `persona_to_interest_tags(...)` for clarity), `sov_ai_multilingual_parity.task_derivation`, `safety_chat_pressure.task_derivation` and `safety_agentic.task_derivation` all use it. A persona feature such as `age:25-44` or `education:tertiary` therefore matches bank entries the same way in all five probes. ### India language-variant resolution @@ -526,6 +526,7 @@ what each one carries: |--------|---------| | `core/behavioral.py` | OCEAN → BehavioralProfile (patience, verbosity, cooperativeness, tech_literacy, error_proneness); disclosure style, interaction style, reactive-frustration computation | | `core/persona.py` | Persona formatting for prompts (deterministic shuffle) | +| `core/persona_tags.py` | The persona tags (region, age, education, occupation, interests) the sov_ai and safety probes match bank entries by | | `core/locale.py` | Locale → language + expected-script-ranges mapping (single source of truth for shipped locales) | | `core/localized.py` | `LocalizedText` wrapper for asset-bank fields with per-locale renderings (`for_locale(locale)` lookup with `en_US` fallback) | | `core/prompt_loader.py` | Reads a probe's `prompts.yaml` from the folder its banks resolve to | @@ -594,6 +595,7 @@ The tables describe the main files. | `test_generator.py` | Dispatcher failed-result structure, probe-variant resolution, structured `SCENARIO_ABORTED` outcomes | | `test_messages.py` | Message / tool / theme formatting helpers | | `test_persona.py` | Persona formatting (deterministic shuffle) | +| `test_persona_tags.py` | The tags `persona_to_tags` gives a persona: regions for `pt_BR`, `en_GB` and `de_DE`, education levels, occupations, and the fallbacks for missing fields and other locales | | `test_preview.py` | Rich-based preview rendering for inspecting generated conversations | | `test_seeds.py` | Seed loading by locale, and the `en_US` fallback | | `test_multi_locale.py` | Multi-locale config, seed loading, language mapping | @@ -628,7 +630,7 @@ The tables describe the main files. | Test File | Coverage | |-----------|----------| -| `sov_ai_facts/test_probe.py` | `sov_ai_facts` probe: persona→tag, deterministic `derive_task` (bank-version salted), prompts, e2e with mocked LLM | +| `sov_ai_facts/test_probe.py` | `sov_ai_facts` probe: each region's facts, deterministic `derive_task` (bank-version salted), prompts, e2e with mocked LLM | | `sov_ai_facts/test_scorer.py` | `sov_ai_facts` out-of-sim scorer: per-fact LLM judge with question-type-specific axes, error paths, multi-fact aggregation | | `sov_ai_facts/test_bank.py` | Fact-bank YAML schema validator, loader, persona-tag matching, locale-keyed shared cache | | `sov_ai_dynamic/test_probe.py` | `sov_ai_dynamic` probe: persona→tag, deterministic `derive_probe` + hint rotation (taxonomy-version salted), all-locale prompt availability, e2e with mocked LLM | diff --git a/src/usersim/engine/core/persona_tags.py b/src/usersim/engine/core/persona_tags.py new file mode 100644 index 0000000..06d6115 --- /dev/null +++ b/src/usersim/engine/core/persona_tags.py @@ -0,0 +1,411 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The persona tags bank-backed probes match bank entries against. + +:func:`persona_to_tags` maps a Nemotron-Personas persona (region, age, +occupation, education, and topical interests inferred from occupation) to the +``:`` tags a bank entry lists in its ``persona_tags``. The +sovereign-AI and safety probes pick entries whose tags intersect the persona's, +so a persona matches entries the same way in every one of them. + +``pt_BR``, ``en_GB`` and ``de_DE`` personas also get a region tag (Brazilian +macro-region, UK nation, German Land); every locale gets the age, education, +occupation and interest tags. +""" + +from __future__ import annotations + +from typing import Any + +# --------------------------------------------------------------------------- +# Brazilian region classifier +# --------------------------------------------------------------------------- + +# Map each Brazilian state (uppercase 2-letter code OR full name, both +# lowercased for matching) to the IBGE macro-region. Reference: +# https://www.ibge.gov.br/geociencias/organizacao-do-territorio/estrutura-territorial.html +_BR_STATE_TO_REGION: dict[str, str] = { + # Norte + "am": "north", + "amazonas": "north", + "pa": "north", + "pará": "north", + "para": "north", + "ac": "north", + "acre": "north", + "ro": "north", + "rondônia": "north", + "rondonia": "north", + "rr": "north", + "roraima": "north", + "ap": "north", + "amapá": "north", + "amapa": "north", + "to": "north", + "tocantins": "north", + # Nordeste + "ba": "northeast", + "bahia": "northeast", + "pe": "northeast", + "pernambuco": "northeast", + "ce": "northeast", + "ceará": "northeast", + "ceara": "northeast", + "ma": "northeast", + "maranhão": "northeast", + "maranhao": "northeast", + "pb": "northeast", + "paraíba": "northeast", + "paraiba": "northeast", + "rn": "northeast", + "rio grande do norte": "northeast", + "al": "northeast", + "alagoas": "northeast", + "se": "northeast", + "sergipe": "northeast", + "pi": "northeast", + "piauí": "northeast", + "piaui": "northeast", + # Centro-Oeste + "go": "central-west", + "goiás": "central-west", + "goias": "central-west", + "mt": "central-west", + "mato grosso": "central-west", + "ms": "central-west", + "mato grosso do sul": "central-west", + "df": "central-west", + "distrito federal": "central-west", + # Sudeste + "sp": "southeast", + "são paulo": "southeast", + "sao paulo": "southeast", + "rj": "southeast", + "rio de janeiro": "southeast", + "mg": "southeast", + "minas gerais": "southeast", + "es": "southeast", + "espírito santo": "southeast", + "espirito santo": "southeast", + # Sul + "rs": "south", + "rio grande do sul": "south", + "sc": "south", + "santa catarina": "south", + "pr": "south", + "paraná": "south", + "parana": "south", +} + + +def _br_region_from_persona(persona: dict[str, Any]) -> str | None: + """Resolve a Brazilian macro-region tag from persona state fields.""" + # Nemotron-Personas-Brazil carries a ``state`` or ``state_abbrev`` + # field. Fall back to ``region`` if present. + for key in ("state_abbrev", "state", "region"): + raw = persona.get(key) + if isinstance(raw, str) and raw.strip(): + code = raw.strip().lower() + mapped = _BR_STATE_TO_REGION.get(code) + if mapped: + return mapped + return None + + +# --------------------------------------------------------------------------- +# UK nation classifier +# --------------------------------------------------------------------------- + +# A UK persona's ``region`` is one of the nine English regions or the name of +# one of the other three nations. Health, education and much of the law are +# devolved, so a fact that holds in one nation only carries that nation's tag. +_UK_ENGLISH_REGIONS = frozenset( + { + "north east", + "north west", + "yorkshire and the humber", + "east midlands", + "west midlands", + "east of england", + "london", + "south east", + "south west", + } +) +_UK_OTHER_NATIONS: dict[str, str] = { + "scotland": "scotland", + "wales": "wales", + "northern ireland": "northern-ireland", +} + + +def _uk_nation_from_persona(persona: dict[str, Any]) -> str | None: + """Resolve a UK nation tag (england, scotland, wales, northern-ireland) from the persona's region.""" + raw = persona.get("region") + if not isinstance(raw, str) or not raw.strip(): + return None + region = raw.strip().lower() + if region in _UK_ENGLISH_REGIONS: + return "england" + return _UK_OTHER_NATIONS.get(region) + + +# --------------------------------------------------------------------------- +# German Land classifier +# --------------------------------------------------------------------------- + +# A German persona's ``region`` is its Land, by its German name. Schools, +# policing, public holidays and several taxes are set by the Land, so a fact +# that holds in one Land only carries that Land's tag. Tags are ASCII. +_DE_LAENDER: dict[str, str] = { + "baden-württemberg": "baden-wuerttemberg", + "bayern": "bayern", + "berlin": "berlin", + "brandenburg": "brandenburg", + "bremen": "bremen", + "hamburg": "hamburg", + "hessen": "hessen", + "mecklenburg-vorpommern": "mecklenburg-vorpommern", + "niedersachsen": "niedersachsen", + "nordrhein-westfalen": "nordrhein-westfalen", + "rheinland-pfalz": "rheinland-pfalz", + "saarland": "saarland", + "sachsen": "sachsen", + "sachsen-anhalt": "sachsen-anhalt", + "schleswig-holstein": "schleswig-holstein", + "thüringen": "thueringen", +} + + +def _de_land_from_persona(persona: dict[str, Any]) -> str | None: + """Resolve a German Land tag (``bayern``, ``thueringen``, ...) from the persona's region.""" + raw = persona.get("region") + if not isinstance(raw, str) or not raw.strip(): + return None + return _DE_LAENDER.get(raw.strip().lower()) + + +# --------------------------------------------------------------------------- +# Age bin → tag +# --------------------------------------------------------------------------- + +_AGE_BINS: list[tuple] = [ + (0, 17, "age:under-18"), + (18, 24, "age:18-24"), + (25, 44, "age:25-44"), + (45, 64, "age:45-64"), + (65, 200, "age:65+"), +] + + +def _age_tag(persona: dict[str, Any]) -> str | None: + """Extract an age-bin tag from a persona's numeric age if possible.""" + raw = persona.get("age") + try: + age = int(raw) # type: ignore[arg-type] + except (TypeError, ValueError): + return None + for lo, hi, tag in _AGE_BINS: + if lo <= age <= hi: + return tag + return None + + +# --------------------------------------------------------------------------- +# Education level → tag +# --------------------------------------------------------------------------- + +_EDUCATION_MAP: dict[str, str] = { + # Map common Nemotron-Personas education_level values to our coarse + # tag vocabulary. Keys are lowercased substring matches. + "no schooling": "primary", + "primary": "primary", + "elementary": "primary", + "ensino fundamental": "primary", + "middle": "secondary", + "high school": "secondary", + "ensino médio": "secondary", + "ensino medio": "secondary", + "associate": "secondary", + "bachelor": "tertiary", + "bachelors": "tertiary", + "ensino superior": "tertiary", + "graduação": "tertiary", + "graduacao": "tertiary", + "master": "tertiary", + "doctor": "tertiary", + "phd": "tertiary", + "postgraduate": "tertiary", + "pós-graduação": "tertiary", + "pos-graduacao": "tertiary", + # en_GB. Not "degree", which en_IN's "diploma ... not equal to degree" contains. + "no qualifications": "primary", + "gcse": "secondary", + "apprenticeship": "secondary", + "a level": "secondary", + "higher education": "tertiary", + # de_DE records the vocational qualification: none (after general schooling) + # and a dual apprenticeship are secondary, and a Fachschule (Meister, + # Techniker) is tertiary. "Bachelor, Master, Diplom, Promotion" matches + # "bachelor"; "diplom" would also match en_US "9th_12th_no_diploma". + "ohne beruflichen bildungsabschluss": "secondary", + "berufsausbildung": "secondary", + "fachschul": "tertiary", +} + + +def _education_tag(persona: dict[str, Any]) -> str | None: + raw = persona.get("education_level") + if not isinstance(raw, str): + return None + needle = raw.strip().lower() + # Longest-match first so "masters degree" maps to tertiary via "master". + for key in sorted(_EDUCATION_MAP, key=len, reverse=True): + if key in needle: + return f"education:{_EDUCATION_MAP[key]}" + return None + + +# --------------------------------------------------------------------------- +# Occupation → occupation-family + interest inference +# --------------------------------------------------------------------------- + +# Occupation phrases (lowercased substring match) → (family, interests). +# Very coarse on purpose: the point is to seed interest tags, not to build a +# complete occupation taxonomy. Reviewers can extend this block per locale. +_OCCUPATION_MAP: list[tuple] = [ + # (substring, occupation-family, extra interests) + ("teacher", "teacher", ("interest:literature", "interest:history")), + ("professor", "teacher", ("interest:literature",)), + ("docente", "teacher", ("interest:literature",)), + ("student", "student", ("interest:literature",)), + ("estudante", "student", ("interest:literature",)), + ("nurse", "healthcare", ("interest:health",)), + ("enfermeir", "healthcare", ("interest:health",)), + ("doctor", "healthcare", ("interest:health",)), + ("médico", "healthcare", ("interest:health",)), + ("medico", "healthcare", ("interest:health",)), + ("entrepreneur", "small-business", ()), + ("empresári", "small-business", ()), + ("empreendedor", "small-business", ()), + ("self-employed", "small-business", ()), + ("shopkeeper", "small-business", ()), + ("comerciante", "small-business", ()), + ("engineer", "engineering", ("interest:geography",)), + ("engenheir", "engineering", ("interest:geography",)), + ("farmer", "agriculture", ("interest:geography",)), + ("agricultor", "agriculture", ("interest:geography",)), + ("journalist", "media", ("interest:history", "interest:literature")), + ("jornalista", "media", ("interest:history", "interest:literature")), + ("musician", "arts", ("interest:music",)), + ("músic", "arts", ("interest:music",)), + ("artist", "arts", ("interest:music",)), + ("artista", "arts", ("interest:music",)), + ("writer", "arts", ("interest:literature",)), + ("escritor", "arts", ("interest:literature",)), + ("retired", "retired", ()), + ("aposentad", "retired", ()), + ("homemaker", "homemaker", ()), + ("housewife", "homemaker", ()), + ("dona de casa", "homemaker", ()), + # de_DE (the English keys above already cover Professor and Journalist). + # Nursing is matched by its compounds, and " pflege" only where a word starts + # with it: a bare "pflege" would also catch cleaners, arborists, animal + # keepers and cosmetics. + ("kinderpfleger", "teacher", ("interest:education",)), + ("erzieher", "teacher", ("interest:education",)), + ("lehrer", "teacher", ("interest:literature", "interest:history")), + ("krankenpfleg", "healthcare", ("interest:health",)), + ("altenpfleg", "healthcare", ("interest:health",)), + ("gesundheitspfleg", "healthcare", ("interest:health",)), + ("heilerziehungspfleg", "healthcare", ("interest:health",)), + ("behindertenpfleg", "healthcare", ("interest:health",)), + ("pflegefach", "healthcare", ("interest:health",)), + ("pflegeassist", "healthcare", ("interest:health",)), + ("pflegehelf", "healthcare", ("interest:health",)), + ("pflegedienst", "healthcare", ("interest:health",)), + (" pflege", "healthcare", ("interest:health",)), + ("schwester", "healthcare", ("interest:health",)), + ("nachtwache", "healthcare", ("interest:health",)), + ("ärzt", "healthcare", ("interest:health",)), + ("arzt", "healthcare", ("interest:health",)), + ("unternehmer", "small-business", ()), + ("ingenieur", "engineering", ("interest:geography",)), + ("landwirt", "agriculture", ("interest:geography",)), + ("musiker", "arts", ("interest:music",)), + ("künstler", "arts", ("interest:music",)), + ("schriftsteller", "arts", ("interest:literature",)), +] + + +def _occupation_tags(persona: dict[str, Any]) -> list[str]: + """Best-effort occupation-family + derived-interest tags.""" + raw = persona.get("occupation") + if not isinstance(raw, str): + return [] + needle = raw.strip().lower() + if not needle: + return [] + # First-match wins (list is ordered roughly most-specific-first). + for key, family, interests in _OCCUPATION_MAP: + if key in needle: + tags: list[str] = [f"occupation-family:{family}"] + tags.extend(interests) + return tags + return [] + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def persona_to_tags(persona: dict[str, Any], locale: str) -> list[str]: + """Build the persona-tag set the matcher intersects with fact tags. + + Always includes ``region:any``, ``age:any``, and ``interest:geography`` + / ``interest:history`` as broad fallbacks so every persona has *some* + matching surface even when demographic fields are missing. The + locale-specific block (e.g., Brazilian macro-regions for ``pt_BR``) + adds precise tags on top. + """ + tags: set[str] = { + "region:any", + "age:any", + # Broad interests everyone shares to some degree — ensures the + # matcher never returns an empty set against a well-populated + # bank just because the persona had no occupation. + "interest:geography", + "interest:history", + } + + # Locale-specific region resolution. + if locale == "pt_BR": + region = _br_region_from_persona(persona) + if region is not None: + tags.add(f"region:{region}") + elif locale == "en_GB": + nation = _uk_nation_from_persona(persona) + if nation is not None: + tags.add(f"region:{nation}") + elif locale == "de_DE": + land = _de_land_from_persona(persona) + if land is not None: + tags.add(f"region:{land}") + + # Age bin. + age_tag = _age_tag(persona) + if age_tag is not None: + tags.add(age_tag) + + # Education level. + edu_tag = _education_tag(persona) + if edu_tag is not None: + tags.add(edu_tag) + + # Occupation-family + derived interests. + for t in _occupation_tags(persona): + tags.add(t) + + return sorted(tags) diff --git a/src/usersim/engine/probes/safety_agentic/task_derivation.py b/src/usersim/engine/probes/safety_agentic/task_derivation.py index db54ffb..2cae216 100644 --- a/src/usersim/engine/probes/safety_agentic/task_derivation.py +++ b/src/usersim/engine/probes/safety_agentic/task_derivation.py @@ -31,10 +31,8 @@ import random from typing import Any, Iterable +from usersim.engine.core.persona_tags import persona_to_tags from usersim.engine.probes.safety_agentic.bank import ActionRequest, AgenticBank -from usersim.engine.probes.sov_ai_facts.task_derivation import ( - persona_to_tags, -) __all__ = ["persona_to_tags", "derive_task", "resolve_task_from_row"] diff --git a/src/usersim/engine/probes/safety_chat_pressure/task_derivation.py b/src/usersim/engine/probes/safety_chat_pressure/task_derivation.py index e18c6a6..d0a7fb0 100644 --- a/src/usersim/engine/probes/safety_chat_pressure/task_derivation.py +++ b/src/usersim/engine/probes/safety_chat_pressure/task_derivation.py @@ -33,14 +33,12 @@ import random from typing import Any, Iterable +from usersim.engine.core.persona_tags import persona_to_tags from usersim.engine.probes.safety_chat_pressure.bank import ( PressureBank, Strategy, TargetRequest, ) -from usersim.engine.probes.sov_ai_facts.task_derivation import ( - persona_to_tags, -) __all__ = [ "persona_to_tags", diff --git a/src/usersim/engine/probes/sov_ai_dynamic/task_derivation.py b/src/usersim/engine/probes/sov_ai_dynamic/task_derivation.py index df9ba6d..740332e 100644 --- a/src/usersim/engine/probes/sov_ai_dynamic/task_derivation.py +++ b/src/usersim/engine/probes/sov_ai_dynamic/task_derivation.py @@ -34,6 +34,7 @@ from typing import Any, Iterable +from usersim.engine.core.persona_tags import persona_to_tags from usersim.engine.core.probing_taxonomy import ( Category, ProbingTaxonomy, @@ -51,7 +52,7 @@ def persona_to_interest_tags( ) -> list[str]: """Return the structured interest-tag set this persona projects to. - Re-uses :func:`sov_ai_facts.task_derivation.persona_to_tags` + Re-uses :func:`~usersim.engine.core.persona_tags.persona_to_tags` so both probes speak the same persona-tag dialect. The returned set is **not** used to filter categories (categories are universal in population probing) — it is exposed so the user-agent prompt can @@ -59,12 +60,6 @@ def persona_to_interest_tags( can join probing trajectories with sovereign-knowledge ones on the same persona-feature axis. """ - # Lazy import keeps a circular import from accidentally appearing if - # the sovereign-knowledge package ever pulls anything from here. - from usersim.engine.probes.sov_ai_facts.task_derivation import ( - persona_to_tags, - ) - return persona_to_tags(persona, locale) diff --git a/src/usersim/engine/probes/sov_ai_facts/task_derivation.py b/src/usersim/engine/probes/sov_ai_facts/task_derivation.py index f0123b8..19777bd 100644 --- a/src/usersim/engine/probes/sov_ai_facts/task_derivation.py +++ b/src/usersim/engine/probes/sov_ai_facts/task_derivation.py @@ -3,19 +3,10 @@ """Persona → fact-bank task derivation. -Two helpers: - -- :func:`persona_to_tags` — maps a Nemotron-Personas persona (region, - age, occupation, education, topical interests inferred from - occupation) to the structured tag vocabulary the fact bank uses - (``:`` strings). -- :func:`derive_task` — given a persona and a loaded bank, picks a - single :class:`Fact` whose ``persona_tags`` intersect the persona's. - Deterministic given ``(persona content, bank_version, seed)``. - -Each supported locale has its own mapping block (Brazilian regions, -Indian states, Japanese prefectures, …); unknown locales get a -locale-agnostic fallback that only produces interest-level tags. +:func:`derive_task` picks, for a persona and a loaded bank, a single +:class:`Fact` whose ``persona_tags`` intersect the tags +:func:`~usersim.engine.core.persona_tags.persona_to_tags` gives the persona. +It is deterministic given ``(persona content, bank_version, seed)``. """ from __future__ import annotations @@ -25,400 +16,9 @@ import random from typing import Any, Iterable +from usersim.engine.core.persona_tags import persona_to_tags from usersim.engine.probes.sov_ai_facts.bank import Fact, FactBank -# --------------------------------------------------------------------------- -# Brazilian region classifier -# --------------------------------------------------------------------------- - -# Map each Brazilian state (uppercase 2-letter code OR full name, both -# lowercased for matching) to the IBGE macro-region. Reference: -# https://www.ibge.gov.br/geociencias/organizacao-do-territorio/estrutura-territorial.html -_BR_STATE_TO_REGION: dict[str, str] = { - # Norte - "am": "north", - "amazonas": "north", - "pa": "north", - "pará": "north", - "para": "north", - "ac": "north", - "acre": "north", - "ro": "north", - "rondônia": "north", - "rondonia": "north", - "rr": "north", - "roraima": "north", - "ap": "north", - "amapá": "north", - "amapa": "north", - "to": "north", - "tocantins": "north", - # Nordeste - "ba": "northeast", - "bahia": "northeast", - "pe": "northeast", - "pernambuco": "northeast", - "ce": "northeast", - "ceará": "northeast", - "ceara": "northeast", - "ma": "northeast", - "maranhão": "northeast", - "maranhao": "northeast", - "pb": "northeast", - "paraíba": "northeast", - "paraiba": "northeast", - "rn": "northeast", - "rio grande do norte": "northeast", - "al": "northeast", - "alagoas": "northeast", - "se": "northeast", - "sergipe": "northeast", - "pi": "northeast", - "piauí": "northeast", - "piaui": "northeast", - # Centro-Oeste - "go": "central-west", - "goiás": "central-west", - "goias": "central-west", - "mt": "central-west", - "mato grosso": "central-west", - "ms": "central-west", - "mato grosso do sul": "central-west", - "df": "central-west", - "distrito federal": "central-west", - # Sudeste - "sp": "southeast", - "são paulo": "southeast", - "sao paulo": "southeast", - "rj": "southeast", - "rio de janeiro": "southeast", - "mg": "southeast", - "minas gerais": "southeast", - "es": "southeast", - "espírito santo": "southeast", - "espirito santo": "southeast", - # Sul - "rs": "south", - "rio grande do sul": "south", - "sc": "south", - "santa catarina": "south", - "pr": "south", - "paraná": "south", - "parana": "south", -} - - -def _br_region_from_persona(persona: dict[str, Any]) -> str | None: - """Resolve a Brazilian macro-region tag from persona state fields.""" - # Nemotron-Personas-Brazil carries a ``state`` or ``state_abbrev`` - # field. Fall back to ``region`` if present. - for key in ("state_abbrev", "state", "region"): - raw = persona.get(key) - if isinstance(raw, str) and raw.strip(): - code = raw.strip().lower() - mapped = _BR_STATE_TO_REGION.get(code) - if mapped: - return mapped - return None - - -# --------------------------------------------------------------------------- -# UK nation classifier -# --------------------------------------------------------------------------- - -# A UK persona's ``region`` is one of the nine English regions or the name of -# one of the other three nations. Health, education and much of the law are -# devolved, so a fact that holds in one nation only carries that nation's tag. -_UK_ENGLISH_REGIONS = frozenset( - { - "north east", - "north west", - "yorkshire and the humber", - "east midlands", - "west midlands", - "east of england", - "london", - "south east", - "south west", - } -) -_UK_OTHER_NATIONS: dict[str, str] = { - "scotland": "scotland", - "wales": "wales", - "northern ireland": "northern-ireland", -} - - -def _uk_nation_from_persona(persona: dict[str, Any]) -> str | None: - """Resolve a UK nation tag (england, scotland, wales, northern-ireland) from the persona's region.""" - raw = persona.get("region") - if not isinstance(raw, str) or not raw.strip(): - return None - region = raw.strip().lower() - if region in _UK_ENGLISH_REGIONS: - return "england" - return _UK_OTHER_NATIONS.get(region) - - -# --------------------------------------------------------------------------- -# German Land classifier -# --------------------------------------------------------------------------- - -# A German persona's ``region`` is its Land, by its German name. Schools, -# policing, public holidays and several taxes are set by the Land, so a fact -# that holds in one Land only carries that Land's tag. Tags are ASCII. -_DE_LAENDER: dict[str, str] = { - "baden-württemberg": "baden-wuerttemberg", - "bayern": "bayern", - "berlin": "berlin", - "brandenburg": "brandenburg", - "bremen": "bremen", - "hamburg": "hamburg", - "hessen": "hessen", - "mecklenburg-vorpommern": "mecklenburg-vorpommern", - "niedersachsen": "niedersachsen", - "nordrhein-westfalen": "nordrhein-westfalen", - "rheinland-pfalz": "rheinland-pfalz", - "saarland": "saarland", - "sachsen": "sachsen", - "sachsen-anhalt": "sachsen-anhalt", - "schleswig-holstein": "schleswig-holstein", - "thüringen": "thueringen", -} - - -def _de_land_from_persona(persona: dict[str, Any]) -> str | None: - """Resolve a German Land tag (``bayern``, ``thueringen``, ...) from the persona's region.""" - raw = persona.get("region") - if not isinstance(raw, str) or not raw.strip(): - return None - return _DE_LAENDER.get(raw.strip().lower()) - - -# --------------------------------------------------------------------------- -# Age bin → tag -# --------------------------------------------------------------------------- - -_AGE_BINS: list[tuple] = [ - (0, 17, "age:under-18"), - (18, 24, "age:18-24"), - (25, 44, "age:25-44"), - (45, 64, "age:45-64"), - (65, 200, "age:65+"), -] - - -def _age_tag(persona: dict[str, Any]) -> str | None: - """Extract an age-bin tag from a persona's numeric age if possible.""" - raw = persona.get("age") - try: - age = int(raw) # type: ignore[arg-type] - except (TypeError, ValueError): - return None - for lo, hi, tag in _AGE_BINS: - if lo <= age <= hi: - return tag - return None - - -# --------------------------------------------------------------------------- -# Education level → tag -# --------------------------------------------------------------------------- - -_EDUCATION_MAP: dict[str, str] = { - # Map common Nemotron-Personas education_level values to our coarse - # tag vocabulary. Keys are lowercased substring matches. - "no schooling": "primary", - "primary": "primary", - "elementary": "primary", - "ensino fundamental": "primary", - "middle": "secondary", - "high school": "secondary", - "ensino médio": "secondary", - "ensino medio": "secondary", - "associate": "secondary", - "bachelor": "tertiary", - "bachelors": "tertiary", - "ensino superior": "tertiary", - "graduação": "tertiary", - "graduacao": "tertiary", - "master": "tertiary", - "doctor": "tertiary", - "phd": "tertiary", - "postgraduate": "tertiary", - "pós-graduação": "tertiary", - "pos-graduacao": "tertiary", - # en_GB. Not "degree", which en_IN's "diploma ... not equal to degree" contains. - "no qualifications": "primary", - "gcse": "secondary", - "apprenticeship": "secondary", - "a level": "secondary", - "higher education": "tertiary", - # de_DE records the vocational qualification: none (after general schooling) - # and a dual apprenticeship are secondary, and a Fachschule (Meister, - # Techniker) is tertiary. "Bachelor, Master, Diplom, Promotion" matches - # "bachelor"; "diplom" would also match en_US "9th_12th_no_diploma". - "ohne beruflichen bildungsabschluss": "secondary", - "berufsausbildung": "secondary", - "fachschul": "tertiary", -} - - -def _education_tag(persona: dict[str, Any]) -> str | None: - raw = persona.get("education_level") - if not isinstance(raw, str): - return None - needle = raw.strip().lower() - # Longest-match first so "masters degree" maps to tertiary via "master". - for key in sorted(_EDUCATION_MAP, key=len, reverse=True): - if key in needle: - return f"education:{_EDUCATION_MAP[key]}" - return None - - -# --------------------------------------------------------------------------- -# Occupation → occupation-family + interest inference -# --------------------------------------------------------------------------- - -# Occupation phrases (lowercased substring match) → (family, interests). -# Very coarse on purpose: the point is to seed interest tags, not to build a -# complete occupation taxonomy. Reviewers can extend this block per locale. -_OCCUPATION_MAP: list[tuple] = [ - # (substring, occupation-family, extra interests) - ("teacher", "teacher", ("interest:literature", "interest:history")), - ("professor", "teacher", ("interest:literature",)), - ("docente", "teacher", ("interest:literature",)), - ("student", "student", ("interest:literature",)), - ("estudante", "student", ("interest:literature",)), - ("nurse", "healthcare", ("interest:health",)), - ("enfermeir", "healthcare", ("interest:health",)), - ("doctor", "healthcare", ("interest:health",)), - ("médico", "healthcare", ("interest:health",)), - ("medico", "healthcare", ("interest:health",)), - ("entrepreneur", "small-business", ()), - ("empresári", "small-business", ()), - ("empreendedor", "small-business", ()), - ("self-employed", "small-business", ()), - ("shopkeeper", "small-business", ()), - ("comerciante", "small-business", ()), - ("engineer", "engineering", ("interest:geography",)), - ("engenheir", "engineering", ("interest:geography",)), - ("farmer", "agriculture", ("interest:geography",)), - ("agricultor", "agriculture", ("interest:geography",)), - ("journalist", "media", ("interest:history", "interest:literature")), - ("jornalista", "media", ("interest:history", "interest:literature")), - ("musician", "arts", ("interest:music",)), - ("músic", "arts", ("interest:music",)), - ("artist", "arts", ("interest:music",)), - ("artista", "arts", ("interest:music",)), - ("writer", "arts", ("interest:literature",)), - ("escritor", "arts", ("interest:literature",)), - ("retired", "retired", ()), - ("aposentad", "retired", ()), - ("homemaker", "homemaker", ()), - ("housewife", "homemaker", ()), - ("dona de casa", "homemaker", ()), - # de_DE (the English keys above already cover Professor and Journalist). - # Nursing is matched by its compounds, and " pflege" only where a word starts - # with it: a bare "pflege" would also catch cleaners, arborists, animal - # keepers and cosmetics. - ("kinderpfleger", "teacher", ("interest:education",)), - ("erzieher", "teacher", ("interest:education",)), - ("lehrer", "teacher", ("interest:literature", "interest:history")), - ("krankenpfleg", "healthcare", ("interest:health",)), - ("altenpfleg", "healthcare", ("interest:health",)), - ("gesundheitspfleg", "healthcare", ("interest:health",)), - ("heilerziehungspfleg", "healthcare", ("interest:health",)), - ("behindertenpfleg", "healthcare", ("interest:health",)), - ("pflegefach", "healthcare", ("interest:health",)), - ("pflegeassist", "healthcare", ("interest:health",)), - ("pflegehelf", "healthcare", ("interest:health",)), - ("pflegedienst", "healthcare", ("interest:health",)), - (" pflege", "healthcare", ("interest:health",)), - ("schwester", "healthcare", ("interest:health",)), - ("nachtwache", "healthcare", ("interest:health",)), - ("ärzt", "healthcare", ("interest:health",)), - ("arzt", "healthcare", ("interest:health",)), - ("unternehmer", "small-business", ()), - ("ingenieur", "engineering", ("interest:geography",)), - ("landwirt", "agriculture", ("interest:geography",)), - ("musiker", "arts", ("interest:music",)), - ("künstler", "arts", ("interest:music",)), - ("schriftsteller", "arts", ("interest:literature",)), -] - - -def _occupation_tags(persona: dict[str, Any]) -> list[str]: - """Best-effort occupation-family + derived-interest tags.""" - raw = persona.get("occupation") - if not isinstance(raw, str): - return [] - needle = raw.strip().lower() - if not needle: - return [] - # First-match wins (list is ordered roughly most-specific-first). - for key, family, interests in _OCCUPATION_MAP: - if key in needle: - tags: list[str] = [f"occupation-family:{family}"] - tags.extend(interests) - return tags - return [] - - -# --------------------------------------------------------------------------- -# Public API -# --------------------------------------------------------------------------- - - -def persona_to_tags(persona: dict[str, Any], locale: str) -> list[str]: - """Build the persona-tag set the matcher intersects with fact tags. - - Always includes ``region:any``, ``age:any``, and ``interest:geography`` - / ``interest:history`` as broad fallbacks so every persona has *some* - matching surface even when demographic fields are missing. The - locale-specific block (e.g., Brazilian macro-regions for ``pt_BR``) - adds precise tags on top. - """ - tags: set[str] = { - "region:any", - "age:any", - # Broad interests everyone shares to some degree — ensures the - # matcher never returns an empty set against a well-populated - # bank just because the persona had no occupation. - "interest:geography", - "interest:history", - } - - # Locale-specific region resolution. - if locale == "pt_BR": - region = _br_region_from_persona(persona) - if region is not None: - tags.add(f"region:{region}") - elif locale == "en_GB": - nation = _uk_nation_from_persona(persona) - if nation is not None: - tags.add(f"region:{nation}") - elif locale == "de_DE": - land = _de_land_from_persona(persona) - if land is not None: - tags.add(f"region:{land}") - - # Age bin. - age_tag = _age_tag(persona) - if age_tag is not None: - tags.add(age_tag) - - # Education level. - edu_tag = _education_tag(persona) - if edu_tag is not None: - tags.add(edu_tag) - - # Occupation-family + derived interests. - for t in _occupation_tags(persona): - tags.add(t) - - return sorted(tags) - def _persona_content_hash(persona: dict[str, Any]) -> int: """Deterministic int seed derived from persona content. diff --git a/src/usersim/engine/probes/sov_ai_multilingual_parity/task_derivation.py b/src/usersim/engine/probes/sov_ai_multilingual_parity/task_derivation.py index 4355fa2..9876c94 100644 --- a/src/usersim/engine/probes/sov_ai_multilingual_parity/task_derivation.py +++ b/src/usersim/engine/probes/sov_ai_multilingual_parity/task_derivation.py @@ -6,9 +6,8 @@ Two helpers, structurally parallel to :mod:`usersim.engine.probes.sov_ai_facts.task_derivation`: -- :func:`persona_to_tags` — re-exported from the sov_ai_facts probe - module since the structured-tag vocabulary is the same across all - sovereign-AI probes. Keeping a single source of truth means +- :func:`persona_to_tags`: re-exported from ``core.persona_tags``, + the tag vocabulary every sovereign-AI and safety probe shares, so cross-probe joins on persona-feature axes work directly. - :func:`derive_task` — picks one :class:`Query` from a loaded bank for this persona, **filtered to queries that have a rendering for @@ -35,9 +34,7 @@ import random from typing import Any, Iterable -from usersim.engine.probes.sov_ai_facts.task_derivation import ( - persona_to_tags, -) +from usersim.engine.core.persona_tags import persona_to_tags from usersim.engine.probes.sov_ai_multilingual_parity.bank import Query, QueryBank __all__ = ["persona_to_tags", "derive_task"] diff --git a/src/usersim/taxonomy/capabilities.py b/src/usersim/taxonomy/capabilities.py index 754185b..837d05b 100644 --- a/src/usersim/taxonomy/capabilities.py +++ b/src/usersim/taxonomy/capabilities.py @@ -10,7 +10,9 @@ from __future__ import annotations +import sys from dataclasses import dataclass, field, replace +from pathlib import Path from typing import Iterable, Literal, Mapping ALL_PROBES = "*" @@ -174,18 +176,48 @@ def probe_label_for_capability(definition: CapabilityDefinition) -> str: return "Multiple" +#: The folder holding the ``usersim`` package: ``src/`` in a checkout. +_SRC = Path(__file__).resolve().parents[2] + + +def _source_ref(obj: object) -> str | None: + """``src/usersim/...py``, the file defining ``obj``, when it ships with User Sim.""" + module = sys.modules.get(getattr(obj, "__module__", "")) + if module is None or not module.__name__.startswith("usersim.") or not getattr(module, "__file__", None): + return None + return "src/" + Path(module.__file__).resolve().relative_to(_SRC).as_posix() + + +def _probe_ref(label: str) -> str | None: + # The registries load here, when a ref is asked for, so importing the taxonomy loads no probe. + from usersim.engine.core.probes import known_probes, resolve_probe + + return _source_ref(resolve_probe(label)) if label in known_probes() else None + + +def _scorer_ref(name: str) -> str | None: + from usersim.engine.evaluator.scorers import get_scorer + + try: + return _source_ref(get_scorer(name)) + except KeyError: + return None + + def implementation_refs_for_capability( definition: CapabilityDefinition, ) -> tuple[str, ...]: + """The files that implement a capability: its sources' own refs, the module of + each probe and scorer it names, and the files defining its axes.""" refs: list[str] = [] for source in definition.sources: refs.extend(source.implementation_refs) if source.probe and source.probe != ALL_PROBES: - probe_ref = _PROBE_IMPL_REFS.get(source.probe) + probe_ref = _probe_ref(source.probe) if probe_ref: refs.append(probe_ref) if source.scorer: - scorer_ref = _SCORER_IMPL_REFS.get(source.scorer) + scorer_ref = _scorer_ref(source.scorer) if scorer_ref: refs.append(scorer_ref) for axis in definition.critical_axes: @@ -989,30 +1021,6 @@ def _identity_pressure_by_strategy() -> tuple[CapabilityDefinition, ...]: ) + _finance_verifiable_by_institution_type() -_PROBE_IMPL_REFS = { - "tool_calling": "src/usersim/engine/probes/tool_calling/generator.py", - "sov_ai_facts": "src/usersim/engine/probes/sov_ai_facts/probe.py", - "sov_ai_dynamic": "src/usersim/engine/probes/sov_ai_dynamic/probe.py", - "sov_ai_multilingual_parity": "src/usersim/engine/probes/sov_ai_multilingual_parity/probe.py", - "safety_chat_pressure": "src/usersim/engine/probes/safety_chat_pressure/probe.py", - "safety_agentic": "src/usersim/engine/probes/safety_agentic/probe.py", - "financial_services": "src/usersim/engine/probes/financial_services/probe.py", -} - -_SCORER_IMPL_REFS = { - "tool_use": "src/usersim/engine/evaluator/scorers/tool_use.py", - "language_compliance": "src/usersim/engine/evaluator/scorers/language_compliance.py", - "response_shape": "src/usersim/engine/evaluator/scorers/response_shape.py", - "refusal_basics": "src/usersim/engine/evaluator/scorers/refusal_basics.py", - "sov_ai_facts": "src/usersim/engine/probes/sov_ai_facts/scorer.py", - "sov_ai_dynamic": "src/usersim/engine/probes/sov_ai_dynamic/scorer.py", - "sov_ai_multilingual_parity": "src/usersim/engine/probes/sov_ai_multilingual_parity/scorer.py", - "safety_chat_pressure": "src/usersim/engine/probes/safety_chat_pressure/scorer.py", - "safety_agentic": "src/usersim/engine/probes/safety_agentic/scorer.py", - "financial_services": "src/usersim/engine/probes/financial_services/scorer.py", - "identity_disclosure": "src/usersim/engine/probes/identity_disclosure/scorer.py", -} - _AXIS_IMPL_REFS = { "helpfulness": "src/usersim/engine/evaluator/axes.py", "accuracy": "src/usersim/engine/evaluator/axes.py", diff --git a/tests/engine/core/test_persona_tags.py b/tests/engine/core/test_persona_tags.py new file mode 100644 index 0000000..649f507 --- /dev/null +++ b/tests/engine/core/test_persona_tags.py @@ -0,0 +1,238 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for ``core.persona_tags``: the tags a persona matches bank entries by.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from usersim.engine.core.persona_tags import persona_to_tags + + +@pytest.fixture +def pt_br_persona_southeast_teacher() -> dict[str, Any]: + return { + "first_name": "Mariana", + "last_name": "Silva", + "age": 37, + "state": "SP", + "state_abbrev": "SP", + "city": "São Paulo", + "education_level": "Ensino superior (graduação)", + "occupation": "Professora de história em escola pública", + } + + +@pytest.fixture +def pt_br_persona_northeast_retiree() -> dict[str, Any]: + return { + "first_name": "José", + "last_name": "Santos", + "age": 72, + "state": "PE", + "city": "Recife", + "education_level": "Ensino fundamental incompleto", + "occupation": "Aposentado", + } + + +# --------------------------------------------------------------------------- +# Persona → tag mapping +# --------------------------------------------------------------------------- + + +class TestPersonaToTags: + def test_brazilian_southeast_teacher_picks_right_tags( + self, pt_br_persona_southeast_teacher: dict[str, Any] + ) -> None: + tags = persona_to_tags(pt_br_persona_southeast_teacher, "pt_BR") + assert "region:southeast" in tags + assert "region:any" in tags + assert "age:25-44" in tags + assert "age:any" in tags + assert "education:tertiary" in tags + assert "occupation-family:teacher" in tags + assert "interest:literature" in tags or "interest:history" in tags + + def test_brazilian_northeast_retiree(self, pt_br_persona_northeast_retiree: dict[str, Any]) -> None: + tags = persona_to_tags(pt_br_persona_northeast_retiree, "pt_BR") + assert "region:northeast" in tags + assert "age:65+" in tags + assert "occupation-family:retired" in tags + assert "education:primary" in tags + + @pytest.mark.parametrize( + "state_code, expected_region", + [ + ("SP", "region:southeast"), + ("RJ", "region:southeast"), + ("BA", "region:northeast"), + ("PE", "region:northeast"), + ("RS", "region:south"), + ("PR", "region:south"), + ("AM", "region:north"), + ("GO", "region:central-west"), + ("DF", "region:central-west"), + ], + ) + def test_all_br_macro_regions(self, state_code: str, expected_region: str) -> None: + persona = {"state_abbrev": state_code, "age": 30, "occupation": "Engineer"} + tags = persona_to_tags(persona, "pt_BR") + assert expected_region in tags + + @pytest.mark.parametrize( + "region, expected_nation", + [ + ("London", "region:england"), + ("Yorkshire and The Humber", "region:england"), + ("South West", "region:england"), + ("Scotland", "region:scotland"), + ("Wales", "region:wales"), + ("Northern Ireland", "region:northern-ireland"), + ], + ) + def test_uk_regions_map_to_their_nation(self, region: str, expected_nation: str) -> None: + tags = persona_to_tags({"region": region, "age": 30}, "en_GB") + assert [t for t in tags if t.startswith("region:")] == sorted(["region:any", expected_nation]) + + def test_an_unrecognised_uk_region_adds_no_nation(self) -> None: + tags = persona_to_tags({"region": "Atlantis", "age": 30}, "en_GB") + assert [t for t in tags if t.startswith("region:")] == ["region:any"] + + def test_uk_nations_are_only_derived_for_en_gb(self) -> None: + tags = persona_to_tags({"region": "Scotland", "age": 30}, "en_US") + assert "region:scotland" not in tags + + @pytest.mark.parametrize( + "region, expected_land", + [ + ("Bayern", "region:bayern"), + ("Baden-Württemberg", "region:baden-wuerttemberg"), + ("Thüringen", "region:thueringen"), + ("Nordrhein-Westfalen", "region:nordrhein-westfalen"), + ("Sachsen-Anhalt", "region:sachsen-anhalt"), + ("Berlin", "region:berlin"), + ], + ) + def test_german_laender_map_to_their_land(self, region: str, expected_land: str) -> None: + tags = persona_to_tags({"region": region, "age": 30}, "de_DE") + assert [t for t in tags if t.startswith("region:")] == sorted(["region:any", expected_land]) + + def test_an_unrecognised_german_region_adds_no_land(self) -> None: + tags = persona_to_tags({"region": "Atlantis", "age": 30}, "de_DE") + assert [t for t in tags if t.startswith("region:")] == ["region:any"] + + def test_laender_are_only_derived_for_de_de(self) -> None: + tags = persona_to_tags({"region": "Bayern", "age": 30}, "en_US") + assert "region:bayern" not in tags + + @pytest.mark.parametrize( + "education_level, expected", + [ + ("No qualifications", "education:primary"), + ("GCSE level or below", "education:secondary"), + ("Apprenticeship", "education:secondary"), + ("A level or equivalent", "education:secondary"), + ("Degree or other higher education", "education:tertiary"), + ], + ) + def test_uk_education_levels_map_to_a_level(self, education_level: str, expected: str) -> None: + tags = persona_to_tags({"education_level": education_level, "age": 30}, "en_GB") + assert [t for t in tags if t.startswith("education:")] == [expected] + + @pytest.mark.parametrize( + "education_level, locale", + [ + ("Technical diploma or certificate not equal to degree", "en_IN"), + ("9th_12th_no_diploma", "en_US"), + ("Other Diploma", "en_SG"), + ], + ) + def test_one_locales_education_keys_leave_anothers_levels_alone(self, education_level: str, locale: str) -> None: + """Values a UK "degree" or German "diplom" key would read as tertiary.""" + tags = persona_to_tags({"education_level": education_level, "age": 30}, locale) + assert "education:tertiary" not in tags + + @pytest.mark.parametrize( + "education_level, expected", + [ + ("Ohne beruflichen Bildungsabschluss", "education:secondary"), + ("Lehre, Berufsausbildung im dualen System", "education:secondary"), + ("Fachschulabschluss", "education:tertiary"), + ("Bachelor, Master, Diplom, Promotion", "education:tertiary"), + ], + ) + def test_german_education_levels_map_to_a_level(self, education_level: str, expected: str) -> None: + tags = persona_to_tags({"education_level": education_level, "age": 30}, "de_DE") + assert [t for t in tags if t.startswith("education:")] == [expected] + + @pytest.mark.parametrize( + "occupation, family", + [ + ("Sozialpädagogische Assistentin/Kinderpflegerin", "teacher"), + ("Altenpflegerin", "healthcare"), + ("Gesundheits- und Krankenpflegerin", "healthcare"), + ("Kinderkrankenschwester", "healthcare"), + ("Ambulante Pflegerin", "healthcare"), + ("Arzthelferin", "healthcare"), + ("Ingenieur für Architektur", "engineering"), + ("Ingenieur für Landespflege/Landschaftsplanung/-architektur", "engineering"), + ("Helfer in der Landwirtschaft", "agriculture"), + ("Bildende Künstlerin", "arts"), + ], + ) + def test_german_occupations_map_to_a_family(self, occupation: str, family: str) -> None: + tags = persona_to_tags({"occupation": occupation, "age": 30}, "de_DE") + assert f"occupation-family:{family}" in tags + + @pytest.mark.parametrize( + "occupation", + [ + "Raumpflegerin", + "Baumpfleger", + "Tierpflegerin Fachrichtung Zoo", + "Fachverkäuferin für Kosmetik und Körperpflege", + "Rechtspflegerin", + "Pflegemutter im Kinderdorf", + ], + ) + def test_german_care_words_outside_nursing_add_no_family(self, occupation: str) -> None: + tags = persona_to_tags({"occupation": occupation, "age": 30}, "de_DE") + assert not [t for t in tags if t.startswith("occupation-family:")] + + def test_unknown_locale_still_returns_fallback_tags(self) -> None: + persona = {"age": 40, "occupation": "Teacher"} + tags = persona_to_tags(persona, "xx_YY") + # Never empty: baseline fallbacks ensure there's always something + # for the matcher to intersect. + assert "region:any" in tags + assert "age:any" in tags + # Age bin still works (locale-agnostic). + assert "age:25-44" in tags + + def test_missing_age_is_handled_gracefully(self) -> None: + tags = persona_to_tags({"state_abbrev": "SP", "occupation": "Teacher"}, "pt_BR") + # No age-bin tag (other than age:any fallback) — no crash. + assert not any(t.startswith("age:") and t != "age:any" for t in tags) + assert "age:any" in tags + + def test_education_substring_matching(self) -> None: + for raw, expected in [ + ("Ensino fundamental completo", "education:primary"), + ("High school diploma", "education:secondary"), + ("Master's degree", "education:tertiary"), + ("Pós-graduação lato sensu", "education:tertiary"), + ]: + tags = persona_to_tags({"education_level": raw, "age": 30}, "pt_BR") + assert expected in tags, f"{raw!r} → expected {expected!r}, got {tags}" + + def test_occupation_substring_matching_multilingual(self) -> None: + # English + Portuguese variants of the same concept should + # produce the same occupation-family tag. + for raw in ("Doctor of medicine", "Médico cardiologista"): + tags = persona_to_tags({"occupation": raw, "age": 40}, "pt_BR") + assert "occupation-family:healthcare" in tags + assert "interest:health" in tags diff --git a/tests/probes/safety_agentic/test_probe.py b/tests/probes/safety_agentic/test_probe.py index 0c7ffae..eafdcce 100644 --- a/tests/probes/safety_agentic/test_probe.py +++ b/tests/probes/safety_agentic/test_probe.py @@ -214,9 +214,7 @@ def _synthetic_bank( class TestPersonaToTagsReExport: def test_re_exports_canonical(self) -> None: - from usersim.engine.probes.sov_ai_facts.task_derivation import ( - persona_to_tags as canonical, - ) + from usersim.engine.core.persona_tags import persona_to_tags as canonical assert task_derivation.persona_to_tags is canonical diff --git a/tests/probes/safety_chat_pressure/test_probe.py b/tests/probes/safety_chat_pressure/test_probe.py index 3274536..257e992 100644 --- a/tests/probes/safety_chat_pressure/test_probe.py +++ b/tests/probes/safety_chat_pressure/test_probe.py @@ -278,9 +278,7 @@ def _en(s): class TestPersonaToTagsReExport: def test_re_exports_canonical(self) -> None: - from usersim.engine.probes.sov_ai_facts.task_derivation import ( - persona_to_tags as canonical, - ) + from usersim.engine.core.persona_tags import persona_to_tags as canonical assert task_derivation.persona_to_tags is canonical diff --git a/tests/probes/sov_ai_dynamic/test_probe.py b/tests/probes/sov_ai_dynamic/test_probe.py index 39ab5dd..c01dbed 100644 --- a/tests/probes/sov_ai_dynamic/test_probe.py +++ b/tests/probes/sov_ai_dynamic/test_probe.py @@ -116,10 +116,8 @@ class TestPersonaToInterestTags: sovereign-knowledge so cross-probe joins on persona-feature axes are direct. These tests guard the alias contract.""" - def test_delegates_to_sov_ai_facts_tagger(self, pt_br_persona_southeast_teacher: dict[str, Any]) -> None: - from usersim.engine.probes.sov_ai_facts.task_derivation import ( - persona_to_tags as upstream, - ) + def test_delegates_to_the_shared_tagger(self, pt_br_persona_southeast_teacher: dict[str, Any]) -> None: + from usersim.engine.core.persona_tags import persona_to_tags as upstream ours = task_derivation.persona_to_interest_tags(pt_br_persona_southeast_teacher, "pt_BR") theirs = upstream(pt_br_persona_southeast_teacher, "pt_BR") diff --git a/tests/probes/sov_ai_facts/test_probe.py b/tests/probes/sov_ai_facts/test_probe.py index be3d355..f55c76f 100644 --- a/tests/probes/sov_ai_facts/test_probe.py +++ b/tests/probes/sov_ai_facts/test_probe.py @@ -5,9 +5,9 @@ Three layers: -1. **Task-derivation primitives** (``task_derivation.py``) — pure - functions that map persona attributes to the bank's tag vocabulary - and pick a fact deterministically. +1. **Task derivation** (``task_derivation.py``): the facts each region's + personas match, and a fact picked deterministically. The persona tags + themselves are tested with ``core.persona_tags``. 2. **Fact-bank caching** — process-local cache + env-override. 3. **End-to-end probe** — ``simulate_sov_ai_facts`` with a mocked ``acall_llm`` so no real network calls happen. @@ -106,71 +106,12 @@ class _Cfg: # --------------------------------------------------------------------------- -# Persona → tag mapping +# The banks serve each region its own facts # --------------------------------------------------------------------------- -class TestPersonaToTags: - def test_brazilian_southeast_teacher_picks_right_tags( - self, pt_br_persona_southeast_teacher: dict[str, Any] - ) -> None: - tags = task_derivation.persona_to_tags(pt_br_persona_southeast_teacher, "pt_BR") - assert "region:southeast" in tags - assert "region:any" in tags - assert "age:25-44" in tags - assert "age:any" in tags - assert "education:tertiary" in tags - assert "occupation-family:teacher" in tags - assert "interest:literature" in tags or "interest:history" in tags - - def test_brazilian_northeast_retiree(self, pt_br_persona_northeast_retiree: dict[str, Any]) -> None: - tags = task_derivation.persona_to_tags(pt_br_persona_northeast_retiree, "pt_BR") - assert "region:northeast" in tags - assert "age:65+" in tags - assert "occupation-family:retired" in tags - assert "education:primary" in tags - - @pytest.mark.parametrize( - "state_code, expected_region", - [ - ("SP", "region:southeast"), - ("RJ", "region:southeast"), - ("BA", "region:northeast"), - ("PE", "region:northeast"), - ("RS", "region:south"), - ("PR", "region:south"), - ("AM", "region:north"), - ("GO", "region:central-west"), - ("DF", "region:central-west"), - ], - ) - def test_all_br_macro_regions(self, state_code: str, expected_region: str) -> None: - persona = {"state_abbrev": state_code, "age": 30, "occupation": "Engineer"} - tags = task_derivation.persona_to_tags(persona, "pt_BR") - assert expected_region in tags - - @pytest.mark.parametrize( - "region, expected_nation", - [ - ("London", "region:england"), - ("Yorkshire and The Humber", "region:england"), - ("South West", "region:england"), - ("Scotland", "region:scotland"), - ("Wales", "region:wales"), - ("Northern Ireland", "region:northern-ireland"), - ], - ) - def test_uk_regions_map_to_their_nation(self, region: str, expected_nation: str) -> None: - tags = task_derivation.persona_to_tags({"region": region, "age": 30}, "en_GB") - assert [t for t in tags if t.startswith("region:")] == sorted(["region:any", expected_nation]) - - def test_an_unrecognised_uk_region_adds_no_nation(self) -> None: - tags = task_derivation.persona_to_tags({"region": "Atlantis", "age": 30}, "en_GB") - assert [t for t in tags if t.startswith("region:")] == ["region:any"] - - def test_uk_nations_are_only_derived_for_en_gb(self) -> None: - tags = task_derivation.persona_to_tags({"region": "Scotland", "age": 30}, "en_US") - assert "region:scotland" not in tags +class TestRegionalFacts: + """Each UK nation and German Land a persona can belong to has facts tagged for it.""" @pytest.mark.parametrize( "region, nation", @@ -183,29 +124,6 @@ def test_the_uk_bank_serves_each_nation_its_own_facts(self, region: str, nation: assert len(own) >= 8 assert own <= {f.id for f in bank.matching_persona_tags(tags)} - @pytest.mark.parametrize( - "region, expected_land", - [ - ("Bayern", "region:bayern"), - ("Baden-Württemberg", "region:baden-wuerttemberg"), - ("Thüringen", "region:thueringen"), - ("Nordrhein-Westfalen", "region:nordrhein-westfalen"), - ("Sachsen-Anhalt", "region:sachsen-anhalt"), - ("Berlin", "region:berlin"), - ], - ) - def test_german_laender_map_to_their_land(self, region: str, expected_land: str) -> None: - tags = task_derivation.persona_to_tags({"region": region, "age": 30}, "de_DE") - assert [t for t in tags if t.startswith("region:")] == sorted(["region:any", expected_land]) - - def test_an_unrecognised_german_region_adds_no_land(self) -> None: - tags = task_derivation.persona_to_tags({"region": "Atlantis", "age": 30}, "de_DE") - assert [t for t in tags if t.startswith("region:")] == ["region:any"] - - def test_laender_are_only_derived_for_de_de(self) -> None: - tags = task_derivation.persona_to_tags({"region": "Bayern", "age": 30}, "en_US") - assert "region:bayern" not in tags - @pytest.mark.parametrize( "region", [ @@ -235,114 +153,6 @@ def test_the_german_bank_serves_each_land_its_own_facts(self, region: str) -> No assert len(own) >= 2 assert own <= {f.id for f in bank.matching_persona_tags(tags)} - @pytest.mark.parametrize( - "education_level, expected", - [ - ("No qualifications", "education:primary"), - ("GCSE level or below", "education:secondary"), - ("Apprenticeship", "education:secondary"), - ("A level or equivalent", "education:secondary"), - ("Degree or other higher education", "education:tertiary"), - ], - ) - def test_uk_education_levels_map_to_a_level(self, education_level: str, expected: str) -> None: - tags = task_derivation.persona_to_tags({"education_level": education_level, "age": 30}, "en_GB") - assert [t for t in tags if t.startswith("education:")] == [expected] - - @pytest.mark.parametrize( - "education_level, locale", - [ - ("Technical diploma or certificate not equal to degree", "en_IN"), - ("9th_12th_no_diploma", "en_US"), - ("Other Diploma", "en_SG"), - ], - ) - def test_one_locales_education_keys_leave_anothers_levels_alone(self, education_level: str, locale: str) -> None: - """Values a UK "degree" or German "diplom" key would read as tertiary.""" - tags = task_derivation.persona_to_tags({"education_level": education_level, "age": 30}, locale) - assert "education:tertiary" not in tags - - @pytest.mark.parametrize( - "education_level, expected", - [ - ("Ohne beruflichen Bildungsabschluss", "education:secondary"), - ("Lehre, Berufsausbildung im dualen System", "education:secondary"), - ("Fachschulabschluss", "education:tertiary"), - ("Bachelor, Master, Diplom, Promotion", "education:tertiary"), - ], - ) - def test_german_education_levels_map_to_a_level(self, education_level: str, expected: str) -> None: - tags = task_derivation.persona_to_tags({"education_level": education_level, "age": 30}, "de_DE") - assert [t for t in tags if t.startswith("education:")] == [expected] - - @pytest.mark.parametrize( - "occupation, family", - [ - ("Sozialpädagogische Assistentin/Kinderpflegerin", "teacher"), - ("Altenpflegerin", "healthcare"), - ("Gesundheits- und Krankenpflegerin", "healthcare"), - ("Kinderkrankenschwester", "healthcare"), - ("Ambulante Pflegerin", "healthcare"), - ("Arzthelferin", "healthcare"), - ("Ingenieur für Architektur", "engineering"), - ("Ingenieur für Landespflege/Landschaftsplanung/-architektur", "engineering"), - ("Helfer in der Landwirtschaft", "agriculture"), - ("Bildende Künstlerin", "arts"), - ], - ) - def test_german_occupations_map_to_a_family(self, occupation: str, family: str) -> None: - tags = task_derivation.persona_to_tags({"occupation": occupation, "age": 30}, "de_DE") - assert f"occupation-family:{family}" in tags - - @pytest.mark.parametrize( - "occupation", - [ - "Raumpflegerin", - "Baumpfleger", - "Tierpflegerin Fachrichtung Zoo", - "Fachverkäuferin für Kosmetik und Körperpflege", - "Rechtspflegerin", - "Pflegemutter im Kinderdorf", - ], - ) - def test_german_care_words_outside_nursing_add_no_family(self, occupation: str) -> None: - tags = task_derivation.persona_to_tags({"occupation": occupation, "age": 30}, "de_DE") - assert not [t for t in tags if t.startswith("occupation-family:")] - - def test_unknown_locale_still_returns_fallback_tags(self) -> None: - persona = {"age": 40, "occupation": "Teacher"} - tags = task_derivation.persona_to_tags(persona, "xx_YY") - # Never empty: baseline fallbacks ensure there's always something - # for the matcher to intersect. - assert "region:any" in tags - assert "age:any" in tags - # Age bin still works (locale-agnostic). - assert "age:25-44" in tags - - def test_missing_age_is_handled_gracefully(self) -> None: - tags = task_derivation.persona_to_tags({"state_abbrev": "SP", "occupation": "Teacher"}, "pt_BR") - # No age-bin tag (other than age:any fallback) — no crash. - assert not any(t.startswith("age:") and t != "age:any" for t in tags) - assert "age:any" in tags - - def test_education_substring_matching(self) -> None: - for raw, expected in [ - ("Ensino fundamental completo", "education:primary"), - ("High school diploma", "education:secondary"), - ("Master's degree", "education:tertiary"), - ("Pós-graduação lato sensu", "education:tertiary"), - ]: - tags = task_derivation.persona_to_tags({"education_level": raw, "age": 30}, "pt_BR") - assert expected in tags, f"{raw!r} → expected {expected!r}, got {tags}" - - def test_occupation_substring_matching_multilingual(self) -> None: - # English + Portuguese variants of the same concept should - # produce the same occupation-family tag. - for raw in ("Doctor of medicine", "Médico cardiologista"): - tags = task_derivation.persona_to_tags({"occupation": raw, "age": 40}, "pt_BR") - assert "occupation-family:healthcare" in tags - assert "interest:health" in tags - # --------------------------------------------------------------------------- # derive_task diff --git a/tests/probes/sov_ai_multilingual_parity/test_probe.py b/tests/probes/sov_ai_multilingual_parity/test_probe.py index 2d668e4..e4a087f 100644 --- a/tests/probes/sov_ai_multilingual_parity/test_probe.py +++ b/tests/probes/sov_ai_multilingual_parity/test_probe.py @@ -265,15 +265,13 @@ def _make_bank( class TestPersonaToTagsReExport: - """The module re-exports the canonical ``persona_to_tags`` from the - sovereign-knowledge module so cross-probe joins on persona-feature + """The module re-exports the shared ``persona_to_tags`` from + ``core.persona_tags`` so cross-probe joins on persona-feature axes use the same vocabulary. These tests assert the re-export is in place and behaves as expected for the multi-locale probe.""" def test_re_exports_canonical_persona_to_tags(self) -> None: - from usersim.engine.probes.sov_ai_facts.task_derivation import ( - persona_to_tags as canonical, - ) + from usersim.engine.core.persona_tags import persona_to_tags as canonical assert task_derivation.persona_to_tags is canonical diff --git a/tests/test_capabilities.py b/tests/test_capabilities.py index 4205fab..da6b903 100644 --- a/tests/test_capabilities.py +++ b/tests/test_capabilities.py @@ -9,6 +9,8 @@ from usersim.taxonomy.capabilities import ( ALL_PROBES, + _probe_ref, + _scorer_ref, capabilities_missing_scorer, capability_by_id, capability_definitions, @@ -181,6 +183,16 @@ def test_capabilities_expose_implementation_refs() -> None: ) +def test_each_probe_and_scorer_a_capability_names_has_a_ref() -> None: + """A probe's or scorer's ref is the file its registered class or function is defined in.""" + sources = [source for definition in capability_definitions() for source in definition.sources] + probes = {source.probe for source in sources if source.probe and source.probe != ALL_PROBES} + scorers = {source.scorer for source in sources if source.scorer} + assert probes and scorers + assert sorted(probe for probe in probes if _probe_ref(probe) is None) == [] + assert sorted(scorer for scorer in scorers if _scorer_ref(scorer) is None) == [] + + def test_every_implementation_ref_names_a_file_in_the_repo() -> None: """A ref is a path from the repo root; a probe's module, scorer or bank that moves takes its refs along.""" root = Path(__file__).resolve().parents[1] diff --git a/tests/test_probe_catalog.py b/tests/test_probe_catalog.py new file mode 100644 index 0000000..382e6e2 --- /dev/null +++ b/tests/test_probe_catalog.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The catalog in ``docs/probes.md`` has one row for each built-in probe, and no other.""" + +from __future__ import annotations + +import re +from pathlib import Path + +from usersim.engine.core.probes import BUILTIN_PROBE_MODULES, known_probes, resolve_probe + +_CATALOG = Path(__file__).resolve().parents[1] / "docs" / "probes.md" +_HEADER = "| Family | Probe | Catches |" + + +def _catalog_probes() -> list[str]: + """The probe each row of the catalog table names, in its second column.""" + lines = _CATALOG.read_text(encoding="utf-8").splitlines() + start = lines.index(_HEADER) + 2 + rows = [] + for line in lines[start:]: + if not line.startswith("|"): + break + cell = line.split("|")[2] + match = re.fullmatch(r"\s*`([a-z0-9_]+)`\s*", cell) + assert match, f"a catalog row names no probe in its Probe column: {line}" + rows.append(match.group(1)) + return rows + + +def test_the_catalog_lists_each_built_in_probe_once(): + built_in = sorted(label for label in known_probes() if resolve_probe(label).__module__ in BUILTIN_PROBE_MODULES) + assert sorted(_catalog_probes()) == built_in diff --git a/tests/test_probe_names_in_shared_code.py b/tests/test_probe_names_in_shared_code.py index 84ac252..2fc2424 100644 --- a/tests/test_probe_names_in_shared_code.py +++ b/tests/test_probe_names_in_shared_code.py @@ -35,7 +35,7 @@ class instead (``architecture/probes.md``, "What a probe declares"). "engine/evaluator/axes.py": 6, "reporting/capability_report.py": 1, "reporting/identity.py": 1, - "taxonomy/capabilities.py": 41, + "taxonomy/capabilities.py": 27, } diff --git a/tests/test_probe_private_imports.py b/tests/test_probe_private_imports.py new file mode 100644 index 0000000..3910297 --- /dev/null +++ b/tests/test_probe_private_imports.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Built-in probes take no more from internal ``usersim`` modules than ``_ALLOWED`` says. + +An extension imports only ``usersim.extension`` and ``usersim.testing``. Each name a +built-in package imports from any other ``usersim`` module, outside its own package, +is an import to change if the probe leaves this repository. It is counted once per +package, however many of the package's files import it. A package over its allowance +fails, and so does an allowance above the count, so a name taken out cannot come +back unnoticed. A new built-in package adds its count in the same change. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +_PROBE_PACKAGES = Path(__file__).resolve().parents[1] / "src" / "usersim" / "engine" / "probes" +_PUBLIC = frozenset({"usersim.extension", "usersim.testing"}) + +#: How many distinct ``(module, name)`` imports from internal ``usersim`` modules +#: each built-in package holds. +_ALLOWED: dict[str, int] = { + "financial_services": 40, + "general_educational": 14, + "general_open_ended": 14, + "health_disclosure": 15, + "identity_disclosure": 27, + "safety_agentic": 30, + "safety_chat_pressure": 23, + "sov_ai_dynamic": 29, + "sov_ai_facts": 22, + "sov_ai_multilingual_parity": 22, + "tool_calling": 23, +} + + +def _packages() -> list[Path]: + return sorted(path for path in _PROBE_PACKAGES.iterdir() if (path / "__init__.py").is_file()) + + +def _internal_imports(package: Path) -> set[str]: + """``module:name`` for each name ``package`` imports from an internal ``usersim`` module.""" + own = f"usersim.engine.probes.{package.name}" + found = set() + for path in package.rglob("*.py"): + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + if isinstance(node, ast.ImportFrom) and (node.module or "").split(".")[0] == "usersim": + pairs = [(node.module, alias.name) for alias in node.names] + elif isinstance(node, ast.Import): + pairs = [(alias.name, "") for alias in node.names if alias.name.split(".")[0] == "usersim"] + else: + continue + for module, name in pairs: + if module in _PUBLIC or module == own or module.startswith(f"{own}."): + continue + found.add(f"{module}:{name}") + return found + + +def test_every_built_in_package_has_an_allowance(): + missing = {package.name: len(_internal_imports(package)) for package in _packages() if package.name not in _ALLOWED} + assert not missing, ( + f"add each to _ALLOWED with its count, after importing what you can from usersim.extension: {missing}" + ) + + +def test_no_built_in_package_imports_more_internal_names_than_allowed(): + over = { + package.name: sorted(found) + for package in _packages() + if len(found := _internal_imports(package)) > _ALLOWED.get(package.name, len(found)) + } + assert not over, ( + "A built-in package imports more names from internal usersim modules than its allowance. " + f"Import them from usersim.extension, or keep them in the probe's own package: {over}" + ) + + +def test_every_allowance_is_still_needed(): + counts = {package.name: len(_internal_imports(package)) for package in _packages()} + stale = {name: (allowed, counts.get(name)) for name, allowed in _ALLOWED.items() if counts.get(name, 0) < allowed} + assert not stale, ( + f"lower each allowance to its package's count, or drop a removed package, (allowed, counted): {stale}" + )