From 513b0c3c897d7024b08a5fb1a0b9505bd74eb962 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 15:40:19 -0700 Subject: [PATCH 01/40] test(rules-doc): example-line grammar parser and annotation registry Co-Authored-By: Claude Fable 5 --- tests/v2/rules_doc.py | 149 +++++++++++++++++++++++++++++ tests/v2/test_rules_doc_grammar.py | 85 ++++++++++++++++ 2 files changed, 234 insertions(+) create mode 100644 tests/v2/rules_doc.py create mode 100644 tests/v2/test_rules_doc_grammar.py diff --git a/tests/v2/rules_doc.py b/tests/v2/rules_doc.py new file mode 100644 index 0000000..e8b72b2 --- /dev/null +++ b/tests/v2/rules_doc.py @@ -0,0 +1,149 @@ +"""Parser and annotation registry for docs/design/rules.md example lines. + +The grammar (spec: docs/superpowers/specs/, library-rules-documentation +design; the committed description lives in rules.md's preamble): + + "INPUT" [annotation] → field=VALUE [· boundary] + [deviates: #N (today: field=VALUE)] + +plus per-rule ``no-boundary: reason`` lines and the trailing pointer +line ``history: ... · interacts: A1, B2 · implemented: path, path``. + +Inside a rule block, any line whose first non-space character is a +double quote (or an opening bracket, the D-section subject form) is an +example line and MUST parse — a silent skip would un-execute a claim, +so a malformed example is a hard error naming the rule. +""" +from __future__ import annotations + +import ast +import re +from dataclasses import dataclass, field as dc_field +from pathlib import Path + +RULES_DOC = Path(__file__).resolve().parents[2] / "docs" / "design" / "rules.md" + +_RULE_RE = re.compile(r"^([A-Z])(\d+)\.\s") +_VALUE = r"\"[^\"]*\"|\([^)]*\)|\[[^\]]*\]" +_EXAMPLE_RE = re.compile( + r'^\s*(?:"(?P[^"]*)"|\[(?P[a-z][a-z0-9_+-]*)\])' + r"(?:\s+(?P[a-z][a-z0-9_+-]*|\[[a-z][a-z0-9_+-]*\]))?" + r"\s+→\s+" + rf"(?P[a-z_]+)=(?P{_VALUE})" + r"(?:\s+·\s+boundary)?" + r"(?:\s+deviates:\s+#(?P\d+)\s+\(today:\s+" + rf"[a-z_]+=(?P{_VALUE})\))?" + r"\s*$") +_NO_BOUNDARY_RE = re.compile(r"^\s*no-boundary:\s+(?P\S.*)$") +_POINTER_RE = re.compile(r"^\s*(history|interacts|implemented):") +_POINTER_PART_RE = re.compile(r"(history|interacts|implemented):\s*([^·]+)") + +ASSERTABLE_FIELDS = frozenset({ + "title", "given", "middle", "family", "suffix", "nickname", "maiden", + "ambiguities", "pieces", "warns"}) + + +@dataclass(frozen=True) +class Example: + text: str + annotation: str | None + field: str + value: object + boundary: bool + deviates_issue: int | None + today_value: object + subject: str | None = None + + +@dataclass +class Rule: + rule_id: str + examples: list[Example] = dc_field(default_factory=list) + no_boundary: str | None = None + interacts: tuple[str, ...] = () + implemented: tuple[str, ...] = () + + def has_boundary_or_waiver(self) -> bool: + return self.no_boundary is not None or any( + e.boundary for e in self.examples) + + +from nameparser._policy import ( # noqa: E402 + FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, Policy) + +#: Named policies example annotations may reference. Grown as +#: extraction demands; each addition is a diff to this dict only. +POLICIES: dict[str, Policy] = { + "family-first": Policy(name_order=FAMILY_FIRST), + "family-first-given-last": Policy(name_order=FAMILY_FIRST_GIVEN_LAST), + "middle_as_family": Policy(middle_as_family=True), +} +#: Extras gates: locale requiring an optional dependency; the examples +#: runner skips these when the import is absent (CI's ja-extra job +#: exercises them). +GATES: dict[str, tuple[str, str]] = {"[ja+segmenter]": ("ja", "namedivider")} + +_LOCALE_ANNOT_RE = re.compile(r"\[[a-z_]{2,5}\]") + + +def resolve_annotation(annot: str) -> tuple[str, object]: + if annot in POLICIES: + return "policy", POLICIES[annot] + if annot in GATES: + return "gated_locale", GATES[annot][0] + if _LOCALE_ANNOT_RE.fullmatch(annot): + return "locale", annot[1:-1] + raise KeyError(annot) + + +def _literal(tok: str) -> object: + return ast.literal_eval(tok) + + +def parse_rules_doc(text: str) -> list[Rule]: + rules: list[Rule] = [] + current: Rule | None = None + for lineno, line in enumerate(text.splitlines(), 1): + m = _RULE_RE.match(line) + if m: + current = Rule(rule_id=m.group(1) + m.group(2)) + rules.append(current) + continue + if current is None: + continue + stripped = line.lstrip() + em = _EXAMPLE_RE.match(line) + if em: + fieldname = em.group("field") + if fieldname not in ASSERTABLE_FIELDS: + raise ValueError( + f"{current.rule_id}: line {lineno}: field " + f"{fieldname!r} not assertable") + current.examples.append(Example( + text=em.group("text") or "", + annotation=em.group("annot"), + field=fieldname, + value=_literal(em.group("value")), + boundary=" · boundary" in line, + deviates_issue=(int(em.group("issue")) + if em.group("issue") else None), + today_value=(_literal(em.group("today")) + if em.group("today") else None), + subject=em.group("subject"))) + continue + if stripped.startswith(('"', "[")): + raise ValueError( + f"{current.rule_id}: line {lineno} looks like an example " + f"but does not parse: {stripped!r}") + nb = _NO_BOUNDARY_RE.match(line) + if nb: + current.no_boundary = nb.group("reason") + continue + if _POINTER_RE.match(line): + for key, val in _POINTER_PART_RE.findall(line): + items = tuple(v.strip() for v in val.split(",") if v.strip()) + if key == "interacts": + current.interacts = items + elif key == "implemented": + current.implemented = items + return rules diff --git a/tests/v2/test_rules_doc_grammar.py b/tests/v2/test_rules_doc_grammar.py new file mode 100644 index 0000000..9fd316a --- /dev/null +++ b/tests/v2/test_rules_doc_grammar.py @@ -0,0 +1,85 @@ +"""Unit tests for the rules.md example-line grammar parser. + +DOC below is a SYNTHETIC fixture: it exists only to exercise the +grammar and is never executed against the parser. Its rule IDs and +content are illustrative (they echo the spec's worked examples for +readability) and deliberately do NOT track docs/design/rules.md — +do not "fix" this fixture when the real doc changes. +""" +from __future__ import annotations + +import pytest + +from tests.v2.rules_doc import Example, parse_rules_doc + +DOC = '''\ +# Parsing rules (synthetic grammar fixture -- not the real rules.md) + +## Particles + +Background: particles link forward to a surname. + +P2. Rationale: particles cannot themselves be a given name. + A never-given particle left alone where the given name would + go folds the whole name into the family. + "de la Vega" → family="de la Vega" + "Mesnil de" family-first → family="Mesnil de" + "de" → given="de" · boundary + Accepted: a bare "de" keeps given="de". + history: decisions.md#P2 · interacts: H1 · implemented: nameparser/_pipeline/_post_rules.py + +P3. Statement with a tracked gap. + "Swami Vivekananda" → family="" deviates: #346 (today: family="Vivekananda") + no-boundary: the rule is total over its vocabulary; no adjacent non-firing shape exists. +''' + + +def test_parses_rule_ids_and_examples() -> None: + rules = parse_rules_doc(DOC) + assert [r.rule_id for r in rules] == ["P2", "P3"] + p2 = rules[0] + assert len(p2.examples) == 3 + ex = p2.examples[0] + assert ex == Example(text="de la Vega", annotation=None, field="family", + value="de la Vega", boundary=False, + deviates_issue=None, today_value=None) + + +def test_annotation_and_boundary_flags() -> None: + p2 = parse_rules_doc(DOC)[0] + assert p2.examples[1].annotation == "family-first" + assert p2.examples[2].boundary is True + assert p2.has_boundary_or_waiver() + + +def test_deviates_marker() -> None: + p3 = parse_rules_doc(DOC)[1] + ex = p3.examples[0] + assert ex.deviates_issue == 346 + assert ex.value == "" + assert ex.today_value == "Vivekananda" + assert p3.no_boundary is not None + + +def test_pointer_line() -> None: + p2 = parse_rules_doc(DOC)[0] + assert p2.interacts == ("H1",) + assert p2.implemented == ("nameparser/_pipeline/_post_rules.py",) + + +def test_unparseable_example_line_is_an_error() -> None: + bad = 'X1. Statement.\n "input" -> family="x"\n' + with pytest.raises(ValueError, match="X1"): + parse_rules_doc(bad) + + +def test_registry_resolves_policies_and_gates() -> None: + from tests.v2.rules_doc import resolve_annotation + kind, obj = resolve_annotation("family-first") + assert kind == "policy" + kind, obj = resolve_annotation("[ru]") + assert kind == "locale" and obj == "ru" + kind, obj = resolve_annotation("[ja+segmenter]") + assert kind == "gated_locale" and obj == "ja" + with pytest.raises(KeyError): + resolve_annotation("no-such-annotation") From c66ac9a81276e4d301f930320b2128396f9dbee6 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 15:41:07 -0700 Subject: [PATCH 02/40] docs(rules): skeletons for rules.md, decisions.md, mechanisms.md Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 28 +++++++++++++ docs/design/mechanisms.md | 29 ++++++++++++++ docs/design/rules.md | 82 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 docs/design/decisions.md create mode 100644 docs/design/mechanisms.md create mode 100644 docs/design/rules.md diff --git a/docs/design/decisions.md b/docs/design/decisions.md new file mode 100644 index 0000000..3a1d96b --- /dev/null +++ b/docs/design/decisions.md @@ -0,0 +1,28 @@ +# Decisions + +The why behind the rules: a lightweight ADR log, keyed by rule ID +(`### P2 — `), by mechanism slug, or by a short slug for +cross-cutting and tooling decisions. Section headings carry the key, +so every `decisions.md#P2` reference in code or rules.md is a live +anchor. + +Entry conventions: + +- **Dated decision entries**, append-only in spirit: a reversed + decision is not edited, a later entry supersedes it. Each entry + cites its issue or PR. +- **`Declined:`** — proposals rejected WITH the evidence that killed + them. Resolved-as-no is a decision; without a home for it, the next + person re-derives the rejected proposal and its measurement. +- **`Excluded:`** — standing prohibitions with indefinite lifetime, + keyed by vocabulary set: entries that must stay OUT of a wordlist, + each with its reason. Distinct from Declined because the failure + mode differs — nobody re-derives a rejected proposal, but someone + sweeping a wordlist ships the excluded entry as a bug. +- **`Open:`** — unresolved questions as issue links with one-line + handles. The ISSUE is canonical; this block never restates it. +- **Weighing entries** for contested questions: the options + considered, each option's intended effect, and the accepted costs + of the option chosen. The costs accepted here are the artifacts + rules.md lists under the rule's `Accepted:` consequences; the two + link by rule ID. diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md new file mode 100644 index 0000000..35de0aa --- /dev/null +++ b/docs/design/mechanisms.md @@ -0,0 +1,29 @@ +# Mechanisms + +How this codebase solves recurring problem shapes. Before proposing +a design, check whether an entry already fits — the catalog is keyed +by problem shape, because that is what you have in hand when +searching. + +Entry format: an `## UPPER-KEBAB-SLUG — one-line slogan` heading, +then **Problem shape** (the situation that should trigger recall), +**Contract statement** (one or two sentences stating what the +mechanism promises — the citable line), **How it works**, **Lives +in**, and **Reach for it when** (the tell-tale sign you should be +using this instead of inventing something). + +The Contract statement is citable under the same discipline as +rules: a committed comment making a mechanism claim — including any +claim about which stage or layer does something — cites the slug and +quotes a verbatim excerpt (`# mechanisms.md#SPANS: ...`), checked by +the citation-integrity test. Free-prose restatements of mechanism +claims are how one wrong sentence has shipped six times. + +This catalog converts discovery into a one-time cost — it does not +remove discovery. What nobody knows yet still has to be found the +hard way, once; the promise is that found things stay found. + +## Verification shapes + +How to measure in this codebase without fooling yourself. (Entries +land with the mechanisms content pass.) diff --git a/docs/design/rules.md b/docs/design/rules.md new file mode 100644 index 0000000..c90e42f --- /dev/null +++ b/docs/design/rules.md @@ -0,0 +1,82 @@ +# Parsing rules + +This document is NORMATIVE, not descriptive: the rules state how +names are written and what should happen when they are parsed, +grounded in how people understand names — not in what the parser +currently does. The parser implements these rules. Where it does not +yet, the gap is a tracked deviation (`deviates:` marker below), not a +counterexample. Statements are implementation-free: no stage names, +no function names, no regexes. + +Authority, scoped: `tests/v2/cases.py` pins CURRENT behavior; this +document states INTENDED behavior. A mismatch between them must be +classified, never defaulted: either the rule is wrong (fix it here) +or the parser is wrong (the example takes a `deviates:` marker and an +issue). Where this document is silent, the behavior is +pinned-but-undocumented — an extraction gap to close, not a +specification, and not license to change the behavior. + +Rule IDs are stable forever: never renumbered, never reused. A +retired rule keeps its ID with a one-line tombstone pointing at +decisions.md. Cross-references use the anchor form `decisions.md#P2` +/ `mechanisms.md#SPANS`; a bare ID is never a citation. The +`interacts:` field on a pointer line is advisory — the +citation-integrity test checks the ID exists, not that the +interaction is real. + +Every example line is EXECUTABLE. The grammar (its executable +definition is `tests/v2/rules_doc.py`; `tests/v2/test_rules_doc.py` +runs every line): + + "INPUT" [annotation] → field="value" [· boundary] + [deviates: #N (today: field="value")] + +An `annotation` names a policy, locale (`[ru]`), or extras gate +(`[ja+segmenter]`) in the registry beside the test. `· boundary` +marks the non-firing example every rule must carry — or the rule +declares `no-boundary: ` instead, so skipping the boundary is +a recorded decision. `deviates:` states the INTENDED output on the +example line while the marker records TODAY's output and the tracking +issue; the runner asserts today's output strictly, so a parser change +that closes the gap fails the suite until the marker is removed in +the same PR. `grep deviates:` on this file is the deviation backlog +(deviations from statable rules — coverage gaps are a separate, +larger category no grep can see). + +## Not in scope + +- **Language detection.** The parser never infers a language from + Latin-script text: transliteration destroys the signal ("Ali", + "Van", "Bin" each belong to several languages with conflicting + readings). Language-specific behavior is opt-in configuration. + Script-conditional behavior exists only where the script itself + settles the convention (see the W section). +- **Grammatical inflection.** Names inflect in many languages + (vocative, genitive); this library neither produces nor consumes + inflected forms. CLDR personNames draws the same line. +- **Validation.** Deciding whether a string IS a person's name is not + parsing; `parse()` is total over strings and never rejects input. + +## Titles & honorifics (H) + +## Particles & surname prefixes (P) + +## Suffixes: generational & credentials (S) + +## Nicknames & quoted names (N) + +## Maiden names (M) + +## Commas & structure (C) + +## Name order (O) + +## Scripts & writing systems (W) + +## Tokens, initials & punctuation (T) + +## Ambiguity & tie-breaking (A) + +## Rendering & views (R) + +## Construction & configuration diagnostics (D) From fb2b141d30a3208a4a5de225471adf53f6971319 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 15:41:42 -0700 Subject: [PATCH 03/40] test(rules-doc): executable-examples runner Co-Authored-By: Claude Fable 5 --- tests/v2/test_rules_doc.py | 79 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 tests/v2/test_rules_doc.py diff --git a/tests/v2/test_rules_doc.py b/tests/v2/test_rules_doc.py new file mode 100644 index 0000000..d7991c3 --- /dev/null +++ b/tests/v2/test_rules_doc.py @@ -0,0 +1,79 @@ +"""Executes every example line in docs/design/rules.md. + +Wrong-on-arrival defense: each example runs against the live parser. +``deviates:`` examples assert TODAY's output (strict, like an xfail) +and additionally that intended != today, so a fixed bug forces the +marker out in the same PR. +""" +from __future__ import annotations + +import importlib.util + +import pytest + +from nameparser import Parser, parse, parser_for +from nameparser import locales +from nameparser._policy import Policy +from tests.v2.rules_doc import ( + RULES_DOC, Example, Rule, parse_rules_doc, resolve_annotation) + +RULES = parse_rules_doc(RULES_DOC.read_text(encoding="utf-8")) + + +@pytest.mark.xfail(strict=True, reason="until the first extraction pass") +def test_doc_has_rules() -> None: + assert RULES, "rules.md holds no rules yet -- extraction not started" + + +@pytest.mark.parametrize("rule", RULES, ids=lambda r: r.rule_id) +def test_every_rule_has_examples_and_boundary(rule: Rule) -> None: + assert rule.examples, f"{rule.rule_id} has no examples" + assert rule.has_boundary_or_waiver(), ( + f"{rule.rule_id}: add a '· boundary' example or an explicit " + f"'no-boundary: '") + + +def _run(example: Example) -> object: + if example.field in ("warns", "ambiguities", "pieces"): + pytest.skip("assertion form lands with its first using rule") + policy: Policy | None = None + locale: str | None = None + if example.annotation is not None: + kind, obj = resolve_annotation(example.annotation) + if kind == "policy": + assert isinstance(obj, Policy) + policy = obj + elif kind == "locale": + assert isinstance(obj, str) + locale = obj + elif kind == "gated_locale": + if importlib.util.find_spec("namedivider") is None: + pytest.skip("optional extra absent; the ja-extra CI job " + "runs this") + assert isinstance(obj, str) + locale = obj + if locale is not None: + parsed = parser_for(locales.get(locale)).parse(example.text) + elif policy is not None: + parsed = Parser(policy=policy).parse(example.text) + else: + parsed = parse(example.text) + return getattr(parsed, example.field) + + +_EXAMPLES = [(r, e) for r in RULES for e in r.examples] +_EXAMPLE_IDS = [f"{r.rule_id}-{i}" for r in RULES + for i, _ in enumerate(r.examples)] + + +@pytest.mark.parametrize("rule, example", _EXAMPLES, ids=_EXAMPLE_IDS) +def test_example(rule: Rule, example: Example) -> None: + actual = _run(example) + if example.deviates_issue is not None: + assert actual == example.today_value, ( + f"{rule.rule_id}: deviation #{example.deviates_issue} moved -- " + f"update or remove the marker in the same PR") + assert example.value != example.today_value, ( + f"{rule.rule_id}: intended equals today -- remove the marker") + else: + assert actual == example.value From 16d59dced0f19328bc2b52312d530d27d5bc2842 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 15:42:52 -0700 Subject: [PATCH 04/40] test(rules-doc): citation integrity (legacy dormant) + spelling denylist The denylist's first run caught its seed entry live in usage.rst and release_log.rst -- Policy(segment_scripts=()) fails mypy; both sites now hand out the frozenset() spelling the warning itself offers. Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 2 +- docs/usage.rst | 2 +- tests/v2/test_doc_citations.py | 118 +++++++++++++++++++++++++++++++++ tests/v2/test_doc_spellings.py | 30 +++++++++ 4 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 tests/v2/test_doc_citations.py create mode 100644 tests/v2/test_doc_spellings.py diff --git a/docs/release_log.rst b/docs/release_log.rst index 45aa83a..f6d3add 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -86,7 +86,7 @@ Release Log - Add ``AmbiguityKind.SEGMENTATION``, reported when a surname split had a vocabulary-supported alternative: ``"남궁민수"`` is 남궁 + 민수 by the compound surname but 남 + 궁민수 by the single-syllable one, and longest-match had to pick. A name with only one possible split reports nothing (#271) - Add the Japanese locale pack ``locales.JA`` and the segmenter factory ``locales.ja_segmenter()``, which together divide an unspaced Japanese name: ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` reads ``山田太郎`` as family ``山田``, given ``太郎``. They are separate because no surname list can do this job, so the pack activates the stage and a third-party divider performs it. ``ja_segmenter()`` wraps `namedivider-python `_, installed with the new ``nameparser[ja]`` extra; the core stays dependency-free. ``locales.available()`` is now ``('ja', 'ru', 'tr_az', 'zh')``. See :doc:`locales` (closes #272) - Add ``Segmentation`` and the ``Segmenter`` type alias to the public API, plus the keyword-only ``Parser(segmenter=...)`` hook: any callable from a token's text to a ``Segmentation`` or to ``None`` to decline. It is consulted only for scripts in ``Policy.segment_scripts``, and only where the surname vocabulary declined first. Note what that ordering means when packs are stacked: a Japanese name opening on a listed Chinese surname never reaches the segmenter, so ``高橋一郎`` still splits ``高`` + ``橋一郎``. The packs are alternatives, one per corpus. See :ref:`segmenter-contract` (#272) - - Add a construction-time ``UserWarning`` when a parser activates segmentation for scripts nothing can divide. ``parser_for(locales.JA)`` without ``segmenter=`` used to build a parser that behaved like a working one minus the feature, silently. It now names the dead scripts and the call to pass. Any configured segmenter or covering surname vocabulary silences it, so the default parser and the ``zh`` pack never warn. A from-scratch lexicon with no hangul surnames warns under the default policy, with ``Policy(segment_scripts=())`` offered as the deactivation. See :ref:`east-asian-names` + - Add a construction-time ``UserWarning`` when a parser activates segmentation for scripts nothing can divide. ``parser_for(locales.JA)`` without ``segmenter=`` used to build a parser that behaved like a working one minus the feature, silently. It now names the dead scripts and the call to pass. Any configured segmenter or covering surname vocabulary silences it, so the default parser and the ``zh`` pack never warn. A from-scratch lexicon with no hangul surnames warns under the default policy, with ``Policy(segment_scripts=frozenset())`` offered as the deactivation. See :ref:`east-asian-names` - Add the ``Script`` members ``HIRAGANA`` and ``KATAKANA``. Two members rather than one ``KANA`` because the parser treats them differently: hiragana never transcribes a foreign name, while a wholly-katakana name usually is one (#272) **Breaking Changes** diff --git a/docs/usage.rst b/docs/usage.rst index 857b262..5aedd40 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -294,7 +294,7 @@ call to pass, because the misconfigured parser would otherwise behave exactly like a working one minus the feature. The same check guards any configuration whose activated scripts nothing can serve. A from-scratch lexicon with no hangul surnames warns under the default -policy, and ``Policy(segment_scripts=())`` is the deactivation the +policy, and ``Policy(segment_scripts=frozenset())`` is the deactivation the message offers. Decomposed text diff --git a/tests/v2/test_doc_citations.py b/tests/v2/test_doc_citations.py new file mode 100644 index 0000000..e9fed6c --- /dev/null +++ b/tests/v2/test_doc_citations.py @@ -0,0 +1,118 @@ +"""Referential integrity for docs/design/ citations. + +Checks: cited rule/mechanism IDs exist; citation sentences are +whitespace-normalized verbatim excerpts of their statements; +``implemented:`` lists match the set of modules actually citing the +rule; ``interacts:`` IDs exist (existence only -- the field is +advisory). The legacy-pattern check stays OFF until the final rewrite +pass arms it. +""" +from __future__ import annotations + +import re +from pathlib import Path + +from tests.v2.rules_doc import RULES_DOC, parse_rules_doc + +REPO = Path(__file__).resolve().parents[2] +MECH_DOC = REPO / "docs" / "design" / "mechanisms.md" +SWEEP_DIRS = ("nameparser", "tests", "tools") +ENFORCE_NO_LEGACY = False # armed by the final rewrite pass +_LEGACY = ("§", "superpowers", "plan deviation") + +_CITE_RE = re.compile( + r"#\s*(?:rules|mechanisms|decisions)\.md#" + r"(?P[A-Z]\d+|[A-Z][A-Z0-9]+(?:-[A-Z0-9]+)+)" + r":\s*(?P.*)") + + +def _norm(s: str) -> str: + return " ".join(s.split()).lower() + + +def _statements() -> dict[str, str]: + out: dict[str, str] = {} + text = RULES_DOC.read_text(encoding="utf-8") + for block in re.split(r"^(?=[A-Z]\d+\.\s)", text, flags=re.M): + m = re.match(r"([A-Z]\d+)\.\s(.*)", block, flags=re.S) + if m: + body = m.group(2).split('\n "')[0] # stop at examples + out[m.group(1)] = _norm(body) + mech = MECH_DOC.read_text(encoding="utf-8") + for mm in re.finditer( + r"^## (?P[A-Z][A-Z0-9-]+)\b.*?Contract statement[.:*]*\s*" + r"(?P.+?)(?=\n\n|\Z)", mech, flags=re.M | re.S): + out[mm.group("slug")] = _norm(mm.group("stmt")) + return out + + +def _citations() -> list[tuple[Path, int, str, str]]: + found = [] + for d in SWEEP_DIRS: + for path in sorted((REPO / d).rglob("*.py")): + lines = path.read_text(encoding="utf-8").splitlines() + for i, line in enumerate(lines): + m = _CITE_RE.search(line) + if not m: + continue + excerpt = [m.group("first")] + for cont in lines[i + 1:]: + cs = cont.strip() + if cs.startswith("#") and not _CITE_RE.search(cont): + excerpt.append(cs.lstrip("# ")) + else: + break + found.append((path, i + 1, m.group("cid"), + _norm(" ".join(excerpt)))) + return found + + +def test_citations_are_verbatim_excerpts() -> None: + statements = _statements() + problems = [] + for path, lineno, cid, excerpt in _citations(): + if cid not in statements: + problems.append(f"{path}:{lineno}: cites unknown ID {cid}") + elif excerpt and excerpt not in statements[cid]: + problems.append( + f"{path}:{lineno}: not a verbatim excerpt of {cid}") + assert not problems, "\n".join(problems) + + +def test_implemented_matches_citing_modules() -> None: + citing: dict[str, set[str]] = {} + for path, _lineno, cid, _x in _citations(): + citing.setdefault(cid, set()).add(str(path.relative_to(REPO))) + problems = [] + for rule in parse_rules_doc(RULES_DOC.read_text(encoding="utf-8")): + if rule.implemented: + actual = citing.get(rule.rule_id, set()) + declared = set(rule.implemented) + if actual != declared: + problems.append( + f"{rule.rule_id}: implemented: says {sorted(declared)} " + f"but citations found in {sorted(actual)}") + assert not problems, "\n".join(problems) + + +def test_interacts_ids_exist() -> None: + rules = parse_rules_doc(RULES_DOC.read_text(encoding="utf-8")) + ids = {r.rule_id for r in rules} + missing = [f"{r.rule_id} -> {t}" for r in rules + for t in r.interacts if t not in ids] + assert not missing, f"advisory interacts: cite unknown IDs: {missing}" + + +def test_no_legacy_citations() -> None: + if not ENFORCE_NO_LEGACY: + return # armed by the final rewrite pass + problems = [] + for d in SWEEP_DIRS: + for path in sorted((REPO / d).rglob("*")): + if path.suffix not in (".py", ".toml"): + continue + text = path.read_text(encoding="utf-8", errors="ignore") + for pat in _LEGACY: + if pat in text: + problems.append(f"{path}: contains {pat!r}") + assert not problems, "\n".join(problems) diff --git a/tests/v2/test_doc_spellings.py b/tests/v2/test_doc_spellings.py new file mode 100644 index 0000000..aca2297 --- /dev/null +++ b/tests/v2/test_doc_spellings.py @@ -0,0 +1,30 @@ +"""Recorded roster of spellings that have actually shipped wrong in +docs, docstrings, or messages. Add an entry when one is caught; never +remove one without a decisions.md note.""" +from __future__ import annotations + +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +DENYLIST: dict[str, str] = { + "Policy(segment_scripts=())": ( + "#334/#337: tuple literal fails mypy on this field; " + "the type-clean spelling is frozenset()"), + "segment_scripts=()": "#334: same arg-type error, keyword form", +} +SWEEP = ("nameparser", "docs") + + +def test_no_denylisted_spellings() -> None: + hits = [] + for d in SWEEP: + for path in sorted((REPO / d).rglob("*")): + if path.suffix not in (".py", ".rst", ".md"): + continue + if "superpowers" in path.parts: + continue + text = path.read_text(encoding="utf-8", errors="ignore") + for bad, why in DENYLIST.items(): + if bad in text: + hits.append(f"{path}: {bad!r} ({why})") + assert not hits, "\n".join(hits) From 7187c8bd98f70113ea01f1644db8fa90e910f42e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 15:55:33 -0700 Subject: [PATCH 05/40] docs(rules): extract post_rules into H/P/O (template pass) Five rules (H1, P1, O1-O3) with executable examples; decisions.md gains P1/O1/O2 history; the 57-line rule-1b comment block becomes a 10-line citation. The adversarial probe falsified O2's 'exactly four words' claim (a suffix-bearing five-word name rotates); the statement now counts name words with titles, suffixes and nicknames set aside, and the falsifying input is pinned as an example. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 44 +++++++++ docs/design/rules.md | 95 ++++++++++++++++++ nameparser/_pipeline/_post_rules.py | 143 +++++++--------------------- tests/v2/test_doc_citations.py | 16 +++- tests/v2/test_rules_doc.py | 1 - 5 files changed, 186 insertions(+), 113 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 3a1d96b..9acdb92 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -26,3 +26,47 @@ Entry conventions: of the option chosen. The costs accepted here are the artifacts rules.md lists under the rule's `Accepted:` consequences; the two link by rule ID. + +### P1 — lone particle fold + +A lone PIECE is the whole test — deliberately narrower than "a +never-given particle is never reported as the given name," which +would be false. Under a family-first order, "Juan de la Vega" holds +the entire chained group in the given position — three words, not a +lone particle — so P1 declines and given="de la Vega" stands; #359 +records that case as working as intended. The MIDDLE position is +deliberately not a fold site. + +- 2026-08 #359 — the opening site is read from joining structure + (pieces), not from assigned roles, so the fold holds under every + name_order. Before this, the role-only read let "de Mesnil" split + under a family-first order. +- 2026-08 #367 — titles are transparent to the fold: "Sir de + Mesnil" now reads like "de Mesnil". Fixed by removing the + title→particle chain in grouping, not by touching this rule. + +Open: [#364](https://github.com/derek73/python-nameparser/issues/364) +how much the fold takes · +[#365](https://github.com/derek73/python-nameparser/issues/365) +should the middle position be a third site · +[#360](https://github.com/derek73/python-nameparser/issues/360) +which particles count as never-given. + +### O1 — East Slavic rotation + +- 2026-07-12 — v1 parity pinned live: the rotation reconstructs + token position from assigned roles, which is faithful to v1 only + under the default given-first order. + +Open: [#270](https://github.com/derek73/python-nameparser/issues/270) +how the rotations interact with non-default name_order values. + +### O2 — Turkic rotation + +- 2026-07-02 — shape fixed at exactly four words (1 given + 2 + middle + 1 marker), v1 parity; other shapes keep their positional + reading even when that leaves the marker in a name field (see the + rule's Accepted consequence). + +Open: [#270](https://github.com/derek73/python-nameparser/issues/270) +same rotation/name_order interaction as O1. diff --git a/docs/design/rules.md b/docs/design/rules.md index c90e42f..a602273 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -59,8 +59,51 @@ larger category no grep can see). ## Titles & honorifics (H) +Background: an honorific title precedes a name and is not itself part +of it; it addresses or ranks the person. Most titles address by +surname ("Mr. Johnson"), but a few — knighthoods, some clerical and +courtesy titles — address by given name ("Sir John"). The library +keeps a vocabulary of titles and, separately, of these given-name +titles. + +H1. Rationale: a title normally addresses by surname, so a title + followed by a single name word usually names the family; but a + given-name title addresses by given name. + A title followed by exactly one name word and nothing else makes + that word the family name, unless the title is a given-name + title, which keeps it the given name. + "Mr. Johnson" → family="Johnson" + "Mrs. Garcia" → family="Garcia" + "Sir John" → given="John" · boundary + implemented: nameparser/_pipeline/_post_rules.py + ## Particles & surname prefixes (P) +Background: particles ("de", "la", "van", "von", "bin") link forward +to a surname and are written as part of it. Some are never anyone's +given name; others ("Van", "Bin") are ordinary given names in some +cultures, so the vocabulary distinguishes never-given particles from +ambiguous ones, and only the never-given ones license special +treatment. Which particles fall on which side of that line is its own +open question (#360). + +P1. Rationale: a never-given particle standing alone cannot be + someone's given name; a name that opens with one, or offers only + one as the given name, is a surname written out in full. + A never-given particle standing alone where the given name would + go — or opening the name — marks the name as surname-only: the + given and middle words fold into the family. It needs another + name word to fold into. An ambiguous particle keeps whatever + reading its position gives it. + "de la Vega" → family="de la Vega" + "Mesnil de" family-first → family="Mesnil de" + "Juan de la Vega" family-first → given="de la Vega" · boundary + "van Gogh" → given="van" · boundary + Accepted: a bare "de" stays the given name — there is nothing to + fold into, and inventing a surname would be worse. + "de" → given="de" + history: decisions.md#P1 · implemented: nameparser/_pipeline/_post_rules.py + ## Suffixes: generational & credentials (S) ## Nicknames & quoted names (N) @@ -71,6 +114,58 @@ larger category no grep can see). ## Name order (O) +Background: written name order varies by convention: given-first +(the library's default reading), family-first, and family-first with +the given name last (Vietnamese). The order is declared by the +caller or a locale pack, never detected — but a few conventions +leave a recognizable trace in the name itself. Patronymics are one: +East Slavic names carry a father's-name derivative with distinctive +endings between given and family, and Turkic names use a standalone +marker word ("oglu" son-of, "qizi" daughter-of) after the father's +name. Where such a trace is present and unambiguous, an opted-in +parser can restore the intended reading from a family-first listing. + +O1. Rationale: an East Slavic name written family-first still shows + its patronymic — the distinctive ending identifies which word is + the patronymic, and the patronymic sits next to the given name. + With East Slavic patronymic handling active and no comma in the + name, a name of exactly three name words — titles, suffixes and + nicknames aside — whose last name word carries a patronymic + ending and whose middle name word does not reads as family-first: + the words are restored to given, patronymic, family. A middle + word that also carries the ending blocks the reading, because + the surname itself may be patronymic-derived. + "Сидоров Иван Петрович" [ru] → family="Сидоров" + "Sidorov Ivan Petrovich Jr." [ru] → family="Sidorov" + "Иван Петрович Абрамович" [ru] → family="Абрамович" · boundary + history: decisions.md#O1 · implemented: nameparser/_pipeline/_post_rules.py + +O2. Rationale: a Turkic patronymic marker is a separate word that + follows the father's name; a four-word name ending in one is a + family-first listing. + With Turkic patronymic handling active and no comma in the name, + a name of exactly four name words — titles, suffixes and + nicknames aside — ending in a standalone patronymic marker reads + family-first: the first name word is the family name, and the + marker stays beside the father's name in the middle. + "Ali Ahmad Vali oglu" [tr_az] → family="Ali" + "Ali Ahmad Vali oglu Jr." [tr_az] → family="Ali" + Accepted: any other count of name words keeps its positional + reading, even when that leaves the marker itself in a name + field. + "Ali Ahmad oglu" [tr_az] → family="oglu" · boundary + history: decisions.md#O2 · implemented: nameparser/_pipeline/_post_rules.py + +O3. Rationale: several traditions write compound family names + unmarked, so that every word after the given name belongs to the + family name. + With compound-family handling active, every middle word joins + the family name and is rendered before it; no word is a middle + name. + "Hassan Mohamad Ali" middle_as_family → family="Mohamad Ali" + "Hassan Mohamad Ali" → family="Ali" · boundary + implemented: nameparser/_pipeline/_post_rules.py + ## Scripts & writing systems (W) ## Tokens, initials & punctuation (T) diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index 78ae161..bd0b321 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -1,47 +1,16 @@ """Stage: post_rules. -Consumes: tokens (roles assigned), plus pieces and structure -- rule 1b -reads the opening piece of segment 0, or of segment 1 under a family -comma (#359). structure was always read here, for the rotation gate. +Consumes: tokens (roles assigned), plus pieces and structure -- the +particle fold reads the opening piece of segment 0, or of segment 1 +under a family comma (#359). structure was always read here, for the +rotation gate. Produces: tokens with roles adjusted by the post rules. Reads: Policy.patronymic_rules, Policy.middle_as_family; Lexicon.given_name_titles. -Rules (each a small pure function over the role-bearing tokens): -1. v1 handle_firstnames: when the parse is exactly a title plus ONE - given token (no other roles), and the title is not a given-name - title ('Sir'), that token is a family name -- "Mr. Johnson". -1b. where a particle that is never a given name stands ALONE as a - piece -- either opening the name or in the given position -- the - name is left with no given name at all: the given and the middles - fold into the family. Opening the name it pulls the rest of it in - ("de la Vega"); in the given position it folds into the family - beside it ("Mesnil de" under a family-first order). Needs another - name token to fold into, so a bare "de" stays as it is. Alone among - these rules it reads the opening position from `pieces` rather than - from the roles assign left, so that shape holds for a lone leading - particle piece under every name_order (#359). -2. EAST_SLAVIC (opt-in): positional GIVEN/MIDDLE/FAMILY each exactly - one token, the FAMILY-position token carries an East Slavic - patronymic ending, and the MIDDLE-position token does NOT (given + - patronymic + patronymic-derived surname like Abramovich must not - rotate) -> rotate: given<-old MIDDLE, middle<-old FAMILY (the - patronymic), family<-old GIVEN (v1 parity, pinned live 2026-07-12). -3. TURKIC (opt-in): exactly 1 GIVEN + 2 MIDDLE + 1 FAMILY tokens and - the FAMILY-position token is a standalone Turkic marker -> - given<-first MIDDLE, middle<-(second MIDDLE, marker), family<-old - GIVEN. - -Both rotations fire only on Structure.NO_COMMA (v1 gates them on -`not self._had_comma`): a comma already established the family. - -The rotations reconstruct token POSITION from roles, which is faithful -to v1 only under the default GIVEN_FIRST order; their interaction with -other name_order values is an open design question for the locale-pack -work (#270). Rule 1b read its particle the same way until #359 gave -it the position test as well, the decision there being that a -never-given particle keeps its particle whatever order the caller -declared. +Implements rules H1, P1, O1, O2 and O3 of docs/design/rules.md; each +is cited at its code below, and the history lives in +docs/design/decisions.md. """ from __future__ import annotations @@ -109,81 +78,30 @@ def post_rules(state: ParseState) -> ParseState: others = any(t.role in (Role.SUFFIX, Role.NICKNAME, Role.MAIDEN) for t in tokens) - # rule 1: title + lone given -> family (v1 handle_firstnames) + # rules.md#H1: "a title followed by exactly one name word and + # nothing else makes that word the family name, unless the title + # is a given-name title" (v1 handle_firstnames) if titles and givens and not middles and not families and not others: joined = _title_key(tokens[i].text for i in titles) if joined not in state.lexicon.given_name_titles: for i in givens: _retag(tokens, i, Role.FAMILY) - # every rule below reads these lists; recompute them the way - # 1b does after its own fold, so no guard can inspect a name - # that has already moved. Measured harmless today -- over the - # 751 differential names in four policies this arm fires 48 - # times, and 1b fires on none of them -- but reading a stale - # token list is the shape of the bug #359 fixed. `middles` - # is empty by the guard above and recomputed anyway, so - # relaxing that guard cannot leave it stale. + # every rule below reads these lists; recompute after any + # retag so no guard can inspect a name that has already + # moved -- a stale index list is the bug shape #359 fixed givens = _idx(tokens, Role.GIVEN) middles = _idx(tokens, Role.MIDDLE) families = _idx(tokens, Role.FAMILY) - # rule 1b enforces one invariant (v1 handle_non_first_name_prefix): - # where a particle that is NEVER a given name stands ALONE as a - # piece -- either opening the name, or in the given position -- the - # name is left with no given name at all, the given and the middles - # joining the family. Two shapes, one repair: - # * the particle OPENS the name, so the whole name is a surname - # and it pulls the rest in -- "de la Vega"; - # * the particle is left ALONE in the given position, so it folds - # into the family beside it -- "Mesnil de" under - # name_order=FAMILY_FIRST, where the given position is the - # trailing piece. - # A lone PIECE is the whole of it, which is a clause narrower than - # "a member is never reported as the given name" -- that reading - # would be false. Under FAMILY_FIRST the given position of "Juan de - # la Vega" holds the whole chain, three tokens rather than a lone - # particle, so 1b declines and given='de la Vega' stands; #359 - # records that case as working as intended. And the degenerate bare - # 'de' keeps given='de', having nothing to fold into. - # "Sir de Mesnil" used to be this guard declining on a chained - # piece, reporting given='de Mesnil' with no family at all. That - # was never a limit this rule meant to draw, and #367 removed the - # chain rather than touching the rule: a title is transparent to - # the leading-particle exception, so 'de' is a lone piece again, - # this guard fires, and the name reads family='de Mesnil' like the - # untitled form. - # Those two sites are the whole scope, and the MIDDLE position is - # deliberately not one of them -- which shows: "Mesnil Garcia de" - # strands middle='de' under FAMILY_FIRST, while under - # FAMILY_FIRST_GIVEN_LAST the same trailing piece IS the given - # position, so it folds to family='Mesnil Garcia de'. Whether that - # difference should stand is #365, not this rule's to settle. How - # much the fold takes once it fires is the other open question: - # "de Mesnil Juan" goes wholly to the family in every order, - # matching the default rather than stopping at the particle group - # (#364). - # Only a never-given particle is in scope: an ambiguous one keeps - # whatever reading name_order gives it -- 'van Gogh' is given - # 'van' in the default order and family 'van' under a family-first - # one -- and #360 tracks the vocabulary line. - # The opening shape is read from `pieces` rather than from the role - # assign left (#359). Under the default order the opening piece IS - # the given, so the one role test used to catch both shapes; under - # FAMILY_FIRST the opening piece is the family and the given sits - # behind it, and reading the role alone let "de Mesnil" split. The - # single-token test says the same thing in each shape: a particle - # group already chained forward is not a lone particle -- the - # FAMILY_FIRST "Juan de la Vega" above is what that looks like. - # "Mr. de Mesnil" is NOT one, and since #367 not even close to - # one: it is three tokens in THREE pieces -- the title, the - # particle, the surname -- because a title no longer displaces the - # particle out of the leading name position, so nothing chains. - # Both sites are one token long, so this guard FIRES and the - # family reading is its own. Rule 1 above cannot be what produces - # it: rule 1 is gated on `not families`, and 'Mesnil' is already - # the family. Both shapes need another name token to fold with, - # which leaves a degenerate bare 'de' as it stands rather than - # inventing a surname. + # rules.md#P1: "a never-given particle standing alone where the + # given name would go — or opening the name — marks the name as + # surname-only: the given and middle words fold into the family. + # It needs another name word to fold into." (v1 + # handle_non_first_name_prefix; history: decisions.md#P1) + # Code-local: a lone PIECE is the test at both sites, so a + # particle group already chained forward is not a lone particle, + # and rule H1 above cannot be what produces the fold's family + # reading -- H1 is gated on `not families`. sites = (_leading_name_piece(state, tokens), tuple(givens)) if len(givens) + len(middles) + len(families) > 1 and any( len(site) == 1 @@ -202,6 +120,10 @@ def post_rules(state: ParseState) -> ParseState: # patronymics first, then handle_middle_name_as_last) rules = state.policy.patronymic_rules rotations_apply = state.structure is Structure.NO_COMMA + # rules.md#O1: "a name of exactly three name words — titles, + # suffixes and nicknames aside — whose last name word carries a + # patronymic ending and whose middle name word does not reads as + # family-first" (history: decisions.md#O1) if rotations_apply and PatronymicRule.EAST_SLAVIC in rules and \ len(givens) == 1 and len(middles) == 1 and len(families) == 1: tail = tokens[families[0]].text @@ -213,6 +135,10 @@ def post_rules(state: ParseState) -> ParseState: _retag(tokens, m, Role.GIVEN) _retag(tokens, f, Role.MIDDLE) _retag(tokens, g, Role.FAMILY) + # rules.md#O2: "a name of exactly four name words — titles, + # suffixes and nicknames aside — ending in a standalone + # patronymic marker reads family-first: the first name word is + # the family name" (history: decisions.md#O2) if rotations_apply and PatronymicRule.TURKIC in rules and \ len(givens) == 1 and len(middles) == 2 and len(families) == 1: tail = tokens[families[0]].text @@ -222,10 +148,11 @@ def post_rules(state: ParseState) -> ParseState: _retag(tokens, m2, Role.MIDDLE) _retag(tokens, f, Role.MIDDLE) _retag(tokens, g, Role.FAMILY) - # rule 4: opt-in fold of middles into family (v1 - # handle_middle_name_as_last). v1 PREPENDED middle_list to - # last_list; spans cannot reorder (anti-#100), so folded tokens - # carry a tag and the family views order them first. + # rules.md#O3: "every middle word joins the family name and is + # rendered before it" (v1 handle_middle_name_as_last). v1 + # PREPENDED middle_list to last_list; spans cannot reorder + # (anti-#100), so folded tokens carry a tag and the family views + # order them first. if state.policy.middle_as_family: for i in _idx(tokens, Role.MIDDLE): tokens[i] = dataclasses.replace( diff --git a/tests/v2/test_doc_citations.py b/tests/v2/test_doc_citations.py index e9fed6c..66e7e5b 100644 --- a/tests/v2/test_doc_citations.py +++ b/tests/v2/test_doc_citations.py @@ -24,6 +24,10 @@ r"#\s*(?:rules|mechanisms|decisions)\.md#" r"(?P[A-Z]\d+|[A-Z][A-Z0-9]+(?:-[A-Z0-9]+)+)" r":\s*(?P.*)") +# The excerpt is the FIRST double-quoted span after the ID, wrapped +# over continuation comment lines; text outside the quotes (v1 names, +# history pointers, code-local notes) is free. +_EXCERPT_RE = re.compile(r'"(.*?)"') def _norm(s: str) -> str: @@ -55,15 +59,16 @@ def _citations() -> list[tuple[Path, int, str, str]]: m = _CITE_RE.search(line) if not m: continue - excerpt = [m.group("first")] + block = [m.group("first")] for cont in lines[i + 1:]: cs = cont.strip() if cs.startswith("#") and not _CITE_RE.search(cont): - excerpt.append(cs.lstrip("# ")) + block.append(cs.lstrip("# ")) else: break + qm = _EXCERPT_RE.search(" ".join(block)) found.append((path, i + 1, m.group("cid"), - _norm(" ".join(excerpt)))) + _norm(qm.group(1)) if qm else "")) return found @@ -73,7 +78,10 @@ def test_citations_are_verbatim_excerpts() -> None: for path, lineno, cid, excerpt in _citations(): if cid not in statements: problems.append(f"{path}:{lineno}: cites unknown ID {cid}") - elif excerpt and excerpt not in statements[cid]: + elif not excerpt: + problems.append( + f"{path}:{lineno}: citation of {cid} has no quoted excerpt") + elif excerpt not in statements[cid]: problems.append( f"{path}:{lineno}: not a verbatim excerpt of {cid}") assert not problems, "\n".join(problems) diff --git a/tests/v2/test_rules_doc.py b/tests/v2/test_rules_doc.py index d7991c3..e6a7414 100644 --- a/tests/v2/test_rules_doc.py +++ b/tests/v2/test_rules_doc.py @@ -20,7 +20,6 @@ RULES = parse_rules_doc(RULES_DOC.read_text(encoding="utf-8")) -@pytest.mark.xfail(strict=True, reason="until the first extraction pass") def test_doc_has_rules() -> None: assert RULES, "rules.md holds no rules yet -- extraction not started" From 4734cd21bc89ac8c78258cdd30c1f51d050ff37c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 16:01:38 -0700 Subject: [PATCH 06/40] docs(rules): spec-review fixes for the foundation Old-numbering map in decisions.md; drop the gitignored-spec pointer from rules_doc.py; PR citations on O1/O2 entries; runner rejects an ungated [ja] annotation (the extra-absent CI split). Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 23 ++++++++++++++++------- tests/v2/rules_doc.py | 4 ++-- tests/v2/test_rules_doc.py | 3 +++ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 9acdb92..228a628 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -27,6 +27,13 @@ Entry conventions: rules.md lists under the rule's `Accepted:` consequences; the two link by rule ID. +### legacy-rule-numbers — the old docstring numbering + +Before rules.md, the post_rules stage docstring numbered its rules +locally, and historical issue comments (#359, #364, #365, #367) use +those numbers. The mapping: rule 1 → H1, rule 1b → P1, rule 2 → O1, +rule 3 → O2, rule 4 → O3. + ### P1 — lone particle fold A lone PIECE is the whole test — deliberately narrower than "a @@ -54,19 +61,21 @@ which particles count as never-given. ### O1 — East Slavic rotation -- 2026-07-12 — v1 parity pinned live: the rotation reconstructs - token position from assigned roles, which is faithful to v1 only - under the default given-first order. +- 2026-07-12 (landed in the v2 core, PR #288) — v1 parity pinned + live: the rotation reconstructs token position from assigned + roles, which is faithful to v1 only under the default given-first + order. Open: [#270](https://github.com/derek73/python-nameparser/issues/270) how the rotations interact with non-default name_order values. ### O2 — Turkic rotation -- 2026-07-02 — shape fixed at exactly four words (1 given + 2 - middle + 1 marker), v1 parity; other shapes keep their positional - reading even when that leaves the marker in a name field (see the - rule's Accepted consequence). +- 2026-07-02 (landed in the v2 core, PR #288) — shape fixed at + exactly four name words (1 given + 2 middle + 1 marker), v1 + parity; other shapes keep their positional reading even when that + leaves the marker in a name field (see the rule's Accepted + consequence). Open: [#270](https://github.com/derek73/python-nameparser/issues/270) same rotation/name_order interaction as O1. diff --git a/tests/v2/rules_doc.py b/tests/v2/rules_doc.py index e8b72b2..29b514a 100644 --- a/tests/v2/rules_doc.py +++ b/tests/v2/rules_doc.py @@ -1,7 +1,7 @@ """Parser and annotation registry for docs/design/rules.md example lines. -The grammar (spec: docs/superpowers/specs/, library-rules-documentation -design; the committed description lives in rules.md's preamble): +The grammar (described for humans in rules.md's preamble; this module +is its executable definition): "INPUT" [annotation] → field=VALUE [· boundary] [deviates: #N (today: field=VALUE)] diff --git a/tests/v2/test_rules_doc.py b/tests/v2/test_rules_doc.py index e6a7414..dcf6c89 100644 --- a/tests/v2/test_rules_doc.py +++ b/tests/v2/test_rules_doc.py @@ -44,6 +44,9 @@ def _run(example: Example) -> object: policy = obj elif kind == "locale": assert isinstance(obj, str) + assert obj != "ja", ( + "ja needs the optional segmenter: use the " + "[ja+segmenter] gate so CI without the extra skips it") locale = obj elif kind == "gated_locale": if importlib.util.find_spec("namedivider") is None: From caa671418a914ee8f0c1030e1c3d2a73ab4880fb Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 16:08:05 -0700 Subject: [PATCH 07/40] test(rules-doc): quality-review fixes -- slug ID classes, nested pieces values Citation ID grammar now admits bare and underscored mechanism slugs (SPANS, FOLDED_TAG); the value grammar accepts one level of list nesting for pieces= assertions, with a unit test. Co-Authored-By: Claude Fable 5 --- tests/v2/rules_doc.py | 2 +- tests/v2/test_doc_citations.py | 5 +++-- tests/v2/test_rules_doc_grammar.py | 10 ++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/v2/rules_doc.py b/tests/v2/rules_doc.py index 29b514a..d4ebe73 100644 --- a/tests/v2/rules_doc.py +++ b/tests/v2/rules_doc.py @@ -24,7 +24,7 @@ RULES_DOC = Path(__file__).resolve().parents[2] / "docs" / "design" / "rules.md" _RULE_RE = re.compile(r"^([A-Z])(\d+)\.\s") -_VALUE = r"\"[^\"]*\"|\([^)]*\)|\[[^\]]*\]" +_VALUE = r"\"[^\"]*\"|\([^)]*\)|\[(?:[^\[\]]|\[[^\]]*\])*\]" _EXAMPLE_RE = re.compile( r'^\s*(?:"(?P[^"]*)"|\[(?P[a-z][a-z0-9_+-]*)\])' r"(?:\s+(?P[a-z][a-z0-9_+-]*|\[[a-z][a-z0-9_+-]*\]))?" diff --git a/tests/v2/test_doc_citations.py b/tests/v2/test_doc_citations.py index 66e7e5b..55c40fd 100644 --- a/tests/v2/test_doc_citations.py +++ b/tests/v2/test_doc_citations.py @@ -22,7 +22,7 @@ _CITE_RE = re.compile( r"#\s*(?:rules|mechanisms|decisions)\.md#" - r"(?P[A-Z]\d+|[A-Z][A-Z0-9]+(?:-[A-Z0-9]+)+)" + r"(?P[A-Z]\d+|[A-Z][A-Z0-9_]*(?:-[A-Z0-9_]+)*)" r":\s*(?P.*)") # The excerpt is the FIRST double-quoted span after the ID, wrapped # over continuation comment lines; text outside the quotes (v1 names, @@ -44,7 +44,8 @@ def _statements() -> dict[str, str]: out[m.group(1)] = _norm(body) mech = MECH_DOC.read_text(encoding="utf-8") for mm in re.finditer( - r"^## (?P[A-Z][A-Z0-9-]+)\b.*?Contract statement[.:*]*\s*" + r"^## (?P[A-Z][A-Z0-9_-]+)(?=[\s—-])" + r".*?Contract statement[.:*]*\s*" r"(?P.+?)(?=\n\n|\Z)", mech, flags=re.M | re.S): out[mm.group("slug")] = _norm(mm.group("stmt")) return out diff --git a/tests/v2/test_rules_doc_grammar.py b/tests/v2/test_rules_doc_grammar.py index 9fd316a..2e34491 100644 --- a/tests/v2/test_rules_doc_grammar.py +++ b/tests/v2/test_rules_doc_grammar.py @@ -73,6 +73,16 @@ def test_unparseable_example_line_is_an_error() -> None: parse_rules_doc(bad) +def test_nested_pieces_value_parses() -> None: + doc = ('X1. Structural claim.\n' + ' "de Mesnil" family-first → ' + 'pieces=[["de"], ["Mesnil"]]\n' + ' no-boundary: structural form.\n') + ex = parse_rules_doc(doc)[0].examples[0] + assert ex.field == "pieces" + assert ex.value == [["de"], ["Mesnil"]] + + def test_registry_resolves_policies_and_gates() -> None: from tests.v2.rules_doc import resolve_annotation kind, obj = resolve_annotation("family-first") From 8dba1b055e315aafb4bb9987873332d0e861af92 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 16:10:38 -0700 Subject: [PATCH 08/40] docs(rules): extract extract_delimited into N/S/M Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 18 +++++++++++ docs/design/rules.md | 55 ++++++++++++++++++++++++++++++++ nameparser/_pipeline/_extract.py | 45 +++++++++++++------------- tests/v2/rules_doc.py | 1 + 4 files changed, 96 insertions(+), 23 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 228a628..10ab977 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -59,6 +59,24 @@ should the middle position be a third site · [#360](https://github.com/derek73/python-nameparser/issues/360) which particles count as never-given. +### M1 — delimited maiden names + +- 2026-07-03 (maiden-bucket design, landed via the v2 core, PR + #288) — the maiden reading of a delimiter pair is opt-in because + enclosure conventions genuinely vary; there is no default pair. +- 2026-07 — bucket overlap is canonicalized before parsing: a pair + listed for maiden is dropped from the effective nickname set + (maiden wins). The v1 facade restores v1's nickname-wins reading + by pre-subtracting on its side. +- 2026-08 #329 — the marker word inside a delimited clause is + dropped from a multi-token clause during grouping; the extraction + itself keeps the whole enclosed span, so nothing is lost when + there is no marker. + +Open: [#335](https://github.com/derek73/python-nameparser/issues/335) +should a marker inside a NICKNAME-delimited clause flip it to maiden +without configuration. + ### O1 — East Slavic rotation - 2026-07-12 (landed in the v2 core, PR #288) — v1 parity pinned diff --git a/docs/design/rules.md b/docs/design/rules.md index a602273..49137c8 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -106,10 +106,65 @@ P1. Rationale: a never-given particle standing alone cannot be ## Suffixes: generational & credentials (S) +S1. Rationale: brackets set off more than nicknames — credentials + are routinely written parenthesized after a name, and a + credential is recognizable by its form. + A bracketed clause whose content is suffix-shaped is not a + nickname: the brackets are dropped and the content reads exactly + as if written bare. + "Andrew Perkins (MBA)" → suffix="MBA" + "Andrew Perkins (Andy)" → nickname="Andy" · boundary + implemented: nameparser/_pipeline/_extract.py + ## Nicknames & quoted names (N) +Background: a nickname is written beside the formal name, set off by +quotes or brackets. Quotation conventions vary by language („…“, +«…», “…”), several share characters — one convention's closer is +another's opener — and the straight apostrophe doubles as a +quotation mark and as a letter-like mark inside names (O'Connor). +Which pairs delimit nicknames is caller configuration. + +N1. Rationale: a quoted or bracketed clause beside a name is an + informal alias, not part of the name. + A clause enclosed by a configured nickname delimiter pair reads + as the nickname and is lifted out of the name; an empty + enclosure is simply dropped. + "Andrew (Andy) Perkins" → nickname="Andy" + "Jean 'JD' Smith" → nickname="JD" + "Anna () Smith" → nickname="" · boundary + implemented: nameparser/_pipeline/_extract.py + +N2. Rationale: only a mark standing at word boundaries is quoting; + anywhere else it is part of the word. + A quote whose open and close are the same character opens only + at a word start and closes only at a word end, so an apostrophe + inside or at the end of a word is literal. Between conventions + that share a character, position in the text decides: the + leftmost valid opener wins. + "Sean O'Connor" → family="O'Connor" + "Hans „Erster“ und “Zweiter” Müller" → nickname="Erster Zweiter" + "Mari' Aube'" → family="Aube'" · boundary + implemented: nameparser/_pipeline/_extract.py + ## Maiden names (M) +Background: a maiden name is written beside the current name, set +off by a marker word (née, geb., 旧姓) or by enclosure. Which +enclosures mean "maiden" rather than "nickname" is a caller +convention, so the maiden reading of a delimiter pair is opt-in. + +M1. Rationale: an enclosure the caller has declared to mean maiden + holds the former family name; a recognized marker word inside it + marks the clause and is not itself part of the name. + With a delimiter pair configured for maiden names, its enclosed + clause reads as the maiden name, a leading recognized marker + word inside the clause being dropped; a pair configured for both + maiden and nickname reads maiden. + "Jane Smith (née Jones)" maiden-parens → maiden="Jones" + "Jane Smith (née Jones)" → nickname="née Jones" · boundary + history: decisions.md#M1 · implemented: nameparser/_pipeline/_extract.py + ## Commas & structure (C) ## Name order (O) diff --git a/nameparser/_pipeline/_extract.py b/nameparser/_pipeline/_extract.py index 5a5d9de..3e33af6 100644 --- a/nameparser/_pipeline/_extract.py +++ b/nameparser/_pipeline/_extract.py @@ -9,31 +9,19 @@ other token, and group drops it from a multi-token clause (#329). Reads: Policy.nickname_delimiters, Policy.maiden_delimiters, and Lexicon.suffix_words / suffix_acronyms / suffix_acronyms_ambiguous -through _suffix_shaped, which lets a clause's CONTENT overrule the -delimiter's verdict: 'Andrew Perkins (MBA)' is not a nickname, so -only the two delimiter spans are masked and the content rejoins the -token stream. - -Matching rules (the #273 mechanism): one left-to-right scan over the -original text, no nesting. At each position the LEFTMOST boundary-valid -opener among ALL configured pairs wins -- position order, never pair -order, decides between conventions that share a character in opposite -roles ('“' closes „…“ but opens “…”; '»' closes «…» but opens »…«), so -"Hans „Erster“ und “Zweiter” Müller" extracts both names. For pairs -whose open == close (quotes), the open must sit at a word boundary -(start of text or after whitespace) and the close before one (end, -whitespace, or a comma char) -- this is what keeps the apostrophe in -O'Connor literal. Empty enclosures are masked (removed from the token -stream) but extract nothing; delimiter characters inside a matched -region are literal content for every other pair. +through _suffix_shaped. + +Implements rules N1, N2, S1 and M1 of docs/design/rules.md (the #273 +matching mechanism); each is cited at its code below. One scan +mechanic worth stating up front: matching is one left-to-right pass, +no nesting, and delimiter characters inside a matched region are +literal content for every other pair. Bucket precedence is NOT decided here: Policy canonicalizes overlap -away before parsing (a pair listed in maiden_delimiters is dropped -from the effective nickname set -- maiden wins; the v1 facade restores -v1's nickname-wins reading via a pre-subtraction in _config_shim), so -the two buckets are always disjoint by the time this stage runs. The -nickname-before-maiden candidate order below is only a same-position -tie-break for exotic configs where two pairs share an OPEN character. +away before parsing, so the two buckets are always disjoint by the +time this stage runs. The nickname-before-maiden candidate order +below is only a same-position tie-break for exotic configs where two +pairs share an OPEN character. """ from __future__ import annotations @@ -48,6 +36,9 @@ from nameparser._types import AmbiguityKind, Role, Span +# rules.md#S1: "a bracketed clause whose content is suffix-shaped is +# not a nickname: the brackets are dropped and the content reads +# exactly as if written bare" def _suffix_shaped(content: str, lexicon: Lexicon) -> bool: """v1 parse_nicknames' escape (parser.py:1125-1141): an unambiguous suffix_words member (edge-normalized), an unambiguous acronym @@ -61,6 +52,9 @@ def _suffix_shaped(content: str, lexicon: Lexicon) -> bool: or content.endswith(".")) +# rules.md#N2: "a quote whose open and close are the same character +# opens only at a word start and closes only at a word end, so an +# apostrophe inside or at the end of a word is literal" def _open_ok(text: str, i: int) -> bool: return i == 0 or text[i - 1].isspace() @@ -130,6 +124,11 @@ def _unmatched(open_: str, offset: int) -> tuple[int, PendingAmbiguity]: )) +# rules.md#N1: "a clause enclosed by a configured nickname delimiter +# pair reads as the nickname and is lifted out of the name; an empty +# enclosure is simply dropped" +# rules.md#M1: "with a delimiter pair configured for maiden names, its +# enclosed clause reads as the maiden name" (history: decisions.md#M1) def extract_delimited(state: ParseState) -> ParseState: text = state.original policy = state.policy diff --git a/tests/v2/rules_doc.py b/tests/v2/rules_doc.py index d4ebe73..6085f92 100644 --- a/tests/v2/rules_doc.py +++ b/tests/v2/rules_doc.py @@ -77,6 +77,7 @@ def has_boundary_or_waiver(self) -> bool: "family-first": Policy(name_order=FAMILY_FIRST), "family-first-given-last": Policy(name_order=FAMILY_FIRST_GIVEN_LAST), "middle_as_family": Policy(middle_as_family=True), + "maiden-parens": Policy(maiden_delimiters=frozenset({("(", ")")})), } #: Extras gates: locale requiring an optional dependency; the examples #: runner skips these when the import is absent (CI's ja-extra job From dc658667617a2642e4ff293f0ed544b374d82397 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 16:12:53 -0700 Subject: [PATCH 09/40] docs(rules): extract tokenize into T (separators, name dots, interpunct) Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 17 ++++++++++++ docs/design/rules.md | 38 +++++++++++++++++++++++++++ nameparser/_pipeline/_tokenize.py | 43 ++++++++++++------------------- tests/v2/rules_doc.py | 1 + 4 files changed, 72 insertions(+), 27 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 10ab977..f720c5b 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -59,6 +59,23 @@ should the middle position be a third site · [#360](https://github.com/derek73/python-nameparser/issues/360) which particles count as never-given. +### T1 — separators, not joiners + +- 2026-07 (v2 core, PR #288) — v1's squash_emoji/squash_bidi + REMOVED the character and joined its neighbors ('A😀B' → 'AB'); + v2 makes an ignorable character a separator ('A😀B' → 'A', 'B'). + The unavoidable consequence of every part being an exact + positioned piece of the input: with no rewriting stage, nothing + can splice two half-words together. + +### T3 — the interpunct's flank guard + +- 2026-08 #298 — U+00B7 divides only between classified-script + characters because it is also the Catalan punt volat, interior to + legitimate names (Gal·la). The nakaguro (T2) needs no such guard: + its codepoints are CJK-only and appear in no other script's + names, which is what licenses an unconditional rule. + ### M1 — delimited maiden names - 2026-07-03 (maiden-bucket design, landed via the v2 core, PR diff --git a/docs/design/rules.md b/docs/design/rules.md index 49137c8..537ee89 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -225,6 +225,44 @@ O3. Rationale: several traditions write compound family names ## Tokens, initials & punctuation (T) +Background: every parsed name part is an exact piece of the input, +located by its position — nothing rewrites the text before parsing. +Some punctuation is a name divider only by convention of a +particular writing system: Japanese writes name parts with a middle +dot between them (マイケル・ジャクソン, 姓・名), while U+00B7 is +both the Chinese divider for transcribed foreign names and the +Catalan punt volat interior to legitimate words (Gal·la). + +T1. Rationale: a character that carries no name content (emoji, an + invisible directionality control) stands between words, not + inside them. + A name splits at whitespace and, when stripping is active, at + ignorable characters: an ignorable character separates its + neighbors and never joins them. + "John😀Smith" → family="Smith" + "John😀Smith" keep-emoji → given="John😀Smith" · boundary + history: decisions.md#T1 · implemented: nameparser/_pipeline/_tokenize.py + +T2. Rationale: the katakana middle dot exists to divide name parts + and appears in no native name. + The katakana middle dot and its halfwidth twin divide a name + like whitespace, always. + "マイケル・ジャクソン" → given="マイケル" + "高橋・一郎" → family="高橋" + no-boundary: the separation is unconditional; the + context-sensitive interpunct is T3's subject. + implemented: nameparser/_pipeline/_tokenize.py + +T3. Rationale: U+00B7 is two marks in one codepoint — the Chinese + 间隔号 dividing a transcribed foreign name, and the Catalan punt + volat interior to words. + The interpunct divides a name only between two characters of a + classified East Asian script; anywhere else it is part of the + word. + "威廉·莎士比亚" → family="莎士比亚" + "Gal·la Serra" → given="Gal·la" · boundary + history: decisions.md#T3 · implemented: nameparser/_pipeline/_tokenize.py + ## Ambiguity & tie-breaking (A) ## Rendering & views (R) diff --git a/nameparser/_pipeline/_tokenize.py b/nameparser/_pipeline/_tokenize.py index 682ae19..9477046 100644 --- a/nameparser/_pipeline/_tokenize.py +++ b/nameparser/_pipeline/_tokenize.py @@ -7,19 +7,10 @@ interpunct_offsets (间隔号 transcription markers, #298; never tokens). Reads: Policy.strip_emoji, Policy.strip_bidi. -There is NO text-rewriting normalize stage: whitespace collapsing, -emoji/bidi stripping, and the katakana name-dot split are all -character-classification rules here -- ignorable characters act as -separators and never enter a token, so spans always index the -original exactly as given. Whitespace and the name-dot are -unconditional; emoji/bidi stripping alone is policy-gated -(Policy.strip_emoji/strip_bidi). The Chinese interpunct U+00B7 is -context-sensitive -- see _INTERPUNCT below. - -v1's squash_emoji/squash_bidi REMOVED the char and joined neighbors -('A\U0001f600B' -> 'AB'); here an ignorable char is a SEPARATOR -('A\U0001f600B' -> 'A', 'B') -- the unavoidable consequence of spans -indexing the original exactly. +Implements rules T1, T2 and T3 of docs/design/rules.md, cited at +their code below. There is NO text-rewriting normalize stage: all +three are character-classification rules, so spans always index the +original exactly as given (v1 contrast: decisions.md#T1). """ from __future__ import annotations @@ -44,23 +35,19 @@ (0x2600, 0x26FF), (0x2700, 0x27BF)) _BIDI = re.compile('[\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+') -# The katakana middle dot and its halfwidth twin divide the parts of -# a foreign name transcribed into katakana (マイケル・ジャクソン) -- -# native names never contain them, so they separate unconditionally, -# like whitespace (amendment 2026-07-29 section 1b). They record no -# offset: the nakaguro also divides kanji roster pairs (高橋・一郎, -# 姓・名 -- read family-first by the script license, #272), so it is -# not a transcription marker. Only the Chinese 间隔号 is (#298), and -# it is context-sensitive -- see _INTERPUNCT below. +# rules.md#T2: "the katakana middle dot and its halfwidth twin divide +# a name like whitespace, always." They record no offset: the +# nakaguro also divides kanji roster pairs (高橋・一郎, read +# family-first by the script license, #272), so it is not a +# transcription marker; only the Chinese 间隔号 is (#298). _NAME_DOT_SEPARATORS = frozenset({"\u30FB", "\uFF65"}) _INTERPUNCT = "\u00B7" -# Per-CHAR classifier for the interpunct's flank guard: a one-char -# string is wholly-classified iff the character is. U+00B7 cannot be -# an unconditional separator like the name dots above -- it is also -# the Catalan punt volat, INTERIOR to legitimate names (Gal\u00B7la) -- so -# it divides only between classified-script characters: the first -# context-sensitive separator rule, which is why it lives in +# rules.md#T3: "the interpunct divides a name only between two +# characters of a classified East Asian script; anywhere else it is +# part of the word" (history: decisions.md#T3). Per-CHAR classifier +# for the flank guard: a one-char string is wholly-classified iff the +# character is. Context-sensitivity is why this lives in # _tokenize_region (where the index exists) and not in _ignorable. _classified_char = _script_matcher(*_SCRIPT_RANGES, whole=True) @@ -70,6 +57,8 @@ def _is_emoji(ch: str) -> bool: return any(lo <= cp <= hi for lo, hi in _EMOJI_RANGES) +# rules.md#T1: "an ignorable character separates its neighbors and +# never joins them" (v1 contrast: decisions.md#T1) def _stripped(ch: str, policy: Policy) -> bool: """True when the strip policy removes `ch` from the token stream. The ONE definition of that set, shared by _ignorable and _flank: a diff --git a/tests/v2/rules_doc.py b/tests/v2/rules_doc.py index 6085f92..a86f0ee 100644 --- a/tests/v2/rules_doc.py +++ b/tests/v2/rules_doc.py @@ -78,6 +78,7 @@ def has_boundary_or_waiver(self) -> bool: "family-first-given-last": Policy(name_order=FAMILY_FIRST_GIVEN_LAST), "middle_as_family": Policy(middle_as_family=True), "maiden-parens": Policy(maiden_delimiters=frozenset({("(", ")")})), + "keep-emoji": Policy(strip_emoji=False), } #: Extras gates: locale requiring an optional dependency; the examples #: runner skips these when the import is absent (CI's ja-extra job From 7004567f8929db412d65c3a8e194bf80a548bc5e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 16:14:26 -0700 Subject: [PATCH 10/40] docs(rules): extract segment into C (comma structures) Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 15 ++++++++++++++ docs/design/rules.md | 35 ++++++++++++++++++++++++++++++++ nameparser/_pipeline/_segment.py | 23 ++++++++------------- tests/v2/rules_doc.py | 1 + 4 files changed, 60 insertions(+), 14 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index f720c5b..c304780 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -59,6 +59,21 @@ should the middle position be a third site · [#360](https://github.com/derek73/python-nameparser/issues/360) which particles count as never-given. +### C1 — the suffix-comma decision + +- 2026-07 (v2 core, PR #288) — v1 parity: only the second segment + decides (v1 parser.py:1318), and ">1 word before the first comma" + is v1's guard, which is why "Smith, PhD" keeps the listing form. +- 2026-07 (plan deviation #3, recorded) — the decision is + definitionally vocabulary-dependent: there is no way to recognize + a credential run without consulting the suffix word lists, so the + structural stage reads vocabulary through one predicate. +- 2026-07-30 #291/#296 — the lenient token test is the default and + `lenient_comma_suffixes=False` restores the strict one. +- 2026-08 #319 — the wholly-suffix predicate was lifted into the + vocabulary layer so the comma decision and the honorific peel's + segment test cannot drift apart. + ### T1 — separators, not joiners - 2026-07 (v2 core, PR #288) — v1's squash_emoji/squash_bidi diff --git a/docs/design/rules.md b/docs/design/rules.md index 537ee89..cd18beb 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -167,6 +167,41 @@ M1. Rationale: an enclosure the caller has declared to mean maiden ## Commas & structure (C) +Background: a comma in a name signals one of two conventions — the +listing form "Family, Given" or trailing credentials "Name, PhD" — +and which is meant can only be judged from what stands after the +first comma. Recognizing a credential run is by nature a vocabulary +judgment, so this is the one structural decision that consults the +suffix word lists. + +C1. Rationale: a credential run after the comma means the name is in + natural order with suffixes appended; anything else after the + comma means the listing form. + With a comma present, the name reads as trailing suffixes when + the part after the first comma is entirely suffix words and more + than one word precedes the comma; otherwise it reads as the + listing form, the part before the comma being the family name. + Only the part after the first comma decides. By default the + suffix judgment is lenient about initials-like abbreviations; + strict mode confines it to the recognized vocabulary. + "Smith, John" → family="Smith" + "John Smith, PhD" → suffix="PhD" + "John Smith, V." → suffix="V." + "John Smith, V." strict-comma-suffixes → family="John Smith" + "Smith, PhD" → family="Smith" · boundary + history: decisions.md#C1 · implemented: nameparser/_pipeline/_segment.py + +C2. Rationale: text beyond the recognized comma parts should be + taken in without silent guessing. + Parts beyond the second are consumed as suffixes either way; a + non-empty extra part that is not entirely suffix words is + flagged as a structural ambiguity rather than rejected — parsing + never fails on content. An empty part between doubled commas is + consumed silently. + "John Smith, MD, Bart" → suffix="MD, Bart" + "John Smith, MD,, Jr." → suffix="MD, Jr." · boundary + history: decisions.md#C1 · implemented: nameparser/_pipeline/_segment.py + ## Name order (O) Background: written name order varies by convention: given-first diff --git a/nameparser/_pipeline/_segment.py b/nameparser/_pipeline/_segment.py index 4fe3d55..c35ff6c 100644 --- a/nameparser/_pipeline/_segment.py +++ b/nameparser/_pipeline/_segment.py @@ -11,19 +11,8 @@ strict token test; Policy.extra_suffix_delimiters gives v1 suffix_delimiter parity, a delimiter-core token being transparent). -Decision (v1 parity): >=1 comma and the SECOND segment entirely -suffix AND >1 word before the first comma -> SUFFIX_COMMA; otherwise -FAMILY_COMMA ("Family, Given ..."). Only the second segment decides -(v1 parser.py:1318) -- segments beyond it are consumed as suffixes -either way, and a NON-EMPTY one that is not entirely suffix is -flagged COMMA_STRUCTURE rather than vetoing the structure (an empty -one is consumed silently, as v1 consumed it, so "John Smith, MD,, Jr." -reports nothing where "John Smith, MD, Bart" does; they are still -best-effort consumed as suffixes by assign, since parse must stay -total over str input and never raise on content). "Entirely suffix" -is is_wholly_suffix's question, so the token test inside it is -lenient by default and strict under -Policy(lenient_comma_suffixes=False). +Implements rules C1 and C2 of docs/design/rules.md, cited at the +decision site below; history in decisions.md#C1. """ from __future__ import annotations @@ -73,7 +62,13 @@ def suffixy(seg: tuple[int, ...]) -> bool: # v1 parity: only parts[1] decides the suffix-comma structure # (parser.py:1318); parts[2:] are consumed as suffixes # unconditionally either way, so a non-suffix tail segment gets the - # COMMA_STRUCTURE flag, not a structure veto + # rules.md#C1: "the name reads as trailing suffixes when the part + # after the first comma is entirely suffix words and more than one + # word precedes the comma; otherwise it reads as the listing form" + # (history: decisions.md#C1) + # rules.md#C2: "a non-empty extra part that is not entirely suffix + # words is flagged as a structural ambiguity rather than rejected" + # -- COMMA_STRUCTURE flag, not a structure veto structure = (Structure.SUFFIX_COMMA if suffixy(groups[1]) and len(groups[0]) > 1 else Structure.FAMILY_COMMA) diff --git a/tests/v2/rules_doc.py b/tests/v2/rules_doc.py index a86f0ee..f930197 100644 --- a/tests/v2/rules_doc.py +++ b/tests/v2/rules_doc.py @@ -79,6 +79,7 @@ def has_boundary_or_waiver(self) -> bool: "middle_as_family": Policy(middle_as_family=True), "maiden-parens": Policy(maiden_delimiters=frozenset({("(", ")")})), "keep-emoji": Policy(strip_emoji=False), + "strict-comma-suffixes": Policy(lenient_comma_suffixes=False), } #: Extras gates: locale requiring an optional dependency; the examples #: runner skips these when the import is absent (CI's ja-extra job From aed37f5374b7176a3eabc945032ef2540efb8d81 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 16:17:21 -0700 Subject: [PATCH 11/40] docs(rules): extract script_segment into W (division, peel, writer's divisions) Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 61 +++++++++++ docs/design/rules.md | 62 +++++++++++ nameparser/_pipeline/_script_segment.py | 135 ++++-------------------- 3 files changed, 145 insertions(+), 113 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index c304780..cd8ff6e 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -59,6 +59,67 @@ should the middle position be a third site · [#360](https://github.com/derek73/python-nameparser/issues/360) which particles count as never-given. +### W1 — unspaced CJK division + +- 2026-08-07 #271 (2.1.0) — Korean division ships as a default: the + census surname list is closed, hangul is self-selecting (a hangul + entry can only match hangul text), and being unsplit is + recoverable while a wrong split is not — which is also why an + unrecognized name stays whole. +- 2026-08-07 #272/amendment 2026-07-29 — Han division is opt-in per + language pack because Han text does not identify its language + (高橋一郎 under a Chinese list divides wrongly); a pluggable + segmenter takes what the vocabulary declines, so pack + segmenter + compose mechanically: a listed surname is a dictionary certainty + and wins. + +- 2026-08 — zh and ja packs are corpus ALTERNATIVES, one per + corpus; stacking them (parser_for(ZH, JA, segmenter=...)) is for + genuinely mixed data that accepts the trade: a listed Chinese + surname wins before the segmenter is consulted, so 高橋一郎 still + divides 高 + 橋一郎 under ZH+JA exactly as under ZH alone. Kana + gating resolves through the same script function order uses, so + kana-licensed composites (高橋みなみ) gate in under JA while + pure-katakana tokens never do (amendment 2026-07-27: activation + is per script because the ambiguity is per script). + +### W3 — the writer's divisions are respected + +- 2026-08 (segmenter amendment 2026-07-29) — a segmenter answers + where an UNDIVIDED name divides, so it is consulted only when the + gated token is the name part's only script-written one: "山田 + 太郎" was divided by its writer and must not have its family + divided again. The peeled tail is exempt (that boundary was + manufactured, not written); a SPACED honorific is not — at that + position it cannot be told from a spaced name element. The trade + was measured before deciding: counting spaced honorifics keeps + four real surnames whole (佐藤 氏, 田中 様, 鈴木 先生, 中村 教授) + and costs the one division 山田太郎 様. +- 2026-08 #312 — under FAMILY_COMMA the whole stage stood down + until the peel moved in front of the gate: the comma doctrine is + about the input's structure, not the split's source, so it covers + vocabulary and segmenter identically, while an honorific is no + part of the name whichever side of the comma it glues to. +- 2026-07 — the stage runs AFTER comma segmentation on purpose: + running before would make the comma structure depend on the + split ("김민준, Jr." pre-split would read suffix-comma on + vocabulary alone; as written it stays the listing form). + +### W2 — the glued-honorific peel + +- 2026-08 #308 — an entry peels only where it can never end a name: + 씨/님/さん/様/先生 peel; 양/군/氏/博士/殿 stay spaced-only; 君 is + in neither set while its kana spelling くん peels. The vocabulary + carries the license, so no structural or per-script gate stands + over the peel. +- 2026-08 #312 — the peel crosses the family comma and the 间隔号: + both answer where a name DIVIDES into surname and given, a + question the peel never asks. +- 2026-08 #319 — a wholly suffix-shaped second run is declined as a + peel site (the "田中さん, V." shape), but only when the name's own + run offers a site, since a glued honorific is itself part of what + makes a run read as suffix-shaped. + ### C1 — the suffix-comma decision - 2026-07 (v2 core, PR #288) — v1 parity: only the second segment diff --git a/docs/design/rules.md b/docs/design/rules.md index cd18beb..af11941 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -258,6 +258,68 @@ O3. Rationale: several traditions write compound family names ## Scripts & writing systems (W) +Background: script-conditional behavior is permitted exactly where +the writing system itself — not statistics about it — settles the +convention; a language can never be inferred from Latin-script text, +because transliteration destroys the signal. The facts this section +builds on: Chinese and Japanese both write the family name first in +native script, so the script settles the order without knowing the +language. Hangul is written by exactly one language and Korean +family names are a small closed census set. Han text does not +identify its language — a Chinese surname list would divide Japanese +高橋一郎 as 高 + 橋一郎 — which is why Han division is opt-in and +there is no Korean pack to opt into. Hiragana never transcribes a +foreign name (transcriptions are katakana alone), so kanji-plus-kana +is a Japanese name in Japanese order, while wholly-katakana is +predominantly a transcribed foreign name already in given-first +order. Real Chinese text is unspaced (毛泽东); the spaced 毛 泽东 is +an artifact. A fuller narrative lives in docs/usage.rst's East Asian +section. + +W1. Rationale: hangul is monoglot Korean and its surnames are a + closed census set, so an unspaced hangul name divides at a + certain point; Han carries no such certainty by default. + An unspaced name in an activated script divides after a + recognized surname, the longest recognized surname first; where + the vocabulary recognizes nothing, an optional segmenter may + divide instead, and with neither the name stays whole rather + than divide in a wrong place. Korean division is active by + default; Han division is opt-in. + "김민준" → family="김" + "남궁민수" → family="남궁" + "毛泽东" → family="毛泽东" · boundary + "毛泽东" [zh] → family="毛" + "高橋一郎" [zh] → family="高" + history: decisions.md#W1 · implemented: nameparser/_pipeline/_script_segment.py + +W2. Rationale: some East Asian honorifics glue directly onto the end + of the name (田中さん); a glued word peels off only if it could + never itself end a name, so the listed vocabulary carries its + own license and needs no other gate. + A listed honorific glued to the end of the name's last name word + splits off and reads as a suffix. The peel crosses a family + comma and ignores surrounding punctuation, but never takes a + part that is not name text as its site. + "田中さん" → suffix="さん" + "김, 민준씨" → suffix="씨" + "田中さん, V." → suffix="さん" + "김지양" → suffix="" · boundary + "王君" → family="王君" · boundary + history: decisions.md#W2 · implemented: nameparser/_pipeline/_script_segment.py + +W3. Rationale: a divided name was divided by its writer, and + re-dividing would invent a boundary nobody drew; a family name + declared by a comma is likewise the writer's own division. + Division applies only to a name whose written form is undivided: + a name part already containing a division divides no further — + a spaced honorific counts as a written division — and under a + family comma the pre-comma text is the family by declaration + and never divides. Only the honorific peel crosses these, + because an honorific is no part of the name on either side. + "남궁민수" → family="남궁" + "남궁민수, 지훈" → family="남궁민수" · boundary + history: decisions.md#W3 · implemented: nameparser/_pipeline/_script_segment.py + ## Tokens, initials & punctuation (T) Background: every parsed name part is an exact piece of the input, diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index a3ddde4..a6e0134 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -35,120 +35,18 @@ split being the half that flips "田中さん, Jr./V." under {"/"} and the bare core the half that flips "田中さん, /"). -Unspaced CJK names give tokenize no separator to find, so this stage -inserts the missing token boundary by vocabulary: the first token -written in an activated script is matched longest-first against -Lexicon.surnames, and a hit splits it in two. Compound-before-single -("夏侯惇" is 夏侯 + 惇, though 夏 is itself a surname) falls out of -longest-first. The split makes sub-slices of the one token, rewriting -nothing -- spans still index the original exactly, so the anti-#100 -invariant holds by construction. - -A second, independent split runs first (#308): a listed honorific -glued to the END of the name's last non-post-nominal token is peeled -off as its own token -- 田中さん -> 田中 + さん -- so that suffix -classification can claim it and the surname match or segmenter consult -below sees the name rather than the name plus an honorific. No -STRUCTURAL gate stands over it -- the only things above it are the -stage's own two preconditions, the ASCII bail and non-empty segments. -Not segment_scripts either: the vocabulary of tails is licensed by the -entries themselves, each of which can never end a name, so no -per-script trust question arises. And since #312 not the FAMILY comma -or the 间隔号 either -- both of those answer where a name divides into -surname and given, which the peel never asks, so a comma or a dot -elsewhere in the string cannot change whether a token ends in a word -that can never end a name. The ASCII bail is the gate a caller adding -a LATIN tail meets -- it sits above everything here, so such a tail +Implements rules W1 (the vocabulary/segmenter division), W2 (the +glued-honorific peel) and W3 (the writer's divisions are respected) +of docs/design/rules.md, cited at their code below; the decision +chain (#308, #312, #319, the vetting bars, the measured +spaced-honorific trade) is decisions.md#W1, #W2 and #W3. Both splits make sub-slices of one token, +rewriting nothing -- spans still index the original exactly, so the +anti-#100 invariant holds by construction. The peel runs FIRST, so +suffix classification can claim the tail and the surname match or +segmenter consult sees the name rather than name-plus-honorific; its +ASCII bail sits above everything here, so a caller-added LATIN tail fires only on a name carrying at least one non-ASCII character (see -the bail's own comment, and honorific_tails' field note). A suffix -comma gates nothing either -- "Dr 김민준씨, Jr." peels within its name -part like any other. - -Where the VOCABULARY declines -- no prefix matched -- an optional -Parser(segmenter=...) gets the token (#272 amendment 2026-07-29). -Vocabulary first, segmenter on decline, so parser_for(ZH, JA, -segmenter=...) composes MECHANICALLY: a listed surname is a -dictionary certainty and wins, and the segmenter takes what is left. -Composing is not a free lunch, and the docs qualify it where they -show the stack: the zh pack's own mis-split warning survives -unchanged, because a Japanese kanji name opening on a listed Chinese -surname never reaches the segmenter at all -- 高橋一郎 still splits -高 + 橋一郎 under ZH+JA, exactly as it does under ZH alone. The two -packs are corpus ALTERNATIVES, one per corpus; stacking them is for -genuinely mixed data that accepts that trade. Its Segmentation may -cut anywhere and any number of times, which is why the split path -below takes n cuts. One precondition guards it that the vocabulary -has no twin of: a segmenter answers where an UNDIVIDED name divides, -so it is consulted only when the gated token is the name part's ONLY -script-written one -- "山田 太郎" was divided by its writer and must -not have its family divided again (a Latin title or suffix draws no -such boundary either, and effective_script gates those out before the -test is reached). The neighbour test reads effective_script -- merely -non-None -- and exempts exactly one token: the tail the peel above -MANUFACTURED (#308), which is a boundary nobody drew. A SPACED -honorific is not exempt, and the distinction is provenance rather -than vocabulary: glued 山田太郎様 was written undivided, while "佐藤 -氏" carries a boundary its writer typed. Not that the writer thereby -declared 佐藤 a unit -- in "山田太郎 様" the unit they drew is the -whole name -- but that at this position a spaced honorific cannot be -told apart from a spaced name ELEMENT, so the precondition counts it -rather than guess. The trade that buys is measured: counting spaced -honorifics keeps four real surnames whole (佐藤 氏, 田中 様, 鈴木 -先生, 中村 教授, all four divided bare under the JA pack) and costs -the one division 山田太郎 様. A segmenter's own exceptions -PROPAGATE -- the single declared exception to parse totality (locales -spec section 4): a user-supplied callable's error is a user-code -error, not a content error. - -Placed AFTER segment, on the comma doctrine that script-conditional -behavior DECIDING WHERE A NAME DIVIDES is ignored where a comma -already decides the family (the rule script_orders follows -- and the -qualifier is load-bearing since #312, which is what the peel crosses -the comma on): under FAMILY_COMMA the pre-comma text IS -the family by declaration, and splitting it would invent a boundary -the writer explicitly did not draw -- "남궁민수, 지훈" must render -family "남궁민수", not "남궁 민수". That doctrine is about the input's -structure, not the split's source, so it covers the segmenter -identically: the opt-out runs before either is consulted. The -post-comma side is given-name text, no surname site either, so that -structure opts out of the SURNAME SPLIT whole -- of the stage entire -until #312 moved the peel in front of the gate, an honorific being no -part of the name whichever side of the comma it was glued to. -NO_COMMA and SUFFIX_COMMA still split, within segments[0] (the name -part) only. Running after segment costs the index remaps below; -running BEFORE it would have made the comma structure itself depend -on the split -- segment's suffix-comma rule needs more than one word -before the comma, so a pre-split "김민준, Jr." would have changed -structure on vocabulary alone. As written it stays FAMILY_COMMA, -which is why the SUFFIX_COMMA path needs a second word ("Dr 김민준, -Jr.") to be reachable at all. - -Activation is per script because the AMBIGUITY is per script -(amendment 2026-07-27): HANGUL is on by default (hangul is -unambiguously Korean, and its surname set is closed and -default-shipped), while HAN is opt-in via locales.ZH -- a Chinese -surname list corrupts Japanese names ("高橋一郎" must not split as -高 + 橋一郎). Japanese divides through the segmenter path below under -locales.JA, not through this table's vocabulary. -The gate resolves each token through effective_script, the same -function order resolution uses, so kana-licensed composites (高橋みなみ --> HIRAGANA) gate in under the JA pack's HIRAGANA entry while -pure-katakana tokens (-> KATAKANA, in no activation set by design -- -they are predominantly transcribed foreign names) never do. -Only the FIRST activated-script token is considered, match or no -match: family-first traditions put the surname at the front of the -name, and a match deeper in the token stream would be a given name or -an ordinary word, not a surname site. Where that first token is one -the vocabulary reads as a POST-NOMINAL, the stage DECLINES rather than -looking further along: an honorific is not part of the name, and a -surname leads, so an honorific in the surname's own position means -there is no surname here to find. Deciding includes deciding that. -Looking further would reach the given name the rule already refuses to -touch -- "양 지훈" would split its own given name, 지 being listed too --- while declining leaves the peel's manufactured tail intact, since -in "Anderson선생님" that tail is the first and only script-written -token (선생님 opens on the listed surname 선, and a spaced -"Anderson 선생님" was mis-split that way before the peel existed). +the bail's own comment, and honorific_tails' field note). """ from __future__ import annotations @@ -373,6 +271,10 @@ def _peel_site(state: ParseState, flat: Sequence[int], return None +# rules.md#W2: "a listed honorific glued to the end of the name's +# last name word splits off and reads as a suffix. The peel crosses a +# family comma and ignores surrounding punctuation, but never takes a +# part that is not name text as its site." (history: decisions.md#W2) def _peel_honorific_tail(state: ParseState) -> ParseState: """#308: split a listed honorific off the END of the name's last NON-POST-NOMINAL token -- 田中さん -> 田中 + さん -- and let @@ -717,6 +619,13 @@ def _split_surname_site(state: ParseState) -> ParseState: return _split(state, i, answer.splits, detail) +# rules.md#W1: "an unspaced name in an activated script divides after +# a recognized surname, the longest recognized surname first; where +# the vocabulary recognizes nothing, an optional segmenter may divide +# instead, and with neither the name stays whole" +# rules.md#W3: "a name part already containing a division divides no +# further" and "under a family comma the pre-comma text is the family +# by declaration and never divides" (history: decisions.md#W3) def script_segment(state: ParseState) -> ParseState: if state.original.isascii(): # spans index the original exactly (the anti-#100 invariant), From 07ae7ad59038abbb95e0b7497447b783407a010d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 16:19:56 -0700 Subject: [PATCH 12/40] docs(rules): extract classify+leading-title into H2/S2 (litmus pass) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spec's drafted litmus statement ('recognized by vocabulary, not by written shape') was falsified by the live parser: the leading- abbreviation shape heuristic is real ('Xyz. John Smith' takes the title), and the abugida gap is its LIMIT, recorded as H2's Accepted consequence. S2 records the bare-ambiguous-acronym behavior ('Jack Wei Ma' → suffix='Ma') as an Accepted consequence pending #326-adjacent design work. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 11 ++++++++ docs/design/rules.md | 44 +++++++++++++++++++++++++++++++ nameparser/_pipeline/_assign.py | 3 +++ nameparser/_pipeline/_classify.py | 9 +++++-- 4 files changed, 65 insertions(+), 2 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index cd8ff6e..5a878e7 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -59,6 +59,17 @@ should the middle position be a third site · [#360](https://github.com/derek73/python-nameparser/issues/360) which particles count as never-given. +### H2 — the leading-abbreviation title + +- 2026-06-30 (leading-period-title design; v2 core, PR #288) — the + shape test is v1 parity (period_abbreviation): two-plus letters + then a period, leading position only, bare initials exempt. The + extraction litmus (2026-08-15): the spec drafted this rule as + "recognized by vocabulary, not by written shape" and the live + parser falsified that framing — the shape heuristic is real, and + what the abugida gap (#342-#345) shows is its LIMIT, not its + absence. Recorded as the rule's Accepted consequence. + ### W1 — unspaced CJK division - 2026-08-07 #271 (2.1.0) — Korean division ships as a default: the diff --git a/docs/design/rules.md b/docs/design/rules.md index af11941..c397309 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -77,6 +77,23 @@ H1. Rationale: a title normally addresses by surname, so a title "Sir John" → given="John" · boundary implemented: nameparser/_pipeline/_post_rules.py +H2. Rationale: before a name, an abbreviation is almost always a + title — "Rev.", "Ing.", "Mag." — and no vocabulary can list + every profession's abbreviations in every language. + A name-opening abbreviation of at least two letters ending in a + period reads as a title even when unlisted; a bare initial does + not. + "Rev. John Smith" → title="Rev." + "Xyz. John Smith" → title="Xyz." + "J. Smith" → given="J." · boundary + Accepted: the shape is only recognizable as an unbroken run of + letters, so an abbreviation in a script whose letters carry + combining vowel signs (Bengali, Devanagari) never reads as a + title by shape — an unlisted abugida honorific stays a name + word, and only vocabulary (#343) can recognize it. + "প্রফেসর. Sen" → given="প্রফেসর." + history: decisions.md#H2 · implemented: nameparser/_pipeline/_assign.py + ## Particles & surname prefixes (P) Background: particles ("de", "la", "van", "von", "bin") link forward @@ -106,6 +123,16 @@ P1. Rationale: a never-given particle standing alone cannot be ## Suffixes: generational & credentials (S) +Background: what follows a name is one of two different things — +generational suffixes (Jr., III), which attach to the name itself, +and credentials (PhD, MD, MBA), which are earned attachments. CLDR +personNames keeps them as separate fields (`generation`, +`credentials`) and formats them differently; this library currently +reports both in one `suffix` field, a merge #326 examines. The +vocabulary is largely split already: a generational word list and a +credential acronym list, plus a short list of acronyms that are also +ordinary names (MA, BA) and so are AMBIGUOUS as bare words. + S1. Rationale: brackets set off more than nicknames — credentials are routinely written parenthesized after a name, and a credential is recognizable by its form. @@ -116,6 +143,23 @@ S1. Rationale: brackets set off more than nicknames — credentials "Andrew Perkins (Andy)" → nickname="Andy" · boundary implemented: nameparser/_pipeline/_extract.py +S2. Rationale: generational suffixes and credentials are recognized + by vocabulary; an acronym that is also an ordinary name is only + unmistakably a credential when its periods are written. + A trailing word of the suffix vocabulary reads as a suffix — + generational forms and credential acronyms alike, and an + ambiguous acronym written with periods counts unambiguously. + "John Smith Jr." → suffix="Jr." + "John Smith M.A." → suffix="M.A." + "John Smith PhD" → suffix="PhD" + Accepted: a BARE ambiguous acronym in the trailing position + still reads as a suffix today, even beside an East Asian + surname it more likely belongs to. + "Jack Wei Ma" → suffix="Ma" + no-boundary: the vocabulary is the boundary; an unlisted + trailing word simply reads as the family name. + implemented: nameparser/_pipeline/_classify.py + ## Nicknames & quoted names (N) Background: a nickname is written beside the formal name, set off by diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 4178fc6..bb4d5a9 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -57,6 +57,9 @@ def _set_roles(tokens: list[WorkToken], piece: tuple[int, ...], tokens[i] = dataclasses.replace(tokens[i], role=role) +# rules.md#H2: "a name-opening abbreviation of at least two letters +# ending in a period reads as a title even when unlisted; a bare +# initial does not" (history: decisions.md#H2) def _is_leading_title(piece: tuple[int, ...], ptags: frozenset[str], tokens: list[WorkToken]) -> bool: if _is_title_piece(piece, ptags, tokens): diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py index a52132c..f6a3ec6 100644 --- a/nameparser/_pipeline/_classify.py +++ b/nameparser/_pipeline/_classify.py @@ -11,8 +11,10 @@ "vocab:suffix", "vocab:suffix-word", "vocab:suffix-ambiguous", "vocab:particle-ambiguous", "vocab:bound-given", "vocab:maiden-marker". "vocab:suffix" means "counts as a suffix as written": unambiguous -suffix vocabulary, or an ambiguous acronym written with periods -('M.A.' yes, 'Ma' no -- 'Ma' gets only "vocab:suffix-ambiguous"). +suffix vocabulary, or an ambiguous acronym written with periods -- +at the TAG level 'M.A.' gets "vocab:suffix" while 'Ma' gets only +"vocab:suffix-ambiguous"; what assign then does with a trailing +ambiguous tag is rule S2's Accepted consequence. The initial veto is assign's job, not classify's: 'V' carries both "vocab:suffix" and "initial". """ @@ -32,6 +34,9 @@ +# rules.md#S2: "a trailing word of the suffix vocabulary reads as a +# suffix — generational forms and credential acronyms alike, and an +# ambiguous acronym written with periods counts unambiguously" def _tags_for(token: WorkToken, state: ParseState) -> frozenset[str]: lex = state.lexicon n = _normalize(token.text) From 8a855c09ad4958fbb33376b11f21ecd9f7796712 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 16:24:07 -0700 Subject: [PATCH 13/40] docs(rules): extract group+assign into H3/P2/P3/M2/N3/O4/W4/A1 Ambiguities assertion form implemented in the runner; plan-deviation phrases retargeted to decisions.md entries; 27 rules total. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 46 ++++++++++++++ docs/design/rules.md | 102 ++++++++++++++++++++++++++++++- nameparser/_pipeline/_assign.py | 17 +++++- nameparser/_pipeline/_group.py | 29 ++++++--- nameparser/_pipeline/_segment.py | 2 +- nameparser/_pipeline/_state.py | 4 ++ tests/v2/test_rules_doc.py | 4 +- 7 files changed, 189 insertions(+), 15 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 5a878e7..99b5a8a 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -59,6 +59,52 @@ should the middle position be a third site · [#360](https://github.com/derek73/python-nameparser/issues/360) which particles count as never-given. +### P2 — particles join forward + +- 2026-08 #367 — a title is transparent to the chain's start: + "Sir de Mesnil" chains de→Mesnil exactly as the untitled form + does. Before, the title displaced the particle out of the leading + position and "Sir de Mesnil" reported given="de Mesnil" with no + family at all — a limit the rule never meant to draw. Fixed in + grouping, which is why P1's fold needed no change (its interacts: + points here). + +### M2 — the maiden-marker rule + +- 2026-07-03 #274 (v2 core, PR #288) — the marker takes everything + after it up to a trailing suffix, greedily: "née Jones Smith" is + a two-word maiden name, matching how the marker is actually used + in running text. The marker itself is dropped as structural, like + a delimiter character. + +### N3 — the lone-word nickname rule + +- 2026-07 (v2 core, PR #288; recorded plan deviation #2 of the core + plan) — v1's rule counted pieces before grouping; the v2 port + counts one non-title piece plus a nonempty nickname. The rule + lives in assignment rather than grouping because that is where + the piece count is settled. + +### phd-merge — the "Ph. D." split + +- 2026-07 (v2 core, PR #288; recorded plan deviation #1 of the core + plan) — "Ph. D." tokenizes as two words and is merged back by + vocabulary (v1 fix_phd), so the spaced and unspaced spellings + read alike. + +### W4 — script-scoped order + +- 2026-07-27 (script-scoped order amendment) — the family-first + override is keyed to the SCRIPT of the written name, never to a + guessed language: wholly-Han, wholly-Hangul, and kana-licensed + Japanese read family-first because zh, ko and ja all write + family-first in native script; wholly-katakana names are + predominantly transcriptions and keep the declared order. Latin + transliterations are never touched. +- 2026-08-07 #272 — the kana license: Han∪kana with at least one + kana cannot be Chinese and is not a transcription, so 高橋みなみ + reads family-first though it is written in two scripts. + ### H2 — the leading-abbreviation title - 2026-06-30 (leading-period-title design; v2 core, PR #288) — the diff --git a/docs/design/rules.md b/docs/design/rules.md index c397309..48ae441 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -94,6 +94,16 @@ H2. Rationale: before a name, an abbreviation is almost always a "প্রফেসর. Sen" → given="প্রফেসর." history: decisions.md#H2 · implemented: nameparser/_pipeline/_assign.py +H3. Rationale: compound titles are written as a run of title words, + connectives included; a title word standing inside the name is + just a name word. + Successive title words at the name's start chain into one + title; a title word elsewhere in the name does not. + "Asst. Vice Chancellor John Smith" → title="Asst. Vice Chancellor" + "Marquess of Bath" → title="Marquess of Bath" + "John Doctor Smith" → middle="Doctor" · boundary + implemented: nameparser/_pipeline/_group.py + ## Particles & surname prefixes (P) Background: particles ("de", "la", "van", "von", "bin") link forward @@ -119,7 +129,27 @@ P1. Rationale: a never-given particle standing alone cannot be Accepted: a bare "de" stays the given name — there is nothing to fold into, and inventing a surname would be worse. "de" → given="de" - history: decisions.md#P1 · implemented: nameparser/_pipeline/_post_rules.py + history: decisions.md#P1 · interacts: P2 · implemented: nameparser/_pipeline/_post_rules.py + +P2. Rationale: a particle is written as part of the surname it + precedes, and a title stands outside the name entirely. + A particle joins forward onto the name word after it, chains + included; the chain begins wherever the name begins, and a + preceding title does not move that point. + "John van der Berg" → family="van der Berg" + "Sir de Mesnil" → family="de Mesnil" + "Juan de" → family="de" · boundary + history: decisions.md#P2 · implemented: nameparser/_pipeline/_group.py + +P3. Rationale: connective words ("y", "of the") bind name words into + one name part; but a single letter in a short name is more + likely an initial than a connective. + A recognized connective joins its neighbors into one name part, + connective runs included — except a single-letter connective in + a three-word name, which stays a name word. + "Juan y Eva Garcia" → given="Juan y Eva" + "Juan y Garcia" → middle="y" · boundary + implemented: nameparser/_pipeline/_group.py ## Suffixes: generational & credentials (S) @@ -191,6 +221,15 @@ N2. Rationale: only a mark standing at word boundaries is quoting; "Mari' Aube'" → family="Aube'" · boundary implemented: nameparser/_pipeline/_extract.py +N3. Rationale: a person set down as a nickname plus one name word is + being identified by surname. + A name that is only a nickname and one name word reads that word + as the family name; with two or more name words the ordinary + positional reading applies. + "'Smitty' Jones" → family="Jones" + "'Smitty' John Jones" → given="John" · boundary + history: decisions.md#N3 · implemented: nameparser/_pipeline/_assign.py + ## Maiden names (M) Background: a maiden name is written beside the current name, set @@ -207,7 +246,19 @@ M1. Rationale: an enclosure the caller has declared to mean maiden maiden and nickname reads maiden. "Jane Smith (née Jones)" maiden-parens → maiden="Jones" "Jane Smith (née Jones)" → nickname="née Jones" · boundary - history: decisions.md#M1 · implemented: nameparser/_pipeline/_extract.py + history: decisions.md#M1 · implemented: nameparser/_pipeline/_extract.py, nameparser/_pipeline/_group.py + +M2. Rationale: a maiden marker announces that what follows it is the + former family name; the marker is an announcement, not a name. + A recognized maiden marker inside the name takes the words after + it — up to any trailing suffix — as the maiden name, and the + marker itself is dropped. A marker with nothing after it is just + a word. + "Jane Smith née Jones" → maiden="Jones" + "Jane née Jones Smith" → maiden="Jones Smith" + "Jane Smith née Jones PhD" → suffix="PhD" + "Jones née" → family="née" · boundary + history: decisions.md#M2 · implemented: nameparser/_pipeline/_group.py ## Commas & structure (C) @@ -300,6 +351,21 @@ O3. Rationale: several traditions write compound family names "Hassan Mohamad Ali" → family="Ali" · boundary implemented: nameparser/_pipeline/_post_rules.py +O4. Rationale: what no vocabulary claims can only be read by where + it stands, under the order the caller declared. + Words no vocabulary has claimed read by position. In the default + given-first order the first name word is the given name, the + last is the family name, and everything between is middle names. + In a family-first order the first name word is the family; in + family-first-given-last the given name comes from the end, the + middles from between. + "Mary Beth Smith" → middle="Beth" + "Garcia Juan Carlos" family-first → family="Garcia" + "Nguyễn Thị Minh Khai" family-first-given-last → given="Khai" + no-boundary: this is the default reading every other rule carves + exceptions from; its boundaries are the other rules. + implemented: nameparser/_pipeline/_assign.py + ## Scripts & writing systems (W) Background: script-conditional behavior is permitted exactly where @@ -364,6 +430,21 @@ W3. Rationale: a divided name was divided by its writer, and "남궁민수, 지훈" → family="남궁민수" · boundary history: decisions.md#W3 · implemented: nameparser/_pipeline/_script_segment.py +W4. Rationale: Chinese, Japanese and Korean all write the family + name first in native script — the script settles the order + without knowing the language — while a wholly-katakana name is + predominantly a transcribed foreign name already in its source + order. + A name written wholly in one East Asian script, or in the + kana-licensed Japanese repertoire, reads family-first whatever + order the caller declared; a wholly-katakana name keeps the + declared order. + "김 민준" → family="김" + "山田 太郎" → family="山田" + "高橋 みなみ" → family="高橋" + "マイケル ジャクソン" → given="マイケル" · boundary + history: decisions.md#W4 · implemented: nameparser/_pipeline/_assign.py + ## Tokens, initials & punctuation (T) Background: every parsed name part is an exact piece of the input, @@ -406,6 +487,23 @@ T3. Rationale: U+00B7 is two marks in one codepoint — the Chinese ## Ambiguity & tie-breaking (A) +Background: some name strings are genuinely ambiguous — the same +written shape carries two readings ("Van Johnson": given name or +particle?), or the text's structure is malformed. Parsing never +fails and never silently discards; it completes on the best reading +and says what it was unsure of. + +A1. Rationale: a caller can only act on doubt that is reported. + Parsing never fails on any input: where the text's structure or + a word's reading is genuinely uncertain, the parse completes on + the best reading and carries an ambiguity report naming the + doubt. + "Van Johnson" → ambiguities=("particle-or-given",) + "Jane „JD Smith" → ambiguities=("unbalanced-delimiter",) + "John Smith, MD, Bart" → ambiguities=("comma-structure",) + "John Smith" → ambiguities=() · boundary + implemented: nameparser/_pipeline/_state.py + ## Rendering & views (R) ## Construction & configuration diagnostics (D) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index bb4d5a9..32e0362 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -8,13 +8,15 @@ across pieces); token/piece tags; Lexicon only through tags already applied by classify (plus the leading-title period rule). -Ports v1's assignment loops. NO_COMMA (per name_order): +Implements rules N3, O4 and W4 of docs/design/rules.md (plus H2 +above), cited at their code below. Ports v1's assignment loops. +NO_COMMA (per name_order): leading title pieces chain while no given-position name has been seen (a title needs a following piece, unless the whole name is one title); then positional assignment per name_order with the trailing-suffix rule: the piece from which everything after is a strict suffix is the last name-position piece, the rest are suffixes. The v1 single-name+ -nickname rule lives here (plan deviation #2): one non-title piece plus +nickname rule lives here (decisions.md#N3): one non-title piece plus a nonempty nickname puts that piece in FAMILY. FAMILY_COMMA: segment 0 wholly FAMILY (v1 parity); segment 1 gets leading titles, then given, then middles with strict-suffix pieces to @@ -85,6 +87,10 @@ def _peel_leading_titles(pieces: tuple[tuple[int, ...], ...], return n +# rules.md#W4: "a name written wholly in one East Asian script, or in +# the kana-licensed Japanese repertoire, reads family-first whatever +# order the caller declared; a wholly-katakana name keeps the declared +# order" (history: decisions.md#W4) def _effective_order(policy: Policy, pieces: list[tuple[int, ...]], tokens: list[WorkToken], @@ -141,6 +147,9 @@ def _effective_order(policy: Policy, if s is resolved), policy.name_order) +# rules.md#O4: "words no vocabulary has claimed read by position. In +# the default given-first order the first name word is the given name, +# the last is the family name, and everything between is middle names" def _name_positions(order: tuple[Role, Role, Role], count: int) -> list[Role]: """Roles for `count` name pieces (titles/suffixes already peeled), @@ -180,7 +189,9 @@ def _assign_main(seg_idx: int, state: ParseState, rest = [k for k in rest if "suffix" not in ptags[k]] if not rest: return - # v1 nickname rule (plan deviation #2): v1's p_len == 1 counted + # rules.md#N3: "a name that is only a nickname and one name word + # reads that word as the family name" (history: decisions.md#N3) + # -- v1's p_len == 1 counted # the WHOLE segment before any title peeling -- 'Xyz. (Bud) Smith' # has two pieces, so the title peel wins and Smith stays the given # name (pinned live 2026-07-17) diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 5fc6af6..c37c908 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -13,13 +13,12 @@ for the same reason). Reads Policy.extra_suffix_delimiters: tail segments drop delimiter-core tokens (v1 suffix_delimiter parity). -Ports v1's join_on_conjunctions + prefix chains + _join_bound_first_name -plus three additions: the "Ph. D."-split merge (v1 fix_phd, recorded -plan deviation #1), the maiden-marker consuming rule (#274: marker plus -following pieces until a suffix become maiden; the marker itself is -structural, like a delimiter char, and is dropped from assembly), and -the same marker dropped inside EXTRACTED maiden content (#329), which -#274 cannot reach because extract's content never enters pieces. +Implements rules H3, P2, P3 and M2 of docs/design/rules.md and the +group half of M1 (#329: the marker dropped inside EXTRACTED maiden +content, which M2's pieces walk cannot reach because extract's +content never enters pieces); each is cited at its code below. Also +ports v1's _join_bound_first_name and the "Ph. D."-split merge +(v1 fix_phd; decisions.md#phd-merge). """ from __future__ import annotations @@ -55,6 +54,8 @@ class BoundJoin(IntEnum): STRICT = 3 # main segments (reserve_last=True: keep a family piece) +# rules.md#H3: "successive title words at the name's start chain into +# one title; a title word elsewhere in the name does not" def _is_title_piece(piece: Sequence[int], ptags: Set[str], tokens: Sequence[WorkToken]) -> bool: if "title" in ptags: @@ -62,6 +63,9 @@ def _is_title_piece(piece: Sequence[int], ptags: Set[str], return len(piece) == 1 and "vocab:title" in tokens[piece[0]].tags +# rules.md#P2: "a particle joins forward onto the name word after it, +# chains included; the chain begins wherever the name begins, and a +# preceding title does not move that point" (history: decisions.md#P2) def _is_prefix_piece(piece: Sequence[int], ptags: Set[str], tokens: Sequence[WorkToken]) -> bool: if "prefix" in ptags: @@ -79,6 +83,9 @@ def _is_suffix_piece(piece: Sequence[int], ptags: Set[str], return "vocab:suffix" in tags and "initial" not in tags +# rules.md#P3: "a recognized connective joins its neighbors into one +# name part, connective runs included — except a single-letter +# connective in a three-word name, which stays a name word" def _is_conj_piece(piece: Sequence[int], ptags: Set[str], tokens: Sequence[WorkToken]) -> bool: if "conjunction" in ptags: @@ -395,6 +402,10 @@ def group(state: ParseState) -> ParseState: for i in piece[1:]: tokens[i] = dataclasses.replace( tokens[i], tags=tokens[i].tags | {"joined"}) + # rules.md#M2: "a recognized maiden marker inside the name + # takes the words after it — up to any trailing suffix — as + # the maiden name, and the marker itself is dropped" + # (history: decisions.md#M2) # maiden markers: a non-leading marker piece consumes following # pieces until a suffix; consumed tokens become MAIDEN, the # marker is dropped (#274) @@ -419,7 +430,9 @@ def group(state: ParseState) -> ParseState: ptags[m:j] = [] all_pieces.append(tuple(tuple(p) for p in pieces)) all_ptags.append(tuple(frozenset(t) for t in ptags)) - # A marker inside EXTRACTED maiden content (#329). classify tags + # rules.md#M1: "a leading recognized marker word inside the clause + # being dropped" — a marker inside EXTRACTED maiden content + # (#329). classify tags # such a marker like any other token -- what the #274 rule above # lacks is not the TAG but the token: extract claims a delimited # clause and tokenize gives its tokens Role.MAIDEN up front, so diff --git a/nameparser/_pipeline/_segment.py b/nameparser/_pipeline/_segment.py index c35ff6c..1690242 100644 --- a/nameparser/_pipeline/_segment.py +++ b/nameparser/_pipeline/_segment.py @@ -6,7 +6,7 @@ COMMA_STRUCTURE ambiguities for unrecognized extra segments. Reads: Lexicon suffix vocabulary and Policy, both through _vocab.is_wholly_suffix -- the suffix-comma decision is definitionally -vocabulary-dependent (recorded plan deviation #3), and the predicate +vocabulary-dependent (decisions.md#C1), and the predicate owns the rest (Policy.lenient_comma_suffixes picks the lenient or strict token test; Policy.extra_suffix_delimiters gives v1 suffix_delimiter parity, a delimiter-core token being transparent). diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index d524e4e..16fa5bb 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -45,6 +45,10 @@ class Structure(Enum): @dataclass(frozen=True, slots=True) +# rules.md#A1: "parsing never fails on any input: where the text's +# structure or a word's reading is genuinely uncertain, the parse +# completes on the best reading and carries an ambiguity report +# naming the doubt" class PendingAmbiguity: """An ambiguity recorded mid-pipeline by token INDEX; assemble materializes real Ambiguity objects over the final tokens. diff --git a/tests/v2/test_rules_doc.py b/tests/v2/test_rules_doc.py index dcf6c89..f7076e8 100644 --- a/tests/v2/test_rules_doc.py +++ b/tests/v2/test_rules_doc.py @@ -33,7 +33,7 @@ def test_every_rule_has_examples_and_boundary(rule: Rule) -> None: def _run(example: Example) -> object: - if example.field in ("warns", "ambiguities", "pieces"): + if example.field in ("warns", "pieces"): pytest.skip("assertion form lands with its first using rule") policy: Policy | None = None locale: str | None = None @@ -60,6 +60,8 @@ def _run(example: Example) -> object: parsed = Parser(policy=policy).parse(example.text) else: parsed = parse(example.text) + if example.field == "ambiguities": + return tuple(a.kind.value for a in parsed.ambiguities) return getattr(parsed, example.field) From 5daf84bbd3304f929ac28326bf715d409bf84e17 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 16:39:30 -0700 Subject: [PATCH 14/40] docs(rules): probe amendments -- nine statements corrected to measured reality The batched adversarial probe falsified or sharpened: P2 (join runs to the end), H2 (unbroken-run definition; comma carve-out), H3 (comma carve-out), C1 (lenient/strict prose), S2 (bare-ambiguous words-to-spare guard), M1 (S1 wins), M2 (comma and particle carve-outs), W1 (family-position precision), W2 (once; vocabulary), W3 (rescoped to the comma doctrine; segmenter clause qualified), W4 (interpunct source-order carve-out), N3 (suffix-count artifact), T1 (wording). Falsifying inputs pinned as examples; all excerpts updated in lockstep. Co-Authored-By: Claude Fable 5 --- docs/design/rules.md | 132 +++++++++++++++--------- nameparser/_pipeline/_assign.py | 6 +- nameparser/_pipeline/_classify.py | 4 +- nameparser/_pipeline/_group.py | 15 +-- nameparser/_pipeline/_script_segment.py | 21 ++-- 5 files changed, 108 insertions(+), 70 deletions(-) diff --git a/docs/design/rules.md b/docs/design/rules.md index 48ae441..109cf83 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -80,19 +80,25 @@ H1. Rationale: a title normally addresses by surname, so a title H2. Rationale: before a name, an abbreviation is almost always a title — "Rev.", "Ing.", "Mag." — and no vocabulary can list every profession's abbreviations in every language. - A name-opening abbreviation of at least two letters ending in a - period reads as a title even when unlisted; a bare initial does - not. + A name-opening abbreviation — an unbroken run of two or more + letters ending in its one period — reads as a title even when + unlisted; a bare initial does not, and neither does anything + with interior periods, hyphens or digits. "Rev. John Smith" → title="Rev." "Xyz. John Smith" → title="Xyz." "J. Smith" → given="J." · boundary + "J.R. Smith" → given="J.R." · boundary Accepted: the shape is only recognizable as an unbroken run of letters, so an abbreviation in a script whose letters carry combining vowel signs (Bengali, Devanagari) never reads as a title by shape — an unlisted abugida honorific stays a name word, and only vocabulary (#343) can recognize it. "প্রফেসর. Sen" → given="প্রফেসর." - history: decisions.md#H2 · implemented: nameparser/_pipeline/_assign.py + Accepted: before a family comma the pre-comma text is wholly the + family name (C1), so no shape or vocabulary reading makes a + title there. + "Xyz. Smith, John" → family="Xyz. Smith" + history: decisions.md#H2 · interacts: C1 · implemented: nameparser/_pipeline/_assign.py H3. Rationale: compound titles are written as a run of title words, connectives included; a title word standing inside the name is @@ -102,7 +108,10 @@ H3. Rationale: compound titles are written as a run of title words, "Asst. Vice Chancellor John Smith" → title="Asst. Vice Chancellor" "Marquess of Bath" → title="Marquess of Bath" "John Doctor Smith" → middle="Doctor" · boundary - implemented: nameparser/_pipeline/_group.py + Accepted: before a family comma the pre-comma text is wholly the + family name (C1), title words included. + "Dr. Smith, John" → family="Dr. Smith" + interacts: C1 · implemented: nameparser/_pipeline/_group.py ## Particles & surname prefixes (P) @@ -133,10 +142,11 @@ P1. Rationale: a never-given particle standing alone cannot be P2. Rationale: a particle is written as part of the surname it precedes, and a title stands outside the name entirely. - A particle joins forward onto the name word after it, chains - included; the chain begins wherever the name begins, and a - preceding title does not move that point. + A particle joins the words after it into one family name, and + the join runs to the end of the name; the chain begins wherever + the name begins, and a preceding title does not move that point. "John van der Berg" → family="van der Berg" + "John van der Berg Smith" → family="van der Berg Smith" "Sir de Mesnil" → family="de Mesnil" "Juan de" → family="de" · boundary history: decisions.md#P2 · implemented: nameparser/_pipeline/_group.py @@ -178,16 +188,20 @@ S2. Rationale: generational suffixes and credentials are recognized unmistakably a credential when its periods are written. A trailing word of the suffix vocabulary reads as a suffix — generational forms and credential acronyms alike, and an - ambiguous acronym written with periods counts unambiguously. + ambiguous acronym written with periods counts unambiguously. A + BARE ambiguous acronym is consumed only when the name has words + to spare: as the second of two words it stays the family name, + flagged ambiguous. "John Smith Jr." → suffix="Jr." "John Smith M.A." → suffix="M.A." "John Smith PhD" → suffix="PhD" - Accepted: a BARE ambiguous acronym in the trailing position - still reads as a suffix today, even beside an East Asian - surname it more likely belongs to. + "John Ma" → family="Ma" · boundary + Accepted: with words to spare, a bare ambiguous acronym reads + as a suffix even beside an East Asian surname it more likely + belongs to; and an unambiguous suffix is consumed even when + that leaves no family name at all. "Jack Wei Ma" → suffix="Ma" - no-boundary: the vocabulary is the boundary; an unlisted - trailing word simply reads as the family name. + "Smith Jr." → family="" implemented: nameparser/_pipeline/_classify.py ## Nicknames & quoted names (N) @@ -228,6 +242,10 @@ N3. Rationale: a person set down as a nickname plus one name word is positional reading applies. "'Smitty' Jones" → family="Jones" "'Smitty' John Jones" → given="John" · boundary + Accepted: the count does not set suffixes or titles aside, so a + nickname plus one name word plus a suffix reads the name word + as given and leaves the family empty. + "'Smitty' Jones Jr." → family="" history: decisions.md#N3 · implemented: nameparser/_pipeline/_assign.py ## Maiden names (M) @@ -241,24 +259,31 @@ M1. Rationale: an enclosure the caller has declared to mean maiden holds the former family name; a recognized marker word inside it marks the clause and is not itself part of the name. With a delimiter pair configured for maiden names, its enclosed - clause reads as the maiden name, a leading recognized marker - word inside the clause being dropped; a pair configured for both - maiden and nickname reads maiden. + clause reads as the maiden name — unless the content is + suffix-shaped, which S1 takes first — a leading recognized + marker word inside the clause being dropped; a pair configured + for both maiden and nickname reads maiden. "Jane Smith (née Jones)" maiden-parens → maiden="Jones" "Jane Smith (née Jones)" → nickname="née Jones" · boundary - history: decisions.md#M1 · implemented: nameparser/_pipeline/_extract.py, nameparser/_pipeline/_group.py + history: decisions.md#M1 · interacts: S1 · implemented: nameparser/_pipeline/_extract.py, nameparser/_pipeline/_group.py M2. Rationale: a maiden marker announces that what follows it is the former family name; the marker is an announcement, not a name. - A recognized maiden marker inside the name takes the words after - it — up to any trailing suffix — as the maiden name, and the - marker itself is dropped. A marker with nothing after it is just - a word. + A recognized maiden marker standing after at least one name + word takes the words after it — up to any trailing suffix — as + the maiden name, and the marker itself is dropped. A marker + with nothing after it, or nothing before it, is just a word. "Jane Smith née Jones" → maiden="Jones" "Jane née Jones Smith" → maiden="Jones Smith" "Jane Smith née Jones PhD" → suffix="PhD" "Jones née" → family="née" · boundary - history: decisions.md#M2 · implemented: nameparser/_pipeline/_group.py + "née Jones" → family="Jones" · boundary + Accepted: a marker straight after a comma is post-comma given + text, not a marker; and a particle chain swallows a marker in + its path, the join (P2) running first. + "Jane Smith, née Jones" → maiden="" + "Jane de la née Jones" → family="de la née Jones" + history: decisions.md#M2 · interacts: P2 · implemented: nameparser/_pipeline/_group.py ## Commas & structure (C) @@ -276,9 +301,10 @@ C1. Rationale: a credential run after the comma means the name is in the part after the first comma is entirely suffix words and more than one word precedes the comma; otherwise it reads as the listing form, the part before the comma being the family name. - Only the part after the first comma decides. By default the - suffix judgment is lenient about initials-like abbreviations; - strict mode confines it to the recognized vocabulary. + Only the part after the first comma decides. Both modes consult + the vocabulary alone; by default a recognized suffix word counts + even written like an initial ("V."), while strict mode vetoes + initial-shaped words. "Smith, John" → family="Smith" "John Smith, PhD" → suffix="PhD" "John Smith, V." → suffix="V." @@ -389,14 +415,16 @@ section. W1. Rationale: hangul is monoglot Korean and its surnames are a closed census set, so an unspaced hangul name divides at a certain point; Han carries no such certainty by default. - An unspaced name in an activated script divides after a - recognized surname, the longest recognized surname first; where - the vocabulary recognizes nothing, an optional segmenter may - divide instead, and with neither the name stays whole rather - than divide in a wrong place. Korean division is active by - default; Han division is opt-in. + An undivided word in the family position of a name written in + an activated script divides after a recognized surname, the + longest recognized surname first; where the vocabulary + recognizes nothing, an optional segmenter may divide instead, + and with neither the word stays whole rather than divide in a + wrong place. Korean division is active by default; Han division + is opt-in. "김민준" → family="김" "남궁민수" → family="남궁" + "남궁민수 지훈" → family="남궁" "毛泽东" → family="毛泽东" · boundary "毛泽东" [zh] → family="毛" "高橋一郎" [zh] → family="高" @@ -406,10 +434,10 @@ W2. Rationale: some East Asian honorifics glue directly onto the end of the name (田中さん); a glued word peels off only if it could never itself end a name, so the listed vocabulary carries its own license and needs no other gate. - A listed honorific glued to the end of the name's last name word - splits off and reads as a suffix. The peel crosses a family - comma and ignores surrounding punctuation, but never takes a - part that is not name text as its site. + A listed honorific glued to the end of the name's last name + word splits off once and reads as a suffix. The split-off + crosses a family comma and ignores surrounding punctuation, but + never treats a part that is not name text as the name's end. "田中さん" → suffix="さん" "김, 민준씨" → suffix="씨" "田中さん, V." → suffix="さん" @@ -417,16 +445,18 @@ W2. Rationale: some East Asian honorifics glue directly onto the end "王君" → family="王君" · boundary history: decisions.md#W2 · implemented: nameparser/_pipeline/_script_segment.py -W3. Rationale: a divided name was divided by its writer, and - re-dividing would invent a boundary nobody drew; a family name - declared by a comma is likewise the writer's own division. - Division applies only to a name whose written form is undivided: - a name part already containing a division divides no further — - a spaced honorific counts as a written division — and under a - family comma the pre-comma text is the family by declaration - and never divides. Only the honorific peel crosses these, - because an honorific is no part of the name on either side. +W3. Rationale: a family name declared by a comma is the writer's + own division, and re-dividing it would invent a boundary nobody + drew. + Under a family comma the pre-comma text is the family by + declaration and never divides, and the post-comma side is given + text with no family to find; only the honorific split-off (W2) + crosses the comma, an honorific being no part of the name on + either side. A segmenter — unlike the vocabulary — is consulted + only for a name whose written form is wholly undivided, a + spaced honorific counting as a written division. "남궁민수" → family="남궁" + "지훈, 남궁민수" → given="남궁민수" "남궁민수, 지훈" → family="남궁민수" · boundary history: decisions.md#W3 · implemented: nameparser/_pipeline/_script_segment.py @@ -443,7 +473,11 @@ W4. Rationale: Chinese, Japanese and Korean all write the family "山田 太郎" → family="山田" "高橋 みなみ" → family="高橋" "マイケル ジャクソン" → given="マイケル" · boundary - history: decisions.md#W4 · implemented: nameparser/_pipeline/_assign.py + Accepted: a name the interpunct divides keeps its source order — + the divider itself marks a transcription (T3) — so the override + stands down there. + "毛·泽东" → given="毛" + history: decisions.md#W4 · interacts: T3 · implemented: nameparser/_pipeline/_assign.py ## Tokens, initials & punctuation (T) @@ -458,9 +492,9 @@ Catalan punt volat interior to legitimate words (Gal·la). T1. Rationale: a character that carries no name content (emoji, an invisible directionality control) stands between words, not inside them. - A name splits at whitespace and, when stripping is active, at - ignorable characters: an ignorable character separates its - neighbors and never joins them. + A name splits at whitespace and — unless the caller opts to + keep them — at ignorable characters: an ignorable character + separates its neighbors and never joins them. "John😀Smith" → family="Smith" "John😀Smith" keep-emoji → given="John😀Smith" · boundary history: decisions.md#T1 · implemented: nameparser/_pipeline/_tokenize.py diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 32e0362..e68dc7e 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -59,9 +59,9 @@ def _set_roles(tokens: list[WorkToken], piece: tuple[int, ...], tokens[i] = dataclasses.replace(tokens[i], role=role) -# rules.md#H2: "a name-opening abbreviation of at least two letters -# ending in a period reads as a title even when unlisted; a bare -# initial does not" (history: decisions.md#H2) +# rules.md#H2: "a name-opening abbreviation — an unbroken run of two +# or more letters ending in its one period — reads as a title even +# when unlisted; a bare initial does not" (history: decisions.md#H2) def _is_leading_title(piece: tuple[int, ...], ptags: frozenset[str], tokens: list[WorkToken]) -> bool: if _is_title_piece(piece, ptags, tokens): diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py index f6a3ec6..fd3b2cf 100644 --- a/nameparser/_pipeline/_classify.py +++ b/nameparser/_pipeline/_classify.py @@ -36,7 +36,9 @@ # rules.md#S2: "a trailing word of the suffix vocabulary reads as a # suffix — generational forms and credential acronyms alike, and an -# ambiguous acronym written with periods counts unambiguously" +# ambiguous acronym written with periods counts unambiguously. A +# bare ambiguous acronym is consumed only when the name has words to +# spare" def _tags_for(token: WorkToken, state: ParseState) -> frozenset[str]: lex = state.lexicon n = _normalize(token.text) diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index c37c908..b4f4305 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -63,9 +63,10 @@ def _is_title_piece(piece: Sequence[int], ptags: Set[str], return len(piece) == 1 and "vocab:title" in tokens[piece[0]].tags -# rules.md#P2: "a particle joins forward onto the name word after it, -# chains included; the chain begins wherever the name begins, and a -# preceding title does not move that point" (history: decisions.md#P2) +# rules.md#P2: "a particle joins the words after it into one family +# name, and the join runs to the end of the name; the chain begins +# wherever the name begins, and a preceding title does not move that +# point" (history: decisions.md#P2) def _is_prefix_piece(piece: Sequence[int], ptags: Set[str], tokens: Sequence[WorkToken]) -> bool: if "prefix" in ptags: @@ -402,10 +403,10 @@ def group(state: ParseState) -> ParseState: for i in piece[1:]: tokens[i] = dataclasses.replace( tokens[i], tags=tokens[i].tags | {"joined"}) - # rules.md#M2: "a recognized maiden marker inside the name - # takes the words after it — up to any trailing suffix — as - # the maiden name, and the marker itself is dropped" - # (history: decisions.md#M2) + # rules.md#M2: "a recognized maiden marker standing after at + # least one name word takes the words after it — up to any + # trailing suffix — as the maiden name, and the marker itself + # is dropped" (history: decisions.md#M2) # maiden markers: a non-leading marker piece consumes following # pieces until a suffix; consumed tokens become MAIDEN, the # marker is dropped (#274) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index a6e0134..00c6dc4 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -272,9 +272,10 @@ def _peel_site(state: ParseState, flat: Sequence[int], # rules.md#W2: "a listed honorific glued to the end of the name's -# last name word splits off and reads as a suffix. The peel crosses a -# family comma and ignores surrounding punctuation, but never takes a -# part that is not name text as its site." (history: decisions.md#W2) +# last name word splits off once and reads as a suffix. The +# split-off crosses a family comma and ignores surrounding +# punctuation, but never treats a part that is not name text as the +# name's end." (history: decisions.md#W2) def _peel_honorific_tail(state: ParseState) -> ParseState: """#308: split a listed honorific off the END of the name's last NON-POST-NOMINAL token -- 田中さん -> 田中 + さん -- and let @@ -619,13 +620,13 @@ def _split_surname_site(state: ParseState) -> ParseState: return _split(state, i, answer.splits, detail) -# rules.md#W1: "an unspaced name in an activated script divides after -# a recognized surname, the longest recognized surname first; where -# the vocabulary recognizes nothing, an optional segmenter may divide -# instead, and with neither the name stays whole" -# rules.md#W3: "a name part already containing a division divides no -# further" and "under a family comma the pre-comma text is the family -# by declaration and never divides" (history: decisions.md#W3) +# rules.md#W1: "an undivided word in the family position of a name +# written in an activated script divides after a recognized surname, +# the longest recognized surname first; where the vocabulary +# recognizes nothing, an optional segmenter may divide instead" +# rules.md#W3: "under a family comma the pre-comma text is the +# family by declaration and never divides, and the post-comma side +# is given text with no family to find" (history: decisions.md#W3) def script_segment(state: ParseState) -> ParseState: if state.original.isascii(): # spans index the original exactly (the anti-#100 invariant), From 246aadcd1221f42867b24ad6391ee8c08450166b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 16:50:38 -0700 Subject: [PATCH 15/40] docs(rules): extract rendering into R (views, fold order, initials, case repair) Co-Authored-By: Claude Fable 5 --- docs/design/rules.md | 43 ++++++++++++++++++++++++++++++++++++++ nameparser/_render.py | 5 +++++ nameparser/_types.py | 7 +++++++ tests/v2/rules_doc.py | 1 + tests/v2/test_rules_doc.py | 4 ++++ 5 files changed, 60 insertions(+) diff --git a/docs/design/rules.md b/docs/design/rules.md index 109cf83..104bdfe 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -540,4 +540,47 @@ A1. Rationale: a caller can only act on doubt that is reported. ## Rendering & views (R) +Background: parsing produces words with roles; every string a caller +reads is assembled from those words on request. Nothing about +rendering changes the parse, and nothing about reading a field +mutates anything. + +R1. Rationale: a field is a way of reading the parse, not a stored + string. + Every field is a view computed from the parsed words at read + time, joining its words in written order — except folded family + words (O3), which render before the rest of the family wherever + they stood in the string. + "Dr. Juan Q. Xavier de la Vega III" → family="de la Vega" + "Hassan, Mohamad Ahmad Ali" middle_as_family → family="Ahmad Ali Hassan" + "Hassan, Mohamad Ahmad Ali" → family="Hassan" · boundary + implemented: nameparser/_types.py + +R2. Rationale: callers need the surname with and without its + particles — sorting wants "Vega", display wants "de la Vega". + The family name splits into further views: the base (the family + without its leading particles) and the particles themselves. + "Dr. Juan Q. Xavier de la Vega III" → family_base="Vega" + "Dr. Juan Q. Xavier de la Vega III" → family_particles="de la" + "Sean O'Connor" → family_base="O'Connor" · boundary + implemented: nameparser/_types.py + +R3. Rationale: initials abbreviate the person's name words; titles, + suffixes, particles and nicknames are not name words. + Initials take the first letter of each given, middle, and base + family word; titles, suffixes, particles and nicknames + contribute nothing. + "Dr. Juan Q. Xavier de la Vega III" → initials="J. Q. X. V." + "Sean O'Connor" → initials="S. O." · boundary + implemented: nameparser/_render.py + +R4. Rationale: case repair is a display concern, applied only on + request and never destructively. + Case repair returns a repaired copy — vocabulary exceptions + (McDonald) included — and never mutates the parse; an + already-correct name comes back unchanged. + "juan mcdonald" → capitalized="Juan McDonald" + "Juan McDonald" → capitalized="Juan McDonald" · boundary + implemented: nameparser/_render.py + ## Construction & configuration diagnostics (D) diff --git a/nameparser/_render.py b/nameparser/_render.py index 702e8e7..f4846f5 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -92,6 +92,9 @@ def render(name: ParsedName, spec: str) -> str: return _format_spec(spec, values, "render", _RENDER_KEYS) +# rules.md#R3: "initials take the first letter of each given, middle, +# and base family word; titles, suffixes, particles and nicknames +# contribute nothing" def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str: """First letter of each contributing token per group, v1 semantics: delimiter follows each initial, separator sits between initials @@ -144,6 +147,8 @@ def _cap_text(text: str, role: Role, lex: Lexicon) -> str: return _WORD.sub(lambda m: _cap_word(m.group(0), role, lex), text) +# rules.md#R4: "case repair returns a repaired copy — vocabulary +# exceptions (McDonald) included — and never mutates the parse" def capitalized(name: ParsedName, lexicon: Lexicon | None, *, force: bool) -> ParsedName: """Case-fixing transform -> new ParsedName, same spans, new token diff --git a/nameparser/_types.py b/nameparser/_types.py index 8da6a6e..322888c 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -608,6 +608,10 @@ def middle(self) -> str: return self._text_for(Role.MIDDLE) @property + # rules.md#R1: "every field is a view computed from the parsed + # words at read time, joining its words in written order — except + # folded family words (O3), which render before the rest of the + # family" def family(self) -> str: return self._text_for(Role.FAMILY) @@ -626,6 +630,9 @@ def maiden(self) -> str: # -- derived views (filters over roles + STABLE tags only) ---------- @property + # rules.md#R2: "the family name splits into further views: the + # base (the family without its leading particles) and the + # particles themselves" def family_particles(self) -> str: return self._text_for(Role.FAMILY, tag="particle") diff --git a/tests/v2/rules_doc.py b/tests/v2/rules_doc.py index f930197..4909c9d 100644 --- a/tests/v2/rules_doc.py +++ b/tests/v2/rules_doc.py @@ -40,6 +40,7 @@ ASSERTABLE_FIELDS = frozenset({ "title", "given", "middle", "family", "suffix", "nickname", "maiden", + "family_base", "family_particles", "initials", "capitalized", "ambiguities", "pieces", "warns"}) diff --git a/tests/v2/test_rules_doc.py b/tests/v2/test_rules_doc.py index f7076e8..26a3f44 100644 --- a/tests/v2/test_rules_doc.py +++ b/tests/v2/test_rules_doc.py @@ -62,6 +62,10 @@ def _run(example: Example) -> object: parsed = parse(example.text) if example.field == "ambiguities": return tuple(a.kind.value for a in parsed.ambiguities) + if example.field == "initials": + return parsed.initials() + if example.field == "capitalized": + return str(parsed.capitalized()) return getattr(parsed, example.field) From 60d0f8e3b49b6b4fa7bdbff3a577519f9006733e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 16:52:23 -0700 Subject: [PATCH 16/40] docs(rules): D-section diagnostics with warns=/raises= forms Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 20 ++++++++++++++++++++ docs/design/rules.md | 34 ++++++++++++++++++++++++++++++++++ nameparser/_parser.py | 7 +++++++ nameparser/_policy.py | 2 ++ tests/v2/rules_doc.py | 23 ++++++++++++++++++++++- tests/v2/test_rules_doc.py | 21 ++++++++++++++++++++- 6 files changed, 105 insertions(+), 2 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 99b5a8a..32684f4 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -105,6 +105,26 @@ which particles count as never-given. kana cannot be Chinese and is not a transcription, so 高橋みなみ reads family-first though it is written in two scripts. +### D1 — the segmenterless-activation warning + +- 2026-08 #337 — the warning exists because parser_for(locales.JA) + without segmenter= used to build a parser that silently behaved + like a working one minus the feature; it re-emits from the + parser_for frame so the reported location is the caller's own + call. +- 2026-08-07 #339 — diagnostics that hand the reader code must hand + code that type-checks: the message's offered deactivation was + Policy(segment_scripts=()) — an arg-type error under mypy in a + py.typed package — and became frozenset(), pinned by a test + asserting the offered spelling. The known-bad spelling is in the + denylist test. + +### D2 — construction raises, parse never does + +- 2026-07 (v2 core, PR #288) — every raise in the locale-apply path + is a plain TypeError/ValueError so the wrap-with-locale-code + rewrap cannot break on exotic exception signatures. + ### H2 — the leading-abbreviation title - 2026-06-30 (leading-period-title design; v2 core, PR #288) — the diff --git a/docs/design/rules.md b/docs/design/rules.md index 104bdfe..1fc886a 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -536,6 +536,10 @@ A1. Rationale: a caller can only act on doubt that is reported. "Jane „JD Smith" → ambiguities=("unbalanced-delimiter",) "John Smith, MD, Bart" → ambiguities=("comma-structure",) "John Smith" → ambiguities=() · boundary + Accepted: the one exception to totality is a user-supplied + segmenter's own error, which propagates — a user-code error is + not a content error. (Needs the optional extra to demonstrate, + so no example line.) implemented: nameparser/_pipeline/_state.py ## Rendering & views (R) @@ -584,3 +588,33 @@ R4. Rationale: case repair is a display concern, applied only on implemented: nameparser/_render.py ## Construction & configuration diagnostics (D) + +Background: configuration mistakes are reported when they are made — +at construction — not when a name happens to hit them; and a +diagnostic that hands the reader code must hand code that works and +type-checks. + +D1. Rationale: a parser whose activated scripts nothing can divide + behaves like a working parser minus a feature, silently — the + one misconfiguration a caller cannot see in output. + Constructing a parser that activates division for scripts with + no covering surnames and no segmenter warns at construction, + naming the dead scripts and each way out. + [segmenterless-ja] → warns="deactivate with Policy(segment_scripts=frozenset())" + no-boundary: any covering surname vocabulary, configured + segmenter, or deactivation silences it — the default parser and + the zh pack never warn, which every other example in this + document exercises. + history: decisions.md#D1 · implemented: nameparser/_parser.py + +D2. Rationale: whatever a name contains, parsing answers; only a + broken configuration may raise, and it must name the field. + Configuration validation raises at construction with the + offending field and value named; applying a locale pack wraps + any such error with the locale's code, so a stacked + configuration names which layer broke. + [bad-name-order] → raises="name_order elements must be Role members" + [bad-order-none] → raises="name_order must be an iterable" + no-boundary: the non-raising side is every other rule's + examples; parse() itself is total (A1). + history: decisions.md#D2 · implemented: nameparser/_policy.py, nameparser/_parser.py diff --git a/nameparser/_parser.py b/nameparser/_parser.py index a13806c..7c22049 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -292,6 +292,13 @@ def parser_for(*locales: Locale, base: Parser | None = None, # a subclass with extra mandatory args would break this rewrap raise type(exc)( f"while applying locale {loc.code!r}: {exc}") from exc + # rules.md#D1: "constructing a parser that activates division for + # scripts with no covering surnames and no segmenter warns at + # construction, naming the dead scripts and each way out" + # (history: decisions.md#D1) + # rules.md#D2: "applying a locale pack wraps any such error with + # the locale's code, so a stacked configuration names which layer + # broke" (history: decisions.md#D2) # Construction warnings (the segmenterless-activation check in # Parser.__post_init__) re-emit from THIS frame: its stacklevel is # sized for direct Parser(...) construction, and through this diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 657717d..c897827 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -654,6 +654,8 @@ class Policy: __getstate__ = _guarded_getstate __setstate__ = _guarded_setstate + # rules.md#D2: "configuration validation raises at construction + # with the offending field and value named" def __post_init__(self) -> None: object.__setattr__( self, "name_order", _validated_order(self.name_order, diff --git a/tests/v2/rules_doc.py b/tests/v2/rules_doc.py index 4909c9d..281e4d0 100644 --- a/tests/v2/rules_doc.py +++ b/tests/v2/rules_doc.py @@ -41,7 +41,7 @@ ASSERTABLE_FIELDS = frozenset({ "title", "given", "middle", "family", "suffix", "nickname", "maiden", "family_base", "family_particles", "initials", "capitalized", - "ambiguities", "pieces", "warns"}) + "ambiguities", "pieces", "warns", "raises"}) @dataclass(frozen=True) @@ -82,6 +82,27 @@ def has_boundary_or_waiver(self) -> bool: "keep-emoji": Policy(strip_emoji=False), "strict-comma-suffixes": Policy(lenient_comma_suffixes=False), } +#: D-section subjects: zero-arg constructions whose diagnostics the +#: warns=/raises= assertion forms exercise. +def _segmenterless_ja() -> object: + from nameparser import locales, parser_for + return parser_for(locales.get("ja")) + + +def _bad_name_order() -> object: + return Policy(name_order=("given",)) + + +def _bad_order_none() -> object: + return Policy(name_order=None) # type: ignore[arg-type] + + +SUBJECTS: dict[str, object] = { + "segmenterless-ja": _segmenterless_ja, + "bad-name-order": _bad_name_order, + "bad-order-none": _bad_order_none, +} + #: Extras gates: locale requiring an optional dependency; the examples #: runner skips these when the import is absent (CI's ja-extra job #: exercises them). diff --git a/tests/v2/test_rules_doc.py b/tests/v2/test_rules_doc.py index 26a3f44..98dc069 100644 --- a/tests/v2/test_rules_doc.py +++ b/tests/v2/test_rules_doc.py @@ -32,8 +32,24 @@ def test_every_rule_has_examples_and_boundary(rule: Rule) -> None: f"'no-boundary: '") +def _check_diagnostic(example: Example) -> None: + from typing import Callable, cast + + import re as _re + + from tests.v2.rules_doc import SUBJECTS + fn = cast(Callable[[], object], SUBJECTS[example.subject or ""]) + pattern = _re.escape(str(example.value)) + if example.field == "warns": + with pytest.warns(UserWarning, match=pattern): + fn() + else: + with pytest.raises((TypeError, ValueError), match=pattern): + fn() + + def _run(example: Example) -> object: - if example.field in ("warns", "pieces"): + if example.field == "pieces": pytest.skip("assertion form lands with its first using rule") policy: Policy | None = None locale: str | None = None @@ -76,6 +92,9 @@ def _run(example: Example) -> object: @pytest.mark.parametrize("rule, example", _EXAMPLES, ids=_EXAMPLE_IDS) def test_example(rule: Rule, example: Example) -> None: + if example.field in ("warns", "raises"): + _check_diagnostic(example) + return actual = _run(example) if example.deviates_issue is not None: assert actual == example.today_value, ( From a18061549cc72dfa58d674c5cf6eb751eb7f3e0d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 16:53:11 -0700 Subject: [PATCH 17/40] docs(rules): D-section diagnostics with warns=/raises= forms The denylist caught its own seed spelling being reintroduced in the D1 decisions entry -- reworded to describe the bad spelling without containing it. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 6 +++--- tests/v2/rules_doc.py | 2 +- tests/v2/test_rules_doc.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 32684f4..fc505e4 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -113,9 +113,9 @@ which particles count as never-given. parser_for frame so the reported location is the caller's own call. - 2026-08-07 #339 — diagnostics that hand the reader code must hand - code that type-checks: the message's offered deactivation was - Policy(segment_scripts=()) — an arg-type error under mypy in a - py.typed package — and became frozenset(), pinned by a test + code that type-checks: the message's offered deactivation used a + bare tuple literal for segment_scripts — an arg-type error under + mypy in a py.typed package — and became frozenset(), pinned by a test asserting the offered spelling. The known-bad spelling is in the denylist test. diff --git a/tests/v2/rules_doc.py b/tests/v2/rules_doc.py index 281e4d0..20ece86 100644 --- a/tests/v2/rules_doc.py +++ b/tests/v2/rules_doc.py @@ -90,7 +90,7 @@ def _segmenterless_ja() -> object: def _bad_name_order() -> object: - return Policy(name_order=("given",)) + return Policy(name_order=("given",)) # type: ignore[arg-type] def _bad_order_none() -> object: diff --git a/tests/v2/test_rules_doc.py b/tests/v2/test_rules_doc.py index 98dc069..511df38 100644 --- a/tests/v2/test_rules_doc.py +++ b/tests/v2/test_rules_doc.py @@ -33,9 +33,9 @@ def test_every_rule_has_examples_and_boundary(rule: Rule) -> None: def _check_diagnostic(example: Example) -> None: - from typing import Callable, cast - import re as _re + from collections.abc import Callable + from typing import cast from tests.v2.rules_doc import SUBJECTS fn = cast(Callable[[], object], SUBJECTS[example.subject or ""]) From bf52e7544e2c4c7fb87894dc89e05ac39463d465 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 16:55:01 -0700 Subject: [PATCH 18/40] docs(rules): mechanisms catalog with contract statements + verification shapes Co-Authored-By: Claude Fable 5 --- docs/design/mechanisms.md | 271 +++++++++++++++++++++++++++- nameparser/_pipeline/_post_rules.py | 6 +- 2 files changed, 272 insertions(+), 5 deletions(-) diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 35de0aa..170f9e9 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -23,7 +23,274 @@ This catalog converts discovery into a one-time cost — it does not remove discovery. What nobody knows yet still has to be found the hard way, once; the promise is that found things stay found. +## SPANS — position is identity, text is not + +Problem shape. A later stage needs to refer to "that word." +Contract statement. Every token carries its (start, end) character +span in the original string, and stages refer to tokens by index, +never by searching for matching text. +How it works. v1 re-found pieces by value, so a name with a repeated +word could rewrite the wrong occurrence (#100 and relatives); a +position cannot be confused with a look-alike. Every parsed part is +an exact slice of the input (rule T1's Background). +Lives in. nameparser/_types.py (Span), threaded through every +_pipeline/ stage. +Reach for it when. New code is about to do `if token.text == ...` to +LOCATE rather than to classify. + +## FOLDED_TAG — reorder at render time, not parse time + +Problem shape. A rule wants words to RENDER in a different order +than they sit in the string. +Contract statement. Tokens never move: a rule that needs different +rendering order tags the token, and the rendering views consult the +tag — family views order folded tokens first. +How it works. Reordering the token tuple would break span math and +reintroduce the #100 family. Parse state stays in string order; only +the view reorders (rule R1, rule O3's render clause). +Lives in. nameparser/_types.py (FOLDED_TAG, the family view), +nameparser/_pipeline/_post_rules.py (the one producer today). +Reach for it when. A new rule needs "X renders before Y" and you are +tempted to swap tokens. Don't swap. Tag. + +## VOCAB-TAGS — the vocabulary layer speaks once + +Problem shape. A later stage needs to know what the vocabulary knew +about a word. +Contract statement. classify tags every token with what the +vocabulary knows about it, and later stages test tags — they never +re-look a word up. +How it works. One lookup site means one answer: a stage that +re-derived vocabulary facts could disagree with the stage before it. +Stable tags ("particle", "conjunction", "initial") are API; +"vocab:"-namespaced ones are not. +Lives in. nameparser/_pipeline/_classify.py (producer); consumers +throughout _group/_assign/_post_rules. +Reach for it when. A stage is about to import Lexicon to ask about a +word classify already saw. + +## PIECES — joining structure survives assignment + +Problem shape. A rule needs to know how words were JOINED (chained +titles, particle groups), not just what roles they got. +Contract statement. group records the joining structure as pieces — +runs of token indices per segment — and that structure remains +readable after roles are assigned. +How it works. Roles alone lose the grouping ("who chained with +whom"); #359's fix made the particle fold read the opening PIECE +rather than the assigned role, which is what makes rule P1 hold +under every name_order. +Lives in. nameparser/_pipeline/_group.py (producer), +_pipeline/_state.py (ParseState.pieces), _post_rules.py (reader). +Reach for it when. A rule keyed on assigned roles behaves +differently under different name_order values — the stable thing to +read is usually the structure. + +## STRUCTURE-GATES — comma shape as an explicit state + +Problem shape. A rule should fire only under one comma convention. +Contract statement. segment decides the comma structure once +(NO_COMMA, FAMILY_COMMA, SUFFIX_COMMA) and every later stage gates +on that single decision rather than re-inspecting commas. +Lives in. nameparser/_pipeline/_state.py (Structure), +_pipeline/_segment.py (the one decider). +Reach for it when. New code is about to count commas. + +## TWO-LAYER-ASSIGN — vocabulary claims, position takes the rest + +Problem shape. Where should a new "recognize X" behavior live? +Contract statement. A vocabulary layer first claims words for what +they ARE, wherever they sit; a positional layer then reads every +unclaimed word by where it STANDS. Every rule belongs to exactly one +layer. +How it works. The two layers compose without ordering bugs because +the positional layer never overrides a vocabulary claim (rule O4 is +the positional layer's contract). +Lives in. _classify/_group (vocabulary side), _assign (positional +side). +Reach for it when. A proposed rule wants a word's identity AND its +position at once — split it, or it will fight both layers. + +## STATE-OFFSET-CHANNELS — early facts ride the state + +Problem shape. A fact known during tokenization matters to a much +later stage. +Contract statement. A pre-token fact is recorded as offsets on the +ParseState (comma_offsets, interpunct_offsets) and consulted later +by position, rather than re-derived from text. +How it works. The offsets survive every intermediate stage +untouched; #298's transcription marker rides this channel from +tokenize to order resolution (rules T3/W4). +Lives in. nameparser/_pipeline/_state.py, produced in _tokenize. +Reach for it when. You are about to re-scan the original string in a +late stage to rediscover something tokenize already knew. + +## PIPELINE-STAGE-CONTRACTS — the ownership map + +Problem shape. "Which stage does X?" — asked before attributing +behavior in prose, comments, or fixes. +Contract statement. Each stage's docstring header declares what it +consumes, produces and reads, and ParseState's docstring holds the +cross-stage map, pinned by tests/v2/pipeline/test_state.py. +How it works. A claim about which stage or layer does something is +CHECKABLE — `parse(s).tokens` prints every token's role and tags — +so check it before writing it; one plausible attribution sentence +once shipped six times wrong (AGENTS.md's stage-attribution note). +Lives in. nameparser/_pipeline/_state.py and every stage header. +Reach for it when. Writing any sentence of the form "X happens +before Y sees it." + +## CLAUSE-CONTENT-OVERRULES-DELIMITER — content wins + +Problem shape. A bracketed clause should be treated as something +other than what its delimiter pair says. +Contract statement. extract may inspect a clause's content against +the lexicon and, when it matches, mask only the two delimiter spans +so the inner content rejoins the main token stream for ordinary +downstream parsing. +How it works. "Andrew Perkins (MBA)" is not a nickname (rule S1): +the parens are masked away and MBA is classified by the normal +machinery — reusing the downstream path, so the delimited and bare +forms cannot drift. +Lives in. nameparser/_pipeline/_extract.py (_suffix_shaped and the +inner-span branch). +Reach for it when. About to add a second code path that duplicates +what the bare form already does — #335's fix is this shape (a +_maiden_marked sibling predicate). + +## CURATED-VOCABULARY-ALTERNATION — the config already splits them + +Problem shape. Two string shapes collide — the same written form +means two different things — and no predicate separates them. +Contract statement. A curated vocabulary that OMITS the ambiguous +entries is itself the separator: membership is the license, and the +per-entry vetting reasons live beside the set. +How it works. GLUED_HONORIFICS is the exemplar (rule W2): 씨 peels +because it can never end a name; 양 stays out because 김지양 is a +given name. The set, not a regex, draws the line. +Lives in. nameparser/config/suffixes.py (the vetting block). +Reach for it when. Arguing that "no regex can separate X from Y" — +check whether a config set already splits them by listing one side. + +## RECORDED-ROSTERS — record the answer, don't re-derive it + +Problem shape. A guard needs to know what the answer WAS, so it can +detect the answer changing. +Contract statement. Store the measured answer as literal data (a +roster) and compare against it; never re-derive the expectation from +the same inputs the check reads, because a derivation from the same +data always agrees with itself. +Lives in. tools/differential/compare.py (_CORPUS_CLAIMS and kin); +tests/v2/test_facade_cases.py (_CORE_ONLY_IDS). +Reach for it when. Writing a check whose expected value is computed +by the code under test, or a comment that enumerates ids/counts — +make it data the suite asserts. + +## LEDGER-RULE-SEPARATION — fields separate rules, file order doesn't + +Problem shape. Two differential-ledger rules claim overlapping +names. +Contract statement. Ledger rules are separated by their fields +subsets and matching predicates, never by their order in the file; +a fields-only rule sorts last and takes what nothing narrower named. +How it works. Detail is owned by tools/differential/README.md. One +standing constraint worth repeating here: sync-pinned rosters select +rules by issue-string substring, so a new rule's issue slug must +avoid the literal #271/#272 substrings unless it means to be +selected. +Lives in. tools/differential/compare.py, the expected_since_*.toml +ledgers. +Reach for it when. A ledger rule's behavior seems to depend on where +it sits in the file — it doesn't, and if moving it changes anything, +the fields are wrong. + +## MAKE-WRONG-STATES-UNREPRESENTABLE — the house meta-pattern + +Problem shape. A convention keeps being violated no matter how +clearly it is written down. +Contract statement. Convert the consistency problem into a +referential-integrity problem: make the wrong state impossible to +express, or mechanically checked, rather than merely documented. +How it works. Three instances built this documentation system: +_CORE_ONLY_IDS replaced an enumerating comment; verbatim-excerpt +citations replaced paraphrase; no-boundary: markers replaced silent +omission. The executable examples in rules.md are the same move at +document scale. +Lives in. tests/v2/test_doc_citations.py, tests/v2/test_rules_doc.py, +and every recorded roster. +Reach for it when. Tempted to write "remember to keep X in sync +with Y." + ## Verification shapes -How to measure in this codebase without fooling yourself. (Entries -land with the mechanisms content pass.) +How to measure in this codebase without fooling yourself. The +inert-measurement class — checks that run, print plausible results, +and measure nothing — has recurred double-digit times; these shapes +are its known antidotes. Convention (AGENTS.md): guard tests SHOULD +carry a RECORDED negative control, the _EXCLUSION_EFFECT shape — the +answer with the guard off, stored as data. Honest limit: these +reduce the inert-measurement class, not the wrong-predicate class; a +guard asserting the wrong invariant is caught only by adversarial +review. + +### VERSION-TELL — know who answered + +Contract statement. A subprocess that speaks for a pinned version +writes its __version__ and __file__ as its first output line, and +the caller aborts before comparing anything if either half disagrees +with what was requested. +Both halves are load-bearing: an editable install reports the tree's +version (agreement proves nothing when tree and baseline share a +number); a genuine wheel at the wrong version passes any path check. +Lives in tools/differential/compare.py (_check_tell). + +### GENERATED-SCRIPT-OUTSIDE-THE-WORKTREE — escape the shadow + +Contract statement. A worker that must run under a pinned dependency +is rendered to a temp directory outside the worktree and spawned by +absolute path, so sys.path[0] contains no copy of the package and +the inline pin is genuine. +Sentinel substitution (@@VERSION@@), not str.format — the worker +body is mostly literal braces. Lives in +tools/differential/compare.py (_worker_source, _run_worker). + +### SENTINEL-SET-OVER-MATCH-CHECK — catching match-everything + +Contract statement. A user-supplied pattern is rejected as +over-matching by probing it against a small set of inputs sharing no +script, vocabulary or punctuation; matching all of them means it +targets no behavior family. +Measured: `.`, `.+`, `\b` and `[\s\S]` all decline the empty string +— the naive probe — and still match every corpus name. Lives in +tools/differential/compare.py (_SENTINELS). + +### FORCE-A-DECISION-TABLE — no silent defaults on growth + +Contract statement. Where adding an enum member or a file must not +silently inherit a default, a local table's key set is asserted +equal to the population, so growth fails the suite until someone +decides — against a local table, not the constant under test. +Exemplar: tests/v2/pipeline/test_vocab.py's per-script initials +check; reused for _CORPUS_FLOORS in tools/differential/compare.py. +Known gap it exposes: DEFAULT_SCRIPT_ORDERS has no such guard. + +### Field notes — the traps themselves + +- Assert which tree you imported, on BOTH sides of a comparison. + `python -c` puts CWD on sys.path; a script's own directory holds + no nameparser in tools/differential/, so a stray PYTHONPATH + outranks the editable install — measured: 89 diffs became 0, exit + 0, both tell halves passing, because both sides had become the + shadow. +- Never pipe a gate's output. Under zsh, `compare.py | tail` makes + `$?` tail's status. Redirect to a file and read the file. +- Mutation-test a new guard before believing it, and mutate the + thing the guard watches — a survivor usually means the fixture + satisfies the invariant for free. +- Verify the restore, not just the mutation: diff against the + pre-mutation copy; do not trust the harness's own restore report. +- Run all the gates, not the ones you remember: ruff runs before + mypy and pytest in CI, and each has caught what the others + passed. +- Purge __pycache__ between same-length source mutations; stale + bytecode makes a changed file measure as unchanged. diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index bd0b321..3577f6b 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -150,9 +150,9 @@ def post_rules(state: ParseState) -> ParseState: _retag(tokens, g, Role.FAMILY) # rules.md#O3: "every middle word joins the family name and is # rendered before it" (v1 handle_middle_name_as_last). v1 - # PREPENDED middle_list to last_list; spans cannot reorder - # (anti-#100), so folded tokens carry a tag and the family views - # order them first. + # PREPENDED middle_list to last_list; mechanisms.md#FOLDED_TAG: + # "tokens never move: a rule that needs different rendering order + # tags the token, and the rendering views consult the tag" if state.policy.middle_as_family: for i in _idx(tokens, Role.MIDDLE): tokens[i] = dataclasses.replace( From cd49ce5b6ecfa971c86961541eade7340c127dfd Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 16:56:39 -0700 Subject: [PATCH 19/40] docs(rules): seed decisions.md -- 2.1-arc harvest, Excluded blocks, 3.0 promotion Harvest measurements spot-checked at landing; the two flagged uncertainties resolved against suffixes.py's own vetting block (the per-90-surnames count became an invariant). The 3.0 shim-shaped list is promoted from session memory into the 3-0-reevaluations section; the memory file is now a pointer. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 133 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index fc505e4..75d8cd5 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -92,6 +92,18 @@ which particles count as never-given. vocabulary (v1 fix_phd), so the spaced and unspaced spellings read alike. +### O4 — positional assignment and declared order + +- 2026-08-07 #83 — romanized Chinese order is answered by + DECLARATION, not detection: pinyin carries no signal and diaspora + makes locale no guide (the issue's own 2019/2021 conclusions); + Policy(name_order=FAMILY_FIRST) is the answer the thread wanted. +- 2026-08-07 #146 — Vietnamese needs the THIRD order constant: + measured on "Nguyễn Thị Minh Khai", FAMILY_FIRST strands + given="Thị" while FAMILY_FIRST_GIVEN_LAST reads + given="Khai", middle="Thị Minh", family="Nguyễn". This is why + three order constants exist rather than two. + ### W4 — script-scoped order - 2026-07-27 (script-scoped order amendment) — the family-first @@ -197,6 +209,34 @@ which particles count as never-given. run offers a site, since a glued honorific is itself part of what makes a run read as suffix-shaped. +Excluded (Lexicon.honorific_tails — a glued tail peels only if it +could never end a name; per-entry reasons live in +nameparser/config/suffixes.py's vetting block): + +- 양, 군 — 양 is also a top-tier surname (Yang), and 김지양 is a + given name; the surname-leads argument covers 군 the same way. +- 氏 — recognized spaced only. +- 博士 — 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka. +- 殿 — Japanese surnames end in it (鵜殿, 真殿). +- 君 — 王君 is a complete Chinese name; its kana spelling くん does + peel. + +Excluded (Policy.script_orders defaults): Script.KATAKANA is +deliberately absent — a pure-katakana token is predominantly a +transcribed foreign name kept in its source order, so nothing +defaults on it (rule W4's boundary). Noted 2026-08-15: of the three +Script-keyed axes, this is the one with no force-a-decision guard +(mechanisms.md#FORCE-A-DECISION-TABLE), so a new Script member +silently gets no order. + +Excluded (MAIDEN_MARKERS, per nameparser/config/maiden_markers.py): + +- Polish "z domu" — a two-token marker; pending the multi-token + matching decision. +- Scandinavian "f." — collides with the initial F.; only the full + participles (født/fødd/född) are safe. Czech masculine "rozený" + awaits the same vetting. + ### C1 — the suffix-comma decision - 2026-07 (v2 core, PR #288) — v1 parity: only the second segment @@ -243,6 +283,14 @@ which particles count as never-given. itself keeps the whole enclosed span, so nothing is lost when there is no marker. +- 2026-08-05 #329/#335 — marker auto-detection inside a + nickname-delimited clause was deferred to #335 on a corpus + measurement: 山田 花子（旧姓 佐藤） is in the CJK differential + corpus so the #329 change was gate-visible, while "Jane Smith + (née Jones)" is in no corpus — shipping auto-detection in 2.1 + would have let a real Latin-affecting change ride under a "0 + Latin-only" gate report. + Open: [#335](https://github.com/derek73/python-nameparser/issues/335) should a marker inside a NICKNAME-delimited clause flip it to maiden without configuration. @@ -267,3 +315,88 @@ how the rotations interact with non-default name_order values. Open: [#270](https://github.com/derek73/python-nameparser/issues/270) same rotation/name_order interaction as O1. + +### differential-ledger — tooling decisions (2.1.0 release arc) + +Harvested 2026-08-15 from the release-arc session; measurements are +that session's, spot-checked at landing. + +- 2026-08-05 #332 — ledger field vocabulary is Role's names, not the + facade's: canonicalizing to first/last would have put an eighth + place naming roles differently from Role inside the durable + record, and the facade vocabulary is removed at 3.0. + +Declined: + +- Policy annotations widened to input unions (2026-08-06 #334) — + five documented spellings fail mypy and every one has a + type-clean equivalent; widening would make every READER see a + union, and reading is the commoner operation. A .pyi stub typing + __init__ wide and attributes narrow was deferred as a 3.0 + candidate (needs a parallel signature list with its own sync + test). +- Closing the differential gate's default-policy ceiling + (2026-08-04 #332) — a per-row policy in the corpus needs defined + behavior for baselines that cannot construct newer policy fields; + tests/v2/cases.py already covers opt-in paths per row, so the + ceiling is documented instead. +- `_check_tree` as is_relative_to(REPO_ROOT) (2026-08-06 #332) — + accepts .venv/, build/ and dist/ copies inside the repo; the + invariant is "is the source package", so the predicate is + is_relative_to(REPO_ROOT / "nameparser"). +- Empty-string probe as the ledger over-match guard (2026-08-06 + #332) — `.`, `.+`, `\b`, `[\s\S]` all decline "" and still match + every corpus name; replaced by the sentinel set + (mechanisms.md#SENTINEL-SET-OVER-MATCH-CHECK). +- "Lists every role" checked against all eight fields entries + (2026-08-06 #332) — a seven-role list passed while omitting + _ambiguities, which below baseline 2.0 cannot enter a diff at + all; the check is against V2_FIELDS. +- A seed ledger rule with name_regex and no fields (2026-08-05 + #332) — it matched every CJK-bearing name and would have + classified all 89 diffs on the first pass, exiting 0 having + distinguished nothing. +- An issue for rotating DEFAULT_BASELINE (2026-08-07) — it recurs + every release; it lives in the AGENTS.md release checklist beside + the VERSION bump instead (#333 must land first). + +Excluded (ledger name_regex patterns — one over-matching rule +shadows the whole ledger, since name_regex rules sort first): "", +"(?:)", ".", ".+", "\b", "[\s\S]". Enforced by the sentinel-set +check. + +### 3-0-reevaluations — decisions shaped by the v1 shim + +Promoted 2026-08-15 from session memory (Derek's 2026-07-30 ask; +promotion approved 2026-08-15). Discipline when appending: mark each +entry (A) "would decide differently without the shim" — real 3.0 +work — or (B) "cited 1.4 parity but stands on its own", recorded so +nobody re-litigates it. Append here whenever a design choice cites +1.4 parity or the shim as a load-bearing reason. + +- (A) v1 field vocabulary at the facade boundary: CJK semantics + squeeze into first/last through HumanName while the core speaks + given/family. 3.0 could drop the aliasing entirely. +- (A) Render's v1-inherited string_format defaults, including + space-joined output for interpunct transcriptions (#298: + str(HumanName("威廉·莎士比亚")) == "威廉 莎士比亚"; users + reinstate the dot via custom format). 3.0 render defaults are a + clean slate. +- (A) The differential harness baseline is 1.4-on-PyPI: post-shim, + the migration promise it verifies dissolves; the successor + baseline is presumably last-2.x. Machinery survives; corpus + contracts change. +- (A) empty_attribute_default left untyped (PR #250): cascades into + the v1-shaped public API. Typeable in 3.0. +- (A) A .pyi stub for Policy (wide __init__, narrow attributes) — + deferred from #334, see the differential-ledger Declined entry. +- (B) Pickle-guard layout breaks landing in minors (2.1's + __setstate__ breaks): the guarded-raise design is right + regardless; only the in-a-minor friction is shim-era. +- (B) Positional-read-when-unlicensed as the safe direction + (script_orders fallbacks, #298 dot-suppression granularity): + coincides with 1.4 parity but stands alone — family-first is the + marked case needing affirmative evidence. +- (B) The FAMILY_COMMA doctrine (rule W3): inherited from v1's + lastname-comma but correct on its own terms — an explicit comma + is stronger evidence than script. From 92c887b7958f6d7d362499bb73825c8cfc87a216 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 16:58:49 -0700 Subject: [PATCH 20/40] docs(rules): retarget nameparser/ spec citations to committed docs All 35 gitignored-spec references (spec/conventions/amendment section numbers) now point at mechanisms.md entries, rules, or decisions.md; three new mechanism entries carry the promoted contracts (LOCALE-PACKS-PURE-DATA, FACADE-CONTRACT, CONFIG-SHIM-SNAPSHOT). Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 7 ++++++ docs/design/mechanisms.md | 45 ++++++++++++++++++++++++++++++++++ nameparser/__main__.py | 2 +- nameparser/_config_shim.py | 21 +++++++++------- nameparser/_facade.py | 15 ++++++++---- nameparser/_parser.py | 14 ++++++----- nameparser/_types.py | 2 +- nameparser/config/__init__.py | 17 ++++++------- nameparser/locales/__init__.py | 7 +++--- nameparser/locales/ja.py | 8 +++--- nameparser/locales/ru.py | 8 +++--- nameparser/locales/tr_az.py | 5 ++-- nameparser/locales/zh.py | 3 ++- nameparser/parser.py | 3 ++- 14 files changed, 113 insertions(+), 44 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 75d8cd5..0c2fd0f 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -390,6 +390,13 @@ nobody re-litigates it. Append here whenever a design choice cites the v1-shaped public API. Typeable in 3.0. - (A) A .pyi stub for Policy (wide __init__, narrow attributes) — deferred from #334, see the differential-ledger Declined entry. +- (A) nameparser.config removal scope: the 3.0 schedule says + "nameparser.config in its entirety" while actually enumerating + only the five shim exports; whether the DATA modules keep the + package as their home or move under the core is open, and + Lexicon's public field docs cross-reference + nameparser.config.particles et al. (see the maintainer note in + nameparser/config/__init__.py). - (B) Pickle-guard layout breaks landing in minors (2.1's __setstate__ breaks): the guarded-raise design is right regardless; only the in-a-minor friction is shim-era. diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 170f9e9..6133eb4 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -221,6 +221,51 @@ and every recorded roster. Reach for it when. Tempted to write "remember to keep X in sync with Y." +## LOCALE-PACKS-PURE-DATA — packs configure, they never compute + +Problem shape. Language-specific behavior needs a home that cannot +drift from the core. +Contract statement. A locale pack is pure data — a Policy patch and +Lexicon additions — applied by parser_for; packs contain no code +paths of their own, and each pack's docstring declares its +deviations from the defaults. +How it works. What is policy for every language (an order constant) +is policy, not pack data; packs are lowercase modules exposing +uppercase constants; a pack error is wrapped with the pack's code +(rule D2). Pure data means a pack can be audited by reading it. +Lives in. nameparser/locales/ (packs), nameparser/_parser.py +(parser_for, the one applier). +Reach for it when. A language fix wants an if-statement — make it +vocabulary or policy in a pack instead. + +## FACADE-CONTRACT — HumanName wraps the core, warning-free v1 keeps working + +Problem shape. Where does v1-compatibility behavior live, and what +may it do? +Contract statement. HumanName is a mutable facade over the immutable +core: code that runs warning-free on 1.4 keeps working with +identical results through 2.x, via validating setters, dirty-tracked +re-parses, and pickle round-trips — and the facade never calls the +v1 parsing hooks it still carries. +Lives in. nameparser/_facade.py; v1 import paths preserved by +nameparser/parser.py and nameparser/config/. +Reach for it when. A core change needs a v1-visible behavior — +the facade, not the core, is where parity lives (and cite +decisions.md#3-0-reevaluations when parity is the only reason). + +## CONFIG-SHIM-SNAPSHOT — v1 config mutations reach the core by snapshot + +Problem shape. v1 code mutates CONSTANTS at runtime; the core is +immutable. +Contract statement. nameparser.config re-exports mutable managers +whose contents are converted to immutable core objects through a +dirty-tracked snapshot: mutations mark the snapshot stale, and the +next parse rebuilds it, so v1 mutation semantics survive over an +immutable core. +Lives in. nameparser/_config_shim.py, nameparser/config/. +Reach for it when. Wiring any new v1 config surface — it must go +through the snapshot, never sideways into a Lexicon. + ## Verification shapes How to measure in this codebase without fooling yourself. The diff --git a/nameparser/__main__.py b/nameparser/__main__.py index b569951..5fafab0 100644 --- a/nameparser/__main__.py +++ b/nameparser/__main__.py @@ -1,4 +1,4 @@ -"""Command-line debug helper over the 2.0 API (migration spec §6). +"""Command-line debug helper over the 2.0 API. python -m nameparser "Dr. Juan Q. Xavier de la Vega III" python -m nameparser --json "Doe, John" diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index baec22d..d93f6c4 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -1,5 +1,6 @@ """v1 ``Constants`` compatibility shim over Lexicon/Policy (migration -spec §3). ``nameparser.config`` re-exports these names from the swap +mechanisms.md#CONFIG-SHIM-SNAPSHOT). ``nameparser.config`` +re-exports these names from the swap commit onward; the whole module is deleted in 3.0 with the facade. Layering: facade layer -- may import anything public; here that's @@ -431,7 +432,7 @@ def __setstate__(self, state: dict[str, object]) -> None: #: The named delimiter buckets, translated to the ``Policy`` -#: (open, close) pairs they stand for (spec §3). The first three are +#: (open, close) pairs they stand for. The first three are #: v1's; the rest are the #273 typographic conventions, named so the #: v1 keyed idioms (pop/move/del) work on them like the originals. #: Keep in sync with DEFAULT_NICKNAME_DELIMITERS in _policy.py (pinned @@ -472,8 +473,8 @@ class -- via ``TupleManager.__reduce__``'s ``(type(self), (), state)`` class _DelimiterManager(TupleManager): """v1 ``nickname_delimiters``/``maiden_delimiters`` bucket. In 2.0 - only the named sentinels in ``_DELIMITER_SENTINELS`` exist (spec - §3; the v1 trio plus the #273 typographic pairs) -- assigning any + only the named sentinels in ``_DELIMITER_SENTINELS`` exist (the v1 + trio plus the #273 typographic pairs) -- assigning any other key raises so a caller reaches for a custom-delimiter Policy kwarg instead of a dict entry that silently does nothing. ``pop()``/ ``__setitem__``/``__delitem__`` stay open (inherited) for the @@ -517,7 +518,7 @@ class _RegexesProxy: ``CONSTANTS.regexes.word`` stays informational -- but 2.0 configures parsing behavior through named ``Policy`` flags, not by mutating a regex, so any attribute *or* item assignment raises ``TypeError`` - (spec §3's uniform read-only rule). + (the shim's uniform read-only rule). """ @staticmethod @@ -691,7 +692,8 @@ def _default_vocab() -> dict[str, frozenset[str]]: class _RenderDefaults(NamedTuple): """v1 scalar rendering knobs that have no home on ``Policy`` - (spec §3): ``__str__``/initials formatting and capitalization stay + (mechanisms.md#CONFIG-SHIM-SNAPSHOT): ``__str__``/initials + formatting and capitalization stay per-Constants defaults, layered onto a shared ``Parser`` by the facade (a later task) rather than folded into the cache key.""" @@ -707,7 +709,8 @@ class _RenderDefaults(NamedTuple): @functools.lru_cache(maxsize=64) def _cached_parser(lexicon: Lexicon, policy: Policy) -> Parser: # keyed on hashable value objects: shared across every facade whose - # Constants resolve to the same snapshot (spec §3) + # Constants resolve to the same snapshot + # (mechanisms.md#CONFIG-SHIM-SNAPSHOT) return Parser(lexicon=lexicon, policy=policy) @@ -716,7 +719,7 @@ class Constants: a frozen ``(Lexicon, Policy, _RenderDefaults)`` snapshot via ``_snapshot()``. ``_generation`` increments on every mutation; facades compare it against a cached value to decide whether their - snapshot is stale (dirty-tracking, spec §3 -- the facade itself is + snapshot is stale (dirty-tracking -- the facade itself is a later task). The module-level ``CONSTANTS`` singleton (below) has ``_shared`` @@ -980,7 +983,7 @@ def _snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: """Resolve this v1-shaped, mutable Constants into the frozen - 2.0 value objects it corresponds to (spec §3). A pure read: no + 2.0 value objects it corresponds to. A pure read: no generation bump, no deprecation warning even on the shared singleton -- only direct attribute mutation is on the 3.0 removal path. diff --git a/nameparser/_facade.py b/nameparser/_facade.py index c72ec46..dc486a5 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -1,4 +1,5 @@ -"""The 2.0 ``HumanName`` facade (migration spec §2): a mutable wrapper +"""The 2.0 ``HumanName`` facade (mechanisms.md#FACADE-CONTRACT): a +mutable wrapper over a frozen ParsedName, delegating parsing to the core Parser resolved from the bound Constants shim. Keeps every v1 spelling. Deleted in 3.0. @@ -41,7 +42,8 @@ -#: v1 parsing hooks the facade never calls (spec §2 exception 2 / #280). +#: v1 parsing hooks the facade never calls +#: (mechanisms.md#FACADE-CONTRACT / #280). _V1_HOOKS = ( "pre_process", "post_process", "parse_full_name", "parse_pieces", "parse_nicknames", "join_on_conjunctions", "squash_emoji", @@ -152,7 +154,8 @@ def _warn_overridden_hooks(cls) -> None: DeprecationWarning, stacklevel=3) # -- render defaults ----------------------------------------------------- - # One-line validating setters (spec §2): assigning a non-str (or, for + # One-line validating setters (mechanisms.md#FACADE-CONTRACT): + # assigning a non-str (or, for # the two fields that allow it, non-str-non-None) raises TypeError at # assignment time instead of failing later inside .format(). @@ -218,7 +221,8 @@ def suffix_delimiter(self, value: str | None) -> None: # -- config / parsing --------------------------------------------------- def _resolve(self) -> Parser: - """Dirty-tracked parser resolution (spec §3): rebuild the + """Dirty-tracked parser resolution + (mechanisms.md#CONFIG-SHIM-SNAPSHOT): rebuild the snapshot only when the bound Constants' generation moved.""" gen = self._C._generation if self._snapshot_gen != gen: @@ -685,7 +689,8 @@ def __setstate__(self, state: dict[str, Any]) -> None: self._suffix_delimiter = state.get("suffix_delimiter", defaults.suffix_delimiter) self._full_name = state.get("_full_name", "") - # Components come back exactly as pickled (spec §2): synthetic + # Components come back exactly as pickled + # (mechanisms.md#FACADE-CONTRACT): synthetic # tokens, never a re-parse. Build them per *_list ENTRY rather # than from one joined string -- an entry may hold several words # ("Ph. D.", "Q.C. M.P."), and re-splitting the joined string on diff --git a/nameparser/_parser.py b/nameparser/_parser.py index 7c22049..f69f95a 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -3,8 +3,8 @@ Layering: sits on _types/_lexicon/_policy/_locale/_pipeline; never imports _render or the v1 facade (enforced by tests/v2/test_layering.py). -_default_parser is THE one sanctioned module-level global (conventions -§8): a functools.cache'd frozen Parser over default config. +_default_parser is THE one sanctioned module-level global: a +functools.cache'd frozen Parser over default config. """ from __future__ import annotations @@ -39,7 +39,8 @@ class Parser: consulted only for a token the segmentation stage gates in and the vocabulary DECLINES, so a locale pack's surnames always win where they match; returning None declines in turn and the token stays - whole. Two promises narrow when one is supplied (locales spec §4): + whole. Two promises narrow when one is supplied +(mechanisms.md#LOCALE-PACKS-PURE-DATA): parse-totality gains its one exception -- an exception raised by the segmenter propagates, because a user-supplied callable's own error is a user-code error, not a content error -- and this Parser @@ -59,7 +60,8 @@ class Parser: #: An optional hook supplying outside knowledge of where an unspaced #: token divides -- see the class docstring; None leaves such tokens #: whole. Keyword-only, so the reserved growth stays additive - #: (locales spec §4): positional construction keeps its two-argument + #: (mechanisms.md#LOCALE-PACKS-PURE-DATA): positional construction +#: keeps its two-argument #: shape. segmenter: Segmenter | None = field(default=None, kw_only=True) @@ -124,7 +126,7 @@ def __post_init__(self) -> None: UserWarning, stacklevel=3) def __repr__(self) -> str: - # composes the two bounded component reprs (spec §2 reprs); the + # composes the two bounded component reprs; the # segmenter shows by name, and only when one is set, so the # default Parser's repr is unchanged seg = "" @@ -237,7 +239,7 @@ def parser_for(*locales: Locale, base: Parser | None = None, """Lexicon fragments unioned left-to-right onto base's; policy patches applied left-to-right (later wins; set-valued fields union per the patch metadata). Validation errors raised while applying a - pack are wrapped with that pack's identity (spec §4 amendment) -- + pack are wrapped with that pack's identity (rule D2) -- PolicyPatch validates lazily, so with stacked packs the raw error would otherwise point at nothing. Two packs setting the same SCALAR field is a declared conflict: UserWarning, later wins. diff --git a/nameparser/_types.py b/nameparser/_types.py index 322888c..7661613 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -34,7 +34,7 @@ class Role(StrEnum): ``sorted()`` yields alphabetical order -- iterate ``Role`` itself for the canonical order.""" - # Declaration order IS the canonical field order (conventions §3): + # Declaration order IS the canonical field order: # every listing of the seven fields anywhere derives from this. #: Pre-nominal titles and honorifics ("Dr.", "Sir", "Capt."). diff --git a/nameparser/config/__init__.py b/nameparser/config/__init__.py index 9528f1b..52967ad 100644 --- a/nameparser/config/__init__.py +++ b/nameparser/config/__init__.py @@ -1,4 +1,5 @@ -"""v1 import-path preservation (migration spec §3): the Constants shim +"""v1 import-path preservation (mechanisms.md#CONFIG-SHIM-SNAPSHOT): +the Constants shim lives in nameparser._config_shim. Two unrelated things share this package. The names re-exported below -- @@ -19,14 +20,12 @@ compatibility failure. """ # Maintainer note, deliberately outside the docstring: the docstring -# above no longer says "this package is deleted in 3.0". The migration -# spec's removal schedule (§4, not §3 -- §3 is the shim itself) does -# say it, listing "nameparser.config in its entirety" while its own -# parenthetical enumerates only the five shim exports re-exported -# below. That gap is the whole reason for this note, so it is stated -# here rather than left to a section number: the spec lives under -# docs/superpowers/, which is gitignored and absent for anyone outside -# this checkout. Whether the DATA modules keep this package as their +# above no longer says "this package is deleted in 3.0". The 3.0 +# removal schedule names "nameparser.config in its entirety" while +# actually enumerating only the five shim exports re-exported below; +# that gap is recorded in docs/design/decisions.md#3-0-reevaluations +# rather than left to a section number of an uncommitted document. +# Whether the DATA modules keep this package as their # home in 3.0 or move under the core is an open decision, not something # to settle in a published docstring -- and it now has a consequence, # since Lexicon's public field docs cross-reference diff --git a/nameparser/locales/__init__.py b/nameparser/locales/__init__.py index 1f45ec9..ab60be9 100644 --- a/nameparser/locales/__init__.py +++ b/nameparser/locales/__init__.py @@ -1,5 +1,6 @@ """Locale packs: named (lexicon fragment, PolicyPatch) deltas folded in -by parser_for (locales spec §2). Packs are pure data with no +by parser_for (mechanisms.md#LOCALE-PACKS-PURE-DATA). Packs are +pure data with no privileged capabilities; they dissolve at parser construction. Loaded lazily (PEP 562): importing nameparser.locales never imports a @@ -20,7 +21,7 @@ from nameparser._types import Segmenter #: attribute name -> (module name, module attribute). Codes are the -#: lowercase module names; attribute constants are uppercase (spec §2). +#: lowercase module names; attribute constants are uppercase. _REGISTRY = { "JA": ("nameparser.locales.ja", "JA"), "RU": ("nameparser.locales.ru", "RU"), @@ -74,7 +75,7 @@ def ja_segmenter(*, gbdt: bool = False) -> Segmenter: def get(code: str) -> Locale: """Dynamic lookup by code ('ru'); raises KeyError listing the - available codes (spec §2).""" + available codes.""" attr = code.upper() if isinstance(code, str) else code if not isinstance(code, str) or attr not in _REGISTRY: raise KeyError( diff --git a/nameparser/locales/ja.py b/nameparser/locales/ja.py index d0c3ca9..8d039a3 100644 --- a/nameparser/locales/ja.py +++ b/nameparser/locales/ja.py @@ -15,7 +15,8 @@ why this pack alone is inert: it activates a stage that then has nothing to segment with. * No order, because a kana-bearing Japanese name already reads - family-first by default (amendment 2026-07-29 §1: hiragana identifies + family-first by default (decisions.md#W4, the kana license: + hiragana identifies native Japanese as certainly as hangul identifies Korean), and wholly Han names have read family-first since the 2026-07-27 amendment. Pure KATAKANA is deliberately outside both the order rule and the @@ -43,9 +44,10 @@ ``ja_segmenter(gbdt=True)`` path, which is what loads that file; the default BasicNameDivider reads namedivider's own bundled kanji.csv and never touches it. Its BERT model for katakana division is CC-BY-SA and -is NOT used (katakana division is out of scope, amendment §6). +is NOT used (katakana division is out of scope; decisions.md#W4). -Declared deviations (spec §2 authoring requirement 3): the pack sets +Declared deviations (mechanisms.md#LOCALE-PACKS-PURE-DATA): the +pack sets one union policy field, self-selecting by script, so it can only change names containing characters of the Japanese repertoire -- DEVIATES below declares exactly that, over-declaring within the repertoire (a diff --git a/nameparser/locales/ru.py b/nameparser/locales/ru.py index cea49b9..6b0738f 100644 --- a/nameparser/locales/ru.py +++ b/nameparser/locales/ru.py @@ -1,16 +1,18 @@ -"""The Russian locale pack (locales spec §3): policy-only -- it turns +"""The Russian locale pack (rule O1): policy-only -- it turns on the EAST_SLAVIC patronymic rule. The morphology data (-ovich/-ovna endings, Cyrillic and transliterated) lives inside the rule implementation in nameparser/_pipeline/_post_rules.py, not in the Lexicon (mirrors v1's patronymic_name_order flag design, v1.3.0). Cyrillic titles/conjunctions are default-lexicon vocabulary (#269), -not pack data (spec §2 sorting rule). +not pack data (mechanisms.md#LOCALE-PACKS-PURE-DATA, the sorting +rule). Data sources: the v1.3.0 patronymic rule (PR #154 discussion and the east-slavic test bank); no external lists -- the pack itself carries no vocabulary. -Declared deviations (spec §2 authoring requirement 3): applying this +Declared deviations (mechanisms.md#LOCALE-PACKS-PURE-DATA): +applying this pack changes only NO_COMMA names whose final token carries an East Slavic patronymic ending while the middle token does not -- DEVIATES(name) below is the machine-readable declaration the diff --git a/nameparser/locales/tr_az.py b/nameparser/locales/tr_az.py index 7e2133a..c92fb75 100644 --- a/nameparser/locales/tr_az.py +++ b/nameparser/locales/tr_az.py @@ -1,4 +1,4 @@ -"""The Turkish/Azerbaijani locale pack (locales spec §3): policy-only +"""The Turkish/Azerbaijani locale pack (rule O2): policy-only -- it turns on the TURKIC patronymic rule. The marker data (oglu/qizi/ uulu/kyzy and Cyrillic forms) lives inside the rule implementation in nameparser/_pipeline/_post_rules.py (mirrors v1's flag design). @@ -7,7 +7,8 @@ bank, tests/test_turkic_patronymic_order.py); the pack carries no vocabulary. -Declared deviations (spec §2 authoring requirement 3): applying this +Declared deviations (mechanisms.md#LOCALE-PACKS-PURE-DATA): +applying this pack changes only NO_COMMA names where some token is a standalone Turkic patronymic marker -- DEVIATES(name) below is the machine-readable declaration the non-interference gate checks. diff --git a/nameparser/locales/zh.py b/nameparser/locales/zh.py index 0bc0b53..d8689ae 100644 --- a/nameparser/locales/zh.py +++ b/nameparser/locales/zh.py @@ -22,7 +22,8 @@ is a silent no-op: Lexicon is frozen, so ``.add`` returns a NEW lexicon that no parser is holding.) -Declared deviations (spec §2 authoring requirement 3): the pack adds +Declared deviations (mechanisms.md#LOCALE-PACKS-PURE-DATA): the +pack adds vocabulary and one union policy field, both self-selecting by script, so it can only change names containing Han characters -- DEVIATES below declares exactly that (over-declaring within the script: only diff --git a/nameparser/parser.py b/nameparser/parser.py index aa85bfe..71ed2d7 100644 --- a/nameparser/parser.py +++ b/nameparser/parser.py @@ -1,4 +1,5 @@ -"""v1 import-path preservation (migration spec §3): the 2.0 HumanName +"""v1 import-path preservation (mechanisms.md#FACADE-CONTRACT): the +2.0 HumanName facade lives in nameparser._facade. This module is deleted in 3.0. """ from nameparser._facade import HumanName as HumanName From f4e8a18abe1f745f54acb52ca33627bc3e6577af Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 16:59:37 -0700 Subject: [PATCH 21/40] docs(rules): retarget tests/ and tools/ legacy citations Co-Authored-By: Claude Fable 5 --- tests/test_python_api.py | 2 +- tests/v2/cases.py | 7 ++++--- tests/v2/test_benchmark.py | 2 +- tests/v2/test_cases.py | 2 +- tests/v2/test_config_shim.py | 5 +++-- tests/v2/test_contracts.py | 2 +- tests/v2/test_facade.py | 4 ++-- tests/v2/test_facade_cases.py | 3 ++- tests/v2/test_layering.py | 8 +++++--- tests/v2/test_locales.py | 21 ++++++++++++-------- tests/v2/test_parser.py | 2 +- tests/v2/test_properties.py | 2 +- tools/differential/expected_since_1.4.0.toml | 4 ++-- 13 files changed, 37 insertions(+), 27 deletions(-) diff --git a/tests/test_python_api.py b/tests/test_python_api.py index fb763d9..d1174c7 100644 --- a/tests/test_python_api.py +++ b/tests/test_python_api.py @@ -550,7 +550,7 @@ def test_override_constants(self) -> None: def test_override_regex_raises(self) -> None: # Custom regexes are not supported in 2.0 (deliberate divergence, - # migration spec §3 uniform rule): the constructor kwarg raises the + # shim's uniform read-only rule): the constructor kwarg raises the # same TypeError as attribute assignment, pointing at named Policy # flags. Reads of the built-in patterns stay available. var = TupleManager([("spaces", re.compile(r"\s+")),]) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 6f8ccd6..19e7e8b 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -1,4 +1,5 @@ -"""THE shared behavior case table (core spec §7.2). +"""THE shared behavior case table (rules.md cites it as the pin of +CURRENT behavior). Format is fixed here, in the first pipeline PR, and never per-PR: one Case per input, expected values for exactly the non-empty fields, @@ -787,7 +788,7 @@ def __post_init__(self) -> None: {"given": "Иван", "family": "Петров"}, locale="ru", notes="a comma is an explicit signal that suppresses the " - "rotation (spec §1)"), + "rotation (rule O1)"), Case("tr_az_pack_marker", "Mammadova Aygun Ali kizi", {"given": "Aygun", "middle": "Ali kizi", "family": "Mammadova"}, locale="tr_az", @@ -826,7 +827,7 @@ def __post_init__(self) -> None: Case("unbalanced_quote", 'Jon "Nick Smith', {"given": "Jon", "middle": '"Nick', "family": "Smith"}, ambiguities=("unbalanced-delimiter",), - notes="quote char stays literal (spec §5a)"), + notes="quote char stays literal (rule N2)"), Case("suffix_stays_suffix", "Johnson PhD", {"given": "Johnson", "suffix": "PhD"}, classification="fix(suffix-routing)", diff --git a/tests/v2/test_benchmark.py b/tests/v2/test_benchmark.py index 122a04d..cca7b1e 100644 --- a/tests/v2/test_benchmark.py +++ b/tests/v2/test_benchmark.py @@ -1,4 +1,4 @@ -"""Perf smoke (core spec §7 tail): parse cost stays v1-comparable +"""Perf smoke: parse cost stays v1-comparable (microseconds per name). Deliberately generous bound -- guards against order-of-magnitude regressions, does not gate normal variance. diff --git a/tests/v2/test_cases.py b/tests/v2/test_cases.py index 3495a26..731c63d 100644 --- a/tests/v2/test_cases.py +++ b/tests/v2/test_cases.py @@ -1,4 +1,4 @@ -"""Core runner over the shared case table (spec §7.2). The facade +"""Core runner over the shared case table. The facade runner (migration plan) consumes the same CASES.""" import pytest diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index eca72bb..6bd41ae 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -1,4 +1,5 @@ -"""Shim Constants/SetManager/TupleManager (migration spec §3).""" +"""Shim Constants/SetManager/TupleManager +(mechanisms.md#CONFIG-SHIM-SNAPSHOT).""" import copy import pickle import warnings @@ -150,7 +151,7 @@ def test_delimiter_manager_sentinels_only() -> None: d2["parenthesis"] = moved # the documented bucket-move idiom assert "parenthesis" in d2 with pytest.raises(TypeError, match="quoted_word"): - d2["angle_brackets"] = "custom" # spec §3: custom keys raise + d2["angle_brackets"] = "custom" # custom keys raise def test_delimiter_manager_no_bypass_via_constructor_or_update() -> None: diff --git a/tests/v2/test_contracts.py b/tests/v2/test_contracts.py index 375993f..7699c12 100644 --- a/tests/v2/test_contracts.py +++ b/tests/v2/test_contracts.py @@ -1,4 +1,4 @@ -"""Stable-string contract tests (core spec §7.4): every enum member and +"""Stable-string contract tests: every enum member and stable tag has a canonical triggering input, parametrized by iterating the registries -- a new member without an entry here fails loudly.""" import pytest diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 31a93ac..fc1dbbf 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -1,4 +1,4 @@ -"""The 2.0 HumanName facade (migration spec §2).""" +"""The 2.0 HumanName facade (mechanisms.md#FACADE-CONTRACT).""" import pickle import warnings from pathlib import Path @@ -123,7 +123,7 @@ def test_field_assignment_str_list_none() -> None: assert n.full_name == "John Smith" # no re-parse (v1 parity) -def test_list_attributes_are_snapshots() -> None: # spec §2 exc. 1 +def test_list_attributes_are_snapshots() -> None: n = HumanName("John Quincy Adams Smith") lst = n.middle_list lst.append("HACKED") diff --git a/tests/v2/test_facade_cases.py b/tests/v2/test_facade_cases.py index da14698..31d73f0 100644 --- a/tests/v2/test_facade_cases.py +++ b/tests/v2/test_facade_cases.py @@ -1,4 +1,5 @@ -"""Facade runner (migration spec §5): the shared case table asserted +"""Facade runner (mechanisms.md#FACADE-CONTRACT): the shared case +table asserted through HumanName. Deleted wholesale in 3.0 with the facade.""" import dataclasses diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index f4495e7..f3c35fb 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -30,7 +30,8 @@ ) # a locale pack: pure data over the base types, no pipeline or config -# access (locales spec §2) -- one contract shared by every pack, like +# access (mechanisms.md#LOCALE-PACKS-PURE-DATA) -- one contract +# shared by every pack, like # _PIPELINE_STAGE_ALLOWED above, so tightening it is a one-line edit. # _types joined the three in #272: the ja pack's segmenter factory # constructs a Segmentation, and _types is the bottom of the graph @@ -76,7 +77,8 @@ "_parser.py": ("nameparser._types", "nameparser._lexicon", "nameparser._policy", "nameparser._locale", "nameparser._pipeline"), - # facade layer (migration spec §2/§3): may import anything public + # facade layer (mechanisms.md#FACADE-CONTRACT): may import anything + # public # plus _render (unused yet) and each other. _facade delegates # parsing to the core Parser resolved from the bound Constants shim. # The bare "nameparser.config" import (not just its submodules) is @@ -103,7 +105,7 @@ # v1 import-path preservation: thin re-exports of the facade/shim "parser.py": ("nameparser._facade",), "config/__init__.py": ("nameparser._config_shim",), - # CLI (migration spec §6): imports only the public package, same as + # CLI: imports only the public package, same as # any other consumer -- no access to internal modules. "__main__.py": ("nameparser",), } diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 269064e..96bd52b 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -1,4 +1,5 @@ -"""The locale pack layer (locales spec §2-3): lazy access, the shipped +"""The locale pack layer (mechanisms.md#LOCALE-PACKS-PURE-DATA): +lazy access, the shipped packs, composition, and the non-interference gate.""" import functools import importlib.util @@ -218,7 +219,7 @@ def test_ja_pack_contents() -> None: assert locales.JA.code == "ja" # segmentation activation ONLY: no vocabulary (no list settles a # kanji name) and no order (the kana license already reads - # Japanese family-first by default, amendment 2026-07-29 §1) + # Japanese family-first by default, decisions.md#W4) assert locales.JA.policy.segment_scripts == frozenset( {Script.HAN, Script.HIRAGANA}) assert locales.JA.policy.name_order is UNSET @@ -404,7 +405,8 @@ def test_ja_end_to_end() -> None: @_needs_ja def test_ja_keeps_a_transcribed_name_in_its_source_order() -> None: # pure katakana is excluded from the pack's activation by design - # (amendment §1: katakana is how Japanese writes FOREIGN names), so + # (rule W4's boundary: katakana is how Japanese writes FOREIGN + # names), so # the nakaguro form divides on the dot and keeps its given-first # source order instead of being read family-first n = _PACKED["ja"].parse("マイケル・ジャクソン") @@ -413,7 +415,7 @@ def test_ja_keeps_a_transcribed_name_in_its_source_order() -> None: @_needs_ja def test_ja_composes_with_zh() -> None: - # vocabulary first, segmenter on decline (amendment §2) + # vocabulary first, segmenter on decline (rule W1) p = parser_for(locales.ZH, locales.JA, segmenter=locales.ja_segmenter()) assert p.parse("毛泽东").family == "毛" # zh surname wins assert p.parse("山田太郎").family == "山田" # zh declines, segmenter @@ -730,7 +732,8 @@ def test_parser_for_results_chain_as_bases() -> None: def test_locales_import_is_lazy(monkeypatch: pytest.MonkeyPatch) -> None: # importing the package must not import any pack module; PEP 562 - # loads them on first attribute access (spec §2: "importing + # loads them on first attribute access (the lazy-access contract: + # "importing # nameparser never pays for pack data"). monkeypatch snapshots # sys.modules AND the parent package attribute (the fresh import # below rebinds nameparser.locales), so everything rolls back even @@ -754,7 +757,7 @@ def _assert_non_interference( packed: Parser, deviates: Callable[[str], bool], corpus: Iterable[str], ) -> int: """Return the number of DECLARED deviations seen; fail on any - undeclared one (spec §5.2 = the pack-acceptance rejection rule).""" + undeclared one (the pack-acceptance rejection rule).""" declared = 0 for name in corpus: base = _default_parse(name) @@ -849,7 +852,7 @@ def _default_corpus() -> list[str]: # ja has no marker regexes and no vocabulary either: its rotators cover # the shapes only the pack PLUS its segmenter changes. Every row is # UNSPACED on purpose -- a spaced kana-licensed name (高橋 みなみ) now -# reads family-first by DEFAULT (amendment 2026-07-29 §1), so it would +# reads family-first by DEFAULT (decisions.md#W4), so it would # not deviate and would fail the must-deviate assertion below. _ROTATORS["ja"] = [ "山田太郎", # 2-kanji family + 2-kanji given, the common shape @@ -889,7 +892,9 @@ def test_range_declaring_packs_stay_out_of_marker_classification() -> None: def test_registry_is_the_pack_contract() -> None: - # spec §2 authoring requirement 3, enforced structurally (design + # the declared-deviations authoring requirement + # (mechanisms.md#LOCALE-PACKS-PURE-DATA), enforced structurally + # (design # note 2026-07-18, option C): every registered pack module must # declare its deviation surface, and every pack must feed the gate's # positive side -- a new pack fails HERE until it ships both, rather diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index a2ae0e2..b0e85a9 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -43,7 +43,7 @@ def test_parse_rejects_non_str_with_decode_hint() -> None: def test_degenerate_inputs_are_total() -> None: - # spec §5a table + # the quote-pair defaults table (rule N2's conventions) assert not parse("") assert not parse(" ") assert parse("").original == "" diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index 586d0c4..1c208e9 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -1,4 +1,4 @@ -"""Property layer (core spec §7.3). Hypothesis is a dev dependency only. +"""Property layer. Hypothesis is a dev dependency only. The alphabet is punctuation-heavy on purpose: plain st.text() spreads over all of Unicode, so commas, quotes, and delimiters almost never diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 407c034..dde79d0 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -1,6 +1,6 @@ # Every rule needs `issue`; optional `name_regex` and `fields` narrow # it. An unexplained diff is a release blocker until classified (spec -# S5). Rules are seeded from docs/superpowers/plans/notes-m12-diffs.md +# S5). Rules were seeded from the 2.0 rewrite's working notes # and the `classification="fix(...)"` rows in tests/v2/cases.py; keep # each entry's `name_regex`/`fields` as tight as the diff allows. # File order is not load-bearing: compare.py sorts `name_regex` rules @@ -205,7 +205,7 @@ issue = "fix(suffix-delimiter-rendering) no-space delimiter core token kept whol # Constants.suffix_delimiters); the corpus runs with default policy, so # this rule is expected to match nothing here. Kept for documentation # parity with tests/v2/cases.py's 'suffix_delimiter_no_space_core' row -# (anti-#100, migration plan deviation 5). +# (anti-#100; mechanisms.md#SPANS). dormant = "the no-space delimiter rendering fires only under a non-default Policy, and the corpora run under the default one, so no corpus name can reach this rule. Kept because the behavior family is real and documented in tests/v2/cases.py: the rule is ready the moment a matching string is added" name_regex = "/" fields = ["suffix"] From 50b675ca052f6e8d8650aa8a82c20e80fb5bc318 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 17:00:45 -0700 Subject: [PATCH 22/40] docs(agents): rules-doc maintenance rules; arm legacy citation check Co-Authored-By: Claude Fable 5 --- AGENTS.md | 60 ++++++++++++++++++++++++++++++++++ docs/release_log.rst | 1 + tests/v2/test_doc_citations.py | 10 ++++-- 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 09abd30..3ecc000 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,6 +9,62 @@ Branch naming: `fix/issue-NNN-short-description` or `feat/short-description`. Before opening the PR, if the change alters parser behavior or internals, *read* the Architecture, Extension Patterns, and Gotchas sections of this file against the change — don't grep for it: AGENTS.md paraphrases behavior in its own words, so text made stale by a code change rarely matches the code's phrasing (a doc-staleness sweep driven by grep terms from the diff will miss it every time). The same applies when scoping a doc-review pass or a subagent prompt: include AGENTS.md in the list of docs to check, or it won't be checked. +## Rules documentation (docs/design/) + +Three committed contributor docs carry the parser's normative rules +and their reasons; three tests enforce them. Before proposing a +design or a fix that touches parser behavior, read +docs/design/mechanisms.md and the relevant docs/design/rules.md +sections — the catalog is keyed by problem shape, and the pattern +you are about to invent is often already there. + +- **rules.md** — NORMATIVE rules (intended behavior, domain-topic + sections, executable examples; `deviates:` markers track known + parser gaps). Cite as `rules.md#P1`; code comments quote a + verbatim excerpt in double quotes, checked by + tests/v2/test_doc_citations.py. +- **decisions.md** — the ADR-style record: dated entries, Declined: + (rejected WITH evidence), Excluded: (what must stay out of a + wordlist and why), Open: (issue links, never restated), + 3-0-reevaluations (append whenever a design cites 1.4 parity as + load-bearing). +- **mechanisms.md** — problem-shape catalog with citable contract + statements; stage-attribution claims in comments must cite an + entry verbatim, never restate it in fresh words. + +**Same-PR amendment rule.** Any PR that changes or clarifies parser +behavior — or the boundaries around a documented rule, since a +neighboring change can invalidate a rule's stated limits without +touching its code — amends rules.md in the same diff (the doc diff +is part of the reviewable change, like tests). A change that +resolves or reverses a design question adds a decisions.md entry; a +new reusable pattern adds a mechanisms.md entry; a fixed deviation +removes its `deviates:` marker in the same PR (the examples test +forces this). Issues proposing behavior changes should be drafted in +rule shape — rationale, statement, examples with boundaries, +accepted consequences, open questions, exclusions — so landing an +accepted proposal is a copy, not a rewrite. + +**Counting claims.** A bare count in prose is either an assertion or +a liability, keyed by who observes its staleness: asserted counts (a +test holds the number) fail CI at change time — the useful kind; +dated snapshots ("51 sites at spec time") cannot go stale; standing +present-tense prose counts are the forbidden class — promote to an +assertion, add a date, or state the invariant and let a test count. +After changing how many times something runs, sweep for counts, not +for the thing's name. + +**Release-log claims.** Quantified or universal behavior claims in +release bullets must come from the differential gate's classified +summary or be verified against rules.md examples, never written from +memory. Per-rule ledger toml comments asserting PARSER behavior cite +rule IDs under the excerpt discipline; free prose is for ledger +mechanics only (owned by tools/differential/README.md). + +**Guard tests** SHOULD carry a recorded negative control — the +answer with the guard off, stored as data (the _EXCLUSION_EFFECT +shape; see mechanisms.md's Verification shapes). + ## Commands ```bash @@ -61,6 +117,10 @@ uv run sphinx-build -b html docs dist/docs # "Reads:". That is checkable, so check it rather than reading it: # compare it against grep -oE '\b(policy|lexicon)\.[a-z_]+' on the module # - tests/v2/cases.py notes, which explain why a row lands where it does +# - docs/design/rules.md, decisions.md and mechanisms.md -- READ, don't +# grep: the excerpt and example tests catch citation and example drift, +# but statement and Background prose can still be wrong about behavior +# that changed # - AGENTS.md itself, for stale commands, architecture notes, or gotchas # And check for open Dependabot PRs on uv.lock (namedivider-python) and merge them # first — pyproject floats >=0.4 so fresh installs get the newest namedivider, but diff --git a/docs/release_log.rst b/docs/release_log.rst index f6d3add..b1c4ed0 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -23,6 +23,7 @@ Release Log **Breaking Changes** + - Add ``docs/design/`` contributor documentation: ``rules.md`` (the parser's normative rules with executable examples), ``decisions.md`` (the decision record), and ``mechanisms.md`` (the solution-pattern catalog), enforced by new tests that execute every documented example and verify every code citation; committed docstrings no longer reference gitignored planning documents - Change every vocabulary set in ``nameparser.config`` to a ``frozenset``: ``TITLES``, ``GIVEN_NAME_TITLES``, ``SUFFIX_WORDS``, ``SUFFIX_ACRONYMS``, ``SUFFIX_ACRONYMS_AMBIGUOUS``, ``GLUED_HONORIFICS``, ``PARTICLES``, ``NON_GIVEN_NAME_PARTICLES``, ``BOUND_GIVEN_NAMES``, ``CONJUNCTIONS`` and ``MAIDEN_MARKERS`` (``KOREAN_SURNAMES`` already was one). Editing one in place -- ``TITLES.add("dean")``, the old way of changing a global default -- now raises ``AttributeError: 'frozenset' object has no attribute 'add'`` at the line that writes it. It was never a reliable way to change a default: whether an edit reached a given parse depended on which config objects had already been built, so one program could hold two disagreeing defaults with nothing to say so. To change the defaults for ``HumanName``, build a private ``Constants`` and pass it (``c = Constants(); c.titles.add("dean"); HumanName(name, constants=c)``); mutating the shared ``CONSTANTS`` still works, but warns and goes away in 3.0. For the 2.0 API, build a lexicon and pass it to a parser (``Parser(lexicon=Lexicon.default().add(titles={"dean"}))``). Neither is affected by this change. ``CAPITALIZATION_EXCEPTIONS`` is a mapping, not a set, and is unchanged. See :doc:`migrate` and :doc:`customize` (#293) **Behavior Changes** diff --git a/tests/v2/test_doc_citations.py b/tests/v2/test_doc_citations.py index 55c40fd..63fd827 100644 --- a/tests/v2/test_doc_citations.py +++ b/tests/v2/test_doc_citations.py @@ -4,8 +4,8 @@ whitespace-normalized verbatim excerpts of their statements; ``implemented:`` lists match the set of modules actually citing the rule; ``interacts:`` IDs exist (existence only -- the field is -advisory). The legacy-pattern check stays OFF until the final rewrite -pass arms it. +advisory). The legacy-pattern check (armed) keeps gitignored-spec +citation forms out of the committed tree. """ from __future__ import annotations @@ -17,7 +17,7 @@ REPO = Path(__file__).resolve().parents[2] MECH_DOC = REPO / "docs" / "design" / "mechanisms.md" SWEEP_DIRS = ("nameparser", "tests", "tools") -ENFORCE_NO_LEGACY = False # armed by the final rewrite pass +ENFORCE_NO_LEGACY = True # armed 2026-08-15, the rewrite complete _LEGACY = ("§", "superpowers", "plan deviation") _CITE_RE = re.compile( @@ -115,11 +115,15 @@ def test_interacts_ids_exist() -> None: def test_no_legacy_citations() -> None: if not ENFORCE_NO_LEGACY: return # armed by the final rewrite pass + self_files = {Path(__file__).name, "test_doc_spellings.py", + "rules_doc.py"} problems = [] for d in SWEEP_DIRS: for path in sorted((REPO / d).rglob("*")): if path.suffix not in (".py", ".toml"): continue + if path.name in self_files: + continue text = path.read_text(encoding="utf-8", errors="ignore") for pat in _LEGACY: if pat in text: From 7749e24549bc45e86db30e8abc9d22f6c5d35ba0 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 17:06:56 -0700 Subject: [PATCH 23/40] docs(rules): repair the interrupted C1/C2 comment in segment (final review) Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_segment.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nameparser/_pipeline/_segment.py b/nameparser/_pipeline/_segment.py index 1690242..a82c7fe 100644 --- a/nameparser/_pipeline/_segment.py +++ b/nameparser/_pipeline/_segment.py @@ -59,16 +59,16 @@ def suffixy(seg: tuple[int, ...]) -> bool: return is_wholly_suffix([state.tokens[i].text for i in seg], state.lexicon, state.policy) - # v1 parity: only parts[1] decides the suffix-comma structure - # (parser.py:1318); parts[2:] are consumed as suffixes - # unconditionally either way, so a non-suffix tail segment gets the # rules.md#C1: "the name reads as trailing suffixes when the part # after the first comma is entirely suffix words and more than one # word precedes the comma; otherwise it reads as the listing form" - # (history: decisions.md#C1) + # (v1 parity: only parts[1] decides, parser.py:1318; history: + # decisions.md#C1) # rules.md#C2: "a non-empty extra part that is not entirely suffix # words is flagged as a structural ambiguity rather than rejected" - # -- COMMA_STRUCTURE flag, not a structure veto + # -- parts[2:] are consumed as suffixes unconditionally either + # way, so a non-suffix tail segment gets the COMMA_STRUCTURE + # flag, not a structure veto structure = (Structure.SUFFIX_COMMA if suffixy(groups[1]) and len(groups[0]) > 1 else Structure.FAMILY_COMMA) From 5b236f0242a86d5808ce2c47541648d722811f6a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 18:15:38 -0700 Subject: [PATCH 24/40] docs(rules): comment-analyzer fixes -- orphaned references, citation seams, stale later-task notes Six branch-introduced defects (orphaned totality-exception reference, legacy rule-1b pointer in particles.py, garbled shim citation seam, two mis-targeted LOCALE-PACKS anchors with broken wrapping, wrong RECORDED-ROSTERS home) plus the improvement items, and the five pre-2.0.0 'a later task' facade notes the analyzer surfaced. Assign's stage header now admits its is_suffix_lenient consult (pre-existing inaccuracy). AST re-verified identical. Co-Authored-By: Claude Fable 5 --- docs/design/mechanisms.md | 3 ++- nameparser/_config_shim.py | 19 ++++++++++--------- nameparser/_parser.py | 10 ++++------ nameparser/_pipeline/_assign.py | 7 ++++--- nameparser/_pipeline/_classify.py | 3 ++- nameparser/_pipeline/_post_rules.py | 2 +- nameparser/_pipeline/_script_segment.py | 5 +++-- nameparser/config/particles.py | 3 ++- nameparser/locales/ja.py | 3 ++- 9 files changed, 30 insertions(+), 25 deletions(-) diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 6133eb4..d19546d 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -180,7 +180,8 @@ Contract statement. Store the measured answer as literal data (a roster) and compare against it; never re-derive the expectation from the same inputs the check reads, because a derivation from the same data always agrees with itself. -Lives in. tools/differential/compare.py (_CORPUS_CLAIMS and kin); +Lives in. tests/v2/test_ledger_guards.py (_CORPUS_CLAIMS), +tools/differential/compare.py (_CORPUS_FLOORS), tests/v2/test_facade_cases.py (_CORE_ONLY_IDS). Reach for it when. Writing a check whose expected value is computed by the code under test, or a comment that enumerates ids/counts — diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index d93f6c4..1bebd65 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -1,5 +1,5 @@ -"""v1 ``Constants`` compatibility shim over Lexicon/Policy (migration -mechanisms.md#CONFIG-SHIM-SNAPSHOT). ``nameparser.config`` +"""v1 ``Constants`` compatibility shim over Lexicon/Policy +(mechanisms.md#CONFIG-SHIM-SNAPSHOT). ``nameparser.config`` re-exports these names from the swap commit onward; the whole module is deleted in 3.0 with the facade. @@ -100,7 +100,7 @@ def _normalize_iterable_of_strings( class SetManager: """v1 ``SetManager`` surface over a plain set of ``lc()``-normalized strings. Mutations call ``_on_change`` (the owning Constants' - generation bump, wired by a later task). ``__call__`` and the + generation bump, wired by the facade). ``__call__`` and the missing-member-tolerant ``remove()`` are gone per the #243 schedule (warned 1.3.0, removed 2.0): ``remove()`` of a missing member raises ``KeyError``, matching ``set.remove``. @@ -317,7 +317,7 @@ class TupleManager(dict[str, object]): ``AttributeError`` naming the key (#256, warned 1.4, enforced 2.0 -- the v1 ``DeprecationWarning`` is gone, this shim only speaks 2.0). Mutations call ``_on_change`` (the owning Constants' generation - bump, wired by a later task). + bump, wired by the facade). """ _on_change: Callable[[], None] | None @@ -695,7 +695,8 @@ class _RenderDefaults(NamedTuple): (mechanisms.md#CONFIG-SHIM-SNAPSHOT): ``__str__``/initials formatting and capitalization stay per-Constants defaults, layered onto a shared ``Parser`` by the - facade (a later task) rather than folded into the cache key.""" + facade (nameparser/_facade.py) rather than folded into the cache + key.""" string_format: str | None initials_format: str @@ -719,8 +720,8 @@ class Constants: a frozen ``(Lexicon, Policy, _RenderDefaults)`` snapshot via ``_snapshot()``. ``_generation`` increments on every mutation; facades compare it against a cached value to decide whether their - snapshot is stale (dirty-tracking -- the facade itself is - a later task). + snapshot is stale (dirty-tracking -- the facade side lives in + nameparser/_facade.py). The module-level ``CONSTANTS`` singleton (below) has ``_shared`` flipped to ``True``: any mutation reached through it emits @@ -1098,8 +1099,8 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: _SENTINEL_PAIRS[k] for k in self.maiden_delimiters if k not in self.nickname_delimiters), # suffix_delimiter is a _RenderDefaults-only field here; the - # facade layers it onto extra_suffix_delimiters per instance - # (a later task) -- _snapshot() itself stays pure translation + # facade layers it onto extra_suffix_delimiters per + # instance -- _snapshot() itself stays pure translation ) defaults = _RenderDefaults( self.string_format, self.initials_format, self.initials_delimiter, diff --git a/nameparser/_parser.py b/nameparser/_parser.py index f69f95a..7d2ea87 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -39,8 +39,8 @@ class Parser: consulted only for a token the segmentation stage gates in and the vocabulary DECLINES, so a locale pack's surnames always win where they match; returning None declines in turn and the token stays - whole. Two promises narrow when one is supplied -(mechanisms.md#LOCALE-PACKS-PURE-DATA): + whole. Two promises narrow when one is supplied (the first is + rules.md#A1's Accepted clause): parse-totality gains its one exception -- an exception raised by the segmenter propagates, because a user-supplied callable's own error is a user-code error, not a content error -- and this Parser @@ -59,10 +59,8 @@ class Parser: policy: Policy = None # type: ignore[assignment] # None -> Policy() #: An optional hook supplying outside knowledge of where an unspaced #: token divides -- see the class docstring; None leaves such tokens - #: whole. Keyword-only, so the reserved growth stays additive - #: (mechanisms.md#LOCALE-PACKS-PURE-DATA): positional construction -#: keeps its two-argument - #: shape. + #: whole. Keyword-only, so the reserved growth stays additive: + #: positional construction keeps its two-argument shape. segmenter: Segmenter | None = field(default=None, kw_only=True) # in the class body so @dataclass(slots=True) keeps them diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index e68dc7e..1c4c2bc 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -2,14 +2,15 @@ Consumes: pieces + piece_tags (grouped), segments, structure, tokens. Produces: tokens with roles set on every main-stream token. -Reads: Policy.name_order (#270) and Policy.script_orders (#271, which +Reads: Policy.name_order (#270), is_suffix_lenient on the trailing +piece of a two-part comma name, and Policy.script_orders (#271, which overrides it when every name piece is written wholly in one script, or in the Han/Hiragana/Katakana repertoire the #272 kana license shares across pieces); token/piece tags; Lexicon only through tags already applied by classify (plus the leading-title period rule). -Implements rules N3, O4 and W4 of docs/design/rules.md (plus H2 -above), cited at their code below. Ports v1's assignment loops. +Implements rules H2, N3, O4 and W4 of docs/design/rules.md, each +cited at its code below. Ports v1's assignment loops. NO_COMMA (per name_order): leading title pieces chain while no given-position name has been seen (a title needs a following piece, unless the whole name is one title); diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py index fd3b2cf..11e45d6 100644 --- a/nameparser/_pipeline/_classify.py +++ b/nameparser/_pipeline/_classify.py @@ -14,7 +14,8 @@ suffix vocabulary, or an ambiguous acronym written with periods -- at the TAG level 'M.A.' gets "vocab:suffix" while 'Ma' gets only "vocab:suffix-ambiguous"; what assign then does with a trailing -ambiguous tag is rule S2's Accepted consequence. +ambiguous tag is the rest of rule S2's statement (the +words-to-spare guard) and its Accepted consequences. The initial veto is assign's job, not classify's: 'V' carries both "vocab:suffix" and "initial". """ diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index 3577f6b..cca1890 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -9,7 +9,7 @@ Lexicon.given_name_titles. Implements rules H1, P1, O1, O2 and O3 of docs/design/rules.md; each -is cited at its code below, and the history lives in +is cited at its code below, and P1/O1/O2's history lives in docs/design/decisions.md. """ from __future__ import annotations diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 00c6dc4..2ceeb1c 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -574,8 +574,9 @@ def _split_surname_site(state: ParseState) -> ParseState: and _PEELED_TAG not in state.tokens[j].tags for j in state.segments[0]): return state - # No try/except around the call: the module docstring's totality - # exception. The two checks below are that same doctrine, curated, + # No try/except around the call: rules.md#A1's Accepted clause + # ("a user-supplied segmenter's own error propagates"). The two + # checks below are that same doctrine, curated, # and they are where the line this module draws is easiest to state: # a PROTOCOL VIOLATION BY THE SEGMENTER AUTHOR RAISES, while an # ADAPTER'S DEFENSE AGAINST ITS LIBRARY DECLINES. Both checks here diff --git a/nameparser/config/particles.py b/nameparser/config/particles.py index 806df8f..a79bc86 100644 --- a/nameparser/config/particles.py +++ b/nameparser/config/particles.py @@ -14,7 +14,8 @@ #: already chained onto the word behind it is part of that piece rather #: than standing alone. #: Opening the name is only the commonest shape. The rule enforcing it -#: (``post_rules`` rule 1b) reaches a member standing alone as a piece in +#: (rules.md#P1; the pre-2.2 docstrings called it rule 1b) reaches a +#: member standing alone as a piece in #: the given position too, folding it into the family beside it, so that #: neither shape leaves a given name behind -- as long as there is another #: name token to fold into. A bare "de" stays as it is. Where a chain is diff --git a/nameparser/locales/ja.py b/nameparser/locales/ja.py index 8d039a3..dfa0a59 100644 --- a/nameparser/locales/ja.py +++ b/nameparser/locales/ja.py @@ -44,7 +44,8 @@ ``ja_segmenter(gbdt=True)`` path, which is what loads that file; the default BasicNameDivider reads namedivider's own bundled kanji.csv and never touches it. Its BERT model for katakana division is CC-BY-SA and -is NOT used (katakana division is out of scope; decisions.md#W4). +is NOT used (katakana division is out of scope; decisions.md#W1's +kana-gating bullet). Declared deviations (mechanisms.md#LOCALE-PACKS-PURE-DATA): the pack sets From 055c3ff5f71c7dce0612c54863bff3019be531db Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 18:23:02 -0700 Subject: [PATCH 25/40] docs(rules): topic-review amendments from the 2.1-arc session (primary source) Attribution: decision dates restored where entries carried release dates (W1/W4/T3); D1 gains the actual decision (warn-not-raise, the filterable-inertness rationale, the conditional ja hint); W2's #312 entry regains its provenance (gates inherited, not designed). New: the codepoint-scope decision and half-flanked-interpunct Declined under T3, the B7 sanctioned-extra Declined-with-measurement, SELF-EXPIRING-GUARD and the extra-environment-split field note in Verification shapes. STATE-OFFSET-CHANNELS contract admits presence-only consumers (both interpunct readers, measured). New verified examples: W1's segmentation ambiguity, W2 peeling across an interpunct-divided name (the review's conjecture, confirmed live), W4's nakaguro/interpunct order contrast. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 46 ++++++++++++++++++++++++++++++++++----- docs/design/mechanisms.md | 20 ++++++++++++++++- docs/design/rules.md | 5 ++++- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 0c2fd0f..c398c4e 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -113,12 +113,19 @@ which particles count as never-given. family-first in native script; wholly-katakana names are predominantly transcriptions and keep the declared order. Latin transliterations are never touched. -- 2026-08-07 #272 — the kana license: Han∪kana with at least one +- 2026-07-29 #272 — the kana license: Han∪kana with at least one kana cannot be Chinese and is not a transcription, so 高橋みなみ reads family-first though it is written in two scripts. ### D1 — the segmenterless-activation warning +- 2026-08 #337 — WARN rather than raise, deliberately: registering + the JA pack without the extra must stay safe (the inert + registration is itself a pinned property), and a warning is + filterable by the caller who wants exactly that inertness. The + Japanese install hint is conditional — it appears only when a + Japanese script is among the dead scripts; a hangul-only gap gets + the generic remedies. - 2026-08 #337 — the warning exists because parser_for(locales.JA) without segmenter= used to build a parser that silently behaved like a working one minus the feature; it re-emits from the @@ -150,12 +157,14 @@ which particles count as never-given. ### W1 — unspaced CJK division -- 2026-08-07 #271 (2.1.0) — Korean division ships as a default: the +- 2026-07-27 #271 (decision; shipped in 2.1.0 via PR #294) — Korean + division ships as a default: the census surname list is closed, hangul is self-selecting (a hangul entry can only match hangul text), and being unsplit is recoverable while a wrong split is not — which is also why an unrecognized name stays whole. -- 2026-08-07 #272/amendment 2026-07-29 — Han division is opt-in per +- 2026-07-29 #272 (the ja amendment; shipped in 2.1.0 via PR #297) — + Han division is opt-in per language pack because Han text does not identify its language (高橋一郎 under a Chinese list divides wrongly); a pluggable segmenter takes what the vocabulary declines, so pack + segmenter @@ -203,7 +212,10 @@ which particles count as never-given. over the peel. - 2026-08 #312 — the peel crosses the family comma and the 间隔号: both answer where a name DIVIDES into surname and given, a - question the peel never asks. + question the peel never asks. Provenance matters here: neither + gate was argued for the peel specifically — both came with the + placement (#312's own framing) — so the crossing was a repair of + inherited gates, not a designed-in property. - 2026-08 #319 — a wholly suffix-shaped second run is declined as a peel site (the "田中さん, V." shape), but only when the name's own run offers a site, since a glued honorific is itself part of what @@ -263,7 +275,24 @@ Excluded (MAIDEN_MARKERS, per nameparser/config/maiden_markers.py): ### T3 — the interpunct's flank guard -- 2026-08 #298 — U+00B7 divides only between classified-script +- 2026-07-30 #298 — codepoint scope, chosen over a blanket + "dot = transcription" rule that would have flipped 高橋・一郎, a + correct and deliberately pinned #272 reading: U+00B7 records the + transcription fact and suppresses division; U+30FB/U+FF65 stay + pure separators whose pieces the script license reads (rule W4's + Accepted pair demonstrates both). Cross-convention input reads by + the codepoint it was actually typed with — a chosen limitation, + recorded in #298's comments. + +Declined: + +- Ambiguity emission on a half-flanked interpunct ("王·Smith") — + proposed in the 2.1 PR review, declined because the dot decides + silently under the no-emission decision; the + undivided-dot-stays-in-the-word behavior went to docs instead. + + +- 2026-07-30 #298 — U+00B7 divides only between classified-script characters because it is also the Catalan punt volat, interior to legitimate names (Gal·la). The nakaguro (T2) needs no such guard: its codepoints are CJK-only and appear in no other script's @@ -356,6 +385,13 @@ Declined: #332) — it matched every CJK-bearing name and would have classified all 89 diffs on the first pass, exiting 0 having distinguished nothing. +- The B7 sanctioned-extra span (pre-#332 arc) — added, then removed + as a pure loss when review measured that every B7-divided name + already matches through its guaranteed classified flanks, so the + span's only effect was absorbing punt-volat Latin regressions + ("Gal·la Marcet" probes on both sides, recorded in PR #305's + history). The cleanest "sanctioned extras must be earned" + precedent the ledger has. - An issue for rotating DEFAULT_BASELINE (2026-08-07) — it recurs every release; it lives in the AGENTS.md release checklist beside the VERSION bump instead (#333 must land first). diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index d19546d..4a7e702 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -117,7 +117,8 @@ Problem shape. A fact known during tokenization matters to a much later stage. Contract statement. A pre-token fact is recorded as offsets on the ParseState (comma_offsets, interpunct_offsets) and consulted later -by position, rather than re-derived from text. +by position — or by presence alone where the fact is name-level, as +both interpunct consumers do — rather than re-derived from text. How it works. The offsets survive every intermediate stage untouched; #298's transcription marker rides this channel from tokenize to order resolution (rules T3/W4). @@ -320,6 +321,18 @@ Exemplar: tests/v2/pipeline/test_vocab.py's per-script initials check; reused for _CORPUS_FLOORS in tools/differential/compare.py. Known gap it exposes: DEFAULT_SCRIPT_ORDERS has no such guard. +### SELF-EXPIRING-GUARD — a decline keyed to a measured defect + +Contract statement. A workaround keyed to a third-party library's +measured defect carries a canary test that pins the defect itself, +so the workaround cannot outlive its reason: when the library fixes +it, the canary fails and the decline gets revisited rather than +fossilizing. +Exemplar: tests/v2/test_locales.py's namedivider shime canary pins +0.4.x cutting 〆木太郎 at offset 1 with confidence 1.0, guarding the +adapter's 〆 decline (#303 arc); its comment says to revisit the +decline, not delete the test. + ### Field notes — the traps themselves - Assert which tree you imported, on BOTH sides of a comparison. @@ -340,3 +353,8 @@ Known gap it exposes: DEFAULT_SCRIPT_ORDERS has no such guard. passed. - Purge __pycache__ between same-length source mutations; stale bytecode makes a changed file measure as unchanged. +- Mind the optional-extra environment split: a local venv's + incidental namedivider makes `if available` branches run PRESENT + locally and ABSENT in CI, so a locally-green suite proves nothing + about the no-extra path (this broke #337's first landing). Run + the decisive check in both states or gate the example. diff --git a/docs/design/rules.md b/docs/design/rules.md index 1fc886a..f6e1cbc 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -441,6 +441,7 @@ W2. Rationale: some East Asian honorifics glue directly onto the end "田中さん" → suffix="さん" "김, 민준씨" → suffix="씨" "田中さん, V." → suffix="さん" + "马丁·路德·金씨" → suffix="씨" "김지양" → suffix="" · boundary "王君" → family="王君" · boundary history: decisions.md#W2 · implemented: nameparser/_pipeline/_script_segment.py @@ -475,8 +476,10 @@ W4. Rationale: Chinese, Japanese and Korean all write the family "マイケル ジャクソン" → given="マイケル" · boundary Accepted: a name the interpunct divides keeps its source order — the divider itself marks a transcription (T3) — so the override - stands down there. + stands down there; the katakana middle dot (T2) carries no such + signal, so a name it divides still reads by the script license. "毛·泽东" → given="毛" + "威廉・莎士比亚" → family="威廉" history: decisions.md#W4 · interacts: T3 · implemented: nameparser/_pipeline/_assign.py ## Tokens, initials & punctuation (T) From 0460b877d7186792a723c7dea9bcb63f99363c83 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 18:28:24 -0700 Subject: [PATCH 26/40] docs(rules): topic-review amendments from the harness/maiden session (primary source) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 양/군 Excluded entry had attached the spaced-safety argument to the glued exclusion — corrected from the vetting block itself, with 김지군 resolving the harvest's flagged uncertainty properly; 氏 gets its real reason (王氏 historical form); 殿 regains the harm clause. Four Declined dates corrected to git author dates (2026-08-05); the _check_tree asymmetry diagnosis restored; D1 gains the autodoc boundary fact. New: the Vietnamese middle_as_family Declined-with-measurement, the fullwidth-colon M2 Accepted consequence (#317 linked), the Japanese spaced-forms asymmetry in W Background, Vietnamese facts in O Background, and the CANONICAL-VOCABULARY-AT-THE-BOUNDARY mechanism. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 42 ++++++++++++++++++++++++++++++--------- docs/design/mechanisms.md | 19 ++++++++++++++++++ docs/design/rules.md | 22 ++++++++++++++++---- 3 files changed, 70 insertions(+), 13 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index c398c4e..ee33826 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -77,6 +77,11 @@ which particles count as never-given. in running text. The marker itself is dropped as structural, like a delimiter character. +Open (M2): +[#317](https://github.com/derek73/python-nameparser/issues/317) +the fullwidth-colon marker (旧姓：佐藤 arrives as one word; the +head-peel question). + ### N3 — the lone-word nickname rule - 2026-07 (v2 core, PR #288; recorded plan deviation #2 of the core @@ -104,6 +109,17 @@ which particles count as never-given. given="Khai", middle="Thị Minh", family="Nguyễn". This is why three order constants exist rather than two. +Declined: + +- middle_as_family as the way to suppress the middle slot for + Vietnamese (2026-08-07 #146) — measured: it merges the middles + into the family, giving family="Thị Minh Nguyễn" for "Nguyễn Thị + Minh Khai" under family-first-given-last — a plausible-looking + wrong answer. There is no policy field that suppresses middle, + and name_order rejects a two-role tuple ("name_order must be one + of the exported orders"); given_names (given + middle) is the + view that stays correct wherever the internal boundary falls. + ### W4 — script-scoped order - 2026-07-27 (script-scoped order amendment) — the family-first @@ -136,7 +152,11 @@ which particles count as never-given. bare tuple literal for segment_scripts — an arg-type error under mypy in a py.typed package — and became frozenset(), pinned by a test asserting the offered spelling. The known-bad spelling is in the - denylist test. + denylist test. The structural fact behind the recurrence: autodoc + renders docstrings into the API reference, so docs/*.rst is NOT + the boundary of "the docs" — a guide and the reference can teach + different spellings on the same rendered page, invisibly to any + .rst-only sweep. ### D2 — construction raises, parse never does @@ -225,11 +245,15 @@ Excluded (Lexicon.honorific_tails — a glued tail peels only if it could never end a name; per-entry reasons live in nameparser/config/suffixes.py's vetting block): -- 양, 군 — 양 is also a top-tier surname (Yang), and 김지양 is a - given name; the surname-leads argument covers 군 the same way. -- 氏 — recognized spaced only. +- 양, 군 — 김지양 and 김지군 are given names ending in these + syllables, and 양 is a top-tier surname (Yang) besides. (The + surname-LEADS argument is a different job: it is why both are + safe in the SPACED set — a leading surname never meets the + trailing-only suffix gate, so 양 미선 keeps family 양.) +- 氏 — 王氏 is a historical name form ("the Wang woman"). - 博士 — 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka. -- 殿 — Japanese surnames end in it (鵜殿, 真殿). +- 殿 — Japanese surnames end in it (鵜殿, 真殿, four-figure + populations); peeling it would cut a real family name in two. - 君 — 王君 is a complete Chinese name; its kana spelling くん does peel. @@ -357,7 +381,7 @@ that session's, spot-checked at landing. Declined: -- Policy annotations widened to input unions (2026-08-06 #334) — +- Policy annotations widened to input unions (2026-08-05 #334) — five documented spellings fail mypy and every one has a type-clean equivalent; widening would make every READER see a union, and reading is the commoner operation. A .pyi stub typing @@ -369,16 +393,16 @@ Declined: behavior for baselines that cannot construct newer policy fields; tests/v2/cases.py already covers opt-in paths per row, so the ceiling is documented instead. -- `_check_tree` as is_relative_to(REPO_ROOT) (2026-08-06 #332) — +- `_check_tree` as is_relative_to(REPO_ROOT) (2026-08-05 #332) — accepts .venv/, build/ and dist/ copies inside the repo; the invariant is "is the source package", so the predicate is is_relative_to(REPO_ROOT / "nameparser"). -- Empty-string probe as the ledger over-match guard (2026-08-06 +- Empty-string probe as the ledger over-match guard (2026-08-05 #332) — `.`, `.+`, `\b`, `[\s\S]` all decline "" and still match every corpus name; replaced by the sentinel set (mechanisms.md#SENTINEL-SET-OVER-MATCH-CHECK). - "Lists every role" checked against all eight fields entries - (2026-08-06 #332) — a seven-role list passed while omitting + (2026-08-05 #332) — a seven-role list passed while omitting _ambiguities, which below baseline 2.0 cannot enter a diff at all; the check is against V2_FIELDS. - A seed ledger rule with name_regex and no fields (2026-08-05 diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 4a7e702..f1df156 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -206,6 +206,25 @@ Reach for it when. A ledger rule's behavior seems to depend on where it sits in the file — it doesn't, and if moving it changes anything, the fields are wrong. +## CANONICAL-VOCABULARY-AT-THE-BOUNDARY — one vocabulary at the comparison + +Problem shape. Two surfaces name the same concept differently +(first/last vs given/family), and a matcher needs to compare across +them. +Contract statement. Canonicalize at the point of comparison: every +compared surface's output is converted to one vocabulary before +matching, and the canonical choice is the one the codebase already +derives everywhere else. +How it works. Teaching the matcher both vocabularies doubles every +rule silently; canonicalizing means existing rules keep matching +when a second surface joins. The ledger canonicalizes v1 field +names to Role's before matching. +Lives in. tools/differential/compare.py (_canonical_field, +_V1_TO_ROLE). +Reach for it when. A second surface joins an existing matcher — +without this, every existing rule silently stops matching the new +surface, which looks like added coverage. + ## MAKE-WRONG-STATES-UNREPRESENTABLE — the house meta-pattern Problem shape. A convention keeps being violated no matter how diff --git a/docs/design/rules.md b/docs/design/rules.md index f6e1cbc..6ef2ea9 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -251,9 +251,15 @@ N3. Rationale: a person set down as a nickname plus one name word is ## Maiden names (M) Background: a maiden name is written beside the current name, set -off by a marker word (née, geb., 旧姓) or by enclosure. Which -enclosures mean "maiden" rather than "nickname" is a caller -convention, so the maiden reading of a delimiter pair is opt-in. +off by a marker word or by enclosure. Markers are attested across +French née/né, German geb./geborene, Dutch geboren, Czech/Slovak +roz./rozená, Scandinavian født/fødd/född, Russian урожд. (both ё +and е spellings), and Japanese 旧姓 — both grammatical genders +where attested. Japanese more often writes the marker with a +fullwidth colon (旧姓：佐藤), which is no separator, so marker and +name arrive as a single word. Which enclosures mean "maiden" rather +than "nickname" is a caller convention, so the maiden reading of a +delimiter pair is opt-in. M1. Rationale: an enclosure the caller has declared to mean maiden holds the former family name; a recognized marker word inside it @@ -278,6 +284,10 @@ M2. Rationale: a maiden marker announces that what follows it is the "Jane Smith née Jones PhD" → suffix="PhD" "Jones née" → family="née" · boundary "née Jones" → family="Jones" · boundary + Accepted: the fullwidth-colon spelling arrives as one word, so + the marker inside it goes unrecognized; #317 tracks whether it + should peel. + "山田 花子 旧姓：佐藤" → maiden="" Accepted: a marker straight after a comma is post-comma given text, not a marker; and a particle chain swallows a marker in its path, the join (P2) running first. @@ -327,7 +337,11 @@ C2. Rationale: text beyond the recognized comma parts should be Background: written name order varies by convention: given-first (the library's default reading), family-first, and family-first with -the given name last (Vietnamese). The order is declared by the +the given name last (Vietnamese, where the person is called by the +last element, given names are frequently two syllables — the +given_names view stays correct wherever the internal boundary +falls — and quốc ngữ is Latin script, so no native-script signal +exists at all). The order is declared by the caller or a locale pack, never detected — but a few conventions leave a recognizable trace in the name itself. Patronymics are one: East Slavic names carry a father's-name derivative with distinctive From f94ecac02f6ce791042ef0142ae8ae92fab2c17e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 18:34:47 -0700 Subject: [PATCH 27/40] docs(rules): topic-review amendments from the #329 session (primary source) rules.md#M1's statement was FALSIFIED as written: the marker drop is conditional on clause size (a one-word clause keeps its word -- Nee is an attested surname), a qualifier decisions.md had and rules.md lost in transit. Statement corrected, the real guard pinned as the boundary example ('Jane Smith (Nee)' maiden 'Nee'), clause independence added as statement + example, and the citation excerpt updated in lockstep. decisions.md#M1 gains the three separately argued decisions, the M1/M2 disjointness guarantee with its 7,775- record verification, and the neighbour-scoping Declined with its measurement. Verification shapes: the honest-limit preamble now credits mutation testing with part of the wrong-predicate class (the #329 survivors), and the skip-reachability trap joins the field notes. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 31 +++++++++++++++++++++++++++---- docs/design/mechanisms.md | 17 +++++++++++++---- docs/design/rules.md | 13 +++++++++---- nameparser/_pipeline/_group.py | 7 ++++--- 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index ee33826..0cce855 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -331,10 +331,33 @@ Declined: listed for maiden is dropped from the effective nickname set (maiden wins). The v1 facade restores v1's nickname-wins reading by pre-subtracting on its side. -- 2026-08 #329 — the marker word inside a delimited clause is - dropped from a multi-token clause during grouping; the extraction - itself keeps the whole enclosed span, so nothing is lost when - there is no marker. +- 2026-08-04 #329 (PR #331) — the marker word inside a delimited + clause is dropped from a multi-word clause during grouping; the + extraction itself keeps the whole enclosed span, so nothing is + lost when there is no marker. Three decisions argued separately: + the clause-size guard exists because Nee is an attested surname + (Irish Ní/Nee, a Chinese romanization) — load-bearing, not + defensive, and mutation-proven; clauses are independent — "(Nee) + (Jones)" reads maiden "Nee Jones", each clause dropping or + keeping its own marker (the first implementation leaked across + clauses, a real defect); and the contentless "(née —)" alone now + parses to every field empty and bool() False (an explicit + alternative keeping maiden "née —" was rejected; pinned by the + maiden_marker_delimited_content_free case). +- M1 and M2 are disjoint by construction, which is the no-conflict + guarantee: M2's walk covers joining structure that role-bearing + words never enter, while M1's drop reaches only extracted + content. Verified independently at review over 7,775 records: + 967 diffs, every one in the maiden field. + +Declined: + +- Neighbour-scoping the drop (drop a marker whose next word also + reads maiden) — implemented, reviewed and rejected: it also fires + on the bare-marker path, eating the surname out of "Jane Smith + nee Nee Jones" (maiden "Jones" instead of "Nee Jones"), and it + leaks across adjacent clauses. It is the obvious implementation, + which is why this entry exists. - 2026-08-05 #329/#335 — marker auto-detection inside a nickname-delimited clause was deferred to #335 on a corpus diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index f1df156..3e108b8 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -294,10 +294,14 @@ inert-measurement class — checks that run, print plausible results, and measure nothing — has recurred double-digit times; these shapes are its known antidotes. Convention (AGENTS.md): guard tests SHOULD carry a RECORDED negative control, the _EXCLUSION_EFFECT shape — the -answer with the guard off, stored as data. Honest limit: these -reduce the inert-measurement class, not the wrong-predicate class; a -guard asserting the wrong invariant is caught only by adversarial -review. +answer with the guard off, stored as data. Honest limit, precisely drawn: these +reduce the inert-measurement class. Mutation testing reaches part +of the wrong-predicate class too — mutating the thing a guard +watches exposes a guard that never depended on it (the #329 +survivors: deleting the tag check left the suite green while every +multi-word maiden clause lost its first word, after adversarial +review had passed that code twice) — but a predicate wrong in a way +the fixture happens to satisfy still needs adversarial review. ### VERSION-TELL — know who answered @@ -372,6 +376,11 @@ decline, not delete the test. passed. - Purge __pycache__ between same-length source mutations; stale bytecode makes a changed file measure as unchanged. +- A skip is indistinguishable from "correctly declined": pytest + turns an empty parametrize into a skip, and a filter that widens + its own skip set cannot fail. After changing any selection shape, + verify the guard still REACHES the code it watches — assert the + selected set is non-empty, or force-a-decision on its size. - Mind the optional-extra environment split: a local venv's incidental namedivider makes `if available` branches run PRESENT locally and ABSENT in CI, so a locally-green suite proves nothing diff --git a/docs/design/rules.md b/docs/design/rules.md index 6ef2ea9..2eb83fe 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -267,11 +267,16 @@ M1. Rationale: an enclosure the caller has declared to mean maiden With a delimiter pair configured for maiden names, its enclosed clause reads as the maiden name — unless the content is suffix-shaped, which S1 takes first — a leading recognized - marker word inside the clause being dropped; a pair configured - for both maiden and nickname reads maiden. + marker word inside a multi-word clause being dropped; a one-word + clause keeps its word, which may itself be a surname (Nee). + Clauses are independent: two enclosures read as one maiden name, + each dropping or keeping its own marker. A pair configured for + both maiden and nickname reads maiden. "Jane Smith (née Jones)" maiden-parens → maiden="Jones" + "Jane Smith (Nee)" maiden-parens → maiden="Nee" · boundary + "Jane Smith (Nee) (Jones)" maiden-parens → maiden="Nee Jones" "Jane Smith (née Jones)" → nickname="née Jones" · boundary - history: decisions.md#M1 · interacts: S1 · implemented: nameparser/_pipeline/_extract.py, nameparser/_pipeline/_group.py + history: decisions.md#M1 · interacts: S1, M2 · implemented: nameparser/_pipeline/_extract.py, nameparser/_pipeline/_group.py M2. Rationale: a maiden marker announces that what follows it is the former family name; the marker is an announcement, not a name. @@ -293,7 +298,7 @@ M2. Rationale: a maiden marker announces that what follows it is the its path, the join (P2) running first. "Jane Smith, née Jones" → maiden="" "Jane de la née Jones" → family="de la née Jones" - history: decisions.md#M2 · interacts: P2 · implemented: nameparser/_pipeline/_group.py + history: decisions.md#M2 · interacts: P2, M1 · implemented: nameparser/_pipeline/_group.py ## Commas & structure (C) diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index b4f4305..c375ec9 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -431,9 +431,10 @@ def group(state: ParseState) -> ParseState: ptags[m:j] = [] all_pieces.append(tuple(tuple(p) for p in pieces)) all_ptags.append(tuple(frozenset(t) for t in ptags)) - # rules.md#M1: "a leading recognized marker word inside the clause - # being dropped" — a marker inside EXTRACTED maiden content - # (#329). classify tags + # rules.md#M1: "a leading recognized marker word inside a + # multi-word clause being dropped; a one-word clause keeps its + # word" — a marker inside EXTRACTED maiden content (#329). + # classify tags # such a marker like any other token -- what the #274 rule above # lacks is not the TAG but the token: extract claims a delimited # clause and tokenize gives its tokens Role.MAIDEN up front, so From bde05d0b45bba9cb80ced1128ccb8933b425ba42 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 18:49:29 -0700 Subject: [PATCH 28/40] docs(rules): topic-review amendments from the particle-arc session (primary source) decisions.md#P2 claimed #367 made 'Sir de Mesnil' chain -- the inverse of the truth (#367 REMOVED a chain; the family reading is P1's fold), contradicting P1's own correct entry; both now agree and the misfiled example moved to P1. The pieces= assertion form is implemented (its first using rule arrived): new P4 states that a leading particle chains nothing, with [[Sir],[de],[Mesnil]] and the chained counterpart as structural examples. P1's family-first boundary becomes the doc's first live deviates: marker (#368 superseded #359's working-as-intended sentence; intended family='de la Vega', today family='Juan'), and its statement now names #364 as open rather than pre-answering it. New: S3 (period-joined vocabulary -- the rule whose absence made a docstring wrong four times), the particle-curation and esq Excluded blocks (mirroring AGENTS' algebra, counts deliberately omitted), two P1 Declined entries (the strict-xfail back door; the not-a-title predicate that st/do/freiherr break), the #365 two-orders-disagree why, the deprecation-bridge template in 3-0-reevaluations, VOCABULARY-OVERLAP-AS-PRECONDITION, and #372's standing caution on LEDGER-RULE-SEPARATION. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 84 +++++++++++++++++++++++++++++----- docs/design/mechanisms.md | 23 +++++++++- docs/design/rules.md | 32 +++++++++++-- nameparser/_pipeline/_group.py | 2 + nameparser/_pipeline/_vocab.py | 2 + tests/v2/test_rules_doc.py | 15 +++++- 6 files changed, 140 insertions(+), 18 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 0cce855..e7336a8 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -38,11 +38,21 @@ rule 3 → O2, rule 4 → O3. A lone PIECE is the whole test — deliberately narrower than "a never-given particle is never reported as the given name," which -would be false. Under a family-first order, "Juan de la Vega" holds -the entire chained group in the given position — three words, not a -lone particle — so P1 declines and given="de la Vega" stands; #359 -records that case as working as intended. The MIDDLE position is -deliberately not a fold site. +would be false (the over-broad invariant shipped to five sites +before #361's review falsified it with three counter-examples). +The counter-example set has since shrunk, and its shrinkage is the +section's history: "Sir de Mesnil" fell to #367 (titles became +transparent); "Juan de la Vega" under family-first — the whole +chain in the given position — was called working-as-intended by +#359, but #368 SUPERSEDES that sentence: the recorded decision is +that the particle wins and a chain becomes the family name +whatever order was declared, so that case is now P1's tracked +deviation, not its boundary. The survivor is the degenerate bare +"de". The MIDDLE position is deliberately not a fold site — and +not merely unimplemented: the two family-first orders disagree +there ("Mesnil Garcia de" strands middle="de" under FAMILY_FIRST +and folds under FAMILY_FIRST_GIVEN_LAST, 464 measured inputs), +which is what makes #365 a decision rather than a gap. - 2026-08 #359 — the opening site is read from joining structure (pieces), not from assigned roles, so the fold holds under every @@ -52,6 +62,18 @@ deliberately not a fold site. Mesnil" now reads like "de Mesnil". Fixed by removing the title→particle chain in grouping, not by touching this rule. +Declined: + +- A strict xfail asserting "de Mesnil" → family under FAMILY_FIRST + (#359 review) — #359 deliberately left those semantics open, and + a strict xfail decides the question by the back door. +- Keying the leading-particle exception on "the first piece that is + not a title" — the obvious implementation, and wrong: st, do and + freiherr are titles AND ambiguous particles, so it collapsed + "St John Smith" into one given name and broke test_add_title + (which adds "te", also a particle). The shipped predicate is + "not a title or a prefix". + Open: [#364](https://github.com/derek73/python-nameparser/issues/364) how much the fold takes · [#365](https://github.com/derek73/python-nameparser/issues/365) @@ -61,13 +83,14 @@ which particles count as never-given. ### P2 — particles join forward -- 2026-08 #367 — a title is transparent to the chain's start: - "Sir de Mesnil" chains de→Mesnil exactly as the untitled form - does. Before, the title displaced the particle out of the leading - position and "Sir de Mesnil" reported given="de Mesnil" with no - family at all — a limit the rule never meant to draw. Fixed in - grouping, which is why P1's fold needed no change (its interacts: - points here). +- 2026-08 #367 — REMOVED a chain, it did not create one: before, + the title displaced the particle out of the leading position, so + "Sir de Mesnil" grouped [Sir][de Mesnil] — a chain — and + reported given="de Mesnil" with no family at all. After, "de" is + the leading name piece, and a leading particle chains nothing + (rule P4): the family reading comes from P1's fold, which is why + P1's fold needed no change (its interacts: points here). Grouped + today: [Sir] [de] [Mesnil]. ### M2 — the maiden-marker rule @@ -241,6 +264,34 @@ Declined: run offers a site, since a glued honorific is itself part of what makes a run read as suffix-shaped. +Excluded (the never-given / ambiguous particle line, +nameparser/config/particles.py — #360 owns the vocabulary +question): + +- Only 9 of the 39 ambiguous members were ever individually + justified; the rest sit there by the conservative default + (ambiguous unless argued never-given). +- mc, ste — measured misparses ("Mc Donald" → given "Mc"), tracked + in #360; st is inert at the head because TITLES claims it first; + mac must stay ambiguous because Mac is a real given name. +- Load-bearing dependency: TITLES ∩ ambiguous == {do, freiherr, + st} is what keeps the particle-or-given ambiguity emitter + reachable at all; moving all three would make it dead code, + which is why test_the_chained_emitter_is_still_reachable + distinguishes "pick another word" from "delete the emitter". + +Excluded (SUFFIX_ACRONYMS / SUFFIX_WORDS — the esq dual +membership, deliberate; AGENTS.md's gotcha carries the full +algebra): + +- esq is in BOTH sets and must not be "deduplicated". The + load-bearing membership is the acronym one (it carries the + multi-dot spellings: removing it costs "John Smith E.S.Q." its + family name); the word membership is inert as shipped but is + what keeps "Esq" matching for a caller who edits suffix_acronyms + themselves. Deliberately no changed-parse count — the count is a + property of the measuring grid, not of the code. + Excluded (Lexicon.honorific_tails — a glued tail peels only if it could never end a name; per-entry reasons live in nameparser/config/suffixes.py's vetting block): @@ -487,6 +538,15 @@ nobody re-litigates it. Append here whenever a design choice cites (script_orders fallbacks, #298 dot-suppression granularity): coincides with 1.4 parity but stands alone — family-first is the marked case needing affirmative evidence. +- (B) The deprecation-bridge shape (#293/#354) is the template for + every remaining 2.x→3.0 shim: PEP 562 module __getattr__ PLUS + __all__ (star imports never reach __getattr__ — measured: `from + ...prefixes import *` bound nothing and leaked the helper); warn + per read-location rather than per process (a write-back let a + vendored dependency's first read consume the only warning); and + the TYPE_CHECKING split, because a module __getattr__ silently + disables mypy's attr-defined checking for the whole module + (measured on a py.typed package). - (B) The FAMILY_COMMA doctrine (rule W3): inherited from v1's lastname-comma but correct on its own terms — an explicit comma is stronger evidence than script. diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 3e108b8..a841d73 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -204,7 +204,13 @@ Lives in. tools/differential/compare.py, the expected_since_*.toml ledgers. Reach for it when. A ledger rule's behavior seems to depend on where it sits in the file — it doesn't, and if moving it changes anything, -the fields are wrong. +the fields are wrong. Standing caution (#372, closed): the contract +is true as written and was measured loose in practice — a +fields-only rule matched all 751 corpus names and owned 1639 of +5257 name×field pairs, and the moving-test above had never actually +been run. #372's two proposed mechanical checks (report every +matching rule, not just the first; a specificity floor for +fields-only rules) are recorded there. ## CANONICAL-VOCABULARY-AT-THE-BOUNDARY — one vocabulary at the comparison @@ -225,6 +231,21 @@ Reach for it when. A second surface joins an existing matcher — without this, every existing rule silently stops matching the new surface, which looks like added coverage. +## VOCABULARY-OVERLAP-AS-PRECONDITION — assert the intersection you stand on + +Problem shape. A test's input depends on two config sets +intersecting (a word that is both a title and a particle), and a +vocabulary edit could quietly unground it. +Contract statement. Assert the intersection as a precondition and +distinguish the two failure modes: word moved (pick another member; +the message prints what is left) versus intersection empty (the +code under test is unreachable — delete it rather than repointing +the test). +Lives in. tests/v2/test_parser.py +(test_the_chained_emitter_is_still_reachable). +Reach for it when. Any config-coupled fixture — a test input chosen +because of what a vocabulary happens to contain. + ## MAKE-WRONG-STATES-UNREPRESENTABLE — the house meta-pattern Problem shape. A convention keeps being violated no matter how diff --git a/docs/design/rules.md b/docs/design/rules.md index 2eb83fe..f0da6aa 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -130,10 +130,12 @@ P1. Rationale: a never-given particle standing alone cannot be go — or opening the name — marks the name as surname-only: the given and middle words fold into the family. It needs another name word to fold into. An ambiguous particle keeps whatever - reading its position gives it. + reading its position gives it. Whether the fold should stop at + the particle group instead of taking everything is open (#364). "de la Vega" → family="de la Vega" + "Sir de Mesnil" → family="de Mesnil" "Mesnil de" family-first → family="Mesnil de" - "Juan de la Vega" family-first → given="de la Vega" · boundary + "Juan de la Vega" family-first → family="de la Vega" deviates: #368 (today: family="Juan") "van Gogh" → given="van" · boundary Accepted: a bare "de" stays the given name — there is nothing to fold into, and inventing a surname would be worse. @@ -147,7 +149,7 @@ P2. Rationale: a particle is written as part of the surname it the name begins, and a preceding title does not move that point. "John van der Berg" → family="van der Berg" "John van der Berg Smith" → family="van der Berg Smith" - "Sir de Mesnil" → family="de Mesnil" + "Dr. John van der Berg" → family="van der Berg" "Juan de" → family="de" · boundary history: decisions.md#P2 · implemented: nameparser/_pipeline/_group.py @@ -161,6 +163,19 @@ P3. Rationale: connective words ("y", "of the") bind name words into "Juan y Garcia" → middle="y" · boundary implemented: nameparser/_pipeline/_group.py +P4. Rationale: a particle links forward from inside a name; at the + very front there is no name yet to be inside. + A particle in the name's leading position chains nothing: the + words stay separate, and any surname reading the name gets + comes from the fold (P1) or from position (O4), never from a + join. This is why a title before a leading particle changes + nothing (the title is not a name word), and why "Van Johnson" + is a given-name reading at all. + "Van Johnson" → given="Van" + "Sir de Mesnil" → pieces=[["Sir"], ["de"], ["Mesnil"]] + "John van der Berg" → pieces=[["John"], ["van", "der", "Berg"]] · boundary + history: decisions.md#P2 · interacts: P1 · implemented: nameparser/_pipeline/_group.py + ## Suffixes: generational & credentials (S) Background: what follows a name is one of two different things — @@ -204,6 +219,17 @@ S2. Rationale: generational suffixes and credentials are recognized "Smith Jr." → family="" implemented: nameparser/_pipeline/_classify.py +S3. Rationale: credentials are often written run together with + periods; the chunks between the periods are what carry the + vocabulary. + A word with interior periods reads as a suffix when any of its + period-separated chunks is suffix vocabulary — any chunk, which + is looser than it sounds, since single letters can be Roman + numerals. + "John Smith J.u.n.i.o.r." → suffix="J.u.n.i.o.r." + "John Smith Q.W.E.R.T." → family="Q.W.E.R.T." · boundary + implemented: nameparser/_pipeline/_vocab.py + ## Nicknames & quoted names (N) Background: a nickname is written beside the formal name, set off by diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index c375ec9..6688be9 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -67,6 +67,8 @@ def _is_title_piece(piece: Sequence[int], ptags: Set[str], # name, and the join runs to the end of the name; the chain begins # wherever the name begins, and a preceding title does not move that # point" (history: decisions.md#P2) +# rules.md#P4: "a particle in the name's leading position chains +# nothing: the words stay separate" (history: decisions.md#P2) def _is_prefix_piece(piece: Sequence[int], ptags: Set[str], tokens: Sequence[WorkToken]) -> bool: if "prefix" in ptags: diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 159170c..61e7ce6 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -178,6 +178,8 @@ def splits_into_suffixes(text: str, cores: frozenset[str], return False +# rules.md#S3: "a word with interior periods reads as a suffix when +# any of its period-separated chunks is suffix vocabulary" def period_joined_vocab(text: str, lexicon: Lexicon) -> str | None: """v1's parse_pieces derivation for interior-period tokens ('Lt.Gov.', 'Msc.Ed.', and by the ANY rule 'Mr.Smith'): ANY title diff --git a/tests/v2/test_rules_doc.py b/tests/v2/test_rules_doc.py index 511df38..4877625 100644 --- a/tests/v2/test_rules_doc.py +++ b/tests/v2/test_rules_doc.py @@ -48,9 +48,18 @@ def _check_diagnostic(example: Example) -> None: fn() +def _pieces(text: str, policy: Policy | None) -> list[list[str]]: + from nameparser._pipeline import run + from nameparser._pipeline._state import ParseState + from nameparser._parser import Parser as _P + p = _P(policy=policy) if policy is not None else _P() + state = run(ParseState(original=text, lexicon=p.lexicon, + policy=p.policy, segmenter=None)) + return [[state.tokens[i].text for i in piece] + for seg in state.pieces for piece in seg] + + def _run(example: Example) -> object: - if example.field == "pieces": - pytest.skip("assertion form lands with its first using rule") policy: Policy | None = None locale: str | None = None if example.annotation is not None: @@ -70,6 +79,8 @@ def _run(example: Example) -> object: "runs this") assert isinstance(obj, str) locale = obj + if example.field == "pieces": + return _pieces(example.text, policy) if locale is not None: parsed = parser_for(locales.get(locale)).parse(example.text) elif policy is not None: From 4a5b7fa836df137c9d4f3938c27ff77268d3b35f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 18:56:35 -0700 Subject: [PATCH 29/40] docs(rules): topic-review amendments from the ledger-arc session (primary source) LEDGER-RULE-SEPARATION's contract was falsified by measurement: a strict fields-subset pair shares the regex tier in the live 1.4 ledger and file order decides between them (a pure reorder reattributes seven names, caught only by _CROSS_RULE_WINNERS). The contract now states the between-tiers/within-tier split, the stale 1639-figure is framed as the dated #372-as-filed measurement with the #375/#376 correction, and 'never actually run' is fixed (run twice in #375, failed). The whole dormancy arc (#328-#376) lands in decisions.md: three shipped decisions ([[never]] monotone exclusions, always-on dormant with three diagnoses, the 2.0.0 baseline gap) and four Declined-with-measurements. New: CROSS-RULE-OUTCOME-PINS mechanism, the alternation-plus-sync-roster second half on CURATED-VOCABULARY-ALTERNATION, the narrowing-relocates-the-bug field note, and the roster list completed on RECORDED-ROSTERS. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 46 ++++++++++++++++++++++++++ docs/design/mechanisms.md | 68 +++++++++++++++++++++++++++++---------- 2 files changed, 97 insertions(+), 17 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index e7336a8..9bcf024 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -499,6 +499,52 @@ shadows the whole ledger, since name_regex rules sort first): "", "(?:)", ".", ".+", "\b", "[\s\S]". Enforced by the sentinel-set check. +### differential-ledger, the dormancy arc (2026-08, #328–#376) + +The second ledger arc; measurements are that session's, the +machinery verified present at landing. + +Decisions that landed: + +- 2026-08 #328 — `[[never]]` exclusions: ledger vocabulary for + "this shape must never be explained". classify() consults + exclusions before rules, which makes them MONOTONE — an entry + only ever removes a name, never moves one between rules — so an + exclusion's blast radius is exactly the names it captures, + independent of rule order. +- 2026-08-12 #373 — `dormant = ""`: a rule explaining + nothing fails the run in both directions, with three diagnoses + (reverted / shadowed by X / refused by a [[never]] entry). Always + on, not behind --strict: a check nobody passes a flag to is a + check that doesn't exist. +- 2026-08 #373 — `--baseline 2.0.0` joined the release checklist: + that ledger's rules had no dynamic coverage at all; measured + clean at 90/0, so the gap was closed rather than documented. + +Declined: + +- Ambiguity reporting on multi-matching rules (#372/#373) — + measured: 28% of claimed name×role pairs already have ≥2 + matching rules (732 of 2619; 432 are one pair of rules alone). A + report firing on 28% of what it inspects is wallpaper. The + actionable slice shipped as the "shadowed by " diagnosis, + which speaks only on FULL shadowing. +- A specificity floor for fields-only rules (#372/#373) — exactly + one fields-only rule exists in any ledger, naming 3 of 7 roles; + a six-of-seven floor matches nothing, and nothing would reveal + it vacuous. +- Specificity reordering of the rule sort (#328) — measured across + all 751 names: width-then-regex-length moves five rule + populations and sends 17 names into the generic fields-only + rule, draining the CJK-specific ones. No reading of the sort + produces the "exactly one label changes" originally claimed; + that figure was corrected on the PR. +- Unanchoring the honorific-suffix rule to reach glued forms + (#376) — it would make a future suffix regression on 김지양 + classify as a recognized honorific. A confidently wrong label is + worse than a catch-all's honest breadth, and the gate reads the + same either way. + ### 3-0-reevaluations — decisions shaped by the v1 shim Promoted 2026-08-15 from session memory (Derek's 2026-07-30 ask; diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index a841d73..d729187 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -172,6 +172,11 @@ given name. The set, not a regex, draws the line. Lives in. nameparser/config/suffixes.py (the vetting block). Reach for it when. Arguing that "no regex can separate X from Y" — check whether a config set already splits them by listing one side. +The second half of the pattern: to USE the set in a regex, hand-copy +the alternation and pin the copy with a sync roster +(_HONORIFIC_SOURCES) — the 2.0.0 ledger has done exactly this since +#308, which is why #376's fix was copying its twin verbatim rather +than inventing one. ## RECORDED-ROSTERS — record the answer, don't re-derive it @@ -181,7 +186,9 @@ Contract statement. Store the measured answer as literal data (a roster) and compare against it; never re-derive the expectation from the same inputs the check reads, because a derivation from the same data always agrees with itself. -Lives in. tests/v2/test_ledger_guards.py (_CORPUS_CLAIMS), +Lives in. tests/v2/test_ledger_guards.py (_CORPUS_CLAIMS, +_EXCLUSION_EFFECT, _CROSS_RULE_WINNERS, _SPAN_BEARING_RULES, +_HONORIFIC_SOURCES, _LATIN_ALTERNATION_SOURCES), tools/differential/compare.py (_CORPUS_FLOORS), tests/v2/test_facade_cases.py (_CORE_ONLY_IDS). Reach for it when. Writing a check whose expected value is computed @@ -192,25 +199,33 @@ make it data the suite asserts. Problem shape. Two differential-ledger rules claim overlapping names. -Contract statement. Ledger rules are separated by their fields -subsets and matching predicates, never by their order in the file; -a fields-only rule sorts last and takes what nothing narrower named. -How it works. Detail is owned by tools/differential/README.md. One -standing constraint worth repeating here: sync-pinned rosters select -rules by issue-string substring, so a new rule's issue slug must -avoid the literal #271/#272 substrings unless it means to be -selected. +Contract statement. Fields subsets and matching predicates separate +ledger rules BETWEEN tiers; within a tier, file order decides and +the narrower rule must be written first. A fields-only rule sorts +last unconditionally and takes what nothing narrower named. +How it works. Detail is owned by tools/differential/README.md. The +within-tier clause is measured, not theoretical: in the 1.4 ledger +the comma-honorific-peel rule's fields are a strict subset of the +comma-compound rule's, both carry a name_regex, and a pure reorder +reattributes seven names — caught by _CROSS_RULE_WINNERS and by +nothing else in the suite (#375's mutation). Whether that pair +should be separated by a predicate instead of by order is an open +question with no issue yet. One standing constraint worth repeating +here: sync-pinned rosters select rules by issue-string substring, so +a new rule's issue slug must avoid the literal #271/#272 substrings +unless it means to be selected. Lives in. tools/differential/compare.py, the expected_since_*.toml ledgers. Reach for it when. A ledger rule's behavior seems to depend on where -it sits in the file — it doesn't, and if moving it changes anything, -the fields are wrong. Standing caution (#372, closed): the contract -is true as written and was measured loose in practice — a -fields-only rule matched all 751 corpus names and owned 1639 of -5257 name×field pairs, and the moving-test above had never actually -been run. #372's two proposed mechanical checks (report every -matching rule, not just the first; a specificity floor for -fields-only rules) are recorded there. +it sits in the file — within a tier it does, and the reorder +mutation is the test (run twice in #375; it fails +_CROSS_RULE_WINNERS). History: #372 (closed) measured the +fields-only rule owning 1639 of 5257 name×field pairs as filed +(2026-08-10); #375/#376 then cut its classifier-of-record share +sharply, and the residual pair ownership is the last-resort tier +working as designed, not a defect. #372's two proposed mechanical +checks were DECLINED with measurements (see +decisions.md#differential-ledger), not left open. ## CANONICAL-VOCABULARY-AT-THE-BOUNDARY — one vocabulary at the comparison @@ -246,6 +261,21 @@ Lives in. tests/v2/test_parser.py Reach for it when. Any config-coupled fixture — a test input chosen because of what a vocabulary happens to contain. +## CROSS-RULE-OUTCOME-PINS — pin who wins the contest + +Problem shape. Every per-rule roster measures a rule alone and the +gate total is per-corpus, but WHICH rule wins a contested name is +neither — and it is exactly what a reorder or a narrowing changes. +Contract statement. Contested outcomes are pinned as data: a roster +records which rule classifies which contested name, so a change in +the winner fails the suite even when every total is unchanged. +How it works. A pure file reorder in the 1.4 ledger fails +_CROSS_RULE_WINNERS and nothing else in the suite — the pin is the +only guard at that granularity. +Lives in. tests/v2/test_ledger_guards.py (_CROSS_RULE_WINNERS). +Reach for it when. Two rules can claim the same name and you are +about to change either one, or their order. + ## MAKE-WRONG-STATES-UNREPRESENTABLE — the house meta-pattern Problem shape. A convention keeps being violated no matter how @@ -397,6 +427,10 @@ decline, not delete the test. passed. - Purge __pycache__ between same-length source mutations; stale bytecode makes a changed file measure as unchanged. +- After NARROWING a rule, check the receiver: the names a narrowed + rule sheds land on a neighbour, and nothing guarantees the + neighbour's prose describes what it inherited — #375 fixed an + over-claiming rule and relocated the bug onto its neighbour. - A skip is indistinguishable from "correctly declined": pytest turns an empty parametrize into a skip, and a filter that widens its own skip set cannot fail. After changing any selection shape, From 7fd1c71428241e3acb91271eaae713479f98c0c3 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 18:59:45 -0700 Subject: [PATCH 30/40] docs(rules): link #382 from the ledger-separation open question Co-Authored-By: Claude Fable 5 --- docs/design/mechanisms.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index d729187..6e2c44e 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -209,8 +209,8 @@ the comma-honorific-peel rule's fields are a strict subset of the comma-compound rule's, both carry a name_regex, and a pure reorder reattributes seven names — caught by _CROSS_RULE_WINNERS and by nothing else in the suite (#375's mutation). Whether that pair -should be separated by a predicate instead of by order is an open -question with no issue yet. One standing constraint worth repeating +should be separated by a predicate instead of by order is +[#382](https://github.com/derek73/python-nameparser/issues/382). One standing constraint worth repeating here: sync-pinned rosters select rules by issue-string substring, so a new rule's issue slug must avoid the literal #271/#272 substrings unless it means to be selected. From ac77e9a40a07db676858a25e6f53cbf167d36994 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 19:04:48 -0700 Subject: [PATCH 31/40] docs(rules): topic-review amendments from the Indic session (primary source) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit H1 gains the empty-family Accepted consequence the reviewer's own worst error called for ('Sir John' → family='', executable); the H Background carries the renunciate criterion (retains-a-family-name, not religious-title — Rabbi Cohen measured correct); S2's ambiguity flag is stated as firing in both readings with the assertable example; decisions#H2's issue range narrowed to #343/#344. New category: contested vocabulary memberships get Open blocks keyed to the vocabulary set (#342/#346/#343/#344) and the rules.md preamble names them as the third grep-invisible class. Excluded blocks gain the non-CJK honorific-tails prohibition (जी, the 殿 criterion in Devanagari) and ঠাকুর-is-Tagore under TITLES. LOCALE-PACKS-PURE-DATA's contract states add-only — a pack can never remove a base entry, which is why #342-class fixes must change shipped vocabulary. W Background gains the script-member no-op fact and the abugida divergence limit. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 24 +++++++++++++++++++++++- docs/design/mechanisms.md | 10 +++++++--- docs/design/rules.md | 21 +++++++++++++++++---- 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 9bcf024..5497dbc 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -195,7 +195,7 @@ Declined: extraction litmus (2026-08-15): the spec drafted this rule as "recognized by vocabulary, not by written shape" and the live parser falsified that framing — the shape heuristic is real, and - what the abugida gap (#342-#345) shows is its LIMIT, not its + what the abugida gap (#343/#344) shows is its LIMIT, not its absence. Recorded as the rule's Accepted consequence. ### W1 — unspaced CJK division @@ -280,6 +280,17 @@ question): which is why test_the_chained_emitter_is_still_reachable distinguishes "pick another word" from "delete the emitter". +Open (contested vocabulary memberships — the rule is right, the +word's set is questioned; the issue is canonical): +[#342](https://github.com/derek73/python-nameparser/issues/342) +rai in SUFFIX_ACRONYMS vs. the South Asian surname · +[#346](https://github.com/derek73/python-nameparser/issues/346) +swami and the renunciate titles absent from the given-name-title +set · +[#343](https://github.com/derek73/python-nameparser/issues/343) / +[#344](https://github.com/derek73/python-nameparser/issues/344) +Bengali and Devanagari honorific vocabulary. + Excluded (SUFFIX_ACRONYMS / SUFFIX_WORDS — the esq dual membership, deliberate; AGENTS.md's gotcha carries the full algebra): @@ -307,6 +318,17 @@ nameparser/config/suffixes.py's vetting block): populations); peeling it would cut a real family name in two. - 君 — 王君 is a complete Chinese name; its kana spelling くん does peel. +- जी (standing prohibition for #344's implementation) — Banerjee, + Mukherjee and Chatterjee end in the substring + (बनर्जी/मुखर्जी/चटर्जी), and glued peeling strands a fragment on + a bare virama (बनर् + जी). The 殿 criterion, in a non-CJK script + — which also shows the criterion is not CJK-specific. + +Excluded (TITLES): + +- ঠাকুর — a genuine Bengali honorific (lord/master) that is also + Tagore, the surname (#343 records it so a wordlist sweep does not + ship it). Excluded (Policy.script_orders defaults): Script.KATAKANA is deliberately absent — a pure-katakana token is predominantly a diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 6e2c44e..d897ac9 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -298,13 +298,17 @@ with Y." Problem shape. Language-specific behavior needs a home that cannot drift from the core. Contract statement. A locale pack is pure data — a Policy patch and -Lexicon additions — applied by parser_for; packs contain no code -paths of their own, and each pack's docstring declares its +Lexicon ADDITIONS, add-only: a pack unions vocabulary in and can +never remove a base entry — applied by parser_for; packs contain no +code paths of their own, and each pack's docstring declares its deviations from the defaults. How it works. What is policy for every language (an order constant) is policy, not pack data; packs are lowercase modules exposing uppercase constants; a pack error is wrapped with the pack's code -(rule D2). Pure data means a pack can be audited by reading it. +(rule D2). Pure data means a pack can be audited by reading it — +and add-only means a bad DEFAULT can never be routed around by a +pack, which is why #342-class fixes must change shipped +vocabulary. Lives in. nameparser/locales/ (packs), nameparser/_parser.py (parser_for, the one applier). Reach for it when. A language fix wants an if-statement — make it diff --git a/docs/design/rules.md b/docs/design/rules.md index f0da6aa..ecc7bb7 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -41,7 +41,9 @@ issue; the runner asserts today's output strictly, so a parser change that closes the gap fails the suite until the marker is removed in the same PR. `grep deviates:` on this file is the deviation backlog (deviations from statable rules — coverage gaps are a separate, -larger category no grep can see). +larger category no grep can see, and contested vocabulary +memberships a third, tracked as Open blocks keyed to the vocabulary +set in decisions.md). ## Not in scope @@ -64,7 +66,13 @@ of it; it addresses or ranks the person. Most titles address by surname ("Mr. Johnson"), but a few — knighthoods, some clerical and courtesy titles — address by given name ("Sir John"). The library keeps a vocabulary of titles and, separately, of these given-name -titles. +titles. The dividing criterion for the given-name list is NOT +"religious title" but whether the tradition retains a family name: +renunciation abolishes the surname, so for Swami, Guru, Baba or +Lama an empty family is the correct output (#346), while rabbi and +imam traditions keep surnames — "Rabbi Cohen" addresses by title +and keeps family "Cohen". Sweeping all religious titles into the +given-name list would break the latter. H1. Rationale: a title normally addresses by surname, so a title followed by a single name word usually names the family; but a @@ -75,6 +83,10 @@ H1. Rationale: a title normally addresses by surname, so a title "Mr. Johnson" → family="Johnson" "Mrs. Garcia" → family="Garcia" "Sir John" → given="John" · boundary + Accepted: a given-name title plus one name word leaves the + family empty — the input names no family, and inventing one + would be worse. + "Sir John" → family="" implemented: nameparser/_pipeline/_post_rules.py H2. Rationale: before a name, an abbreviation is almost always a @@ -205,8 +217,8 @@ S2. Rationale: generational suffixes and credentials are recognized generational forms and credential acronyms alike, and an ambiguous acronym written with periods counts unambiguously. A BARE ambiguous acronym is consumed only when the name has words - to spare: as the second of two words it stays the family name, - flagged ambiguous. + to spare — as the second of two words it stays the family + name — and either reading carries the ambiguity flag. "John Smith Jr." → suffix="Jr." "John Smith M.A." → suffix="M.A." "John Smith PhD" → suffix="PhD" @@ -216,6 +228,7 @@ S2. Rationale: generational suffixes and credentials are recognized belongs to; and an unambiguous suffix is consumed even when that leaves no family name at all. "Jack Wei Ma" → suffix="Ma" + "Jack Wei Ma" → ambiguities=("suffix-or-name",) "Smith Jr." → family="" implemented: nameparser/_pipeline/_classify.py From 3148aace8884f883d4fe5b8a6870d3e2776d657b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 19:09:02 -0700 Subject: [PATCH 32/40] docs(rules): document the contested-membership Open keying convention Co-Authored-By: Claude Fable 5 --- AGENTS.md | 4 +++- docs/design/decisions.md | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 3ecc000..1c08d9d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,7 +25,9 @@ you are about to invent is often already there. tests/v2/test_doc_citations.py. - **decisions.md** — the ADR-style record: dated entries, Declined: (rejected WITH evidence), Excluded: (what must stay out of a - wordlist and why), Open: (issue links, never restated), + wordlist and why), Open: (issue links, never restated; also keyed + to a vocabulary set for contested memberships — the + right-rule-wrong-set class), 3-0-reevaluations (append whenever a design cites 1.4 parity as load-bearing). - **mechanisms.md** — problem-shape catalog with citable contract diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 5497dbc..cf09ad2 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -21,6 +21,13 @@ Entry conventions: sweeping a wordlist ships the excluded entry as a bug. - **`Open:`** — unresolved questions as issue links with one-line handles. The ISSUE is canonical; this block never restates it. + Two keyings: under a rule ID for questions about the rule, and — + like `Excluded:` — keyed to a VOCABULARY SET for contested + memberships, the category neither `deviates:` nor `Excluded:` + covers: the rule is right and a word's set membership is the + question (rai in the suffix acronyms, swami absent from the + given-name titles). Place the block beside the set's `Excluded:` + entries so a wordlist editor meets both. - **Weighing entries** for contested questions: the options considered, each option's intended effect, and the accepted costs of the option chosen. The costs accepted here are the artifacts From 71ca544db5adcfcc78c8d6285fc689d51740a52f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 19:10:58 -0700 Subject: [PATCH 33/40] docs(rules): document the remaining review-uncovered conventions Decision dates (not release dates; git over narrative), harvest- landing provenance discipline (uncertainties resolved from the source, never by inference), and the primary-source review protocol -- each practiced and review-corrected during PR #381, none previously written down. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 8 ++++++++ docs/design/decisions.md | 12 +++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 1c08d9d..13e79b7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,6 +63,14 @@ memory. Per-rule ledger toml comments asserting PARSER behavior cite rule IDs under the excerpt discipline; free prose is for ledger mechanics only (owned by tools/differential/README.md). +**Primary-source review.** When doc content is distilled from a +session's work, have that session (or its transcript) review its +own sections before or soon after landing — attribution flattening +and inverted arguments are visible only to the source. Reviewers +state which tree each measurement ran on (stale fetches produced +three rounds of already-fixed findings), and landed corrections are +re-verified here before committing. + **Guard tests** SHOULD carry a recorded negative control — the answer with the guard off, stored as data (the _EXCLUSION_EFFECT shape; see mechanisms.md's Verification shapes). diff --git a/docs/design/decisions.md b/docs/design/decisions.md index cf09ad2..e3c2556 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -10,7 +10,17 @@ Entry conventions: - **Dated decision entries**, append-only in spirit: a reversed decision is not edited, a later entry supersedes it. Each entry - cites its issue or PR. + cites its issue or PR. The date is the DECISION's, never the + release's (add "shipped in X" alongside if useful), and git + author dates outrank remembered ones — two review rounds + corrected exactly these two errors. +- **Harvested entries** (content landed from a session's report) + keep their provenance: a "measurements are that session's, + spot-checked at landing" framing line, the contributor's + measured-vs-remembered markings where they matter, and — the + hard-won one — a flagged uncertainty is resolved by reading the + source artifact, never by inference from the neighboring + argument. - **`Declined:`** — proposals rejected WITH the evidence that killed them. Resolved-as-no is a decision; without a home for it, the next person re-derives the rejected proposal and its measurement. From 3685f869fa3a44126ea949f22e8a6a3eed591279 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 19:24:12 -0700 Subject: [PATCH 34/40] docs(rules): topic-review amendments from the comma-suffix session (primary source) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit H2's scope was FALSE as landed ('name-opening'): the shape rule fires at the head of the part carrying the given name, post-comma included ('Smith, Major. John' → title='Major.') — a correction PR #315 made to the older docs two weeks before this branch, which the extraction failed to pick up; H3 had the identical narrowing ('Morse, Det. Insp. Jane' now pins it). H2 also states its precedence over the suffix vocabulary ('Esq. Smith' → title), with the matching named exception carved into TWO-LAYER-ASSIGN's contract. C1's lenient/strict bullet reattributed to the v2 Policy commits per git author dates. Two new deviates: markers carry the settled-but-unshipped bundle intent (#296 'Smith, PhD', #291's family-name loss made visible). New: the comma-suffix-arc decisions section (milestone moves, two Declined-with-limits), the trailing- titles Open (#316) with the surname-collision Excluded entry, VOCABULARY-FEEDS-STRUCTURE, the out-of-vocabulary corpus limit field note, and the esq singleton-intersection clause. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 63 ++++++++++++++++++++++++++++++--- docs/design/mechanisms.md | 28 ++++++++++++++- docs/design/rules.md | 29 +++++++++++---- nameparser/_pipeline/_assign.py | 7 ++-- nameparser/_pipeline/_group.py | 5 +-- 5 files changed, 114 insertions(+), 18 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index e3c2556..d2e3700 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -208,12 +208,24 @@ Declined: - 2026-06-30 (leading-period-title design; v2 core, PR #288) — the shape test is v1 parity (period_abbreviation): two-plus letters - then a period, leading position only, bare initials exempt. The + then a period, bare initials exempt. Its site is the head of the + part CARRYING THE GIVEN NAME — the whole name, or the post-comma + part under a family comma — not "the head of the name"; that + scope correction is PR #315 (2026-08-01, docs-only), verified + against 1.4.0 from PyPI, so the parity claim is real and the + narrower description never was. The extraction litmus (2026-08-15): the spec drafted this rule as "recognized by vocabulary, not by written shape" and the live parser falsified that framing — the shape heuristic is real, and what the abugida gap (#343/#344) shows is its LIMIT, not its - absence. Recorded as the rule's Accepted consequence. + absence. Recorded as the rule's Accepted consequence — and the + 2026-08-15 landing initially re-narrowed the scope to "name- + opening", correcting one error while preserving another; the + eighth review round fixed it. + +Open: [#316](https://github.com/derek73/python-nameparser/issues/316) +what a trailing title-vocabulary word should do (the comma paths +disagree today). ### W1 — unspaced CJK division @@ -317,7 +329,10 @@ algebra): multi-dot spellings: removing it costs "John Smith E.S.Q." its family name); the word membership is inert as shipped but is what keeps "Esq" matching for a caller who edits suffix_acronyms - themselves. Deliberately no changed-parse count — the count is a + themselves. esq is the ONLY member of SUFFIX_ACRONYMS ∩ + SUFFIX_WORDS — that singleton is why the two sets cannot carry a + disjointness assert, which is the standing cost this entry + defends. Deliberately no changed-parse count — the count is a property of the measuring grid, not of the code. Excluded (Lexicon.honorific_tails — a glued tail peels only if it @@ -346,6 +361,15 @@ Excluded (TITLES): - ঠাকুর — a genuine Bengali honorific (lord/master) that is also Tagore, the surname (#343 records it so a wordlist sweep does not ship it). +- The trailing-position rule that must NOT be adopted: TITLES holds + hundreds of words in no suffix set, at least nineteen of them + ordinary English surnames (king, judge, bishop, baron, sheriff, + ...), so a blanket "vocabulary outranks position in the trailing + slot" reading would turn "Mary Jane King" into title="King" with + the family name gone. The leading half of this argument is + AGENTS.md's "Dean is deliberately absent" gotcha; this is the + trailing half, and it shadows the family name rather than the + given (#316). Excluded (Policy.script_orders defaults): Script.KATAKANA is deliberately absent — a pure-katakana token is predominantly a @@ -372,8 +396,11 @@ Excluded (MAIDEN_MARKERS, per nameparser/config/maiden_markers.py): definitionally vocabulary-dependent: there is no way to recognize a credential run without consulting the suffix word lists, so the structural stage reads vocabulary through one predicate. -- 2026-07-30 #291/#296 — the lenient token test is the default and - `lenient_comma_suffixes=False` restores the strict one. +- 2026-07-12/13 (v2 Policy work, PR #288) — the lenient token test + is the default and `lenient_comma_suffixes=False` restores the + strict one. (An earlier entry here credited #291/#296 — git + author dates place both the field and its wiring in the Policy + commits, and the #291/#296 design doc never mentions the knob.) - 2026-08 #319 — the wholly-suffix predicate was lifted into the vocabulary layer so the comma decision and the honorific peel's segment test cannot drift apart. @@ -584,6 +611,32 @@ Declined: worse than a catch-all's honest breadth, and the gate reads the same either way. +### comma-suffix-arc — #291/#296/#316 (2026-07-30 → 2026-08-01) + +Intent for #291 and #296 is settled by the approved bundle spec but +UNSHIPPED — rules.md carries both as deviates: markers on C1. The +arc's bookkeeping: #291/#296 moved milestone v2.1 → v2.2 on +2026-08-01 (with #289/#293); the 2026-07-30 design doc was amended +in place 2026-08-01 with an A1–A6 amendments section, and citations +should be to the amended form; #316 (trailing titles) was filed the +same day, carrying the esq cleanup as a Related section. + +Declined: + +- The trailing-abbreviation structural fallback — with a + measurement LIMIT rather than a measurement: the differential + corpora structurally cannot evidence it, because they hold only + names someone wrote down, and an unrecognized abbreviation is by + definition outside the vocabulary. A green run there proves + nothing (the reusable harness fact is in mechanisms.md's field + notes). +- SUFFIX_PHRASES matching in assignment — measured cost: it + renders suffix="LEED, AP", because the suffix view comma-joins + suffix words unless they carry the stable "joined" tag, which + only grouping applies. The general form: multi-word vocabulary + must merge where the render tag is applied, not where the role + is assigned. + ### 3-0-reevaluations — decisions shaped by the v1 shim Promoted 2026-08-15 from session memory (Derek's 2026-07-30 ask; diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index d897ac9..4de13b3 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -102,7 +102,9 @@ Problem shape. Where should a new "recognize X" behavior live? Contract statement. A vocabulary layer first claims words for what they ARE, wherever they sit; a positional layer then reads every unclaimed word by where it STANDS. Every rule belongs to exactly one -layer. +layer — with one named exception: the leading-abbreviation shape +(rule H2) fires before the suffix vocabulary is consulted, so +"Esq. Smith" reads title, not suffix. How it works. The two layers compose without ordering bugs because the positional layer never overrides a vocabulary claim (rule O4 is the positional layer's contract). @@ -276,6 +278,25 @@ Lives in. tests/v2/test_ledger_guards.py (_CROSS_RULE_WINNERS). Reach for it when. Two rules can claim the same name and you are about to change either one, or their order. +## VOCABULARY-FEEDS-STRUCTURE — a wordlist edit can move the comma decision + +Problem shape. Editing suffix vocabulary looks field-local, but the +comma-structure decision (rule C1) reads that vocabulary, so a +removal can flip which segment is the family name. +Contract statement. Suffix-set membership is an input to the +structure decision: removing a word from the suffix vocabulary can +change a suffix-comma name into a listing-form name, relocating the +family. +How it works. Measured in the comma-suffix arc: dropping "dr" from +the suffix vocabulary flips "John Smith, Dr." from family="Smith" +suffix="Dr." to family="John Smith" — three individually correct +documented facts composing into a family-name loss. +Lives in. nameparser/_pipeline/_segment.py (the C1 decision) reading +the suffix sets through _vocab.is_wholly_suffix. +Reach for it when. Editing suffix membership for any word that +occurs after a comma in real data — check the structure flip, not +just the field. + ## MAKE-WRONG-STATES-UNREPRESENTABLE — the house meta-pattern Problem shape. A convention keeps being violated no matter how @@ -440,6 +461,11 @@ decline, not delete the test. its own skip set cannot fail. After changing any selection shape, verify the guard still REACHES the code it watches — assert the selected set is non-empty, or force-a-decision on its size. +- A differential corpus cannot evidence behavior keyed to + OUT-of-vocabulary shapes: it holds only names someone wrote + down, and an unrecognized word is by definition outside the + vocabulary — a green run over the corpus proves nothing about + such a rule. - Mind the optional-extra environment split: a local venv's incidental namedivider makes `if available` branches run PRESENT locally and ABSENT in CI, so a locally-green suite proves nothing diff --git a/docs/design/rules.md b/docs/design/rules.md index ecc7bb7..f7832cd 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -66,7 +66,12 @@ of it; it addresses or ranks the person. Most titles address by surname ("Mr. Johnson"), but a few — knighthoods, some clerical and courtesy titles — address by given name ("Sir John"). The library keeps a vocabulary of titles and, separately, of these given-name -titles. The dividing criterion for the given-name list is NOT +titles. What a TRAILING title-vocabulary word should do is +unresolved (#316): today "John Smith Prof." keeps Prof. a name word +while "Smith, Prof." reads it as a title — the two comma paths +disagree, and TITLES holding ordinary surnames (king, judge, +bishop) is what bars the blanket vocabulary-wins answer. +The dividing criterion for the given-name list is NOT "religious title" but whether the tradition retains a family name: renunciation abolishes the surname, so for Swami, Guru, Baba or Lama an empty family is the correct output (#346), while rabbi and @@ -92,12 +97,18 @@ H1. Rationale: a title normally addresses by surname, so a title H2. Rationale: before a name, an abbreviation is almost always a title — "Rev.", "Ing.", "Mag." — and no vocabulary can list every profession's abbreviations in every language. - A name-opening abbreviation — an unbroken run of two or more - letters ending in its one period — reads as a title even when - unlisted; a bare initial does not, and neither does anything - with interior periods, hyphens or digits. + An abbreviation opening the part of the name that carries the + given name — the whole name, or the part after a family comma — + reads as a title even when unlisted, provided it is an unbroken + run of two or more letters ending in its one period; a bare + initial does not, and neither does anything with interior + periods, hyphens or digits. Where it fires, the shape outranks + vocabulary: a period-marked opening word is a title even when + the word is suffix vocabulary. "Rev. John Smith" → title="Rev." "Xyz. John Smith" → title="Xyz." + "Smith, Major. John" → title="Major." + "Esq. Smith" → title="Esq." "J. Smith" → given="J." · boundary "J.R. Smith" → given="J.R." · boundary Accepted: the shape is only recognizable as an unbroken run of @@ -115,10 +126,12 @@ H2. Rationale: before a name, an abbreviation is almost always a H3. Rationale: compound titles are written as a run of title words, connectives included; a title word standing inside the name is just a name word. - Successive title words at the name's start chain into one - title; a title word elsewhere in the name does not. + Successive title words at the start of the part carrying the + given name chain into one title; a title word elsewhere in the + name does not. "Asst. Vice Chancellor John Smith" → title="Asst. Vice Chancellor" "Marquess of Bath" → title="Marquess of Bath" + "Morse, Det. Insp. Jane" → title="Det. Insp." "John Doctor Smith" → middle="Doctor" · boundary Accepted: before a family comma the pre-comma text is wholly the family name (C1), title words included. @@ -364,6 +377,8 @@ C1. Rationale: a credential run after the comma means the name is in "John Smith, V." → suffix="V." "John Smith, V." strict-comma-suffixes → family="John Smith" "Smith, PhD" → family="Smith" · boundary + "Smith, PhD" → suffix="PhD" deviates: #296 (today: suffix="") + "John Smith, LEED AP" → family="Smith" deviates: #291 (today: family="John Smith") history: decisions.md#C1 · implemented: nameparser/_pipeline/_segment.py C2. Rationale: text beyond the recognized comma parts should be diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 1c4c2bc..334d37c 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -60,9 +60,10 @@ def _set_roles(tokens: list[WorkToken], piece: tuple[int, ...], tokens[i] = dataclasses.replace(tokens[i], role=role) -# rules.md#H2: "a name-opening abbreviation — an unbroken run of two -# or more letters ending in its one period — reads as a title even -# when unlisted; a bare initial does not" (history: decisions.md#H2) +# rules.md#H2: "an abbreviation opening the part of the name that +# carries the given name — the whole name, or the part after a +# family comma — reads as a title even when unlisted" +# (history: decisions.md#H2) def _is_leading_title(piece: tuple[int, ...], ptags: frozenset[str], tokens: list[WorkToken]) -> bool: if _is_title_piece(piece, ptags, tokens): diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 6688be9..232cba0 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -54,8 +54,9 @@ class BoundJoin(IntEnum): STRICT = 3 # main segments (reserve_last=True: keep a family piece) -# rules.md#H3: "successive title words at the name's start chain into -# one title; a title word elsewhere in the name does not" +# rules.md#H3: "successive title words at the start of the part +# carrying the given name chain into one title; a title word +# elsewhere in the name does not" def _is_title_piece(piece: Sequence[int], ptags: Set[str], tokens: Sequence[WorkToken]) -> bool: if "title" in ptags: From 5f5859894e17b0e06ef82c2eb22bd958460c15f1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 22:18:57 -0700 Subject: [PATCH 35/40] docs(rules): topic-review amendments from the 2.0-vocabulary session (primary source) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comma-suffix arc gains its true origin (#291 filed 2026-07-26) and the missing earlier Declined -- splitting multi-word entries into single words, declined on the name-swallowing measurements in the issue body, which is why the seven entries were removed rather than split. New Excluded block for the eight dead multi-word entries (with the chargé/charge split and its precedent); 'born' recorded as a never-shipped drafting invention; the never-given ENCODING and freeze rationales (#293) land beside the particle curation block. S Background states the per-word matching truth the docs assumed everywhere. FACADE-CONTRACT closes the stays-warning- free misreading; FOLDED_TAG gains its consumer-side strip (Parser.revise, with the measured reordering hazard); two new mechanisms from the session's fixes (WARN-AT-THE-CALLER, LEGACY-STATE-SIGNATURE); C1's #296 marker gains the title-field note so a future measurement doesn't misread it as stale. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 40 +++++++++++++++++++++++++++++++++++--- docs/design/mechanisms.md | 41 ++++++++++++++++++++++++++++++++++++--- docs/design/rules.md | 9 ++++++++- 3 files changed, 83 insertions(+), 7 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index d2e3700..67a498b 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -303,6 +303,13 @@ question): - mc, ste — measured misparses ("Mc Donald" → given "Mc"), tracked in #360; st is inert at the head because TITLES claims it first; mac must stay ambiguous because Mac is a real given name. +- Encoding rationale (#293, predating #360's membership questions): + the data layer stores the NEVER-GIVEN set and derives the + ambiguous one, because that is safe-by-default for new particles + — a one-place addition — and the v1 shim translates by + one-directional complement. And the constants are FROZEN + specifically to kill the cached-Lexicon.default()-vs-fresh- + Constants desync that runtime module-constant mutation caused. - Load-bearing dependency: TITLES ∩ ambiguous == {do, freiherr, st} is what keeps the particle-or-given ambiguity emitter reachable at all; moving all three would make it dead code, @@ -335,6 +342,18 @@ algebra): defends. Deliberately no changed-parse count — the count is a property of the measuring grid, not of the code. +Excluded (multi-word vocabulary entries — every set matches one +written word, so a multi-word entry is silently inert and now warns +at configuration): + +- Eight entries shipped unmatchable from 2013 to 2.0. chargé + d'affaires was SPLIT into the chainable chargé + d'affaires, plus + unaccented charge (the attaché/attache precedent, Derek's call); + leed ap, nicet i–iv and psm i/ii were REMOVED rather than split, + on the name-swallowing measurements recorded in #291 (see the + comma-suffix-arc Declined entry). The multi-word UserWarning is + this story's enforcement. + Excluded (Lexicon.honorific_tails — a glued tail peels only if it could never end a name; per-entry reasons live in nameparser/config/suffixes.py's vetting block): @@ -382,7 +401,11 @@ silently gets no order. Excluded (MAIDEN_MARKERS, per nameparser/config/maiden_markers.py): - Polish "z domu" — a two-token marker; pending the multi-token - matching decision. + matching decision, tracked in #291 since 2026-07-27. +- "born" — never shipped: a release-log drafting invention, caught + by the 2.0 milestone audit and corrected (5ccf9f3). Recorded so + nobody "restores" it; if ever proposed for real, Max Born is the + counterexample to analyze. - Scandinavian "f." — collides with the initial F.; only the full participles (født/fødd/född) are safe. Czech masculine "rozený" awaits the same vetting. @@ -611,9 +634,12 @@ Declined: worse than a catch-all's honest breadth, and the gate reads the same either way. -### comma-suffix-arc — #291/#296/#316 (2026-07-30 → 2026-08-01) +### comma-suffix-arc — #291/#296/#316 (2026-07-26 → 2026-08-01) -Intent for #291 and #296 is settled by the approved bundle spec but +#291 was filed 2026-07-26 out of the 2.0 vocabulary cleanup, with +its decline-by-measurement evidence in the issue body; "z domu" +folded into it 2026-07-27 (see Excluded, MAIDEN_MARKERS). Intent +for #291 and #296 is settled by the approved bundle spec but UNSHIPPED — rules.md carries both as deviates: markers on C1. The arc's bookkeeping: #291/#296 moved milestone v2.1 → v2.2 on 2026-08-01 (with #289/#293); the 2026-07-30 design doc was amended @@ -623,6 +649,14 @@ same day, carrying the esq cleanup as a Related section. Declined: +- Splitting the dead multi-word suffix entries into single-word + entries (2026-07-26, the evidence in #291's body) — measured: + "Smith, A.P." has the suffix steal the given initials; "John + Leed" and "Mary Nicet" lose family names; and the period-gate + escape is equivalent to removal because nobody writes "L.E.E.D.". + This decline is why the seven removable entries were REMOVED in + 2.0 rather than split, and it is the missing history behind C1's + #291 marker. - The trailing-abbreviation structural fallback — with a measurement LIMIT rather than a measurement: the differential corpora structurally cannot evidence it, because they hold only diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 4de13b3..0cb201c 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -49,7 +49,11 @@ How it works. Reordering the token tuple would break span math and reintroduce the #100 family. Parse state stays in string order; only the view reorders (rule R1, rule O3's render clause). Lives in. nameparser/_types.py (FOLDED_TAG, the family view), -nameparser/_pipeline/_post_rules.py (the one producer today). +nameparser/_pipeline/_post_rules.py (the one producer today), and +one deliberate CONSUMER-side strip: Parser.revise removes the tag +from harvested tokens (a revised value must not inherit fold +ordering) — losing that strip is this mechanism's measured hazard, +a family rendering "García Gabriel Márquez". Reach for it when. A new rule needs "X renders before Y" and you are tempted to swap tokens. Don't swap. Tag. @@ -297,6 +301,34 @@ Reach for it when. Editing suffix membership for any word that occurs after a comma in real data — check the structure flip, not just the field. +## WARN-AT-THE-CALLER — walk out of the library, don't count frames + +Problem shape. A warning should point at the caller's code, but the +entry depth varies (constructor, add(), unpickle, +dataclasses.replace, the shim snapshot), so any fixed stacklevel +lands on library internals for most paths. +Contract statement. The warner walks the stack outward until the +first frame outside the library's own modules and warns there, +instead of counting frames. +Lives in. nameparser/_lexicon.py (_warn_dead_entry), and the +related but distinct per-read-location choice is in the #293/#354 +bridge (decisions.md#3-0-reevaluations). +Reach for it when. Adding any warning reachable through more than +one public entry point. + +## LEGACY-STATE-SIGNATURE — subtract legacy defaults only as a complete set + +Problem shape. A retired default rides in on old pickles, but a +user may have deliberately re-added one of the same entries. +Contract statement. Known-dead legacy defaults are subtracted from +restored state only when the state carries ALL of them — the +complete pre-retirement signature — so real legacy blobs clean up +silently while a deliberate single re-add survives round-trips. +Lives in. nameparser/_lexicon.py / _config_shim.py +(_LEGACY_DEAD_ENTRIES). +Reach for it when. Retiring any default vocabulary entry that +existing pickles may carry. + ## MAKE-WRONG-STATES-UNREPRESENTABLE — the house meta-pattern Problem shape. A convention keeps being violated no matter how @@ -335,7 +367,7 @@ Lives in. nameparser/locales/ (packs), nameparser/_parser.py Reach for it when. A language fix wants an if-statement — make it vocabulary or policy in a pack instead. -## FACADE-CONTRACT — HumanName wraps the core, warning-free v1 keeps working +## FACADE-CONTRACT — HumanName wraps the core; 1.4-warning-free code keeps working Problem shape. Where does v1-compatibility behavior live, and what may it do? @@ -343,7 +375,10 @@ Contract statement. HumanName is a mutable facade over the immutable core: code that runs warning-free on 1.4 keeps working with identical results through 2.x, via validating setters, dirty-tracked re-parses, and pickle round-trips — and the facade never calls the -v1 parsing hooks it still carries. +v1 parsing hooks it still carries. "Warning-free" describes the +code's 1.4 behavior, not a promise it STAYS warning-free: 2.x adds +deliberate warnings (the multi-word-vocabulary and field-assignment +ones), each release-log-classified, always with identical results. Lives in. nameparser/_facade.py; v1 import paths preserved by nameparser/parser.py and nameparser/config/. Reach for it when. A core change needs a v1-visible behavior — diff --git a/docs/design/rules.md b/docs/design/rules.md index f7832cd..cbbb262 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -205,7 +205,11 @@ P4. Rationale: a particle links forward from inside a name; at the Background: what follows a name is one of two different things — generational suffixes (Jr., III), which attach to the name itself, -and credentials (PhD, MD, MBA), which are earned attachments. CLDR +and credentials (PhD, MD, MBA), which are earned attachments. All +vocabulary sets match one written word at a time: a multi-word +entry can never match anything and is warned about at +configuration (the eight that shipped dead for years are the +Excluded story in decisions.md). CLDR personNames keeps them as separate fields (`generation`, `credentials`) and formats them differently; this library currently reports both in one `suffix` field, a merge #326 examines. The @@ -378,6 +382,9 @@ C1. Rationale: a credential run after the comma means the name is in "John Smith, V." strict-comma-suffixes → family="John Smith" "Smith, PhD" → family="Smith" · boundary "Smith, PhD" → suffix="PhD" deviates: #296 (today: suffix="") + (Today PhD lands in TITLE — the #316 trailing-title tangle + crossing C1; the marker tracks the suffix field only, so a + measured title="PhD" does not mean the marker is stale.) "John Smith, LEED AP" → family="Smith" deviates: #291 (today: family="John Smith") history: decisions.md#C1 · implemented: nameparser/_pipeline/_segment.py From b3cc3772988f05de9da84d7ded52794a98877ed5 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 22:31:44 -0700 Subject: [PATCH 36/40] docs(rules): topic-review amendments from the July-arc session (primary source) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two Background falsifications corrected: the H criterion was conflating list membership (how the title ADDRESSES -- the Arabic honorifics qualify while fully retaining family names) with the separate #346 empty-family question (surname retention); the N Background still called the typographic pairs caller-opt-in two rounds after #273 shipped them as defaults. FACADE-CONTRACT's 'identical results' gains the RFC's own release-log-classified carve-out (Kennedy-nickname measured). New decisions sections from the arc: N2's cross-pair mis-extraction history with its Declined offset-filter, ma-do, deviates-registry (option C -- why packs carry no callables, now cross-linked from LOCALE-PACKS and FORCE-A-DECISION), normalization-fold (the deliberate lower/casefold asymmetry), render-default (the declined lossless née template), and P3's Google Code provenance with the GC-namespace note on legacy-rule-numbers. M1's canonicalization bullet gains its date, origin, and the through-patches ruling; C1 gains the i/v blast radius. Excluded: curly single quotes; the 2026-07-19 transliteration deferrals (Arabic, Greek, Ottoman, Hebrew, sri/shri). N2 states the dangling-suppression filter A1 depends on; R Background names the default view. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 122 ++++++++++++++++++++++++++++++++++++-- docs/design/mechanisms.md | 11 +++- docs/design/rules.md | 31 ++++++---- 3 files changed, 146 insertions(+), 18 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 67a498b..3d8864f 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -49,7 +49,10 @@ Entry conventions: Before rules.md, the post_rules stage docstring numbered its rules locally, and historical issue comments (#359, #364, #365, #367) use those numbers. The mapping: rule 1 → H1, rule 1b → P1, rule 2 → O1, -rule 3 → O2, rule 4 → O3. +rule 3 → O2, rule 4 → O3. Older still: bare "#11"-style citations +from v1 comments live in the GOOGLE CODE namespace, not GitHub — +GC issue 11 is decisions.md#P3's provenance — so a v1-era number +that doesn't fit its GitHub issue is probably a GC number. ### P1 — lone particle fold @@ -130,6 +133,76 @@ head-peel question). lives in assignment rather than grouping because that is where the piece count is settled. +### N2 — same-character quotes and shared-character conventions + +- 2026-07-13 #273 (7ee6e3a) — the typographic delimiter pairs + shipped with a per-pair full-text scan; three independent review + agents broke it the same day: with two shared-character + conventions genuinely present, the earlier-sorted pair stole the + close and the legitimate match dropped silently, zero ambiguity. +- 2026-07-19 (05fe693) — the interleaved leftmost-match restructure + replaced it (an altitude review called it the right depth); the + author's own first fix — post-hoc offset filtering of unbalanced + candidates alone — was superseded within the day and survives + only as the bulk-recorded dangler filter rules.md#N2 now names. A + cached delimiter-charset prescreen landed in the same commit, + measured: no-delimiter names parse faster than the pre-#273 + baseline. + +Declined: + +- The offset-filter as the whole fix — it repaired the reported + symptom (spurious unbalanced flags) while leaving the stolen- + close mis-extraction in place. + +### ma-do — ambiguous acronyms by decision + +- 2026-07-17 (M12, Derek-approved) — ma and do joined the ambiguous + acronym set because both are common surnames; the periods gate is + what keeps "Jack Ma" intact. Documented side effect: parenthesized + bare "(MA)"/"(DO)" no longer escape to suffix as in 1.x. + +### deviates-registry — packs stay pure data (option C) + +- 2026-07-18 (d4aaafa; the DEVIATES design note) — a `deviates` + field ON Locale was rejected: callables break value equality and + pickle-by-reference, and DEVIATES is acceptance metadata, not + runtime behavior (revisit only if a runtime consumer like + explain() appears). The REGISTRY became the contract instead: + test_registry_is_the_pack_contract fails structurally unless + every registered pack ships DEVIATES and its rotator list. This + is why LOCALE-PACKS-PURE-DATA can promise "no code paths of + their own" and mean it. + +### normalization-fold — lower(), not casefold(); comparison folds harder + +- 2026-07-17/19 — vocabulary storage normalizes with lower(), NOT + casefold(): casefold mutated stored spellings (κος→κοσ, + großfürst→grossfürst) while lower() keeps authored forms; the + accepted cost, pinned deliberately, is that ASCII-SS GROSSFÜRST + no longer matches. The paired half: comparison_key()/matches() + use casefold() ON PURPOSE — comparison is the one surface that + wants aggressive folding, a documented 1.4 deviation (ß and + final-sigma forms compare equal). The storage-vs-comparison + split is deliberate asymmetry; do not "fix" it symmetric. + +### render-default — the default view's format + +- 2026-07-19 (bf1141c) — the default spec is + '{title} {given} "{nickname}" {middle} {family} ({maiden}) + {suffix}'. Declined the same day: a née-template maiden + ('née {maiden}') that round-tripped LOSSLESSLY (née is a maiden + marker), rejected on presentation grounds. Accepted cost of the + shipped form: a parenthesized maiden re-parses as a nickname. + +### P3 — connectives + +- Provenance: the single-letter-connective guard is v1's fix for + Google Code issue 11 ("john e smith", 2014, commit 33676c9) — + the "#11" citations that circulated pointed at a 2014 GitHub + accident, not the real source. Recorded so the archaeology stays + done. + ### phd-merge — the "Ph. D." split - 2026-07 (v2 core, PR #288; recorded plan deviation #1 of the core @@ -342,6 +415,32 @@ algebra): defends. Deliberately no changed-parse count — the count is a property of the measuring grid, not of the code. +Excluded (DEFAULT_NICKNAME_DELIMITERS): + +- Curly single quotes ('‘','’') are deliberately absent from the + default pairs: U+2019 is the typographic apostrophe (O'Connor in + curly type), so shipping the pair would eat real names. #273's + own proposal excluded them; pinned by the + curly_apostrophe_stays_literal case. A sweeper "completing the + typographic set" ships the regression. + +Excluded (given-name titles and post-nominals — the 2026-07-19 +transliteration deferrals, each living in a data-module comment +until now): + +- Arabic bare سيد/شيخ/أمير/سلطان — given-name collisions (Sayyid, + Shaikha, Amir, Sultan); the honorific forms with the article + (الدكتور, الشيخ) ship instead. +- The abbreviation د. — edge-period normalization leaves bare د, + the single-letter trap; Greek bare κ deferred on the same + grounds. +- Ottoman post-nominals باشا/بك/أفندي — surname collisions. +- Hebrew bare רב (an ordinary word, "many") and בר (Bar is a + common modern Israeli given name) — deferred, #269's territory. +- Latin sri/shri deliberately absent while Devanagari श्री ships: + the transliteration collides (Sri Mulyani), the native script + cannot. + Excluded (multi-word vocabulary entries — every set matches one written word, so a multi-word entry is silently inert and now warns at configuration): @@ -415,6 +514,10 @@ Excluded (MAIDEN_MARKERS, per nameparser/config/maiden_markers.py): - 2026-07 (v2 core, PR #288) — v1 parity: only the second segment decides (v1 parser.py:1318), and ">1 word before the first comma" is v1's guard, which is why "Smith, PhD" keeps the listing form. + The strict/lenient knob's blast radius on default vocabulary is + exactly the single-letter Roman numerals i and v — the only + initial-shaped members of the shipped suffix words — which is + what makes rules.md#C1's "V." examples the canonical pair. - 2026-07 (plan deviation #3, recorded) — the decision is definitionally vocabulary-dependent: there is no way to recognize a credential run without consulting the suffix word lists, so the @@ -467,10 +570,19 @@ Declined: - 2026-07-03 (maiden-bucket design, landed via the v2 core, PR #288) — the maiden reading of a delimiter pair is opt-in because enclosure conventions genuinely vary; there is no default pair. -- 2026-07 — bucket overlap is canonicalized before parsing: a pair - listed for maiden is dropped from the effective nickname set - (maiden wins). The v1 facade restores v1's nickname-wins reading - by pre-subtracting on its side. +- 2026-07-19 (cc7063f; Derek's proposal out of a docs-review pain + point — routing parens to maiden used to require editing both + buckets in tandem, and the nickname default's contents were not + discoverable) — bucket overlap is canonicalized before parsing: a + pair listed for maiden is dropped from the effective nickname set + (maiden wins), and the public DEFAULT_NICKNAME_DELIMITERS + constant landed with it. The v1 facade restores v1's + nickname-wins reading by pre-subtracting on its side. Weighed and + ruled the same day: maiden-wins applies THROUGH apply_patch too — + a pack's maiden pair silently removes a user's unrelated explicit + nickname pair; warn-on-removal was considered (a silent-failure + review flagged the site) and rejected, the silence ruled + intended. - 2026-08-04 #329 (PR #331) — the marker word inside a delimited clause is dropped from a multi-word clause during grouping; the extraction itself keeps the whole enclosed span, so nothing is diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 0cb201c..6a8b9f2 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -372,8 +372,10 @@ vocabulary or policy in a pack instead. Problem shape. Where does v1-compatibility behavior live, and what may it do? Contract statement. HumanName is a mutable facade over the immutable -core: code that runs warning-free on 1.4 keeps working with -identical results through 2.x, via validating setters, dirty-tracked +core: code that runs warning-free on 1.4 keeps working through 2.x +with identical results EXCEPT release-log-classified fixes, each +verified in the differential ledger (the RFC's compatibility promise +carries the same carve-out) — via validating setters, dirty-tracked re-parses, and pickle round-trips — and the facade never calls the v1 parsing hooks it still carries. "Warning-free" describes the code's 1.4 behavior, not a promise it STAYS warning-free: 2.x adds @@ -452,7 +454,10 @@ silently inherit a default, a local table's key set is asserted equal to the population, so growth fails the suite until someone decides — against a local table, not the constant under test. Exemplar: tests/v2/pipeline/test_vocab.py's per-script initials -check; reused for _CORPUS_FLOORS in tools/differential/compare.py. +check; reused for _CORPUS_FLOORS in tools/differential/compare.py, +and for the pack registry (test_registry_is_the_pack_contract: +every registered pack must ship DEVIATES and its rotator list or +the suite fails structurally — decisions.md#deviates-registry). Known gap it exposes: DEFAULT_SCRIPT_ORDERS has no such guard. ### SELF-EXPIRING-GUARD — a decline keyed to a measured defect diff --git a/docs/design/rules.md b/docs/design/rules.md index cbbb262..f6a7c04 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -71,13 +71,18 @@ unresolved (#316): today "John Smith Prof." keeps Prof. a name word while "Smith, Prof." reads it as a title — the two comma paths disagree, and TITLES holding ordinary surnames (king, judge, bishop) is what bars the blanket vocabulary-wins answer. -The dividing criterion for the given-name list is NOT -"religious title" but whether the tradition retains a family name: -renunciation abolishes the surname, so for Swami, Guru, Baba or -Lama an empty family is the correct output (#346), while rabbi and -imam traditions keep surnames — "Rabbi Cohen" addresses by title -and keeps family "Cohen". Sweeping all religious titles into the -given-name list would break the latter. +Two criteria govern two different questions here. +Membership in the given-name-title list follows HOW THE TITLE +ADDRESSES: a title that precedes and addresses by the given name +belongs (Sir, Sheikh, the Arabic honorifics الدكتور/الشيخ — which +qualify even though those traditions fully retain family names). +Whether an EMPTY FAMILY is correct output is the separate question, +governed by surname retention: renunciation abolishes the surname, +so for Swami, Guru, Baba or Lama family="" is right (#346), while +rabbi and imam traditions keep surnames — "Rabbi Cohen" addresses +by title and keeps family "Cohen". Conflating the two criteria +either ejects the Arabic entries or sweeps in titles that break +"Rabbi Cohen". H1. Rationale: a title normally addresses by surname, so a title followed by a single name word usually names the family; but a @@ -285,11 +290,14 @@ N2. Rationale: only a mark standing at word boundaries is quoting; at a word start and closes only at a word end, so an apostrophe inside or at the end of a word is literal. Between conventions that share a character, position in the text decides: the - leftmost valid opener wins. + leftmost valid opener wins — and a dangling-open report is + suppressed where its character sits inside another pair's + successful match, being literal content there rather than an + imbalance (A1 depends on that filter). "Sean O'Connor" → family="O'Connor" "Hans „Erster“ und “Zweiter” Müller" → nickname="Erster Zweiter" "Mari' Aube'" → family="Aube'" · boundary - implemented: nameparser/_pipeline/_extract.py + history: decisions.md#N2 · implemented: nameparser/_pipeline/_extract.py N3. Rationale: a person set down as a nickname plus one name word is being identified by surname. @@ -630,7 +638,10 @@ A1. Rationale: a caller can only act on doubt that is reported. Background: parsing produces words with roles; every string a caller reads is assembled from those words on request. Nothing about rendering changes the parse, and nothing about reading a field -mutates anything. +mutates anything. The default view renders +'{title} {given} "{nickname}" {middle} {family} ({maiden}) +{suffix}' — the choice and its declined née-template alternative +are decisions.md#render-default. R1. Rationale: a field is a way of reading the parse, not a stored string. From 436ff68d6fcf0cc823715f661c5b6cb6b4d03747 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 22:44:32 -0700 Subject: [PATCH 37/40] docs(rules): topic-review additions from the rc1 session (primary source) Zero falsifications this round; all additive. New rule A2 (content- free input parses to the empty name; born-empty ambiguities survive, with the v1 contrast and the review-fix history). normalization-fold gains the fixed-point requirement and the _title_key corollary (the shipped-then-fixed silently-inert key). New decisions: given-name-titles' twice-declined validation (the 'sir and dame' counterexample), N2's why-only-the-apostrophe (Derek's double-quote distinguishing test), and the two-corpus provenance (2026-07-24: tracker-harvested corpus because v1 test banks are structurally blind to 2.0 additions). New mechanism: AMBIGUITY-AT-THE-DECISION-SITE (every deciding stage carries an emitter; a branch that changes nothing is not a decision). Two verification field notes: guard-the-family parametrization, and growth-guard calibration. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 43 ++++++++++++++++++++++++++++++- docs/design/mechanisms.md | 31 ++++++++++++++++++++++ docs/design/rules.md | 10 +++++++ nameparser/_pipeline/_assemble.py | 4 +++ 4 files changed, 87 insertions(+), 1 deletion(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 3d8864f..4cbfe7d 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -44,6 +44,14 @@ Entry conventions: rules.md lists under the rule's `Accepted:` consequences; the two link by rule ID. +### A2 — the empty name + +- 2026-07 (v2 core, PR #288) — v1 kept parse(".") as first="."; + 2.0 empties content-free input instead. The born-empty-ambiguity + survival was a review fix in the rc1 arc: an unbalanced-delimiter + report must outlive the emptying, or malformed input becomes + indistinguishable from blank input. + ### legacy-rule-numbers — the old docstring numbering Before rules.md, the post_rules stage docstring numbered its rules @@ -144,7 +152,12 @@ head-peel question). replaced it (an altitude review called it the right depth); the author's own first fix — post-hoc offset filtering of unbalanced candidates alone — was superseded within the day and survives - only as the bulk-recorded dangler filter rules.md#N2 now names. A + only as the bulk-recorded dangler filter rules.md#N2 now names. + The word-internal carve-out covers ONLY the straight apostrophe + because it is the one delimiter character that occurs mid- and + end-of-word in real names; Derek's distinguishing test: with + double quotes the same shape ("Mari\" Aube\"") is genuinely + ambiguous, while Mari' Aube' is not. A cached delimiter-charset prescreen landed in the same commit, measured: no-delimiter names parse faster than the pre-#273 baseline. @@ -185,6 +198,13 @@ Declined: wants aggressive folding, a documented 1.4 deviation (ß and final-sigma forms compare equal). The storage-vs-comparison split is deliberate asymmetry; do not "fix" it symmetric. +- 2026-07 (rc1 arc) — the fold must reach a FIXED POINT, and + anything built on it must converge too: _title_key joined + per-word folds and kept empty slots, so given_name_titles with a + foldable word stored a key match-time could never rebuild — + silently inert, and pickle round-trips then rejected the state. + Shipped in the rc, caught in review, fixed by dropping words + that fold away. ### render-default — the default view's format @@ -203,6 +223,19 @@ Declined: accident, not the real source. Recorded so the archaeology stays done. +### given-name-titles — deliberately unvalidated + +Declined (rc1 arc; the full argument is AGENTS.md's gotcha): + +- Validating given_name_titles against titles, twice: the + whole-entry check rejected legitimate multi-word entries; the + per-word check rejected "sir and dame" (a conjunction inside a + title run is itself a TITLE token, so the key is matchable while + its middle word lives in conjunctions). No static relation over + the vocabulary sets decides reachability, and an unreachable + entry is inert — each guard cost a working configuration to + forbid a condition that costs nothing. + ### phd-merge — the "Ph. D." split - 2026-07 (v2 core, PR #288; recorded plan deviation #1 of the core @@ -649,6 +682,14 @@ same rotation/name_order interaction as O1. Harvested 2026-08-15 from the release-arc session; measurements are that session's, spot-checked at landing. +- 2026-07-24 (rc1 arc, predating this section's window) — the + TWO-CORPUS design: corpus_issues.jsonl (198 tracker-harvested + names, 166 absent from the v1-test-bank corpus) exists because + v1 test banks are structurally blind to anything 2.0 added; + compare.py globs corpus*.jsonl and fails loudly on none. Its + first catch was the leading-credential case ("Ph. D. John + Smith" → suffix). + - 2026-08-05 #332 — ledger field vocabulary is Role's names, not the facade's: canonicalizing to first/last would have put an eighth place naming roles differently from Role inside the durable diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 6a8b9f2..9e5a4d8 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -329,6 +329,26 @@ Lives in. nameparser/_lexicon.py / _config_shim.py Reach for it when. Retiring any default vocabulary entry that existing pickles may carry. +## AMBIGUITY-AT-THE-DECISION-SITE — emit where the branch is taken + +Problem shape. An ambiguity report should fire exactly when the +parse chose between live readings — no more, no less — and the +choosing happens in code, not in vocabulary. +Contract statement. Emit at the site that takes the branch, not +where an ambiguous tag sits; when a fork's two branches are decided +in different stages, EVERY deciding stage carries an emitter; and a +branch that runs but changes nothing is not a decision and must not +report. +How it works. PARTICLE_OR_GIVEN fires from assignment for a lone +leading particle and from grouping when a title shifts it off the +front — for two years only the first site emitted. And keying on +"the code got here" instead of "the outcome differed" once reported +a fork for all 39 ambiguous particles on "Dr. Van Jr.". +Lives in. The AmbiguityKind emitters across _pipeline/ (rule A1 is +the observable contract). +Reach for it when. Adding any ambiguous vocabulary or any new fork +— count the deciding sites, then count the emitters. + ## MAKE-WRONG-STATES-UNREPRESENTABLE — the house meta-pattern Problem shape. A convention keeps being violated no matter how @@ -506,6 +526,17 @@ decline, not delete the test. down, and an unrecognized word is by definition outside the vocabulary — a green run over the corpus proves nothing about such a rule. +- Guard the whole family, parametrize over it: a defect on one of + N parallel entry points hides behind a per-example test — three + times in one session (a guard on one class of two, a decode hint + on 3 of 5 entry points, a sync roster missing 4 copies) — and a + {class}×{field}×{bad-value} parametrization is what caught each. +- A growth guard needs calibration, not just existence: benchmark + guards that compare n vs 4n catch the quadratic the absolute-time + tests are blind to, but calibrate against the WEAKEST signal you + must detect and confirm a planted regression fails across + repeated runs, not once — a stochastic check "verified" on one + sample verifies nothing. - Mind the optional-extra environment split: a local venv's incidental namedivider makes `if available` branches run PRESENT locally and ABSENT in CI, so a locally-green suite proves nothing diff --git a/docs/design/rules.md b/docs/design/rules.md index f6a7c04..b4aaaee 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -633,6 +633,16 @@ A1. Rationale: a caller can only act on doubt that is reported. so no example line.) implemented: nameparser/_pipeline/_state.py +A2. Rationale: an input with no name content names nobody, and + saying so beats inventing fields from punctuation. + A name with no name content parses to the empty name — every + field empty, false as a boolean — while ambiguities born from + its punctuation survive on the empty result. + ".," → family="" + "(" → ambiguities=("unbalanced-delimiter",) + "John . Smith" → family="Smith" · boundary + history: decisions.md#A2 · implemented: nameparser/_pipeline/_assemble.py + ## Rendering & views (R) Background: parsing produces words with roles; every string a caller diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py index a29f366..7f78bd7 100644 --- a/nameparser/_pipeline/_assemble.py +++ b/nameparser/_pipeline/_assemble.py @@ -34,6 +34,10 @@ def assemble(state: ParseState) -> ParsedName: # name: a bare '.' or '- -' is not a person. v1 kept such input # (parse('.') -> first '.'); 2.0 empties it so bool() stays an # honest "did I get a name?" check. isalnum() is Unicode-aware, so + # rules.md#A2: "a name with no name content parses to the empty + # name — every field empty, false as a boolean — while + # ambiguities born from its punctuation survive on the empty + # result" (history: decisions.md#A2) # every real name in any script has content and only pure # punctuation/symbols empty out. (Embedded junk in a name with # content -- 'John . Smith' -- is left alone: that parse is truthy, From ba007b0461abf7e36f66860c5cdccbe841d2ae10 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 22:56:24 -0700 Subject: [PATCH 38/40] docs(rules): topic-review amendments from the issue-filing session (primary source) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P3 falsified and corrected: the Latin-capital initial veto on single-letter connectives is now stated ('Jose E Maria Santos' pinned) with the script asymmetry as an Accepted consequence — Cyrillic blessed by #267's closure, the Latin half never separately adjudicated (recorded as such). C section gains which commas count (U+060C and U+FF0C segment, U+3001 does not — #265) with executable examples. Dates disciplined per the convention: M2 and O2 now distinguish design date from issue and landing; T1 credits #266 for bidi existing at all; O4 gains its #270-body provenance and the free-form-tuples Declined. W1 records the opt-in→default-on delta from the filed proposal and the JMnedict Declined (staleness, no frequency data, licensing). The stale Open-on-closed-#270 blocks in O1/O2 become flagged status notes; nee's silently-resolved risk gets its contrast entry; no_vowels' removal is recorded. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 67 ++++++++++++++++++++++++++++++---- docs/design/rules.md | 20 ++++++++-- nameparser/_pipeline/_group.py | 4 +- 3 files changed, 79 insertions(+), 12 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 4cbfe7d..fa89443 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -122,7 +122,8 @@ which particles count as never-given. ### M2 — the maiden-marker rule -- 2026-07-03 #274 (v2 core, PR #288) — the marker takes everything +- 2026-07-03 (maiden-bucket design; #274 filed 2026-07-07, landed + in the v2 core, PR #288) — the marker takes everything after it up to a trailing suffix, greedily: "née Jones Smith" is a two-word maiden name, matching how the marker is actually used in running text. The marker itself is dropped as structural, like @@ -217,6 +218,13 @@ Declined: ### P3 — connectives +- 2026-07-30 #267 — the four-word single-letter asymmetry: a bare + Latin capital connective is vetoed as an initial while a Cyrillic + capital joins. #267's closure ("v2.0 behaves this way by + default … verify it was the right call") showed only the + Cyrillic half; the Latin-capital veto was never separately + adjudicated, which rules.md#P3 now records as an Accepted + consequence pending anyone caring. - Provenance: the single-letter-connective guard is v1's fix for Google Code issue 11 ("john e smith", 2014, commit 33676c9) — the "#11" citations that circulated pointed at a 2014 GitHub @@ -255,8 +263,18 @@ Declined (rc1 arc; the full argument is AGENTS.md's gotcha): given="Khai", middle="Thị Minh", family="Nguyễn". This is why three order constants exist rather than two. +- Provenance: the three-constants argument originates in #270's + own body (2026-07-07): "A boolean family_name_first flag was + considered and rejected: Vietnamese order is + [family][middle][given]." The 2026-08-07 #146 measurement below + is the verification; #270 is the origin. + Declined: +- Free-form order tuples (('last','middle','first')-style, #270's + original draft shape) — the shipped design is exported order + constants with tuple rejection at construction (rule D2's + [bad-name-order] example is the pinned message). - middle_as_family as the way to suppress the middle slot for Vietnamese (2026-08-07 #146) — measured: it merges the middles into the family, giving family="Thị Minh Nguyễn" for "Nguyễn Thị @@ -340,7 +358,11 @@ disagree today). census surname list is closed, hangul is self-selecting (a hangul entry can only match hangul text), and being unsplit is recoverable while a wrong split is not — which is also why an - unrecognized name stays whole. + unrecognized name stays whole. The filed proposal (#271, + 2026-07-07) asked for OPT-IN segmentation for Korean too, "like + all localization"; default-on is the later refinement, and the + census/self-selecting argument above is what justified promoting + Korean past the blanket opt-in stance. - 2026-07-29 #272 (the ja amendment; shipped in 2.1.0 via PR #297) — Han division is opt-in per language pack because Han text does not identify its language @@ -349,6 +371,16 @@ disagree today). compose mechanically: a listed surname is a dictionary certainty and wins. +Declined: + +- JMnedict as bundled segmentation data (#272's body, 2026-07-07) + — the pip packaging (jamdict-data) was last compiled 2021-04; + JMnedict carries no frequency data, so it cannot resolve the + ambiguous 2+2/3+1 splits that motivate a segmenter at all; and + CC BY-SA bundling raises questions the LGPL core avoids by + delegating to namedivider (MIT). The kind of decline someone + re-proposes in two years. + - 2026-08 — zh and ja packs are corpus ALTERNATIVES, one per corpus; stacking them (parser_for(ZH, JA, segmenter=...)) is for genuinely mixed data that accepts the trade: a listed Chinese @@ -534,6 +566,12 @@ Excluded (MAIDEN_MARKERS, per nameparser/config/maiden_markers.py): - Polish "z domu" — a two-token marker; pending the multi-token matching decision, tracked in #291 since 2026-07-27. +- Contrast entry — unaccented "nee" SHIPS as a marker despite + #274 flagging "is nee safe as a default (it's also a rare + surname)" as open; the question resolved silently with the + shipped set. Recorded here because the included risky member + deserves its analysis as much as the excluded ones; M1's (Nee) + boundary covers only the enclosure path, not this marker path. - "born" — never shipped: a release-log drafting invention, caught by the 2.0 milestone audit and corrected (5ccf9f3). Recorded so nobody "restores" it; if ever proposed for real, Max Born is the @@ -567,7 +605,9 @@ Excluded (MAIDEN_MARKERS, per nameparser/config/maiden_markers.py): ### T1 — separators, not joiners - 2026-07 (v2 core, PR #288) — v1's squash_emoji/squash_bidi - REMOVED the character and joined its neighbors ('A😀B' → 'AB'); + REMOVED the character and joined its neighbors. (v1.3.0 had no + bidi handling at all: squash_bidi entered late v1 via #266, + 2026-07-07, on the emoji precedent's shape.) ('A😀B' → 'AB'); v2 makes an ignorable character a separator ('A😀B' → 'A', 'B'). The unavoidable consequence of every part being an exact positioned piece of the input: with no rewriting stage, nothing @@ -663,19 +703,23 @@ without configuration. roles, which is faithful to v1 only under the default given-first order. -Open: [#270](https://github.com/derek73/python-nameparser/issues/270) -how the rotations interact with non-default name_order values. +- 2026-08-15 — the rotation × non-default name_order interaction + question rode #270, which closed 2026-07-28 with the order + constants and no recorded answer for the rotations; no successor + issue tracks it. Flagged at review: needs either a resolution + note or a live issue. ### O2 — Turkic rotation -- 2026-07-02 (landed in the v2 core, PR #288) — shape fixed at +- 2026-07-02 (Turkic design; landed in the v2 core, PR #288) — + shape fixed at exactly four name words (1 given + 2 middle + 1 marker), v1 parity; other shapes keep their positional reading even when that leaves the marker in a name field (see the rule's Accepted consequence). -Open: [#270](https://github.com/derek73/python-nameparser/issues/270) -same rotation/name_order interaction as O1. +- 2026-08-15 — same rotation/name_order interaction status as O1 + (#270 closed without a recorded answer; no successor issue). ### differential-ledger — tooling decisions (2.1.0 release arc) @@ -824,6 +868,13 @@ Declined: must merge where the render tag is applied, not where the role is assigned. +### removed-v1-surface + +- no_vowels: removed in 2.0 (#268, filed 2026-07-07, closed + 2026-07-28) — never consulted by any parser version, ASCII-only; + the facade carries no replacement because there was nothing to + replace. + ### 3-0-reevaluations — decisions shaped by the v1 shim Promoted 2026-08-15 from session memory (Derek's 2026-07-30 ask; diff --git a/docs/design/rules.md b/docs/design/rules.md index b4aaaee..7c32378 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -188,10 +188,19 @@ P3. Rationale: connective words ("y", "of the") bind name words into likely an initial than a connective. A recognized connective joins its neighbors into one name part, connective runs included — except a single-letter connective in - a three-word name, which stays a name word. + a three-word name, which stays a name word, and a single-letter + connective written as a bare Latin capital, which reads as an + initial and never joins. "Juan y Eva Garcia" → given="Juan y Eva" + "Jose E Maria Santos" → middle="E Maria" "Juan y Garcia" → middle="y" · boundary - implemented: nameparser/_pipeline/_group.py + Accepted: the initial veto is a LATIN shape — a Cyrillic + capital joins ("И".isupper() is true, so this is not a + Unicode-uppercase rule); #267's closure blessed the Cyrillic + side while the Latin-capital half was never separately + adjudicated. + "Хосе И Мария Сантос" → given="Хосе И Мария" + history: decisions.md#P3 · implemented: nameparser/_pipeline/_group.py P4. Rationale: a particle links forward from inside a name; at the very front there is no name yet to be inside. @@ -371,7 +380,10 @@ listing form "Family, Given" or trailing credentials "Name, PhD" — and which is meant can only be judged from what stands after the first comma. Recognizing a credential run is by nature a vocabulary judgment, so this is the one structural decision that consults the -suffix word lists. +suffix word lists. Which characters COUNT as the comma is part of +the rule: the Arabic comma (U+060C) and the fullwidth comma +(U+FF0C) both signal the listing form, while the ideographic comma +(U+3001) is not a name-structure comma at all (#265). C1. Rationale: a credential run after the comma means the name is in natural order with suffixes appended; anything else after the @@ -385,6 +397,8 @@ C1. Rationale: a credential run after the comma means the name is in even written like an initial ("V."), while strict mode vetoes initial-shaped words. "Smith, John" → family="Smith" + "سلمان، محمد" → family="سلمان" + "田中、太郎" → family="" "John Smith, PhD" → suffix="PhD" "John Smith, V." → suffix="V." "John Smith, V." strict-comma-suffixes → family="John Smith" diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 232cba0..1de5d7b 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -89,7 +89,9 @@ def _is_suffix_piece(piece: Sequence[int], ptags: Set[str], # rules.md#P3: "a recognized connective joins its neighbors into one # name part, connective runs included — except a single-letter -# connective in a three-word name, which stays a name word" +# connective in a three-word name, which stays a name word, and a +# single-letter connective written as a bare Latin capital, which +# reads as an initial and never joins" (history: decisions.md#P3) def _is_conj_piece(piece: Sequence[int], ptags: Set[str], tokens: Sequence[WorkToken]) -> bool: if "conjunction" in ptags: From 47cc545b0ec73abe1f84226f61833714b9f4cf35 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 23:03:55 -0700 Subject: [PATCH 39/40] docs(rules): link #383 and #384 from their flagged questions Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 24 +++++++++++++++++------- docs/design/rules.md | 3 +-- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index fa89443..68aa985 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -223,8 +223,12 @@ Declined: capital joins. #267's closure ("v2.0 behaves this way by default … verify it was the right call") showed only the Cyrillic half; the Latin-capital veto was never separately - adjudicated, which rules.md#P3 now records as an Accepted - consequence pending anyone caring. + adjudicated, which rules.md#P3 records as an Accepted + consequence. + +Open: +[#383](https://github.com/derek73/python-nameparser/issues/383) +should the Latin-capital veto stand (bless / drop / extend). - Provenance: the single-letter-connective guard is v1's fix for Google Code issue 11 ("john e smith", 2014, commit 33676c9) — the "#11" citations that circulated pointed at a 2014 GitHub @@ -705,9 +709,12 @@ without configuration. - 2026-08-15 — the rotation × non-default name_order interaction question rode #270, which closed 2026-07-28 with the order - constants and no recorded answer for the rotations; no successor - issue tracks it. Flagged at review: needs either a resolution - note or a live issue. + constants and no recorded answer for the rotations. + +Open: +[#384](https://github.com/derek73/python-nameparser/issues/384) +what the rotations should do under a non-default name_order (the +divergent measurement is in the issue). ### O2 — Turkic rotation @@ -718,8 +725,11 @@ without configuration. leaves the marker in a name field (see the rule's Accepted consequence). -- 2026-08-15 — same rotation/name_order interaction status as O1 - (#270 closed without a recorded answer; no successor issue). +- 2026-08-15 — same rotation/name_order interaction status as O1. + +Open: +[#384](https://github.com/derek73/python-nameparser/issues/384) +same question as O1. ### differential-ledger — tooling decisions (2.1.0 release arc) diff --git a/docs/design/rules.md b/docs/design/rules.md index 7c32378..a60d88c 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -197,8 +197,7 @@ P3. Rationale: connective words ("y", "of the") bind name words into Accepted: the initial veto is a LATIN shape — a Cyrillic capital joins ("И".isupper() is true, so this is not a Unicode-uppercase rule); #267's closure blessed the Cyrillic - side while the Latin-capital half was never separately - adjudicated. + side, and whether the Latin-capital half should stand is #383. "Хосе И Мария Сантос" → given="Хосе И Мария" history: decisions.md#P3 · implemented: nameparser/_pipeline/_group.py From 3176a5e6770c100a49f8dfa86a1962140bee4e32 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 15 Aug 2026 23:14:19 -0700 Subject: [PATCH 40/40] docs(rules): topic-review amendments from the 1.x stink-test session (primary source) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit normalization-fold's 'documented 1.4 deviation' inverted the history: v1.4 has no casefold anywhere (measured against the tag) — casefold is a classified 2.0 change, and the 1.x line was symmetric BY DESIGN, which strengthens the do-not-fix-symmetric warning with a real alternative rather than a hypothetical. The stale empty_attribute_default 3-0 bullet (promoted from a memory already three days stale at promotion) is replaced by the actual removal record under removed-v1-surface (#44 origin, the z-Smith in-band- signaling bug, substitute-before-format, tombstone-not-deletion, readers-not-just-writers). New comparison-surface section: the eq/hash trilemma, the cross-constants asymmetry, and two Declined entries including the deliberately-issueless lenient-matching resolved-as-no. The bridge discipline (warn in a released version first) becomes a standing 3-0 rule; WARN-AT-THE-CALLER gains its first measured instance; comparison joins rules.md's Not-in-scope with a pointer. Co-Authored-By: Claude Fable 5 --- docs/design/decisions.md | 67 +++++++++++++++++++++++++++++++++++---- docs/design/mechanisms.md | 4 ++- docs/design/rules.md | 3 ++ 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 68aa985..629bdf9 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -196,9 +196,14 @@ Declined: accepted cost, pinned deliberately, is that ASCII-SS GROSSFÜRST no longer matches. The paired half: comparison_key()/matches() use casefold() ON PURPOSE — comparison is the one surface that - wants aggressive folding, a documented 1.4 deviation (ß and - final-sigma forms compare equal). The storage-vs-comparison - split is deliberate asymmetry; do not "fix" it symmetric. + wants aggressive folding; this is a release-log-classified 2.0 + CHANGE from 1.4's lower() (ß and final-sigma forms compare equal + only since the v2 core; v1.4 has no casefold anywhere). The 1.x + line was symmetric BY DESIGN — lower() everywhere, chosen for + symmetry with the parser's lc() convention — so the + storage-vs-comparison split is a v2 decision moving away from a + real, deliberate alternative, not a fix of an oversight. Do not + "fix" it symmetric: symmetric is what 1.x was. - 2026-07 (rc1 arc) — the fold must reach a FIXED POINT, and anything built on it must converge too: _title_key joined per-word folds and kept empty slots, so given_name_titles with a @@ -878,8 +883,51 @@ Declined: must merge where the render tag is applied, not where the role is assigned. +### comparison-surface — why value equality died, and what replaced it + +- 2026-07 (1.3.0 arc, #223/#224; executed in the v2 core — + measured: HumanName == HumanName is False, matches() and + comparison_key() live, the facade cites #223) — the trilemma: + case-insensitive equality, equality-with-plain-strings, and + hashability are mutually inconsistent (a hash cannot equal both + hash("John Smith") and hash("john smith")), so 1.2.0's design + was unfixable in place, not merely disliked. Additional costs of + the old surface: equality ran through str(self) and so depended + on mutable string_format, and maiden was invisible to == + (absent from the default format). +- 2026-07-06 — the cross-constants asymmetry, pinned: a str + operand is reparsed with self.C; a HumanName operand is compared + as already parsed. + +Declined: + +- Component-based __eq__/__hash__ — workable, considered, rejected: + it still privileged one equality semantic for a domain that has + none, and kept the mutate-while-hashed hazard. +- Lenient matching (initials-compatible, Bob/Robert) — declined + WITHOUT an issue, deliberately: matches() is exact-components by + design. Someone will propose could_match() within a year; this + entry is the resolved-as-no they should find. + ### removed-v1-surface +- empty_attribute_default: removed in 2.0 (#255; deprecated in 1.4 + per the bridge discipline). Origin #44 (2016): a DB-NULL + convenience whose first answer — `name.title or None` — became + the migration path. The in-band-signaling bug that sealed it + (#254): the 2016 `.replace('None','')` scrub could not tell + interpolated None from name text, so "Nonez Smith" rendered + "z Smith" — the fix shape, worth keeping as a one-liner, is + SUBSTITUTE BEFORE FORMAT, never scrub after. Two removal + cautions shaped the implementation: tombstone-not-deletion (a + plain deleted attribute would make assignment silently + accepted-and-ignored, the #241 failure family) and + readers-not-just-writers (#44's own thread advised asserting + against the attribute, so reads existed downstream). The + tombstone and pickle-tolerant load die with the shim. (An + earlier 3-0-reevaluations bullet said "left untyped, typeable in + 3.0" — promoted from a memory that was already three days stale + when written; nothing is left to type.) - no_vowels: removed in 2.0 (#268, filed 2026-07-07, closed 2026-07-28) — never consulted by any parser version, ASCII-only; the facade carries no replacement because there was nothing to @@ -892,7 +940,12 @@ promotion approved 2026-08-15). Discipline when appending: mark each entry (A) "would decide differently without the shim" — real 3.0 work — or (B) "cited 1.4 parity but stands on its own", recorded so nobody re-litigates it. Append here whenever a design choice cites -1.4 parity or the shim as a load-bearing reason. +1.4 parity or the shim as a load-bearing reason. Standing +discipline for the removals themselves: every removal warns in a +RELEASED version first — the rule established by the 1.3.0 eq/hash +work (#223/#224), the reason the v1.4 milestone existed, and the +reason FACADE-CONTRACT's "warning-free on 1.4" anchor works at all. +3.0's shim removals follow the same bridge. - (A) v1 field vocabulary at the facade boundary: CJK semantics squeeze into first/last through HumanName while the core speaks @@ -906,8 +959,6 @@ nobody re-litigates it. Append here whenever a design choice cites the migration promise it verifies dissolves; the successor baseline is presumably last-2.x. Machinery survives; corpus contracts change. -- (A) empty_attribute_default left untyped (PR #250): cascades into - the v1-shaped public API. Typeable in 3.0. - (A) A .pyi stub for Policy (wide __init__, narrow attributes) — deferred from #334, see the differential-ledger Declined entry. - (A) nameparser.config removal scope: the 3.0 schedule says @@ -925,7 +976,9 @@ nobody re-litigates it. Append here whenever a design choice cites coincides with 1.4 parity but stands alone — family-first is the marked case needing affirmative evidence. - (B) The deprecation-bridge shape (#293/#354) is the template for - every remaining 2.x→3.0 shim: PEP 562 module __getattr__ PLUS + every remaining 2.x→3.0 shim (the bridge DISCIPLINE — warn in a + released version first — predates it, from #223/#224; this entry + is the module-__getattr__ mechanics): PEP 562 module __getattr__ PLUS __all__ (star imports never reach __getattr__ — measured: `from ...prefixes import *` bound nothing and leaked the helper); warn per read-location rather than per process (a write-back let a diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 9e5a4d8..32024ba 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -312,7 +312,9 @@ first frame outside the library's own modules and warns there, instead of counting frames. Lives in. nameparser/_lexicon.py (_warn_dead_entry), and the related but distinct per-read-location choice is in the #293/#354 -bridge (decisions.md#3-0-reevaluations). +bridge (decisions.md#3-0-reevaluations). First measured instance +of the hazard: the delegated bytes-deprecation warnings shipped +with fixed stacklevel=2 and needed the follow-up fix (0044073). Reach for it when. Adding any warning reachable through more than one public entry point. diff --git a/docs/design/rules.md b/docs/design/rules.md index a60d88c..6f047e7 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -58,6 +58,9 @@ set in decisions.md). inflected forms. CLDR personNames draws the same line. - **Validation.** Deciding whether a string IS a person's name is not parsing; `parse()` is total over strings and never rejects input. +- **Comparison.** matches()/comparison_key() are a value-API + surface, not parsing; their design record is + decisions.md#comparison-surface. ## Titles & honorifics (H)