Cover non-BMP characters, and refuse lone surrogates with LiftWriteError

CHANGELOG.md

@@ -31,7 +31,10 @@ releases may contain breaking changes.
   documents and untouched entries are written byte-identically; touched entries
   re-serialize canonically with all out-of-schema content preserved. Fidelity
   contract documented in `docs/en/fidelity.md` and enforced by corpus
-  byte-identity tests plus Hypothesis round-trip properties.
+  byte-identity tests plus Hypothesis round-trip properties. Content XML cannot
+  represent — a lone surrogate, which only an API assignment can introduce — is
+  refused with `LiftWriteError` naming the node, and reported by validation as
+  `lone-surrogate`.
 - Change detection against the loaded document, reading the same parse-time
   digests. `Lexicon.changed_entries()` reports entries whose content differs
   (an entry's digest covers its whole subtree, so an edit at any depth reports

docs/en/fidelity.md

@@ -26,6 +26,12 @@ Exceptions (the writer falls back to full canonical serialization, which is sema
 !!! note ""Canonical" here is not related to any other Canonical XML"
     Canonical form on this page means `sil-lift`'s own documented shape, described in a bullet above. It is unrelated to W3C's Canonical XML (C14N) process. It is unrelated to `SIL.Core`'s `CanonicalXmlSettings` class.
 
+## Content XML cannot represent
+
+Non-BMP characters — emoji, CJK Extension B, Adlam, anything above U+FFFF — are ordinary content and round-trip byte-identically. A "surrogate pair" is a UTF-16 encoding detail: Python strings are sequences of codepoints, so nothing in the reader, the byte scanner, or the writer ever sees one.
+
+A _lone_ surrogate (U+D800–U+DFFF) is different: a Python string may hold one, an XML document may not, in any encoding. It can never arrive from a file — the parser rejects both spellings, a `�` character reference and CESU-8/WTF-8 bytes — only from a string assigned through the API. Saving such a model raises `LiftWriteError` naming the node and the codepoint and writes nothing; validation reports it as a single `lone-surrogate` error, since the document cannot be serialized for the schema layers to check.
+
 ## Known approximations (touched nodes only)
 
 - Comments _inside_ a `<text>` run are preserved but moved next to the run, not kept at their exact character offset.

docs/en/guides/validate.md

@@ -44,6 +44,8 @@ Every finding carries one of these, whichever layer produced it — `schema` and
 | `undefined-range-value`  | warning | a grammatical-info or range-keyed trait value the range does not list      |
 | `uri-not-rfc`            | warning | an href that is not a valid URI — FLEx's `file://C:/...`                   |
 
+All three layers work from what `save()` would write, so a document that cannot be serialized at all is reported as a single `lone-surrogate` error instead — see [Fidelity guarantees](../fidelity.md#content-xml-cannot-represent).
+
 ## Real-world FieldWorks (FLEx) output
 
 FieldWorks systematically writes some content that strict tooling rejects. Here is sil-lift's policy, so that real lexicons validate usefully:

src/sil_lift/__init__.py

@@ -9,7 +9,7 @@
 from typing import TYPE_CHECKING
 
 from ._canonical import canonicalize
-from ._errors import LiftError, LiftParseError, LiftValidationError
+from ._errors import LiftError, LiftParseError, LiftValidationError, LiftWriteError
 from ._extras import Extras
 from ._header import FieldDefinition, Header, Range, RangeElement
 from ._model import (
@@ -59,6 +59,7 @@
     "LiftParseError",
     "LiftReader",
     "LiftValidationError",
+    "LiftWriteError",
     "LiftWriter",
     "MediaRef",
     "Multitext",

src/sil_lift/_errors.py

@@ -7,7 +7,7 @@
 if TYPE_CHECKING:
     from ._validate import Problem
 
-__all__ = ["LiftError", "LiftParseError", "LiftValidationError"]
+__all__ = ["LiftError", "LiftParseError", "LiftValidationError", "LiftWriteError"]
 
 
 class LiftError(Exception):
@@ -23,6 +23,10 @@ class LiftParseError(LiftError):
     """
 
 
+class LiftWriteError(LiftError):
+    """An in-memory document holds content that XML cannot represent, so it cannot be written."""
+
+
 class LiftValidationError(LiftError):
     """Raised by the fail-fast validation wrappers on the first error-level
     :class:`~sil_lift.Problem` (warnings never raise)."""

src/sil_lift/_model.py

@@ -565,7 +565,9 @@ def save(self, path: str | os.PathLike[str] | None = None) -> None:
         paths (they are shared with the original document, not copied).
 
         Raises :class:`ValueError` if no target path is available (none was
-        passed and the lexicon was not loaded from a file).
+        passed and the lexicon was not loaded from a file), and
+        :class:`~sil_lift.LiftWriteError` if the model holds content XML cannot
+        represent (a lone surrogate) — nothing is written in that case.
         """
         from ._writer import render_document
 

src/sil_lift/_validate.py

@@ -24,6 +24,10 @@
    (``missing-id`` opt-in via ``require_ids``). The codes are named on
    ``Problem.code`` below; ``docs/en/guides/validate.md`` tabulates each one's
    level and what it flags.
+
+A document that cannot be serialized at all — a lone surrogate assigned through
+the API — is reported as a single ``lone-surrogate`` error instead of the layers
+above, all of which need the rendered bytes.
 """
 
 from __future__ import annotations
@@ -35,7 +39,7 @@
 
 from lxml import etree
 
-from ._errors import LiftValidationError
+from ._errors import LiftValidationError, LiftWriteError
 from ._model import GrammaticalInfo, Lexicon, _normalize_href
 from ._text import Multitext, Trait
 
@@ -57,9 +61,7 @@ class Problem:
     """One validation finding, addressable to a file/entry/line."""
 
     level: Literal["error", "warning"]
-    code: str  # "schema", "duplicate-guid", "dangling-ref", "range-parent",
-    # "undefined-range-value", "normalization-mismatch", "duplicate-form-lang",
-    # "missing-media", "uri-not-rfc", "dangling-ranges-href", "missing-id"
+    code: str  # e.g. "schema", "duplicate-guid", "dangling-ref", ...
     message: str
     file: Path | None = None
     entry_id: str | None = None
@@ -101,11 +103,25 @@ def iter_lexicon_problems(lexicon: Lexicon, *, require_ids: bool = False) -> Ite
     # source, so line numbers keep matching the file on disk — and rendered
     # entry order always matches lexicon.entries, keeping the entry_lines
     # table aligned for semantic addressing even after edits or sort().
-    data = render_document(lexicon)
+    #
+    # A lone surrogate makes the document unrenderable, so it is reported as
+    # the one finding and nothing else runs: every layer below needs the
+    # rendered bytes (the schema layers parse them, and the semantic layer
+    # addresses findings by their line numbers). Reporting it here is what
+    # makes it diagnosable at all — save() would raise the same refusal.
+    try:
+        data = render_document(lexicon)
+    except LiftWriteError as exc:
+        yield Problem("error", "lone-surrogate", str(exc), file=lexicon.path)
+        return
     entry_lines, problems = _schema_problems(data, lift_schema, lexicon.path)
     yield from problems
     for ranges_file in lexicon.ranges_files.values():
-        rdata = render_ranges_document(ranges_file)
+        try:
+            rdata = render_ranges_document(ranges_file)
+        except LiftWriteError as exc:
+            yield Problem("error", "lone-surrogate", str(exc), file=ranges_file.path)
+            continue
         _, range_problems = _schema_problems(rdata, ranges_schema, ranges_file.path)
         yield from range_problems
     yield from _semantic_problems(lexicon, entry_lines, require_ids=require_ids)

src/sil_lift/_writer.py

@@ -15,6 +15,8 @@
   reassembles byte-identically.
 
 Snapshots are sha256 digests of canonical bytes, taken at parse time.
+
+Both paths refuse content XML cannot represent: see :func:`_guarded`.
 """
 
 from __future__ import annotations
@@ -26,6 +28,7 @@
 
 from lxml import etree
 
+from ._errors import LiftWriteError
 from ._extras import Extras
 from ._header import FieldDefinition, Header, Range, RangeElement
 from ._model import (
@@ -69,6 +72,34 @@
 _FRAGMENT_PARSER = etree.XMLParser(resolve_entities=False, no_network=True)
 
 
+def _guarded(what: str, build: Callable[[], bytes]) -> bytes:
+    """Serialize one node, turning unrepresentable content into a LIFT error.
+
+    A Python string may hold a lone surrogate; XML may not, in any encoding.
+    lxml reports it as a bare ``UnicodeEncodeError`` from wherever the text or
+    attribute was set — deep inside the builders below, naming no node. Every
+    entry point that builds and serializes a node passes through here so the
+    failure arrives as a :class:`~sil_lift.LiftWriteError` that says which node
+    and which codepoint. Nothing else in a ``str`` is unencodable as UTF-8, so
+    anything else is re-raised untouched rather than mislabelled.
+    """
+    try:
+        return build()
+    except UnicodeEncodeError as exc:
+        char = exc.object[exc.start]
+        if not 0xD800 <= ord(char) <= 0xDFFF:
+            raise
+        raise LiftWriteError(
+            f"{what}: U+{ord(char):04X} is a lone surrogate, which XML cannot "
+            f"represent in any encoding (in {exc.object!r})"
+        ) from exc
+
+
+def _entry_label(entry: Entry) -> str:
+    name = entry.id or entry.guid
+    return f"entry {name!r}" if name else "entry (no id or guid)"
+
+
 # --- byte-reuse state (created by the reader, consumed here) ---------------------
 
 
@@ -111,15 +142,15 @@ class _RangesSourceInfo:
 
 
 def entry_digest(entry: Entry) -> bytes:
-    return hashlib.sha256(_node_bytes(_entry_el(entry))).digest()
+    return hashlib.sha256(canonical_entry_bytes(entry)).digest()
 
 
 def header_digest(header: Header) -> bytes:
-    return hashlib.sha256(_node_bytes(_header_el(header))).digest()
+    return hashlib.sha256(canonical_header_bytes(header)).digest()
 
 
 def range_digest(range_: Range) -> bytes:
-    return hashlib.sha256(_node_bytes(_range_el(range_))).digest()
+    return hashlib.sha256(canonical_range_bytes(range_)).digest()
 
 
 # --- canonical building blocks ---------------------------------------------------
@@ -624,24 +655,31 @@ def _node_bytes(el: etree._Element) -> bytes:
 
 
 def canonical_entry_bytes(entry: Entry) -> bytes:
-    return _node_bytes(_entry_el(entry))
+    return _guarded(_entry_label(entry), lambda: _node_bytes(_entry_el(entry)))
 
 
 def canonical_header_bytes(header: Header) -> bytes:
-    return _node_bytes(_header_el(header))
+    return _guarded("header", lambda: _node_bytes(_header_el(header)))
+
+
+def canonical_range_bytes(range_: Range) -> bytes:
+    return _guarded(f"range {range_.id!r}", lambda: _node_bytes(_range_el(range_)))
 
 
 # --- document rendering ------------------------------------------------------------
 
 
 def _root_open_bytes(lexicon: Lexicon) -> bytes:
-    el = _element(
-        "lift",
-        [("version", "0.13"), ("producer", lexicon.producer)],
-        lexicon.extra,
-    )
-    serialized = etree.tostring(el, encoding="unicode").encode("utf-8")
-    return serialized[:-2] + b">"  # "<lift .../>" -> "<lift ...>"
+    def build() -> bytes:
+        el = _element(
+            "lift",
+            [("version", "0.13"), ("producer", lexicon.producer)],
+            lexicon.extra,
+        )
+        serialized = etree.tostring(el, encoding="unicode").encode("utf-8")
+        return serialized[:-2] + b">"  # "<lift .../>" -> "<lift ...>"
+
+    return _guarded("<lift> root", build)
 
 
 def canonical_document(
@@ -659,7 +697,7 @@ def canonical_document(
         if node.kind == "text":
             continue  # character data at root level is not representable
         position = min(node.index, len(chunks))
-        chunks.insert(position, node.xml.encode("utf-8") + b"\n")
+        chunks.insert(position, _guarded("root-level residue", node.xml.encode) + b"\n")
     parts = [b'<?xml version="1.0" encoding="UTF-8"?>\n', _root_open_bytes(lexicon), b"\n"]
     # Each chunk's trailing newline is the inter-chunk separator. Byte-reused
     # (untouched) regions end at ">", so append the newline they lack — without
@@ -818,20 +856,32 @@ def render_document(lexicon: Lexicon) -> bytes:
 # --- .lift-ranges documents ----------------------------------------------------------
 
 
+def _ranges_root_open_bytes(ranges_file: RangesFile) -> bytes:
+    def build() -> bytes:
+        root = _element("lift-ranges", [], ranges_file.extra)
+        serialized = etree.tostring(root, encoding="unicode").encode("utf-8")
+        return serialized[:-2] + b">"  # "<lift-ranges .../>" -> "<lift-ranges ...>"
+
+    return _guarded("<lift-ranges> root", build)
+
+
 def canonical_ranges_document(
     ranges_file: RangesFile,
     range_bytes: Callable[[Range], bytes] | None = None,
 ) -> bytes:
     if range_bytes is None:
-        range_bytes = lambda r: _node_bytes(_range_el(r))  # noqa: E731
+        range_bytes = canonical_range_bytes
     chunks = [range_bytes(range_) for range_ in ranges_file.ranges]
     for node in sorted(ranges_file.extra._nodes, key=lambda n: n.index):
         if node.kind == "text":
             continue
-        chunks.insert(min(node.index, len(chunks)), node.xml.encode("utf-8") + b"\n")
-    root = _element("lift-ranges", [], ranges_file.extra)
-    serialized = etree.tostring(root, encoding="unicode").encode("utf-8")
-    parts = [b'<?xml version="1.0" encoding="UTF-8"?>\n', serialized[:-2] + b">", b"\n"]
+        fragment = _guarded("root-level residue", node.xml.encode)
+        chunks.insert(min(node.index, len(chunks)), fragment + b"\n")
+    parts = [
+        b'<?xml version="1.0" encoding="UTF-8"?>\n',
+        _ranges_root_open_bytes(ranges_file),
+        b"\n",
+    ]
     # See canonical_document: reused regions need the newline they lack, or the
     # chunk runs into its neighbor.
     parts.extend(chunk if chunk.endswith(b"\n") else chunk + b"\n" for chunk in chunks)
@@ -852,7 +902,7 @@ def fn(range_: Range) -> bytes:
             record, region = found
             if range_digest(range_) == record.digest:
                 return source.data[region.start : region.end]
-        return _node_bytes(_range_el(range_))
+        return canonical_range_bytes(range_)
 
     return fn
 
@@ -883,9 +933,7 @@ def render_ranges_document(ranges_file: RangesFile) -> bytes:
     if root_unchanged:
         parts.append(data[source.root_open_start : source.root_open_end])
     else:
-        root = _element("lift-ranges", [], ranges_file.extra)
-        serialized = etree.tostring(root, encoding="unicode").encode("utf-8")
-        parts.append(serialized[:-2] + b">")
+        parts.append(_ranges_root_open_bytes(ranges_file))
     position = source.root_open_end
     range_index = 0
     for region in source.children:

tests/test_property_roundtrip.py

@@ -26,10 +26,16 @@
 # non-characters; \r is excluded (parsers normalize it), \t and \n are added
 # back via explicit alternatives where allowed.
 _CHARS = st.characters(min_codepoint=0x20, codec="utf-8", exclude_characters="￾￿")
-_TEXT = st.text(alphabet=st.one_of(_CHARS, st.sampled_from("\t\n")), max_size=30)
+# Non-BMP codepoints — 4-byte UTF-8, one surrogate pair each in a UTF-16 source
+# — drawn explicitly: they are what a byte scanner guessing at character
+# boundaries would break on, and _CHARS alone reaches them too rarely to count
+# as coverage. See tests/test_unicode.py for the deterministic cases.
+_NON_BMP = st.sampled_from("\U0001f389\U00020000\U0001e900\U0001d11e\U000e0021")
+_CHARS_INCL_NON_BMP = st.one_of(_CHARS, _NON_BMP)
+_TEXT = st.text(alphabet=st.one_of(_CHARS_INCL_NON_BMP, st.sampled_from("\t\n")), max_size=30)
 # Attribute values: XML parsers normalize \t\n in attributes to spaces, so keep
 # tokens to characters that round-trip verbatim.
-_TOKEN = st.text(alphabet=_CHARS, min_size=1, max_size=15)
+_TOKEN = st.text(alphabet=_CHARS_INCL_NON_BMP, min_size=1, max_size=15)
 _LANG = st.sampled_from(["en", "fr", "th", "sg", "es", "qaa-x-test"])
 
 _WHEN = st.one_of(