diff --git a/CHANGELOG.md b/CHANGELOG.md index ec04d9b..05cee3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,13 +51,15 @@ releases may contain breaking changes. against the most recent `save()`. - LIFT-folder handling: `RangesFile` (standalone `.lift-ranges` documents, same fidelity guarantees), automatic companion discovery/tracking on load - (`Lexicon.ranges_files`), `save()` writes companions together, + (`Lexicon.ranges_files`, matching companion filenames across case and + Unicode normalization differences, and loading nothing for a name several + files answer to that way), `save()` writes companions together, `all_ranges()` merged view, `media_refs()` / `missing_media()` helpers, build-from-scratch helpers `Lexicon.add_ranges_file()` / `RangesFile.add_range()` / `Range.add_element()` (`save()` writes and header-references a new companion beside the `.lift`); vendored - `schemas/lift-ranges-0.13.rng` — the first schema for standalone - ranges documents. + `schemas/lift-ranges-0.13.rng` — the first schema for standalone ranges + documents. - Zipped LIFT packages: `sil_lift.load()` reads a `.zip` (both the flat and folder-wrapped layouts, junk entries like `__MACOSX` ignored), `Lexicon.save_zip()` writes one (carrying media, `WritingSystems/`, and other @@ -72,7 +74,7 @@ releases may contain breaking changes. file, entry, and line it concerns. RELAX NG layer with two documented departures from strict validation (invalid `file://` hrefs downgraded to `uri-not-rfc` warnings; legal interleaving not falsely flagged); vendored - ranges schema over companions; and nine semantic checks the grammar cannot + ranges schema over companions; and ten semantic checks the grammar cannot express, one `Problem` code each (with missing-id opt-in via `require_ids`). Names resolve against range and range-element ids under NFC; a match that needed normalizing is reported as normalization-mismatch, once per id. diff --git a/docs/en/guides/validate.md b/docs/en/guides/validate.md index b6fb006..c1249b8 100644 --- a/docs/en/guides/validate.md +++ b/docs/en/guides/validate.md @@ -24,14 +24,15 @@ Each `Problem` carries `level` (`"error"`/`"warning"`), a stable `code`, `messag 1. **RELAX NG** against the LIFT 0.13 grammar (vendored from lift-standard — a byte-identical copy committed into this package). 2. **Ranges schema** — this project's `lift-ranges-0.13.rng` — over every tracked `.lift-ranges` companion, addressed to the companion rather than the `.lift`. -3. **Semantic checks** the grammar cannot express — nine of them, one code each. +3. **Semantic checks** the grammar cannot express — ten of them, one code each. ## Problem codes -Every finding carries one of these, whichever layer produced it — `schema` and `uri-not-rfc` come from the schema layers, the other nine are semantic checks. The strings are a supported interface; `--strict` promotes every warning to an error. +Every finding carries one of these, whichever layer produced it — `schema` and `uri-not-rfc` come from the schema layers, the other ten are semantic checks. The strings are a supported interface; `--strict` promotes every warning to an error. | code | level | what it flags | | ------------------------ | ------- | -------------------------------------------------------------------------- | +| `ambiguous-ranges-file` | warning | several files answering to one companion name under case folding and NFC | | `dangling-ranges-href` | warning | a header `range/@href` resolving to no companion file | | `dangling-ref` | error | a `relation/@ref` or `variant/@ref` matching no entry or sense | | `duplicate-form-lang` | warning | two forms in one multitext sharing a language | @@ -46,6 +47,8 @@ Every finding carries one of these, whichever layer produced it — `schema` and All three layers work from what `save()` would write, so a document that cannot be serialized at all is reported as a single `lone-surrogate` error instead — see [Fidelity guarantees](../fidelity.md#content-xml-cannot-represent). +A companion name matching several files loads none of them: the ranges they define go absent until all but one is renamed or removed. + ## Real-world FieldWorks (FLEx) output FieldWorks systematically writes some content that strict tooling rejects. Here is sil-lift's policy, so that real lexicons validate usefully: diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 8597750..7e2cc55 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -11,6 +11,7 @@ from __future__ import annotations +import unicodedata from dataclasses import dataclass, field from datetime import date, datetime from pathlib import Path, PurePosixPath, PureWindowsPath @@ -23,7 +24,7 @@ if TYPE_CHECKING: import os import tempfile - from collections.abc import Iterator + from collections.abc import Iterable, Iterator from typing import Literal from ._validate import Problem @@ -452,6 +453,107 @@ def _normalize_href(href: str) -> Path | None: return Path(normalized) +def _fold(text: str) -> str: + """A filename reduced to what a case-folding filesystem treats as one name. + + ``casefold`` for what ``lower`` gets wrong (e.g. the Turkish dotless i); + NFC for names that arrive decomposed (e.g. ones zipped on macOS). An + approximation of NTFS's and APFS's tables, not a general equivalence. + """ + return unicodedata.normalize("NFC", text).casefold() + + +def _folded_matches(candidate: Path, listings: dict[Path, dict[str, list[Path]]]) -> list[Path]: + """The files ``candidate`` names: the exact one, or those folding onto it. + + Where the authoring filesystem folds case, as on Windows and macOS, an + inconsistently spelled pair goes unnoticed: ``Dict.LIFT`` beside + ``Dict.lift-ranges`` is a pair there but not everywhere. + + An exact hit is the only match: folding runs only after the exact name + misses, and then only on the final component — the hrefs this serves are + basenames or same-folder relatives. + + Several matches mean the folder holds names the filesystem that wrote them + could not have told apart, so which one the name meant is not recoverable. + + ``listings`` caches one directory read per folder. + """ + try: + if candidate.is_file(): + return [candidate] + if candidate.is_dir(): + # An href of "" or "sub/" lands here; folding a folder's own name + # would search its parent and match anything spelled like it. + return [] + except OSError: + pass # unstattable exact spelling: a case variant of it may still stat + folder = candidate.parent + if folder not in listings: + files: dict[str, list[Path]] = {} + try: + for path in folder.iterdir(): + if path.is_file(): + files.setdefault(_fold(path.name), []).append(path) + except OSError: + pass # unreadable folder: no candidate resolves out of it + listings[folder] = files + return listings[folder].get(_fold(candidate.name), []) + + +def _existing_file(candidate: Path, listings: dict[Path, dict[str, list[Path]]]) -> Path | None: + """The one file ``candidate`` names, or None if no file or several do. + + Guessing between several would pick a companion by a rule no exporter + knows, so an ambiguous name resolves to nothing at all — visibly, as + ``ambiguous-ranges-file`` from validation, rather than by silent choice. + """ + matches = _folded_matches(candidate, listings) + return matches[0] if len(matches) == 1 else None + + +def _ranges_candidates(lift_path: Path, ranges: Iterable[Range]) -> list[Path]: + """Where a companion may be found, in the order :meth:`Lexicon.load` tries. + + Each path once: an href shared by several ranges, or agreeing with the + sibling, is one candidate however many times it is written. + """ + base = lift_path.parent + # with_name and with_suffix agree on every name that has an extension, but + # with_suffix would raise on a name that has none — which parse_document + # accepts, since it never inspects the extension. + candidates = [lift_path.with_name(lift_path.name + "-ranges")] + for range_ in ranges: + if range_.href is None: + continue + relative = _normalize_href(range_.href) + if relative is not None: + candidates.append(base / relative) + basename = range_.href.replace("\\", "/").rpartition("/")[2] + if basename: + candidates.append(base / basename) + return list(dict.fromkeys(candidates)) + + +def _same_file(left: Path, right: Path) -> bool: + """Whether two paths that fold together denote one file. + + ``Path.resolve()`` canonicalizes case on Windows but not on macOS, where + one file reached under two spellings yields two keys — tracked twice, and + written twice by :meth:`Lexicon.save`. + + Both sides resolve first, so ``..`` segments and symlinks compare alike; + the fold pre-check then keeps the inode comparison from conflating + distinct files where ``st_ino`` is 0. + """ + try: + if _fold(str(left.resolve())) != _fold(str(right.resolve())): + return False + return left.samefile(right) + except OSError: + return False + + def _same_dir(left: Path, right: Path | None) -> bool: """Whether two paths denote the same directory, spelling aside. @@ -505,13 +607,19 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L With ``resolve_ranges`` (the default), companion ``.lift-ranges`` files are loaded and tracked in :attr:`ranges_files`. Several - candidates are tried and every one that exists is loaded: the - conventional ``.lift-ranges`` sibling, and for each header + candidates are tried and every distinct file among them is loaded: + the conventional ``.lift-ranges`` sibling, and for each header ``range/@href`` both the href resolved as a path relative to the ``.lift`` file and its bare basename in the same directory (FLEx hrefs are usually dangling absolute ``file://C:/...`` paths from the exporting machine, so the basename is what resolves locally). + A candidate matching no file exactly resolves across differences in + case or Unicode normalization, so a folder authored on Windows loads + the same way everywhere. One that several files answer to that way is + ambiguous and resolves to none of them, which validation reports as + ``ambiguous-ranges-file``; the ``.lift`` is never its own companion. + A ``.zip`` path is treated as a packaged LIFT folder: it is extracted to a temporary directory (kept alive for the returned lexicon's lifetime) and the single contained ``.lift`` is loaded. @@ -531,27 +639,24 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L def _resolve_ranges(self) -> None: if self.path is None: return - base = self.path.parent - candidates: list[Path] = [] - sibling = self.path.with_suffix(self.path.suffix + "-ranges") - candidates.append(sibling) - for range_ in self.header.ranges: - if range_.href is None: + listings: dict[Path, dict[str, list[Path]]] = {} + for candidate in _ranges_candidates(self.path, self.header.ranges): + found = _existing_file(candidate, listings) + if found is None: continue - relative = _normalize_href(range_.href) - if relative is not None: - candidates.append(base / relative) - basename = range_.href.replace("\\", "/").rpartition("/")[2] - if basename: - candidates.append(base / basename) - for candidate in candidates: try: - resolved = candidate.resolve() - exists = candidate.is_file() + resolved = found.resolve() except OSError: continue - if exists and resolved not in self.ranges_files: - self.ranges_files[resolved] = RangesFile.load(candidate) + # A header href naming the .lift in another case folds onto it, and + # RangesFile.load rejects that root, failing the whole load; two + # spellings of one companion, which resolve() leaves distinct on + # macOS, would load and write it twice. + if resolved in self.ranges_files or any( + _same_file(resolved, other) for other in (self.path, *self.ranges_files) + ): + continue + self.ranges_files[resolved] = RangesFile.load(found) def save(self, path: str | os.PathLike[str] | None = None) -> None: """Write the ``.lift`` file and every tracked ``.lift-ranges`` companion. diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index 5961640..ec17898 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -40,7 +40,15 @@ from lxml import etree from ._errors import LiftValidationError, LiftWriteError -from ._model import GrammaticalInfo, Lexicon, _normalize_href +from ._model import ( + GrammaticalInfo, + Lexicon, + _existing_file, + _folded_matches, + _normalize_href, + _ranges_candidates, + _same_file, +) from ._text import Multitext, Trait if TYPE_CHECKING: @@ -509,13 +517,41 @@ def range_named(name: str) -> str | None: # here, not inside the file check below, which a lexicon with no path skips. header_ranges = {range_.id: range_named(range_.id) for range_ in lexicon.header.ranges} - # Header references (relative) that resolve to no companion. - # Absolute/file:// hrefs are ones FLEx writes knowing they will not resolve - # (they are resolved by basename when the companion is in the same folder) - # and are not checked here; this catches an exporter that writes a relative - # href but not the file. if lexicon.path is not None: base = lexicon.path.parent + listings: dict[Path, dict[str, list[Path]]] = {} + + # Every candidate, not just the hrefs below: a collision on the sibling + # name has nothing else to report it. Keyed by colliding group, since + # several candidate names can fold onto the same one. + reported: set[tuple[Path, tuple[str, ...]]] = set() + for candidate in _ranges_candidates(lexicon.path, lexicon.header.ranges): + matches = _folded_matches(candidate, listings) + if len(matches) < 2: + continue + # One of the colliding files loaded, named exactly by another candidate. + if any(_same_file(path, loaded) for path in matches for loaded in lexicon.ranges_files): + continue + names = sorted(path.name for path in matches) + key = (candidate.parent, tuple(names)) + if key in reported: + continue + reported.add(key) + # Spellings differing only in normalization render identically. + spellings = ", ".join(f"{name!a}" for name in names) + yield Problem( + "warning", + "ambiguous-ranges-file", + f"companion {candidate.name!a} matches {spellings}; they differ only " + "in case or Unicode normalization, so none of them is loaded", + file=lexicon.path, + ) + + # Header references that resolve to no companion — an + # exporter that wrote the href but not the file. Absolute and file:// + # hrefs are skipped: FLEx writes those knowing they will not resolve, + # and load reaches their companions by basename in the same folder + # instead. for range_ in lexicon.header.ranges: if not range_.href or range_.elements: continue @@ -524,7 +560,10 @@ def range_named(name: str) -> str | None: continue if header_ranges[range_.id] is not None: continue # supplied by a sibling companion instead - if not (base / relative).is_file(): + found = _existing_file(base / relative, listings) + # _resolve_ranges refuses the lexicon as its own companion, so an + # href folding onto it supplies nothing and dangles too. + if found is None or _same_file(found, lexicon.path): yield Problem( "warning", "dangling-ranges-href", diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 9544db9..b1a8862 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -1,4 +1,5 @@ import shutil +import unicodedata from pathlib import Path import pytest @@ -225,6 +226,209 @@ def test_missing_media_flags_broken_ref(tmp_path: Path) -> None: assert [r.href for r in missing] == ["pictures\\sdd.png"] +def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> Path: + """A loadable .lift plus companion under arbitrary filename spellings. + + Deliberately not named after the fixture, so the header's ``range/@href`` + basename candidate finds nothing — only the sibling candidate resolves these. + """ + folder.mkdir(parents=True, exist_ok=True) + (folder / lift_name).write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + (folder / ranges_name).write_bytes((PAIR_DIR / "test20080407.lift-ranges").read_bytes()) + return folder / lift_name + + +def _write_lift_with_href(folder: Path, lift_name: str, href: str) -> Path: + """The fixture .lift under another name, its companion href rewritten.""" + folder.mkdir(parents=True, exist_ok=True) + source = (PAIR_DIR / "test20080407.lift").read_bytes() + patched = source.replace(b'"file://test20080407.lift-ranges"', f'"{href}"'.encode()) + assert patched != source, "fixture href changed; the replacement no longer matches" + (folder / lift_name).write_bytes(patched) + return folder / lift_name + + +def _case_sensitive(folder: Path) -> bool: + probe = folder / "CaseProbe" + probe.mkdir(exist_ok=True) + sensitive = not (folder / "caseprobe").exists() + probe.rmdir() + return sensitive + + +def test_companion_resolves_when_lift_suffix_is_uppercase(tmp_path: Path) -> None: + lift = _write_case_variant_pair(tmp_path / "pkg", "Dict.LIFT", "Dict.lift-ranges") + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_companion_resolves_when_companion_suffix_is_uppercase(tmp_path: Path) -> None: + lift = _write_case_variant_pair(tmp_path / "pkg", "Dict.lift", "Dict.LIFT-RANGES") + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_case_folded_companions_resolve_to_neither(tmp_path: Path) -> None: + if not _case_sensitive(tmp_path): + pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") + # Neither spelling matches the Dict.LIFT-ranges candidate exactly, and + # nothing says which one it meant, so no companion is loaded for it. + folder = tmp_path / "pkg" + lift = _write_case_variant_pair(folder, "Dict.LIFT", "Dict.lift-ranges") + (folder / "Dict.Lift-ranges").write_bytes((folder / "Dict.lift-ranges").read_bytes()) + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files == {} + ambiguous = [p for p in lexicon.iter_problems() if p.code == "ambiguous-ranges-file"] + assert [p.level for p in ambiguous] == ["warning"] + assert "'Dict.LIFT-ranges' matches 'Dict.Lift-ranges', 'Dict.lift-ranges'" in ( + ambiguous[0].message + ) + + +def test_an_exactly_named_companion_ignores_the_variant_beside_it(tmp_path: Path) -> None: + if not _case_sensitive(tmp_path): + pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") + # The candidate names one of the two exactly, so folding never runs: a case + # variant is only ambiguous when nothing answers to the name as written. + folder = tmp_path / "pkg" + lift = _write_case_variant_pair(folder, "Dict.lift", "Dict.lift-ranges") + (folder / "Dict.LIFT-ranges").write_bytes((folder / "Dict.lift-ranges").read_bytes()) + lexicon = sil_lift.load(lift) + assert [path.name for path in lexicon.ranges_files] == ["Dict.lift-ranges"] + assert [p for p in lexicon.iter_problems() if p.code == "ambiguous-ranges-file"] == [] + + +# One stem in the four spellings a filesystem that folds case still keeps +# apart: each accent composed or decomposed, independently. +_COMPOSED = "Ñandú" +_N_SPLIT = "Ñandú" +_U_SPLIT = "Ñandú" +_BOTH_SPLIT = "Ñandú" + + +def _normalization_sensitive(folder: Path) -> bool: + probe = folder / "NormProbé" + probe.mkdir() + sensitive = not (folder / unicodedata.normalize("NFC", probe.name)).exists() + probe.rmdir() + return sensitive + + +def test_normalization_folded_companions_resolve_to_neither(tmp_path: Path) -> None: + if not _normalization_sensitive(tmp_path): + pytest.skip("needs a filesystem that keeps normalization forms apart") + # Which accent is decomposed, rather than case, is what separates these two + # companions — the only way to reach the collision where the exact-name stat + # is itself case-insensitive, as it is on NTFS and APFS. + folder = tmp_path / "pkg" + # The sibling candidate is fully composed and the href fully decomposed: + # two names folding onto the same pair, so one finding covers both. + lift = _write_lift_with_href(folder, f"{_COMPOSED}.lift", f"{_BOTH_SPLIT}.lift-ranges") + source = (PAIR_DIR / "test20080407.lift-ranges").read_bytes() + (folder / f"{_N_SPLIT}.lift-ranges").write_bytes(source) + (folder / f"{_U_SPLIT}.lift-ranges").write_bytes(source) + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files == {} + problems = list(lexicon.iter_problems()) + ambiguous = [p for p in problems if p.code == "ambiguous-ranges-file"] + assert [p.level for p in ambiguous] == ["warning"] + # Spellings that render identically, named by code point. + assert ascii(f"{_N_SPLIT}.lift-ranges") in ambiguous[0].message + assert ascii(f"{_U_SPLIT}.lift-ranges") in ambiguous[0].message + # The collision says why nothing resolved; the href, which range went unmet. + assert "dangling-ranges-href" in [p.code for p in problems] + + +def test_a_collision_including_the_loaded_companion_is_not_reported(tmp_path: Path) -> None: + if not _normalization_sensitive(tmp_path): + pytest.skip("needs a filesystem that keeps normalization forms apart") + # The sibling names the composed spelling exactly and loads it; the href, + # spelled a third way, folds onto both files and resolves to neither. + folder = tmp_path / "pkg" + lift = _write_lift_with_href(folder, f"{_COMPOSED}.lift", f"{_BOTH_SPLIT}.lift-ranges") + source = (PAIR_DIR / "test20080407.lift-ranges").read_bytes() + (folder / f"{_COMPOSED}.lift-ranges").write_bytes(source) + (folder / f"{_N_SPLIT}.lift-ranges").write_bytes(source) + lexicon = sil_lift.load(lift) + assert [path.name for path in lexicon.ranges_files] == [f"{_COMPOSED}.lift-ranges"] + assert lexicon.all_ranges()["grammatical-info"].elements + assert [p for p in lexicon.iter_problems() if p.code == "ambiguous-ranges-file"] == [] + + +def test_absent_companion_stays_absent(tmp_path: Path) -> None: + # The fallback must not look outside the folder for a name not in it. + folder = tmp_path / "pkg" + folder.mkdir() + (folder / "Dict.lift").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + assert sil_lift.load(folder / "Dict.lift").ranges_files == {} + + +def test_companion_resolves_across_unicode_normalization(tmp_path: Path) -> None: + # FLEx mixes NFC and NFD within one export, and the mismatch reaches the + # filenames; only macOS folds the two forms together on its own. + composed = "Caf\N{LATIN SMALL LETTER E WITH ACUTE}.lift" + decomposed = unicodedata.normalize("NFD", f"{composed}-ranges") + lift = _write_case_variant_pair(tmp_path / "pkg", composed, decomposed) + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_lift_without_an_extension_loads(tmp_path: Path) -> None: + # Loading never inspects the extension, so the sibling candidate is built + # from a name that may have none; this companion is the href's basename. + folder = tmp_path / "pkg" + folder.mkdir() + (folder / "Dict").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + shutil.copy(PAIR_DIR / "test20080407.lift-ranges", folder) + lexicon = sil_lift.load(folder / "Dict") + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_href_folding_onto_the_lift_itself_is_not_a_companion(tmp_path: Path) -> None: + # Dict.lift beside a Dict.LIFT is the lexicon, not its ranges: loading it + # as one would raise on the root and take the whole load down. + lift = _write_lift_with_href(tmp_path / "pkg", "Dict.LIFT", "Dict.lift") + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files == {} + assert "dangling-ranges-href" in [p.code for p in lexicon.iter_problems()] + + +def test_self_referencing_href_dangles_however_it_is_spelled(tmp_path: Path) -> None: + # The ".." keeps the href from matching the lexicon's path as spelled, so + # both sides have to resolve before deciding what the reference supplies. + lift = _write_lift_with_href(tmp_path / "pkg", "Dict.LIFT", "../pkg/Dict.lift") + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files == {} + assert "dangling-ranges-href" in [p.code for p in lexicon.iter_problems()] + + +def test_folder_shaped_href_stays_inside_the_folder(tmp_path: Path) -> None: + if not _case_sensitive(tmp_path): + pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") + # An empty href names the folder itself; folding it would search the parent. + lift = _write_lift_with_href(tmp_path / "pkg", "Dict.lift", "") + (tmp_path / "PKG").write_bytes(b"") + assert sil_lift.load(lift).ranges_files == {} + + +# Defines the range the header points at, but no elements — so the merged view +# cannot vouch for the href and the check falls through to the filesystem. +ELEMENTLESS_RANGES = b""" + + + +""" + + +def test_case_variant_companion_is_not_reported_dangling(tmp_path: Path) -> None: + folder = tmp_path / "pkg" + lift = _write_lift_with_href(folder, "Dict.LIFT", "Dict.LIFT-ranges") + (folder / "Dict.lift-ranges").write_bytes(ELEMENTLESS_RANGES) + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files # the companion resolved + assert [p for p in lexicon.iter_problems() if p.code == "dangling-ranges-href"] == [] + + @pytest.mark.parametrize( ("href", "expected"), [