From 37a1d9f780b071f9daf234255718c5a7c76d5a33 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 4 Aug 2026 16:37:00 -0400 Subject: [PATCH 01/16] Resolve companion ranges across filename case differences MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A LIFT folder written on Windows can spell its pair inconsistently — Dict.LIFT beside Dict.lift-ranges, or the reverse — and load fine there, because the filesystem folds case. On Linux the sibling candidate is built from the .lift's own suffix, so it missed, the companion was skipped without a word, and every range it defined went absent. Candidates that match no file exactly now fall back to one whose name differs only in case. The fallback is reached only after an exact miss, so a case-folding filesystem never enters it and behaves as before; a case-sensitive one gets one directory read per folder, cached across the candidate list. Where several names fold together the lexicographically first wins. The choice is arbitrary but fixed, which matters more than which file it picks: directory order varies between filesystems and runs, and a companion that loads differently on consecutive reads would be worse than one that never loads. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 10 ++++++++ src/sil_lift/_model.py | 48 ++++++++++++++++++++++++++++++++---- tests/test_ranges_folder.py | 49 +++++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec04d9b..770cf2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,16 @@ releases may contain breaking changes. ## [Unreleased] +### Fixed + +- Companion `.lift-ranges` files now resolve when the folder's filenames + disagree in case (`Dict.LIFT` beside `Dict.lift-ranges`, or the reverse). + Such a folder loads on Windows and macOS, whose filesystems fold case, but + on Linux the companion was silently skipped and its ranges went missing. + A candidate that matches no file exactly now falls back to one whose name + differs only in case; where several fold together the lexicographically + first wins, so resolution is stable across runs. + ## [0.1.0] - 2026-07-TBD ### Added diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 8597750..f828391 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -452,6 +452,38 @@ def _normalize_href(href: str) -> Path | None: return Path(normalized) +def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Path | None: + """``candidate`` if it is a file, else one whose name differs only in case. + + LIFT folders are written on Windows, where the filesystem folds case, and + read everywhere. A folder whose ``.lift`` and ``.lift-ranges`` disagree in + case (``Dict.LIFT`` beside ``Dict.lift-ranges``) resolves there and, before + this fallback, silently did not on a case-sensitive filesystem. + + The fallback fires only where the exact name missed, so a case-folding + filesystem never reaches it and nothing changes there. Where several names + fold together, the lexicographically first wins — arbitrary, but stable + across runs, which "whatever the directory yields first" would not be. + ``listings`` caches one directory read per folder. + """ + try: + if candidate.is_file(): + return candidate + except OSError: + return None + folder = candidate.parent + if folder not in listings: + entries: dict[str, Path] = {} + try: + for path in sorted(folder.iterdir()): + if path.is_file(): + entries.setdefault(path.name.lower(), path) + except OSError: + pass # unreadable folder: no candidate resolves out of it + listings[folder] = entries + return listings[folder].get(candidate.name.lower()) + + def _same_dir(left: Path, right: Path | None) -> bool: """Whether two paths denote the same directory, spelling aside. @@ -510,7 +542,10 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L ``range/@href`` both the href resolved as a path relative to the ``.lift`` file and its bare basename in the same directory (FLEx hrefs are usually dangling absolute ``file://C:/...`` paths from the - exporting machine, so the basename is what resolves locally). + exporting machine, so the basename is what resolves locally). A + candidate no file matches exactly still resolves to one whose name + differs only in case, so a folder authored on Windows loads the same + way on a case-sensitive filesystem. A ``.zip`` path is treated as a packaged LIFT folder: it is extracted to a temporary directory (kept alive for the returned lexicon's @@ -544,14 +579,17 @@ def _resolve_ranges(self) -> None: basename = range_.href.replace("\\", "/").rpartition("/")[2] if basename: candidates.append(base / basename) + listings: dict[Path, dict[str, Path]] = {} for candidate in candidates: + found = _existing_file(candidate, listings) + if found is None: + continue try: - resolved = candidate.resolve() - exists = candidate.is_file() + resolved = found.resolve() except OSError: continue - if exists and resolved not in self.ranges_files: - self.ranges_files[resolved] = RangesFile.load(candidate) + if resolved not in self.ranges_files: + self.ranges_files[resolved] = RangesFile.load(found) def save(self, path: str | os.PathLike[str] | None = None) -> None: """Write the ``.lift`` file and every tracked ``.lift-ranges`` companion. diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 9544db9..336a6bc 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -225,6 +225,55 @@ def test_missing_media_flags_broken_ref(tmp_path: Path) -> None: assert [r.href for r in missing] == ["pictures\\sdd.png"] +def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> Path: + """A loadable .lift plus companion under arbitrary filename casing. + + Named off the fixture stem so the header's ``range/@href`` basename + candidate finds nothing — only the sibling candidate can resolve these. + """ + folder.mkdir(parents=True, exist_ok=True) + (folder / lift_name).write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + (folder / ranges_name).write_bytes((PAIR_DIR / "test20080407.lift-ranges").read_bytes()) + return folder / lift_name + + +def _case_sensitive_fs(folder: Path) -> bool: + (folder / "CaseProbe").mkdir() + return not (folder / "caseprobe").exists() + + +def test_companion_resolves_when_lift_suffix_is_uppercase(tmp_path: Path) -> None: + lift = _write_case_variant_pair(tmp_path / "pkg", "Dict.LIFT", "Dict.lift-ranges") + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_companion_resolves_when_companion_suffix_is_uppercase(tmp_path: Path) -> None: + lift = _write_case_variant_pair(tmp_path / "pkg", "Dict.lift", "Dict.LIFT-RANGES") + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_case_folded_companions_resolve_deterministically(tmp_path: Path) -> None: + if not _case_sensitive_fs(tmp_path): + pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") + # Neither spelling matches the Dict.LIFT-ranges candidate exactly, so the + # tie-break picks one: lexicographically first, the same one every run. + folder = tmp_path / "pkg" + lift = _write_case_variant_pair(folder, "Dict.LIFT", "Dict.lift-ranges") + (folder / "Dict.Lift-ranges").write_bytes((folder / "Dict.lift-ranges").read_bytes()) + lexicon = sil_lift.load(lift) + assert [path.name for path in lexicon.ranges_files] == ["Dict.Lift-ranges"] + + +def test_absent_companion_stays_absent(tmp_path: Path) -> None: + # The fallback must not reach past a folder for a name that isn't in it. + folder = tmp_path / "pkg" + folder.mkdir() + (folder / "Dict.lift").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + assert sil_lift.load(folder / "Dict.lift").ranges_files == {} + + @pytest.mark.parametrize( ("href", "expected"), [ From 66f52d712c57b84e89ea3fa13e892021ac0a1752 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 4 Aug 2026 16:43:47 -0400 Subject: [PATCH 02/16] Describe case-tolerant companion discovery under 0.1.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0.1.0 has not shipped, so there is no released behavior for an Unreleased entry to be fixing — the tolerance is simply part of what companion discovery does in the first release. Fold it into that bullet. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 770cf2d..1af5527 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,16 +18,6 @@ releases may contain breaking changes. ## [Unreleased] -### Fixed - -- Companion `.lift-ranges` files now resolve when the folder's filenames - disagree in case (`Dict.LIFT` beside `Dict.lift-ranges`, or the reverse). - Such a folder loads on Windows and macOS, whose filesystems fold case, but - on Linux the companion was silently skipped and its ranges went missing. - A candidate that matches no file exactly now falls back to one whose name - differs only in case; where several fold together the lexicographically - first wins, so resolution is stable across runs. - ## [0.1.0] - 2026-07-TBD ### Added @@ -61,13 +51,14 @@ releases may contain breaking changes. against the most recent `save()`. - LIFT-folder handling: `RangesFile` (standalone `.lift-ranges` documents, same fidelity guarantees), automatic companion discovery/tracking on load - (`Lexicon.ranges_files`), `save()` writes companions together, - `all_ranges()` merged view, `media_refs()` / `missing_media()` helpers, - build-from-scratch helpers `Lexicon.add_ranges_file()` / - `RangesFile.add_range()` / `Range.add_element()` (`save()` writes and - header-references a new companion beside the `.lift`); vendored - `schemas/lift-ranges-0.13.rng` — the first schema for standalone - ranges documents. + (`Lexicon.ranges_files`, resolving a companion whose filename differs from + the `.lift` only in case, as Windows-authored folders often do), `save()` + writes companions together, `all_ranges()` merged view, `media_refs()` / + `missing_media()` helpers, build-from-scratch helpers + `Lexicon.add_ranges_file()` / `RangesFile.add_range()` / + `Range.add_element()` (`save()` writes and header-references a new companion + beside the `.lift`); vendored `schemas/lift-ranges-0.13.rng` — the first + schema for standalone ranges documents. - Zipped LIFT packages: `sil_lift.load()` reads a `.zip` (both the flat and folder-wrapped layouts, junk entries like `__MACOSX` ignored), `Lexicon.save_zip()` writes one (carrying media, `WritingSystems/`, and other From b863047104b0ca42f8e2125528895adde75c6f00 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 12 Aug 2026 15:16:08 -0400 Subject: [PATCH 03/16] Keep "entry" for LIFT entries in the companion-case fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The directory listing the fallback builds called its files "entries", the word this module uses for a LIFT everywhere else — the same collision that keeps byte regions from being called spans. Name them files. Spell the surrounding prose the way the rest of the package does: a fallback that runs rather than fires, a name that matched no file rather than missed, a helper named for the filesystem it probes rather than abbreviating it, and fixture names deliberately not taken from the corpus file. Unpack the two densest clauses so each reads in one pass. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 22 +++++++++++----------- tests/test_ranges_folder.py | 10 +++++----- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index f828391..18b3db9 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -460,11 +460,11 @@ def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Pa case (``Dict.LIFT`` beside ``Dict.lift-ranges``) resolves there and, before this fallback, silently did not on a case-sensitive filesystem. - The fallback fires only where the exact name missed, so a case-folding - filesystem never reaches it and nothing changes there. Where several names - fold together, the lexicographically first wins — arbitrary, but stable - across runs, which "whatever the directory yields first" would not be. - ``listings`` caches one directory read per folder. + The fallback runs only where the exact name matched no file, so a + case-folding filesystem never reaches it and nothing changes there. Where + several names fold together, the lexicographically first wins — arbitrary, + but stable across runs, which "whatever the directory yields first" would + not be. ``listings`` caches one directory read per folder. """ try: if candidate.is_file(): @@ -473,14 +473,14 @@ def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Pa return None folder = candidate.parent if folder not in listings: - entries: dict[str, Path] = {} + files: dict[str, Path] = {} try: for path in sorted(folder.iterdir()): if path.is_file(): - entries.setdefault(path.name.lower(), path) + files.setdefault(path.name.lower(), path) except OSError: pass # unreadable folder: no candidate resolves out of it - listings[folder] = entries + listings[folder] = files return listings[folder].get(candidate.name.lower()) @@ -543,9 +543,9 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L ``.lift`` file and its bare basename in the same directory (FLEx hrefs are usually dangling absolute ``file://C:/...`` paths from the exporting machine, so the basename is what resolves locally). A - candidate no file matches exactly still resolves to one whose name - differs only in case, so a folder authored on Windows loads the same - way on a case-sensitive filesystem. + candidate that no file matches exactly still resolves to a file whose + name differs only in case, so a folder authored on Windows loads the + same way on a case-sensitive filesystem. A ``.zip`` path is treated as a packaged LIFT folder: it is extracted to a temporary directory (kept alive for the returned lexicon's diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 336a6bc..4869bc2 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -228,8 +228,8 @@ def test_missing_media_flags_broken_ref(tmp_path: Path) -> None: def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> Path: """A loadable .lift plus companion under arbitrary filename casing. - Named off the fixture stem so the header's ``range/@href`` basename - candidate finds nothing — only the sibling candidate can resolve these. + Deliberately not named after the fixture, so the header's ``range/@href`` + basename candidate finds nothing — only the sibling candidate resolves these. """ folder.mkdir(parents=True, exist_ok=True) (folder / lift_name).write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) @@ -237,7 +237,7 @@ def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> return folder / lift_name -def _case_sensitive_fs(folder: Path) -> bool: +def _case_sensitive_filesystem(folder: Path) -> bool: (folder / "CaseProbe").mkdir() return not (folder / "caseprobe").exists() @@ -255,7 +255,7 @@ def test_companion_resolves_when_companion_suffix_is_uppercase(tmp_path: Path) - def test_case_folded_companions_resolve_deterministically(tmp_path: Path) -> None: - if not _case_sensitive_fs(tmp_path): + if not _case_sensitive_filesystem(tmp_path): pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") # Neither spelling matches the Dict.LIFT-ranges candidate exactly, so the # tie-break picks one: lexicographically first, the same one every run. @@ -267,7 +267,7 @@ def test_case_folded_companions_resolve_deterministically(tmp_path: Path) -> Non def test_absent_companion_stays_absent(tmp_path: Path) -> None: - # The fallback must not reach past a folder for a name that isn't in it. + # The fallback must not look outside the folder for a name not in it. folder = tmp_path / "pkg" folder.mkdir() (folder / "Dict.lift").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) From ed2d26bdac33ea4659016766f25d1cfb1377e2a7 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 18 Aug 2026 17:23:33 -0400 Subject: [PATCH 04/16] Resolve companions by folded name, and never load the .lift as one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion lookup folds names with casefold() over NFC rather than lower(), so a Turkish-cased or NFD-spelled name resolves the way it does on the filesystem that wrote it — FLEx mixes normalization forms within one export. Ties break in code point order on every platform; sorting Path objects left the choice to directory order on Windows, where PurePath ordering is itself case-folded. An unstattable exact spelling now falls through to the folded lookup instead of giving up. A candidate that folds onto the .lift itself is skipped: RangesFile.load rejects a root, so a header href naming the lexicon in another case took the whole load down. One that folds onto a companion already tracked is skipped too — Path.resolve() leaves case alone on macOS, so a single file reached under two spellings was loaded and tracked twice, and written twice by save(). The sibling candidate is built with with_name, which agrees with with_suffix on every name that has an extension and does not raise on a name without one. Nothing upstream requires the .lift extension: parse_document never inspects it. dangling-ranges-href decides existence with that same lookup, so a companion spelled in another case is no longer reported missing on a case-sensitive filesystem. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 85 ++++++++++++++++++++++++++++++------- src/sil_lift/_validate.py | 9 ++-- tests/test_ranges_folder.py | 66 ++++++++++++++++++++++++++-- 3 files changed, 138 insertions(+), 22 deletions(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 18b3db9..59f7614 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -11,6 +11,7 @@ from __future__ import annotations +import unicodedata from dataclasses import dataclass, field from datetime import date, datetime from pathlib import Path, PurePosixPath, PureWindowsPath @@ -452,36 +453,73 @@ def _normalize_href(href: str) -> Path | None: return Path(normalized) +def _fold(text: str) -> str: + """A filename reduced to what a forgiving filesystem treats as one name. + + ``casefold`` rather than ``lower`` for the non-ASCII cases (Turkish dotted + and dotless I), and NFC because normalization forms get mixed within a + single export — FLEx writes the ``.lift`` in NFC and its companion in NFD — + and macOS folds them together where Linux does not. Neither NTFS's nor + APFS's own folding table is reproduced exactly; this is an approximation + over LIFT filenames, not a general equivalence. + """ + return unicodedata.normalize("NFC", text).casefold() + + def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Path | None: - """``candidate`` if it is a file, else one whose name differs only in case. + """``candidate`` if it is a file, else one whose name differs only in spelling. LIFT folders are written on Windows, where the filesystem folds case, and read everywhere. A folder whose ``.lift`` and ``.lift-ranges`` disagree in case (``Dict.LIFT`` beside ``Dict.lift-ranges``) resolves there and, before - this fallback, silently did not on a case-sensitive filesystem. + this fallback, silently did not on a case-sensitive filesystem. See + :func:`_fold` for what counts as the same name. The fallback runs only where the exact name matched no file, so a - case-folding filesystem never reaches it and nothing changes there. Where - several names fold together, the lexicographically first wins — arbitrary, - but stable across runs, which "whatever the directory yields first" would - not be. ``listings`` caches one directory read per folder. + case-folding filesystem never reaches it and nothing changes there. Only + the final path component is folded: a candidate under a *directory* spelled + in another case still does not resolve, which the hrefs this serves — bare + basenames, or relatives within the folder — do not need. Where several + names fold together, the first in code point order wins (so ``Dict.LIFT`` + ahead of ``Dict.lift``) — arbitrary, but stable across runs and platforms, + which "whatever the directory yields first" would not be. ``listings`` + caches one directory read per folder. """ try: if candidate.is_file(): return candidate except OSError: - return None + pass # unstattable exact spelling: a case variant of it may still stat folder = candidate.parent if folder not in listings: files: dict[str, Path] = {} try: - for path in sorted(folder.iterdir()): + # By name, not by Path: PurePath ordering is case-folded on Windows, + # which would leave the tie-break to directory order there. + for path in sorted(folder.iterdir(), key=lambda entry: entry.name): if path.is_file(): - files.setdefault(path.name.lower(), path) + files.setdefault(_fold(path.name), path) except OSError: pass # unreadable folder: no candidate resolves out of it listings[folder] = files - return listings[folder].get(candidate.name.lower()) + return listings[folder].get(_fold(candidate.name)) + + +def _same_file(left: Path, right: Path) -> bool: + """Whether two paths differing only in spelling denote one file. + + ``Path.resolve()`` canonicalizes case on Windows but not on macOS, so on + the latter one file reached under two spellings yields two distinct keys — + tracked twice, and written twice by :meth:`Lexicon.save`. Only paths that + :func:`_fold` together are compared, since the inode check alone would + conflate distinct files on the filesystems that report ``st_ino`` as 0. + """ + if _fold(str(left)) != _fold(str(right)): + return False + try: + return left.samefile(right) + except OSError: + return False def _same_dir(left: Path, right: Path | None) -> bool: @@ -544,8 +582,9 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L hrefs are usually dangling absolute ``file://C:/...`` paths from the exporting machine, so the basename is what resolves locally). A candidate that no file matches exactly still resolves to a file whose - name differs only in case, so a folder authored on Windows loads the - same way on a case-sensitive filesystem. + name differs only in case or Unicode normalization, so a folder + authored on Windows loads the same way on a case-sensitive filesystem. + The ``.lift`` itself is never taken as its own companion. A ``.zip`` path is treated as a packaged LIFT folder: it is extracted to a temporary directory (kept alive for the returned lexicon's @@ -567,9 +606,16 @@ def _resolve_ranges(self) -> None: if self.path is None: return base = self.path.parent + try: + own = self.path.resolve() + except OSError: + own = self.path candidates: list[Path] = [] - sibling = self.path.with_suffix(self.path.suffix + "-ranges") - candidates.append(sibling) + # with_name, not with_suffix: they agree on every name that has an + # extension, but with_suffix rejects "-ranges" outright on a name + # without one, and nothing upstream requires the document to be named + # ``.lift`` — parse_document never looks at the extension. + candidates.append(self.path.with_name(self.path.name + "-ranges")) for range_ in self.header.ranges: if range_.href is None: continue @@ -588,8 +634,15 @@ def _resolve_ranges(self) -> None: resolved = found.resolve() except OSError: continue - if resolved not in self.ranges_files: - self.ranges_files[resolved] = RangesFile.load(found) + # Skip a spelling of something already tracked (macOS keeps two + # keys for one file) and the .lift itself, which a header href + # naming it in another case now folds onto — RangesFile.load would + # reject its root and take the whole load down with it. + if resolved in self.ranges_files or any( + _same_file(resolved, other) for other in (own, *self.ranges_files) + ): + continue + self.ranges_files[resolved] = RangesFile.load(found) def save(self, path: str | os.PathLike[str] | None = None) -> None: """Write the ``.lift`` file and every tracked ``.lift-ranges`` companion. diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index 5961640..070ad14 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -40,7 +40,7 @@ from lxml import etree from ._errors import LiftValidationError, LiftWriteError -from ._model import GrammaticalInfo, Lexicon, _normalize_href +from ._model import GrammaticalInfo, Lexicon, _existing_file, _normalize_href from ._text import Multitext, Trait if TYPE_CHECKING: @@ -513,9 +513,12 @@ def range_named(name: str) -> str | None: # Absolute/file:// hrefs are ones FLEx writes knowing they will not resolve # (they are resolved by basename when the companion is in the same folder) # and are not checked here; this catches an exporter that writes a relative - # href but not the file. + # href but not the file. Existence is the same notion load resolves + # companions by (_existing_file), so a companion spelled in another case is + # not reported missing on a case-sensitive filesystem. if lexicon.path is not None: base = lexicon.path.parent + listings: dict[Path, dict[str, Path]] = {} for range_ in lexicon.header.ranges: if not range_.href or range_.elements: continue @@ -524,7 +527,7 @@ def range_named(name: str) -> str | None: continue if header_ranges[range_.id] is not None: continue # supplied by a sibling companion instead - if not (base / relative).is_file(): + if _existing_file(base / relative, listings) is None: yield Problem( "warning", "dangling-ranges-href", diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 4869bc2..fc0d2b2 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -1,4 +1,5 @@ import shutil +import unicodedata from pathlib import Path import pytest @@ -226,7 +227,7 @@ def test_missing_media_flags_broken_ref(tmp_path: Path) -> None: def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> Path: - """A loadable .lift plus companion under arbitrary filename casing. + """A loadable .lift plus companion under arbitrary filename spellings. Deliberately not named after the fixture, so the header's ``range/@href`` basename candidate finds nothing — only the sibling candidate resolves these. @@ -237,9 +238,22 @@ def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> return folder / lift_name +def _write_lift_with_href(folder: Path, lift_name: str, href: str) -> Path: + """The fixture .lift under another name, its companion href rewritten.""" + folder.mkdir(parents=True, exist_ok=True) + source = (PAIR_DIR / "test20080407.lift").read_bytes() + patched = source.replace(b'"file://test20080407.lift-ranges"', f'"{href}"'.encode()) + assert patched != source, "fixture href changed; the replacement no longer matches" + (folder / lift_name).write_bytes(patched) + return folder / lift_name + + def _case_sensitive_filesystem(folder: Path) -> bool: - (folder / "CaseProbe").mkdir() - return not (folder / "caseprobe").exists() + probe = folder / "CaseProbe" + probe.mkdir(exist_ok=True) + sensitive = not (folder / "caseprobe").exists() + probe.rmdir() + return sensitive def test_companion_resolves_when_lift_suffix_is_uppercase(tmp_path: Path) -> None: @@ -274,6 +288,52 @@ def test_absent_companion_stays_absent(tmp_path: Path) -> None: assert sil_lift.load(folder / "Dict.lift").ranges_files == {} +def test_companion_resolves_across_unicode_normalization(tmp_path: Path) -> None: + # FLEx mixes NFC and NFD within one export, and the mismatch reaches the + # filenames; only macOS folds the two forms together on its own. + composed = "Caf\N{LATIN SMALL LETTER E WITH ACUTE}.lift" + decomposed = unicodedata.normalize("NFD", f"{composed}-ranges") + lift = _write_case_variant_pair(tmp_path / "pkg", composed, decomposed) + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_lift_without_an_extension_loads(tmp_path: Path) -> None: + # Loading never inspects the extension, so the sibling candidate is built + # from a name that may have none; this companion is the href's basename. + folder = tmp_path / "pkg" + folder.mkdir() + (folder / "Dict").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + shutil.copy(PAIR_DIR / "test20080407.lift-ranges", folder) + lexicon = sil_lift.load(folder / "Dict") + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_href_folding_onto_the_lift_itself_is_not_a_companion(tmp_path: Path) -> None: + # Dict.lift beside a Dict.LIFT is the lexicon, not its ranges: loading it + # as one would raise on the root and take the whole load down. + lift = _write_lift_with_href(tmp_path / "pkg", "Dict.LIFT", "Dict.lift") + assert sil_lift.load(lift).ranges_files == {} + + +# Defines the range the header points at, but no elements — so the merged view +# cannot vouch for the href and the check falls through to the filesystem. +ELEMENTLESS_RANGES = b""" + + + +""" + + +def test_case_variant_companion_is_not_reported_dangling(tmp_path: Path) -> None: + folder = tmp_path / "pkg" + lift = _write_lift_with_href(folder, "Dict.LIFT", "Dict.LIFT-ranges") + (folder / "Dict.lift-ranges").write_bytes(ELEMENTLESS_RANGES) + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files # the companion resolved + assert [p for p in lexicon.iter_problems() if p.code == "dangling-ranges-href"] == [] + + @pytest.mark.parametrize( ("href", "expected"), [ From 0cd9f95616e99b7ec590ee14a1ad5e5438c5143e Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 18 Aug 2026 17:23:53 -0400 Subject: [PATCH 05/16] Describe case- and normalization-tolerant companion discovery The 0.1.0 entry said companion names fold on case alone; they fold on Unicode normalization form as well. The folder guide listed the candidates tried but never mentioned the folding at all. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 15 ++++++++------- docs/en/guides/folder-media.md | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1af5527..190cc62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,13 +52,14 @@ releases may contain breaking changes. - LIFT-folder handling: `RangesFile` (standalone `.lift-ranges` documents, same fidelity guarantees), automatic companion discovery/tracking on load (`Lexicon.ranges_files`, resolving a companion whose filename differs from - the `.lift` only in case, as Windows-authored folders often do), `save()` - writes companions together, `all_ranges()` merged view, `media_refs()` / - `missing_media()` helpers, build-from-scratch helpers - `Lexicon.add_ranges_file()` / `RangesFile.add_range()` / - `Range.add_element()` (`save()` writes and header-references a new companion - beside the `.lift`); vendored `schemas/lift-ranges-0.13.rng` — the first - schema for standalone ranges documents. + the `.lift` only in case or Unicode normalization form, as Windows- and + FLEx-authored folders do), `save()` writes companions together, + `all_ranges()` merged view, `media_refs()` / `missing_media()` helpers, + build-from-scratch helpers `Lexicon.add_ranges_file()` / + `RangesFile.add_range()` / `Range.add_element()` (`save()` writes and + header-references a new companion beside the `.lift`); vendored + `schemas/lift-ranges-0.13.rng` — the first schema for standalone ranges + documents. - Zipped LIFT packages: `sil_lift.load()` reads a `.zip` (both the flat and folder-wrapped layouts, junk entries like `__MACOSX` ignored), `Lexicon.save_zip()` writes one (carrying media, `WritingSystems/`, and other diff --git a/docs/en/guides/folder-media.md b/docs/en/guides/folder-media.md index 16497fb..ddbeb56 100644 --- a/docs/en/guides/folder-media.md +++ b/docs/en/guides/folder-media.md @@ -12,7 +12,7 @@ lex.all_ranges() # merged {id: Range} view lex.all_ranges()["grammatical-info"].elements ``` -Companion discovery handles the real world: a `range/@href` that points at an existing file is used; FieldWorks' dangling absolute `file://C:/...` hrefs fall back to the href's basename next to the `.lift`; and the conventional `.lift-ranges` sibling is picked up even when nothing references it. +Companion discovery handles the real world: a `range/@href` that points at an existing file is used; FieldWorks' dangling absolute `file://C:/...` hrefs fall back to the href's basename next to the `.lift`; and the conventional `.lift-ranges` sibling is picked up even when nothing references it. Each of those resolves a name that differs only in case or Unicode normalization form, so a folder written on Windows — `Dict.LIFT` beside `Dict.lift-ranges` — loads the same way on a case-sensitive filesystem. `lex.save()` writes the `.lift` and every tracked companion together. Edits to a `RangesFile` save back to _its_ file; untouched ranges keep their exact bytes. Standalone use: From 8d1c24a6b2220fed38edc0e0e842b833cb8402b4 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 18 Aug 2026 17:55:45 -0400 Subject: [PATCH 06/16] Trim the companion-folding prose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three helpers each stated a rule, defended it, then disclaimed it. What is left is the reasoning the code cannot show: casefold over lower, NFC, why only the final component folds, why code point order, and what the fold pre-check protects the inode comparison from. The folder guide drops the folding sentence outright — it describes behavior no reader acts on, in a paragraph otherwise about which candidate wins. The 0.1.0 entry keeps the fact and loses the justification, which now lives only in the docstrings. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 5 ++-- docs/en/guides/folder-media.md | 2 +- src/sil_lift/_model.py | 49 ++++++++++++++-------------------- 3 files changed, 23 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 190cc62..dadecbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,9 +51,8 @@ releases may contain breaking changes. against the most recent `save()`. - LIFT-folder handling: `RangesFile` (standalone `.lift-ranges` documents, same fidelity guarantees), automatic companion discovery/tracking on load - (`Lexicon.ranges_files`, resolving a companion whose filename differs from - the `.lift` only in case or Unicode normalization form, as Windows- and - FLEx-authored folders do), `save()` writes companions together, + (`Lexicon.ranges_files`, matching companion filenames across case and + Unicode normalization differences), `save()` writes companions together, `all_ranges()` merged view, `media_refs()` / `missing_media()` helpers, build-from-scratch helpers `Lexicon.add_ranges_file()` / `RangesFile.add_range()` / `Range.add_element()` (`save()` writes and diff --git a/docs/en/guides/folder-media.md b/docs/en/guides/folder-media.md index ddbeb56..16497fb 100644 --- a/docs/en/guides/folder-media.md +++ b/docs/en/guides/folder-media.md @@ -12,7 +12,7 @@ lex.all_ranges() # merged {id: Range} view lex.all_ranges()["grammatical-info"].elements ``` -Companion discovery handles the real world: a `range/@href` that points at an existing file is used; FieldWorks' dangling absolute `file://C:/...` hrefs fall back to the href's basename next to the `.lift`; and the conventional `.lift-ranges` sibling is picked up even when nothing references it. Each of those resolves a name that differs only in case or Unicode normalization form, so a folder written on Windows — `Dict.LIFT` beside `Dict.lift-ranges` — loads the same way on a case-sensitive filesystem. +Companion discovery handles the real world: a `range/@href` that points at an existing file is used; FieldWorks' dangling absolute `file://C:/...` hrefs fall back to the href's basename next to the `.lift`; and the conventional `.lift-ranges` sibling is picked up even when nothing references it. `lex.save()` writes the `.lift` and every tracked companion together. Edits to a `RangesFile` save back to _its_ file; untouched ranges keep their exact bytes. Standalone use: diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 59f7614..3984f7d 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -454,36 +454,28 @@ def _normalize_href(href: str) -> Path | None: def _fold(text: str) -> str: - """A filename reduced to what a forgiving filesystem treats as one name. - - ``casefold`` rather than ``lower`` for the non-ASCII cases (Turkish dotted - and dotless I), and NFC because normalization forms get mixed within a - single export — FLEx writes the ``.lift`` in NFC and its companion in NFD — - and macOS folds them together where Linux does not. Neither NTFS's nor - APFS's own folding table is reproduced exactly; this is an approximation - over LIFT filenames, not a general equivalence. + """A filename reduced to what a case-folding filesystem treats as one name. + + ``casefold`` for what ``lower`` gets wrong (the Turkish dotless i), NFC + because FLEx mixes normalization forms within one export. An approximation + of NTFS's and APFS's tables, not a general equivalence. """ return unicodedata.normalize("NFC", text).casefold() def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Path | None: - """``candidate`` if it is a file, else one whose name differs only in spelling. + """``candidate`` if it is a file, else one whose name folds onto it. LIFT folders are written on Windows, where the filesystem folds case, and - read everywhere. A folder whose ``.lift`` and ``.lift-ranges`` disagree in - case (``Dict.LIFT`` beside ``Dict.lift-ranges``) resolves there and, before - this fallback, silently did not on a case-sensitive filesystem. See - :func:`_fold` for what counts as the same name. - - The fallback runs only where the exact name matched no file, so a - case-folding filesystem never reaches it and nothing changes there. Only - the final path component is folded: a candidate under a *directory* spelled - in another case still does not resolve, which the hrefs this serves — bare - basenames, or relatives within the folder — do not need. Where several - names fold together, the first in code point order wins (so ``Dict.LIFT`` - ahead of ``Dict.lift``) — arbitrary, but stable across runs and platforms, - which "whatever the directory yields first" would not be. ``listings`` - caches one directory read per folder. + read everywhere: ``Dict.LIFT`` beside ``Dict.lift-ranges`` resolves there + and, before this fallback, silently did not on a case-sensitive filesystem. + + Only the final component folds — the hrefs this serves are basenames or + same-folder relatives — and only after the exact name misses, so a + case-folding filesystem never reaches this. Among names that fold together + the first in code point order wins: arbitrary, but stable across runs and + platforms, which directory order is not. ``listings`` caches one directory + read per folder. """ try: if candidate.is_file(): @@ -506,13 +498,12 @@ def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Pa def _same_file(left: Path, right: Path) -> bool: - """Whether two paths differing only in spelling denote one file. + """Whether two paths that fold together denote one file. - ``Path.resolve()`` canonicalizes case on Windows but not on macOS, so on - the latter one file reached under two spellings yields two distinct keys — - tracked twice, and written twice by :meth:`Lexicon.save`. Only paths that - :func:`_fold` together are compared, since the inode check alone would - conflate distinct files on the filesystems that report ``st_ino`` as 0. + ``Path.resolve()`` canonicalizes case on Windows but not on macOS, where + one file reached under two spellings yields two keys — tracked twice, and + written twice by :meth:`Lexicon.save`. The fold pre-check keeps the inode + comparison from conflating distinct files where ``st_ino`` is 0. """ if _fold(str(left)) != _fold(str(right)): return False From ca36d340eb1f639c21af6a2ead5f903b7ba6bd8e Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 19 Aug 2026 14:11:36 -0400 Subject: [PATCH 07/16] Keep folder-shaped and self-referencing hrefs out of companion resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A candidate that exists as a directory now stops the lookup instead of falling through to the folded listing. An href of "" normalizes to the LIFT folder itself and one of "sub/" to a subfolder, and folding a folder's own name searches its *parent*: any file there spelled like the folder was returned as the companion, and RangesFile.load then rejected its root and failed the whole load. dangling-ranges-href also treats a match that is the .lift itself as no match. _resolve_ranges refuses to take the lexicon for its own companion, so a header href folding onto it resolves to nothing that supplies the range — the reference is as dangling as a missing file, and went unreported on a case-sensitive filesystem. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 4 ++++ src/sil_lift/_validate.py | 7 +++++-- tests/test_ranges_folder.py | 11 +++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 3984f7d..01891b3 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -480,6 +480,10 @@ def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Pa try: if candidate.is_file(): return candidate + if candidate.is_dir(): + # An href of "" or "sub/" lands here; folding a folder's own name + # would search its parent and match anything spelled like it. + return None except OSError: pass # unstattable exact spelling: a case variant of it may still stat folder = candidate.parent diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index 070ad14..a59f667 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -40,7 +40,7 @@ from lxml import etree from ._errors import LiftValidationError, LiftWriteError -from ._model import GrammaticalInfo, Lexicon, _existing_file, _normalize_href +from ._model import GrammaticalInfo, Lexicon, _existing_file, _normalize_href, _same_file from ._text import Multitext, Trait if TYPE_CHECKING: @@ -527,7 +527,10 @@ def range_named(name: str) -> str | None: continue if header_ranges[range_.id] is not None: continue # supplied by a sibling companion instead - if _existing_file(base / relative, listings) is None: + found = _existing_file(base / relative, listings) + # _resolve_ranges refuses the lexicon as its own companion, so an + # href folding onto it supplies nothing and dangles too. + if found is None or _same_file(found, lexicon.path): yield Problem( "warning", "dangling-ranges-href", diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index fc0d2b2..9411afd 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -313,6 +313,17 @@ def test_href_folding_onto_the_lift_itself_is_not_a_companion(tmp_path: Path) -> # Dict.lift beside a Dict.LIFT is the lexicon, not its ranges: loading it # as one would raise on the root and take the whole load down. lift = _write_lift_with_href(tmp_path / "pkg", "Dict.LIFT", "Dict.lift") + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files == {} + assert "dangling-ranges-href" in [p.code for p in lexicon.iter_problems()] + + +def test_folder_shaped_href_stays_inside_the_folder(tmp_path: Path) -> None: + if not _case_sensitive_filesystem(tmp_path): + pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") + # An empty href names the folder itself; folding it would search the parent. + lift = _write_lift_with_href(tmp_path / "pkg", "Dict.lift", "") + (tmp_path / "PKG").write_bytes(b"") assert sil_lift.load(lift).ranges_files == {} From 8ffa816b96a92fea83e04532fe42d0656fea586c Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 19 Aug 2026 15:32:59 -0400 Subject: [PATCH 08/16] Resolve both sides before deciding two paths are one file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _same_file compared the spellings it was handed, so a caller had to canonicalize first to get a true answer. _resolve_ranges did; the dangling-ranges-href check did not, and an href reaching the .lift through a ".." segment or a symlink read there as some other file. The loader skipped that candidate as self-referential while validation counted it as a companion that exists, leaving the header range both unsupplied and unreported. Resolving inside _same_file makes the answer independent of how the caller spelled its arguments, and retires the loader's own pre-resolution — the duplicate that let the two drift apart. A path that will not resolve now compares false instead of falling back to the spelling as given; it would fail the samefile stat on the next line regardless. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 15 ++++++--------- tests/test_ranges_folder.py | 9 +++++++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 01891b3..a49a10b 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -506,12 +506,13 @@ def _same_file(left: Path, right: Path) -> bool: ``Path.resolve()`` canonicalizes case on Windows but not on macOS, where one file reached under two spellings yields two keys — tracked twice, and - written twice by :meth:`Lexicon.save`. The fold pre-check keeps the inode - comparison from conflating distinct files where ``st_ino`` is 0. + written twice by :meth:`Lexicon.save`. Both sides resolve first, so an + href's ``..`` or a symlink compares alike; the fold pre-check then keeps + the inode comparison from conflating distinct files where ``st_ino`` is 0. """ - if _fold(str(left)) != _fold(str(right)): - return False try: + if _fold(str(left.resolve())) != _fold(str(right.resolve())): + return False return left.samefile(right) except OSError: return False @@ -601,10 +602,6 @@ def _resolve_ranges(self) -> None: if self.path is None: return base = self.path.parent - try: - own = self.path.resolve() - except OSError: - own = self.path candidates: list[Path] = [] # with_name, not with_suffix: they agree on every name that has an # extension, but with_suffix rejects "-ranges" outright on a name @@ -634,7 +631,7 @@ def _resolve_ranges(self) -> None: # naming it in another case now folds onto — RangesFile.load would # reject its root and take the whole load down with it. if resolved in self.ranges_files or any( - _same_file(resolved, other) for other in (own, *self.ranges_files) + _same_file(resolved, other) for other in (self.path, *self.ranges_files) ): continue self.ranges_files[resolved] = RangesFile.load(found) diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 9411afd..3669b07 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -318,6 +318,15 @@ def test_href_folding_onto_the_lift_itself_is_not_a_companion(tmp_path: Path) -> assert "dangling-ranges-href" in [p.code for p in lexicon.iter_problems()] +def test_self_referencing_href_dangles_however_it_is_spelled(tmp_path: Path) -> None: + # The ".." keeps the href from matching the lexicon's path as spelled, so + # both sides have to resolve before deciding what the reference supplies. + lift = _write_lift_with_href(tmp_path / "pkg", "Dict.LIFT", "../pkg/Dict.lift") + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files == {} + assert "dangling-ranges-href" in [p.code for p in lexicon.iter_problems()] + + def test_folder_shaped_href_stays_inside_the_folder(tmp_path: Path) -> None: if not _case_sensitive_filesystem(tmp_path): pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") From b0d66535bed36ed9cb5741dcbb32fc106aeb07d8 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 19 Aug 2026 16:28:29 -0400 Subject: [PATCH 09/16] Say what the folding rules are, not how they got there MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _fold cited FLEx's mixed NFC/NFD content as its reason to normalize filenames. That is a different code path and no evidence for the names; decomposed filenames arrive from macOS, so say that instead. _existing_file described itself against the behavior it replaced, which reads oddly once nothing remembers that behavior. Lexicon.load's candidate list and its matching rules split into separate paragraphs, and "every one that exists is loaded" becomes "every distinct file among them" — candidates resolving to the lexicon, to a directory, or to a file already tracked all exist and are deliberately skipped. The dangling-ranges-href comment leads with what the check catches rather than closing with it, and loses a restatement of _existing_file's own docstring. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 29 ++++++++++++++--------------- src/sil_lift/_validate.py | 11 ++++------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index a49a10b..d7cfb89 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -456,9 +456,9 @@ def _normalize_href(href: str) -> Path | None: def _fold(text: str) -> str: """A filename reduced to what a case-folding filesystem treats as one name. - ``casefold`` for what ``lower`` gets wrong (the Turkish dotless i), NFC - because FLEx mixes normalization forms within one export. An approximation - of NTFS's and APFS's tables, not a general equivalence. + ``casefold`` for what ``lower`` gets wrong (the Turkish dotless i), NFC for + names that arrive decomposed, as ones written or zipped on macOS do. An + approximation of NTFS's and APFS's tables, not a general equivalence. """ return unicodedata.normalize("NFC", text).casefold() @@ -467,15 +467,14 @@ def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Pa """``candidate`` if it is a file, else one whose name folds onto it. LIFT folders are written on Windows, where the filesystem folds case, and - read everywhere: ``Dict.LIFT`` beside ``Dict.lift-ranges`` resolves there - and, before this fallback, silently did not on a case-sensitive filesystem. + read everywhere. ``Dict.LIFT`` beside ``Dict.lift-ranges`` is a pair there + and nowhere else. Only the final component folds — the hrefs this serves are basenames or same-folder relatives — and only after the exact name misses, so a case-folding filesystem never reaches this. Among names that fold together - the first in code point order wins: arbitrary, but stable across runs and - platforms, which directory order is not. ``listings`` caches one directory - read per folder. + the first in code point order wins: arbitrary, but stable, which directory + order is not. ``listings`` caches one directory read per folder. """ try: if candidate.is_file(): @@ -571,16 +570,16 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L With ``resolve_ranges`` (the default), companion ``.lift-ranges`` files are loaded and tracked in :attr:`ranges_files`. Several - candidates are tried and every one that exists is loaded: the - conventional ``.lift-ranges`` sibling, and for each header + candidates are tried and every distinct file among them is loaded: + the conventional ``.lift-ranges`` sibling, and for each header ``range/@href`` both the href resolved as a path relative to the ``.lift`` file and its bare basename in the same directory (FLEx hrefs are usually dangling absolute ``file://C:/...`` paths from the - exporting machine, so the basename is what resolves locally). A - candidate that no file matches exactly still resolves to a file whose - name differs only in case or Unicode normalization, so a folder - authored on Windows loads the same way on a case-sensitive filesystem. - The ``.lift`` itself is never taken as its own companion. + exporting machine, so the basename is what resolves locally). + + A candidate matching no file exactly resolves across differences in + case or Unicode normalization, so a folder authored on Windows loads + the same way everywhere; the ``.lift`` is never its own companion. A ``.zip`` path is treated as a packaged LIFT folder: it is extracted to a temporary directory (kept alive for the returned lexicon's diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index a59f667..85f27ab 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -509,13 +509,10 @@ def range_named(name: str) -> str | None: # here, not inside the file check below, which a lexicon with no path skips. header_ranges = {range_.id: range_named(range_.id) for range_ in lexicon.header.ranges} - # Header references (relative) that resolve to no companion. - # Absolute/file:// hrefs are ones FLEx writes knowing they will not resolve - # (they are resolved by basename when the companion is in the same folder) - # and are not checked here; this catches an exporter that writes a relative - # href but not the file. Existence is the same notion load resolves - # companions by (_existing_file), so a companion spelled in another case is - # not reported missing on a case-sensitive filesystem. + # Header references that resolve to no companion — an exporter + # that wrote the href but not the file. Absolute and file:// hrefs are + # skipped: FLEx writes those knowing they will not resolve, and load reaches + # their companions by basename in the same folder instead. if lexicon.path is not None: base = lexicon.path.parent listings: dict[Path, dict[str, Path]] = {} From 1202a1eb386a13750eec8ff5472a2ac5b410ed6e Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Thu, 20 Aug 2026 08:18:45 -0400 Subject: [PATCH 10/16] Rework the companion-folding prose, and name a predicate like one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docstrings and comments only, plus one rename. _existing_file claimed a case-folding filesystem never reaches the fallback. That is false on NTFS, which folds case but not normalization: an NFD companion misses the exact stat and only the fallback finds it. The guarantee that does hold everywhere — an exact hit is returned unchanged — leads instead. Its summary named the argument rather than the return value, and its motivation read as though LIFT folders can only be written on Windows, when what matters is whether the authoring filesystem folds case, as macOS also does. The with_name comment justified a choice against with_suffix rather than warning about it. Naming the hazard is what stops someone reaching for the tidier call and reintroducing the raise it avoids. _case_sensitive_filesystem returned a bool under a noun phrase that promises a filesystem; _same_file and _same_dir are the house pattern for a predicate. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 48 ++++++++++++++++++++----------------- tests/test_ranges_folder.py | 6 ++--- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index d7cfb89..e99a30e 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -456,25 +456,28 @@ def _normalize_href(href: str) -> Path | None: def _fold(text: str) -> str: """A filename reduced to what a case-folding filesystem treats as one name. - ``casefold`` for what ``lower`` gets wrong (the Turkish dotless i), NFC for - names that arrive decomposed, as ones written or zipped on macOS do. An + ``casefold`` for what ``lower`` gets wrong (e.g. the Turkish dotless i); + NFC for names that arrive decomposed (e.g. ones zipped on macOS). An approximation of NTFS's and APFS's tables, not a general equivalence. """ return unicodedata.normalize("NFC", text).casefold() def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Path | None: - """``candidate`` if it is a file, else one whose name folds onto it. + """The file ``candidate`` names, matched exactly or by folded name. - LIFT folders are written on Windows, where the filesystem folds case, and - read everywhere. ``Dict.LIFT`` beside ``Dict.lift-ranges`` is a pair there - and nowhere else. + Where the authoring filesystem folds case, as on Windows and macOS, an + inconsistently spelled pair goes unnoticed: ``Dict.LIFT`` beside + ``Dict.lift-ranges`` is a pair there but not everywhere. - Only the final component folds — the hrefs this serves are basenames or - same-folder relatives — and only after the exact name misses, so a - case-folding filesystem never reaches this. Among names that fold together - the first in code point order wins: arbitrary, but stable, which directory - order is not. ``listings`` caches one directory read per folder. + An exact hit is always returned unchanged: folding runs only after the + exact name misses, and then only on the final component — the hrefs this + serves are basenames or same-folder relatives. + + Among names that fold together the first in code point order wins: + arbitrary, but stable, which directory order is not. + + ``listings`` caches one directory read per folder. """ try: if candidate.is_file(): @@ -505,9 +508,11 @@ def _same_file(left: Path, right: Path) -> bool: ``Path.resolve()`` canonicalizes case on Windows but not on macOS, where one file reached under two spellings yields two keys — tracked twice, and - written twice by :meth:`Lexicon.save`. Both sides resolve first, so an - href's ``..`` or a symlink compares alike; the fold pre-check then keeps - the inode comparison from conflating distinct files where ``st_ino`` is 0. + written twice by :meth:`Lexicon.save`. + + Both sides resolve first, so ``..`` segments and symlinks compare alike; + the fold pre-check then keeps the inode comparison from conflating + distinct files where ``st_ino`` is 0. """ try: if _fold(str(left.resolve())) != _fold(str(right.resolve())): @@ -602,10 +607,9 @@ def _resolve_ranges(self) -> None: return base = self.path.parent candidates: list[Path] = [] - # with_name, not with_suffix: they agree on every name that has an - # extension, but with_suffix rejects "-ranges" outright on a name - # without one, and nothing upstream requires the document to be named - # ``.lift`` — parse_document never looks at the extension. + # with_name and with_suffix agree on every name that has an extension, + # but with_suffix would raise on a name that has none — which + # parse_document accepts, since it never inspects the extension. candidates.append(self.path.with_name(self.path.name + "-ranges")) for range_ in self.header.ranges: if range_.href is None: @@ -625,10 +629,10 @@ def _resolve_ranges(self) -> None: resolved = found.resolve() except OSError: continue - # Skip a spelling of something already tracked (macOS keeps two - # keys for one file) and the .lift itself, which a header href - # naming it in another case now folds onto — RangesFile.load would - # reject its root and take the whole load down with it. + # A header href naming the .lift in another case folds onto it, and + # RangesFile.load rejects that root, failing the whole load; two + # spellings of one companion, which resolve() leaves distinct on + # macOS, would load and write it twice. if resolved in self.ranges_files or any( _same_file(resolved, other) for other in (self.path, *self.ranges_files) ): diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 3669b07..dde9835 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -248,7 +248,7 @@ def _write_lift_with_href(folder: Path, lift_name: str, href: str) -> Path: return folder / lift_name -def _case_sensitive_filesystem(folder: Path) -> bool: +def _case_sensitive(folder: Path) -> bool: probe = folder / "CaseProbe" probe.mkdir(exist_ok=True) sensitive = not (folder / "caseprobe").exists() @@ -269,7 +269,7 @@ def test_companion_resolves_when_companion_suffix_is_uppercase(tmp_path: Path) - def test_case_folded_companions_resolve_deterministically(tmp_path: Path) -> None: - if not _case_sensitive_filesystem(tmp_path): + if not _case_sensitive(tmp_path): pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") # Neither spelling matches the Dict.LIFT-ranges candidate exactly, so the # tie-break picks one: lexicographically first, the same one every run. @@ -328,7 +328,7 @@ def test_self_referencing_href_dangles_however_it_is_spelled(tmp_path: Path) -> def test_folder_shaped_href_stays_inside_the_folder(tmp_path: Path) -> None: - if not _case_sensitive_filesystem(tmp_path): + if not _case_sensitive(tmp_path): pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") # An empty href names the folder itself; folding it would search the parent. lift = _write_lift_with_href(tmp_path / "pkg", "Dict.lift", "") From d5c12d9d8501c4294905884210457b3405be16ea Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Thu, 20 Aug 2026 08:56:19 -0400 Subject: [PATCH 11/16] Call the tie-break deterministic rather than stable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Stable" names a specific property in sorting — preserving the relative order of equal elements — which is close enough to what is meant here to be read as a claim about the sort rather than about the outcome. The choice among fold-equal names is deterministic: same folder, same winner, on every platform and every run. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index e99a30e..02ad08e 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -475,7 +475,7 @@ def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Pa serves are basenames or same-folder relatives. Among names that fold together the first in code point order wins: - arbitrary, but stable, which directory order is not. + arbitrary, but deterministic, which directory order is not. ``listings`` caches one directory read per folder. """ From 86c901355cf707edb1f22461692f5d4ffaf5a807 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 25 Aug 2026 12:43:21 -0400 Subject: [PATCH 12/16] Refuse a companion name that several files answer to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Where the exact name missed and more than one file folded onto it, the lookup took the first in code point order. That puts the conventionally spelled Dict.lift-ranges last, behind any oddly cased twin beside it, and picks between files whose contents can differ — a stale export, or a rename git recorded as an add — with nothing said about it. Which one the name meant is not recoverable, so none of them is loaded now. Validation reports the collision as ambiguous-ranges-file, naming every spelling by code point: two can differ only by normalization and render identically. It walks the same candidates load does, so it covers the sibling name as well, which no header href reports on and which is what a folder authored on Windows most often collides over. One finding per colliding group, however many candidates fold onto it; a dangling href beside it says which range went unmet. A name that matches a file exactly still resolves to it. Folding runs only after that miss, so an odd-cased variant sitting beside an exactly named companion is no collision at all. The candidate list moves out of _resolve_ranges into _ranges_candidates for validation to walk, and now yields each path once: an href repeated across ranges, or agreeing with the sibling, was looked up as many times as it was written. A case-only collision cannot be created where the filesystem folds case, and there the exact-name stat folds too — so the test that runs everywhere separates its two companions by which of two accents is decomposed, the one difference NTFS and APFS keep. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 5 ++- docs/en/guides/validate.md | 7 ++- src/sil_lift/_model.py | 87 ++++++++++++++++++++++--------------- src/sil_lift/_validate.py | 52 +++++++++++++++++++--- tests/test_ranges_folder.py | 67 ++++++++++++++++++++++++++-- 5 files changed, 170 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dadecbe..05cee3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,7 +52,8 @@ releases may contain breaking changes. - LIFT-folder handling: `RangesFile` (standalone `.lift-ranges` documents, same fidelity guarantees), automatic companion discovery/tracking on load (`Lexicon.ranges_files`, matching companion filenames across case and - Unicode normalization differences), `save()` writes companions together, + Unicode normalization differences, and loading nothing for a name several + files answer to that way), `save()` writes companions together, `all_ranges()` merged view, `media_refs()` / `missing_media()` helpers, build-from-scratch helpers `Lexicon.add_ranges_file()` / `RangesFile.add_range()` / `Range.add_element()` (`save()` writes and @@ -73,7 +74,7 @@ releases may contain breaking changes. file, entry, and line it concerns. RELAX NG layer with two documented departures from strict validation (invalid `file://` hrefs downgraded to `uri-not-rfc` warnings; legal interleaving not falsely flagged); vendored - ranges schema over companions; and nine semantic checks the grammar cannot + ranges schema over companions; and ten semantic checks the grammar cannot express, one `Problem` code each (with missing-id opt-in via `require_ids`). Names resolve against range and range-element ids under NFC; a match that needed normalizing is reported as normalization-mismatch, once per id. diff --git a/docs/en/guides/validate.md b/docs/en/guides/validate.md index b6fb006..6069cae 100644 --- a/docs/en/guides/validate.md +++ b/docs/en/guides/validate.md @@ -24,14 +24,15 @@ Each `Problem` carries `level` (`"error"`/`"warning"`), a stable `code`, `messag 1. **RELAX NG** against the LIFT 0.13 grammar (vendored from lift-standard — a byte-identical copy committed into this package). 2. **Ranges schema** — this project's `lift-ranges-0.13.rng` — over every tracked `.lift-ranges` companion, addressed to the companion rather than the `.lift`. -3. **Semantic checks** the grammar cannot express — nine of them, one code each. +3. **Semantic checks** the grammar cannot express — ten of them, one code each. ## Problem codes -Every finding carries one of these, whichever layer produced it — `schema` and `uri-not-rfc` come from the schema layers, the other nine are semantic checks. The strings are a supported interface; `--strict` promotes every warning to an error. +Every finding carries one of these, whichever layer produced it — `schema` and `uri-not-rfc` come from the schema layers, the other ten are semantic checks. The strings are a supported interface; `--strict` promotes every warning to an error. | code | level | what it flags | | ------------------------ | ------- | -------------------------------------------------------------------------- | +| `ambiguous-ranges-file` | warning | a companion name matching several files that differ only by case or NFC | | `dangling-ranges-href` | warning | a header `range/@href` resolving to no companion file | | `dangling-ref` | error | a `relation/@ref` or `variant/@ref` matching no entry or sense | | `duplicate-form-lang` | warning | two forms in one multitext sharing a language | @@ -46,6 +47,8 @@ Every finding carries one of these, whichever layer produced it — `schema` and All three layers work from what `save()` would write, so a document that cannot be serialized at all is reported as a single `lone-surrogate` error instead — see [Fidelity guarantees](../fidelity.md#content-xml-cannot-represent). +Companions are matched by folded filename — case and Unicode normalization — so a folder authored on Windows loads the same way on a case-sensitive filesystem. Where that folding leaves one name matching several files, none of them is loaded: which one it meant is not recoverable. `ambiguous-ranges-file` reports the collision; renaming or removing all but one resolves it. + ## Real-world FieldWorks (FLEx) output FieldWorks systematically writes some content that strict tooling rejects. Here is sil-lift's policy, so that real lexicons validate usefully: diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 02ad08e..7e2cc55 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -24,7 +24,7 @@ if TYPE_CHECKING: import os import tempfile - from collections.abc import Iterator + from collections.abc import Iterable, Iterator from typing import Literal from ._validate import Problem @@ -463,44 +463,76 @@ def _fold(text: str) -> str: return unicodedata.normalize("NFC", text).casefold() -def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Path | None: - """The file ``candidate`` names, matched exactly or by folded name. +def _folded_matches(candidate: Path, listings: dict[Path, dict[str, list[Path]]]) -> list[Path]: + """The files ``candidate`` names: the exact one, or those folding onto it. Where the authoring filesystem folds case, as on Windows and macOS, an inconsistently spelled pair goes unnoticed: ``Dict.LIFT`` beside ``Dict.lift-ranges`` is a pair there but not everywhere. - An exact hit is always returned unchanged: folding runs only after the - exact name misses, and then only on the final component — the hrefs this - serves are basenames or same-folder relatives. + An exact hit is the only match: folding runs only after the exact name + misses, and then only on the final component — the hrefs this serves are + basenames or same-folder relatives. - Among names that fold together the first in code point order wins: - arbitrary, but deterministic, which directory order is not. + Several matches mean the folder holds names the filesystem that wrote them + could not have told apart, so which one the name meant is not recoverable. ``listings`` caches one directory read per folder. """ try: if candidate.is_file(): - return candidate + return [candidate] if candidate.is_dir(): # An href of "" or "sub/" lands here; folding a folder's own name # would search its parent and match anything spelled like it. - return None + return [] except OSError: pass # unstattable exact spelling: a case variant of it may still stat folder = candidate.parent if folder not in listings: - files: dict[str, Path] = {} + files: dict[str, list[Path]] = {} try: - # By name, not by Path: PurePath ordering is case-folded on Windows, - # which would leave the tie-break to directory order there. - for path in sorted(folder.iterdir(), key=lambda entry: entry.name): + for path in folder.iterdir(): if path.is_file(): - files.setdefault(_fold(path.name), path) + files.setdefault(_fold(path.name), []).append(path) except OSError: pass # unreadable folder: no candidate resolves out of it listings[folder] = files - return listings[folder].get(_fold(candidate.name)) + return listings[folder].get(_fold(candidate.name), []) + + +def _existing_file(candidate: Path, listings: dict[Path, dict[str, list[Path]]]) -> Path | None: + """The one file ``candidate`` names, or None if no file or several do. + + Guessing between several would pick a companion by a rule no exporter + knows, so an ambiguous name resolves to nothing at all — visibly, as + ``ambiguous-ranges-file`` from validation, rather than by silent choice. + """ + matches = _folded_matches(candidate, listings) + return matches[0] if len(matches) == 1 else None + + +def _ranges_candidates(lift_path: Path, ranges: Iterable[Range]) -> list[Path]: + """Where a companion may be found, in the order :meth:`Lexicon.load` tries. + + Each path once: an href shared by several ranges, or agreeing with the + sibling, is one candidate however many times it is written. + """ + base = lift_path.parent + # with_name and with_suffix agree on every name that has an extension, but + # with_suffix would raise on a name that has none — which parse_document + # accepts, since it never inspects the extension. + candidates = [lift_path.with_name(lift_path.name + "-ranges")] + for range_ in ranges: + if range_.href is None: + continue + relative = _normalize_href(range_.href) + if relative is not None: + candidates.append(base / relative) + basename = range_.href.replace("\\", "/").rpartition("/")[2] + if basename: + candidates.append(base / basename) + return list(dict.fromkeys(candidates)) def _same_file(left: Path, right: Path) -> bool: @@ -584,7 +616,9 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L A candidate matching no file exactly resolves across differences in case or Unicode normalization, so a folder authored on Windows loads - the same way everywhere; the ``.lift`` is never its own companion. + the same way everywhere. One that several files answer to that way is + ambiguous and resolves to none of them, which validation reports as + ``ambiguous-ranges-file``; the ``.lift`` is never its own companion. A ``.zip`` path is treated as a packaged LIFT folder: it is extracted to a temporary directory (kept alive for the returned lexicon's @@ -605,23 +639,8 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L def _resolve_ranges(self) -> None: if self.path is None: return - base = self.path.parent - candidates: list[Path] = [] - # with_name and with_suffix agree on every name that has an extension, - # but with_suffix would raise on a name that has none — which - # parse_document accepts, since it never inspects the extension. - candidates.append(self.path.with_name(self.path.name + "-ranges")) - for range_ in self.header.ranges: - if range_.href is None: - continue - relative = _normalize_href(range_.href) - if relative is not None: - candidates.append(base / relative) - basename = range_.href.replace("\\", "/").rpartition("/")[2] - if basename: - candidates.append(base / basename) - listings: dict[Path, dict[str, Path]] = {} - for candidate in candidates: + listings: dict[Path, dict[str, list[Path]]] = {} + for candidate in _ranges_candidates(self.path, self.header.ranges): found = _existing_file(candidate, listings) if found is None: continue diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index 85f27ab..f4da66e 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -40,7 +40,15 @@ from lxml import etree from ._errors import LiftValidationError, LiftWriteError -from ._model import GrammaticalInfo, Lexicon, _existing_file, _normalize_href, _same_file +from ._model import ( + GrammaticalInfo, + Lexicon, + _existing_file, + _folded_matches, + _normalize_href, + _ranges_candidates, + _same_file, +) from ._text import Multitext, Trait if TYPE_CHECKING: @@ -509,13 +517,45 @@ def range_named(name: str) -> str | None: # here, not inside the file check below, which a lexicon with no path skips. header_ranges = {range_.id: range_named(range_.id) for range_ in lexicon.header.ranges} - # Header references that resolve to no companion — an exporter - # that wrote the href but not the file. Absolute and file:// hrefs are - # skipped: FLEx writes those knowing they will not resolve, and load reaches - # their companions by basename in the same folder instead. + # The two companion checks that need a folder to look in, sharing its + # listing: whether a candidate name picks out one file, and whether a + # header href reaches one at all. if lexicon.path is not None: base = lexicon.path.parent - listings: dict[Path, dict[str, Path]] = {} + listings: dict[Path, dict[str, list[Path]]] = {} + + # A candidate name that several files answer to once case and Unicode + # normalization are folded together. Load refuses to guess between + # them, so nothing is loaded for that name — including the sibling + # candidate, which no href reports on. One finding per colliding group, + # however many candidates fold onto it. + reported: set[tuple[Path, tuple[str, ...]]] = set() + for candidate in _ranges_candidates(lexicon.path, lexicon.header.ranges): + matches = _folded_matches(candidate, listings) + if len(matches) < 2: + continue + names = sorted(path.name for path in matches) + key = (candidate.parent, tuple(names)) + if key in reported: + continue + reported.add(key) + # Two spellings can differ only by normalization and render + # identically, so name them by code point, as the mismatch + # findings above do. + spellings = ", ".join(f"{name!a}" for name in names) + yield Problem( + "warning", + "ambiguous-ranges-file", + f"companion {candidate.name!a} matches {spellings}; they differ only " + "by case or Unicode normalization, so none of them is loaded", + file=lexicon.path, + ) + + # Header references that resolve to no companion — an + # exporter that wrote the href but not the file. Absolute and file:// + # hrefs are skipped: FLEx writes those knowing they will not resolve, + # and load reaches their companions by basename in the same folder + # instead. for range_ in lexicon.header.ranges: if not range_.href or range_.elements: continue diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index dde9835..958cb45 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -268,16 +268,75 @@ def test_companion_resolves_when_companion_suffix_is_uppercase(tmp_path: Path) - assert lexicon.all_ranges()["grammatical-info"].elements -def test_case_folded_companions_resolve_deterministically(tmp_path: Path) -> None: +def test_case_folded_companions_resolve_to_neither(tmp_path: Path) -> None: if not _case_sensitive(tmp_path): pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") - # Neither spelling matches the Dict.LIFT-ranges candidate exactly, so the - # tie-break picks one: lexicographically first, the same one every run. + # Neither spelling matches the Dict.LIFT-ranges candidate exactly, and + # nothing says which one it meant, so no companion is loaded for it. folder = tmp_path / "pkg" lift = _write_case_variant_pair(folder, "Dict.LIFT", "Dict.lift-ranges") (folder / "Dict.Lift-ranges").write_bytes((folder / "Dict.lift-ranges").read_bytes()) lexicon = sil_lift.load(lift) - assert [path.name for path in lexicon.ranges_files] == ["Dict.Lift-ranges"] + assert lexicon.ranges_files == {} + ambiguous = [p for p in lexicon.iter_problems() if p.code == "ambiguous-ranges-file"] + assert [p.level for p in ambiguous] == ["warning"] + assert "'Dict.LIFT-ranges' matches 'Dict.Lift-ranges', 'Dict.lift-ranges'" in ( + ambiguous[0].message + ) + + +def test_an_exactly_named_companion_ignores_the_variant_beside_it(tmp_path: Path) -> None: + if not _case_sensitive(tmp_path): + pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") + # The candidate names one of the two exactly, so folding never runs: a case + # variant is only ambiguous when nothing answers to the name as written. + folder = tmp_path / "pkg" + lift = _write_case_variant_pair(folder, "Dict.lift", "Dict.lift-ranges") + (folder / "Dict.LIFT-ranges").write_bytes((folder / "Dict.lift-ranges").read_bytes()) + lexicon = sil_lift.load(lift) + assert [path.name for path in lexicon.ranges_files] == ["Dict.lift-ranges"] + assert [p for p in lexicon.iter_problems() if p.code == "ambiguous-ranges-file"] == [] + + +# One stem in the four spellings a filesystem that folds case still keeps +# apart: each accent composed or decomposed, independently. +_COMPOSED = "Ñandú" +_N_SPLIT = "Ñandú" +_U_SPLIT = "Ñandú" +_BOTH_SPLIT = "Ñandú" + + +def _normalization_sensitive(folder: Path) -> bool: + probe = folder / "NormProbé" + probe.mkdir() + sensitive = not (folder / unicodedata.normalize("NFC", probe.name)).exists() + probe.rmdir() + return sensitive + + +def test_normalization_folded_companions_resolve_to_neither(tmp_path: Path) -> None: + if not _normalization_sensitive(tmp_path): + pytest.skip("needs a filesystem that keeps normalization forms apart") + # Which accent is decomposed, rather than case, is what separates these two + # companions — the only way to reach the collision where the exact-name stat + # is itself case-insensitive, as it is on NTFS and APFS. + folder = tmp_path / "pkg" + # The sibling candidate is fully composed and the href fully decomposed: + # two names folding onto the same pair, so one finding covers both. + lift = _write_lift_with_href(folder, f"{_COMPOSED}.lift", f"{_BOTH_SPLIT}.lift-ranges") + source = (PAIR_DIR / "test20080407.lift-ranges").read_bytes() + (folder / f"{_N_SPLIT}.lift-ranges").write_bytes(source) + (folder / f"{_U_SPLIT}.lift-ranges").write_bytes(source) + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files == {} + problems = list(lexicon.iter_problems()) + ambiguous = [p for p in problems if p.code == "ambiguous-ranges-file"] + assert [p.level for p in ambiguous] == ["warning"] + # Spellings that render identically, named by code point. + assert ascii(f"{_N_SPLIT}.lift-ranges") in ambiguous[0].message + assert ascii(f"{_U_SPLIT}.lift-ranges") in ambiguous[0].message + # The collision says why nothing resolved; the href, which range went unmet. + assert "dangling-ranges-href" in [p.code for p in problems] def test_absent_companion_stays_absent(tmp_path: Path) -> None: From cda1d7b4b2d753301d71f9645d48929a50f925ed Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 25 Aug 2026 13:29:02 -0400 Subject: [PATCH 13/16] Say what folds the two spellings, not that they differ by a form MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NFC is a normalization form, not a dimension along which two filenames differ: names collide because folding them applies NFC, and neither of the two need be in that form — a stem with two accents has four spellings that fold onto one key. So the problem code's description says the files answer to one name under case folding and NFC, rather than that they differ by it. Two spellings differ *in* case or normalization, not *by* it. Co-Authored-By: Claude Opus 5 (1M context) --- docs/en/guides/validate.md | 2 +- src/sil_lift/_validate.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/guides/validate.md b/docs/en/guides/validate.md index 6069cae..e932f51 100644 --- a/docs/en/guides/validate.md +++ b/docs/en/guides/validate.md @@ -32,7 +32,7 @@ Every finding carries one of these, whichever layer produced it — `schema` and | code | level | what it flags | | ------------------------ | ------- | -------------------------------------------------------------------------- | -| `ambiguous-ranges-file` | warning | a companion name matching several files that differ only by case or NFC | +| `ambiguous-ranges-file` | warning | several files answering to one companion name under case folding and NFC | | `dangling-ranges-href` | warning | a header `range/@href` resolving to no companion file | | `dangling-ref` | error | a `relation/@ref` or `variant/@ref` matching no entry or sense | | `duplicate-form-lang` | warning | two forms in one multitext sharing a language | diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index f4da66e..a562413 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -539,7 +539,7 @@ def range_named(name: str) -> str | None: if key in reported: continue reported.add(key) - # Two spellings can differ only by normalization and render + # Two spellings can differ only in normalization and render # identically, so name them by code point, as the mismatch # findings above do. spellings = ", ".join(f"{name!a}" for name in names) @@ -547,7 +547,7 @@ def range_named(name: str) -> str | None: "warning", "ambiguous-ranges-file", f"companion {candidate.name!a} matches {spellings}; they differ only " - "by case or Unicode normalization, so none of them is loaded", + "in case or Unicode normalization, so none of them is loaded", file=lexicon.path, ) From 2f97d63fca607445bfa8dab20aca187050d4cb57 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 25 Aug 2026 13:30:06 -0400 Subject: [PATCH 14/16] Cut the companion-collision note down to effect and remedy The problem-code reference told the reader how folding works and why the collision is unresolvable. Both belong where the lookup lives; what a reader needs here is that the ranges go absent and that leaving one file brings them back. Co-Authored-By: Claude Opus 5 (1M context) --- docs/en/guides/validate.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/guides/validate.md b/docs/en/guides/validate.md index e932f51..c1249b8 100644 --- a/docs/en/guides/validate.md +++ b/docs/en/guides/validate.md @@ -47,7 +47,7 @@ Every finding carries one of these, whichever layer produced it — `schema` and All three layers work from what `save()` would write, so a document that cannot be serialized at all is reported as a single `lone-surrogate` error instead — see [Fidelity guarantees](../fidelity.md#content-xml-cannot-represent). -Companions are matched by folded filename — case and Unicode normalization — so a folder authored on Windows loads the same way on a case-sensitive filesystem. Where that folding leaves one name matching several files, none of them is loaded: which one it meant is not recoverable. `ambiguous-ranges-file` reports the collision; renaming or removing all but one resolves it. +A companion name matching several files loads none of them: the ranges they define go absent until all but one is renamed or removed. ## Real-world FieldWorks (FLEx) output From 06def3a418fba7db28d21e1c292b71b8f1ff53ed Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 25 Aug 2026 13:33:36 -0400 Subject: [PATCH 15/16] Keep only what the companion-collision code cannot say itself The comments restated the lookup helpers' docstrings and described the shape of the two loops below them. What is left is the part the code does not show: that the check walks every candidate because a collision on the sibling name has no href to report it, and why the spellings are named by code point. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_validate.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index a562413..041febf 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -517,18 +517,13 @@ def range_named(name: str) -> str | None: # here, not inside the file check below, which a lexicon with no path skips. header_ranges = {range_.id: range_named(range_.id) for range_ in lexicon.header.ranges} - # The two companion checks that need a folder to look in, sharing its - # listing: whether a candidate name picks out one file, and whether a - # header href reaches one at all. if lexicon.path is not None: base = lexicon.path.parent listings: dict[Path, dict[str, list[Path]]] = {} - # A candidate name that several files answer to once case and Unicode - # normalization are folded together. Load refuses to guess between - # them, so nothing is loaded for that name — including the sibling - # candidate, which no href reports on. One finding per colliding group, - # however many candidates fold onto it. + # Every candidate, not just the hrefs below: a collision on the sibling + # name has nothing else to report it. Keyed by colliding group, since + # several candidate names can fold onto the same one. reported: set[tuple[Path, tuple[str, ...]]] = set() for candidate in _ranges_candidates(lexicon.path, lexicon.header.ranges): matches = _folded_matches(candidate, listings) @@ -539,9 +534,7 @@ def range_named(name: str) -> str | None: if key in reported: continue reported.add(key) - # Two spellings can differ only in normalization and render - # identically, so name them by code point, as the mismatch - # findings above do. + # Spellings differing only in normalization render identically. spellings = ", ".join(f"{name!a}" for name in names) yield Problem( "warning", From 3345e290fb56096ad31dfa562af83415bed54480 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 25 Aug 2026 13:43:34 -0400 Subject: [PATCH 16/16] Skip a companion collision when one of its files loaded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A candidate folding onto several files warned even when another candidate had named one of them exactly and loaded it — so the finding claimed that none of them is loaded while one was, and pointed a reader at renaming the very file supplying their ranges. Reachable whenever the sibling name resolves and a header href is spelled a third way: the sibling supplies the range, which also keeps dangling-ranges-href quiet, so the false finding was the only thing said about a folder that had lost nothing. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_validate.py | 3 +++ tests/test_ranges_folder.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index 041febf..ec17898 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -529,6 +529,9 @@ def range_named(name: str) -> str | None: matches = _folded_matches(candidate, listings) if len(matches) < 2: continue + # One of the colliding files loaded, named exactly by another candidate. + if any(_same_file(path, loaded) for path in matches for loaded in lexicon.ranges_files): + continue names = sorted(path.name for path in matches) key = (candidate.parent, tuple(names)) if key in reported: diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 958cb45..b1a8862 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -339,6 +339,22 @@ def test_normalization_folded_companions_resolve_to_neither(tmp_path: Path) -> N assert "dangling-ranges-href" in [p.code for p in problems] +def test_a_collision_including_the_loaded_companion_is_not_reported(tmp_path: Path) -> None: + if not _normalization_sensitive(tmp_path): + pytest.skip("needs a filesystem that keeps normalization forms apart") + # The sibling names the composed spelling exactly and loads it; the href, + # spelled a third way, folds onto both files and resolves to neither. + folder = tmp_path / "pkg" + lift = _write_lift_with_href(folder, f"{_COMPOSED}.lift", f"{_BOTH_SPLIT}.lift-ranges") + source = (PAIR_DIR / "test20080407.lift-ranges").read_bytes() + (folder / f"{_COMPOSED}.lift-ranges").write_bytes(source) + (folder / f"{_N_SPLIT}.lift-ranges").write_bytes(source) + lexicon = sil_lift.load(lift) + assert [path.name for path in lexicon.ranges_files] == [f"{_COMPOSED}.lift-ranges"] + assert lexicon.all_ranges()["grammatical-info"].elements + assert [p for p in lexicon.iter_problems() if p.code == "ambiguous-ranges-file"] == [] + + def test_absent_companion_stays_absent(tmp_path: Path) -> None: # The fallback must not look outside the folder for a name not in it. folder = tmp_path / "pkg"