From 9d03b0de0b09bf81439b83842bee2eb70d820c8d Mon Sep 17 00:00:00 2001 From: Claudius Ellsel Date: Thu, 16 Jul 2026 16:29:22 +0200 Subject: [PATCH] Preserve Biber comments within entries * Keep percent-comment lines out of active bibliography fields. * Retain comment content and field-boundary positions during canonical writes. * Preserve comments across edits, CRLF input, and entry copying. --- bibtexparser/model.py | 52 +++++++++ bibtexparser/splitter.py | 68 ++++++++++-- bibtexparser/writer.py | 18 +++- tests/resources/biber_comments.bib | 8 ++ .../test_splitter_biber_comments.py | 102 ++++++++++++++++++ tests/test_model.py | 39 ++++++- 6 files changed, 279 insertions(+), 8 deletions(-) create mode 100644 tests/resources/biber_comments.bib create mode 100644 tests/splitter_tests/test_splitter_biber_comments.py diff --git a/bibtexparser/model.py b/bibtexparser/model.py index 29837b8d..070fccd7 100644 --- a/bibtexparser/model.py +++ b/bibtexparser/model.py @@ -306,6 +306,47 @@ def __repr__(self) -> str: ) +class EntryComment: + """A Biber-style ``%`` comment positioned between fields of an entry. + + ``field_index`` records how many fields preceded the comment in the parsed + source. It lets the canonical writer retain comments while keeping + ``Entry.fields`` restricted to active bibliography data. + """ + + def __init__(self, comment: str, field_index: int): + if field_index < 0: + raise ValueError("field_index must be non-negative") + self._comment = comment + self._field_index = field_index + + @property + def comment(self) -> str: + """Content following the source line's leading ``%`` marker.""" + return self._comment + + @comment.setter + def comment(self, value: str): + self._comment = value + + @property + def field_index(self) -> int: + """Number of active fields that preceded this comment when parsed.""" + return self._field_index + + @field_index.setter + def field_index(self, value: int): + if value < 0: + raise ValueError("field_index must be non-negative") + self._field_index = value + + def __eq__(self, other: object) -> bool: + return isinstance(other, EntryComment) and self.__dict__ == other.__dict__ + + def __repr__(self) -> str: + return f"EntryComment(comment={self.comment!r}, field_index={self.field_index})" + + class Entry(Block): """Bibtex Blocks of the ``@entry`` type, e.g. ``@article{Cesar2013, ...}``.""" @@ -316,11 +357,13 @@ def __init__( fields: list[Field], start_line: int | None = None, raw: str | None = None, + comments: list[EntryComment] | None = None, ): super().__init__(start_line, raw) self._entry_type = entry_type self._key = key self._fields = fields + self._comments = [] if comments is None else comments @property def entry_type(self) -> str: @@ -349,6 +392,15 @@ def fields(self) -> list[Field]: def fields(self, value: list[Field]): self._fields = value + @property + def comments(self) -> list[EntryComment]: + """Biber-style percent comments nested within this entry, in source order.""" + return self._comments + + @comments.setter + def comments(self, value: list[EntryComment]): + self._comments = value + @property def fields_dict(self) -> dict[str, Field]: """A dict of fields, with field keys as keys. diff --git a/bibtexparser/splitter.py b/bibtexparser/splitter.py index bbc3e97f..05c66c73 100644 --- a/bibtexparser/splitter.py +++ b/bibtexparser/splitter.py @@ -7,6 +7,7 @@ from .library import Library from .model import DuplicateFieldKeyBlock from .model import Entry +from .model import EntryComment from .model import ExplicitComment from .model import Field from .model import ImplicitComment @@ -17,6 +18,11 @@ logger = logging.getLogger(__name__) +def _is_line_comment_mark(mark: re.Match) -> bool: + """Return whether a splitter mark starts a percent-comment line.""" + return mark.group(0).lstrip().startswith("%") + + class Splitter: """Object responsible for splitting a BibTeX string into blocks. @@ -66,6 +72,20 @@ def _is_at_line_start(self, pos: int) -> bool: # Start of string counts as line start return True + def _skip_marks_before(self, end_index: int) -> None: + """Advance the mark iterator to the first mark at or after ``end_index``.""" + while True: + mark = self._next_mark(accept_eof=True) + if mark is None: + break + if mark.start() >= end_index: + self._unaccepted_mark = mark + break + + # Normal block handlers leave this index at their closing delimiter. + # Mirror that state so line tracking resumes after the consumed comment. + self._current_char_index = end_index - 1 + def _end_implicit_comment(self, end_char_index) -> ImplicitComment | None: if self._implicit_comment_start is None: return # No implicit comment started @@ -192,6 +212,12 @@ def _is_escaped(): continue # Check for end of field + elif _is_line_comment_mark(next_mark) and not _is_escaped(): + # A Biber line comment is whitespace at this grammar level. Returning + # its boundary prevents the comment from becoming part of the value; + # the entry parser consumes and positions the comment itself. + self._unaccepted_mark = next_mark + return next_mark.start() elif next_mark.group(0) == "," and not _is_escaped(): self._unaccepted_mark = next_mark return next_mark.start() @@ -220,18 +246,38 @@ def _is_escaped(): end_index=next_mark.start() - 1, ) - def _move_to_end_of_entry(self, first_key_start: int) -> tuple[list[Field], int, set[str]]: - """Move to the end of the entry and return the fields and the end index.""" + def _consume_entry_comment(self, mark: re.Match, field_index: int) -> EntryComment: + """Consume one Biber-style line comment and retain its field boundary.""" + percent_index = mark.end() - 1 + line_end = self.bibstr.find("\n", percent_index) + if line_end == -1: + line_end = len(self.bibstr) + comment_end = line_end + if comment_end > percent_index and self.bibstr[comment_end - 1] == "\r": + comment_end -= 1 + comment = self.bibstr[percent_index + 1 : comment_end] + self._skip_marks_before(line_end + 1) + return EntryComment(comment=comment, field_index=field_index) + + def _move_to_end_of_entry( + self, first_key_start: int + ) -> tuple[list[Field], list[EntryComment], int, set[str]]: + """Move to the end of the entry and return its fields and comments.""" result = [] + comments = [] keys = set() duplicate_keys = set() key_start = first_key_start while True: equals_mark = self._next_mark(accept_eof=False) + if _is_line_comment_mark(equals_mark): + comments.append(self._consume_entry_comment(equals_mark, field_index=len(result))) + key_start = self._current_char_index + 1 + continue if equals_mark.group(0) == "}": # End of entry - return result, equals_mark.end(), duplicate_keys + return result, comments, equals_mark.end(), duplicate_keys if equals_mark.group(0) != "=": self._unaccepted_mark = equals_mark @@ -261,6 +307,11 @@ def _move_to_end_of_entry(self, first_key_start: int) -> tuple[list[Field], int, # If next mark is a comma, continue after_field_mark = self._next_mark(accept_eof=False) + while _is_line_comment_mark(after_field_mark): + comments.append( + self._consume_entry_comment(after_field_mark, field_index=len(result)) + ) + after_field_mark = self._next_mark(accept_eof=False) if after_field_mark.group(0) == ",": key_start = after_field_mark.end() elif after_field_mark.group(0) == "}": @@ -284,7 +335,9 @@ def split(self, library: Library | None = None) -> Library: The library with the added blocks. """ self._markiter = re.finditer( - r"(? Entry | ParsingFailedBlock: # This is an entry without any comma after the key, and with no fields # Used e.g. by RefTeX (see issue #384) key = self.bibstr[m.end() + 1 : comma_mark.start()].strip() - fields, end_index, duplicate_keys = [], comma_mark.end(), [] + fields, comments, end_index, duplicate_keys = [], [], comma_mark.end(), [] elif comma_mark.group(0) != ",": self._unaccepted_mark = comma_mark raise BlockAbortedException( @@ -412,7 +465,9 @@ def _handle_entry(self, m, m_val) -> Entry | ParsingFailedBlock: ) else: key = self.bibstr[m.end() + 1 : comma_mark.start()].strip() - fields, end_index, duplicate_keys = self._move_to_end_of_entry(comma_mark.end()) + fields, comments, end_index, duplicate_keys = self._move_to_end_of_entry( + comma_mark.end() + ) entry = Entry( start_line=start_line, @@ -420,6 +475,7 @@ def _handle_entry(self, m, m_val) -> Entry | ParsingFailedBlock: key=key, fields=fields, raw=self.bibstr[m.start() : end_index], + comments=comments, ) # If there were duplicate field keys, we return a DuplicateFieldKeyBlock wrapping diff --git a/bibtexparser/writer.py b/bibtexparser/writer.py index 421b274e..30cc3d30 100644 --- a/bibtexparser/writer.py +++ b/bibtexparser/writer.py @@ -3,6 +3,7 @@ from .library import Library from .model import Entry +from .model import EntryComment from .model import ExplicitComment from .model import Field from .model import ImplicitComment @@ -16,16 +17,31 @@ def _treat_entry(block: Entry, bibtex_format) -> list[str]: res = ["@", block.entry_type, "{", block.key, ",\n"] + comments_by_field_index: dict[int, list[EntryComment]] = {} + for comment in block.comments: + # If callers removed fields without repositioning comments, retaining the + # comment at the end is safer than losing it or failing serialization. + field_index = min(comment.field_index, len(block.fields)) + comments_by_field_index.setdefault(field_index, []).append(comment) + field: Field for i, field in enumerate(block.fields): + for comment in comments_by_field_index.get(i, []): + res.extend([bibtex_format.indent, "%", comment.comment, "\n"]) res.append(bibtex_format.indent) res.append(field.key) res.append(_val_indent_string(bibtex_format, field.key)) res.append(VAL_SEP) res.append(field.value) - if bibtex_format.trailing_comma or i < len(block.fields) - 1: + if ( + bibtex_format.trailing_comma + or i < len(block.fields) - 1 + or len(block.fields) in comments_by_field_index + ): res.append(",") res.append("\n") + for comment in comments_by_field_index.get(len(block.fields), []): + res.extend([bibtex_format.indent, "%", comment.comment, "\n"]) res.append("}\n") return res diff --git a/tests/resources/biber_comments.bib b/tests/resources/biber_comments.bib new file mode 100644 index 00000000..4b9e0b29 --- /dev/null +++ b/tests/resources/biber_comments.bib @@ -0,0 +1,8 @@ +@online{record, + % before all fields + title = {Current title}, + % between fields + date = {2024}, + url = {https://example.invalid} + % after all fields +} diff --git a/tests/splitter_tests/test_splitter_biber_comments.py b/tests/splitter_tests/test_splitter_biber_comments.py new file mode 100644 index 00000000..08bb4080 --- /dev/null +++ b/tests/splitter_tests/test_splitter_biber_comments.py @@ -0,0 +1,102 @@ +"""Regression tests for Biber-style percent comments within entries. + +Biber permits a line whose first non-whitespace character is ``%`` inside an +entry. Such a line is commentary, even when it resembles a field or contains an +equals sign. Treating it as a regular field can silently absorb the following +field and corrupt both the parsed model and subsequent output. +""" + +from pathlib import Path + +from bibtexparser import parse_string +from bibtexparser import write_string + +RESOURCE = Path(__file__).parents[1] / "resources" / "biber_comments.bib" + + +class TestBiberEntryComments: + """Protect comment visibility without exposing comments as bibliography fields.""" + + def test_comment_with_equals_sign_does_not_absorb_following_field(self): + """Comment punctuation cannot change the boundaries of real fields.""" + source = """@article{record, + author = {Fartsy}, + % explanatory = sign, braces={ignored} + date = {2024}, + }""" + + entry = parse_string(source).entries_dict["record"] + + assert [(field.key, field.value) for field in entry.fields] == [ + ("author", "Fartsy"), + ("date", "2024"), + ] + assert [comment.comment for comment in entry.comments] == [ + " explanatory = sign, braces={ignored}" + ] + + def test_commented_out_field_is_not_available_as_data(self): + """A field-shaped comment remains commentary rather than active data.""" + source = """@online{record, + %title={Withdrawn title}, + title={Current title}, + url={https://example.invalid}, + }""" + + entry = parse_string(source).entries_dict["record"] + + assert entry["title"] == "Current title" + assert "%title" not in entry + assert [comment.comment for comment in entry.comments] == ["title={Withdrawn title},"] + + def test_roundtrip_preserves_comments_at_their_field_boundaries(self): + """Canonical formatting retains comments before, between, and after fields.""" + source = RESOURCE.read_text(encoding="utf-8") + + written = write_string(parse_string(source)) + reparsed = parse_string(written).entries_dict["record"] + + assert written.index("% before all fields") < written.index("title =") + assert written.index("title =") < written.index("% between fields") + assert written.index("% between fields") < written.index("url =") + assert written.index("url =") < written.index("% after all fields") + assert [comment.comment for comment in reparsed.comments] == [ + " before all fields", + " between fields", + " after all fields", + ] + assert [(field.key, field.value) for field in reparsed.fields] == [ + ("title", "Current title"), + ("date", "2024"), + ("url", "https://example.invalid"), + ] + + def test_crlf_comment_does_not_retain_carriage_return(self): + """The line ending is syntax and must not become part of comment content.""" + source = "@article{record,\r\n % note\r\n title={Current title}\r\n}" + + entry = parse_string(source).entries_dict["record"] + + assert [comment.comment for comment in entry.comments] == [" note"] + assert "\r" not in write_string(parse_string(source)) + + def test_top_level_percent_comment_remains_an_implicit_comment(self): + """The in-entry marker must not consume established top-level comments.""" + source = "% top-level note\n@article{record, title={Current title}}" + + library = parse_string(source) + + assert [comment.comment for comment in library.comments] == ["% top-level note"] + assert [entry.key for entry in library.entries] == ["record"] + + def test_comment_survives_when_preceding_fields_are_removed(self): + """Editing fields must move an out-of-range comment boundary, not drop it.""" + library = parse_string(RESOURCE.read_text(encoding="utf-8")) + entry = library.entries_dict["record"] + entry.fields = [] + + written = write_string(library) + + assert "% before all fields" in written + assert "% between fields" in written + assert "% after all fields" in written diff --git a/tests/test_model.py b/tests/test_model.py index d24f42b6..2de19299 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -5,6 +5,7 @@ import pytest from bibtexparser.model import Entry +from bibtexparser.model import EntryComment from bibtexparser.model import ExplicitComment from bibtexparser.model import Field from bibtexparser.model import ImplicitComment @@ -49,6 +50,16 @@ def test_entry_equality(): "raw", ) assert entry_1 != entry_4 + # Not equal to an otherwise identical entry with a nested source comment + entry_5 = Entry( + "article", + "key", + [Field("field", "value", 1)], + 1, + "raw", + comments=[EntryComment(" note", field_index=0)], + ) + assert entry_1 != entry_5 def test_entry_copy(): @@ -66,7 +77,14 @@ def test_entry_copy(): def test_entry_deepcopy(): - entry_1 = Entry("article", "key", [Field("field", "value", 1)], 1, "raw") + entry_1 = Entry( + "article", + "key", + [Field("field", "value", 1)], + 1, + "raw", + comments=[EntryComment(" note", field_index=0)], + ) entry_2 = deepcopy(entry_1) assert entry_1 == entry_2 assert entry_1 is not entry_2 @@ -74,6 +92,25 @@ def test_entry_deepcopy(): assert entry_1.fields == entry_2.fields assert entry_1.fields_dict["field"] is not entry_2.fields_dict["field"] assert entry_1.fields_dict["field"] == entry_2.fields_dict["field"] + assert entry_1.comments is not entry_2.comments + assert entry_1.comments[0] is not entry_2.comments[0] + assert entry_1.comments == entry_2.comments + + +def test_entry_comment_rejects_negative_field_index(): + """Comment positions are field boundaries and therefore cannot be negative.""" + with pytest.raises(ValueError, match="non-negative"): + EntryComment(" note", field_index=-1) + + comment = EntryComment(" note", field_index=0) + with pytest.raises(ValueError, match="non-negative"): + comment.field_index = -1 + + +def test_entry_comment_is_unhashable_while_mutable(): + """Mutable comment content and positions must not become unstable mapping keys.""" + with pytest.raises(TypeError): + hash(EntryComment(" note", field_index=0)) def test_entry_get():