Preserve Biber comments within entries

bibtexparser/model.py

@@ -306,6 +306,47 @@ def __repr__(self) -> str:
         )
 
 
+class EntryComment:
+    """A Biber-style ``%`` comment positioned between fields of an entry.
+
+    ``field_index`` records how many fields preceded the comment in the parsed
+    source. It lets the canonical writer retain comments while keeping
+    ``Entry.fields`` restricted to active bibliography data.
+    """
+
+    def __init__(self, comment: str, field_index: int):
+        if field_index < 0:
+            raise ValueError("field_index must be non-negative")
+        self._comment = comment
+        self._field_index = field_index
+
+    @property
+    def comment(self) -> str:
+        """Content following the source line's leading ``%`` marker."""
+        return self._comment
+
+    @comment.setter
+    def comment(self, value: str):
+        self._comment = value
+
+    @property
+    def field_index(self) -> int:
+        """Number of active fields that preceded this comment when parsed."""
+        return self._field_index
+
+    @field_index.setter
+    def field_index(self, value: int):
+        if value < 0:
+            raise ValueError("field_index must be non-negative")
+        self._field_index = value
+
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, EntryComment) and self.__dict__ == other.__dict__
+
+    def __repr__(self) -> str:
+        return f"EntryComment(comment={self.comment!r}, field_index={self.field_index})"
+
+
 class Entry(Block):
     """Bibtex Blocks of the ``@entry`` type, e.g. ``@article{Cesar2013, ...}``."""
 
@@ -316,11 +357,13 @@ def __init__(
         fields: list[Field],
         start_line: int | None = None,
         raw: str | None = None,
+        comments: list[EntryComment] | None = None,
     ):
         super().__init__(start_line, raw)
         self._entry_type = entry_type
         self._key = key
         self._fields = fields
+        self._comments = [] if comments is None else comments
 
     @property
     def entry_type(self) -> str:
@@ -349,6 +392,15 @@ def fields(self) -> list[Field]:
     def fields(self, value: list[Field]):
         self._fields = value
 
+    @property
+    def comments(self) -> list[EntryComment]:
+        """Biber-style percent comments nested within this entry, in source order."""
+        return self._comments
+
+    @comments.setter
+    def comments(self, value: list[EntryComment]):
+        self._comments = value
+
     @property
     def fields_dict(self) -> dict[str, Field]:
         """A dict of fields, with field keys as keys.

bibtexparser/splitter.py

@@ -7,6 +7,7 @@
 from .library import Library
 from .model import DuplicateFieldKeyBlock
 from .model import Entry
+from .model import EntryComment
 from .model import ExplicitComment
 from .model import Field
 from .model import ImplicitComment
@@ -17,6 +18,11 @@
 logger = logging.getLogger(__name__)
 
 
+def _is_line_comment_mark(mark: re.Match) -> bool:
+    """Return whether a splitter mark starts a percent-comment line."""
+    return mark.group(0).lstrip().startswith("%")
+
+
 class Splitter:
     """Object responsible for splitting a BibTeX string into blocks.
 
@@ -66,6 +72,20 @@ def _is_at_line_start(self, pos: int) -> bool:
         # Start of string counts as line start
         return True
 
+    def _skip_marks_before(self, end_index: int) -> None:
+        """Advance the mark iterator to the first mark at or after ``end_index``."""
+        while True:
+            mark = self._next_mark(accept_eof=True)
+            if mark is None:
+                break
+            if mark.start() >= end_index:
+                self._unaccepted_mark = mark
+                break
+
+        # Normal block handlers leave this index at their closing delimiter.
+        # Mirror that state so line tracking resumes after the consumed comment.
+        self._current_char_index = end_index - 1
+
     def _end_implicit_comment(self, end_char_index) -> ImplicitComment | None:
         if self._implicit_comment_start is None:
             return  # No implicit comment started
@@ -192,6 +212,12 @@ def _is_escaped():
                 continue
 
             # Check for end of field
+            elif _is_line_comment_mark(next_mark) and not _is_escaped():
+                # A Biber line comment is whitespace at this grammar level. Returning
+                # its boundary prevents the comment from becoming part of the value;
+                # the entry parser consumes and positions the comment itself.
+                self._unaccepted_mark = next_mark
+                return next_mark.start()
             elif next_mark.group(0) == "," and not _is_escaped():
                 self._unaccepted_mark = next_mark
                 return next_mark.start()
@@ -220,18 +246,38 @@ def _is_escaped():
                     end_index=next_mark.start() - 1,
                 )
 
-    def _move_to_end_of_entry(self, first_key_start: int) -> tuple[list[Field], int, set[str]]:
-        """Move to the end of the entry and return the fields and the end index."""
+    def _consume_entry_comment(self, mark: re.Match, field_index: int) -> EntryComment:
+        """Consume one Biber-style line comment and retain its field boundary."""
+        percent_index = mark.end() - 1
+        line_end = self.bibstr.find("\n", percent_index)
+        if line_end == -1:
+            line_end = len(self.bibstr)
+        comment_end = line_end
+        if comment_end > percent_index and self.bibstr[comment_end - 1] == "\r":
+            comment_end -= 1
+        comment = self.bibstr[percent_index + 1 : comment_end]
+        self._skip_marks_before(line_end + 1)
+        return EntryComment(comment=comment, field_index=field_index)
+
+    def _move_to_end_of_entry(
+        self, first_key_start: int
+    ) -> tuple[list[Field], list[EntryComment], int, set[str]]:
+        """Move to the end of the entry and return its fields and comments."""
         result = []
+        comments = []
         keys = set()
         duplicate_keys = set()
 
         key_start = first_key_start
         while True:
             equals_mark = self._next_mark(accept_eof=False)
+            if _is_line_comment_mark(equals_mark):
+                comments.append(self._consume_entry_comment(equals_mark, field_index=len(result)))
+                key_start = self._current_char_index + 1
+                continue
             if equals_mark.group(0) == "}":
                 # End of entry
-                return result, equals_mark.end(), duplicate_keys
+                return result, comments, equals_mark.end(), duplicate_keys
 
             if equals_mark.group(0) != "=":
                 self._unaccepted_mark = equals_mark
@@ -261,6 +307,11 @@ def _move_to_end_of_entry(self, first_key_start: int) -> tuple[list[Field], int,
 
             # If next mark is a comma, continue
             after_field_mark = self._next_mark(accept_eof=False)
+            while _is_line_comment_mark(after_field_mark):
+                comments.append(
+                    self._consume_entry_comment(after_field_mark, field_index=len(result))
+                )
+                after_field_mark = self._next_mark(accept_eof=False)
             if after_field_mark.group(0) == ",":
                 key_start = after_field_mark.end()
             elif after_field_mark.group(0) == "}":
@@ -284,7 +335,9 @@ def split(self, library: Library | None = None) -> Library:
             The library with the added blocks.
         """
         self._markiter = re.finditer(
-            r"(?<!\\)[\{\}\",=\n]|@[\w]*( |\t)*(?={)", self.bibstr, re.MULTILINE
+            r"(?<!\\)[\{\}\",=\n]|(?<=\n)[ \t]*%|@[\w]*( |\t)*(?={)",
+            self.bibstr,
+            re.MULTILINE,
         )
 
         if library is None:
@@ -403,7 +456,7 @@ def _handle_entry(self, m, m_val) -> Entry | ParsingFailedBlock:
             # This is an entry without any comma after the key, and with no fields
             #   Used e.g. by RefTeX (see issue #384)
             key = self.bibstr[m.end() + 1 : comma_mark.start()].strip()
-            fields, end_index, duplicate_keys = [], comma_mark.end(), []
+            fields, comments, end_index, duplicate_keys = [], [], comma_mark.end(), []
         elif comma_mark.group(0) != ",":
             self._unaccepted_mark = comma_mark
             raise BlockAbortedException(
@@ -412,14 +465,17 @@ def _handle_entry(self, m, m_val) -> Entry | ParsingFailedBlock:
             )
         else:
             key = self.bibstr[m.end() + 1 : comma_mark.start()].strip()
-            fields, end_index, duplicate_keys = self._move_to_end_of_entry(comma_mark.end())
+            fields, comments, end_index, duplicate_keys = self._move_to_end_of_entry(
+                comma_mark.end()
+            )
 
         entry = Entry(
             start_line=start_line,
             entry_type=entry_type,
             key=key,
             fields=fields,
             raw=self.bibstr[m.start() : end_index],
+            comments=comments,
         )
 
         # If there were duplicate field keys, we return a DuplicateFieldKeyBlock wrapping

bibtexparser/writer.py

@@ -3,6 +3,7 @@
 
 from .library import Library
 from .model import Entry
+from .model import EntryComment
 from .model import ExplicitComment
 from .model import Field
 from .model import ImplicitComment
@@ -16,16 +17,31 @@
 
 def _treat_entry(block: Entry, bibtex_format) -> list[str]:
     res = ["@", block.entry_type, "{", block.key, ",\n"]
+    comments_by_field_index: dict[int, list[EntryComment]] = {}
+    for comment in block.comments:
+        # If callers removed fields without repositioning comments, retaining the
+        # comment at the end is safer than losing it or failing serialization.
+        field_index = min(comment.field_index, len(block.fields))
+        comments_by_field_index.setdefault(field_index, []).append(comment)
+
     field: Field
     for i, field in enumerate(block.fields):
+        for comment in comments_by_field_index.get(i, []):
+            res.extend([bibtex_format.indent, "%", comment.comment, "\n"])
         res.append(bibtex_format.indent)
         res.append(field.key)
         res.append(_val_indent_string(bibtex_format, field.key))
         res.append(VAL_SEP)
         res.append(field.value)
-        if bibtex_format.trailing_comma or i < len(block.fields) - 1:
+        if (
+            bibtex_format.trailing_comma
+            or i < len(block.fields) - 1
+            or len(block.fields) in comments_by_field_index
+        ):
             res.append(",")
         res.append("\n")
+    for comment in comments_by_field_index.get(len(block.fields), []):
+        res.extend([bibtex_format.indent, "%", comment.comment, "\n"])
     res.append("}\n")
     return res
 

tests/resources/biber_comments.bib

@@ -0,0 +1,8 @@
+@online{record,
+  % before all fields
+  title = {Current title},
+  % between fields
+  date = {2024},
+  url = {https://example.invalid}
+  % after all fields
+}

tests/splitter_tests/test_splitter_biber_comments.py

@@ -0,0 +1,102 @@
+"""Regression tests for Biber-style percent comments within entries.
+
+Biber permits a line whose first non-whitespace character is ``%`` inside an
+entry. Such a line is commentary, even when it resembles a field or contains an
+equals sign. Treating it as a regular field can silently absorb the following
+field and corrupt both the parsed model and subsequent output.
+"""
+
+from pathlib import Path
+
+from bibtexparser import parse_string
+from bibtexparser import write_string
+
+RESOURCE = Path(__file__).parents[1] / "resources" / "biber_comments.bib"
+
+
+class TestBiberEntryComments:
+    """Protect comment visibility without exposing comments as bibliography fields."""
+
+    def test_comment_with_equals_sign_does_not_absorb_following_field(self):
+        """Comment punctuation cannot change the boundaries of real fields."""
+        source = """@article{record,
+            author = {Fartsy},
+            % explanatory = sign, braces={ignored}
+            date = {2024},
+        }"""
+
+        entry = parse_string(source).entries_dict["record"]
+
+        assert [(field.key, field.value) for field in entry.fields] == [
+            ("author", "Fartsy"),
+            ("date", "2024"),
+        ]
+        assert [comment.comment for comment in entry.comments] == [
+            " explanatory = sign, braces={ignored}"
+        ]
+
+    def test_commented_out_field_is_not_available_as_data(self):
+        """A field-shaped comment remains commentary rather than active data."""
+        source = """@online{record,
+            %title={Withdrawn title},
+            title={Current title},
+            url={https://example.invalid},
+        }"""
+
+        entry = parse_string(source).entries_dict["record"]
+
+        assert entry["title"] == "Current title"
+        assert "%title" not in entry
+        assert [comment.comment for comment in entry.comments] == ["title={Withdrawn title},"]
+
+    def test_roundtrip_preserves_comments_at_their_field_boundaries(self):
+        """Canonical formatting retains comments before, between, and after fields."""
+        source = RESOURCE.read_text(encoding="utf-8")
+
+        written = write_string(parse_string(source))
+        reparsed = parse_string(written).entries_dict["record"]
+
+        assert written.index("% before all fields") < written.index("title =")
+        assert written.index("title =") < written.index("% between fields")
+        assert written.index("% between fields") < written.index("url =")
+        assert written.index("url =") < written.index("% after all fields")
+        assert [comment.comment for comment in reparsed.comments] == [
+            " before all fields",
+            " between fields",
+            " after all fields",
+        ]
+        assert [(field.key, field.value) for field in reparsed.fields] == [
+            ("title", "Current title"),
+            ("date", "2024"),
+            ("url", "https://example.invalid"),
+        ]
+
+    def test_crlf_comment_does_not_retain_carriage_return(self):
+        """The line ending is syntax and must not become part of comment content."""
+        source = "@article{record,\r\n  % note\r\n  title={Current title}\r\n}"
+
+        entry = parse_string(source).entries_dict["record"]
+
+        assert [comment.comment for comment in entry.comments] == [" note"]
+        assert "\r" not in write_string(parse_string(source))
+
+    def test_top_level_percent_comment_remains_an_implicit_comment(self):
+        """The in-entry marker must not consume established top-level comments."""
+        source = "% top-level note\n@article{record, title={Current title}}"
+
+        library = parse_string(source)
+
+        assert [comment.comment for comment in library.comments] == ["% top-level note"]
+        assert [entry.key for entry in library.entries] == ["record"]
+
+    def test_comment_survives_when_preceding_fields_are_removed(self):
+        """Editing fields must move an out-of-range comment boundary, not drop it."""
+        library = parse_string(RESOURCE.read_text(encoding="utf-8"))
+        entry = library.entries_dict["record"]
+        entry.fields = []
+
+        written = write_string(library)
+
+        assert "% before all fields" in written
+        assert "% between fields" in written
+        assert "% after all fields" in written