From 0b2d011771e46a1fa528c47004b38006d7ea8369 Mon Sep 17 00:00:00 2001 From: Claudius Ellsel Date: Thu, 16 Jul 2026 13:59:49 +0200 Subject: [PATCH 1/4] Report unsupported parenthesis-delimited blocks * Preserve unsupported entry, string, preamble, and comment source as explicit failures. * Recover subsequent supported blocks on the same or following line. --- bibtexparser/splitter.py | 82 ++++++++++++++++++- .../test_splitter_block_start_detection.py | 49 +++++++++++ 2 files changed, 129 insertions(+), 2 deletions(-) diff --git a/bibtexparser/splitter.py b/bibtexparser/splitter.py index bbc3e97..7287191 100644 --- a/bibtexparser/splitter.py +++ b/bibtexparser/splitter.py @@ -66,6 +66,82 @@ def _is_at_line_start(self, pos: int) -> bool: # Start of string counts as line start return True + def _find_parenthesis_block_end(self, opening_parenthesis_index: int) -> int: + """Find the recoverable end of an unsupported parenthesis-delimited block. + + Parentheses inside braces or quotes are field content. If the outer + parenthesis is unclosed, a new block at the start of a line is used as + the recovery boundary so subsequent supported blocks can still parse. + """ + parenthesis_depth = 0 + brace_depth = 0 + in_quote = False + + for index in range(opening_parenthesis_index, len(self.bibstr)): + char = self.bibstr[index] + escaped = index > 0 and self.bibstr[index - 1] == "\\" + + if ( + index > opening_parenthesis_index + and char == "@" + and parenthesis_depth == 1 + and brace_depth == 0 + and not in_quote + and self._is_at_line_start(index) + ): + return index + + if escaped: + continue + if char == '"' and brace_depth == 0: + in_quote = not in_quote + elif in_quote: + continue + elif char == "{": + brace_depth += 1 + elif char == "}" and brace_depth > 0: + brace_depth -= 1 + elif brace_depth == 0 and char == "(": + parenthesis_depth += 1 + elif brace_depth == 0 and char == ")": + parenthesis_depth -= 1 + if parenthesis_depth == 0: + return index + 1 + + return len(self.bibstr) + + def _skip_marks_before(self, end_index: int) -> None: + """Advance the mark iterator to the first mark at or after ``end_index``.""" + while True: + mark = self._next_mark(accept_eof=True) + if mark is None: + break + if mark.start() >= end_index: + self._unaccepted_mark = mark + break + + # The normal block handlers leave this index at their closing delimiter. + # Mirror that state so the next implicit-comment boundary starts correctly. + self._current_char_index = end_index - 1 + + def _handle_parenthesis_block(self, mark: re.Match) -> ParsingFailedBlock: + """Return an explicit failure for an unsupported parenthesis-delimited block.""" + start_line = self._current_line + end_index = self._find_parenthesis_block_end(mark.end()) + raw = self.bibstr[mark.start() : end_index].rstrip() + self._skip_marks_before(end_index) + return ParsingFailedBlock( + start_line=start_line, + raw=raw, + error=BlockAbortedException( + abort_reason=( + "Parenthesis-delimited blocks are not supported. " + "Use curly braces or handle this failed block explicitly." + ), + end_index=end_index, + ), + ) + def _end_implicit_comment(self, end_char_index) -> ImplicitComment | None: if self._implicit_comment_start is None: return # No implicit comment started @@ -284,7 +360,7 @@ def split(self, library: Library | None = None) -> Library: The library with the added blocks. """ self._markiter = re.finditer( - r"(? Library: start_line = self._current_line try: # Start new block parsing - if m_val.startswith("@comment"): + if self.bibstr[m.end()] == "(": + library.add(self._handle_parenthesis_block(m)) + elif m_val.startswith("@comment"): library.add(self._handle_explicit_comment()) elif m_val.startswith("@preamble"): library.add(self._handle_preamble()) diff --git a/tests/splitter_tests/test_splitter_block_start_detection.py b/tests/splitter_tests/test_splitter_block_start_detection.py index 3d7553e..a3740d1 100644 --- a/tests/splitter_tests/test_splitter_block_start_detection.py +++ b/tests/splitter_tests/test_splitter_block_start_detection.py @@ -154,6 +154,55 @@ def test_mixed_blocks_same_line( assert len(library.comments) == expected_comments +@pytest.mark.parametrize( + "unsupported_block", + [ + pytest.param( + "@article(test, title = {A (parenthesized) title})", + id="entry", + ), + pytest.param( + '@string(name = "value (draft)")', + id="string", + ), + pytest.param( + '@preamble("value (draft)")', + id="preamble", + ), + pytest.param( + "@comment(text (with nested parentheses))", + id="comment", + ), + ], +) +@pytest.mark.parametrize("same_line", [False, True], ids=["next-line", "same-line"]) +def test_parenthesis_delimited_block_is_explicit_failure(unsupported_block: str, same_line: bool): + """Unsupported standard syntax must never disappear into an implicit comment.""" + separator = " " if same_line else "\n" + library = Splitter( + f"{unsupported_block}{separator}@book{{valid, title = {{Supported block}}}}" + ).split() + + assert len(library.failed_blocks) == 1 + assert library.failed_blocks[0].raw == unsupported_block + assert "Parenthesis-delimited blocks are not supported" in ( + library.failed_blocks[0].error.abort_reason + ) + assert len(library.comments) == 0 + assert [entry.key for entry in library.entries] == ["valid"] + + +def test_unclosed_parenthesis_block_recovers_at_next_line_block(): + """An unclosed unsupported block does not hide a later supported block.""" + library = Splitter(dedent("""\ + @article(broken, title = {Unclosed parenthesis block} + @book{valid, title = {Supported block}}""")).split() + + assert len(library.failed_blocks) == 1 + assert library.failed_blocks[0].raw == ("@article(broken, title = {Unclosed parenthesis block}") + assert [entry.key for entry in library.entries] == ["valid"] + + # ============================================================================= # Test: Error recovery when new block starts at line start # ============================================================================= From 4379bd326edf0b445682e2f889e430aeaf6e3342 Mon Sep 17 00:00:00 2001 From: Claudius Ellsel Date: Thu, 16 Jul 2026 15:15:57 +0200 Subject: [PATCH 2/4] Require exact special command types Treat custom entry types that extend comment, preamble, or string names as ordinary entries. --- bibtexparser/splitter.py | 7 ++++--- .../test_splitter_block_start_detection.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/bibtexparser/splitter.py b/bibtexparser/splitter.py index 7287191..15c00d7 100644 --- a/bibtexparser/splitter.py +++ b/bibtexparser/splitter.py @@ -376,6 +376,7 @@ def split(self, library: Library | None = None) -> Library: m_val = m.group(0).lower() if m_val.startswith("@"): + block_type = m_val[1:].strip() # Clean up previous block implicit_comment implicit_comment = self._end_implicit_comment(m.start()) if implicit_comment is not None: @@ -387,11 +388,11 @@ def split(self, library: Library | None = None) -> Library: # Start new block parsing if self.bibstr[m.end()] == "(": library.add(self._handle_parenthesis_block(m)) - elif m_val.startswith("@comment"): + elif block_type == "comment": library.add(self._handle_explicit_comment()) - elif m_val.startswith("@preamble"): + elif block_type == "preamble": library.add(self._handle_preamble()) - elif m_val.startswith("@string"): + elif block_type == "string": library.add(self._handle_string(m), fail_on_duplicate_key=False) else: library.add(self._handle_entry(m, m_val), fail_on_duplicate_key=False) diff --git a/tests/splitter_tests/test_splitter_block_start_detection.py b/tests/splitter_tests/test_splitter_block_start_detection.py index a3740d1..70b0354 100644 --- a/tests/splitter_tests/test_splitter_block_start_detection.py +++ b/tests/splitter_tests/test_splitter_block_start_detection.py @@ -203,6 +203,25 @@ def test_unclosed_parenthesis_block_recovers_at_next_line_block(): assert [entry.key for entry in library.entries] == ["valid"] +@pytest.mark.parametrize( + "entry_type", + [ + pytest.param("commentary", id="comment-prefix"), + pytest.param("preamble_study", id="preamble-prefix"), + pytest.param("string_theory", id="string-prefix"), + ], +) +def test_entry_type_extending_special_command_name_is_entry(entry_type: str): + """Special commands require an exact type match; longer custom types remain entries.""" + library = Splitter(f"@{entry_type}{{key, title = {{Custom entry type}}}}").split() + + assert len(library.failed_blocks) == 0 + assert len(library.comments) == 0 + assert len(library.preambles) == 0 + assert len(library.strings) == 0 + assert [(entry.entry_type, entry.key) for entry in library.entries] == [(entry_type, "key")] + + # ============================================================================= # Test: Error recovery when new block starts at line start # ============================================================================= From f8e24e531539c8063cd28471c0d8462e4223535c Mon Sep 17 00:00:00 2001 From: Claudius Ellsel Date: Thu, 16 Jul 2026 16:16:31 +0200 Subject: [PATCH 3/4] Recognize complete BibTeX block starts * Accept standard whitespace before outer delimiters. * Preserve punctuation permitted in entry-type identifiers. * Keep line-separated parenthesis blocks visible as explicit failures. --- bibtexparser/splitter.py | 42 +++++++++++++--- .../test_splitter_block_start_detection.py | 49 +++++++++++++++++++ 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/bibtexparser/splitter.py b/bibtexparser/splitter.py index 15c00d7..bd8105d 100644 --- a/bibtexparser/splitter.py +++ b/bibtexparser/splitter.py @@ -66,6 +66,27 @@ def _is_at_line_start(self, pos: int) -> bool: # Start of string counts as line start return True + def _find_outer_delimiter(self, mark: re.Match) -> tuple[int, str]: + """Return the standard outer delimiter following a block-type mark. + + The mark expression guarantees that only whitespace separates the type + from the delimiter. Keeping newlines outside the mark lets ``_next_mark`` + continue to maintain accurate source-line information. + """ + delimiter_index = mark.end() + while self.bibstr[delimiter_index].isspace(): + delimiter_index += 1 + + delimiter = self.bibstr[delimiter_index] + if delimiter not in "{(": + raise ParserStateException( + message=( + "A block-start mark was not followed by a supported outer delimiter. " + "Please report this bug." + ) + ) + return delimiter_index, delimiter + def _find_parenthesis_block_end(self, opening_parenthesis_index: int) -> int: """Find the recoverable end of an unsupported parenthesis-delimited block. @@ -124,10 +145,12 @@ def _skip_marks_before(self, end_index: int) -> None: # Mirror that state so the next implicit-comment boundary starts correctly. self._current_char_index = end_index - 1 - def _handle_parenthesis_block(self, mark: re.Match) -> ParsingFailedBlock: + def _handle_parenthesis_block( + self, mark: re.Match, opening_parenthesis_index: int + ) -> ParsingFailedBlock: """Return an explicit failure for an unsupported parenthesis-delimited block.""" start_line = self._current_line - end_index = self._find_parenthesis_block_end(mark.end()) + end_index = self._find_parenthesis_block_end(opening_parenthesis_index) raw = self.bibstr[mark.start() : end_index].rstrip() self._skip_marks_before(end_index) return ParsingFailedBlock( @@ -360,7 +383,9 @@ def split(self, library: Library | None = None) -> Library: The library with the added blocks. """ self._markiter = re.finditer( - r"(? Library: if m_val.startswith("@"): block_type = m_val[1:].strip() + opening_delimiter_index, opening_delimiter = self._find_outer_delimiter(m) # Clean up previous block implicit_comment implicit_comment = self._end_implicit_comment(m.start()) if implicit_comment is not None: @@ -386,8 +412,8 @@ def split(self, library: Library | None = None) -> Library: start_line = self._current_line try: # Start new block parsing - if self.bibstr[m.end()] == "(": - library.add(self._handle_parenthesis_block(m)) + if opening_delimiter == "(": + library.add(self._handle_parenthesis_block(m, opening_delimiter_index)) elif block_type == "comment": library.add(self._handle_explicit_comment()) elif block_type == "preamble": @@ -481,7 +507,7 @@ def _handle_entry(self, m, m_val) -> Entry | ParsingFailedBlock: if comma_mark.group(0) == "}": # This is an entry without any comma after the key, and with no fields # Used e.g. by RefTeX (see issue #384) - key = self.bibstr[m.end() + 1 : comma_mark.start()].strip() + key = self.bibstr[start_bracket_mark.end() : comma_mark.start()].strip() fields, end_index, duplicate_keys = [], comma_mark.end(), [] elif comma_mark.group(0) != ",": self._unaccepted_mark = comma_mark @@ -490,7 +516,7 @@ def _handle_entry(self, m, m_val) -> Entry | ParsingFailedBlock: end_index=comma_mark.end(), ) else: - key = self.bibstr[m.end() + 1 : comma_mark.start()].strip() + key = self.bibstr[start_bracket_mark.end() : comma_mark.start()].strip() fields, end_index, duplicate_keys = self._move_to_end_of_entry(comma_mark.end()) entry = Entry( @@ -528,7 +554,7 @@ def _handle_string(self, m) -> String: f" but found {equals_mark.group(0)}", end_index=equals_mark.end(), ) - key = self.bibstr[m.end() + 1 : equals_mark.start()].strip() + key = self.bibstr[start_bracket_mark.end() : equals_mark.start()].strip() value_start = equals_mark.end() end_i = self._move_to_closed_bracket() value = self.bibstr[value_start:end_i].strip() diff --git a/tests/splitter_tests/test_splitter_block_start_detection.py b/tests/splitter_tests/test_splitter_block_start_detection.py index 70b0354..cf0b6d7 100644 --- a/tests/splitter_tests/test_splitter_block_start_detection.py +++ b/tests/splitter_tests/test_splitter_block_start_detection.py @@ -10,6 +10,10 @@ import pytest +from bibtexparser.model import Entry +from bibtexparser.model import ExplicitComment +from bibtexparser.model import Preamble +from bibtexparser.model import String from bibtexparser.splitter import Splitter # ============================================================================= @@ -154,6 +158,51 @@ def test_mixed_blocks_same_line( assert len(library.comments) == expected_comments +@pytest.mark.parametrize( + "bibtex, expected_block_type", + [ + pytest.param( + "@article\n {key, title = {Line-separated delimiter}}", + Entry, + id="entry", + ), + pytest.param('@string\n {name = "Line-separated delimiter"}', String, id="string"), + pytest.param('@preamble\n {"Line-separated delimiter"}', Preamble, id="preamble"), + pytest.param("@comment\n {Line-separated delimiter}", ExplicitComment, id="comment"), + ], +) +def test_newline_before_curly_delimiter_starts_block(bibtex: str, expected_block_type: type): + """BibTeX whitespace may separate a block type from its outer delimiter.""" + library = Splitter(bibtex).split() + + assert len(library.failed_blocks) == 0 + assert len(library.blocks) == 1 + assert isinstance(library.blocks[0], expected_block_type) + assert library.blocks[0].raw == bibtex + + +@pytest.mark.parametrize("entry_type", ["systematic-review", "review:type", "software/package"]) +def test_punctuation_in_entry_type_is_not_silently_ignored(entry_type: str): + """BibTeX identifiers permit punctuation beyond Python word characters.""" + library = Splitter(f"@{entry_type}{{key, title = {{Custom entry type}}}}").split() + + assert len(library.failed_blocks) == 0 + assert len(library.comments) == 0 + assert [(entry.entry_type, entry.key) for entry in library.entries] == [(entry_type, "key")] + + +def test_newline_before_parenthesis_delimiter_is_explicit_failure(): + """Unsupported parenthesis syntax remains visible when whitespace precedes its delimiter.""" + bibtex = "@article\n (key, title = {Line-separated delimiter})" + + library = Splitter(bibtex).split() + + assert len(library.blocks) == 1 + assert len(library.failed_blocks) == 1 + assert len(library.comments) == 0 + assert library.failed_blocks[0].raw == bibtex + + @pytest.mark.parametrize( "unsupported_block", [ From f0c4a722bea45febb9a63de87938db302f9c286a Mon Sep 17 00:00:00 2001 From: Claudius Ellsel Date: Thu, 16 Jul 2026 11:43:54 +0200 Subject: [PATCH 4/4] Lock structural block visibility Require every standard-delimited block candidate to become either a parsed entry or an explicit failed block across whitespace and identifier variants. --- .../test_splitter_block_start_detection.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/splitter_tests/test_splitter_block_start_detection.py b/tests/splitter_tests/test_splitter_block_start_detection.py index cf0b6d7..f62a523 100644 --- a/tests/splitter_tests/test_splitter_block_start_detection.py +++ b/tests/splitter_tests/test_splitter_block_start_detection.py @@ -203,6 +203,34 @@ def test_newline_before_parenthesis_delimiter_is_explicit_failure(): assert library.failed_blocks[0].raw == bibtex +@pytest.mark.parametrize( + "entry_type", ["article", "systematic-review", "review:type", "software/package"] +) +@pytest.mark.parametrize("separator", ["", " ", "\n ", "\r\n\t"]) +@pytest.mark.parametrize( + "opening, closing, expected_kind", + [("{", "}", "entry"), ("(", ")", "failed")], + ids=["supported-curly", "unsupported-parenthesis"], +) +def test_structural_block_candidate_never_becomes_implicit_comment( + entry_type: str, separator: str, opening: str, closing: str, expected_kind: str +): + """A standard-delimited candidate must be parsed or explicitly retained as failed.""" + source = f"@{entry_type}{separator}{opening}key, title = {{Visible block}}{closing}" + + library = Splitter(source).split() + + assert len(library.comments) == 0 + assert len(library.blocks) == 1 + if expected_kind == "entry": + assert len(library.entries) == 1 + assert len(library.failed_blocks) == 0 + else: + assert len(library.entries) == 0 + assert len(library.failed_blocks) == 1 + assert library.failed_blocks[0].raw == source + + @pytest.mark.parametrize( "unsupported_block", [