diff --git a/bibtexparser/splitter.py b/bibtexparser/splitter.py index bbc3e97..bd8105d 100644 --- a/bibtexparser/splitter.py +++ b/bibtexparser/splitter.py @@ -66,6 +66,105 @@ def _is_at_line_start(self, pos: int) -> bool: # Start of string counts as line start return True + def _find_outer_delimiter(self, mark: re.Match) -> tuple[int, str]: + """Return the standard outer delimiter following a block-type mark. + + The mark expression guarantees that only whitespace separates the type + from the delimiter. Keeping newlines outside the mark lets ``_next_mark`` + continue to maintain accurate source-line information. + """ + delimiter_index = mark.end() + while self.bibstr[delimiter_index].isspace(): + delimiter_index += 1 + + delimiter = self.bibstr[delimiter_index] + if delimiter not in "{(": + raise ParserStateException( + message=( + "A block-start mark was not followed by a supported outer delimiter. " + "Please report this bug." + ) + ) + return delimiter_index, delimiter + + def _find_parenthesis_block_end(self, opening_parenthesis_index: int) -> int: + """Find the recoverable end of an unsupported parenthesis-delimited block. + + Parentheses inside braces or quotes are field content. If the outer + parenthesis is unclosed, a new block at the start of a line is used as + the recovery boundary so subsequent supported blocks can still parse. + """ + parenthesis_depth = 0 + brace_depth = 0 + in_quote = False + + for index in range(opening_parenthesis_index, len(self.bibstr)): + char = self.bibstr[index] + escaped = index > 0 and self.bibstr[index - 1] == "\\" + + if ( + index > opening_parenthesis_index + and char == "@" + and parenthesis_depth == 1 + and brace_depth == 0 + and not in_quote + and self._is_at_line_start(index) + ): + return index + + if escaped: + continue + if char == '"' and brace_depth == 0: + in_quote = not in_quote + elif in_quote: + continue + elif char == "{": + brace_depth += 1 + elif char == "}" and brace_depth > 0: + brace_depth -= 1 + elif brace_depth == 0 and char == "(": + parenthesis_depth += 1 + elif brace_depth == 0 and char == ")": + parenthesis_depth -= 1 + if parenthesis_depth == 0: + return index + 1 + + return len(self.bibstr) + + def _skip_marks_before(self, end_index: int) -> None: + """Advance the mark iterator to the first mark at or after ``end_index``.""" + while True: + mark = self._next_mark(accept_eof=True) + if mark is None: + break + if mark.start() >= end_index: + self._unaccepted_mark = mark + break + + # The normal block handlers leave this index at their closing delimiter. + # Mirror that state so the next implicit-comment boundary starts correctly. + self._current_char_index = end_index - 1 + + def _handle_parenthesis_block( + self, mark: re.Match, opening_parenthesis_index: int + ) -> ParsingFailedBlock: + """Return an explicit failure for an unsupported parenthesis-delimited block.""" + start_line = self._current_line + end_index = self._find_parenthesis_block_end(opening_parenthesis_index) + raw = self.bibstr[mark.start() : end_index].rstrip() + self._skip_marks_before(end_index) + return ParsingFailedBlock( + start_line=start_line, + raw=raw, + error=BlockAbortedException( + abort_reason=( + "Parenthesis-delimited blocks are not supported. " + "Use curly braces or handle this failed block explicitly." + ), + end_index=end_index, + ), + ) + def _end_implicit_comment(self, end_char_index) -> ImplicitComment | None: if self._implicit_comment_start is None: return # No implicit comment started @@ -284,7 +383,9 @@ def split(self, library: Library | None = None) -> Library: The library with the added blocks. """ self._markiter = re.finditer( - r"(? Library: m_val = m.group(0).lower() if m_val.startswith("@"): + block_type = m_val[1:].strip() + opening_delimiter_index, opening_delimiter = self._find_outer_delimiter(m) # Clean up previous block implicit_comment implicit_comment = self._end_implicit_comment(m.start()) if implicit_comment is not None: @@ -309,11 +412,13 @@ def split(self, library: Library | None = None) -> Library: start_line = self._current_line try: # Start new block parsing - if m_val.startswith("@comment"): + if opening_delimiter == "(": + library.add(self._handle_parenthesis_block(m, opening_delimiter_index)) + elif block_type == "comment": library.add(self._handle_explicit_comment()) - elif m_val.startswith("@preamble"): + elif block_type == "preamble": library.add(self._handle_preamble()) - elif m_val.startswith("@string"): + elif block_type == "string": library.add(self._handle_string(m), fail_on_duplicate_key=False) else: library.add(self._handle_entry(m, m_val), fail_on_duplicate_key=False) @@ -402,7 +507,7 @@ def _handle_entry(self, m, m_val) -> Entry | ParsingFailedBlock: if comma_mark.group(0) == "}": # This is an entry without any comma after the key, and with no fields # Used e.g. by RefTeX (see issue #384) - key = self.bibstr[m.end() + 1 : comma_mark.start()].strip() + key = self.bibstr[start_bracket_mark.end() : comma_mark.start()].strip() fields, end_index, duplicate_keys = [], comma_mark.end(), [] elif comma_mark.group(0) != ",": self._unaccepted_mark = comma_mark @@ -411,7 +516,7 @@ def _handle_entry(self, m, m_val) -> Entry | ParsingFailedBlock: end_index=comma_mark.end(), ) else: - key = self.bibstr[m.end() + 1 : comma_mark.start()].strip() + key = self.bibstr[start_bracket_mark.end() : comma_mark.start()].strip() fields, end_index, duplicate_keys = self._move_to_end_of_entry(comma_mark.end()) entry = Entry( @@ -449,7 +554,7 @@ def _handle_string(self, m) -> String: f" but found {equals_mark.group(0)}", end_index=equals_mark.end(), ) - key = self.bibstr[m.end() + 1 : equals_mark.start()].strip() + key = self.bibstr[start_bracket_mark.end() : equals_mark.start()].strip() value_start = equals_mark.end() end_i = self._move_to_closed_bracket() value = self.bibstr[value_start:end_i].strip() diff --git a/tests/splitter_tests/test_splitter_block_start_detection.py b/tests/splitter_tests/test_splitter_block_start_detection.py index 3d7553e..f62a523 100644 --- a/tests/splitter_tests/test_splitter_block_start_detection.py +++ b/tests/splitter_tests/test_splitter_block_start_detection.py @@ -10,6 +10,10 @@ import pytest +from bibtexparser.model import Entry +from bibtexparser.model import ExplicitComment +from bibtexparser.model import Preamble +from bibtexparser.model import String from bibtexparser.splitter import Splitter # ============================================================================= @@ -154,6 +158,147 @@ def test_mixed_blocks_same_line( assert len(library.comments) == expected_comments +@pytest.mark.parametrize( + "bibtex, expected_block_type", + [ + pytest.param( + "@article\n {key, title = {Line-separated delimiter}}", + Entry, + id="entry", + ), + pytest.param('@string\n {name = "Line-separated delimiter"}', String, id="string"), + pytest.param('@preamble\n {"Line-separated delimiter"}', Preamble, id="preamble"), + pytest.param("@comment\n {Line-separated delimiter}", ExplicitComment, id="comment"), + ], +) +def test_newline_before_curly_delimiter_starts_block(bibtex: str, expected_block_type: type): + """BibTeX whitespace may separate a block type from its outer delimiter.""" + library = Splitter(bibtex).split() + + assert len(library.failed_blocks) == 0 + assert len(library.blocks) == 1 + assert isinstance(library.blocks[0], expected_block_type) + assert library.blocks[0].raw == bibtex + + +@pytest.mark.parametrize("entry_type", ["systematic-review", "review:type", "software/package"]) +def test_punctuation_in_entry_type_is_not_silently_ignored(entry_type: str): + """BibTeX identifiers permit punctuation beyond Python word characters.""" + library = Splitter(f"@{entry_type}{{key, title = {{Custom entry type}}}}").split() + + assert len(library.failed_blocks) == 0 + assert len(library.comments) == 0 + assert [(entry.entry_type, entry.key) for entry in library.entries] == [(entry_type, "key")] + + +def test_newline_before_parenthesis_delimiter_is_explicit_failure(): + """Unsupported parenthesis syntax remains visible when whitespace precedes its delimiter.""" + bibtex = "@article\n (key, title = {Line-separated delimiter})" + + library = Splitter(bibtex).split() + + assert len(library.blocks) == 1 + assert len(library.failed_blocks) == 1 + assert len(library.comments) == 0 + assert library.failed_blocks[0].raw == bibtex + + +@pytest.mark.parametrize( + "entry_type", ["article", "systematic-review", "review:type", "software/package"] +) +@pytest.mark.parametrize("separator", ["", " ", "\n ", "\r\n\t"]) +@pytest.mark.parametrize( + "opening, closing, expected_kind", + [("{", "}", "entry"), ("(", ")", "failed")], + ids=["supported-curly", "unsupported-parenthesis"], +) +def test_structural_block_candidate_never_becomes_implicit_comment( + entry_type: str, separator: str, opening: str, closing: str, expected_kind: str +): + """A standard-delimited candidate must be parsed or explicitly retained as failed.""" + source = f"@{entry_type}{separator}{opening}key, title = {{Visible block}}{closing}" + + library = Splitter(source).split() + + assert len(library.comments) == 0 + assert len(library.blocks) == 1 + if expected_kind == "entry": + assert len(library.entries) == 1 + assert len(library.failed_blocks) == 0 + else: + assert len(library.entries) == 0 + assert len(library.failed_blocks) == 1 + assert library.failed_blocks[0].raw == source + + +@pytest.mark.parametrize( + "unsupported_block", + [ + pytest.param( + "@article(test, title = {A (parenthesized) title})", + id="entry", + ), + pytest.param( + '@string(name = "value (draft)")', + id="string", + ), + pytest.param( + '@preamble("value (draft)")', + id="preamble", + ), + pytest.param( + "@comment(text (with nested parentheses))", + id="comment", + ), + ], +) +@pytest.mark.parametrize("same_line", [False, True], ids=["next-line", "same-line"]) +def test_parenthesis_delimited_block_is_explicit_failure(unsupported_block: str, same_line: bool): + """Unsupported standard syntax must never disappear into an implicit comment.""" + separator = " " if same_line else "\n" + library = Splitter( + f"{unsupported_block}{separator}@book{{valid, title = {{Supported block}}}}" + ).split() + + assert len(library.failed_blocks) == 1 + assert library.failed_blocks[0].raw == unsupported_block + assert "Parenthesis-delimited blocks are not supported" in ( + library.failed_blocks[0].error.abort_reason + ) + assert len(library.comments) == 0 + assert [entry.key for entry in library.entries] == ["valid"] + + +def test_unclosed_parenthesis_block_recovers_at_next_line_block(): + """An unclosed unsupported block does not hide a later supported block.""" + library = Splitter(dedent("""\ + @article(broken, title = {Unclosed parenthesis block} + @book{valid, title = {Supported block}}""")).split() + + assert len(library.failed_blocks) == 1 + assert library.failed_blocks[0].raw == ("@article(broken, title = {Unclosed parenthesis block}") + assert [entry.key for entry in library.entries] == ["valid"] + + +@pytest.mark.parametrize( + "entry_type", + [ + pytest.param("commentary", id="comment-prefix"), + pytest.param("preamble_study", id="preamble-prefix"), + pytest.param("string_theory", id="string-prefix"), + ], +) +def test_entry_type_extending_special_command_name_is_entry(entry_type: str): + """Special commands require an exact type match; longer custom types remain entries.""" + library = Splitter(f"@{entry_type}{{key, title = {{Custom entry type}}}}").split() + + assert len(library.failed_blocks) == 0 + assert len(library.comments) == 0 + assert len(library.preambles) == 0 + assert len(library.strings) == 0 + assert [(entry.entry_type, entry.key) for entry in library.entries] == [(entry_type, "key")] + + # ============================================================================= # Test: Error recovery when new block starts at line start # =============================================================================