diff --git a/bibtexparser/middlewares/__init__.py b/bibtexparser/middlewares/__init__.py index 7202200d..fd5befa1 100644 --- a/bibtexparser/middlewares/__init__.py +++ b/bibtexparser/middlewares/__init__.py @@ -1,5 +1,6 @@ from bibtexparser.middlewares.enclosing import AddEnclosingMiddleware from bibtexparser.middlewares.enclosing import RemoveEnclosingMiddleware +from bibtexparser.middlewares.entrytypes import NormalizeEntryTypes from bibtexparser.middlewares.fieldkeys import NormalizeFieldKeys from bibtexparser.middlewares.interpolate import ResolveStringReferencesMiddleware from bibtexparser.middlewares.latex_encoding import LatexDecodingMiddleware diff --git a/bibtexparser/middlewares/entrytypes.py b/bibtexparser/middlewares/entrytypes.py new file mode 100644 index 00000000..863c0bf4 --- /dev/null +++ b/bibtexparser/middlewares/entrytypes.py @@ -0,0 +1,24 @@ +from bibtexparser.library import Library +from bibtexparser.model import Entry + +from .middleware import BlockMiddleware + + +class NormalizeEntryTypes(BlockMiddleware): + """Normalize entry-type identifiers to lowercase. + + Parsing preserves the declared spelling by default so a read/write cycle + does not create case-only changes. Add this middleware when canonical + lowercase entry types are preferred over source preservation. + """ + + def __init__(self, allow_inplace_modification: bool = True): + super().__init__( + allow_inplace_modification=allow_inplace_modification, + allow_parallel_execution=True, + ) + + # docstr-coverage: inherited + def transform_entry(self, entry: Entry, library: Library) -> Entry: + entry.entry_type = entry.entry_type.lower() + return entry diff --git a/bibtexparser/splitter.py b/bibtexparser/splitter.py index bbc3e97f..3ca5ead5 100644 --- a/bibtexparser/splitter.py +++ b/bibtexparser/splitter.py @@ -316,7 +316,7 @@ def split(self, library: Library | None = None) -> Library: elif m_val.startswith("@string"): library.add(self._handle_string(m), fail_on_duplicate_key=False) else: - library.add(self._handle_entry(m, m_val), fail_on_duplicate_key=False) + library.add(self._handle_entry(m), fail_on_duplicate_key=False) except BlockAbortedException as e: logger.warning( @@ -385,10 +385,12 @@ def _handle_explicit_comment(self) -> ExplicitComment: raw=self.bibstr[start_index : end_bracket_index + 1], ) - def _handle_entry(self, m, m_val) -> Entry | ParsingFailedBlock: + def _handle_entry(self, m) -> Entry | ParsingFailedBlock: """Handle entry block. Return end index""" start_line = self._current_line - entry_type = m_val[1:].strip() + # Dispatch uses a normalized marker, but retaining the source spelling here prevents + # parse-write round trips from creating case-only changes in BibTeX files. + entry_type = m.group(0)[1:].strip() start_bracket_mark = self._next_mark(accept_eof=False) if start_bracket_mark.group(0) != "{": self._unaccepted_mark = start_bracket_mark diff --git a/docs/source/biber.rst b/docs/source/biber.rst index fb58c3fc..66303f1b 100644 --- a/docs/source/biber.rst +++ b/docs/source/biber.rst @@ -1,10 +1,36 @@ ================ -Biber & Biblatex +Biber & BibLaTeX ================ +BibLaTeX normally uses Biber to process BibTeX-format data sources. It extends +the traditional BibTeX data model with additional entry types, fields, lists, +dates, inheritance mechanisms, annotations, and name forms. Biber also permits +custom data models. The structural parser therefore accepts entry-type and field +names without restricting them to either project's built-in schema. -Due to its simple and high-level nature, this library should not only support BibTeX, but also be able to parse biblatex and biber files -with ease, as they all share the same general syntax. +The test suite covers representative BibLaTeX and Biber constructs separately +from traditional BibTeX examples, including: -That said, we did not explicitly check against all of biber and biblatex features. Should you detect anything which is not supported, -please open an issue or send a pull request. +* ``@set`` and ``@xdata`` entries; +* extended name forms and data annotations; +* date ranges, UTF-8 values, and BibLaTeX-specific entry types and fields; +* names supplied by a custom Biber data model. + +These constructs use the normal parser and writer defaults. No dialect option is +needed merely to retain them. Their subgrammars remain uninterpreted strings +unless a caller explicitly applies suitable middleware. In particular, the +parser does not validate a BibLaTeX data model, resolve entry sets or inheritance, +or interpret names, lists, annotations, and dates as typed values. Use Biber when +those semantics or validation are required. + +This is not a claim that every concrete syntax accepted by Biber is implemented. +In particular, parenthesis-delimited blocks are outside the currently tested +structural-compatibility profile. Curly-brace-delimited data sources provide the +profile covered here. + +Converting between the traditional BibTeX and BibLaTeX data models is also not a +default parsing operation. BibLaTeX contains information with no general +lossless BibTeX equivalent, so a future converter must use an explicit mapping +profile and report every lossy or ambiguous decision. Such conversion belongs in +opt-in transformation middleware or a higher-level bibliography tool rather than +in the structural codec defaults. diff --git a/docs/source/customize.rst b/docs/source/customize.rst index 9b8a2ae4..c6065305 100644 --- a/docs/source/customize.rst +++ b/docs/source/customize.rst @@ -71,6 +71,17 @@ Core Middleware bibtexparser comes with a number of middleware options: +Identifier Normalization +:::::::::::::::::::::::: + +Parsing preserves entry-type spelling by default. Use +:class:`bibtexparser.middlewares.NormalizeEntryTypes` when canonical lowercase +types are preferred. Field-key normalization remains a separate choice because +it has different collision behavior: + +* :class:`bibtexparser.middlewares.NormalizeEntryTypes` +* :class:`bibtexparser.middlewares.NormalizeFieldKeys` + .. _middleware_encoding: Encoding and Enclosing of Values diff --git a/tests/middleware_tests/test_entrytypes.py b/tests/middleware_tests/test_entrytypes.py new file mode 100644 index 00000000..4da399e1 --- /dev/null +++ b/tests/middleware_tests/test_entrytypes.py @@ -0,0 +1,26 @@ +"""Behavioral contracts for explicit entry-type normalization.""" + +from bibtexparser import parse_string +from bibtexparser import write_string +from bibtexparser.middlewares import NormalizeEntryTypes + + +def test_entry_type_normalization_is_explicit(): + """Callers can request the lowercase behavior that source-preserving parsing replaced.""" + library = parse_string( + "@Article{Mixed, title = {Case}}", + append_middleware=[NormalizeEntryTypes()], + ) + + assert library.entries[0].entry_type == "article" + assert write_string(library).startswith("@article{Mixed,") + + +def test_entry_type_normalization_can_avoid_mutating_input(): + """The standard middleware copy policy remains available for reusable libraries.""" + library = parse_string("@SoftwarePackage{Mixed, title = {Case}}") + + normalized = NormalizeEntryTypes(allow_inplace_modification=False).transform(library) + + assert library.entries[0].entry_type == "SoftwarePackage" + assert normalized.entries[0].entry_type == "softwarepackage" diff --git a/tests/resources/biblatex_contract.bib b/tests/resources/biblatex_contract.bib new file mode 100644 index 00000000..a8648aab --- /dev/null +++ b/tests/resources/biblatex_contract.bib @@ -0,0 +1,32 @@ +% A synthetic BibLaTeX data source exercising syntax beyond traditional BibTeX. +@XData{shared-publisher, + publisher = {Example Press}, + location = {Berlin and London}, +} + +@Set{review-set, + entryset = {online-record,software-record}, + date = {2024}, +} + +@Online{online-record, + author = {Family, Given and given=Sam, family=Researcher, prefix=von}, + author+an = {1:family=lead;2=corresponding}, + title = {A UTF-8 Überblick}, + title+an:translation = {="An overview"}, + date = {2024-03/2024-05}, + urldate = {2025-01-31}, + url = {https://example.invalid/overview}, + xdata = {shared-publisher}, + related = {software-record}, + relatedtype = {reviewof}, + keywords = {review and reproducibility}, +} + +@Software{software-record, + author = {Tooling Team}, + title = {Review Helper}, + date = {2024-02-29}, + version = {2.0}, + url = {https://example.invalid/review-helper}, +} diff --git a/tests/splitter_tests/test_splitter_basic.py b/tests/splitter_tests/test_splitter_basic.py index 58be5c46..ccc9a3bb 100644 --- a/tests/splitter_tests/test_splitter_basic.py +++ b/tests/splitter_tests/test_splitter_basic.py @@ -348,7 +348,7 @@ def test_handles_duplicate_strings(): }, { "key": "9837531", - "type": "inproceedings", + "type": "INPROCEEDINGS", "raw": "@INPROCEEDINGS{9837531,\n" " author={Hassan, Mona Bakri and Saeed, Rashid A. and Khalifa, Othman and Ali, Elmustafa Sayed and Mokhtar, Rania A. and Hashim, Aisha A.},\n" " title={Green Machine Learning for Green Cloud Energy Efficiency},\n" @@ -357,7 +357,7 @@ def test_handles_duplicate_strings(): }, { "key": "9372936", - "type": "article", + "type": "ARTICLE", "raw": "@ARTICLE{9372936,\n" " author={Hu, Ning and Tian, Zhihong and Du, Xiaojiang and Guizani, Nadra and Zhu, Zhihan},\n" " title={Deep-Green: A Dispersed Energy-Efficiency Computing Paradigm for Green Industrial IoT},\n" @@ -366,7 +366,7 @@ def test_handles_duplicate_strings(): }, { "key": "5445167", - "type": "article", + "type": "ARTICLE", "raw": "@ARTICLE{5445167,\n" " author={Kumar, Karthik and Lu, Yung-Hsiang},\n" " title={Cloud Computing for Mobile Users: Can Offloading Computation Save Energy?},\n" diff --git a/tests/splitter_tests/test_splitter_entry.py b/tests/splitter_tests/test_splitter_entry.py index 5d57da0e..89f1d0cc 100644 --- a/tests/splitter_tests/test_splitter_entry.py +++ b/tests/splitter_tests/test_splitter_entry.py @@ -48,27 +48,27 @@ def test_field_enclosings(field_key: str, value: str, line: int): @pytest.mark.parametrize( - "declared_block_type,expected", + "declared_block_type", [ - ("@article", "article"), - ("@ARTICLE", "article"), - ("@Article", "article"), - ("@book", "book"), - ("@BOOK", "book"), - ("@Book", "book"), - ("@inbook", "inbook"), - ("@INBOOK", "inbook"), - ("@Inbook", "inbook"), - ("@incollection", "incollection"), - ("@INCOLLECTION", "incollection"), - ("@Incollection", "incollection"), - ("@inproceedings", "inproceedings"), - ("@INPROCEEDINGS", "inproceedings"), - ("@InprocEEdings", "inproceedings"), + "@article", + "@ARTICLE", + "@Article", + "@book", + "@BOOK", + "@Book", + "@inbook", + "@INBOOK", + "@Inbook", + "@incollection", + "@INCOLLECTION", + "@Incollection", + "@inproceedings", + "@INPROCEEDINGS", + "@InprocEEdings", ], ) -def test_entry_type(declared_block_type, expected): - """Test that the entry type is case insensitive.""" +def test_entry_type_case_is_preserved(declared_block_type): + """Entry type recognition is case-insensitive without normalizing its spelling.""" bibtex_str = """@article{test, author = "John Doe" }""" @@ -77,7 +77,7 @@ def test_entry_type(declared_block_type, expected): assert len(library.failed_blocks) == 0 assert len(library.entries) == 1 - assert library.entries[0].entry_type == expected + assert library.entries[0].entry_type == declared_block_type[1:] @pytest.mark.parametrize("field_value", EDGE_CASE_VALUES) @@ -287,7 +287,7 @@ def test_entry_with_space_before_bracket(entry: str): assert library.entries[0].key == "normal_entry" assert len(library.entries[0].fields) == 2 - assert library.entries[1].entry_type == "article" + assert library.entries[1].entry_type == "Article" assert library.entries[1].key == "articleTestKey" assert len(library.entries[1].fields) == 1 diff --git a/tests/test_biblatex_contract.py b/tests/test_biblatex_contract.py new file mode 100644 index 00000000..6666512d --- /dev/null +++ b/tests/test_biblatex_contract.py @@ -0,0 +1,69 @@ +"""Compatibility contracts for BibLaTeX data sources processed by Biber. + +BibLaTeX uses BibTeX-format data sources but defines a larger, configurable data +model and several Biber extensions. These tests deliberately exercise those +constructs separately from traditional BibTeX examples so compatibility cannot +regress under the misleading assumption that only the classic types and fields +matter. +""" + +from pathlib import Path + +from bibtexparser import parse_string +from bibtexparser import write_string + +RESOURCE = Path(__file__).parent / "resources" / "biblatex_contract.bib" + + +def _entry_inventory(source: str) -> list[tuple[str, str, list[tuple[str, str]]]]: + """Return the ordered BibLaTeX information the structural codec must retain.""" + library = parse_string(source) + assert library.failed_blocks == [] + return [ + ( + entry.entry_type, + entry.key, + [(field.key, field.value) for field in entry.fields], + ) + for entry in library.entries + ] + + +class TestBibLaTeXDataSourceContract: + """Protect schema-agnostic parsing of the richer BibLaTeX data model.""" + + def test_default_roundtrip_preserves_biblatex_extensions(self): + """Default I/O retains types, fields, values, spelling, and source order.""" + source = RESOURCE.read_text(encoding="utf-8") + + written = write_string(parse_string(source)) + + assert _entry_inventory(written) == _entry_inventory(source) + + def test_set_and_xdata_remain_first_class_entries(self): + """Structural parsing must not discard inheritance or entry-set records.""" + library = parse_string(RESOURCE.read_text(encoding="utf-8")) + + assert library.entries_dict["shared-publisher"].entry_type == "XData" + assert library.entries_dict["review-set"].entry_type == "Set" + assert library.entries_dict["review-set"]["entryset"] == ("online-record,software-record") + assert library.entries_dict["online-record"]["xdata"] == "shared-publisher" + + def test_annotations_and_extended_names_remain_uninterpreted(self): + """Biber-only subgrammars stay lossless until explicitly interpreted.""" + entry = parse_string(RESOURCE.read_text(encoding="utf-8")).entries_dict["online-record"] + + assert entry["author"] == ("Family, Given and given=Sam, family=Researcher, prefix=von") + assert entry["author+an"] == "1:family=lead;2=corresponding" + assert entry["title+an:translation"] == '="An overview"' + + def test_custom_data_model_names_do_not_require_a_parser_option(self): + """Unknown types and fields are data, not syntax errors for the codec.""" + source = ( + "@CustomEvidence{custom-record," + "customfield={A value permitted by a custom Biber data model}}" + ) + entry = parse_string(source).entries_dict["custom-record"] + + assert entry.entry_type == "CustomEvidence" + assert entry["customfield"] == ("A value permitted by a custom Biber data model") diff --git a/tests/test_entrypoint.py b/tests/test_entrypoint.py index 43ace6c7..f1a9c110 100644 --- a/tests/test_entrypoint.py +++ b/tests/test_entrypoint.py @@ -7,6 +7,7 @@ import pytest from bibtexparser import parse_file +from bibtexparser import parse_string from bibtexparser import write_file from bibtexparser import write_string from bibtexparser.library import Library @@ -14,6 +15,26 @@ from bibtexparser.model import Field +def test_entry_type_case_is_preserved_on_string_roundtrip(): + """Writing a parsed entry must not create case-only changes in versioned BibTeX files.""" + library = parse_string('@Article{test, title = "Case-sensitive entry type"}') + + assert write_string(library).startswith("@Article{test,") + + +def test_entry_and_field_key_case_is_preserved_on_string_roundtrip(): + """Default parsing and writing retain citation-key and field-key spelling.""" + library = parse_string("@article{CamelCaseKey, Author_Keywords = {Systems}}") + + entry = library.entries[0] + assert entry.key == "CamelCaseKey" + assert entry.fields[0].key == "Author_Keywords" + + written = write_string(library) + assert written.startswith("@article{CamelCaseKey,") + assert "\tAuthor_Keywords = {Systems}" in written + + def test_gbk(): library = parse_file("tests/resources/gbk_test.bib", encoding="gbk") assert library.entries[0]["author"] == "凯撒"