🐛 FIX: Make a document's own front matter available in env.metadata

docs/syntax/optional.md

@@ -401,6 +401,35 @@ Therefore you can do things like:
 - {{ "a" + "b" }}
 :::
 
+This includes `env.metadata`, which holds the [front matter](syntax/frontmatter) of each document,
+keyed by docname, and so allows a document to refer to its own metadata:
+
+````md
+---
+last_review_date: 1970-09-08
+---
+
+Last reviewed on {{ env.metadata[env.docname]["last_review_date"] }}
+````
+
+```{versionadded} 5.2.0
+Previously, only *other* documents' metadata was available; a document could not read its own.
+```
+
+Note that these values are strings (non-scalars are JSON encoded), since this is how Sphinx
+stores them, so use e.g. string methods or Jinja2 filters rather than date methods.
+The exceptions are `tocdepth`, which is an integer,
+and `authors`, which a document reads about itself as a single string (such as `Alice; Bob`),
+but which Sphinx stores as a list once the document has been read.
+
+:::{warning}
+Referring to *another* document's metadata, e.g. `env.metadata["other-doc"]["key"]`, is fragile:
+the value is only present if that document has already been read in this build,
+which depends on build order, and is not the case at all under parallel reads.
+It is also not registered as a dependency, so the referring document will not be rebuilt
+when the other document's front matter changes.
+:::
+
 A substitution whose content is exactly one key is looked up directly, before any expression evaluation.
 So a key that is not a valid Jinja2 identifier, such as `foo-bar`, can be used on its own as `{{foo-bar}}`,
 but not inside a filter or expression (`{{ foo-bar | upper }}` is parsed as `foo - (bar | upper)`).

myst_parser/mdit_to_docutils/base.py

@@ -17,6 +17,7 @@
 )
 from contextlib import contextmanager, suppress
 from datetime import date, datetime
+from pathlib import Path
 from types import ModuleType
 from typing import (
     TYPE_CHECKING,
@@ -1361,10 +1362,65 @@ def render_front_matter(self, token: SyntaxTreeNode) -> None:
                 fields, language_code=self.document.settings.language_code
             )
             self.current_node.append(field_list)
+            self._add_fm_env_metadata(field_list, self.document.settings.language_code)
 
         if data.get("title") and self.md_config.title_to_header:
             self.nested_render_text(f"# {data['title']}", 0)
 
+    def _add_fm_env_metadata(
+        self, field_list: nodes.field_list, language_code: str
+    ) -> None:
+        """Eagerly add the front matter to the sphinx ``env.metadata``.
+
+        Sphinx's ``MetadataCollector`` only populates ``env.metadata`` on the
+        ``doctree-read`` event, i.e. after the whole document has been parsed.
+        Adding the metadata here as well makes it available *during* the parse,
+        so that a document can reference its own front matter,
+        e.g. ``{{ env.metadata[env.docname]["key"] }}`` in a substitution.
+
+        The values are read back off the rendered fields,
+        and keyed as the collector keys them,
+        so that the collector later overwrites each of them with its own value.
+        These are the same, except for ``authors``,
+        which is a single string here, but a list once collected.
+
+        Nothing is added unless the document being rendered is the one Sphinx is
+        reading: a Markdown file parsed into another document, for example by an
+        rST ``include`` with ``:parser: myst_parser.sphinx_``, is parsed into a
+        separate document whose ``docinfo`` the collector never sees,
+        so its front matter would otherwise remain in the including
+        document's metadata.
+
+        :param field_list: the field list rendered from the front matter.
+        :param language_code: used to resolve localised bibliographic field names.
+        """
+        env = self.sphinx_env
+        if env is None:
+            return
+        source = self.document.get("source")
+        if not source or (
+            Path(source).resolve() != Path(env.doc2path(env.docname)).resolve()
+        ):
+            return
+        bibliofields = get_language(language_code).bibliographic_fields
+        meta = env.metadata[env.docname]
+        for field in field_list:
+            name: str = field[0].astext()
+            value: Any = field[1].astext()
+            # the `DocInfo` transform converts bibliographic fields to dedicated
+            # nodes, which the collector then keys by the canonical (English) name
+            name = bibliofields.get(nodes.fully_normalize_name(name), name)
+            if name in ("dedication", "abstract"):
+                # these are placed outside the `docinfo`, so are never collected
+                continue
+            if name == "tocdepth":
+                # as per MetadataCollector
+                try:
+                    value = int(value)
+                except ValueError:
+                    value = 0
+            meta[name] = value
+
     def dict_to_fm_field_list(
         self, data: dict[str, Any], language_code: str, line: int = 0
     ) -> nodes.field_list:
@@ -1398,7 +1454,10 @@ def dict_to_fm_field_list(
         by the `DoctreeReadEvent` transform (priority 880),
         calling `MetadataCollector.process_doc`.
         In this case keys and values will be converted to strings and stored in
-        `app.env.metadata[app.env.docname]`
+        `app.env.metadata[app.env.docname]`.
+        Note that this only happens once the whole document has been parsed,
+        so `_add_fm_env_metadata` additionally seeds these keys during the
+        render, to make them readable from within the document itself.
 
         See
         https://www.sphinx-doc.org/en/master/usage/restructuredtext/field-lists.html

tests/test_sphinx/sourcedirs/frontmatter_metadata/conf.py

@@ -0,0 +1,3 @@
+extensions = ["myst_parser"]
+exclude_patterns = ["_build", "inc.md"]
+myst_enable_extensions = ["substitution"]

tests/test_sphinx/sourcedirs/frontmatter_metadata/host.rst

@@ -0,0 +1,8 @@
+Host
+====
+
+An rST document including a Markdown file,
+whose front matter must not end up in this document's metadata.
+
+.. include:: inc.md
+   :parser: myst_parser.sphinx_

tests/test_sphinx/sourcedirs/frontmatter_metadata/inc.md

@@ -0,0 +1,5 @@
+---
+leaked: yes
+---
+
+Included Markdown content.

tests/test_sphinx/sourcedirs/frontmatter_metadata/index.md

@@ -0,0 +1,22 @@
+---
+tocdepth: 2
+last_review_date: 1970-09-08
+reviewers:
+  - alice
+  - bob
+links:
+  home: https://example.com
+---
+
+# Index
+
+Own metadata: {{ env.metadata[env.docname]["last_review_date"] }}
+
+Own list: {{ env.metadata[env.docname]["reviewers"] }}
+
+Own dict: {{ env.metadata[env.docname]["links"] }}
+
+```{toctree}
+other.md
+host.rst
+```

tests/test_sphinx/sourcedirs/frontmatter_metadata/other.md

@@ -0,0 +1,9 @@
+---
+last_review_date: 2037-09-08
+---
+
+# Other
+
+Own metadata: {{ env.metadata[env.docname]["last_review_date"] }}
+
+Metadata of an already-read document: {{ env.metadata["index"]["last_review_date"] }}

tests/test_sphinx/test_sphinx_builds.py

@@ -451,6 +451,48 @@ def test_substitutions(
         get_sphinx_app_output(app, filename="index.html", regress_html=True)
 
 
+@pytest.mark.sphinx(
+    buildername="html",
+    srcdir=os.path.join(SOURCE_DIR, "frontmatter_metadata"),
+    freshenv=True,
+)
+def test_frontmatter_metadata(
+    app,
+    status,
+    warning,
+    get_sphinx_app_doctree,
+    file_regression,
+    normalize_doctree_xml,
+):
+    """Test that a document can access its own front matter via ``env.metadata``."""
+    app.build()
+    assert "build succeeded" in status.getvalue()
+    assert warning.getvalue().strip() == ""
+
+    # values should be the same strings that sphinx's MetadataCollector stores,
+    # except for ``tocdepth``, which it converts to an integer
+    ignore = {"myst_slugs", "wordcount"}  # added separately by the renderer
+    metadata = {k: v for k, v in app.env.metadata["index"].items() if k not in ignore}
+    assert metadata == {
+        "tocdepth": 2,
+        "last_review_date": "1970-09-08",
+        "reviewers": '["alice", "bob"]',
+        "links": '{"home": "https://example.com"}',
+    }
+
+    # front matter of a Markdown file parsed into another document
+    # (here an rST ``include`` with ``:parser:``) must not be added to the
+    # including document's metadata: the collector never sees that front matter,
+    # so nothing would ever overwrite or remove it
+    assert "leaked" not in app.env.metadata["host"]
+
+    get_sphinx_app_doctree(app, docname="index", regress=True)
+    file_regression.check(
+        normalize_doctree_xml(get_sphinx_app_doctree(app, docname="other").pformat()),
+        extension=".other.xml",
+    )
+
+
 @pytest.mark.sphinx(
     buildername="html",
     srcdir=os.path.join(SOURCE_DIR, "substitutions_missing"),

tests/test_sphinx/test_sphinx_builds/test_frontmatter_metadata.other.xml

@@ -0,0 +1,10 @@
+<document source="other.md">
+    <section ids="other" names="other">
+        <title>
+            Other
+        <paragraph>
+            Own metadata: 
+            2037-09-08
+        <paragraph>
+            Metadata of an already-read document: 
+            1970-09-08

tests/test_sphinx/test_sphinx_builds/test_frontmatter_metadata.xml

@@ -0,0 +1,15 @@
+<document source="index.md">
+    <section ids="index" names="index">
+        <title>
+            Index
+        <paragraph>
+            Own metadata: 
+            1970-09-08
+        <paragraph>
+            Own list: 
+            [“alice”, “bob”]
+        <paragraph>
+            Own dict: 
+            {“home”: “https://example.com”}
+        <compound classes="toctree-wrapper">
+            <toctree caption="True" entries="(None,\ 'other') (None,\ 'host')" glob="False" hidden="False" includefiles="other host" includehidden="False" maxdepth="-1" numbered="0" parent="index" rawentries="" titlesonly="False">