Fix wrapped {table} captions becoming body paragraphs

docs/syntax/tables.md

@@ -52,6 +52,25 @@ The `table` directive can be used to create a table with a caption:
 :::
 ::::
 
+A long caption may continue on the lines directly after the opening line,
+before the table:
+
+::::{syntax-example}
+:::{table} A table caption that is too long
+to fit on the opening line
+
+| foo | bar |
+| --- | --- |
+| baz | bim |
+:::
+::::
+
+There must be no blank line between the opening line and the rest of the caption.
+If the table has options, they must come first, directly after the opening line,
+with the rest of the caption after them.
+This applies to the `table` directive only:
+`list-table` and `csv-table` captions must stay on the opening line.
+
 The following options are recognized:
 
 :::{admonition} Table options

myst_parser/mdit_to_docutils/base.py

@@ -34,6 +34,7 @@
 from docutils.parsers.rst import Parser as RSTParser
 from docutils.parsers.rst.directives.body import ParsedLiteral
 from docutils.parsers.rst.directives.misc import Include
+from docutils.parsers.rst.directives.tables import RSTTable
 from docutils.parsers.rst.languages import get_language as get_language_rst
 from docutils.statemachine import StringList
 from docutils.utils import Reporter, SystemMessage, new_document
@@ -53,6 +54,7 @@
     MockRSTParser,
     MockState,
     MockStateMachine,
+    MockTableState,
 )
 from myst_parser.parsers.directives import (
     YAML_LOAD_ERRORS,
@@ -1925,6 +1927,7 @@ def run_directive(
             )
 
         # initialise directive
+        state: MockState | None = None
         if issubclass(directive_class, Include):
             directive_instance = MockIncludeDirective(
                 self,
@@ -1937,7 +1940,14 @@ def run_directive(
             )
         else:
             state_machine = MockStateMachine(self, position)
-            state = MockState(
+            # a table caption may wrap onto the lines directly after the opening line
+            # (or the options block), only where docutils' run would otherwise fail
+            state_class = (
+                MockTableState
+                if directive_class.run is RSTTable.run and not parsed.blank_line_removed
+                else MockState
+            )
+            state = state_class(
                 self,
                 state_machine,
                 position,
@@ -1983,6 +1993,8 @@ def run_directive(
         assert isinstance(result, list), (
             f'Directive "{name}" must return a list of nodes.'
         )
+        if isinstance(state, MockTableState):
+            result = state.add_caption_continuation(directive_instance, result)
         for i in range(len(result)):
             assert isinstance(result[i], nodes.Node), (
                 f'Directive "{name}" returned non-Node object (index {i}): {result[i]}'

myst_parser/mocking.py

@@ -19,7 +19,7 @@
 from docutils.utils import unescape
 from markdown_it.rules_inline import StateInline
 
-from .parsers.directives import MarkupError, parse_directive_text
+from .parsers.directives import MarkupError, is_option_line, parse_directive_text
 
 if TYPE_CHECKING:
     from .mdit_to_docutils.base import DocutilsRenderer
@@ -354,6 +354,100 @@ def __getattr__(self, name: str):
         raise MockingError(msg).with_traceback(sys.exc_info()[2])
 
 
+class MockTableState(MockState):
+    """A mock state for the docutils ``table`` directive,
+    whose caption may wrap onto the lines directly after the opening line.
+
+    MyST takes a directive's argument from its opening line only,
+    so the rest of a hard-wrapped caption is parsed as a paragraph before the table,
+    and the directive fails with "exactly one table expected".
+    The body is parsed exactly as by `MockState`, and is only changed in that case:
+    when it parses to a paragraph that starts on its first line,
+    followed by a table (system messages aside),
+    the paragraph is taken out (so the directive gets its one table),
+    and `add_caption_continuation` then adds its content to the table's caption.
+    """
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self._caption_continuation: nodes.paragraph | None = None
+        self._caption_messages: list[nodes.Element] = []
+
+    def nested_parse(
+        self,
+        block: StringList,
+        input_offset: int,
+        node: nodes.Element,
+        match_titles: bool = False,
+        state_machine_class=None,
+        state_machine_kwargs=None,
+    ) -> None:
+        super().nested_parse(
+            block,
+            input_offset,
+            node,
+            match_titles,
+            state_machine_class,
+            state_machine_kwargs,
+        )
+        messages = [c for c in node.children if isinstance(c, nodes.system_message)]
+        blocks = [c for c in node.children if not isinstance(c, nodes.system_message)]
+        if not (
+            len(blocks) == 2
+            and isinstance(blocks[0], nodes.paragraph)
+            and isinstance(blocks[1], nodes.table)
+        ):
+            return
+        paragraph, table = blocks
+        first_line = self._lineno + input_offset + 1
+        if paragraph.line != first_line:
+            return
+        # options go before the caption's continuation: an option line in it
+        # would be silently lost in the caption
+        if any(is_option_line(line) for line in paragraph.rawsource.splitlines()):
+            return
+        self._caption_continuation = paragraph
+        self._caption_messages = messages
+        node[:] = [table]
+
+    def add_caption_continuation(
+        self, directive: Directive, result: list[nodes.Node]
+    ) -> list[nodes.Node]:
+        """Add the wrapped caption's continuation to the table's caption.
+
+        :param directive: The table directive that was run with this state
+        :param result: The nodes returned by the directive
+        :returns: The nodes to output
+        """
+        paragraph = self._caption_continuation
+        if paragraph is None or not result or not isinstance(result[0], nodes.table):
+            return result
+        table = result[0]
+        continuation = paragraph.children[:]
+        paragraph.children = []
+        if directive.arguments and table.children and isinstance(table[0], nodes.title):
+            # the caption from the opening line, joined to its continuation by a space
+            title = table[0]
+            title.rawsource = f"{title.rawsource} {paragraph.rawsource}"
+            if (
+                title.children
+                and isinstance(title[-1], nodes.Text)
+                and continuation
+                and isinstance(continuation[0], nodes.Text)
+            ):
+                title[-1] = nodes.Text(f"{title[-1]!s} {continuation.pop(0)!s}")
+            else:
+                title += nodes.Text(" ")
+        else:
+            title = nodes.title(paragraph.rawsource, "")
+            title.source, title.line = self.state_machine.get_source_and_line(
+                directive.lineno
+            )
+            table.insert(0, title)
+        title.extend(continuation)
+        return [*result, *self._caption_messages]
+
+
 class MockStateMachine:
     """A mock version of `docutils.parsers.rst.states.RSTStateMachine`.
 

myst_parser/parsers/directives.py

@@ -85,6 +85,10 @@ class DirectiveParsingResult:
     """List of non-fatal errors encountered during parsing.
     (message, line_number)
     """
+    blank_line_removed: bool = False
+    """Whether a blank line between the opening line (or the options block)
+    and the body content was removed.
+    """
 
 
 def parse_directive_text(
@@ -161,7 +165,8 @@ def parse_directive_text(
 
     # remove first line of body if blank
     # this is to allow space between the options and the content
-    if body_lines and not body_lines[0].strip():
+    blank_line_removed = bool(body_lines) and not body_lines[0].strip()
+    if blank_line_removed:
         body_lines = body_lines[1:]
         content_offset += 1
 
@@ -170,7 +175,12 @@ def parse_directive_text(
         parse_warnings.append(ParseWarnings("Has content, but none permitted"))
 
     return DirectiveParsingResult(
-        arguments, options, body_lines, content_offset, parse_warnings
+        arguments,
+        options,
+        body_lines,
+        content_offset,
+        parse_warnings,
+        blank_line_removed,
     )
 
 
@@ -190,6 +200,16 @@ class _DirectiveOptions:
     """
 
 
+def is_option_line(line: str) -> bool:
+    """Return True if ``line`` is read as a ``:key: value`` directive option line,
+    when it is in the options block.
+    """
+    stripped = line.lstrip()
+    # lines that don't start with a colon or have 3+ colons, which are colon fences
+    # (e.g. nested directives like `::::{other}`), are not options
+    return stripped.startswith(":") and not stripped.startswith(":::")
+
+
 def _parse_directive_options(
     content: str,
     directive_class: type[Directive],
@@ -245,10 +265,7 @@ def _parse_directive_options(
         content_lines = content.splitlines()
         yaml_lines = []
         while content_lines:
-            stripped = content_lines[0].lstrip()
-            # Stop at lines that don't start with a colon or have 3+ colons, which are colon fences
-            # (e.g. nested directives like `::::{other}`)
-            if not stripped.startswith(":") or stripped.startswith(":::"):
+            if not is_option_line(content_lines[0]):
                 break
             yaml_lines.append(content_lines.pop(0).lstrip()[1:])
         options_block = "\n".join(yaml_lines)