fix(ingest): keep the word boundary a styled space carries

graphify/ingest.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 import json
 import re
+from functools import lru_cache
 import uuid
 import urllib.error
 import urllib.parse
@@ -86,14 +87,49 @@ def _fetch_html(url: str) -> str:
     return safe_fetch_text(url)
 
 
+# Inline tags whose markdownify conversion runs the text through chomp(), which lifts the
+# surrounding whitespace out of the text and then returns an empty string once nothing is
+# left. An element holding only whitespace therefore disappears together with its whitespace
+# and the words on either side run together: a page that styles the space between two words,
+# which is what an editor emitting one element per styled run produces, is ingested with
+# `Hello<b> </b>world` as `Helloworld`.
+_WHITESPACE_ONLY_PRESERVING_TAGS = frozenset(
+    {"a", "b", "code", "del", "em", "i", "kbd", "s", "samp", "strike", "strong", "sub", "sup", "u"}
+)
+
+
+@lru_cache(maxsize=1)
+def _markdown_converter():
+    """markdownify's converter, with a whitespace-only inline element left as its whitespace.
+
+    Raises ImportError when markdownify is absent, which is what the caller falls back on.
+    """
+    from markdownify import MarkdownConverter
+
+    class _KeepWhitespaceOnly(MarkdownConverter):
+        def get_conv_fn(self, tag_name):
+            convert_fn = super().get_conv_fn(tag_name)
+            if convert_fn is None or tag_name.lower() not in _WHITESPACE_ONLY_PRESERVING_TAGS:
+                return convert_fn
+
+            def keep_whitespace_only(el, text, *args, **kwargs):
+                if not text.strip():
+                    return text
+                return convert_fn(el, text, *args, **kwargs)
+
+            return keep_whitespace_only
+
+    return _KeepWhitespaceOnly
+
+
 def _html_to_markdown(html: str, url: str) -> str:
     """Convert HTML to clean markdown. Uses markdownify if available, else basic strip."""
     # Always pre-strip script/style so their text content never leaks into output
     html = re.sub(r"<script[^>]*>.*?</script>", "", html, flags=re.DOTALL | re.IGNORECASE)
     html = re.sub(r"<style[^>]*>.*?</style>", "", html, flags=re.DOTALL | re.IGNORECASE)
     try:
-        from markdownify import markdownify
-        return markdownify(html, heading_style="ATX", bullets="-", strip=["img"])
+        converter = _markdown_converter()
+        return converter(heading_style="ATX", bullets="-", strip=["img"]).convert(html)
     except ImportError:
         # Fallback: basic tag strip
         text = re.sub(r"<[^>]+>", " ", html)

tests/test_ingest_html_markdown.py

@@ -0,0 +1,55 @@
+"""`_html_to_markdown` keeps the word boundaries a page carries.
+
+markdownify drops an inline element that holds nothing but whitespace, and takes the
+whitespace with it, so two words are ingested as one.
+"""
+from __future__ import annotations
+
+import pytest
+
+from graphify.ingest import _html_to_markdown
+
+pytest.importorskip("markdownify")
+
+URL = "https://example.test/page"
+
+
+@pytest.mark.parametrize(
+    "tag", ["a", "b", "strong", "em", "i", "s", "del", "code", "sub", "sup"]
+)
+def test_whitespace_only_element_keeps_the_word_boundary(tag: str) -> None:
+    attributes = ' href="https://example.test"' if tag == "a" else ""
+
+    assert _html_to_markdown(f"<p>Hello<{tag}{attributes}> </{tag}>world</p>", URL).strip() == "Hello world"
+
+
+def test_space_between_two_styled_runs_survives() -> None:
+    """An editor that emits one element per styled run puts the space in its own element."""
+    html = "<p><b>First</b><b> </b><b>Last</b></p>"
+
+    assert _html_to_markdown(html, URL).strip() == "**First** **Last**"
+
+
+@pytest.mark.parametrize(
+    ("html", "expected"),
+    [
+        ("<p>Hello <b>bold</b> world</p>", "Hello **bold** world"),
+        ("<p>Hello <em>it</em> world</p>", "Hello *it* world"),
+        ("<p>Hello <code>x</code> world</p>", "Hello `x` world"),
+        ("<h1>Title</h1>", "# Title"),
+        ("<ul><li>item</li></ul>", "- item"),
+        ('<p>a<img src="x.png">b</p>', "ab"),
+    ],
+)
+def test_conversion_options_and_content_are_unchanged(html: str, expected: str) -> None:
+    """The control: headings stay ATX, bullets stay `-`, images stay stripped."""
+    assert _html_to_markdown(html, URL).strip() == expected
+
+
+def test_script_and_style_text_never_reaches_the_output() -> None:
+    html = "<p>Hello</p><script>var secret = 1;</script><style>p{color:red}</style>"
+
+    markdown = _html_to_markdown(html, URL)
+
+    assert "secret" not in markdown
+    assert "color:red" not in markdown