From 6e53ea5cd903979ecde2fff2f74dd176f76fc6da Mon Sep 17 00:00:00 2001 From: L4XB Date: Wed, 16 Sep 2026 04:43:27 +0200 Subject: [PATCH] fix(ingest): keep the word boundary a styled space carries markdownify runs an inline element's text through chomp(), which lifts the surrounding whitespace out and then returns an empty string once nothing is left, so an element holding only whitespace disappears together with its whitespace: Hello world -> Helloworld First Last -> **First****Last** The second shape is what an editor emitting one element per styled run produces for a space between two bold words, so an ingested page can lose word boundaries anywhere a phrase is styled mid-sentence. Whatever is searched and linked in the graph then carries the merged word. Wrap the conversion function markdownify resolves per tag, which covers every inline tag at once, and keep the ImportError fallback intact: the converter is built lazily so an install without markdownify still takes the tag strip. --- graphify/ingest.py | 40 ++++++++++++++++++++-- tests/test_ingest_html_markdown.py | 55 ++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 tests/test_ingest_html_markdown.py diff --git a/graphify/ingest.py b/graphify/ingest.py index 7dc92c3418..e5346a326a 100644 --- a/graphify/ingest.py +++ b/graphify/ingest.py @@ -2,6 +2,7 @@ from __future__ import annotations import json import re +from functools import lru_cache import uuid import urllib.error import urllib.parse @@ -86,14 +87,49 @@ def _fetch_html(url: str) -> str: return safe_fetch_text(url) +# Inline tags whose markdownify conversion runs the text through chomp(), which lifts the +# surrounding whitespace out of the text and then returns an empty string once nothing is +# left. An element holding only whitespace therefore disappears together with its whitespace +# and the words on either side run together: a page that styles the space between two words, +# which is what an editor emitting one element per styled run produces, is ingested with +# `Hello world` as `Helloworld`. +_WHITESPACE_ONLY_PRESERVING_TAGS = frozenset( + {"a", "b", "code", "del", "em", "i", "kbd", "s", "samp", "strike", "strong", "sub", "sup", "u"} +) + + +@lru_cache(maxsize=1) +def _markdown_converter(): + """markdownify's converter, with a whitespace-only inline element left as its whitespace. + + Raises ImportError when markdownify is absent, which is what the caller falls back on. + """ + from markdownify import MarkdownConverter + + class _KeepWhitespaceOnly(MarkdownConverter): + def get_conv_fn(self, tag_name): + convert_fn = super().get_conv_fn(tag_name) + if convert_fn is None or tag_name.lower() not in _WHITESPACE_ONLY_PRESERVING_TAGS: + return convert_fn + + def keep_whitespace_only(el, text, *args, **kwargs): + if not text.strip(): + return text + return convert_fn(el, text, *args, **kwargs) + + return keep_whitespace_only + + return _KeepWhitespaceOnly + + def _html_to_markdown(html: str, url: str) -> str: """Convert HTML to clean markdown. Uses markdownify if available, else basic strip.""" # Always pre-strip script/style so their text content never leaks into output html = re.sub(r"]*>.*?", "", html, flags=re.DOTALL | re.IGNORECASE) html = re.sub(r"]*>.*?", "", html, flags=re.DOTALL | re.IGNORECASE) try: - from markdownify import markdownify - return markdownify(html, heading_style="ATX", bullets="-", strip=["img"]) + converter = _markdown_converter() + return converter(heading_style="ATX", bullets="-", strip=["img"]).convert(html) except ImportError: # Fallback: basic tag strip text = re.sub(r"<[^>]+>", " ", html) diff --git a/tests/test_ingest_html_markdown.py b/tests/test_ingest_html_markdown.py new file mode 100644 index 0000000000..82be2ebcbb --- /dev/null +++ b/tests/test_ingest_html_markdown.py @@ -0,0 +1,55 @@ +"""`_html_to_markdown` keeps the word boundaries a page carries. + +markdownify drops an inline element that holds nothing but whitespace, and takes the +whitespace with it, so two words are ingested as one. +""" +from __future__ import annotations + +import pytest + +from graphify.ingest import _html_to_markdown + +pytest.importorskip("markdownify") + +URL = "https://example.test/page" + + +@pytest.mark.parametrize( + "tag", ["a", "b", "strong", "em", "i", "s", "del", "code", "sub", "sup"] +) +def test_whitespace_only_element_keeps_the_word_boundary(tag: str) -> None: + attributes = ' href="https://example.test"' if tag == "a" else "" + + assert _html_to_markdown(f"

Hello<{tag}{attributes}> world

", URL).strip() == "Hello world" + + +def test_space_between_two_styled_runs_survives() -> None: + """An editor that emits one element per styled run puts the space in its own element.""" + html = "

First Last

" + + assert _html_to_markdown(html, URL).strip() == "**First** **Last**" + + +@pytest.mark.parametrize( + ("html", "expected"), + [ + ("

Hello bold world

", "Hello **bold** world"), + ("

Hello it world

", "Hello *it* world"), + ("

Hello x world

", "Hello `x` world"), + ("

Title

", "# Title"), + ("
  • item
", "- item"), + ('

ab

', "ab"), + ], +) +def test_conversion_options_and_content_are_unchanged(html: str, expected: str) -> None: + """The control: headings stay ATX, bullets stay `-`, images stay stripped.""" + assert _html_to_markdown(html, URL).strip() == expected + + +def test_script_and_style_text_never_reaches_the_output() -> None: + html = "

Hello

" + + markdown = _html_to_markdown(html, URL) + + assert "secret" not in markdown + assert "color:red" not in markdown