diff --git a/graphify/ingest.py b/graphify/ingest.py index 7dc92c3418..e5346a326a 100644 --- a/graphify/ingest.py +++ b/graphify/ingest.py @@ -2,6 +2,7 @@ from __future__ import annotations import json import re +from functools import lru_cache import uuid import urllib.error import urllib.parse @@ -86,14 +87,49 @@ def _fetch_html(url: str) -> str: return safe_fetch_text(url) +# Inline tags whose markdownify conversion runs the text through chomp(), which lifts the +# surrounding whitespace out of the text and then returns an empty string once nothing is +# left. An element holding only whitespace therefore disappears together with its whitespace +# and the words on either side run together: a page that styles the space between two words, +# which is what an editor emitting one element per styled run produces, is ingested with +# `Hello world` as `Helloworld`. +_WHITESPACE_ONLY_PRESERVING_TAGS = frozenset( + {"a", "b", "code", "del", "em", "i", "kbd", "s", "samp", "strike", "strong", "sub", "sup", "u"} +) + + +@lru_cache(maxsize=1) +def _markdown_converter(): + """markdownify's converter, with a whitespace-only inline element left as its whitespace. + + Raises ImportError when markdownify is absent, which is what the caller falls back on. + """ + from markdownify import MarkdownConverter + + class _KeepWhitespaceOnly(MarkdownConverter): + def get_conv_fn(self, tag_name): + convert_fn = super().get_conv_fn(tag_name) + if convert_fn is None or tag_name.lower() not in _WHITESPACE_ONLY_PRESERVING_TAGS: + return convert_fn + + def keep_whitespace_only(el, text, *args, **kwargs): + if not text.strip(): + return text + return convert_fn(el, text, *args, **kwargs) + + return keep_whitespace_only + + return _KeepWhitespaceOnly + + def _html_to_markdown(html: str, url: str) -> str: """Convert HTML to clean markdown. Uses markdownify if available, else basic strip.""" # Always pre-strip script/style so their text content never leaks into output html = re.sub(r"]*>.*?", "", html, flags=re.DOTALL | re.IGNORECASE) html = re.sub(r"]*>.*?", "", html, flags=re.DOTALL | re.IGNORECASE) try: - from markdownify import markdownify - return markdownify(html, heading_style="ATX", bullets="-", strip=["img"]) + converter = _markdown_converter() + return converter(heading_style="ATX", bullets="-", strip=["img"]).convert(html) except ImportError: # Fallback: basic tag strip text = re.sub(r"<[^>]+>", " ", html) diff --git a/tests/test_ingest_html_markdown.py b/tests/test_ingest_html_markdown.py new file mode 100644 index 0000000000..82be2ebcbb --- /dev/null +++ b/tests/test_ingest_html_markdown.py @@ -0,0 +1,55 @@ +"""`_html_to_markdown` keeps the word boundaries a page carries. + +markdownify drops an inline element that holds nothing but whitespace, and takes the +whitespace with it, so two words are ingested as one. +""" +from __future__ import annotations + +import pytest + +from graphify.ingest import _html_to_markdown + +pytest.importorskip("markdownify") + +URL = "https://example.test/page" + + +@pytest.mark.parametrize( + "tag", ["a", "b", "strong", "em", "i", "s", "del", "code", "sub", "sup"] +) +def test_whitespace_only_element_keeps_the_word_boundary(tag: str) -> None: + attributes = ' href="https://example.test"' if tag == "a" else "" + + assert _html_to_markdown(f"

Hello<{tag}{attributes}> world

", URL).strip() == "Hello world" + + +def test_space_between_two_styled_runs_survives() -> None: + """An editor that emits one element per styled run puts the space in its own element.""" + html = "

First Last

" + + assert _html_to_markdown(html, URL).strip() == "**First** **Last**" + + +@pytest.mark.parametrize( + ("html", "expected"), + [ + ("

Hello bold world

", "Hello **bold** world"), + ("

Hello it world

", "Hello *it* world"), + ("

Hello x world

", "Hello `x` world"), + ("

Title

", "# Title"), + ("", "- item"), + ('

ab

', "ab"), + ], +) +def test_conversion_options_and_content_are_unchanged(html: str, expected: str) -> None: + """The control: headings stay ATX, bullets stay `-`, images stay stripped.""" + assert _html_to_markdown(html, URL).strip() == expected + + +def test_script_and_style_text_never_reaches_the_output() -> None: + html = "

Hello

" + + markdown = _html_to_markdown(html, URL) + + assert "secret" not in markdown + assert "color:red" not in markdown