diff --git a/README.md b/README.md index d47edc5..1189208 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ Contributor commands and validation steps live in ## Tools -The server currently exposes six MCP tools: +The server currently exposes seven MCP tools: | Tool | Description | |------|-------------| @@ -214,6 +214,7 @@ The server currently exposes six MCP tools: | `list_versions` | List all indexed Python versions with metadata. | | `detect_python_version` | Detect the user's local Python version and report whether that version has been indexed. | | `compare_versions` | Diff a Python stdlib symbol between two indexed versions. Returns `change=added|removed|changed|unchanged` with optional `new_in`, `changed_in`, `deprecated_in`, `signature_delta` (advisory heuristic), `see_also_added/removed`, `section_diff`, and `note` deltas. Token-frugal — emits only changed fields, not full content. | +| `whatsnew_for_version` | Browse nonempty sections of the already-indexed official What's New page for one Python version. Optional `kind` filters by conservative heading-hierarchy category. `start_index` and `max_sections` paginate the filtered results; the structured response is capped at 20,000 characters, and truncated excerpts include a stable anchor and `get_docs` follow-up hint. Querying is offline. | ## Why not Context7 or generic docs retrieval? diff --git a/src/mcp_server_python_docs/app_context.py b/src/mcp_server_python_docs/app_context.py index 5237198..0e46654 100644 --- a/src/mcp_server_python_docs/app_context.py +++ b/src/mcp_server_python_docs/app_context.py @@ -16,6 +16,7 @@ from mcp_server_python_docs.services.persistent_cache import PersistentDocsCache from mcp_server_python_docs.services.search import SearchService from mcp_server_python_docs.services.version import VersionService +from mcp_server_python_docs.services.whatsnew import WhatsNewService @dataclass @@ -28,6 +29,7 @@ class AppContext: content_service: ContentService compare_service: CompareService version_service: VersionService + whatsnew_service: WhatsNewService package_docs_service: PackageDocsService = field(default_factory=PackageDocsService) persistent_docs_cache: PersistentDocsCache | None = None synonyms: dict[str, list[str]] = field(default_factory=dict) diff --git a/src/mcp_server_python_docs/models.py b/src/mcp_server_python_docs/models.py index 2fa5654..83bb0f4 100644 --- a/src/mcp_server_python_docs/models.py +++ b/src/mcp_server_python_docs/models.py @@ -301,3 +301,29 @@ class CompareVersionsResult(BaseModel): "therefore based on symbol presence alone." ), ) + + +# --- whatsnew_for_version models --- + +WhatsNewKind = Literal[ + "new_module", "new_feature", "deprecation", "removal", + "performance", "syntax", "other", +] + + +class WhatsNewSection(BaseModel): + """A nonempty, bounded section from an indexed official release page.""" + + title: str = Field(description="Official section heading") + anchor: str = Field(description="Stable section anchor for get_docs follow-up") + body: str = Field( + description="Bounded section text; truncated sections include a get_docs hint" + ) + kind: WhatsNewKind = Field(description="Conservative heading-hierarchy category") + + +class WhatsNewResult(BaseModel): + """Paginated sections from What's New in one Python version.""" + + sections: list[WhatsNewSection] = Field(default_factory=list) + next_start_index: int | None = Field(default=None) diff --git a/src/mcp_server_python_docs/server.py b/src/mcp_server_python_docs/server.py index db7887e..5cec0b3 100644 --- a/src/mcp_server_python_docs/server.py +++ b/src/mcp_server_python_docs/server.py @@ -34,6 +34,8 @@ ListVersionsResult, PackageDocsResult, SearchDocsResult, + WhatsNewKind, + WhatsNewResult, ) from mcp_server_python_docs.services.compare import CompareService from mcp_server_python_docs.services.content import ContentService @@ -41,6 +43,7 @@ from mcp_server_python_docs.services.persistent_cache import PersistentDocsCache from mcp_server_python_docs.services.search import SearchService from mcp_server_python_docs.services.version import VersionService +from mcp_server_python_docs.services.whatsnew import WhatsNewService from mcp_server_python_docs.storage.db import ( get_cache_dir, get_index_path, @@ -162,6 +165,7 @@ async def app_lifespan(server: FastMCP) -> AsyncIterator[AppContext]: content_svc = ContentService(db, persistent_cache=persistent_docs_cache) compare_svc = CompareService(db, content_svc) version_svc = VersionService(db) + whatsnew_svc = WhatsNewService(db) package_docs_svc = PackageDocsService() # Detect user's Python version and match to indexed versions @@ -188,6 +192,7 @@ async def app_lifespan(server: FastMCP) -> AsyncIterator[AppContext]: content_service=content_svc, compare_service=compare_svc, version_service=version_svc, + whatsnew_service=whatsnew_svc, package_docs_service=package_docs_svc, persistent_docs_cache=persistent_docs_cache, detected_python_version=matched, @@ -411,6 +416,29 @@ def compare_versions( logger.exception("Unexpected error in compare_versions") raise ToolError(f"Internal error: {type(e).__name__}") + @mcp.tool(annotations=_TOOL_ANNOTATIONS) + def whatsnew_for_version( + version: CompareVersionParam, + kind: WhatsNewKind | None = None, + start_index: StartIndexParam = 0, + max_sections: MaxResultsParam = 20, + ctx: Context = None, # type: ignore[assignment] + ) -> WhatsNewResult: + """Browse indexed official What's New sections in document order. + + Use kind to narrow the release topics. Sections are bounded; for full + text use get_docs(slug='whatsnew/X.Y', version='X.Y', anchor=section.anchor). + Pagination start_index counts nonempty sections after kind filtering. + """ + app_ctx: AppContext = ctx.request_context.lifespan_context + try: + return app_ctx.whatsnew_service.get(version, kind, start_index, max_sections) + except DocsServerError as e: + raise ToolError(str(e)) + except Exception as e: + logger.exception("Unexpected error in whatsnew_for_version") + raise ToolError(f"Internal error: {type(e).__name__}") + # SRVR-07: _meta hint for get_docs tool. # FastMCP 1.27 does not expose a public API for setting _meta on tool # definitions. Deferred until the mcp SDK adds _meta support to the diff --git a/src/mcp_server_python_docs/services/whatsnew.py b/src/mcp_server_python_docs/services/whatsnew.py new file mode 100644 index 0000000..b206c62 --- /dev/null +++ b/src/mcp_server_python_docs/services/whatsnew.py @@ -0,0 +1,131 @@ +"""Offline, version-scoped discovery of sections on the official What's New page.""" +from __future__ import annotations + +import sqlite3 + +from mcp_server_python_docs.errors import PageNotFoundError +from mcp_server_python_docs.models import WhatsNewKind, WhatsNewResult, WhatsNewSection +from mcp_server_python_docs.retrieval.budget import apply_budget +from mcp_server_python_docs.services.version_resolution import validate_version + +# Cap the whole structured response, including metadata and continuation hints. +_RESULT_CHARS = 20_000 +_BODY_CHARS = 7800 + + +def _classify(headings: tuple[str, ...]) -> WhatsNewKind: + """Classify only when an explicit heading or its ancestry supports it. + + A leaf such as ``asyncio`` or an opaque ``#id6`` cannot classify itself. + Nearest relevant ancestor wins, so a future pending removal inside a + deprecations chapter remains a deprecation, not a removal in this release. + """ + for heading in reversed(headings): + title = heading.casefold().strip() + if title.startswith("pending removal"): + return "deprecation" + if title in {"deprecated", "new deprecations", "deprecated c apis"}: + return "deprecation" + if title in {"removed", "removed modules and apis", "removed c apis"}: + return "removal" + if title in {"new modules"}: + return "new_module" + if title in {"optimizations", "performance"}: + return "performance" + if "syntax" in title or title == "syntax changes": + return "syntax" + if title in {"new features", "new features related to type hints", "improved modules"}: + return "new_feature" + return "other" + + +class WhatsNewService: + """Read the already-indexed release page; never fetch at query time.""" + + def __init__(self, db: sqlite3.Connection) -> None: + self._db = db + + def get( + self, + version: str, + kind: WhatsNewKind | None = None, + start_index: int = 0, + max_sections: int = 20, + ) -> WhatsNewResult: + validate_version(self._db, version) + if start_index < 0 or not 1 <= max_sections <= 20: + raise ValueError("start_index must be nonnegative and max_sections must be 1..20") + slug = f"whatsnew/{version}" + document = self._db.execute( + "SELECT d.id FROM documents d JOIN doc_sets ds ON ds.id = d.doc_set_id " + "WHERE ds.version = ? AND ds.source = 'python-docs' " + "AND ds.language = 'en' AND d.slug = ?", + (version, slug), + ).fetchone() + if document is None: + raise PageNotFoundError( + f"official What's New page {slug!r} is not indexed for Python {version}; " + "rebuild the full documentation index (without --skip-content)" + ) + + rows = self._db.execute( + "SELECT anchor, heading, level, content_text FROM sections " + "WHERE document_id = ? ORDER BY ordinal, id", + (document["id"],), + ).fetchall() + ancestors: list[tuple[int, str]] = [] + matches: list[tuple[sqlite3.Row, WhatsNewKind]] = [] + for row in rows: + level = max(1, int(row["level"])) + # Sphinx levels may skip; compare actual levels, not stack depth. + while ancestors and ancestors[-1][0] >= level: + ancestors.pop() + ancestors.append((level, row["heading"])) + body = row["content_text"].strip() + if not body: + continue + category = _classify(tuple(title for _, title in ancestors)) + if kind is not None and category != kind: + continue + matches.append((row, category)) + page = matches[start_index : start_index + max_sections] + next_index = start_index + len(page) + sections = [ + WhatsNewSection(title=row["heading"], anchor=row["anchor"], body="", kind=category) + for row, category in page + ] + result = WhatsNewResult( + sections=sections, + next_start_index=next_index if next_index < len(matches) else None, + ) + remaining = _RESULT_CHARS - len(result.model_dump_json()) + for index, (row, _) in enumerate(page): + section = sections[index] + body = row["content_text"].strip() + hint = ( + "\n\n[Section truncated. Continue with " + f"get_docs(slug={slug!r}, version={version!r}, " + f"anchor={row['anchor']!r}).]" + ) + share = remaining // (len(page) - index) + empty_length = len(section.model_dump_json()) + # Binary-search the excerpt so escaped JSON plus the follow-up hint + # fits a fair share. Short sections leave more space for later ones. + low, high = 0, min(_BODY_CHARS, len(body)) + chosen = "" + while low <= high: + middle = (low + high) // 2 + excerpt, truncated, _ = apply_budget(body, middle) + candidate = excerpt + hint if truncated else excerpt + cost = ( + len(section.model_copy(update={"body": candidate}).model_dump_json()) + - empty_length + ) + if cost <= share: + chosen = candidate + low = middle + 1 + else: + high = middle - 1 + section.body = chosen + remaining -= len(section.model_dump_json()) - empty_length + return result diff --git a/tests/test_retrieval_regression.py b/tests/test_retrieval_regression.py index cecfc44..449b86d 100644 --- a/tests/test_retrieval_regression.py +++ b/tests/test_retrieval_regression.py @@ -14,6 +14,7 @@ from mcp_server_python_docs.services.content import ContentService from mcp_server_python_docs.services.search import SearchService from mcp_server_python_docs.services.version import VersionService +from mcp_server_python_docs.services.whatsnew import WhatsNewService from mcp_server_python_docs.storage.db import bootstrap_schema, get_readwrite_connection _CASES_PATH = Path(__file__).parent / "fixtures" / "retrieval_regression_cases.json" @@ -134,6 +135,7 @@ def _make_app_context(db, detected_python_version: str | None) -> AppContext: content_service=content_service, compare_service=CompareService(db, content_service), version_service=VersionService(db), + whatsnew_service=WhatsNewService(db), detected_python_version=detected_python_version, detected_python_source="test fixture", ) diff --git a/tests/test_services.py b/tests/test_services.py index 29adf86..772852a 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -447,12 +447,15 @@ def test_all_tools_have_annotations(self): annotations.openWorldHint is False ), f"{name} openWorldHint should be False" - def test_six_tools_registered(self): + def test_seven_tools_registered(self): from mcp_server_python_docs.server import create_server server = create_server() tools = server._tool_manager._tools - assert len(tools) == 6 + assert set(tools) == { + "search_docs", "get_docs", "lookup_package_docs", "list_versions", + "detect_python_version", "compare_versions", "whatsnew_for_version", + } def test_runtime_tool_schemas_include_input_constraints(self): import anyio diff --git a/tests/test_whatsnew.py b/tests/test_whatsnew.py new file mode 100644 index 0000000..17da766 --- /dev/null +++ b/tests/test_whatsnew.py @@ -0,0 +1,236 @@ +"""Offline release-note discovery over production-shaped indexed sections.""" +from __future__ import annotations + +import socket + +import pytest + +from mcp_server_python_docs.errors import PageNotFoundError, VersionNotFoundError +from mcp_server_python_docs.services.whatsnew import WhatsNewService +from mcp_server_python_docs.storage.db import bootstrap_schema, get_readwrite_connection + + +@pytest.fixture +def release_db(tmp_path): + db = get_readwrite_connection(tmp_path / "release.db") + bootstrap_schema(db) + cases = { + "3.12": [ + (1, "what-s-new-in-python-3-12", "What's New In Python 3.12", "Intro"), + (2, "new-features", "New Features", ""), + (3, "pep-695-type-parameter-syntax", "PEP 695: Type Parameter Syntax", "Syntax detail"), + (2, "improved-modules", "Improved Modules", ""), + (3, "asyncio", "asyncio", "Asyncio improvements"), + (2, "deprecated", "Deprecated", "D" * 9000), + ( + 3, "pending-removal-in-python-3-14", + "Pending Removal in Python 3.14", "Future warning", + ), + (2, "c-api-changes", "C API Changes", ""), + (3, "id6", "Deprecated", "C API deprecation"), + (4, "id7", "Pending Removal in Python 3.14", "C API future warning"), + (3, "id10", "Removed", "C API removal"), + (2, "removed", "Removed", ""), + (3, "imp", "imp", "Removed module"), + ], + "3.13": [ + (1, "what-s-new-in-python-3-13", "What's New In Python 3.13", "Intro"), + (2, "new-modules", "New Modules", "New module summary"), + (2, "improved-modules", "Improved Modules", ""), + (3, "asyncio", "asyncio", "Asyncio improvements"), + (2, "new-deprecations", "New Deprecations", "Deprecation summary"), + ( + 3, "pending-removal-in-python-3-14", + "Pending Removal in Python 3.14", "Future warning", + ), + ], + } + for version, sections in cases.items(): + db.execute( + "INSERT INTO doc_sets (version, label) VALUES (?, ?)", + (version, f"Python {version}"), + ) + ds_id = db.execute("SELECT last_insert_rowid()").fetchone()[0] + slug = f"whatsnew/{version}" + db.execute( + "INSERT INTO documents (doc_set_id, uri, slug, title, content_text, char_count) " + "VALUES (?, ?, ?, ?, '', 0)", + (ds_id, f"{slug}.html", slug, f"What's New {version}"), + ) + doc_id = db.execute("SELECT last_insert_rowid()").fetchone()[0] + for ordinal, (level, anchor, heading, body) in enumerate(sections): + db.execute( + "INSERT INTO sections (document_id, uri, anchor, heading, level, ordinal, " + "content_text, char_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (doc_id, f"{slug}.html#{anchor}", anchor, heading, level, ordinal, body, len(body)), + ) + db.commit() + yield db + db.close() + + +def test_312_hierarchy_empty_headings_and_bounded_body(release_db): + result = WhatsNewService(release_db).get("3.12") + assert all(section.body for section in result.sections) + assert "new-features" not in {section.anchor for section in result.sections} + kinds = {section.anchor: section.kind for section in result.sections} + assert kinds["id6"] == "deprecation" + assert kinds["id7"] == "deprecation" + assert kinds["pending-removal-in-python-3-14"] == "deprecation" + assert kinds["id10"] == "removal" + assert kinds["imp"] == "removal" + assert kinds["pep-695-type-parameter-syntax"] == "syntax" + deprecated = next(section for section in result.sections if section.anchor == "deprecated") + assert len(deprecated.body) < 8000 + assert "Section truncated" in deprecated.body + assert "get_docs(slug='whatsnew/3.12', version='3.12', anchor='deprecated')" in deprecated.body + + +def test_313_asyncio_and_filter(release_db): + service = WhatsNewService(release_db) + result = service.get("3.13") + asyncio = next(section for section in result.sections if section.anchor == "asyncio") + assert asyncio.kind == "new_feature" + filtered = service.get("3.13", kind="deprecation") + assert filtered.sections + assert all(section.kind == "deprecation" for section in filtered.sections) + assert [section.anchor for section in filtered.sections] == [ + "new-deprecations", "pending-removal-in-python-3-14" + ] + + +def test_pagination_counts_filtered_nonempty_sections(release_db): + service = WhatsNewService(release_db) + complete = service.get("3.12", kind="deprecation", max_sections=20) + first = service.get("3.12", kind="deprecation", max_sections=2) + assert first.next_start_index == 2 + second = service.get("3.12", kind="deprecation", start_index=2, max_sections=2) + assert [s.anchor for s in first.sections + second.sections] == [ + s.anchor for s in complete.sections + ] + assert second.next_start_index is None + assert service.get("3.12", start_index=100).sections == [] + + +def test_skipped_levels_discard_previous_sibling_category(release_db): + doc_id = release_db.execute( + "SELECT id FROM documents WHERE slug = 'whatsnew/3.13'" + ).fetchone()[0] + for ordinal, (level, anchor, heading) in enumerate( + [ + (1, "new-features-skipped", "New Features"), + (3, "removed-skipped", "Removed"), + (4, "child-skipped", "Opaque child"), + (3, "sibling-skipped", "asyncio"), + ], + start=100, + ): + release_db.execute( + "INSERT INTO sections (document_id, uri, anchor, heading, level, ordinal, " + "content_text, char_count) VALUES (?, ?, ?, ?, ?, ?, 'Excerpt', 7)", + (doc_id, f"whatsnew/3.13.html#{anchor}", anchor, heading, level, ordinal), + ) + result = WhatsNewService(release_db).get("3.13") + kinds = {section.anchor: section.kind for section in result.sections} + assert kinds["removed-skipped"] == "removal" + assert kinds["child-skipped"] == "removal" + assert kinds["sibling-skipped"] == "new_feature" + + +def test_aggregate_budget_preserves_excerpts_and_pagination(release_db): + doc_id = release_db.execute( + "SELECT id FROM documents WHERE slug = 'whatsnew/3.13'" + ).fetchone()[0] + body = 'A long "release-note" excerpt with \\ escapes. ' * 400 + for ordinal in range(25): + anchor = f"budget-{ordinal}" + release_db.execute( + "INSERT INTO sections (document_id, uri, anchor, heading, level, ordinal, " + "content_text, char_count) VALUES (?, ?, ?, ?, 2, ?, ?, ?)", + (doc_id, f"whatsnew/3.13.html#{anchor}", anchor, "Performance", 100 + ordinal, + body, len(body)), + ) + service = WhatsNewService(release_db) + first = service.get("3.13") + second = service.get("3.13", start_index=first.next_start_index) + assert len(first.sections) == 20 + assert first.next_start_index == 20 + assert second.next_start_index is None + assert len(first.model_dump_json()) <= 20_000 + assert len(second.model_dump_json()) <= 20_000 + assert [section.anchor for section in first.sections + second.sections][-25:] == [ + f"budget-{ordinal}" for ordinal in range(25) + ] + long_sections = [section for section in first.sections if section.anchor.startswith("budget-")] + assert all(len(section.body) >= 300 for section in long_sections) + assert all("get_docs(slug='whatsnew/3.13'" in section.body for section in long_sections) + + +def test_missing_version_and_missing_release_page(release_db): + service = WhatsNewService(release_db) + with pytest.raises(VersionNotFoundError, match="available:.*3.12.*3.13"): + service.get("3.14") + release_db.execute("INSERT INTO doc_sets (version, label) VALUES ('3.14', 'Python 3.14')") + with pytest.raises(PageNotFoundError, match="rebuild the full documentation index"): + service.get("3.14") + + +def test_query_never_opens_network(release_db, monkeypatch): + def reject(*args, **kwargs): + raise AssertionError("runtime network call") + + monkeypatch.setattr(socket, "create_connection", reject) + monkeypatch.setattr(socket.socket, "connect", reject) + assert WhatsNewService(release_db).get("3.12").sections + + +def test_stdio_tool_registration_and_round_trip(tmp_path): + """Real JSON-RPC transport exposes the seventh tool and structured result.""" + from mcp_server_python_docs.storage.db import get_readwrite_connection + from tests.test_stdio_smoke import ( + _assert_protocol_on_stdout_only, + _create_test_index, + _find_response, + _isolated_cache_env, + _make_notification, + _make_request, + _run_server_until_responses, + ) + + env, cache_dir = _isolated_cache_env(tmp_path) + db_path = _create_test_index(cache_dir) + db = get_readwrite_connection(db_path) + db.execute( + "INSERT INTO documents (doc_set_id, uri, slug, title, content_text, char_count) " + "VALUES (1, 'whatsnew/3.13.html', 'whatsnew/3.13', 'Whats New', '', 0)" + ) + doc_id = db.execute("SELECT last_insert_rowid()").fetchone()[0] + db.execute( + "INSERT INTO sections (document_id, uri, anchor, heading, level, ordinal, " + "content_text, char_count) VALUES (?, 'whatsnew/3.13.html#asyncio', " + "'asyncio', 'asyncio', 2, 0, 'New asyncio behavior', 20)", + (doc_id,), + ) + db.commit() + db.close() + stdin_data = ( + _make_request("initialize", { + "protocolVersion": "2024-11-05", "capabilities": {}, + "clientInfo": {"name": "test", "version": "0.1"}, + }, req_id=1) + + _make_notification("notifications/initialized") + + _make_request("tools/list", {}, req_id=2) + + _make_request("tools/call", { + "name": "whatsnew_for_version", "arguments": {"version": "3.13"} + }, req_id=3) + ) + responses = _assert_protocol_on_stdout_only(_run_server_until_responses(stdin_data, env)) + listed = _find_response(responses, 2) + assert listed is not None + assert [tool["name"] for tool in listed["result"]["tools"]] == [ + "search_docs", "get_docs", "lookup_package_docs", "list_versions", + "detect_python_version", "compare_versions", "whatsnew_for_version", + ] + called = _find_response(responses, 3) + assert called is not None + assert called["result"]["structuredContent"]["sections"][0]["anchor"] == "asyncio"