diff --git a/README.md b/README.md index cc28e94..582ac79 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ tools for lookup and section retrieval. - page and section retrieval with truncation and pagination - a local SQLite + FTS5 index; no runtime web scraping - results for each Python version you index -- six read-only MCP tools +- seven read-only MCP tools ## Quick example @@ -218,7 +218,7 @@ Contributor commands and validation steps live in ## Tools -The server currently exposes six MCP tools: +The server currently exposes seven MCP tools: | Tool | Description | |------|-------------| @@ -228,6 +228,7 @@ The server currently exposes six MCP tools: | `list_versions` | List all indexed Python versions with metadata. | | `detect_python_version` | Detect the user's local Python version and report whether that version has been indexed. | | `compare_versions` | Diff a Python stdlib symbol between two indexed versions. Returns `change=added|removed|changed|unchanged` with optional `new_in`, `changed_in`, `deprecated_in`, `signature_delta` (advisory heuristic), `see_also_added/removed`, `section_diff`, and `note` deltas. Token-frugal — emits only changed fields, not full content. | +| `whatsnew_for_version` | Browse nonempty sections of the already-indexed official What's New page for one Python version. Optional `kind` filters by conservative heading-hierarchy category. `start_index` and `max_sections` paginate the filtered results; the structured response is capped at 20,000 characters, and truncated excerpts include a stable anchor and `get_docs` follow-up hint. Canonical titles and anchors are unchanged; if one section cannot fit its metadata plus nonempty indexed text and any required continuation hint, an explicit bounded error is returned. Page size may be reduced to reserve usable excerpts. The cap counts Unicode characters in the model JSON, excluding the MCP envelope. Querying is offline. | For `search_docs` with `kind="auto"` and an explicit indexed `version`, the original lookup and full-text search run first. If both return no hits, the diff --git a/src/mcp_server_python_docs/app_context.py b/src/mcp_server_python_docs/app_context.py index 5237198..0e46654 100644 --- a/src/mcp_server_python_docs/app_context.py +++ b/src/mcp_server_python_docs/app_context.py @@ -16,6 +16,7 @@ from mcp_server_python_docs.services.persistent_cache import PersistentDocsCache from mcp_server_python_docs.services.search import SearchService from mcp_server_python_docs.services.version import VersionService +from mcp_server_python_docs.services.whatsnew import WhatsNewService @dataclass @@ -28,6 +29,7 @@ class AppContext: content_service: ContentService compare_service: CompareService version_service: VersionService + whatsnew_service: WhatsNewService package_docs_service: PackageDocsService = field(default_factory=PackageDocsService) persistent_docs_cache: PersistentDocsCache | None = None synonyms: dict[str, list[str]] = field(default_factory=dict) diff --git a/src/mcp_server_python_docs/models.py b/src/mcp_server_python_docs/models.py index 2fa5654..83bb0f4 100644 --- a/src/mcp_server_python_docs/models.py +++ b/src/mcp_server_python_docs/models.py @@ -301,3 +301,29 @@ class CompareVersionsResult(BaseModel): "therefore based on symbol presence alone." ), ) + + +# --- whatsnew_for_version models --- + +WhatsNewKind = Literal[ + "new_module", "new_feature", "deprecation", "removal", + "performance", "syntax", "other", +] + + +class WhatsNewSection(BaseModel): + """A nonempty, bounded section from an indexed official release page.""" + + title: str = Field(description="Official section heading") + anchor: str = Field(description="Stable section anchor for get_docs follow-up") + body: str = Field( + description="Bounded section text; truncated sections include a get_docs hint" + ) + kind: WhatsNewKind = Field(description="Conservative heading-hierarchy category") + + +class WhatsNewResult(BaseModel): + """Paginated sections from What's New in one Python version.""" + + sections: list[WhatsNewSection] = Field(default_factory=list) + next_start_index: int | None = Field(default=None) diff --git a/src/mcp_server_python_docs/server.py b/src/mcp_server_python_docs/server.py index db7887e..5cec0b3 100644 --- a/src/mcp_server_python_docs/server.py +++ b/src/mcp_server_python_docs/server.py @@ -34,6 +34,8 @@ ListVersionsResult, PackageDocsResult, SearchDocsResult, + WhatsNewKind, + WhatsNewResult, ) from mcp_server_python_docs.services.compare import CompareService from mcp_server_python_docs.services.content import ContentService @@ -41,6 +43,7 @@ from mcp_server_python_docs.services.persistent_cache import PersistentDocsCache from mcp_server_python_docs.services.search import SearchService from mcp_server_python_docs.services.version import VersionService +from mcp_server_python_docs.services.whatsnew import WhatsNewService from mcp_server_python_docs.storage.db import ( get_cache_dir, get_index_path, @@ -162,6 +165,7 @@ async def app_lifespan(server: FastMCP) -> AsyncIterator[AppContext]: content_svc = ContentService(db, persistent_cache=persistent_docs_cache) compare_svc = CompareService(db, content_svc) version_svc = VersionService(db) + whatsnew_svc = WhatsNewService(db) package_docs_svc = PackageDocsService() # Detect user's Python version and match to indexed versions @@ -188,6 +192,7 @@ async def app_lifespan(server: FastMCP) -> AsyncIterator[AppContext]: content_service=content_svc, compare_service=compare_svc, version_service=version_svc, + whatsnew_service=whatsnew_svc, package_docs_service=package_docs_svc, persistent_docs_cache=persistent_docs_cache, detected_python_version=matched, @@ -411,6 +416,29 @@ def compare_versions( logger.exception("Unexpected error in compare_versions") raise ToolError(f"Internal error: {type(e).__name__}") + @mcp.tool(annotations=_TOOL_ANNOTATIONS) + def whatsnew_for_version( + version: CompareVersionParam, + kind: WhatsNewKind | None = None, + start_index: StartIndexParam = 0, + max_sections: MaxResultsParam = 20, + ctx: Context = None, # type: ignore[assignment] + ) -> WhatsNewResult: + """Browse indexed official What's New sections in document order. + + Use kind to narrow the release topics. Sections are bounded; for full + text use get_docs(slug='whatsnew/X.Y', version='X.Y', anchor=section.anchor). + Pagination start_index counts nonempty sections after kind filtering. + """ + app_ctx: AppContext = ctx.request_context.lifespan_context + try: + return app_ctx.whatsnew_service.get(version, kind, start_index, max_sections) + except DocsServerError as e: + raise ToolError(str(e)) + except Exception as e: + logger.exception("Unexpected error in whatsnew_for_version") + raise ToolError(f"Internal error: {type(e).__name__}") + # SRVR-07: _meta hint for get_docs tool. # FastMCP 1.27 does not expose a public API for setting _meta on tool # definitions. Deferred until the mcp SDK adds _meta support to the diff --git a/src/mcp_server_python_docs/services/whatsnew.py b/src/mcp_server_python_docs/services/whatsnew.py new file mode 100644 index 0000000..6f553e9 --- /dev/null +++ b/src/mcp_server_python_docs/services/whatsnew.py @@ -0,0 +1,172 @@ +"""Offline, version-scoped discovery of sections on the official What's New page.""" +from __future__ import annotations + +import sqlite3 + +from mcp_server_python_docs.errors import DocsServerError, PageNotFoundError +from mcp_server_python_docs.models import WhatsNewKind, WhatsNewResult, WhatsNewSection +from mcp_server_python_docs.retrieval.budget import apply_budget +from mcp_server_python_docs.services.version_resolution import validate_version + +# Cap the whole structured response, including metadata and continuation hints. +_RESULT_CHARS = 20_000 +_BODY_CHARS = 7800 + + +def _body_cost(section: WhatsNewSection, body: str) -> int: + return ( + len(section.model_copy(update={"body": body}).model_dump_json()) + - len(section.model_copy(update={"body": ""}).model_dump_json()) + ) + + +def _fit_body(section: WhatsNewSection, body: str, hint: str, share: int) -> str: + # Full text has no hint: its cost is outside the monotonic truncated search. + if len(body) <= _BODY_CHARS and _body_cost(section, body) <= share: + return body + low, high = 1, min(_BODY_CHARS, len(body) - 1) + chosen = section.body + while low <= high: + middle = (low + high) // 2 + excerpt, truncated, _ = apply_budget(body, middle) + candidate = excerpt + hint if truncated else excerpt + if _body_cost(section, candidate) <= share: + chosen = candidate + low = middle + 1 + else: + high = middle - 1 + return chosen + + +def _classify(headings: tuple[str, ...]) -> WhatsNewKind: + """Classify only when an explicit heading or its ancestry supports it. + + A leaf such as ``asyncio`` or an opaque ``#id6`` cannot classify itself. + Nearest relevant ancestor wins, so a future pending removal inside a + deprecations chapter remains a deprecation, not a removal in this release. + """ + for heading in reversed(headings): + title = heading.casefold().strip() + if title.startswith("pending removal"): + return "deprecation" + if title in {"deprecated", "new deprecations", "deprecated c apis"}: + return "deprecation" + if title in {"removed", "removed modules and apis", "removed c apis"}: + return "removal" + if title in {"new modules"}: + return "new_module" + if title in {"optimizations", "performance"}: + return "performance" + if "syntax" in title or title == "syntax changes": + return "syntax" + if title in {"new features", "new features related to type hints", "improved modules"}: + return "new_feature" + return "other" + + +class WhatsNewService: + """Read the already-indexed release page; never fetch at query time.""" + + def __init__(self, db: sqlite3.Connection) -> None: + self._db = db + + def get( + self, + version: str, + kind: WhatsNewKind | None = None, + start_index: int = 0, + max_sections: int = 20, + ) -> WhatsNewResult: + validate_version(self._db, version) + if start_index < 0 or not 1 <= max_sections <= 20: + raise ValueError("start_index must be nonnegative and max_sections must be 1..20") + slug = f"whatsnew/{version}" + document = self._db.execute( + "SELECT d.id FROM documents d JOIN doc_sets ds ON ds.id = d.doc_set_id " + "WHERE ds.version = ? AND ds.source = 'python-docs' " + "AND ds.language = 'en' AND d.slug = ?", + (version, slug), + ).fetchone() + if document is None: + raise PageNotFoundError( + f"official What's New page {slug!r} is not indexed for Python {version}; " + "rebuild the full documentation index (without --skip-content)" + ) + + rows = self._db.execute( + "SELECT anchor, heading, level, content_text FROM sections " + "WHERE document_id = ? ORDER BY ordinal, id", + (document["id"],), + ).fetchall() + ancestors: list[tuple[int, str]] = [] + matches: list[tuple[sqlite3.Row, WhatsNewKind]] = [] + for row in rows: + level = max(1, int(row["level"])) + # Sphinx levels may skip; compare actual levels, not stack depth. + while ancestors and ancestors[-1][0] >= level: + ancestors.pop() + ancestors.append((level, row["heading"])) + body = row["content_text"].strip() + if not body: + continue + category = _classify(tuple(title for _, title in ancestors)) + if kind is not None and category != kind: + continue + matches.append((row, category)) + # Reserve indexed text and any required hint before admitting each section. + # Stop at the first non-fitting section; the next offset must never skip it. + page: list[tuple[str, str]] = [] + sections: list[WhatsNewSection] = [] + minimum_costs: list[int] = [] + result = WhatsNewResult() + for row, category in matches[start_index : start_index + max_sections]: + section = WhatsNewSection( + title=row["heading"], anchor=row["anchor"], body="", kind=category + ) + next_index = start_index + len(sections) + 1 + candidate = WhatsNewResult( + sections=[*sections, section], + next_start_index=next_index if next_index < len(matches) else None, + ) + if not sections and len(candidate.model_dump_json()) > _RESULT_CHARS: + raise DocsServerError( + f"What's New section at start_index={start_index} has metadata " + "larger than the 20,000-character response limit" + ) + body = row["content_text"].strip() + hint = ( + "\n\n[Section truncated. Continue with " + f"get_docs(slug={slug!r}, version={version!r}, " + f"anchor={row['anchor']!r}).]" + ) + excerpt, truncated, _ = apply_budget(body, 1) + minimum = excerpt + hint if truncated else excerpt + if len(body) <= _BODY_CHARS and ( + _body_cost(section, body) <= _body_cost(section, minimum) + ): + minimum = body + section.body = minimum + if len(candidate.model_dump_json()) > _RESULT_CHARS: + if not sections: + raise DocsServerError( + f"What's New section at start_index={start_index} cannot fit " + "nonempty indexed text and its required continuation within " + "the 20,000-character response limit" + ) + break + page.append((body, hint)) + minimum_costs.append(_body_cost(section, minimum)) + sections.append(section) + result = candidate + remaining = _RESULT_CHARS - len(result.model_dump_json()) + sum(minimum_costs) + reserved = sum(minimum_costs) + for index, (body, hint) in enumerate(page): + section = sections[index] + reserved -= minimum_costs[index] + share = min( + max(remaining // (len(page) - index), minimum_costs[index]), + remaining - reserved, + ) + section.body = _fit_body(section, body, hint, share) + remaining -= _body_cost(section, section.body) + return result diff --git a/tests/test_retrieval_regression.py b/tests/test_retrieval_regression.py index cecfc44..449b86d 100644 --- a/tests/test_retrieval_regression.py +++ b/tests/test_retrieval_regression.py @@ -14,6 +14,7 @@ from mcp_server_python_docs.services.content import ContentService from mcp_server_python_docs.services.search import SearchService from mcp_server_python_docs.services.version import VersionService +from mcp_server_python_docs.services.whatsnew import WhatsNewService from mcp_server_python_docs.storage.db import bootstrap_schema, get_readwrite_connection _CASES_PATH = Path(__file__).parent / "fixtures" / "retrieval_regression_cases.json" @@ -134,6 +135,7 @@ def _make_app_context(db, detected_python_version: str | None) -> AppContext: content_service=content_service, compare_service=CompareService(db, content_service), version_service=VersionService(db), + whatsnew_service=WhatsNewService(db), detected_python_version=detected_python_version, detected_python_source="test fixture", ) diff --git a/tests/test_services.py b/tests/test_services.py index 29adf86..772852a 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -447,12 +447,15 @@ def test_all_tools_have_annotations(self): annotations.openWorldHint is False ), f"{name} openWorldHint should be False" - def test_six_tools_registered(self): + def test_seven_tools_registered(self): from mcp_server_python_docs.server import create_server server = create_server() tools = server._tool_manager._tools - assert len(tools) == 6 + assert set(tools) == { + "search_docs", "get_docs", "lookup_package_docs", "list_versions", + "detect_python_version", "compare_versions", "whatsnew_for_version", + } def test_runtime_tool_schemas_include_input_constraints(self): import anyio diff --git a/tests/test_whatsnew.py b/tests/test_whatsnew.py new file mode 100644 index 0000000..570acab --- /dev/null +++ b/tests/test_whatsnew.py @@ -0,0 +1,415 @@ +"""Offline release-note discovery over production-shaped indexed sections.""" +from __future__ import annotations + +import socket + +import pytest + +from mcp_server_python_docs.errors import DocsServerError, PageNotFoundError, VersionNotFoundError +from mcp_server_python_docs.services.whatsnew import WhatsNewService +from mcp_server_python_docs.storage.db import bootstrap_schema, get_readwrite_connection + + +@pytest.fixture +def release_db(tmp_path): + db = get_readwrite_connection(tmp_path / "release.db") + bootstrap_schema(db) + cases = { + "3.12": [ + (1, "what-s-new-in-python-3-12", "What's New In Python 3.12", "Intro"), + (2, "new-features", "New Features", ""), + (3, "pep-695-type-parameter-syntax", "PEP 695: Type Parameter Syntax", "Syntax detail"), + (2, "improved-modules", "Improved Modules", ""), + (3, "asyncio", "asyncio", "Asyncio improvements"), + (2, "deprecated", "Deprecated", "D" * 9000), + ( + 3, "pending-removal-in-python-3-14", + "Pending Removal in Python 3.14", "Future warning", + ), + (2, "c-api-changes", "C API Changes", ""), + (3, "id6", "Deprecated", "C API deprecation"), + (4, "id7", "Pending Removal in Python 3.14", "C API future warning"), + (3, "id10", "Removed", "C API removal"), + (2, "removed", "Removed", ""), + (3, "imp", "imp", "Removed module"), + ], + "3.13": [ + (1, "what-s-new-in-python-3-13", "What's New In Python 3.13", "Intro"), + (2, "new-modules", "New Modules", "New module summary"), + (2, "improved-modules", "Improved Modules", ""), + (3, "asyncio", "asyncio", "Asyncio improvements"), + (2, "new-deprecations", "New Deprecations", "Deprecation summary"), + ( + 3, "pending-removal-in-python-3-14", + "Pending Removal in Python 3.14", "Future warning", + ), + ], + } + for version, sections in cases.items(): + db.execute( + "INSERT INTO doc_sets (version, label) VALUES (?, ?)", + (version, f"Python {version}"), + ) + ds_id = db.execute("SELECT last_insert_rowid()").fetchone()[0] + slug = f"whatsnew/{version}" + db.execute( + "INSERT INTO documents (doc_set_id, uri, slug, title, content_text, char_count) " + "VALUES (?, ?, ?, ?, '', 0)", + (ds_id, f"{slug}.html", slug, f"What's New {version}"), + ) + doc_id = db.execute("SELECT last_insert_rowid()").fetchone()[0] + for ordinal, (level, anchor, heading, body) in enumerate(sections): + db.execute( + "INSERT INTO sections (document_id, uri, anchor, heading, level, ordinal, " + "content_text, char_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (doc_id, f"{slug}.html#{anchor}", anchor, heading, level, ordinal, body, len(body)), + ) + db.commit() + yield db + db.close() + + +def test_312_hierarchy_empty_headings_and_bounded_body(release_db): + result = WhatsNewService(release_db).get("3.12") + assert all(section.body for section in result.sections) + assert "new-features" not in {section.anchor for section in result.sections} + kinds = {section.anchor: section.kind for section in result.sections} + assert kinds["id6"] == "deprecation" + assert kinds["id7"] == "deprecation" + assert kinds["pending-removal-in-python-3-14"] == "deprecation" + assert kinds["id10"] == "removal" + assert kinds["imp"] == "removal" + assert kinds["pep-695-type-parameter-syntax"] == "syntax" + deprecated = next(section for section in result.sections if section.anchor == "deprecated") + assert len(deprecated.body) < 8000 + assert "Section truncated" in deprecated.body + assert "get_docs(slug='whatsnew/3.12', version='3.12', anchor='deprecated')" in deprecated.body + + +def test_313_asyncio_and_filter(release_db): + service = WhatsNewService(release_db) + result = service.get("3.13") + asyncio = next(section for section in result.sections if section.anchor == "asyncio") + assert asyncio.kind == "new_feature" + filtered = service.get("3.13", kind="deprecation") + assert filtered.sections + assert all(section.kind == "deprecation" for section in filtered.sections) + assert [section.anchor for section in filtered.sections] == [ + "new-deprecations", "pending-removal-in-python-3-14" + ] + + +def test_pagination_counts_filtered_nonempty_sections(release_db): + service = WhatsNewService(release_db) + complete = service.get("3.12", kind="deprecation", max_sections=20) + first = service.get("3.12", kind="deprecation", max_sections=2) + assert first.next_start_index == 2 + second = service.get("3.12", kind="deprecation", start_index=2, max_sections=2) + assert [s.anchor for s in first.sections + second.sections] == [ + s.anchor for s in complete.sections + ] + assert second.next_start_index is None + assert service.get("3.12", start_index=100).sections == [] + + +def test_skipped_levels_discard_previous_sibling_category(release_db): + doc_id = release_db.execute( + "SELECT id FROM documents WHERE slug = 'whatsnew/3.13'" + ).fetchone()[0] + for ordinal, (level, anchor, heading) in enumerate( + [ + (1, "new-features-skipped", "New Features"), + (3, "removed-skipped", "Removed"), + (4, "child-skipped", "Opaque child"), + (3, "sibling-skipped", "asyncio"), + ], + start=100, + ): + release_db.execute( + "INSERT INTO sections (document_id, uri, anchor, heading, level, ordinal, " + "content_text, char_count) VALUES (?, ?, ?, ?, ?, ?, 'Excerpt', 7)", + (doc_id, f"whatsnew/3.13.html#{anchor}", anchor, heading, level, ordinal), + ) + result = WhatsNewService(release_db).get("3.13") + kinds = {section.anchor: section.kind for section in result.sections} + assert kinds["removed-skipped"] == "removal" + assert kinds["child-skipped"] == "removal" + assert kinds["sibling-skipped"] == "new_feature" + + +def test_aggregate_budget_preserves_excerpts_and_pagination(release_db): + doc_id = release_db.execute( + "SELECT id FROM documents WHERE slug = 'whatsnew/3.13'" + ).fetchone()[0] + body = 'A long "release-note" excerpt with \\ escapes. ' * 400 + for ordinal in range(25): + anchor = f"budget-{ordinal}" + release_db.execute( + "INSERT INTO sections (document_id, uri, anchor, heading, level, ordinal, " + "content_text, char_count) VALUES (?, ?, ?, ?, 2, ?, ?, ?)", + (doc_id, f"whatsnew/3.13.html#{anchor}", anchor, "Performance", 100 + ordinal, + body, len(body)), + ) + service = WhatsNewService(release_db) + first = service.get("3.13") + second = service.get("3.13", start_index=first.next_start_index) + assert len(first.sections) == 20 + assert first.next_start_index == 20 + assert second.next_start_index is None + assert len(first.model_dump_json()) <= 20_000 + assert len(second.model_dump_json()) <= 20_000 + assert [section.anchor for section in first.sections + second.sections][-25:] == [ + f"budget-{ordinal}" for ordinal in range(25) + ] + long_sections = [section for section in first.sections if section.anchor.startswith("budget-")] + assert all(len(section.body) >= 300 for section in long_sections) + assert all("get_docs(slug='whatsnew/3.13'" in section.body for section in long_sections) + + +def test_large_metadata_paginates_without_changing_canonical_fields(release_db): + doc_id = release_db.execute( + "SELECT id FROM documents WHERE slug = 'whatsnew/3.13'" + ).fetchone()[0] + release_db.execute("DELETE FROM sections WHERE document_id = ?", (doc_id,)) + heading = 'Official "heading" ' * 340 + anchor = "official-anchor-" * 370 + cases = [ + (heading + " A", anchor + "-a", "Body A"), + (heading + " B", anchor + "-b", "Body B"), + ("Ordinary section", "ordinary", "Ordinary body"), + ] + for ordinal, (title, section_anchor, body) in enumerate(cases): + release_db.execute( + "INSERT INTO sections (document_id, uri, anchor, heading, level, ordinal, " + "content_text, char_count) VALUES (?, ?, ?, ?, 2, ?, ?, ?)", + (doc_id, f"whatsnew/3.13.html#{section_anchor}", section_anchor, + title, ordinal, body, len(body)), + ) + service = WhatsNewService(release_db) + pages = [] + start_index = 0 + while True: + page = service.get("3.13", start_index=start_index) + pages.append(page) + assert len(page.model_dump_json()) <= 20_000 + if page.next_start_index is None: + break + assert page.next_start_index > start_index + start_index = page.next_start_index + assert [section.title for page in pages for section in page.sections] == [ + title for title, _, _ in cases + ] + assert [section.anchor for page in pages for section in page.sections] == [ + section_anchor for _, section_anchor, _ in cases + ] + assert pages[0].next_start_index == 1 + assert pages[-1].sections[-1].body == "Ordinary body" + + +@pytest.mark.parametrize("max_sections", [1, 2, 20]) +def test_escaped_metadata_preserves_filtered_pagination(release_db, max_sections): + doc_id = release_db.execute( + "SELECT id FROM documents WHERE slug = 'whatsnew/3.13'" + ).fetchone()[0] + release_db.execute("DELETE FROM sections WHERE document_id = ?", (doc_id,)) + cases = [("New Deprecations", "category", "", 2)] + expected = [] + for index in range(3): + title = ('Escaped "heading" \\ \t\n\x00é ' * 230) + str(index) + anchor = ('anchor-"\\\t\n-' * 300) + str(index) + expected.append((title, anchor)) + cases.append((title, anchor, "Body", 3)) + cases.extend([("Removed", "removed", "Excluded", 2), + ("New Deprecations", "empty-category", "", 2)]) + for ordinal, (title, anchor, body, level) in enumerate(cases): + release_db.execute( + "INSERT INTO sections (document_id, uri, anchor, heading, level, ordinal, " + "content_text, char_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (doc_id, f"whatsnew/3.13.html#{anchor}", anchor, title, level, + ordinal, body, len(body)), + ) + service = WhatsNewService(release_db) + actual = [] + start = 0 + for _ in expected: + page = service.get("3.13", kind="deprecation", start_index=start, + max_sections=max_sections) + assert len(page.model_dump_json()) <= 20_000 + assert page.sections + assert all(section.kind == "deprecation" for section in page.sections) + actual.extend((section.title, section.anchor) for section in page.sections) + if page.next_start_index is None: + break + assert page.next_start_index == start + len(page.sections) + start = page.next_start_index + else: + pytest.fail("pagination did not terminate") + assert actual == expected + + +def test_single_section_metadata_over_limit_is_explicit_error(release_db): + doc_id = release_db.execute( + "SELECT id FROM documents WHERE slug = 'whatsnew/3.13'" + ).fetchone()[0] + release_db.execute("DELETE FROM sections WHERE document_id = ?", (doc_id,)) + for ordinal, (heading, anchor) in enumerate([ + ("Ordinary", "ordinary"), + ('"' * 9000, "a" * 9000), + ]): + release_db.execute( + "INSERT INTO sections (document_id, uri, anchor, heading, level, ordinal, " + "content_text, char_count) VALUES (?, ?, ?, ?, 2, ?, 'Body', 4)", + (doc_id, f"whatsnew/3.13.html#{anchor}", anchor, heading, ordinal), + ) + service = WhatsNewService(release_db) + first = service.get("3.13") + assert [section.anchor for section in first.sections] == ["ordinary"] + assert first.next_start_index == 1 + assert len(first.model_dump_json()) <= 20_000 + with pytest.raises(DocsServerError, match="start_index=1.*20,000-character"): + service.get("3.13", start_index=1) + + +def test_missing_version_and_missing_release_page(release_db): + service = WhatsNewService(release_db) + with pytest.raises(VersionNotFoundError, match="available:.*3.12.*3.13"): + service.get("3.14") + release_db.execute("INSERT INTO doc_sets (version, label) VALUES ('3.14', 'Python 3.14')") + with pytest.raises(PageNotFoundError, match="rebuild the full documentation index"): + service.get("3.14") + + +def test_query_never_opens_network(release_db, monkeypatch): + def reject(*args, **kwargs): + raise AssertionError("runtime network call") + + monkeypatch.setattr(socket, "create_connection", reject) + monkeypatch.setattr(socket.socket, "connect", reject) + assert WhatsNewService(release_db).get("3.12").sections + + +def test_stdio_tool_registration_and_round_trip(tmp_path): + """Real JSON-RPC transport exposes the seventh tool and structured result.""" + from mcp_server_python_docs.storage.db import get_readwrite_connection + from tests.test_stdio_smoke import ( + _assert_protocol_on_stdout_only, + _create_test_index, + _find_response, + _isolated_cache_env, + _make_notification, + _make_request, + _run_server_until_responses, + ) + + env, cache_dir = _isolated_cache_env(tmp_path) + db_path = _create_test_index(cache_dir) + db = get_readwrite_connection(db_path) + db.execute( + "INSERT INTO documents (doc_set_id, uri, slug, title, content_text, char_count) " + "VALUES (1, 'whatsnew/3.13.html', 'whatsnew/3.13', 'Whats New', '', 0)" + ) + doc_id = db.execute("SELECT last_insert_rowid()").fetchone()[0] + db.execute( + "INSERT INTO sections (document_id, uri, anchor, heading, level, ordinal, " + "content_text, char_count) VALUES (?, 'whatsnew/3.13.html#asyncio', " + "'asyncio', 'asyncio', 2, 0, 'New asyncio behavior', 20)", + (doc_id,), + ) + db.commit() + db.close() + stdin_data = ( + _make_request("initialize", { + "protocolVersion": "2024-11-05", "capabilities": {}, + "clientInfo": {"name": "test", "version": "0.1"}, + }, req_id=1) + + _make_notification("notifications/initialized") + + _make_request("tools/list", {}, req_id=2) + + _make_request("tools/call", { + "name": "whatsnew_for_version", "arguments": {"version": "3.13"} + }, req_id=3) + ) + responses = _assert_protocol_on_stdout_only(_run_server_until_responses(stdin_data, env)) + listed = _find_response(responses, 2) + assert listed is not None + assert [tool["name"] for tool in listed["result"]["tools"]] == [ + "search_docs", "get_docs", "lookup_package_docs", "list_versions", + "detect_python_version", "compare_versions", "whatsnew_for_version", + ] + called = _find_response(responses, 3) + assert called is not None + assert called["result"]["structuredContent"]["sections"][0]["anchor"] == "asyncio" + + +def _replace_release_sections(db, cases): + doc_id = db.execute( + "SELECT id FROM documents WHERE slug = 'whatsnew/3.13'" + ).fetchone()[0] + db.execute("DELETE FROM sections WHERE document_id = ?", (doc_id,)) + for ordinal, (title, anchor, body, level) in enumerate(cases): + db.execute( + "INSERT INTO sections (document_id, uri, anchor, heading, level, ordinal, " + "content_text, char_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (doc_id, f"whatsnew/3.13.html#{anchor}", anchor, title, level, + ordinal, body, len(body)), + ) + + +@pytest.mark.parametrize("max_sections", [1, 2, 20]) +@pytest.mark.parametrize( + "body", ["Indexed body", ('Quoted "\\" text ' * 100).strip()], + ids=["short", "escaped"], +) +def test_full_body_fit_despite_long_hint(release_db, max_sections, body): + title, anchor = "T" * 5000, "a" * 9800 + _replace_release_sections(release_db, [(title, anchor, body, 2)]) + page = WhatsNewService(release_db).get("3.13", max_sections=max_sections) + assert len(page.model_dump_json()) <= 20_000 + assert [(s.title, s.anchor) for s in page.sections] == [(title, anchor)] + assert page.sections[0].body == body + + +@pytest.mark.parametrize("max_sections", [1, 2, 20]) +def test_nonempty_content_and_hint_cannot_fit_is_bounded_error(release_db, max_sections): + cases = [("Ordinary", "ordinary", "Body", 2), + ("T" * 5000, "a" * 9800, "Content " * 5000, 2)] + _replace_release_sections(release_db, cases) + service = WhatsNewService(release_db) + first = service.get("3.13", max_sections=max_sections) + assert [s.anchor for s in first.sections] == ["ordinary"] + assert first.next_start_index == 1 + with pytest.raises(DocsServerError, match="start_index=1.*nonempty.*20,000-character") as error: + service.get("3.13", start_index=1, max_sections=max_sections) + assert len(str(error.value)) < 200 + + +@pytest.mark.parametrize("max_sections", [1, 2, 20]) +@pytest.mark.parametrize("escaped", [False, True]) +def test_metadata_cannot_starve_filtered_excerpts(release_db, max_sections, escaped): + title = ('T"\\ \x00é' * 200) if escaped else "T" * 2500 + anchor = ('a"\\ ' * 220) if escaped else "a" * 1900 + body = ('Indexed "\\" text é ' * 1500) if escaped else "Indexed text " * 1500 + expected = [(title + str(i), anchor + str(i), body, 3) for i in range(5)] + cases = [("New Deprecations", "category", "", 2), *expected, + ("Removed", "excluded", "Not selected", 2)] + _replace_release_sections(release_db, cases) + service = WhatsNewService(release_db) + actual, start = [], 0 + for _ in expected: + page = service.get("3.13", kind="deprecation", start_index=start, + max_sections=max_sections) + assert page.sections + assert len(page.sections) <= max_sections + assert len(page.model_dump_json()) <= 20_000 + for section in page.sections: + excerpt, separator, hint = section.body.partition("\n\n[Section truncated.") + assert excerpt and body.startswith(excerpt) + assert separator + assert f"anchor={section.anchor!r}" in hint + actual.extend((s.title, s.anchor) for s in page.sections) + if page.next_start_index is None: + break + assert page.next_start_index == start + len(page.sections) + start = page.next_start_index + else: + pytest.fail("pagination did not terminate") + assert actual == [(t, a) for t, a, _, _ in expected]