diff --git a/.github/workflows/_checks.yml b/.github/workflows/_checks.yml index 2a48d19..2b5bb9f 100644 --- a/.github/workflows/_checks.yml +++ b/.github/workflows/_checks.yml @@ -105,7 +105,7 @@ jobs: fastapi fastapi-sentry fastapi-otl fastapi-logging fastapi-metrics fastapi-all \ litestar litestar-sentry litestar-otl litestar-logging litestar-metrics litestar-all \ faststream faststream-sentry faststream-otl faststream-logging faststream-metrics faststream-all \ - fastmcp fastmcp-metrics fastmcp-all" + fastmcp fastmcp-otl fastmcp-metrics fastmcp-all" failed="" uv venv --python 3.11 .v-bare >/dev/null if uv pip install --python .v-bare/bin/python . >/dev/null 2>&1 \ diff --git a/docs/integrations/fastmcp.md b/docs/integrations/fastmcp.md index 784dd9f..1196390 100644 --- a/docs/integrations/fastmcp.md +++ b/docs/integrations/fastmcp.md @@ -64,6 +64,18 @@ nanoseconds. A message that raises is logged at exception level and the exceptio Set `health_checks_enabled=False` to omit the health route. +## Tracing + +With `opentelemetry_endpoint` set (and the `fastmcp-otl` extra installed), every ASGI application +`application.http_app()` builds, including the one `application.run(transport="http")` builds, is +wrapped in OpenTelemetry's ASGI middleware. Each request produces a server span named after its +route, such as `POST /mcp` or `GET /health/`, with the route in `http.route`. A request to an +unknown path produces a span named after the HTTP method alone, without `http.route`. + +The metrics path, the health-check path (unless `opentelemetry_generate_health_check_spans` is on) +and `opentelemetry_excluded_urls` produce no spans. An application already instrumented by +`StarletteInstrumentor` is left as is, so no request is traced twice. + Teardown is wired through FastMCP's provider lifecycle — `bootstrapper.teardown()` runs automatically when the FastMCP server's ASGI lifespan shuts down (i.e. when the application that serves `application.http_app()` shuts down). diff --git a/docs/introduction/configuration.md b/docs/introduction/configuration.md index 5285ff6..c3c7c42 100644 --- a/docs/introduction/configuration.md +++ b/docs/introduction/configuration.md @@ -122,7 +122,7 @@ Additional parameters: ### Metrics Left unset, no `MeterProvider` is installed and the metrics signal stays off. Set it, and the -FastAPI, Litestar and FastStream instrumentations start recording their duration histograms +FastAPI, Litestar, FastStream and FastMCP instrumentations start recording their duration histograms against it; until then they build those instruments against a no-op provider and the data goes nowhere. diff --git a/docs/introduction/installation.md b/docs/introduction/installation.md index 6707bb0..d1ca5ea 100644 --- a/docs/introduction/installation.md +++ b/docs/introduction/installation.md @@ -8,7 +8,7 @@ You can choose required framework and instruments using this table: |---------------|--------------------|----------------------|-------------------|---------------------|--------------------------------------| | sentry | `litestar-sentry` | `faststream-sentry` | `fastapi-sentry` | `sentry` (compose) | `sentry` | | prometheus | `litestar-metrics` | `faststream-metrics` | `fastapi-metrics` | `fastmcp-metrics` | not used | -| opentelemetry | `litestar-otl` | `faststream-otl` | `fastapi-otl` | not used | `otl` | +| opentelemetry | `litestar-otl` | `faststream-otl` | `fastapi-otl` | `fastmcp-otl` | `otl` | | pyroscope | `pyroscope` | `pyroscope` | `pyroscope` | `pyroscope` | `pyroscope` | | structlog | `litestar-logging` | `faststream-logging` | `fastapi-logging` | `logging` (compose) | `logging` | | cors | no extra | not used | no extra | not used | not used | diff --git a/lite_bootstrap/bootstrappers/fastmcp_bootstrapper.py b/lite_bootstrap/bootstrappers/fastmcp_bootstrapper.py index cee2550..dbbea26 100644 --- a/lite_bootstrap/bootstrappers/fastmcp_bootstrapper.py +++ b/lite_bootstrap/bootstrappers/fastmcp_bootstrapper.py @@ -1,5 +1,7 @@ import contextlib import dataclasses +import functools +import re import time import typing from collections.abc import AsyncGenerator @@ -8,6 +10,7 @@ from lite_bootstrap.bootstrappers.base import BaseBootstrapper from lite_bootstrap.instruments.healthchecks_instrument import HealthChecksConfig, HealthChecksInstrument from lite_bootstrap.instruments.logging_instrument import LoggingConfig, LoggingInstrument +from lite_bootstrap.instruments.opentelemetry_instrument import OpenTelemetryConfig, OpenTelemetryInstrument from lite_bootstrap.instruments.prometheus_instrument import PrometheusConfig, PrometheusInstrument from lite_bootstrap.instruments.pyroscope_instrument import PyroscopeConfig, PyroscopeInstrument from lite_bootstrap.instruments.sentry_instrument import SentryConfig, SentryInstrument @@ -20,6 +23,15 @@ from starlette.requests import Request from starlette.responses import JSONResponse, Response +if import_checker.is_fastmcp_opentelemetry_installed: + from fastmcp.server.http import StarletteWithLifespan + from opentelemetry.instrumentation.asgi import OpenTelemetryMiddleware + from opentelemetry.metrics import get_meter_provider + from opentelemetry.trace import get_tracer_provider + from opentelemetry.util.http import parse_excluded_urls + from starlette.routing import BaseRoute, Match, Mount, Route + from starlette.types import Scope + if import_checker.is_structlog_installed: import structlog @@ -29,10 +41,55 @@ import prometheus_client +# OpenTelemetryMiddleware matches its patterns against a full URL, not a bare path. +_EXCLUDED_URL_SCHEME_AND_HOST: typing.Final = r"^\w+://[^/]*" + +# Set by StarletteInstrumentor too, so an application is never traced twice +_OPENTELEMETRY_INSTRUMENTED_MARKER: typing.Final = "_is_instrumented_by_opentelemetry" + + def _make_fastmcp() -> "FastMCP[typing.Any]": return FastMCP() +def _postprocess_http_apps( + application: "FastMCP[typing.Any]", + postprocess: "typing.Callable[[StarletteWithLifespan], StarletteWithLifespan]", +) -> typing.Callable[[], None]: + """Pass every ASGI application ``http_app()`` builds through ``postprocess``; return the undo. + + FastMCP builds its ASGI application lazily, when the user calls ``http_app()`` after bootstrap or + ``run(transport="http")`` calls it, so there is nothing to instrument at bootstrap time. + """ + previous_override: typing.Final = vars(application).get("http_app") + build_http_app: typing.Final = application.http_app + + def http_app(*args: typing.Any, **kwargs: typing.Any) -> "StarletteWithLifespan": # noqa: ANN401 + return postprocess(build_http_app(*args, **kwargs)) + + application.http_app = http_app # ty: ignore[invalid-assignment] + + def restore() -> None: + if previous_override is None: + del application.http_app + else: + application.http_app = previous_override + + return restore + + +def build_fastmcp_route_details_from_scope( + scope: "Scope", + routes: "typing.Iterable[BaseRoute]", +) -> tuple[str, dict[str, str]]: + method: typing.Final = str(scope.get("method", "HTTP")).strip() + for route in routes: + if isinstance(route, (Route, Mount)) and route.path and route.matches(scope)[0] == Match.FULL: + return f"{method} {route.path}", {"http.route": route.path} + # Unmatched paths get no `http.route`: a raw path would let scanners explode span cardinality + return method, {} + + if import_checker.is_fastmcp_installed: class _TeardownProvider(Provider): @@ -80,7 +137,9 @@ async def on_message( @dataclasses.dataclass(kw_only=True, slots=True, frozen=True) -class FastMcpConfig(HealthChecksConfig, LoggingConfig, PrometheusConfig, PyroscopeConfig, SentryConfig): +class FastMcpConfig( + HealthChecksConfig, LoggingConfig, OpenTelemetryConfig, PrometheusConfig, PyroscopeConfig, SentryConfig +): application: "FastMCP[typing.Any]" = dataclasses.field(default_factory=_make_fastmcp) fastmcp_logging_middleware_enabled: bool = False @@ -102,6 +161,63 @@ async def health_check_handler(_: "Request") -> "JSONResponse": return JSONResponse(dict(self.render_health_check_data())) +@dataclasses.dataclass(kw_only=True) +class FastMcpOpenTelemetryInstrument(OpenTelemetryInstrument): + bootstrap_config: FastMcpConfig + missing_dependency_message = "opentelemetry-instrumentation-asgi is not installed" + _restore_http_app: typing.Callable[[], None] | None = dataclasses.field( + default=None, init=False, repr=False, compare=False + ) + + @staticmethod + def dependencies_installed() -> bool: + return OpenTelemetryInstrument.dependencies_installed() and import_checker.is_fastmcp_opentelemetry_installed + + def _build_excluded_url_patterns(self) -> list[str]: + """Anchored patterns for the derived paths, plus the caller's own entries verbatim. + + The trailing slash is stripped and matched optionally, so ``/health/`` excludes the path with + or without it. Anchoring is needed because ``ExcludeList`` searches unanchored: a bare + ``/health`` would also silence ``/healthy``. Caller-supplied entries stay untouched because + OpenTelemetry documents them as regexes. + """ + anchored_patterns: typing.Final = { + rf"{_EXCLUDED_URL_SCHEME_AND_HOST}{re.escape(normalized_path)}(?:/|$)" + for excluded_path in self._build_infrastructure_excluded_paths() + # A bare "/" would anchor to every URL, so it is dropped along with empty values. + if (normalized_path := excluded_path.rstrip("/")) + } + return sorted(anchored_patterns | set(self.bootstrap_config.opentelemetry_excluded_urls)) + + def _instrument_http_app(self, http_application: "StarletteWithLifespan") -> "StarletteWithLifespan": + if getattr(http_application, _OPENTELEMETRY_INSTRUMENTED_MARKER, False): + return http_application + http_application.add_middleware( + OpenTelemetryMiddleware, + default_span_details=functools.partial( + build_fastmcp_route_details_from_scope, routes=http_application.routes + ), + # OpenTelemetryMiddleware only parses a raw string from 0.56b0; the floor is 0.49b0. + excluded_urls=parse_excluded_urls(",".join(self._build_excluded_url_patterns())), + tracer_provider=get_tracer_provider(), + meter_provider=get_meter_provider(), + ) + setattr(http_application, _OPENTELEMETRY_INSTRUMENTED_MARKER, True) + return http_application + + def bootstrap(self) -> None: + super().bootstrap() + self._restore_http_app = _postprocess_http_apps(self.bootstrap_config.application, self._instrument_http_app) + + def teardown(self) -> None: + try: + super().teardown() + finally: + if self._restore_http_app is not None: + self._restore_http_app() + self._restore_http_app = None + + @dataclasses.dataclass(kw_only=True) class FastMcpPrometheusInstrument(PrometheusInstrument): bootstrap_config: FastMcpConfig @@ -142,6 +258,7 @@ class FastMcpBootstrapper(BaseBootstrapper["FastMCP[typing.Any]"]): __slots__ = "bootstrap_config", "instruments" instruments_types: typing.ClassVar = [ + FastMcpOpenTelemetryInstrument, PyroscopeInstrument, SentryInstrument, FastMcpHealthChecksInstrument, diff --git a/lite_bootstrap/import_checker.py b/lite_bootstrap/import_checker.py index e3b0e4b..29da5ef 100644 --- a/lite_bootstrap/import_checker.py +++ b/lite_bootstrap/import_checker.py @@ -37,4 +37,7 @@ def _safe_find_spec(module_name: str) -> bool: is_otlp_http_exporter_installed = _safe_find_spec("opentelemetry.exporter.otlp.proto.http.trace_exporter") is_pyroscope_installed = find_spec("pyroscope") is not None is_fastmcp_installed = find_spec("fastmcp") is not None +is_fastmcp_opentelemetry_installed = ( + is_opentelemetry_installed and is_fastmcp_installed and _safe_find_spec("opentelemetry.instrumentation.asgi") +) is_orjson_installed = find_spec("orjson") is not None diff --git a/pyproject.toml b/pyproject.toml index 55f4eda..bd3635f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,12 +196,16 @@ fastmcp = [ # fastmcp 3.0 pulls py-key-value-aio, which requires typing-extensions>=4.15. "typing-extensions>=4.15", ] +fastmcp-otl = [ + "lite-bootstrap[fastmcp,otl]", + "opentelemetry-instrumentation-asgi>=0.49b0", +] fastmcp-metrics = [ "lite-bootstrap[fastmcp]", "prometheus-client>=0.20", ] fastmcp-all = [ - "lite-bootstrap[fastmcp,fastmcp-metrics,sentry,logging,pyroscope]", + "lite-bootstrap[fastmcp,fastmcp-otl,fastmcp-metrics,sentry,logging,pyroscope]", ] [dependency-groups] diff --git a/scripts/floor_smoke.py b/scripts/floor_smoke.py index a91871f..e40a519 100644 --- a/scripts/floor_smoke.py +++ b/scripts/floor_smoke.py @@ -198,10 +198,18 @@ def _fastmcp() -> None: sentry_dsn=SENTRY_DSN, sentry_additional_params=SENTRY_PARAMS, pyroscope_endpoint=PYROSCOPE_ENDPOINT, + opentelemetry_endpoint=OTLP_ENDPOINT, + opentelemetry_metrics_endpoint=OTLP_ENDPOINT, + opentelemetry_log_traces=True, ) ) - bootstrapper.bootstrap() - bootstrapper.teardown() + application = bootstrapper.bootstrap() + try: + # FastMcpOpenTelemetryInstrument adds its middleware to the application http_app() builds. + application.http_app() + _emit_span() + finally: + bootstrapper.teardown() TARGETS: typing.Final = { diff --git a/tests/test_fastmcp_bootstrap.py b/tests/test_fastmcp_bootstrap.py index 0c9ce42..63b2438 100644 --- a/tests/test_fastmcp_bootstrap.py +++ b/tests/test_fastmcp_bootstrap.py @@ -1,17 +1,32 @@ +import contextlib import typing import uuid import warnings +from collections.abc import Generator from unittest.mock import MagicMock import prometheus_client import pytest from fastmcp import FastMCP +from fastmcp.server.http import StarletteWithLifespan from fastmcp.server.middleware import MiddlewareContext +from opentelemetry.instrumentation.asgi import OpenTelemetryMiddleware +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.sdk.trace import TracerProvider as SDKTracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import SpanKind, get_tracer_provider from starlette import status +from starlette.applications import Starlette from starlette.testclient import TestClient from lite_bootstrap import BootstrapperNotReadyError, FastMcpBootstrapper, FastMcpConfig -from lite_bootstrap.bootstrappers.fastmcp_bootstrapper import FastMcpLoggingMiddleware +from lite_bootstrap.bootstrappers import fastmcp_bootstrapper +from lite_bootstrap.bootstrappers.fastmcp_bootstrapper import ( + FastMcpLoggingMiddleware, + FastMcpOpenTelemetryInstrument, + _postprocess_http_apps, +) from lite_bootstrap.exceptions import ConfigurationError from tests.conftest import ( emulate_package_missing, @@ -338,3 +353,141 @@ def test_second_fastmcp_bootstrapper_bootstrap_raises() -> None: second.bootstrap() finally: first.teardown() + + +def _count_opentelemetry_middlewares(http_application: Starlette) -> int: + return sum(middleware.cls is OpenTelemetryMiddleware for middleware in http_application.user_middleware) + + +def _server_spans(exporter: InMemorySpanExporter) -> list[ReadableSpan]: + return [span for span in exporter.get_finished_spans() if span.kind == SpanKind.SERVER] + + +@contextlib.contextmanager +def _bootstrapped_with_span_exporter( + **overrides: typing.Any, # noqa: ANN401 +) -> Generator[tuple[FastMcpBootstrapper, FastMCP, InMemorySpanExporter]]: + bootstrapper = FastMcpBootstrapper(bootstrap_config=_make_test_config(opentelemetry_log_traces=True, **overrides)) + application = bootstrapper.bootstrap() + tracer_provider = get_tracer_provider() + assert isinstance(tracer_provider, SDKTracerProvider) + exporter = InMemorySpanExporter() + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + try: + yield bootstrapper, application, exporter + finally: + bootstrapper.teardown() + + +def test_fastmcp_otel_span_carries_route_template() -> None: + with ( + _bootstrapped_with_span_exporter() as (bootstrapper, application, exporter), + TestClient(application.http_app()) as client, + ): + assert client.get(bootstrapper.bootstrap_config.health_checks_path).status_code == status.HTTP_200_OK + client.post("/mcp", json={}) + + server_spans = _server_spans(exporter) + health_checks_path = bootstrapper.bootstrap_config.health_checks_path + assert [span.name for span in server_spans] == [f"GET {health_checks_path}", "POST /mcp"] + assert [(span.attributes or {}).get("http.route") for span in server_spans] == [health_checks_path, "/mcp"] + + +def test_fastmcp_otel_unmatched_path_span_is_named_by_method_only() -> None: + with ( + _bootstrapped_with_span_exporter() as (_, application, exporter), + TestClient(application.http_app()) as client, + ): + assert client.get("/missing/abc").status_code == status.HTTP_404_NOT_FOUND + + server_spans = _server_spans(exporter) + assert [span.name for span in server_spans] == ["GET"] + assert "http.route" not in (server_spans[0].attributes or {}) + + +def test_fastmcp_otel_excludes_infrastructure_and_configured_paths() -> None: + with ( + _bootstrapped_with_span_exporter( + health_checks_path="/custom-health/", + opentelemetry_generate_health_check_spans=False, + opentelemetry_excluded_urls=["/mcp"], + ) as (bootstrapper, application, exporter), + TestClient(application.http_app()) as client, + ): + client.get("/custom-health/") + client.get("/custom-health") + client.get(bootstrapper.bootstrap_config.prometheus_metrics_path) + client.post("/mcp", json={}) + client.get("/custom-healthy") + + assert [span.name for span in _server_spans(exporter)] == ["GET"] + + +def test_fastmcp_otel_instruments_every_http_app_once() -> None: + with _bootstrapped_with_span_exporter() as (bootstrapper, application, _): + first_http_application = application.http_app() + second_http_application = application.http_app(path="/other") + instrument = next(one for one in bootstrapper.instruments if isinstance(one, FastMcpOpenTelemetryInstrument)) + instrument._instrument_http_app(first_http_application) # noqa: SLF001 + + assert _count_opentelemetry_middlewares(first_http_application) == 1 + assert _count_opentelemetry_middlewares(second_http_application) == 1 + + +def test_fastmcp_otel_teardown_restores_http_app() -> None: + with _bootstrapped_with_span_exporter() as (_, application, _): + assert "http_app" in vars(application) + + assert "http_app" not in vars(application) + assert _count_opentelemetry_middlewares(application.http_app()) == 0 + + +def test_fastmcp_otel_leaves_http_app_alone_when_not_configured() -> None: + bootstrapper = FastMcpBootstrapper(bootstrap_config=_make_test_config()) + application = bootstrapper.bootstrap() + try: + assert "http_app" not in vars(application) + assert _count_opentelemetry_middlewares(application.http_app()) == 0 + finally: + bootstrapper.teardown() + + +def test_fastmcp_otel_is_skipped_without_asgi_instrumentation() -> None: + with emulate_package_missing_with_module_reload( + "opentelemetry.instrumentation.asgi", + ["lite_bootstrap.bootstrappers.fastmcp_bootstrapper"], + ): + with pytest.warns(UserWarning, match="opentelemetry-instrumentation-asgi"): + bootstrapper = fastmcp_bootstrapper.FastMcpBootstrapper( + bootstrap_config=_make_test_config(opentelemetry_log_traces=True) + ) + application = bootstrapper.bootstrap() + try: + assert "http_app" not in vars(application) + finally: + bootstrapper.teardown() + + +def test_fastmcp_http_app_postprocessors_chain_and_restore_in_reverse() -> None: + application = FastMCP() + applied: list[str] = [] + + def postprocessor(name: str) -> typing.Callable[[StarletteWithLifespan], StarletteWithLifespan]: + def postprocess(http_application: StarletteWithLifespan) -> StarletteWithLifespan: + applied.append(name) + return http_application + + return postprocess + + restore_first = _postprocess_http_apps(application, postprocessor("first")) + restore_second = _postprocess_http_apps(application, postprocessor("second")) + application.http_app() + assert applied == ["first", "second"] + + restore_second() + applied.clear() + application.http_app() + assert applied == ["first"] + + restore_first() + assert "http_app" not in vars(application)