diff --git a/dev/integration/tests/adapter/test_aiohttp_cloud_adapter.py b/dev/integration/tests/adapter/test_aiohttp_cloud_adapter.py index d1a21492..b704f5c8 100644 --- a/dev/integration/tests/adapter/test_aiohttp_cloud_adapter.py +++ b/dev/integration/tests/adapter/test_aiohttp_cloud_adapter.py @@ -4,14 +4,110 @@ from typing import cast import pytest +from aiohttp import web from microsoft_agents.activity import Activity, ConversationAccount from microsoft_agents.hosting.aiohttp import CloudAdapter -from microsoft_agents.hosting.core import TurnContext +from microsoft_agents.hosting.core import OutboundHostValidator, TurnContext from microsoft_agents.hosting.core.authorization import ClaimsIdentity, Connections from microsoft_agents.hosting.core.connector.client import UserTokenClient +class RecordingAgent: + def __init__(self): + self.turn_count = 0 + + async def on_turn(self, context: TurnContext): + self.turn_count += 1 + + +def _create_app(host_validator: OutboundHostValidator): + adapter = CloudAdapter( + connection_manager=cast(Connections, object()), + host_validator=host_validator, + ) + agent = RecordingAgent() + app = web.Application() + + async def messages(request: web.Request): + return await adapter.process(request, agent) + + app.router.add_post("/api/messages", messages) + return app, agent + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("host_validator", "service_url", "expected_status", "expected_turn_count"), + [ + pytest.param( + OutboundHostValidator(enabled=False), + "https://evil.example.com/relay", + 202, + 1, + id="disabled-allows-unknown-host", + ), + pytest.param( + OutboundHostValidator(enabled=True), + "https://smba.trafficmanager.net/teams/", + 202, + 1, + id="enabled-allows-default-microsoft-host", + ), + pytest.param( + OutboundHostValidator(enabled=True), + "https://evil.example.com/relay", + 401, + 0, + id="enabled-denies-unknown-host", + ), + pytest.param( + OutboundHostValidator( + enabled=True, + hosts=["contoso.com"], + include_default_microsoft_hosts=False, + ), + "https://api.contoso.com/messages", + 202, + 1, + id="enabled-allows-configured-host", + ), + pytest.param( + OutboundHostValidator( + enabled=True, + hosts=["contoso.com"], + include_default_microsoft_hosts=False, + ), + "https://graph.microsoft.com/v1.0", + 401, + 0, + id="enabled-without-defaults-denies-microsoft-host", + ), + ], +) +async def test_cloud_adapter_applies_outbound_host_validator( + aiohttp_client, + host_validator: OutboundHostValidator, + service_url: str, + expected_status: int, + expected_turn_count: int, +): + app, agent = _create_app(host_validator) + client = await aiohttp_client(app) + + response = await client.post( + "/api/messages", + json={ + "type": "message", + "conversation": {"id": "conversation-id"}, + "serviceUrl": service_url, + }, + ) + + assert response.status == expected_status + assert agent.turn_count == expected_turn_count + + @pytest.mark.asyncio @pytest.mark.parametrize( "token_service_endpoint", diff --git a/dev/integration/tests/adapter/test_fastapi_cloud_adapter.py b/dev/integration/tests/adapter/test_fastapi_cloud_adapter.py index 54bf339f..a32f77e2 100644 --- a/dev/integration/tests/adapter/test_fastapi_cloud_adapter.py +++ b/dev/integration/tests/adapter/test_fastapi_cloud_adapter.py @@ -4,14 +4,108 @@ from typing import cast import pytest +from fastapi import FastAPI, Request +from fastapi.testclient import TestClient from microsoft_agents.activity import Activity, ConversationAccount -from microsoft_agents.hosting.core import TurnContext +from microsoft_agents.hosting.core import OutboundHostValidator, TurnContext from microsoft_agents.hosting.core.authorization import ClaimsIdentity, Connections from microsoft_agents.hosting.core.connector.client import UserTokenClient from microsoft_agents.hosting.fastapi import CloudAdapter +class RecordingAgent: + def __init__(self): + self.turn_count = 0 + + async def on_turn(self, context: TurnContext): + self.turn_count += 1 + + +def _create_client(host_validator: OutboundHostValidator): + adapter = CloudAdapter( + connection_manager=cast(Connections, object()), + host_validator=host_validator, + ) + agent = RecordingAgent() + app = FastAPI() + + @app.post("/api/messages") + async def messages(request: Request): + return await adapter.process(request, agent) + + return TestClient(app), agent + + +@pytest.mark.parametrize( + ("host_validator", "service_url", "expected_status", "expected_turn_count"), + [ + pytest.param( + OutboundHostValidator(enabled=False), + "https://evil.example.com/relay", + 202, + 1, + id="disabled-allows-unknown-host", + ), + pytest.param( + OutboundHostValidator(enabled=True), + "https://smba.trafficmanager.net/teams/", + 202, + 1, + id="enabled-allows-default-microsoft-host", + ), + pytest.param( + OutboundHostValidator(enabled=True), + "https://evil.example.com/relay", + 401, + 0, + id="enabled-denies-unknown-host", + ), + pytest.param( + OutboundHostValidator( + enabled=True, + hosts=["contoso.com"], + include_default_microsoft_hosts=False, + ), + "https://api.contoso.com/messages", + 202, + 1, + id="enabled-allows-configured-host", + ), + pytest.param( + OutboundHostValidator( + enabled=True, + hosts=["contoso.com"], + include_default_microsoft_hosts=False, + ), + "https://graph.microsoft.com/v1.0", + 401, + 0, + id="enabled-without-defaults-denies-microsoft-host", + ), + ], +) +def test_cloud_adapter_applies_outbound_host_validator( + host_validator: OutboundHostValidator, + service_url: str, + expected_status: int, + expected_turn_count: int, +): + client, agent = _create_client(host_validator) + + response = client.post( + "/api/messages", + json={ + "type": "message", + "conversation": {"id": "conversation-id"}, + "serviceUrl": service_url, + }, + ) + + assert response.status_code == expected_status + assert agent.turn_count == expected_turn_count + + @pytest.mark.asyncio @pytest.mark.parametrize( "token_service_endpoint", diff --git a/libraries/microsoft-agents-hosting-aiohttp/microsoft_agents/hosting/aiohttp/cloud_adapter.py b/libraries/microsoft-agents-hosting-aiohttp/microsoft_agents/hosting/aiohttp/cloud_adapter.py index 9ca08b5a..3fe3936a 100644 --- a/libraries/microsoft-agents-hosting-aiohttp/microsoft_agents/hosting/aiohttp/cloud_adapter.py +++ b/libraries/microsoft-agents-hosting-aiohttp/microsoft_agents/hosting/aiohttp/cloud_adapter.py @@ -10,7 +10,10 @@ from microsoft_agents.hosting.core.http import ( HttpResponse, ) -from microsoft_agents.hosting.core import ChannelServiceClientFactoryBase +from microsoft_agents.hosting.core import ( + ChannelServiceClientFactoryBase, + OutboundHostValidator, +) from .agent_http_adapter import AgentHttpAdapter from ._aiohttp_request_adapter import AiohttpRequestAdapter @@ -25,6 +28,7 @@ def __init__( connection_manager: Connections | None = None, channel_service_client_factory: ChannelServiceClientFactoryBase | None = None, channel_service_client_factory_options: dict | None = None, + host_validator: OutboundHostValidator | None = None, ): """ Initializes a new instance of the CloudAdapter class. @@ -38,6 +42,7 @@ def __init__( connection_manager=connection_manager, channel_service_client_factory=channel_service_client_factory, channel_service_client_factory_options=channel_service_client_factory_options, + host_validator=host_validator, ) async def process(self, request: Request, agent: Agent) -> Optional[Response]: diff --git a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/__init__.py b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/__init__.py index 2b9e817f..d72d1eb2 100644 --- a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/__init__.py +++ b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/__init__.py @@ -9,6 +9,7 @@ from .middleware_set import Middleware, MiddlewareSet from .rest_channel_service_client_factory import RestChannelServiceClientFactory from .turn_context import TurnContext +from .outbound_host_validator import OutboundHostValidator # HTTP abstractions from .http import ( @@ -122,6 +123,7 @@ "Middleware", "RestChannelServiceClientFactory", "TurnContext", + "OutboundHostValidator", "HttpRequestProtocol", "HttpResponse", "HttpResponseFactory", diff --git a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/_http_adapter_base.py b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/_http_adapter_base.py index 5d6cfaa1..4b928a8d 100644 --- a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/_http_adapter_base.py +++ b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/_http_adapter_base.py @@ -7,6 +7,8 @@ from http import HTTPStatus from traceback import format_exc +import logging + from microsoft_agents.activity import Activity, DeliveryModes from microsoft_agents.hosting.core.telemetry.adapter import spans @@ -20,6 +22,9 @@ from .message_factory import MessageFactory from .rest_channel_service_client_factory import RestChannelServiceClientFactory from .turn_context import TurnContext +from .outbound_host_validator import OutboundHostValidator, _try_create_url + +logger = logging.getLogger(__name__) class HttpAdapterBase(ChannelServiceAdapter, ABC): @@ -35,6 +40,7 @@ def __init__( connection_manager: Connections | None = None, channel_service_client_factory: ChannelServiceClientFactoryBase | None = None, channel_service_client_factory_options: dict | None = None, + host_validator: OutboundHostValidator | None = None, ): """Initialize the HTTP adapter. @@ -73,6 +79,7 @@ async def on_turn_error(context: TurnContext, error: Exception): connection_manager, **(channel_service_client_factory_options or {}), ) + self._host_validator = host_validator or OutboundHostValidator() super().__init__(factory) @@ -126,6 +133,11 @@ async def process_request( "Activity must have type and conversation.id" ) + if not self._validate_service_url(claims_identity, activity): + return HttpResponseFactory.unauthorized( + "Service URL is not allowed by the host validator." + ) + try: # Process the inbound activity with the agent invoke_response = await self.process_activity( @@ -151,3 +163,57 @@ async def process_request( except PermissionError: return HttpResponseFactory.unauthorized() + + def _validate_service_url( + self, claims_identity: ClaimsIdentity, activity: Activity + ) -> bool: + """Validate the service URL against the claims identity. + + Args: + claims_identity: The claims identity to validate against. + activity: The activity containing the service URL to validate. + + Returns: + True if the service URL is valid, False otherwise. + """ + + if ( + self._host_validator.enabled + and activity.service_url + and not self._host_validator.is_allowed(activity.service_url) + ): + logger.warning( + "Service URL %s is not allowed by the host validator.", + activity.service_url, + ) + return False + + if not claims_identity: + return True + + claims_service_url = claims_identity.get_claim_value("serviceurl") + if activity.service_url and claims_service_url: + claim_url = _try_create_url(claims_service_url) + activity_url = _try_create_url(activity.service_url) + claim_url_host = (claim_url.host or "") if claim_url else "" + activity_url_host = (activity_url.host or "") if activity_url else "" + if ( + not claim_url + or not activity_url + or claim_url_host.casefold() != activity_url_host.casefold() + ): + if self._host_validator and self._host_validator.enabled: + logger.warning( + "Service URL host mismatch: %s vs %s", + claim_url_host, + activity_url_host, + ) + return False + else: + logger.warning( + "Service URL host mismatch (host validator disabled): %s vs %s", + claim_url_host, + activity_url_host, + ) + + return True diff --git a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/outbound_host_validator.py b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/outbound_host_validator.py new file mode 100644 index 00000000..ea58f10e --- /dev/null +++ b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/outbound_host_validator.py @@ -0,0 +1,117 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from yarl import URL + +_DEFAULT_MICROSOFT_HOSTS = [ + "botframework.com", # Bot Connector / channel services URLs + "smba.trafficmanager.net", # Teams service URLs + "teams.microsoft.com", + "teams.microsoft.us", + "graph.microsoft.com", # Microsoft Graph + "sharepoint.com", # Sharepoint / OneDrive hosted attachments + "svc.ms", # Teams attachment CDN + "blob.core.windows.net", # Azure Blob Storage / Attachment Management Service +] + + +def _try_create_url(url: str | URL) -> URL | None: + """Attempts to create a URL object from the given string or URL. + + :param url: The URL string or URL object to create. + :return: A URL object if successful, None otherwise. + """ + try: + return URL(url) if isinstance(url, str) else url + except (ValueError, TypeError): + return None + + +def _normalize(host: str) -> str | None: + """Normalizes a host string to a suffix for comparison. + + :param host: The host string to normalize. + :return: The normalized host suffix, or None if the input is invalid. + """ + if not host: + return None + + host = host.strip().casefold() + + if host.startswith("*."): + host = host[2:] + + url_obj = _try_create_url(host) + if url_obj and url_obj.host: + host = url_obj.host + else: + slash = host.find("/") + if slash >= 0: + host = host[:slash] + + colon = host.find(":") + if colon >= 0: + host = host[:colon] + + return host if host else None + + +class OutboundHostValidator: + """Validates that an outbound URL targets an allowed host before the SDK makes a + server-side, often token-bearing, request to it (e.g. Activity.service_url callbacks or attachment + downloads). This is the SDK's shared anti-SSRF ("allow_hosts") control.""" + + _enabled: bool + _suffixes: set[str] + + def __init__( + self, + enabled: bool = False, + hosts: list[str] | None = None, + include_default_microsoft_hosts: bool = True, + ): + self._enabled = enabled + suffixes: list[str] = [] + if include_default_microsoft_hosts: + suffixes.extend(_DEFAULT_MICROSOFT_HOSTS) + + if hosts: + for host in hosts: + normalized = _normalize(host) + if normalized is not None: + suffixes.append(normalized) + + self._suffixes = set(suffixes) + + @property + def enabled(self) -> bool: + """Gets whether the validator is enabled. If disabled, all outbound hosts are allowed.""" + return self._enabled + + def is_allowed(self, url: str | URL) -> bool: + """Checks whether the given URL is allowed by the validator. + + :param url: The URL to check. + :return: True if the URL is allowed, False otherwise. + """ + if not self._enabled: + return True + + url_obj = _try_create_url(url) + if not url_obj: + return False + + if not url_obj.absolute: + return False + + host = url_obj.host + if not host: + return False + + host = host.casefold() + + for suffix in self._suffixes: + if host == suffix or host.endswith("." + suffix): + return True + + return False diff --git a/libraries/microsoft-agents-hosting-core/setup.py b/libraries/microsoft-agents-hosting-core/setup.py index 06a48045..6703d855 100644 --- a/libraries/microsoft-agents-hosting-core/setup.py +++ b/libraries/microsoft-agents-hosting-core/setup.py @@ -18,5 +18,7 @@ "python-dotenv>=1.1.1", "opentelemetry-api>=1.27.0", "opentelemetry-sdk>=1.27.0", + "aiohttp>=3.11.11", + "yarl>=1.17.0,<2.0", ], ) diff --git a/libraries/microsoft-agents-hosting-fastapi/microsoft_agents/hosting/fastapi/cloud_adapter.py b/libraries/microsoft-agents-hosting-fastapi/microsoft_agents/hosting/fastapi/cloud_adapter.py index 731ccb9c..c56f913e 100644 --- a/libraries/microsoft-agents-hosting-fastapi/microsoft_agents/hosting/fastapi/cloud_adapter.py +++ b/libraries/microsoft-agents-hosting-fastapi/microsoft_agents/hosting/fastapi/cloud_adapter.py @@ -11,7 +11,10 @@ from microsoft_agents.hosting.core.http import ( HttpResponse, ) -from microsoft_agents.hosting.core import ChannelServiceClientFactoryBase +from microsoft_agents.hosting.core import ( + ChannelServiceClientFactoryBase, + OutboundHostValidator, +) from .agent_http_adapter import AgentHttpAdapter from ._fastapi_request_adapter import FastApiRequestAdapter @@ -26,6 +29,7 @@ def __init__( connection_manager: Connections | None = None, channel_service_client_factory: ChannelServiceClientFactoryBase | None = None, channel_service_client_factory_options: dict | None = None, + host_validator: OutboundHostValidator | None = None, ): """ Initializes a new instance of the CloudAdapter class. @@ -39,6 +43,7 @@ def __init__( connection_manager=connection_manager, channel_service_client_factory=channel_service_client_factory, channel_service_client_factory_options=channel_service_client_factory_options, + host_validator=host_validator, ) async def process(self, request: Request, agent: Agent) -> Optional[Response]: diff --git a/tests/hosting_core/test_outbound_host_validator.py b/tests/hosting_core/test_outbound_host_validator.py new file mode 100644 index 00000000..a0fc6698 --- /dev/null +++ b/tests/hosting_core/test_outbound_host_validator.py @@ -0,0 +1,179 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest +from yarl import URL + +from microsoft_agents.hosting.core.outbound_host_validator import ( + OutboundHostValidator, + _normalize, + _try_create_url, +) + + +class TestTryCreateUrl: + def test_creates_url_from_string(self): + assert _try_create_url("https://example.com/path") == URL( + "https://example.com/path" + ) + + def test_returns_existing_url(self): + url = URL("https://example.com/path") + + assert _try_create_url(url) is url + + @pytest.mark.parametrize("url", ["http://[invalid", "https://example.com:invalid"]) + def test_returns_none_for_invalid_url(self, url): + assert _try_create_url(url) is None + + +class TestNormalize: + @pytest.mark.parametrize( + ("host", "expected"), + [ + ("EXAMPLE.COM", "example.com"), + (" example.com ", "example.com"), + ("*.example.com", "example.com"), + ("example.com:443", "example.com"), + ("example.com/path", "example.com"), + ("https://Example.COM:443/path", "example.com"), + ], + ) + def test_normalizes_host(self, host, expected): + assert _normalize(host) == expected + + @pytest.mark.parametrize("host", ["", " "]) + def test_returns_none_for_empty_normalized_host(self, host): + assert _normalize(host) is None + + +class TestOutboundHostValidator: + @pytest.mark.parametrize( + "url", + [ + "https://evil.example.com/relay", + "https://169.254.169.254/latest/meta-data", + "http://localhost/admin", + "not-a-uri", + None, + ], + ) + def test_disabled_allows_everything(self, url): + validator = OutboundHostValidator(enabled=False) + + assert validator.enabled is False + assert validator.is_allowed(url) is True + + def test_default_options_disable_enforcement(self): + validator = OutboundHostValidator() + + assert validator.enabled is False + assert validator.is_allowed("https://evil.example.com/relay") is True + + @pytest.mark.parametrize( + "url", + [ + "https://smba.trafficmanager.net/teams/", + "https://graph.microsoft.com/v1.0/me", + "https://contoso.sharepoint.com/file", + "https://foo.svc.ms/download", + "https://account.blob.core.windows.net/container/blob", + "https://webchat.botframework.com/callback", + ], + ) + def test_enabled_allows_first_party_microsoft_hosts(self, url): + validator = OutboundHostValidator(enabled=True) + + assert validator.enabled is True + assert validator.is_allowed(url) is True + + @pytest.mark.parametrize( + "url", + [ + "https://evil.example.com/relay", + "https://169.254.169.254/latest/meta-data", + "https://internal-test.local:8443/secret", + "http://localhost/admin", + "https://localhost/admin", + "https://evil.trafficmanager.net/relay", + ], + ) + def test_enabled_denies_unknown_hosts(self, url): + validator = OutboundHostValidator(enabled=True) + + assert validator.is_allowed(url) is False + + @pytest.mark.parametrize( + "configured_host", + [ + "https://contoso.com", + "https://contoso.com/some/path", + "contoso.com:8443", + "contoso.com/path", + ], + ) + def test_enabled_normalizes_configured_host(self, configured_host): + validator = OutboundHostValidator(enabled=True, hosts=[configured_host]) + + assert validator.is_allowed("https://contoso.com/api") is True + assert validator.is_allowed("https://files.contoso.com/api") is True + + def test_enabled_allows_configured_host_exact_and_subdomain(self): + validator = OutboundHostValidator(enabled=True, hosts=["contoso.com"]) + + assert validator.is_allowed("https://contoso.com/api") is True + assert validator.is_allowed("https://files.contoso.com/api") is True + assert validator.is_allowed("https://notcontoso.com/api") is False + assert validator.is_allowed("https://contoso.com.evil.com/api") is False + + def test_enabled_accepts_wildcard_prefix_in_configured_host(self): + validator = OutboundHostValidator(enabled=True, hosts=["*.fabrikam.com"]) + + assert validator.is_allowed("https://api.fabrikam.com/x") is True + assert validator.is_allowed("https://fabrikam.com/x") is True + + def test_enabled_without_defaults_denies_microsoft_hosts(self): + validator = OutboundHostValidator( + enabled=True, + hosts=["contoso.com"], + include_default_microsoft_hosts=False, + ) + + assert validator.is_allowed("https://graph.microsoft.com/v1.0/me") is False + assert validator.is_allowed("https://contoso.com/x") is True + + def test_enabled_host_match_is_case_insensitive(self): + validator = OutboundHostValidator(enabled=True) + + assert validator.is_allowed("https://GRAPH.MICROSOFT.COM/v1.0/me") is True + + @pytest.mark.parametrize( + "url", + [ + "not-a-uri", + "/relative/path", + None, + ], + ) + def test_enabled_denies_non_absolute_or_invalid_urls(self, url): + validator = OutboundHostValidator(enabled=True) + + assert validator.is_allowed(url) is False + + def test_accepts_url_object(self): + validator = OutboundHostValidator( + enabled=True, + hosts=["example.com"], + include_default_microsoft_hosts=False, + ) + + assert validator.is_allowed(URL("https://example.com/path")) is True + + def test_rejects_userinfo_host_confusion(self): + validator = OutboundHostValidator( + enabled=True, + hosts=["example.com"], + include_default_microsoft_hosts=False, + ) + + assert validator.is_allowed("https://example.com@evil.example/path") is False