diff --git a/python/packages/hyperlight/README.md b/python/packages/hyperlight/README.md index 76e5237793..ead73e4506 100644 --- a/python/packages/hyperlight/README.md +++ b/python/packages/hyperlight/README.md @@ -19,6 +19,25 @@ Hyperlight-backed CodeAct integrations for Microsoft Agent Framework. > Sandbox isolation does not supply FIDES enforcement. > See the [FIDES guide](../../samples/02-agents/security/FIDES_DEVELOPER_GUIDE.md). +## Error diagnostics + +Host tool callbacks expose a generic `RuntimeError("Host tool execution failed.")` +by default, before diagnostics reach generated code. Raw sandbox failure details +are omitted from error contents. Successful outputs and known-safe output-limit +or cleanup errors retain their existing behavior. + +Both `HyperlightCodeActProvider` and `HyperlightExecuteCodeTool` accept the +host-side option `include_detailed_errors=True` for trusted channels. It preserves +original callback exceptions and sandbox error details, which may contain +sensitive data. It is not a parameter of `execute_code` and is independent of the +outer chat client's error policy and sensitive-data logging. + +The policy is captured in each run snapshot and serialized state. Sandbox cache +keys include it, so detailed-error callbacks are not reused for redacted runs. +Initialization, snapshot restore, and execution failures follow the same policy. +Failed sandbox entries are retired; cancellation and system-exit signals still +propagate without becoming ordinary execution errors. + ## Installation ```bash diff --git a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py index 3c97dcce5a..645e0f2de4 100644 --- a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py +++ b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py @@ -112,6 +112,7 @@ class _RunConfig: max_output_files: int = DEFAULT_MAX_OUTPUT_FILES max_output_file_bytes: int = DEFAULT_MAX_OUTPUT_FILE_BYTES max_output_total_bytes: int = DEFAULT_MAX_OUTPUT_TOTAL_BYTES + include_detailed_errors: bool = False @property def mounted_paths(self) -> tuple[str, ...]: @@ -135,6 +136,7 @@ def cache_key(self) -> tuple[Any, ...]: self.max_output_files, self.max_output_file_bytes, self.max_output_total_bytes, + self.include_detailed_errors, ) @@ -173,7 +175,8 @@ class _SandboxWorker: are dropped on the wrong thread and PyO3 panics. To prevent this, every exception raised inside a worker closure is caught on the worker, the traceback is dropped while still on the worker thread, and a sanitized - copy (preserving message and exception type) is re-raised on the caller. + copy is re-raised on the caller. Ordinary failures expose their original + type and message only when detailed errors are enabled. """ __slots__ = ("_executor", "_initialized", "_reusable", "_sandbox", "_snapshot") @@ -186,20 +189,23 @@ def __init__(self, *, name: str = "hl-sandbox") -> None: self._initialized = False self._reusable = True - def _run_on_worker(self, fn: Callable[[], _T]) -> _T: + def _run_on_worker(self, fn: Callable[[], _T], *, include_detailed_errors: bool = False) -> _T: """Run ``fn`` on the worker thread; sanitize any exception's traceback there. If ``fn`` raises, the exception's ``__traceback__`` is dropped on the worker thread (so any PyO3 unsendable locals captured in frame locals are released - on the owner thread) and a fresh exception of the same type is raised on - the caller's thread carrying only the original message. + on the owner thread). Ordinary failures are replaced with a generic + exception unless detailed errors are enabled. Cancellation and other + system signals retain their type. """ def _wrapped() -> tuple[bool, Any]: try: return True, fn() except BaseException as exc: - exc_type = type(exc) + self._reusable = False + exc_type: type[BaseException] + exc_args: tuple[str, ...] # Capture args (usually (message,)) so the re-raised exception keeps the # original shape for types whose constructor doesn't accept a single str. # Coerce each arg to ``str`` on the worker thread: if a caller-supplied @@ -207,7 +213,12 @@ def _wrapped() -> tuple[bool, Any]: # unsendable object in args, forwarding it as-is would re-introduce the # same cross-thread Drop hazard the traceback nulling avoids. Strings # are always sendable. Fall back to the str() form if args is empty. - exc_args: tuple[str, ...] = tuple(str(a) for a in exc.args) if exc.args else (str(exc),) + if isinstance(exc, Exception) and not include_detailed_errors: + exc_type = RuntimeError + exc_args = ("Code execution failed.",) + else: + exc_type = type(exc) + exc_args = tuple(str(a) for a in exc.args) if exc.args else (str(exc),) # Drop the traceback on the worker thread so frame locals (which # may include PyO3 unsendable objects) are released here, not on # the caller thread that will receive the wrapped exception. @@ -224,11 +235,12 @@ def _wrapped() -> tuple[bool, Any]: # If the exception type's constructor rejects the captured args (rare), fall # back to a RuntimeError carrying the string form so we never lose the signal. try: - raise exc_type(*exc_args) + error = exc_type(*exc_args) except TypeError: raise RuntimeError(f"{exc_type.__name__}: {exc_args}") from None + raise error - def initialize(self, build_fn: Callable[[], tuple[Any, Any]]) -> None: + def initialize(self, build_fn: Callable[[], tuple[Any, Any]], *, include_detailed_errors: bool = False) -> None: """Build and install the sandbox+snapshot on the worker thread. ``build_fn`` is invoked with no arguments on the worker thread. It must @@ -244,7 +256,7 @@ def _init_on_worker() -> None: # Locals fall out of scope on the worker thread; the worker-local # attributes hold the only strong refs from now on. - self._run_on_worker(_init_on_worker) + self._run_on_worker(_init_on_worker, include_detailed_errors=include_detailed_errors) def execute( self, @@ -255,6 +267,7 @@ def execute( max_output_files: int, max_output_file_bytes: int, max_output_total_bytes: int, + include_detailed_errors: bool = False, ) -> list[Content]: """Restore + run + build sendable contents — all on the worker thread. @@ -290,6 +303,7 @@ def _on_worker() -> list[Content]: max_output_files=max_output_files, max_output_file_bytes=max_output_file_bytes, max_output_total_bytes=max_output_total_bytes, + include_detailed_errors=include_detailed_errors, ) if output_dir is not None and not any(item.type == "data" for item in contents): # Isolate any file that becomes visible after the finite discovery window @@ -313,7 +327,7 @@ def _on_worker() -> list[Content]: if contents is not None and not any(item.type == "error" for item in contents): contents.append(_output_cleanup_error_content(exc)) - return self._run_on_worker(_on_worker) + return self._run_on_worker(_on_worker, include_detailed_errors=include_detailed_errors) def is_reusable(self) -> bool: return self._reusable @@ -1209,6 +1223,7 @@ def _build_execution_contents( max_output_files: int, max_output_file_bytes: int, max_output_total_bytes: int, + include_detailed_errors: bool = False, ) -> list[Content]: success = bool(getattr(result, "success", False)) stdout = str(getattr(result, "stdout", "") or "").replace("\r\n", "\n") or None @@ -1246,7 +1261,7 @@ def _build_execution_contents( outputs.append(Content.from_text("Code executed successfully without output.")) return outputs - error_details = stderr or "Unknown sandbox error" + error_details = (stderr or "Unknown sandbox error") if include_detailed_errors else None outputs.append( Content.from_error( message="Execution error", @@ -1257,7 +1272,7 @@ def _build_execution_contents( return outputs -def _make_sandbox_callback(tool_obj: FunctionTool) -> Callable[..., Any]: +def _make_sandbox_callback(tool_obj: FunctionTool, *, include_detailed_errors: bool = False) -> Callable[..., Any]: sandbox_tool = copy(tool_obj) def _callback(**kwargs: Any) -> Any: @@ -1282,6 +1297,8 @@ def _run() -> None: worker.start() worker.join() if error_box: + if not include_detailed_errors and isinstance(error_box[0], Exception): + raise RuntimeError("Host tool execution failed.") from None raise error_box[0] # Return the raw value. The Hyperlight FFI marshals primitives (dict, list, # str, int, float, bool, None) natively into the guest, and falls back to @@ -1418,6 +1435,7 @@ def execute(self, *, config: _RunConfig, code: str) -> list[Content]: max_output_files=config.max_output_files, max_output_file_bytes=config.max_output_file_bytes, max_output_total_bytes=config.max_output_total_bytes, + include_detailed_errors=config.include_detailed_errors, ) finally: if not entry.worker.is_reusable(): @@ -1487,7 +1505,10 @@ def _create_sandbox() -> Any: def _configure_sandbox(*, sandbox: Any, expand_missing_scheme: bool) -> None: for tool_obj in config.tools: - sandbox.register_tool(tool_obj.name, _make_sandbox_callback(tool_obj)) + sandbox.register_tool( + tool_obj.name, + _make_sandbox_callback(tool_obj, include_detailed_errors=config.include_detailed_errors), + ) for allowed_domain in config.allowed_domains: for target in _allowed_domain_registration_targets( @@ -1518,7 +1539,7 @@ def _build_sandbox() -> tuple[Any, Any]: worker = _SandboxWorker() try: - worker.initialize(_build_sandbox) + worker.initialize(_build_sandbox, include_detailed_errors=config.include_detailed_errors) except BaseException: worker.dispose() raise @@ -1573,6 +1594,8 @@ class HyperlightExecuteCodeTool(FunctionTool): or ``"json"``, globally or mapped by exact, case-sensitive tool name. Missing names use compact format. Schemas that cannot be represented faithfully in compact form use JSON Schema. Mappings are copied, including entries for tools registered later. + include_detailed_errors: Expose host exception and execution diagnostics only for a trusted + channel. Defaults to ``False`` and is independent of outer-client error/logging settings. """ def __init__( @@ -1580,6 +1603,7 @@ def __init__( *, tools: FunctionTool | Callable[..., Any] | Sequence[FunctionTool | Callable[..., Any]] | None = None, tool_description_format: _ToolDescriptionFormat = "compact", + include_detailed_errors: bool = False, approval_mode: ApprovalMode | None = None, workspace_root: str | Path | None = None, file_mounts: FileMountInput | Sequence[FileMountInput] | None = None, @@ -1593,6 +1617,8 @@ def __init__( _registry: SandboxRuntime | None = None, _fides_warned_metadata: list[tuple[FunctionTool, dict[str, str]]] | None = None, ) -> None: + if not isinstance(include_detailed_errors, bool): + raise TypeError("include_detailed_errors must be a bool.") normalized_description_format = _normalize_tool_description_format(tool_description_format) max_output_files = _validate_positive_integer(name="max_output_files", value=max_output_files) max_output_file_bytes = _validate_positive_integer(name="max_output_file_bytes", value=max_output_file_bytes) @@ -1608,6 +1634,7 @@ def __init__( self._tool_description_format: _NormalizedToolDescriptionFormat = normalized_description_format self._registry = _registry or _SandboxRegistry() self._default_approval_mode: ApprovalMode = approval_mode or "never_require" + self._include_detailed_errors = include_detailed_errors self._workspace_root = _resolve_workspace_root(workspace_root) self._backend: str = backend self._module: str | None = module @@ -1785,6 +1812,7 @@ def create_run_tool(self) -> HyperlightExecuteCodeTool: tools=tools, tool_description_format=self._tool_description_format, approval_mode=self._default_approval_mode, + include_detailed_errors=self._include_detailed_errors, workspace_root=self._workspace_root, file_mounts=file_mounts or None, allowed_domains=allowed_domains or None, @@ -1805,6 +1833,7 @@ def build_serializable_state(self) -> dict[str, Any]: "module": config.module, "module_path": config.module_path, "approval_mode": config.approval_mode, + "include_detailed_errors": config.include_detailed_errors, "max_output_files": config.max_output_files, "max_output_file_bytes": config.max_output_file_bytes, "max_output_total_bytes": config.max_output_total_bytes, @@ -1852,6 +1881,7 @@ def _build_run_config(self) -> _RunConfig: max_output_files = self._max_output_files max_output_file_bytes = self._max_output_file_bytes max_output_total_bytes = self._max_output_total_bytes + include_detailed_errors = self._include_detailed_errors approval_mode = _resolve_execute_code_approval_mode( base_approval_mode=self._default_approval_mode, tools=managed_tools, @@ -1880,8 +1910,15 @@ def _build_run_config(self) -> _RunConfig: max_output_files=max_output_files, max_output_file_bytes=max_output_file_bytes, max_output_total_bytes=max_output_total_bytes, + include_detailed_errors=include_detailed_errors, ) async def _run_code(self, *, code: str) -> list[Content]: - config = self._build_run_config() - return await asyncio.to_thread(self._registry.execute, config=config, code=code) + include_detailed_errors = self._include_detailed_errors + try: + config = self._build_run_config() + return await asyncio.to_thread(self._registry.execute, config=config, code=code) + except Exception: + if include_detailed_errors: + raise + raise RuntimeError("Code execution failed.") from None diff --git a/python/packages/hyperlight/agent_framework_hyperlight/_provider.py b/python/packages/hyperlight/agent_framework_hyperlight/_provider.py index 119e7fbcdc..a72fd032f4 100644 --- a/python/packages/hyperlight/agent_framework_hyperlight/_provider.py +++ b/python/packages/hyperlight/agent_framework_hyperlight/_provider.py @@ -29,6 +29,8 @@ class HyperlightCodeActProvider(ContextProvider): ``"compact"`` (default) or ``"json"``, globally or mapped by exact, case-sensitive tool name. Missing names use compact format; rich schemas fall back to full JSON Schema. Mappings are copied, including entries for tools registered later. + include_detailed_errors: Expose host exception and execution diagnostics only for a trusted + channel. Defaults to ``False`` and is independent of outer-client error/logging settings. """ DEFAULT_SOURCE_ID = "hyperlight_codeact" @@ -39,6 +41,7 @@ def __init__( *, tools: FunctionTool | Callable[..., Any] | Sequence[FunctionTool | Callable[..., Any]] | None = None, tool_description_format: _ToolDescriptionFormat = "compact", + include_detailed_errors: bool = False, approval_mode: ApprovalMode | None = None, workspace_root: str | Path | None = None, file_mounts: FileMountInput | Sequence[FileMountInput] | None = None, @@ -55,6 +58,7 @@ def __init__( self._execute_code_tool = HyperlightExecuteCodeTool( tools=tools, tool_description_format=tool_description_format, + include_detailed_errors=include_detailed_errors, approval_mode=approval_mode, workspace_root=workspace_root, file_mounts=file_mounts, diff --git a/python/packages/hyperlight/tests/hyperlight/conftest.py b/python/packages/hyperlight/tests/hyperlight/conftest.py new file mode 100644 index 0000000000..23d4bfa851 --- /dev/null +++ b/python/packages/hyperlight/tests/hyperlight/conftest.py @@ -0,0 +1,84 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +import asyncio +import threading +from collections.abc import Callable +from types import SimpleNamespace +from typing import Any + +import pytest +from agent_framework import FunctionTool + + +@pytest.fixture +def host_error_detail() -> str: + return "synthetic backend diagnostic" + + +@pytest.fixture +def failing_host_tool(request: pytest.FixtureRequest, host_error_detail: str) -> FunctionTool: + def fail() -> str: + raise ValueError(host_error_detail) + + async def fail_async() -> str: + await asyncio.sleep(0) + raise ValueError(host_error_detail) + + return FunctionTool( + name="failing_host_tool", + description="Raise a synthetic diagnostic.", + func=fail_async if request.param else fail, + ) + + +@pytest.fixture +def cancelled_host_tool() -> FunctionTool: + async def cancel() -> str: + raise asyncio.CancelledError + + return FunctionTool(name="cancelled_host_tool", func=cancel) + + +@pytest.fixture +def lifecycle_sandbox_factory(host_error_detail: str) -> Callable[..., type[Any]]: + def factory(failure_stage: str, error_type: type[BaseException] = ValueError) -> type[Any]: + class LifecycleSandbox: + events: list[tuple[str, int]] = [] + + def __init__(self, **kwargs: Any) -> None: + self._owner_thread = threading.get_ident() + self._record("build") + + def _record(self, stage: str) -> None: + self.events.append((stage, threading.get_ident())) + if stage == failure_stage: + raise error_type(host_error_detail) + + def register_tool(self, name: str, callback: Any) -> None: + self._record("register_tool") + + def allow_domain(self, target: str, methods: Any = None) -> None: + self._record("allow_domain") + + def run(self, code: str) -> Any: + self._record("warmup" if code == "None" else "run") + return SimpleNamespace(success=True, stdout="ok\n", stderr="") + + def snapshot(self) -> object: + self._record("snapshot") + return object() + + def restore(self, snapshot: object) -> None: + self._record("restore") + + def close(self) -> None: + self._record("close") + + def __del__(self) -> None: + self.events.append(("drop", threading.get_ident())) + + return LifecycleSandbox + + return factory diff --git a/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py b/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py index 2e4cfcb186..9c024d1b7f 100644 --- a/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py +++ b/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py @@ -39,15 +39,14 @@ ResponseStream, tool, ) +from agent_framework_hyperlight import AllowedDomain, FileMount, HyperlightCodeActProvider, HyperlightExecuteCodeTool +from agent_framework_hyperlight import _execute_code_tool as execute_code_module +from agent_framework_hyperlight import _instructions as instructions_module from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from pytest import fixture -from agent_framework_hyperlight import AllowedDomain, FileMount, HyperlightCodeActProvider, HyperlightExecuteCodeTool -from agent_framework_hyperlight import _execute_code_tool as execute_code_module -from agent_framework_hyperlight import _instructions as instructions_module - def _hyperlight_integration_static_skip_reason() -> str | None: if sys.platform not in {"linux", "win32"}: @@ -1477,7 +1476,7 @@ def fail_post_cleanup(path: str | os.PathLike[str]) -> Any: recovered = await execute_code.invoke(arguments={"code": "fail"}) assert not any(item.type == "data" for item in recovered) - assert any(item.type == "error" and item.error_details == "sandbox boom" for item in recovered) + assert any(item.type == "error" and item.error_details is None for item in recovered) assert len(_FakeSandbox.instances) == 2 assert registry._entries == {} finally: @@ -1540,13 +1539,15 @@ def blocking_cleanup_temp_dirs(entry_to_clean: Any) -> None: _close_execute_code_registry(execute_code) +@pytest.mark.parametrize("include_detailed_errors", [False, True]) async def test_execute_code_tool_preserves_exception_when_post_cleanup_fails( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + include_detailed_errors: bool, ) -> None: _FakeSandbox.instances.clear() monkeypatch.setattr(execute_code_module, "_load_sandbox_class", lambda: _FakeSandbox) - execute_code = HyperlightExecuteCodeTool(workspace_root=tmp_path) + execute_code = HyperlightExecuteCodeTool(workspace_root=tmp_path, include_detailed_errors=include_detailed_errors) config = execute_code._build_run_config() registry = cast(Any, execute_code._registry) output_root = Path(registry._get_or_create_entry(config).output_dir.name) @@ -1569,7 +1570,8 @@ def fail_build_contents(**kwargs: Any) -> list[Content]: monkeypatch.setattr(execute_code_module, "_build_execution_contents", fail_build_contents) try: - with pytest.raises(RuntimeError, match="primary build failure"): + expected = "primary build failure" if include_detailed_errors else "Code execution failed." + with pytest.raises(RuntimeError, match=expected): await execute_code.invoke(arguments={"code": "create-output"}) assert registry._entries == {} finally: @@ -2438,22 +2440,29 @@ def blocking_cleanup_temp_dirs(entry_to_clean: Any) -> None: second_result = await execute_code.invoke(arguments={"code": "fail"}) assert not any(item.type == "data" for item in second_result) - assert any(item.type == "error" and item.error_details == "sandbox boom" for item in second_result) + assert any(item.type == "error" and item.error_details is None for item in second_result) assert len(_FakeSandbox.instances) == 2 finally: release_cleanup.set() _close_execute_code_registry(execute_code) -async def test_execute_code_tool_failure_returns_error_content(monkeypatch: pytest.MonkeyPatch) -> None: +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_execute_code_tool_failure_returns_error_content( + monkeypatch: pytest.MonkeyPatch, include_detailed_errors: bool +) -> None: _FakeSandbox.instances.clear() monkeypatch.setattr(execute_code_module, "_load_sandbox_class", lambda: _FakeSandbox) - execute_code = HyperlightExecuteCodeTool() - result = await execute_code.invoke(arguments={"code": "fail"}) + execute_code = HyperlightExecuteCodeTool(include_detailed_errors=include_detailed_errors) + try: + result = await execute_code.invoke(arguments={"code": "fail"}) - assert result[0].type == "error" - assert result[0].error_details == "sandbox boom" + assert result[0].type == "error" + assert result[0].error_details == ("sandbox boom" if include_detailed_errors else None) + assert ("sandbox boom" in json.dumps([item.to_dict() for item in result])) is include_detailed_errors + finally: + _close_execute_code_registry(execute_code) async def test_execute_code_tool_retries_allowed_domains_with_urls_when_backend_rejects_host_targets( @@ -2829,7 +2838,7 @@ async def test_execute_code_tool_rejects_sparse_file_with_real_sandbox(tmp_path: errors = [item for item in contents if item.type == "error"] assert len(errors) == 1 assert errors[0].message == "Execution error" - assert errors[0].error_details + assert errors[0].message == "Execution error" @pytest.mark.flaky @@ -3258,11 +3267,141 @@ def boom(x: int) -> int: """Always fails.""" raise RuntimeError("nope") - callback = execute_code_module._make_sandbox_callback(boom) + callback = execute_code_module._make_sandbox_callback(boom, include_detailed_errors=True) with pytest.raises(RuntimeError, match="nope"): callback(x=1) +@pytest.mark.parametrize("failing_host_tool", [False, True], indirect=True) +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_sandbox_callback_redacts_errors_unless_explicitly_enabled( + failing_host_tool: FunctionTool, host_error_detail: str, include_detailed_errors: bool +) -> None: + callback = execute_code_module._make_sandbox_callback( + failing_host_tool, include_detailed_errors=include_detailed_errors + ) + + with pytest.raises(ValueError if include_detailed_errors else RuntimeError) as error: + callback() + + assert (host_error_detail in str(error.value)) is include_detailed_errors + if not include_detailed_errors: + assert str(error.value) == "Host tool execution failed." + assert error.value.__suppress_context__ + + +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_error_policy_preserves_callback_cancellation( + cancelled_host_tool: FunctionTool, include_detailed_errors: bool +) -> None: + callback = execute_code_module._make_sandbox_callback( + cancelled_host_tool, include_detailed_errors=include_detailed_errors + ) + + with pytest.raises(asyncio.CancelledError): + callback() + + +@pytest.mark.parametrize( + "failure_stage", ["build", "register_tool", "allow_domain", "warmup", "snapshot", "restore", "run"] +) +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_lifecycle_failures_obey_error_policy_and_retire_worker( + monkeypatch: pytest.MonkeyPatch, + lifecycle_sandbox_factory: Callable[..., type[Any]], + host_error_detail: str, + failure_stage: str, + include_detailed_errors: bool, +) -> None: + sandbox_cls = lifecycle_sandbox_factory(failure_stage) + monkeypatch.setattr(execute_code_module, "_load_sandbox_class", lambda: sandbox_cls) + execute_code = HyperlightExecuteCodeTool( + tools=[lambda: "ok"], allowed_domains=["example.com"], include_detailed_errors=include_detailed_errors + ) + registry = cast(Any, execute_code._registry) + try: + with pytest.raises(ValueError if include_detailed_errors else RuntimeError) as error: + await execute_code.invoke(arguments={"code": "print('ok')"}) + assert (host_error_detail in str(error.value)) is include_detailed_errors + if not include_detailed_errors: + assert str(error.value) == "Code execution failed." + assert registry._entries == {} + finally: + _close_execute_code_registry(execute_code) + gc.collect() + owner_threads = {ident for _, ident in sandbox_cls.events} + assert len(owner_threads) == 1 + assert threading.get_ident() not in owner_threads + + +@pytest.mark.parametrize("signal_type", [asyncio.CancelledError, SystemExit, KeyboardInterrupt]) +@pytest.mark.parametrize("failure_stage", ["warmup", "restore", "run"]) +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_lifecycle_error_policy_preserves_system_signals( + monkeypatch: pytest.MonkeyPatch, + lifecycle_sandbox_factory: Callable[..., type[Any]], + host_error_detail: str, + signal_type: type[BaseException], + failure_stage: str, + include_detailed_errors: bool, +) -> None: + sandbox_cls = lifecycle_sandbox_factory(failure_stage, signal_type) + monkeypatch.setattr(execute_code_module, "_load_sandbox_class", lambda: sandbox_cls) + execute_code = HyperlightExecuteCodeTool(include_detailed_errors=include_detailed_errors) + try: + with pytest.raises(signal_type, match=host_error_detail): + await execute_code.invoke(arguments={"code": "print('ok')"}) + finally: + _close_execute_code_registry(execute_code) + + +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +def test_error_policy_is_snapshotted_and_part_of_the_sandbox_cache_key(include_detailed_errors: bool) -> None: + registry = _FakeRuntime() + codeact = HyperlightExecuteCodeTool(include_detailed_errors=include_detailed_errors, _registry=registry) + provider = HyperlightCodeActProvider(include_detailed_errors=include_detailed_errors, _registry=registry) + opposite = HyperlightExecuteCodeTool(include_detailed_errors=not include_detailed_errors, _registry=registry) + + assert codeact.create_run_tool().build_serializable_state()["include_detailed_errors"] is include_detailed_errors + assert ( + provider._execute_code_tool.create_run_tool().build_serializable_state()["include_detailed_errors"] + is include_detailed_errors + ) + assert codeact._build_run_config().cache_key() != opposite._build_run_config().cache_key() + assert set(codeact.parameters()["properties"]) == {"code"} + + +@pytest.mark.parametrize("constructor", [HyperlightExecuteCodeTool, HyperlightCodeActProvider]) +@pytest.mark.parametrize("value", ["false", 1, None]) +def test_detailed_error_option_rejects_non_boolean_values(constructor: Any, value: Any) -> None: + with pytest.raises(TypeError, match="include_detailed_errors must be a bool"): + constructor(include_detailed_errors=value) + + +@pytest.mark.integration +@skip_if_hyperlight_integration_tests_disabled +@pytest.mark.parametrize("failing_host_tool", [False, True], indirect=True) +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_host_error_policy_with_real_sandbox( + failing_host_tool: FunctionTool, host_error_detail: str, include_detailed_errors: bool +) -> None: + _skip_if_hyperlight_integration_runtime_disabled() + execute_code = HyperlightExecuteCodeTool(tools=[failing_host_tool], include_detailed_errors=include_detailed_errors) + try: + result = await execute_code.invoke( + arguments={ + "code": "try:\n call_tool('failing_host_tool')\nexcept Exception as exc:\n print(str(exc))" + }, + skip_parsing=True, + ) + serialized = json.dumps([content.to_dict() for content in result]) + assert (host_error_detail in serialized) is include_detailed_errors + if not include_detailed_errors: + assert "Host tool execution failed." in serialized + finally: + _close_execute_code_registry(execute_code) + + class _OwnerThreadTrackedResult: """Fake sandbox.run() return value that mirrors a PyO3 ``unsendable`` object's Drop. diff --git a/python/packages/monty/AGENTS.md b/python/packages/monty/AGENTS.md index e366b78750..390b7de63b 100644 --- a/python/packages/monty/AGENTS.md +++ b/python/packages/monty/AGENTS.md @@ -36,6 +36,10 @@ from agent_framework.monty import ( - `file_mounts` — sequence of `FileMountInput` (str shorthand, `(host_path, mount_path)` tuple, or `FileMount`) - `resource_limits` — Monty `ResourceLimits` TypedDict +- `include_detailed_errors` — keyword-only Boolean, default `False`. Host failures become generic + `RuntimeError`s and execution errors omit raw details. Trusted opt-in preserves original diagnostics. + The flag is not a code input, is independent of outer-client error/logging settings, and is copied + into run snapshots and serializable state. - `tool_description_format` — keyword-only `Literal["compact", "json"] | Mapping[str, Literal["compact", "json"]] = "compact"`. Selects parameter documentation in both the description and instructions. diff --git a/python/packages/monty/README.md b/python/packages/monty/README.md index 6b7603acde..b9ec3e9e82 100644 --- a/python/packages/monty/README.md +++ b/python/packages/monty/README.md @@ -22,6 +22,24 @@ Monty-backed CodeAct integrations for Microsoft Agent Framework. > enforce authorization and destination restrictions inside host functions. > See the [FIDES guide](../../samples/02-agents/security/FIDES_DEVELOPER_GUIDE.md). +## Error diagnostics + +Host tool failures raise a generic `RuntimeError("Host tool execution failed.")` +inside generated code by default. Execution error contents omit raw diagnostic +details. This applies to both direct tool calls and `call_tool(...)`, including +failures that generated code catches and prints or returns. + +Both `MontyCodeActProvider` and `MontyExecuteCodeTool` accept the host-side option +`include_detailed_errors=True` to retain original host exception types/messages +and execution diagnostics. Enable it only for a trusted channel: exception +messages can contain credentials or other sensitive data. This option is not a +parameter of `execute_code` and is independent of the outer chat client's error +policy and sensitive-data logging. Run snapshots and serialized state retain it. + +Code that catches a specific host exception type, such as `ValueError`, must +instead handle the generic `RuntimeError` in the default mode. Failure signaling +and successful tool results are unchanged. + ## Installation ```bash diff --git a/python/packages/monty/agent_framework_monty/_execute_code_tool.py b/python/packages/monty/agent_framework_monty/_execute_code_tool.py index 38fe9db8d0..fc2cbb6aa5 100644 --- a/python/packages/monty/agent_framework_monty/_execute_code_tool.py +++ b/python/packages/monty/agent_framework_monty/_execute_code_tool.py @@ -260,6 +260,11 @@ class MontyExecuteCodeTool(FunctionTool): All mutators (``add_tools``, ``add_file_mounts`` etc.) must be called from the same task/thread that owns the tool. Monty itself runs on the event loop, so no internal locking is needed. + + Keyword Args: + include_detailed_errors: Include host exception types/messages and execution error details. + Defaults to ``False``. Enable only for a trusted channel; diagnostics can contain secrets. + This setting is independent of the outer client's error and sensitive-logging settings. """ def __init__( @@ -271,8 +276,11 @@ def __init__( file_mounts: FileMountInput | Sequence[FileMountInput] | None = None, resource_limits: dict[str, Any] | None = None, tool_description_format: _ToolDescriptionFormat = "compact", + include_detailed_errors: bool = False, _fides_warned_metadata: list[tuple[FunctionTool, dict[str, str]]] | None = None, ) -> None: + if not isinstance(include_detailed_errors, bool): + raise TypeError("include_detailed_errors must be a bool.") super().__init__( name=EXECUTE_CODE_TOOL_NAME, description=EXECUTE_CODE_TOOL_DESCRIPTION, @@ -281,6 +289,7 @@ def __init__( input_model=EXECUTE_CODE_INPUT_SCHEMA, ) self._default_approval_mode: ApprovalMode = approval_mode or "never_require" + self._include_detailed_errors = include_detailed_errors self._tool_description_format: _NormalizedToolDescriptionFormat = _normalize_tool_description_format( tool_description_format ) @@ -413,6 +422,7 @@ def create_run_tool(self) -> MontyExecuteCodeTool: file_mounts=list(self._file_mounts.values()) or None, resource_limits=self._resource_limits, tool_description_format=self._tool_description_format, + include_detailed_errors=self._include_detailed_errors, ) def build_serializable_state(self) -> dict[str, Any]: @@ -425,6 +435,7 @@ def build_serializable_state(self) -> dict[str, Any]: return { "runtime": "monty", "approval_mode": approval_mode, + "include_detailed_errors": self._include_detailed_errors, "tool_names": [tool_obj.name for tool_obj in self._managed_tools], "tool_description_format": ( dict(self._tool_description_format) @@ -489,6 +500,7 @@ def _effective_mounts(self) -> list[FileMount]: return mounts async def _run_code(self, *, code: str) -> list[Content]: + include_detailed_errors = self._include_detailed_errors tools = list(self._managed_tools) mounts = self._effective_mounts() @@ -505,6 +517,7 @@ async def _run_code(self, *, code: str) -> list[Content]: type_stubs=type_stubs, mounts=[_to_monty_mount(mount) for mount in mounts] or None, resource_limits=self._resource_limits, + include_detailed_errors=include_detailed_errors, ) try: @@ -513,7 +526,7 @@ async def _run_code(self, *, code: str) -> list[Content]: return [ Content.from_error( message="Execution error", - error_details=f"{type(exc).__name__}: {exc}", + error_details=f"{type(exc).__name__}: {exc}" if include_detailed_errors else None, ), ] diff --git a/python/packages/monty/agent_framework_monty/_monty_bridge.py b/python/packages/monty/agent_framework_monty/_monty_bridge.py index a7f9291dd9..008d3ed177 100644 --- a/python/packages/monty/agent_framework_monty/_monty_bridge.py +++ b/python/packages/monty/agent_framework_monty/_monty_bridge.py @@ -39,8 +39,10 @@ def _ensure_json_value(value: Any) -> Any: raise ValueError(f"Value of type {type(value).__name__} is not JSON-safe.") -def _external_error(exc: Exception) -> dict[str, str]: - return {"exc_type": type(exc).__name__, "message": str(exc)} +def _external_error(exc: Exception, *, include_detailed_errors: bool = False) -> dict[str, str]: + if include_detailed_errors: + return {"exc_type": type(exc).__name__, "message": str(exc)} + return {"exc_type": "RuntimeError", "message": "Host tool execution failed."} def _parse_call_tool(args: tuple[Any, ...], kwargs: dict[str, Any]) -> tuple[str, dict[str, Any]]: @@ -210,11 +212,13 @@ def __init__( type_stubs: str | None = None, mounts: Sequence[Any] | None = None, resource_limits: dict[str, Any] | None = None, + include_detailed_errors: bool = False, ) -> None: self.tool_map: dict[str, Callable[..., Any]] = dict(tool_map) self.type_stubs: str | None = type_stubs self._mounts = tuple(mounts) if mounts else () self._resource_limits = resource_limits + self._include_detailed_errors = include_detailed_errors self._pending_calls: dict[int, tuple[str, dict[str, Any]]] = {} async def run(self, code: str) -> dict[str, Any]: @@ -301,7 +305,7 @@ def _schedule_call_tool(self, snapshot: Any) -> Any: raise ValueError(f"Tool {name!r} is not registered. Available tools: {allowed}") self._pending_calls[int(snapshot.call_id)] = (name, kwargs) except Exception as exc: - return snapshot.resume(_external_error(exc)) + return snapshot.resume(_external_error(exc, include_detailed_errors=True)) return snapshot.resume({"future": ...}) async def _handle_future(self, snapshot: Any) -> Any: @@ -331,4 +335,4 @@ async def _invoke_tool(self, cid: int, name: str, kwargs: dict[str, Any]) -> tup result = await self.tool_map[name](**kwargs) return cid, {"return_value": _ensure_json_value(result)} except Exception as exc: - return cid, _external_error(exc) + return cid, _external_error(exc, include_detailed_errors=self._include_detailed_errors) diff --git a/python/packages/monty/agent_framework_monty/_provider.py b/python/packages/monty/agent_framework_monty/_provider.py index 81192a038e..4b4a79fb90 100644 --- a/python/packages/monty/agent_framework_monty/_provider.py +++ b/python/packages/monty/agent_framework_monty/_provider.py @@ -31,6 +31,10 @@ class MontyCodeActProvider(ContextProvider): names to either format. Missing names use compact; rich schemas fall back to full JSON Schema. Mappings are copied on construction and for each run, retaining entries for tools registered later. + + Keyword Args: + include_detailed_errors: Expose host exception and execution diagnostics only for a trusted + channel. Defaults to ``False`` and is independent of outer-client error/logging settings. """ DEFAULT_SOURCE_ID = "monty_codeact" @@ -45,6 +49,7 @@ def __init__( file_mounts: FileMountInput | Sequence[FileMountInput] | None = None, resource_limits: dict[str, Any] | None = None, tool_description_format: _ToolDescriptionFormat = "compact", + include_detailed_errors: bool = False, ) -> None: super().__init__(source_id) self._execute_code_tool = MontyExecuteCodeTool( @@ -54,6 +59,7 @@ def __init__( file_mounts=file_mounts, resource_limits=resource_limits, tool_description_format=tool_description_format, + include_detailed_errors=include_detailed_errors, ) def add_tools( diff --git a/python/packages/monty/tests/monty/conftest.py b/python/packages/monty/tests/monty/conftest.py new file mode 100644 index 0000000000..d94bdb7558 --- /dev/null +++ b/python/packages/monty/tests/monty/conftest.py @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +import asyncio + +import pytest +from agent_framework import FunctionTool + + +@pytest.fixture +def host_error_detail() -> str: + return "synthetic backend diagnostic" + + +@pytest.fixture +def failing_host_tool(request: pytest.FixtureRequest, host_error_detail: str) -> FunctionTool: + def fail() -> str: + raise ValueError(host_error_detail) + + async def fail_async() -> str: + await asyncio.sleep(0) + raise ValueError(host_error_detail) + + return FunctionTool( + name="failing_host_tool", + description="Raise a synthetic diagnostic.", + func=fail_async if request.param else fail, + ) + + +@pytest.fixture +def cancelled_host_tool() -> FunctionTool: + async def cancel() -> str: + raise asyncio.CancelledError + + return FunctionTool(name="cancelled_host_tool", func=cancel) diff --git a/python/packages/monty/tests/monty/test_monty_codeact.py b/python/packages/monty/tests/monty/test_monty_codeact.py index 4f2cd10e1f..d3e81887ed 100644 --- a/python/packages/monty/tests/monty/test_monty_codeact.py +++ b/python/packages/monty/tests/monty/test_monty_codeact.py @@ -28,7 +28,6 @@ from agent_framework import Content, FunctionTool, Message, tool from agent_framework._sessions import SessionContext from agent_framework.exceptions import ToolException - from agent_framework_monty import MontyCodeActProvider, MontyExecuteCodeTool from agent_framework_monty import _execute_code_tool as execute_code_module from agent_framework_monty import _instructions as instructions_module @@ -946,7 +945,10 @@ async def test_run_code_call_tool_with_unregistered_name_returns_error() -> None ) -async def test_run_code_returns_error_content_on_runtime_failure(monkeypatch: pytest.MonkeyPatch) -> None: +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_run_code_returns_error_content_on_runtime_failure( + monkeypatch: pytest.MonkeyPatch, include_detailed_errors: bool +) -> None: class _BoomBridge: def __init__(self, tool_map: Any, **_: Any) -> None: pass @@ -956,11 +958,32 @@ async def run(self, code: str) -> dict[str, Any]: monkeypatch.setattr(execute_code_module, "InlineCodeBridge", _BoomBridge) - monty_tool = MontyExecuteCodeTool() + monty_tool = MontyExecuteCodeTool(include_detailed_errors=include_detailed_errors) result = await monty_tool._run_code(code="x = 1") assert len(result) == 1 assert result[0].type == "error" - assert "boom" in (result[0].error_details or "") + assert ("boom" in (result[0].error_details or "")) is include_detailed_errors + assert ("boom" in json.dumps([item.to_dict() for item in result])) is include_detailed_errors + + +@pytest.mark.parametrize("constructor", [MontyExecuteCodeTool, MontyCodeActProvider]) +@pytest.mark.parametrize("value", ["false", 1, None]) +def test_detailed_error_option_rejects_non_boolean_values(constructor: Any, value: Any) -> None: + with pytest.raises(TypeError, match="include_detailed_errors must be a bool"): + constructor(include_detailed_errors=value) + + +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +def test_detailed_error_policy_is_snapshotted_and_not_a_code_parameter(include_detailed_errors: bool) -> None: + codeact = MontyExecuteCodeTool(include_detailed_errors=include_detailed_errors) + provider = MontyCodeActProvider(include_detailed_errors=include_detailed_errors) + + assert codeact.create_run_tool().build_serializable_state()["include_detailed_errors"] is include_detailed_errors + assert ( + provider._execute_code_tool.create_run_tool().build_serializable_state()["include_detailed_errors"] + is include_detailed_errors + ) + assert set(codeact.parameters()["properties"]) == {"code"} def test_build_execution_contents_handles_truncation_and_non_json_output() -> None: @@ -1836,7 +1859,10 @@ async def limited() -> int: results = await asyncio.gather(*(bridge._invoke_tool(index, "limited", {}) for index, bridge in enumerate(bridges))) assert sum(payload == {"return_value": 42} for _, payload in results) == 1 - assert sum(payload.get("exc_type") == "ToolException" for _, payload in results) == 7 + assert ( + sum(payload == {"exc_type": "RuntimeError", "message": "Host tool execution failed."} for _, payload in results) + == 7 + ) assert limited.invocation_count == 1 @@ -1893,8 +1919,7 @@ def limited() -> int: if run_index == 0: assert payload == {"return_value": 42} else: - assert payload["exc_type"] == "ToolException" - assert "maximum invocation limit" in payload["message"] + assert payload == {"exc_type": "RuntimeError", "message": "Host tool execution failed."} assert limited.invocation_count == 1 diff --git a/python/packages/monty/tests/monty/test_monty_codeact_integration.py b/python/packages/monty/tests/monty/test_monty_codeact_integration.py index c52fb0e00e..4f851c5c32 100644 --- a/python/packages/monty/tests/monty/test_monty_codeact_integration.py +++ b/python/packages/monty/tests/monty/test_monty_codeact_integration.py @@ -12,6 +12,7 @@ import asyncio import importlib.util +import json import time from typing import Annotated, Any from unittest.mock import MagicMock @@ -19,7 +20,6 @@ import pytest from agent_framework import Agent, Content, FunctionTool, Message, tool from agent_framework._sessions import SessionContext - from agent_framework_monty import MontyCodeActProvider, MontyExecuteCodeTool @@ -138,6 +138,62 @@ async def test_async_host_tool_is_awaited() -> None: assert any("ping" in text for text in texts) +@pytest.mark.parametrize("failing_host_tool", [False, True], indirect=True) +@pytest.mark.parametrize("call", ["await failing_host_tool()", "await call_tool('failing_host_tool')"]) +@pytest.mark.parametrize("handling", ["print", "value", "uncaught"]) +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_host_error_details_require_trusted_opt_in( + failing_host_tool: FunctionTool, + host_error_detail: str, + call: str, + handling: str, + include_detailed_errors: bool, +) -> None: + if handling == "print": + code = f"try:\n {call}\nexcept Exception as exc:\n print(str(exc))" + elif handling == "value": + code = f"message = ''\ntry:\n {call}\nexcept Exception as exc:\n message = str(exc)\nmessage" + else: + code = call + execute_code = MontyExecuteCodeTool(tools=[failing_host_tool], include_detailed_errors=include_detailed_errors) + + contents = await execute_code._run_code(code=code) + + serialized = json.dumps([content.to_dict() for content in contents]) + assert (host_error_detail in serialized) is include_detailed_errors + if handling == "uncaught": + assert contents[0].type == "error" + elif not include_detailed_errors: + assert "Host tool execution failed." in serialized + + +@pytest.mark.parametrize("failing_host_tool", [True], indirect=True) +async def test_concurrent_executions_keep_error_policies_separate( + failing_host_tool: FunctionTool, host_error_detail: str +) -> None: + redacted = MontyExecuteCodeTool(tools=[failing_host_tool]) + detailed = MontyExecuteCodeTool(tools=[failing_host_tool], include_detailed_errors=True) + code = "try:\n await failing_host_tool()\nexcept Exception as exc:\n print(str(exc))" + + redacted_result, detailed_result = await asyncio.gather( + redacted._run_code(code=code), detailed._run_code(code=code) + ) + + assert host_error_detail not in "\n".join(_text_outputs(redacted_result)) + assert host_error_detail in "\n".join(_text_outputs(detailed_result)) + assert failing_host_tool.invocation_count == 2 + + +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_error_policy_preserves_host_cancellation( + cancelled_host_tool: FunctionTool, include_detailed_errors: bool +) -> None: + execute_code = MontyExecuteCodeTool(tools=[cancelled_host_tool], include_detailed_errors=include_detailed_errors) + + with pytest.raises(asyncio.CancelledError): + await execute_code._run_code(code="await cancelled_host_tool()") + + @pytest.mark.parametrize("is_async", [False, True]) @pytest.mark.parametrize("call", ["await limited()", "await call_tool('limited')"]) async def test_host_tool_invocation_limit_across_code_executions(is_async: bool, call: str) -> None: diff --git a/skills/agent-framework-builder-python/references/integrations.md b/skills/agent-framework-builder-python/references/integrations.md index 415d4b3336..6690a7d41a 100644 --- a/skills/agent-framework-builder-python/references/integrations.md +++ b/skills/agent-framework-builder-python/references/integrations.md @@ -46,6 +46,11 @@ application owns authorization, durable snapshots, scaling, and rendering policy ## Code execution +Monty and Hyperlight redact host tool exception diagnostics by default. Enable +`include_detailed_errors=True` only for a trusted channel. This host-side setting +is separate from the outer chat client's error policy and sensitive-data logging; +outer redaction cannot repair a diagnostic already caught and returned by generated code. + For Monty, distinguish interpreter `resource_limits` from host-side attachment capture. Capture has per-call file-count and raw-byte limits shared across read-write mounts. Omitted attachments do not undo filesystem writes. Check the [Monty file-mount and resource-limit guidance](https://github.com/microsoft/agent-framework/blob/main/python/packages/monty/README.md#file-mounts-and-resource-limits)