From 858c047f5ade7091688a4099a4c158945e01a11c Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 6 Nov 2026 10:33:01 -0500 Subject: [PATCH] fix(hyperlight): interrupt Python guest execution and enforce time limits Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3d6807e7-70a6-4c43-8dc1-7bab20376cd4 --- python/packages/hyperlight/README.md | 35 ++++ .../_execute_code_tool.py | 149 ++++++++++++++++-- .../agent_framework_hyperlight/_provider.py | 5 + .../hyperlight/tests/hyperlight/conftest.py | 64 +++++++- .../hyperlight/test_execution_control.py | 67 ++++++++ .../tests/hyperlight/test_execution_limits.py | 144 +++++++++++++++++ .../hyperlight/test_hyperlight_codeact.py | 32 +++- .../references/integrations.md | 8 + 8 files changed, 491 insertions(+), 13 deletions(-) create mode 100644 python/packages/hyperlight/tests/hyperlight/test_execution_control.py create mode 100644 python/packages/hyperlight/tests/hyperlight/test_execution_limits.py diff --git a/python/packages/hyperlight/README.md b/python/packages/hyperlight/README.md index 13abf69f832..216de3d19ee 100644 --- a/python/packages/hyperlight/README.md +++ b/python/packages/hyperlight/README.md @@ -38,6 +38,41 @@ Initialization, snapshot restore, and execution failures follow the same policy. Failed sandbox entries are retired; cancellation and system-exit signals still propagate without becoming ordinary execution errors. +## Execution limits + +Guest execution has a finite 30-second budget by default. Both +`HyperlightCodeActProvider` and `HyperlightExecuteCodeTool` accept the host-side +`execution_timeout_seconds` option, which must be a positive finite number. +Applications that intentionally run longer must configure a larger budget. +The option is captured in provider run snapshots and serialized state; it is +not a model-controlled `execute_code` argument. + +Reaching the budget returns an error content with +`Code execution exceeded its time limit.` Caller task cancellation interrupts +guest execution and propagates `asyncio.CancelledError`. Cancellation waits for +native execution and its interruption thread to finish, even if the caller +cancels again. Queued cancellation does not interrupt another invocation. +The next invocation restores the warm snapshot before reusing the sandbox. + +The budget starts on the sandbox's owning worker, not while waiting for that +worker. Initialization and warm-up share the invocation's budget, but runtime +initialization and host tool callbacks cannot be interrupted. Host tools must +enforce their own cancellation and time limits. Completed results and unrelated +execution failures are not replaced with a timeout that arrives late. + +This behavior requires an SDK with the cross-thread `Sandbox.interrupt_handle()` +API from [hyperlight-dev/hyperlight-sandbox#275](https://github.com/hyperlight-dev/hyperlight-sandbox/pull/275). +The currently declared released dependency does not yet include this API; +a compatible SDK release and dependency-floor update are required before shipping. + +To run the native regressions on a KVM host with the linked SDK and both +backends installed: + +```bash +HYPERLIGHT_RUN_GUEST_TESTS=true uv run --directory python/packages/hyperlight pytest \ + tests/hyperlight/test_execution_limits.py -m integration +``` + ## Installation ```bash diff --git a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py index 645e0f2de45..673f71a3692 100644 --- a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py +++ b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py @@ -5,6 +5,7 @@ import asyncio import contextvars import logging +import math import mimetypes import os import shutil @@ -57,6 +58,10 @@ OUTPUT_TRAVERSAL_ENTRIES_PER_FILE = 10 OUTPUT_TRAVERSAL_MAX_ENTRIES = 10_000 OUTPUT_TRAVERSAL_MAX_DEPTH = 32 +DEFAULT_EXECUTION_TIMEOUT_SECONDS = 30.0 +_execution_cancellation: contextvars.ContextVar[threading.Event | None] = contextvars.ContextVar( + "hyperlight_execution_cancellation", default=None +) EXECUTE_CODE_INPUT_SCHEMA: dict[str, Any] = { "type": "object", @@ -80,6 +85,68 @@ class _OutputCleanupError(RuntimeError): """Raised when sandbox output cannot be safely cleaned within configured bounds.""" +class _GuestExecutionTimeout(TimeoutError): + """Raised when the invocation's guest execution budget expires.""" + + +class _ExecutionControl: + def __init__(self, timeout_seconds: float) -> None: + self._timeout_seconds = timeout_seconds + self._deadline: float | None = None + self._cancelled = _execution_cancellation.get() or threading.Event() + + def start(self) -> float: + if self._deadline is None: + self._deadline = time.monotonic() + self._timeout_seconds + self.check() + return self._deadline + + def check(self) -> None: + if self._cancelled.is_set(): + raise asyncio.CancelledError + if self._deadline is not None and time.monotonic() >= self._deadline: + raise _GuestExecutionTimeout("Code execution exceeded its time limit.") + + def run(self, sandbox: Any, code: str) -> Any: + deadline = self.start() + handle = sandbox.interrupt_handle() + self.check() + completed = threading.Event() + timed_out = threading.Event() + interruption_errors: list[str] = [] + + def _interrupt() -> None: + try: + while not completed.is_set(): + if time.monotonic() >= deadline: + timed_out.set() + if (self._cancelled.is_set() or timed_out.is_set()) and handle.kill(): + return + completed.wait(0.01) + except Exception as exc: + interruption_errors.append(str(exc)) + + # Only the independently owned handle crosses threads, never the sandbox. + interruption = threading.Thread(target=_interrupt, name="hl-interrupt") + interruption.start() + try: + self.check() + try: + result = sandbox.run(code=code) + except InterruptedError: + if self._cancelled.is_set(): + raise asyncio.CancelledError from None + if timed_out.is_set(): + raise _GuestExecutionTimeout("Code execution exceeded its time limit.") from None + raise + finally: + completed.set() + interruption.join() + if interruption_errors: + raise RuntimeError("Could not interrupt Hyperlight guest execution.") + return result + + def _output_cleanup_error_content(error: _OutputCleanupError) -> Content: return Content.from_error(message="Execution error", error_details=str(error)) @@ -113,6 +180,7 @@ class _RunConfig: max_output_file_bytes: int = DEFAULT_MAX_OUTPUT_FILE_BYTES max_output_total_bytes: int = DEFAULT_MAX_OUTPUT_TOTAL_BYTES include_detailed_errors: bool = False + execution_timeout_seconds: float = DEFAULT_EXECUTION_TIMEOUT_SECONDS @property def mounted_paths(self) -> tuple[str, ...]: @@ -203,7 +271,8 @@ def _wrapped() -> tuple[bool, Any]: try: return True, fn() except BaseException as exc: - self._reusable = False + if not isinstance(exc, (asyncio.CancelledError, _GuestExecutionTimeout)): + self._reusable = False exc_type: type[BaseException] exc_args: tuple[str, ...] # Capture args (usually (message,)) so the re-raised exception keeps the @@ -213,7 +282,11 @@ def _wrapped() -> tuple[bool, Any]: # unsendable object in args, forwarding it as-is would re-introduce the # same cross-thread Drop hazard the traceback nulling avoids. Strings # are always sendable. Fall back to the str() form if args is empty. - if isinstance(exc, Exception) and not include_detailed_errors: + if ( + isinstance(exc, Exception) + and not include_detailed_errors + and not isinstance(exc, _GuestExecutionTimeout) + ): exc_type = RuntimeError exc_args = ("Code execution failed.",) else: @@ -267,6 +340,7 @@ def execute( max_output_files: int, max_output_file_bytes: int, max_output_total_bytes: int, + control: _ExecutionControl, include_detailed_errors: bool = False, ) -> list[Content]: """Restore + run + build sendable contents — all on the worker thread. @@ -277,6 +351,7 @@ def execute( cleanup_max_entries = _output_traversal_entry_limit(max_output_files) def _on_worker() -> list[Content]: + control.start() if not self._reusable: return [_output_cleanup_error_content(_OutputCleanupError("Could not clear sandbox output safely."))] @@ -295,7 +370,7 @@ def _on_worker() -> list[Content]: contents: list[Content] | None = None try: - result = sandbox.run(code=code) + result = control.run(sandbox, code) try: contents = build_contents( result=result, @@ -1426,8 +1501,10 @@ def execute(self, *, config: _RunConfig, code: str) -> list[Content]: Entries whose output cannot be cleaned safely are evicted before another invocation. """ cache_key = config.cache_key() - entry = self._get_or_create_entry(config) + control = _ExecutionControl(config.execution_timeout_seconds) + entry: _SandboxEntry | None = None try: + entry = self._get_or_create_entry(config, control=control) return entry.worker.execute( code=code, output_dir=entry.output_dir, @@ -1435,10 +1512,15 @@ def execute(self, *, config: _RunConfig, code: str) -> list[Content]: max_output_files=config.max_output_files, max_output_file_bytes=config.max_output_file_bytes, max_output_total_bytes=config.max_output_total_bytes, + control=control, include_detailed_errors=config.include_detailed_errors, ) + except _GuestExecutionTimeout: + return [ + Content.from_error(message="Execution error", error_details="Code execution exceeded its time limit.") + ] finally: - if not entry.worker.is_reusable(): + if entry is not None and not entry.worker.is_reusable(): self._discard_entry(cache_key=cache_key, entry=entry) def _discard_entry(self, *, cache_key: tuple[Any, ...], entry: _SandboxEntry) -> None: @@ -1449,12 +1531,14 @@ def _discard_entry(self, *, cache_key: tuple[Any, ...], entry: _SandboxEntry) -> entry.worker.dispose() self._cleanup_executor.submit(entry.cleanup_temp_dirs) - def _get_or_create_entry(self, config: _RunConfig) -> _SandboxEntry: + def _get_or_create_entry(self, config: _RunConfig, *, control: _ExecutionControl | None = None) -> _SandboxEntry: cache_key = config.cache_key() with self._entries_lock: entry = self._entries.get(cache_key) if entry is None: - entry = self._create_entry(config) + entry = self._create_entry( + config, control=control or _ExecutionControl(config.execution_timeout_seconds) + ) self._entries[cache_key] = entry return entry @@ -1476,7 +1560,7 @@ def close(self) -> None: # were already moved into the per-worker disposal closure inside dispose(). del entries - def _create_entry(self, config: _RunConfig) -> _SandboxEntry: + def _create_entry(self, config: _RunConfig, *, control: _ExecutionControl) -> _SandboxEntry: input_dir_handle = TemporaryDirectory() if config.filesystem_enabled else None output_dir_handle = TemporaryDirectory() if config.filesystem_enabled else None @@ -1521,18 +1605,19 @@ def _configure_sandbox(*, sandbox: Any, expand_missing_scheme: bool) -> None: ) def _build_sandbox() -> tuple[Any, Any]: + control.start() sandbox = _create_sandbox() _configure_sandbox(sandbox=sandbox, expand_missing_scheme=False) try: - sandbox.run("None") + control.run(sandbox, "None") except RuntimeError as exc: if not _should_retry_allowed_domain_registration(error=exc, allowed_domains=config.allowed_domains): raise sandbox = _create_sandbox() _configure_sandbox(sandbox=sandbox, expand_missing_scheme=True) - sandbox.run("None") + control.run(sandbox, "None") snapshot = sandbox.snapshot() return sandbox, snapshot @@ -1596,6 +1681,9 @@ class HyperlightExecuteCodeTool(FunctionTool): Mappings are copied, including entries for tools registered later. include_detailed_errors: Expose host exception and execution diagnostics only for a trusted channel. Defaults to ``False`` and is independent of outer-client error/logging settings. + execution_timeout_seconds: Positive finite guest execution budget, in seconds. Defaults to 30. + Cancellation and timeout wait for native execution to stop. Runtime initialization and + host tool callbacks cannot be interrupted; host tools must enforce their own limits. """ def __init__( @@ -1611,6 +1699,7 @@ def __init__( max_output_files: int = DEFAULT_MAX_OUTPUT_FILES, max_output_file_bytes: int = DEFAULT_MAX_OUTPUT_FILE_BYTES, max_output_total_bytes: int = DEFAULT_MAX_OUTPUT_TOTAL_BYTES, + execution_timeout_seconds: float = DEFAULT_EXECUTION_TIMEOUT_SECONDS, backend: str = DEFAULT_HYPERLIGHT_BACKEND, module: str | None = DEFAULT_HYPERLIGHT_MODULE, module_path: str | None = None, @@ -1619,6 +1708,10 @@ def __init__( ) -> None: if not isinstance(include_detailed_errors, bool): raise TypeError("include_detailed_errors must be a bool.") + if isinstance(execution_timeout_seconds, bool) or not isinstance(execution_timeout_seconds, (int, float)): + raise TypeError("execution_timeout_seconds must be a number.") + if not math.isfinite(execution_timeout_seconds) or execution_timeout_seconds <= 0: + raise ValueError("execution_timeout_seconds must be positive and finite.") normalized_description_format = _normalize_tool_description_format(tool_description_format) max_output_files = _validate_positive_integer(name="max_output_files", value=max_output_files) max_output_file_bytes = _validate_positive_integer(name="max_output_file_bytes", value=max_output_file_bytes) @@ -1635,6 +1728,7 @@ def __init__( self._registry = _registry or _SandboxRegistry() self._default_approval_mode: ApprovalMode = approval_mode or "never_require" self._include_detailed_errors = include_detailed_errors + self._execution_timeout_seconds = execution_timeout_seconds self._workspace_root = _resolve_workspace_root(workspace_root) self._backend: str = backend self._module: str | None = module @@ -1819,6 +1913,7 @@ def create_run_tool(self) -> HyperlightExecuteCodeTool: max_output_files=self._max_output_files, max_output_file_bytes=self._max_output_file_bytes, max_output_total_bytes=self._max_output_total_bytes, + execution_timeout_seconds=self._execution_timeout_seconds, backend=self._backend, module=self._module, module_path=self._module_path, @@ -1834,6 +1929,7 @@ def build_serializable_state(self) -> dict[str, Any]: "module_path": config.module_path, "approval_mode": config.approval_mode, "include_detailed_errors": config.include_detailed_errors, + "execution_timeout_seconds": config.execution_timeout_seconds, "max_output_files": config.max_output_files, "max_output_file_bytes": config.max_output_file_bytes, "max_output_total_bytes": config.max_output_total_bytes, @@ -1911,13 +2007,44 @@ def _build_run_config(self) -> _RunConfig: max_output_file_bytes=max_output_file_bytes, max_output_total_bytes=max_output_total_bytes, include_detailed_errors=include_detailed_errors, + execution_timeout_seconds=self._execution_timeout_seconds, ) async def _run_code(self, *, code: str) -> list[Content]: include_detailed_errors = self._include_detailed_errors try: config = self._build_run_config() - return await asyncio.to_thread(self._registry.execute, config=config, code=code) + cancelled = threading.Event() + token = _execution_cancellation.set(cancelled) + try: + current_context = contextvars.copy_context() + finally: + _execution_cancellation.reset(token) + execution = asyncio.get_running_loop().run_in_executor( + None, lambda: current_context.run(self._registry.execute, config=config, code=code) + ) + try: + return await asyncio.shield(execution) + except asyncio.CancelledError: + cancelled.set() + # Shield the owning worker until native execution and its interruption + # thread have finished, including when the caller cancels again. + while not execution.done(): + try: + await asyncio.shield(execution) + except asyncio.CancelledError: + continue + except Exception: + break + try: + execution.result() + except asyncio.CancelledError: + pass + except Exception: + logger.warning( + "Hyperlight execution failed during cancellation cleanup.", exc_info=include_detailed_errors + ) + raise except Exception: if include_detailed_errors: raise diff --git a/python/packages/hyperlight/agent_framework_hyperlight/_provider.py b/python/packages/hyperlight/agent_framework_hyperlight/_provider.py index a72fd032f44..bfb76daf88a 100644 --- a/python/packages/hyperlight/agent_framework_hyperlight/_provider.py +++ b/python/packages/hyperlight/agent_framework_hyperlight/_provider.py @@ -11,6 +11,7 @@ from agent_framework._tools import ApprovalMode, _ToolDescriptionFormat # pyright: ignore[reportPrivateUsage] from ._execute_code_tool import ( + DEFAULT_EXECUTION_TIMEOUT_SECONDS, DEFAULT_MAX_OUTPUT_FILE_BYTES, DEFAULT_MAX_OUTPUT_FILES, DEFAULT_MAX_OUTPUT_TOTAL_BYTES, @@ -31,6 +32,8 @@ class HyperlightCodeActProvider(ContextProvider): Mappings are copied, including entries for tools registered later. include_detailed_errors: Expose host exception and execution diagnostics only for a trusted channel. Defaults to ``False`` and is independent of outer-client error/logging settings. + execution_timeout_seconds: Positive finite guest execution budget, in seconds. Defaults to 30. + Runtime initialization and host tool callbacks cannot be interrupted. """ DEFAULT_SOURCE_ID = "hyperlight_codeact" @@ -49,6 +52,7 @@ def __init__( max_output_files: int = DEFAULT_MAX_OUTPUT_FILES, max_output_file_bytes: int = DEFAULT_MAX_OUTPUT_FILE_BYTES, max_output_total_bytes: int = DEFAULT_MAX_OUTPUT_TOTAL_BYTES, + execution_timeout_seconds: float = DEFAULT_EXECUTION_TIMEOUT_SECONDS, backend: str = "wasm", module: str | None = "python_guest.path", module_path: str | None = None, @@ -66,6 +70,7 @@ def __init__( max_output_files=max_output_files, max_output_file_bytes=max_output_file_bytes, max_output_total_bytes=max_output_total_bytes, + execution_timeout_seconds=execution_timeout_seconds, backend=backend, module=module, module_path=module_path, diff --git a/python/packages/hyperlight/tests/hyperlight/conftest.py b/python/packages/hyperlight/tests/hyperlight/conftest.py index 23d4bfa8515..21db573f11b 100644 --- a/python/packages/hyperlight/tests/hyperlight/conftest.py +++ b/python/packages/hyperlight/tests/hyperlight/conftest.py @@ -6,11 +6,70 @@ import threading from collections.abc import Callable from types import SimpleNamespace -from typing import Any +from typing import Any, cast import pytest from agent_framework import FunctionTool +from agent_framework_hyperlight import HyperlightExecuteCodeTool +from agent_framework_hyperlight import _execute_code_tool as execute_code_module + + +@pytest.fixture +def interruptible_execution(monkeypatch: pytest.MonkeyPatch): + entered = threading.Event() + release = threading.Event() + interrupted = threading.Event() + kill_attempted = threading.Event() + completed = threading.Event() + + class Sandbox: + def __init__(self, **kwargs: Any) -> None: + self._host_callback = False + + def interrupt_handle(self) -> SimpleNamespace: + def kill() -> bool: + kill_attempted.set() + interrupted.set() + return not self._host_callback + + return SimpleNamespace(kill=kill) + + def run(self, code: str) -> SimpleNamespace: + if code in {"guest", "host"}: + self._host_callback = code == "host" + entered.set() + try: + if self._host_callback: + if not release.wait(5): + raise RuntimeError("Host callback was not released.") + elif not interrupted.wait(5): + raise RuntimeError("Guest was not interrupted.") + if interrupted.is_set(): + raise InterruptedError("Guest execution interrupted.") + finally: + completed.set() + self._host_callback = False + return SimpleNamespace(success=True, stdout="ok\n", stderr="") + + def snapshot(self) -> object: + return object() + + def restore(self, snapshot: object) -> None: + interrupted.clear() + + monkeypatch.setattr(execute_code_module, "_load_sandbox_class", lambda: Sandbox) + execute_code = HyperlightExecuteCodeTool(execution_timeout_seconds=0.1) + yield SimpleNamespace( + tool=execute_code, + entered=entered, + release=release, + kill_attempted=kill_attempted, + completed=completed, + ) + release.set() + cast(Any, execute_code._registry).close() + @pytest.fixture def host_error_detail() -> str: @@ -70,6 +129,9 @@ def snapshot(self) -> object: self._record("snapshot") return object() + def interrupt_handle(self) -> SimpleNamespace: + return SimpleNamespace(kill=lambda: False) + def restore(self, snapshot: object) -> None: self._record("restore") diff --git a/python/packages/hyperlight/tests/hyperlight/test_execution_control.py b/python/packages/hyperlight/tests/hyperlight/test_execution_control.py new file mode 100644 index 00000000000..9118c97e732 --- /dev/null +++ b/python/packages/hyperlight/tests/hyperlight/test_execution_control.py @@ -0,0 +1,67 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +import asyncio + + +async def test_timeout_returns_error_and_allows_reuse(interruptible_execution) -> None: + native = interruptible_execution + result = await native.tool.invoke(arguments={"code": "guest"}) + + assert native.completed.is_set() + assert len(result) == 1 + assert result[0].type == "error" + assert result[0].error_details == "Code execution exceeded its time limit." + recovered = await native.tool.invoke(arguments={"code": "None"}) + assert recovered[0].text == "ok\n" + + +async def test_repeated_cancellation_waits_for_host_callback_to_finish(interruptible_execution) -> None: + native = interruptible_execution + execution = asyncio.create_task(native.tool.invoke(arguments={"code": "host"})) + try: + assert await asyncio.to_thread(native.entered.wait, 2) + execution.cancel() + assert await asyncio.to_thread(native.kill_attempted.wait, 2) + execution.cancel() + await asyncio.sleep(0) + assert not execution.done() + assert not native.completed.is_set() + finally: + native.release.set() + try: + await asyncio.wait_for(execution, 2) + except asyncio.CancelledError: + pass + else: + raise AssertionError("Caller cancellation was not preserved.") + assert native.completed.is_set() + recovered = await native.tool.invoke(arguments={"code": "None"}) + assert recovered[0].text == "ok\n" + + +async def test_queued_cancellation_does_not_interrupt_another_invocation(interruptible_execution) -> None: + native = interruptible_execution + native.tool._execution_timeout_seconds = 5 + active = asyncio.create_task(native.tool.invoke(arguments={"code": "host"})) + queued = None + try: + assert await asyncio.to_thread(native.entered.wait, 2) + queued = asyncio.create_task(native.tool.invoke(arguments={"code": "None"})) + await asyncio.sleep(0.02) + queued.cancel() + await asyncio.sleep(0) + assert not native.kill_attempted.is_set() + finally: + native.release.set() + result = await asyncio.wait_for(active, 2) + assert result[0].text == "ok\n" + assert queued is not None + try: + await asyncio.wait_for(queued, 2) + except asyncio.CancelledError: + pass + else: + raise AssertionError("Queued cancellation was not preserved.") + assert not native.kill_attempted.is_set() diff --git a/python/packages/hyperlight/tests/hyperlight/test_execution_limits.py b/python/packages/hyperlight/tests/hyperlight/test_execution_limits.py new file mode 100644 index 00000000000..d1dc4c80c50 --- /dev/null +++ b/python/packages/hyperlight/tests/hyperlight/test_execution_limits.py @@ -0,0 +1,144 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +import asyncio +import os +import time +from typing import Any, cast + +import pytest +from agent_framework import tool + +from agent_framework_hyperlight import HyperlightExecuteCodeTool + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + os.environ.get("HYPERLIGHT_RUN_GUEST_TESTS") != "true", + reason="Requires explicit Hyperlight KVM guest-test configuration.", + ), +] + + +@pytest.mark.parametrize("backend", ["wasm", "hyperlight-js"]) +async def test_cancellation_stops_guest_before_reuse(backend: str) -> None: + loop = asyncio.get_running_loop() + entered = asyncio.Event() + completed = asyncio.Event() + deadline = 0.0 + + @tool + def started() -> None: + """Signal guest entry.""" + nonlocal deadline + deadline = time.monotonic() + 5 + loop.call_soon_threadsafe(entered.set) + + @tool + def keep_running() -> bool: + """Bound the baseline workload using the host clock.""" + return time.monotonic() < deadline + + @tool + def finished() -> None: + """Signal guest completion.""" + loop.call_soon_threadsafe(completed.set) + + execute_code = HyperlightExecuteCodeTool( + tools=[started, keep_running, finished], + backend=backend, + module=None if backend == "hyperlight-js" else "python_guest.path", + ) + code = ( + "call_tool('started')\nwhile call_tool('keep_running'):\n pass\ncall_tool('finished')" + if backend == "wasm" + else "call_tool('started'); while (call_tool('keep_running')) {} call_tool('finished');" + ) + try: + warmup = await execute_code.invoke(arguments={"code": "print('ready')" if backend == "wasm" else "1;"}) + assert not any(item.type == "error" for item in warmup), warmup + execution = asyncio.create_task(execute_code.invoke(arguments={"code": code})) + await asyncio.wait_for(entered.wait(), 10) + start = time.monotonic() + execution.cancel() + with pytest.raises(asyncio.CancelledError): + await execution + recovered = await asyncio.wait_for( + execute_code.invoke( + arguments={"code": "print('recovered')" if backend == "wasm" else "console.log('recovered');"} + ), + 10, + ) + assert not any(item.type == "error" for item in recovered), recovered + assert not completed.is_set(), "Guest continued executing after caller cancellation." + assert time.monotonic() - start < 3, "Cancellation did not promptly release guest execution." + assert any(item.type == "text" and "recovered" in (item.text or "") for item in recovered) + finally: + cast(Any, execute_code._registry).close() + + +@pytest.mark.parametrize("backend", ["wasm", "hyperlight-js"]) +async def test_configured_timeout_interrupts_cpu_work_and_allows_reuse(backend: str) -> None: + warm_tool = HyperlightExecuteCodeTool( + backend=backend, module=None if backend == "hyperlight-js" else "python_guest.path" + ) + limited_tool = HyperlightExecuteCodeTool( + backend=backend, + module=None if backend == "hyperlight-js" else "python_guest.path", + execution_timeout_seconds=0.1, + _registry=warm_tool._registry, + ) + try: + warmup = await warm_tool.invoke(arguments={"code": "print('ready')" if backend == "wasm" else "1;"}) + assert not any(item.type == "error" for item in warmup), warmup + start = time.monotonic() + result = await limited_tool.invoke( + arguments={ + "code": ( + "for value in range(100_000_000):\n value * value" + if backend == "wasm" + else "for (let value = 0; value < 100000000; value++) { value * value; }" + ) + } + ) + assert time.monotonic() - start < 3 + errors = [item for item in result if item.type == "error"] + assert len(errors) == 1 + assert errors[0].error_details == "Code execution exceeded its time limit." + recovered = await asyncio.wait_for( + warm_tool.invoke( + arguments={"code": "print('recovered')" if backend == "wasm" else "console.log('recovered');"} + ), + 10, + ) + assert not any(item.type == "error" for item in recovered), recovered + assert any(item.type == "text" and "recovered" in (item.text or "") for item in recovered) + finally: + cast(Any, warm_tool._registry).close() + + +async def test_default_timeout_stops_guest_before_reuse() -> None: + deadline = 0.0 + + @tool + def keep_running() -> bool: + """Bound the baseline workload using the host clock.""" + return time.monotonic() < deadline + + execute_code = HyperlightExecuteCodeTool(tools=[keep_running]) + try: + warmup = await execute_code.invoke(arguments={"code": "print('ready')"}) + assert not any(item.type == "error" for item in warmup), warmup + deadline = time.monotonic() + 32 + result = await execute_code.invoke( + arguments={"code": "while call_tool('keep_running'):\n pass\nprint('finished')"} + ) + errors = [item for item in result if item.type == "error"] + assert len(errors) == 1, "The default execution limit was not enforced." + assert errors[0].error_details == "Code execution exceeded its time limit." + recovered = await asyncio.wait_for(execute_code.invoke(arguments={"code": "print('recovered')"}), 10) + assert any(item.type == "text" and "recovered" in (item.text or "") for item in recovered) + assert not any(item.type == "error" for item in recovered), recovered + finally: + cast(Any, execute_code._registry).close() diff --git a/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py b/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py index d7ee0381d60..80f4da03aad 100644 --- a/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py +++ b/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py @@ -21,7 +21,7 @@ from dataclasses import dataclass from pathlib import Path from tempfile import TemporaryDirectory -from types import MappingProxyType +from types import MappingProxyType, SimpleNamespace from typing import Any, Literal, cast from unittest.mock import DEFAULT, patch @@ -325,6 +325,9 @@ def run(self, code: str) -> _FakeResult: def snapshot(self) -> str: return "snapshot" + def interrupt_handle(self) -> SimpleNamespace: + return SimpleNamespace(kill=lambda: False) + def restore(self, snapshot: Any) -> None: self.restore_calls.append(snapshot) @@ -2504,6 +2507,9 @@ def run(self, code: str) -> _FakeResult: def snapshot(self) -> str: return "snapshot" + def interrupt_handle(self) -> SimpleNamespace: + return SimpleNamespace(kill=lambda: False) + def restore(self, snapshot: Any) -> None: del snapshot @@ -2611,6 +2617,30 @@ async def test_provider_forwards_output_limits_to_run_tool_and_serializable_stat json.dumps(state) +@pytest.mark.parametrize("constructor", [HyperlightExecuteCodeTool, HyperlightCodeActProvider]) +@pytest.mark.parametrize("value", [True, "30", None, 0, -1, float("inf"), float("-inf"), float("nan")]) +def test_execution_timeout_requires_a_positive_finite_number(constructor: Any, value: Any) -> None: + with pytest.raises((TypeError, ValueError), match="execution_timeout_seconds"): + constructor(execution_timeout_seconds=value) + + +async def test_provider_captures_execution_timeout_without_changing_sandbox_identity() -> None: + runtime = _FakeRuntime() + provider = HyperlightCodeActProvider(execution_timeout_seconds=0.25, _registry=runtime) + context = _FakeSessionContext() + state: dict[str, Any] = {} + + await provider.before_run(agent=object(), session=None, context=cast(Any, context), state=state) + run_tool = context.tools[0][1][0] + assert isinstance(run_tool, HyperlightExecuteCodeTool) + await run_tool.invoke(arguments={"code": "None"}) + + assert runtime.calls[0][0].execution_timeout_seconds == 0.25 + assert state[provider.source_id]["execution_timeout_seconds"] == 0.25 + default_config = HyperlightExecuteCodeTool(_registry=runtime)._build_run_config() + assert runtime.calls[0][0].cache_key() == default_config.cache_key() + + async def test_output_limits_are_isolated_when_registry_is_shared( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/skills/agent-framework-builder-python/references/integrations.md b/skills/agent-framework-builder-python/references/integrations.md index c19bb242bb4..3a9f8570dba 100644 --- a/skills/agent-framework-builder-python/references/integrations.md +++ b/skills/agent-framework-builder-python/references/integrations.md @@ -63,6 +63,14 @@ Monty and Hyperlight redact host tool exception diagnostics by default. Enable is separate from the outer chat client's error policy and sensitive-data logging; outer redaction cannot repair a diagnostic already caught and returned by generated code. +Hyperlight guest execution defaults to a 30-second budget. Configure the positive finite +host-side `execution_timeout_seconds` option on `HyperlightCodeActProvider` or +`HyperlightExecuteCodeTool` for longer work. Caller task cancellation interrupts guest execution +and waits for the owning worker to finish before propagating. Neither cancellation nor the budget +can interrupt runtime initialization or host tool callbacks; enforce limits inside host tools too. +This requires a compatible SDK with `Sandbox.interrupt_handle()`; verify the selected package +release supports it before relying on these limits. + For Monty, distinguish interpreter `resource_limits` from host-side attachment capture. Capture has per-call file-count and raw-byte limits shared across read-write mounts. Omitted attachments do not undo filesystem writes. Monty requires `pydantic-monty>=1.0`. Use `max_feed_duration_secs` for the total sandbox execution budget or