[BREAKING] Python: fix(hyperlight): interrupt guest execution and enforce time limits

python/packages/hyperlight/README.md

@@ -38,6 +38,41 @@ Initialization, snapshot restore, and execution failures follow the same policy.
 Failed sandbox entries are retired; cancellation and system-exit signals still
 propagate without becoming ordinary execution errors.
 
+## Execution limits
+
+Guest execution has a finite 30-second budget by default. Both
+`HyperlightCodeActProvider` and `HyperlightExecuteCodeTool` accept the host-side
+`execution_timeout_seconds` option, which must be a positive finite number.
+Applications that intentionally run longer must configure a larger budget.
+The option is captured in provider run snapshots and serialized state; it is
+not a model-controlled `execute_code` argument.
+
+Reaching the budget returns an error content with
+`Code execution exceeded its time limit.` Caller task cancellation interrupts
+guest execution and propagates `asyncio.CancelledError`. Cancellation waits for
+native execution and its interruption thread to finish, even if the caller
+cancels again. Queued cancellation does not interrupt another invocation.
+The next invocation restores the warm snapshot before reusing the sandbox.
+
+The budget starts on the sandbox's owning worker, not while waiting for that
+worker. Initialization and warm-up share the invocation's budget, but runtime
+initialization and host tool callbacks cannot be interrupted. Host tools must
+enforce their own cancellation and time limits. Completed results and unrelated
+execution failures are not replaced with a timeout that arrives late.
+
+This behavior requires an SDK with the cross-thread `Sandbox.interrupt_handle()`
+API from [hyperlight-dev/hyperlight-sandbox#275](https://github.com/hyperlight-dev/hyperlight-sandbox/pull/275).
+The currently declared released dependency does not yet include this API;
+a compatible SDK release and dependency-floor update are required before shipping.
+
+To run the native regressions on a KVM host with the linked SDK and both
+backends installed:
+
+```bash
+HYPERLIGHT_RUN_GUEST_TESTS=true uv run --directory python/packages/hyperlight pytest \
+  tests/hyperlight/test_execution_limits.py -m integration
+```
+
 ## Installation
 
 ```bash

python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py

@@ -5,6 +5,7 @@
 import asyncio
 import contextvars
 import logging
+import math
 import mimetypes
 import os
 import shutil
@@ -57,6 +58,10 @@
 OUTPUT_TRAVERSAL_ENTRIES_PER_FILE = 10
 OUTPUT_TRAVERSAL_MAX_ENTRIES = 10_000
 OUTPUT_TRAVERSAL_MAX_DEPTH = 32
+DEFAULT_EXECUTION_TIMEOUT_SECONDS = 30.0
+_execution_cancellation: contextvars.ContextVar[threading.Event | None] = contextvars.ContextVar(
+    "hyperlight_execution_cancellation", default=None
+)
 
 EXECUTE_CODE_INPUT_SCHEMA: dict[str, Any] = {
     "type": "object",
@@ -80,6 +85,68 @@ class _OutputCleanupError(RuntimeError):
     """Raised when sandbox output cannot be safely cleaned within configured bounds."""
 
 
+class _GuestExecutionTimeout(TimeoutError):
+    """Raised when the invocation's guest execution budget expires."""
+
+
+class _ExecutionControl:
+    def __init__(self, timeout_seconds: float) -> None:
+        self._timeout_seconds = timeout_seconds
+        self._deadline: float | None = None
+        self._cancelled = _execution_cancellation.get() or threading.Event()
+
+    def start(self) -> float:
+        if self._deadline is None:
+            self._deadline = time.monotonic() + self._timeout_seconds
+        self.check()
+        return self._deadline
+
+    def check(self) -> None:
+        if self._cancelled.is_set():
+            raise asyncio.CancelledError
+        if self._deadline is not None and time.monotonic() >= self._deadline:
+            raise _GuestExecutionTimeout("Code execution exceeded its time limit.")
+
+    def run(self, sandbox: Any, code: str) -> Any:
+        deadline = self.start()
+        handle = sandbox.interrupt_handle()
+        self.check()
+        completed = threading.Event()
+        timed_out = threading.Event()
+        interruption_errors: list[str] = []
+
+        def _interrupt() -> None:
+            try:
+                while not completed.is_set():
+                    if time.monotonic() >= deadline:
+                        timed_out.set()
+                    if (self._cancelled.is_set() or timed_out.is_set()) and handle.kill():
+                        return
+                    completed.wait(0.01)
+            except Exception as exc:
+                interruption_errors.append(str(exc))
+
+        # Only the independently owned handle crosses threads, never the sandbox.
+        interruption = threading.Thread(target=_interrupt, name="hl-interrupt")
+        interruption.start()
+        try:
+            self.check()
+            try:
+                result = sandbox.run(code=code)
+            except InterruptedError:
+                if self._cancelled.is_set():
+                    raise asyncio.CancelledError from None
+                if timed_out.is_set():
+                    raise _GuestExecutionTimeout("Code execution exceeded its time limit.") from None
+                raise
+        finally:
+            completed.set()
+            interruption.join()
+        if interruption_errors:
+            raise RuntimeError("Could not interrupt Hyperlight guest execution.")
+        return result
+
+
 def _output_cleanup_error_content(error: _OutputCleanupError) -> Content:
     return Content.from_error(message="Execution error", error_details=str(error))
 
@@ -113,6 +180,7 @@ class _RunConfig:
     max_output_file_bytes: int = DEFAULT_MAX_OUTPUT_FILE_BYTES
     max_output_total_bytes: int = DEFAULT_MAX_OUTPUT_TOTAL_BYTES
     include_detailed_errors: bool = False
+    execution_timeout_seconds: float = DEFAULT_EXECUTION_TIMEOUT_SECONDS
 
     @property
     def mounted_paths(self) -> tuple[str, ...]:
@@ -203,7 +271,8 @@ def _wrapped() -> tuple[bool, Any]:
             try:
                 return True, fn()
             except BaseException as exc:
-                self._reusable = False
+                if not isinstance(exc, (asyncio.CancelledError, _GuestExecutionTimeout)):
+                    self._reusable = False
                 exc_type: type[BaseException]
                 exc_args: tuple[str, ...]
                 # Capture args (usually (message,)) so the re-raised exception keeps the
@@ -213,7 +282,11 @@ def _wrapped() -> tuple[bool, Any]:
                 # unsendable object in args, forwarding it as-is would re-introduce the
                 # same cross-thread Drop hazard the traceback nulling avoids. Strings
                 # are always sendable. Fall back to the str() form if args is empty.
-                if isinstance(exc, Exception) and not include_detailed_errors:
+                if (
+                    isinstance(exc, Exception)
+                    and not include_detailed_errors
+                    and not isinstance(exc, _GuestExecutionTimeout)
+                ):
                     exc_type = RuntimeError
                     exc_args = ("Code execution failed.",)
                 else:
@@ -267,6 +340,7 @@ def execute(
         max_output_files: int,
         max_output_file_bytes: int,
         max_output_total_bytes: int,
+        control: _ExecutionControl,
         include_detailed_errors: bool = False,
     ) -> list[Content]:
         """Restore + run + build sendable contents — all on the worker thread.
@@ -277,6 +351,7 @@ def execute(
         cleanup_max_entries = _output_traversal_entry_limit(max_output_files)
 
         def _on_worker() -> list[Content]:
+            control.start()
             if not self._reusable:
                 return [_output_cleanup_error_content(_OutputCleanupError("Could not clear sandbox output safely."))]
 
@@ -295,7 +370,7 @@ def _on_worker() -> list[Content]:
 
             contents: list[Content] | None = None
             try:
-                result = sandbox.run(code=code)
+                result = control.run(sandbox, code)
                 try:
                     contents = build_contents(
                         result=result,
@@ -1426,19 +1501,26 @@ def execute(self, *, config: _RunConfig, code: str) -> list[Content]:
         Entries whose output cannot be cleaned safely are evicted before another invocation.
         """
         cache_key = config.cache_key()
-        entry = self._get_or_create_entry(config)
+        control = _ExecutionControl(config.execution_timeout_seconds)
+        entry: _SandboxEntry | None = None
         try:
+            entry = self._get_or_create_entry(config, control=control)
             return entry.worker.execute(
                 code=code,
                 output_dir=entry.output_dir,
                 build_contents=_build_execution_contents,
                 max_output_files=config.max_output_files,
                 max_output_file_bytes=config.max_output_file_bytes,
                 max_output_total_bytes=config.max_output_total_bytes,
+                control=control,
                 include_detailed_errors=config.include_detailed_errors,
             )
+        except _GuestExecutionTimeout:
+            return [
+                Content.from_error(message="Execution error", error_details="Code execution exceeded its time limit.")
+            ]
         finally:
-            if not entry.worker.is_reusable():
+            if entry is not None and not entry.worker.is_reusable():
                 self._discard_entry(cache_key=cache_key, entry=entry)
 
     def _discard_entry(self, *, cache_key: tuple[Any, ...], entry: _SandboxEntry) -> None:
@@ -1449,12 +1531,14 @@ def _discard_entry(self, *, cache_key: tuple[Any, ...], entry: _SandboxEntry) ->
             entry.worker.dispose()
             self._cleanup_executor.submit(entry.cleanup_temp_dirs)
 
-    def _get_or_create_entry(self, config: _RunConfig) -> _SandboxEntry:
+    def _get_or_create_entry(self, config: _RunConfig, *, control: _ExecutionControl | None = None) -> _SandboxEntry:
         cache_key = config.cache_key()
         with self._entries_lock:
             entry = self._entries.get(cache_key)
             if entry is None:
-                entry = self._create_entry(config)
+                entry = self._create_entry(
+                    config, control=control or _ExecutionControl(config.execution_timeout_seconds)
+                )
                 self._entries[cache_key] = entry
             return entry
 
@@ -1476,7 +1560,7 @@ def close(self) -> None:
             # were already moved into the per-worker disposal closure inside dispose().
             del entries
 
-    def _create_entry(self, config: _RunConfig) -> _SandboxEntry:
+    def _create_entry(self, config: _RunConfig, *, control: _ExecutionControl) -> _SandboxEntry:
         input_dir_handle = TemporaryDirectory() if config.filesystem_enabled else None
         output_dir_handle = TemporaryDirectory() if config.filesystem_enabled else None
 
@@ -1521,18 +1605,19 @@ def _configure_sandbox(*, sandbox: Any, expand_missing_scheme: bool) -> None:
                     )
 
         def _build_sandbox() -> tuple[Any, Any]:
+            control.start()
             sandbox = _create_sandbox()
             _configure_sandbox(sandbox=sandbox, expand_missing_scheme=False)
 
             try:
-                sandbox.run("None")
+                control.run(sandbox, "None")
             except RuntimeError as exc:
                 if not _should_retry_allowed_domain_registration(error=exc, allowed_domains=config.allowed_domains):
                     raise
 
                 sandbox = _create_sandbox()
                 _configure_sandbox(sandbox=sandbox, expand_missing_scheme=True)
-                sandbox.run("None")
+                control.run(sandbox, "None")
 
             snapshot = sandbox.snapshot()
             return sandbox, snapshot
@@ -1596,6 +1681,9 @@ class HyperlightExecuteCodeTool(FunctionTool):
             Mappings are copied, including entries for tools registered later.
         include_detailed_errors: Expose host exception and execution diagnostics only for a trusted
             channel. Defaults to ``False`` and is independent of outer-client error/logging settings.
+        execution_timeout_seconds: Positive finite guest execution budget, in seconds. Defaults to 30.
+            Cancellation and timeout wait for native execution to stop. Runtime initialization and
+            host tool callbacks cannot be interrupted; host tools must enforce their own limits.
     """
 
     def __init__(
@@ -1611,6 +1699,7 @@ def __init__(
         max_output_files: int = DEFAULT_MAX_OUTPUT_FILES,
         max_output_file_bytes: int = DEFAULT_MAX_OUTPUT_FILE_BYTES,
         max_output_total_bytes: int = DEFAULT_MAX_OUTPUT_TOTAL_BYTES,
+        execution_timeout_seconds: float = DEFAULT_EXECUTION_TIMEOUT_SECONDS,
         backend: str = DEFAULT_HYPERLIGHT_BACKEND,
         module: str | None = DEFAULT_HYPERLIGHT_MODULE,
         module_path: str | None = None,
@@ -1619,6 +1708,10 @@ def __init__(
     ) -> None:
         if not isinstance(include_detailed_errors, bool):
             raise TypeError("include_detailed_errors must be a bool.")
+        if isinstance(execution_timeout_seconds, bool) or not isinstance(execution_timeout_seconds, (int, float)):
+            raise TypeError("execution_timeout_seconds must be a number.")
+        if not math.isfinite(execution_timeout_seconds) or execution_timeout_seconds <= 0:
+            raise ValueError("execution_timeout_seconds must be positive and finite.")
         normalized_description_format = _normalize_tool_description_format(tool_description_format)
         max_output_files = _validate_positive_integer(name="max_output_files", value=max_output_files)
         max_output_file_bytes = _validate_positive_integer(name="max_output_file_bytes", value=max_output_file_bytes)
@@ -1635,6 +1728,7 @@ def __init__(
         self._registry = _registry or _SandboxRegistry()
         self._default_approval_mode: ApprovalMode = approval_mode or "never_require"
         self._include_detailed_errors = include_detailed_errors
+        self._execution_timeout_seconds = execution_timeout_seconds
         self._workspace_root = _resolve_workspace_root(workspace_root)
         self._backend: str = backend
         self._module: str | None = module
@@ -1819,6 +1913,7 @@ def create_run_tool(self) -> HyperlightExecuteCodeTool:
             max_output_files=self._max_output_files,
             max_output_file_bytes=self._max_output_file_bytes,
             max_output_total_bytes=self._max_output_total_bytes,
+            execution_timeout_seconds=self._execution_timeout_seconds,
             backend=self._backend,
             module=self._module,
             module_path=self._module_path,
@@ -1834,6 +1929,7 @@ def build_serializable_state(self) -> dict[str, Any]:
             "module_path": config.module_path,
             "approval_mode": config.approval_mode,
             "include_detailed_errors": config.include_detailed_errors,
+            "execution_timeout_seconds": config.execution_timeout_seconds,
             "max_output_files": config.max_output_files,
             "max_output_file_bytes": config.max_output_file_bytes,
             "max_output_total_bytes": config.max_output_total_bytes,
@@ -1911,13 +2007,44 @@ def _build_run_config(self) -> _RunConfig:
             max_output_file_bytes=max_output_file_bytes,
             max_output_total_bytes=max_output_total_bytes,
             include_detailed_errors=include_detailed_errors,
+            execution_timeout_seconds=self._execution_timeout_seconds,
         )
 
     async def _run_code(self, *, code: str) -> list[Content]:
         include_detailed_errors = self._include_detailed_errors
         try:
             config = self._build_run_config()
-            return await asyncio.to_thread(self._registry.execute, config=config, code=code)
+            cancelled = threading.Event()
+            token = _execution_cancellation.set(cancelled)
+            try:
+                current_context = contextvars.copy_context()
+            finally:
+                _execution_cancellation.reset(token)
+            execution = asyncio.get_running_loop().run_in_executor(
+                None, lambda: current_context.run(self._registry.execute, config=config, code=code)
+            )
+            try:
+                return await asyncio.shield(execution)
+            except asyncio.CancelledError:
+                cancelled.set()
+                # Shield the owning worker until native execution and its interruption
+                # thread have finished, including when the caller cancels again.
+                while not execution.done():
+                    try:
+                        await asyncio.shield(execution)
+                    except asyncio.CancelledError:
+                        continue
+                    except Exception:
+                        break
+                try:
+                    execution.result()
+                except asyncio.CancelledError:
+                    pass
+                except Exception:
+                    logger.warning(
+                        "Hyperlight execution failed during cancellation cleanup.", exc_info=include_detailed_errors
+                    )
+                raise
         except Exception:
             if include_detailed_errors:
                 raise

python/packages/hyperlight/agent_framework_hyperlight/_provider.py

@@ -11,6 +11,7 @@
 from agent_framework._tools import ApprovalMode, _ToolDescriptionFormat  # pyright: ignore[reportPrivateUsage]
 
 from ._execute_code_tool import (
+    DEFAULT_EXECUTION_TIMEOUT_SECONDS,
     DEFAULT_MAX_OUTPUT_FILE_BYTES,
     DEFAULT_MAX_OUTPUT_FILES,
     DEFAULT_MAX_OUTPUT_TOTAL_BYTES,
@@ -31,6 +32,8 @@ class HyperlightCodeActProvider(ContextProvider):
             Mappings are copied, including entries for tools registered later.
         include_detailed_errors: Expose host exception and execution diagnostics only for a trusted
             channel. Defaults to ``False`` and is independent of outer-client error/logging settings.
+        execution_timeout_seconds: Positive finite guest execution budget, in seconds. Defaults to 30.
+            Runtime initialization and host tool callbacks cannot be interrupted.
     """
 
     DEFAULT_SOURCE_ID = "hyperlight_codeact"
@@ -49,6 +52,7 @@ def __init__(
         max_output_files: int = DEFAULT_MAX_OUTPUT_FILES,
         max_output_file_bytes: int = DEFAULT_MAX_OUTPUT_FILE_BYTES,
         max_output_total_bytes: int = DEFAULT_MAX_OUTPUT_TOTAL_BYTES,
+        execution_timeout_seconds: float = DEFAULT_EXECUTION_TIMEOUT_SECONDS,
         backend: str = "wasm",
         module: str | None = "python_guest.path",
         module_path: str | None = None,
@@ -66,6 +70,7 @@ def __init__(
             max_output_files=max_output_files,
             max_output_file_bytes=max_output_file_bytes,
             max_output_total_bytes=max_output_total_bytes,
+            execution_timeout_seconds=execution_timeout_seconds,
             backend=backend,
             module=module,
             module_path=module_path,