From 37e96174e8e32637cff3a729c9b141b9479d6472 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Mon, 2 Nov 2026 14:08:29 +0100 Subject: [PATCH 1/3] Redact CodeAct host-tool diagnostics by default --- .../HyperlightCodeActProviderOptions.cs | 10 ++ .../Internal/SandboxExecutor.cs | 14 ++- .../Internal/ToolBridge.cs | 26 ++-- .../Microsoft.Agents.AI.Hyperlight/README.md | 14 +++ .../Internal/CodeExecutor.cs | 8 +- .../Internal/ProcessBridge.cs | 9 +- .../LocalCodeActProvider.cs | 5 +- .../LocalCodeActProviderOptions.cs | 10 ++ .../LocalExecuteCodeFunction.cs | 2 +- .../README.md | 13 ++ .../CodeActEndToEndTests.cs | 39 ++++++ .../SandboxExecutorTests.cs | 6 + .../ToolBridgeTests.cs | 29 +++-- ...ocalExecuteCodeFunctionIntegrationTests.cs | 116 +++++++++++++++++- python/packages/hyperlight/README.md | 16 +++ .../_execute_code_tool.py | 27 +++- .../agent_framework_hyperlight/_provider.py | 4 + .../hyperlight/tests/hyperlight/conftest.py | 37 ++++++ .../hyperlight/test_hyperlight_codeact.py | 109 ++++++++++++++-- python/packages/monty/AGENTS.md | 4 + python/packages/monty/README.md | 18 +++ .../_execute_code_tool.py | 15 ++- .../agent_framework_monty/_monty_bridge.py | 12 +- .../monty/agent_framework_monty/_provider.py | 6 + python/packages/monty/tests/monty/conftest.py | 37 ++++++ .../monty/tests/monty/test_monty_codeact.py | 39 ++++-- .../monty/test_monty_codeact_integration.py | 58 ++++++++- .../references/integrations.md | 5 + .../references/integrations.md | 5 + 29 files changed, 636 insertions(+), 57 deletions(-) create mode 100644 python/packages/hyperlight/tests/hyperlight/conftest.py create mode 100644 python/packages/monty/tests/monty/conftest.py diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs b/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs index 93e5a09c395..d394d328259 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs @@ -81,6 +81,16 @@ public static HyperlightCodeActProviderOptions CreateForJavaScript() /// public CodeActApprovalMode ApprovalMode { get; set; } = CodeActApprovalMode.NeverRequire; + /// + /// Gets or sets whether host exception details are exposed to generated code and in execution errors. + /// Defaults to . + /// + /// + /// Enable only for a trusted channel: diagnostics can contain sensitive host data. + /// This is independent of the outer chat client's error policy and sensitive-data logging. + /// + public bool IncludeDetailedErrors { get; set; } + /// /// Gets or sets an optional host directory exposed to the sandbox as its /// /input directory. diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs index 4c13b10d60a..6ae6dfbb1d0 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs @@ -52,6 +52,7 @@ public RunSnapshot( this.HeapSize = options.HeapSize; this.StackSize = options.StackSize; this.HostInputDirectory = options.HostInputDirectory; + this.IncludeDetailedErrors = options.IncludeDetailedErrors; this.ToolRegistryVersion = toolRegistryVersion; this.ConfigFingerprint = ComputeFingerprint( this.Tools, @@ -62,7 +63,8 @@ public RunSnapshot( this.Backend, this.ModulePath, this.HeapSize, - this.StackSize); + this.StackSize, + this.IncludeDetailedErrors); } public IReadOnlyList Tools { get; } @@ -81,6 +83,8 @@ public RunSnapshot( public string? HostInputDirectory { get; } + public bool IncludeDetailedErrors { get; } + public Guid ToolRegistryVersion { get; } /// @@ -100,7 +104,8 @@ internal static string ComputeFingerprint( SandboxBackend backend = SandboxBackend.JavaScript, string? modulePath = null, string? heapSize = null, - string? stackSize = null) + string? stackSize = null, + bool includeDetailedErrors = false) { var buffer = new ArrayBufferWriter(); using (var writer = new Utf8JsonWriter(buffer)) @@ -111,6 +116,7 @@ internal static string ComputeFingerprint( writer.WriteString("heapSize", heapSize); writer.WriteString("stackSize", stackSize); writer.WriteString("hostInputDirectory", hostInputDirectory); + writer.WriteBoolean("includeDetailedErrors", includeDetailedErrors); writer.WriteString("toolRegistryVersion", toolRegistryVersion); writer.WritePropertyName("tools"); @@ -234,7 +240,7 @@ public async Task ExecuteAsync(RunSnapshot snapshot, string code, Cancel catch (Exception ex) #pragma warning restore CA1031 { - return BuildErrorResult(ex.Message); + return BuildErrorResult(snapshot.IncludeDetailedErrors ? ex.Message : "Code execution failed."); } return BuildResult(result); @@ -300,7 +306,7 @@ private void BuildAndWarmUp(RunSnapshot snapshot) var sandbox = builder.Build(); // Tools must be registered before the first Run() call. - ToolBridge.RegisterAll(sandbox, snapshot.Tools); + ToolBridge.RegisterAll(sandbox, snapshot.Tools, snapshot.IncludeDetailedErrors); foreach (var allowedDomain in snapshot.AllowedDomains) { diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs index b2f735474e4..aab8072b6ed 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs @@ -21,24 +21,33 @@ internal static class ToolBridge /// Registers every entry against the provided /// as a raw JSON-in / JSON-out async tool. /// - public static void RegisterAll(Sandbox sandbox, IReadOnlyList tools) + public static void RegisterAll(Sandbox sandbox, IReadOnlyList tools, bool includeDetailedErrors = false) { foreach (var tool in tools) { - RegisterOne(sandbox, tool); + RegisterOne(sandbox, tool, includeDetailedErrors); } } - private static void RegisterOne(Sandbox sandbox, AIFunction tool) + private static void RegisterOne(Sandbox sandbox, AIFunction tool, bool includeDetailedErrors) => sandbox.RegisterToolAsync( tool.Name, - async (string argsJson) => await InvokeAsync(tool, argsJson).ConfigureAwait(false)); + async (string argsJson) => await InvokeAsync(tool, argsJson, includeDetailedErrors).ConfigureAwait(false)); - internal static async Task InvokeAsync(AIFunction tool, string argsJson) + internal static async Task InvokeAsync(AIFunction tool, string argsJson, bool includeDetailedErrors = false) { try { - var arguments = ParseArguments(argsJson); + IDictionary arguments; + try + { + arguments = ParseArguments(argsJson); + } + catch (Exception ex) when (ex is JsonException or ArgumentException) + { + return SerializeError(ex.Message); + } + var result = await tool.InvokeAsync(new AIFunctionArguments(arguments)).ConfigureAwait(false); return SerializeResult(result); } @@ -46,10 +55,13 @@ internal static async Task InvokeAsync(AIFunction tool, string argsJson) catch (Exception ex) #pragma warning restore CA1031 { - return JsonSerializer.Serialize(new HyperlightToolError(ex.Message), HyperlightJsonContext.Default.HyperlightToolError); + return SerializeError(includeDetailedErrors ? ex.Message : "Host tool execution failed."); } } + private static string SerializeError(string message) => + JsonSerializer.Serialize(new HyperlightToolError(message), HyperlightJsonContext.Default.HyperlightToolError); + internal static IDictionary ParseArguments(string argsJson) { if (string.IsNullOrWhiteSpace(argsJson)) diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md b/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md index 8596db97644..bdd7f3063bb 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md @@ -35,6 +35,20 @@ Both surfaces support: the meantime. * A Hyperlight Python guest module when using `SandboxBackend.Wasm`. +## Error diagnostics + +Host tool failures return a generic JSON error by default. Host execution +exceptions returned by the executor also use a generic message. +`HyperlightCodeActProviderOptions.IncludeDetailedErrors` defaults to `false` +for both the provider and standalone function. Set it to `true` only for a +trusted channel: host exception messages can contain sensitive data. + +This option is independent of the outer `FunctionInvokingChatClient` error policy +and sensitive-data logging. Outer error handling cannot redact diagnostics that +generated code has already caught and included in normal output. Run snapshots +capture the policy, and sandbox configuration fingerprints include it so cached +callbacks cannot retain a different disclosure policy. + ## Network access and guest packages `HyperlightCodeActProviderOptions.AllowedDomains` controls outbound requests diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/CodeExecutor.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/CodeExecutor.cs index 51d6a7136d1..abbf23dcfe2 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/CodeExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/CodeExecutor.cs @@ -39,15 +39,18 @@ public CodeExecutor( /// Immutable snapshot of provider state captured at the start of an invocation. public sealed class RunSnapshot { - public RunSnapshot(IReadOnlyList tools, IReadOnlyList fileMounts) + public RunSnapshot(IReadOnlyList tools, IReadOnlyList fileMounts, bool includeDetailedErrors = false) { this.Tools = tools; this.FileMounts = fileMounts; + this.IncludeDetailedErrors = includeDetailedErrors; } public IReadOnlyList Tools { get; } public IReadOnlyList FileMounts { get; } + + public bool IncludeDetailedErrors { get; } } public async Task> ExecuteAsync(RunSnapshot snapshot, string code, CancellationToken cancellationToken) @@ -65,7 +68,8 @@ public async Task> ExecuteAsync(RunSnapshot snapshot, string cod snapshot.Tools, this._limits, this._environment, - this._workingDirectory); + this._workingDirectory, + snapshot.IncludeDetailedErrors); var result = await bridge.RunAsync(code, cancellationToken).ConfigureAwait(false); diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs index dee86154c38..6e234779b38 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs @@ -31,6 +31,7 @@ internal sealed class ProcessBridge private readonly ProcessExecutionLimits _limits; private readonly IReadOnlyDictionary? _environment; private readonly string? _workingDirectory; + private readonly bool _includeDetailedErrors; public ProcessBridge( string pythonExecutable, @@ -38,7 +39,8 @@ public ProcessBridge( IReadOnlyList tools, ProcessExecutionLimits limits, IReadOnlyDictionary? environment, - string? workingDirectory) + string? workingDirectory, + bool includeDetailedErrors = false) { this._pythonExecutable = pythonExecutable; this._runnerScript = runnerScript; @@ -46,6 +48,7 @@ public ProcessBridge( this._limits = limits; this._environment = environment; this._workingDirectory = workingDirectory; + this._includeDetailedErrors = includeDetailedErrors; } /// Represents the parsed final result returned by the Python runner. @@ -264,7 +267,9 @@ await SendToolResponseAsync(process, callId, ok: false, result: null, #pragma warning restore CA1031 { await SendToolResponseAsync(process, callId, ok: false, result: null, - excType: ex.GetType().Name, excMessage: ex.Message, cancellationToken).ConfigureAwait(false); + excType: this._includeDetailedErrors ? ex.GetType().Name : "RuntimeError", + excMessage: this._includeDetailedErrors ? ex.Message : "Host tool execution failed.", + cancellationToken).ConfigureAwait(false); } } diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProvider.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProvider.cs index 33f9205f735..4cdb3b38d86 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProvider.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProvider.cs @@ -37,6 +37,7 @@ public sealed class LocalCodeActProvider : AIContextProvider, IDisposable private readonly CodeExecutor _executor; private readonly LocalCodeActApprovalMode _approvalMode; + private readonly bool _includeDetailedErrors; private readonly ConcurrentDictionary _tools = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _fileMounts = new(StringComparer.Ordinal); @@ -68,6 +69,7 @@ public LocalCodeActProvider(string pythonExecutablePath, LocalCodeActProviderOpt } this._approvalMode = options.ApprovalMode; + this._includeDetailedErrors = options.IncludeDetailedErrors; this._executor = new CodeExecutor( pythonExecutablePath, @@ -190,7 +192,8 @@ protected override ValueTask ProvideAIContextAsync(InvokingContext co this.ThrowIfDisposed(); snapshot = new CodeExecutor.RunSnapshot( this._tools.Values.ToList(), - this._fileMounts.Values.ToList()); + this._fileMounts.Values.ToList(), + this._includeDetailedErrors); FeatureUsageMarker.MarkUsed(); var approvalRequired = ComputeApprovalRequired(this._approvalMode, snapshot.Tools); diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs index 51a161a20ea..65c8a759dcd 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs @@ -31,6 +31,16 @@ public sealed class LocalCodeActProviderOptions /// public LocalCodeActApprovalMode ApprovalMode { get; set; } = LocalCodeActApprovalMode.NeverRequire; + /// + /// Gets or sets whether host exception details are exposed to generated code. + /// Defaults to . + /// + /// + /// Enable only for a trusted channel: diagnostics can contain sensitive host data. + /// This is independent of the outer chat client's error policy and sensitive-data logging. + /// + public bool IncludeDetailedErrors { get; set; } + /// /// Gets or sets the initial set of file mounts exposed to generated code. /// diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalExecuteCodeFunction.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalExecuteCodeFunction.cs index a8b0fd34e22..5bb0804208d 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalExecuteCodeFunction.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalExecuteCodeFunction.cs @@ -72,7 +72,7 @@ public LocalExecuteCodeFunction(string pythonExecutablePath, LocalCodeActProvide options.Environment, options.WorkingDirectory); - this._snapshot = new CodeExecutor.RunSnapshot(tools, fileMounts); + this._snapshot = new CodeExecutor.RunSnapshot(tools, fileMounts, options.IncludeDetailedErrors); this._inner = AIFunctionFactory.Create( this.ExecuteCodeAsync, new AIFunctionFactoryOptions diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md index e4ae51298bf..d8d3b5cd5cf 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md @@ -37,6 +37,19 @@ using var provider = new LocalCodeActProvider("/usr/bin/python3", options); // Register provider with your AIAgent's context providers. ``` +## Error diagnostics + +Host tool failures send a generic failed response to generated Python code by +default. `LocalCodeActProviderOptions.IncludeDetailedErrors` defaults to `false` +for both the provider and standalone function. Set it to `true` only for a +trusted channel: original exception types and messages can contain sensitive data. + +The policy is captured at construction and passed through run snapshots. It is +independent of the outer `FunctionInvokingChatClient` error policy and +sensitive-data logging. Outer error handling still applies to uncaught failures, +but cannot redact diagnostics that generated code has already caught and printed +or returned. Successful results and failure signaling are unchanged. + ## What the Package Controls - **AST validation** (default on): Validates generated code against allow-lists diff --git a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs index 58b9ebceb6a..255e946a266 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs @@ -56,4 +56,43 @@ public async Task ExecuteCode_PythonPrint_ReturnsStdoutAsync() Assert.Contains("hi", doc.RootElement.GetProperty("stdout").GetString()!); Assert.Equal(0, doc.RootElement.GetProperty("exit_code").GetInt32()); } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteCode_HostErrorDetailsRequireOptInAsync(bool includeDetailedErrors) + { + // Arrange + if (string.IsNullOrWhiteSpace(GuestPath)) + { + Assert.Skip(SkipReason); + return; + } + + const string Detail = "synthetic backend diagnostic"; + var options = HyperlightCodeActProviderOptions.CreateForWasm(GuestPath!); + options.IncludeDetailedErrors = includeDetailedErrors; + options.Tools = [AIFunctionFactory.Create( + (Func)(() => throw new InvalidOperationException(Detail)), name: "failing_tool")]; + using var function = new HyperlightExecuteCodeFunction(options); + + // Act + var result = await function.InvokeAsync(new AIFunctionArguments + { + ["code"] = """ + try: + print(call_tool('failing_tool')) + except Exception as exc: + print(str(exc)) + """, + }); + + // Assert + var json = Assert.IsType(result); + Assert.Equal(includeDetailedErrors, json.Contains(Detail, StringComparison.Ordinal)); + if (!includeDetailedErrors) + { + Assert.Contains("Host tool execution failed.", json, StringComparison.Ordinal); + } + } } diff --git a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs index 5a528606b60..386a839e65d 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs @@ -254,12 +254,15 @@ public void Fingerprint_DifferentSandboxOptions_DifferentFingerprints() [], [], [], hostInputDirectory: null, heapSize: "20Mi"); var differentStack = SandboxExecutor.RunSnapshot.ComputeFingerprint( [], [], [], hostInputDirectory: null, stackSize: "10Mi"); + var detailedErrors = SandboxExecutor.RunSnapshot.ComputeFingerprint( + [], [], [], hostInputDirectory: null, includeDetailedErrors: true); // Assert Assert.NotEqual(baseline, differentBackend); Assert.NotEqual(baseline, differentModule); Assert.NotEqual(baseline, differentHeap); Assert.NotEqual(baseline, differentStack); + Assert.NotEqual(baseline, detailedErrors); } [Fact] @@ -285,6 +288,7 @@ public void RunSnapshot_CapturesMutableInputs() HeapSize = "10Mi", StackSize = "5Mi", HostInputDirectory = "/host/input", + IncludeDetailedErrors = true, }; // Act @@ -296,6 +300,7 @@ public void RunSnapshot_CapturesMutableInputs() options.HeapSize = "20Mi"; options.StackSize = "10Mi"; options.HostInputDirectory = "/host/other"; + options.IncludeDetailedErrors = false; // Assert Assert.Single(snapshot.Tools); @@ -305,6 +310,7 @@ public void RunSnapshot_CapturesMutableInputs() Assert.Equal("10Mi", snapshot.HeapSize); Assert.Equal("5Mi", snapshot.StackSize); Assert.Equal("/host/input", snapshot.HostInputDirectory); + Assert.True(snapshot.IncludeDetailedErrors); } [Fact] diff --git a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs index 9d94d719302..d4bd084f3f8 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs @@ -24,20 +24,34 @@ public async Task InvokeAsync_PassesArgumentsAndReturnsSerializedResultAsync() Assert.Equal("\"echo:hello\"", result); } - [Fact] - public async Task InvokeAsync_ReturnsErrorJsonOnExceptionAsync() + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task InvokeAsync_ReturnsErrorJsonOnExceptionAsync(bool includeDetailedErrors, bool isAsync) { // Arrange - static int Boom() => throw new InvalidOperationException("nope"); - var tool = AIFunctionFactory.Create(Boom, name: "boom"); + const string Detail = "synthetic backend diagnostic"; + static int Boom() => throw new InvalidOperationException(Detail); + static async Task BoomAsync() + { + await Task.Yield(); + throw new InvalidOperationException(Detail); + } + + var tool = isAsync + ? AIFunctionFactory.Create(BoomAsync, name: "boom") + : AIFunctionFactory.Create(Boom, name: "boom"); // Act - var result = await ToolBridge.InvokeAsync(tool, "{}"); + var result = await ToolBridge.InvokeAsync(tool, "{}", includeDetailedErrors); // Assert using var doc = JsonDocument.Parse(result); Assert.True(doc.RootElement.TryGetProperty("error", out var err)); - Assert.Contains("nope", err.GetString()!); + Assert.Equal(includeDetailedErrors ? Detail : "Host tool execution failed.", err.GetString()); + Assert.Equal(includeDetailedErrors, result.Contains(Detail, StringComparison.Ordinal)); } [Fact] @@ -66,6 +80,7 @@ public async Task InvokeAsync_NonObjectJson_ReturnsErrorAsync() // Assert using var doc = JsonDocument.Parse(result); - Assert.True(doc.RootElement.TryGetProperty("error", out _)); + Assert.True(doc.RootElement.TryGetProperty("error", out var error)); + Assert.Contains("Tool arguments must be a JSON object.", error.GetString()!, StringComparison.Ordinal); } } diff --git a/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs b/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs index c7118a452ab..9db177a097c 100644 --- a/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs @@ -49,6 +49,110 @@ public async Task ExecuteCode_PrintsAndReturnsResultAsync() Assert.Contains("3", combined); } + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task ExecuteCode_HostErrorsRespectDetailedPolicyAsync(bool includeDetailedErrors, bool isAsync) + { + // Arrange + SkipIfNoPython(); + const string Detail = "synthetic backend diagnostic"; + static string Fail() => throw new InvalidOperationException(Detail); + static async Task FailAsync() + { + await Task.Yield(); + throw new InvalidOperationException(Detail); + } + + var tool = isAsync + ? AIFunctionFactory.Create(FailAsync, name: "failing_tool") + : AIFunctionFactory.Create(Fail, name: "failing_tool"); + var function = new LocalExecuteCodeFunction(s_python!, new LocalCodeActProviderOptions + { + Tools = [tool], + IncludeDetailedErrors = includeDetailedErrors, + }); + var args = new AIFunctionArguments + { + ["code"] = """ + try: + await failing_tool() + except Exception as exc: + print(str(exc)) + try: + await call_tool('failing_tool') + except Exception as exc: + print(str(exc)) + """, + }; + + // Act + var result = await function.InvokeAsync(args, CancellationToken.None); + + // Assert + var text = GetResultText(result); + Assert.Contains(includeDetailedErrors ? Detail : "Host tool execution failed.", text, StringComparison.Ordinal); + Assert.Equal(includeDetailedErrors, text.Contains(Detail, StringComparison.Ordinal)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteCode_UncaughtHostErrorsPreserveFailureAsync(bool includeDetailedErrors) + { + // Arrange + SkipIfNoPython(); + const string Detail = "synthetic backend diagnostic"; + var tool = AIFunctionFactory.Create( + (Func)(() => throw new InvalidOperationException(Detail)), name: "failing_tool"); + var function = new LocalExecuteCodeFunction(s_python!, new LocalCodeActProviderOptions + { + Tools = [tool], + IncludeDetailedErrors = includeDetailedErrors, + }); + + // Act + var error = await Assert.ThrowsAsync(async () => + await function.InvokeAsync(new AIFunctionArguments { ["code"] = "await failing_tool()" })); + + // Assert + Assert.Contains(includeDetailedErrors ? Detail : "Host tool execution failed.", error.Message, StringComparison.Ordinal); + Assert.Equal(includeDetailedErrors, error.Message.Contains(Detail, StringComparison.Ordinal)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Provider_HostErrorPolicyIsCapturedAtConstructionAsync(bool includeDetailedErrors) + { + // Arrange + SkipIfNoPython(); + const string Detail = "synthetic backend diagnostic"; + var tool = AIFunctionFactory.Create( + (Func)(() => throw new InvalidOperationException(Detail)), name: "failing_tool"); + var options = new LocalCodeActProviderOptions + { + Tools = [tool], + IncludeDetailedErrors = includeDetailedErrors, + }; + using var provider = new LocalCodeActProvider(s_python!, options); + options.IncludeDetailedErrors = !includeDetailedErrors; + var context = await provider.InvokingAsync( + new AIContextProvider.InvokingContext(new Moq.Mock().Object, session: null, new AIContext())); + var function = Assert.IsAssignableFrom(Assert.Single(context.Tools!)); + + // Act + var result = await function.InvokeAsync(new AIFunctionArguments + { + ["code"] = "try:\n await failing_tool()\nexcept Exception as exc:\n print(str(exc))", + }); + + // Assert + Assert.Equal(includeDetailedErrors, GetResultText(result).Contains(Detail, StringComparison.Ordinal)); + } + [Fact] public async Task ExecuteCode_AllowsAstValidatedFormattingAsync() { @@ -883,8 +987,10 @@ await call_tool('definitely_not_registered', x=1) Assert.DoesNotContain("NO_ERROR", combined); } - [Fact] - public async Task ExecuteCode_ToolThrowingExceptionPropagatesToGeneratedCodeAsync() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteCode_ToolThrowingExceptionPropagatesToGeneratedCodeAsync(bool includeDetailedErrors) { SkipIfNoPython(); @@ -896,6 +1002,7 @@ public async Task ExecuteCode_ToolThrowingExceptionPropagatesToGeneratedCodeAsyn var options = new LocalCodeActProviderOptions { Tools = new[] { faultyTool }, + IncludeDetailedErrors = includeDetailedErrors, }; var function = new LocalExecuteCodeFunction(s_python!, options); @@ -912,8 +1019,9 @@ await call_tool('faulty', message='boom') var result = await function.InvokeAsync(args, CancellationToken.None); var combined = GetResultText(result); Assert.Contains("GOT_ERROR", combined); - Assert.Contains("InvalidOperationException", combined); - Assert.Contains("intentional: boom", combined); + Assert.Contains(includeDetailedErrors ? "intentional: boom" : "Host tool execution failed.", combined); + Assert.Equal(includeDetailedErrors, combined.Contains("InvalidOperationException", StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, combined.Contains("intentional: boom", StringComparison.Ordinal)); } [Fact] diff --git a/python/packages/hyperlight/README.md b/python/packages/hyperlight/README.md index 76e52377936..94bd4b9aeb5 100644 --- a/python/packages/hyperlight/README.md +++ b/python/packages/hyperlight/README.md @@ -19,6 +19,22 @@ Hyperlight-backed CodeAct integrations for Microsoft Agent Framework. > Sandbox isolation does not supply FIDES enforcement. > See the [FIDES guide](../../samples/02-agents/security/FIDES_DEVELOPER_GUIDE.md). +## Error diagnostics + +Host tool callbacks expose a generic `RuntimeError("Host tool execution failed.")` +by default, before diagnostics reach generated code. Raw sandbox failure details +are omitted from error contents. Successful outputs and known-safe output-limit +or cleanup errors retain their existing behavior. + +Both `HyperlightCodeActProvider` and `HyperlightExecuteCodeTool` accept the +host-side option `include_detailed_errors=True` for trusted channels. It preserves +original callback exceptions and sandbox error details, which may contain +sensitive data. It is not a parameter of `execute_code` and is independent of the +outer chat client's error policy and sensitive-data logging. + +The policy is captured in each run snapshot and serialized state. Sandbox cache +keys include it, so detailed-error callbacks are not reused for redacted runs. + ## Installation ```bash diff --git a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py index 3c97dcce5a2..c0d88cb042b 100644 --- a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py +++ b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py @@ -112,6 +112,7 @@ class _RunConfig: max_output_files: int = DEFAULT_MAX_OUTPUT_FILES max_output_file_bytes: int = DEFAULT_MAX_OUTPUT_FILE_BYTES max_output_total_bytes: int = DEFAULT_MAX_OUTPUT_TOTAL_BYTES + include_detailed_errors: bool = False @property def mounted_paths(self) -> tuple[str, ...]: @@ -135,6 +136,7 @@ def cache_key(self) -> tuple[Any, ...]: self.max_output_files, self.max_output_file_bytes, self.max_output_total_bytes, + self.include_detailed_errors, ) @@ -255,6 +257,7 @@ def execute( max_output_files: int, max_output_file_bytes: int, max_output_total_bytes: int, + include_detailed_errors: bool = False, ) -> list[Content]: """Restore + run + build sendable contents — all on the worker thread. @@ -290,6 +293,7 @@ def _on_worker() -> list[Content]: max_output_files=max_output_files, max_output_file_bytes=max_output_file_bytes, max_output_total_bytes=max_output_total_bytes, + include_detailed_errors=include_detailed_errors, ) if output_dir is not None and not any(item.type == "data" for item in contents): # Isolate any file that becomes visible after the finite discovery window @@ -1209,6 +1213,7 @@ def _build_execution_contents( max_output_files: int, max_output_file_bytes: int, max_output_total_bytes: int, + include_detailed_errors: bool = False, ) -> list[Content]: success = bool(getattr(result, "success", False)) stdout = str(getattr(result, "stdout", "") or "").replace("\r\n", "\n") or None @@ -1246,7 +1251,7 @@ def _build_execution_contents( outputs.append(Content.from_text("Code executed successfully without output.")) return outputs - error_details = stderr or "Unknown sandbox error" + error_details = (stderr or "Unknown sandbox error") if include_detailed_errors else None outputs.append( Content.from_error( message="Execution error", @@ -1257,7 +1262,7 @@ def _build_execution_contents( return outputs -def _make_sandbox_callback(tool_obj: FunctionTool) -> Callable[..., Any]: +def _make_sandbox_callback(tool_obj: FunctionTool, *, include_detailed_errors: bool = False) -> Callable[..., Any]: sandbox_tool = copy(tool_obj) def _callback(**kwargs: Any) -> Any: @@ -1282,6 +1287,8 @@ def _run() -> None: worker.start() worker.join() if error_box: + if not include_detailed_errors and isinstance(error_box[0], Exception): + raise RuntimeError("Host tool execution failed.") from None raise error_box[0] # Return the raw value. The Hyperlight FFI marshals primitives (dict, list, # str, int, float, bool, None) natively into the guest, and falls back to @@ -1418,6 +1425,7 @@ def execute(self, *, config: _RunConfig, code: str) -> list[Content]: max_output_files=config.max_output_files, max_output_file_bytes=config.max_output_file_bytes, max_output_total_bytes=config.max_output_total_bytes, + include_detailed_errors=config.include_detailed_errors, ) finally: if not entry.worker.is_reusable(): @@ -1487,7 +1495,10 @@ def _create_sandbox() -> Any: def _configure_sandbox(*, sandbox: Any, expand_missing_scheme: bool) -> None: for tool_obj in config.tools: - sandbox.register_tool(tool_obj.name, _make_sandbox_callback(tool_obj)) + sandbox.register_tool( + tool_obj.name, + _make_sandbox_callback(tool_obj, include_detailed_errors=config.include_detailed_errors), + ) for allowed_domain in config.allowed_domains: for target in _allowed_domain_registration_targets( @@ -1573,6 +1584,8 @@ class HyperlightExecuteCodeTool(FunctionTool): or ``"json"``, globally or mapped by exact, case-sensitive tool name. Missing names use compact format. Schemas that cannot be represented faithfully in compact form use JSON Schema. Mappings are copied, including entries for tools registered later. + include_detailed_errors: Expose host exception and execution diagnostics only for a trusted + channel. Defaults to ``False`` and is independent of outer-client error/logging settings. """ def __init__( @@ -1580,6 +1593,7 @@ def __init__( *, tools: FunctionTool | Callable[..., Any] | Sequence[FunctionTool | Callable[..., Any]] | None = None, tool_description_format: _ToolDescriptionFormat = "compact", + include_detailed_errors: bool = False, approval_mode: ApprovalMode | None = None, workspace_root: str | Path | None = None, file_mounts: FileMountInput | Sequence[FileMountInput] | None = None, @@ -1593,6 +1607,8 @@ def __init__( _registry: SandboxRuntime | None = None, _fides_warned_metadata: list[tuple[FunctionTool, dict[str, str]]] | None = None, ) -> None: + if not isinstance(include_detailed_errors, bool): + raise TypeError("include_detailed_errors must be a bool.") normalized_description_format = _normalize_tool_description_format(tool_description_format) max_output_files = _validate_positive_integer(name="max_output_files", value=max_output_files) max_output_file_bytes = _validate_positive_integer(name="max_output_file_bytes", value=max_output_file_bytes) @@ -1608,6 +1624,7 @@ def __init__( self._tool_description_format: _NormalizedToolDescriptionFormat = normalized_description_format self._registry = _registry or _SandboxRegistry() self._default_approval_mode: ApprovalMode = approval_mode or "never_require" + self._include_detailed_errors = include_detailed_errors self._workspace_root = _resolve_workspace_root(workspace_root) self._backend: str = backend self._module: str | None = module @@ -1785,6 +1802,7 @@ def create_run_tool(self) -> HyperlightExecuteCodeTool: tools=tools, tool_description_format=self._tool_description_format, approval_mode=self._default_approval_mode, + include_detailed_errors=self._include_detailed_errors, workspace_root=self._workspace_root, file_mounts=file_mounts or None, allowed_domains=allowed_domains or None, @@ -1805,6 +1823,7 @@ def build_serializable_state(self) -> dict[str, Any]: "module": config.module, "module_path": config.module_path, "approval_mode": config.approval_mode, + "include_detailed_errors": config.include_detailed_errors, "max_output_files": config.max_output_files, "max_output_file_bytes": config.max_output_file_bytes, "max_output_total_bytes": config.max_output_total_bytes, @@ -1852,6 +1871,7 @@ def _build_run_config(self) -> _RunConfig: max_output_files = self._max_output_files max_output_file_bytes = self._max_output_file_bytes max_output_total_bytes = self._max_output_total_bytes + include_detailed_errors = self._include_detailed_errors approval_mode = _resolve_execute_code_approval_mode( base_approval_mode=self._default_approval_mode, tools=managed_tools, @@ -1880,6 +1900,7 @@ def _build_run_config(self) -> _RunConfig: max_output_files=max_output_files, max_output_file_bytes=max_output_file_bytes, max_output_total_bytes=max_output_total_bytes, + include_detailed_errors=include_detailed_errors, ) async def _run_code(self, *, code: str) -> list[Content]: diff --git a/python/packages/hyperlight/agent_framework_hyperlight/_provider.py b/python/packages/hyperlight/agent_framework_hyperlight/_provider.py index 119e7fbcdc4..a72fd032f44 100644 --- a/python/packages/hyperlight/agent_framework_hyperlight/_provider.py +++ b/python/packages/hyperlight/agent_framework_hyperlight/_provider.py @@ -29,6 +29,8 @@ class HyperlightCodeActProvider(ContextProvider): ``"compact"`` (default) or ``"json"``, globally or mapped by exact, case-sensitive tool name. Missing names use compact format; rich schemas fall back to full JSON Schema. Mappings are copied, including entries for tools registered later. + include_detailed_errors: Expose host exception and execution diagnostics only for a trusted + channel. Defaults to ``False`` and is independent of outer-client error/logging settings. """ DEFAULT_SOURCE_ID = "hyperlight_codeact" @@ -39,6 +41,7 @@ def __init__( *, tools: FunctionTool | Callable[..., Any] | Sequence[FunctionTool | Callable[..., Any]] | None = None, tool_description_format: _ToolDescriptionFormat = "compact", + include_detailed_errors: bool = False, approval_mode: ApprovalMode | None = None, workspace_root: str | Path | None = None, file_mounts: FileMountInput | Sequence[FileMountInput] | None = None, @@ -55,6 +58,7 @@ def __init__( self._execute_code_tool = HyperlightExecuteCodeTool( tools=tools, tool_description_format=tool_description_format, + include_detailed_errors=include_detailed_errors, approval_mode=approval_mode, workspace_root=workspace_root, file_mounts=file_mounts, diff --git a/python/packages/hyperlight/tests/hyperlight/conftest.py b/python/packages/hyperlight/tests/hyperlight/conftest.py new file mode 100644 index 00000000000..d94bdb75583 --- /dev/null +++ b/python/packages/hyperlight/tests/hyperlight/conftest.py @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +import asyncio + +import pytest +from agent_framework import FunctionTool + + +@pytest.fixture +def host_error_detail() -> str: + return "synthetic backend diagnostic" + + +@pytest.fixture +def failing_host_tool(request: pytest.FixtureRequest, host_error_detail: str) -> FunctionTool: + def fail() -> str: + raise ValueError(host_error_detail) + + async def fail_async() -> str: + await asyncio.sleep(0) + raise ValueError(host_error_detail) + + return FunctionTool( + name="failing_host_tool", + description="Raise a synthetic diagnostic.", + func=fail_async if request.param else fail, + ) + + +@pytest.fixture +def cancelled_host_tool() -> FunctionTool: + async def cancel() -> str: + raise asyncio.CancelledError + + return FunctionTool(name="cancelled_host_tool", func=cancel) diff --git a/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py b/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py index 2e4cfcb1862..0255cb608e6 100644 --- a/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py +++ b/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py @@ -39,15 +39,14 @@ ResponseStream, tool, ) +from agent_framework_hyperlight import AllowedDomain, FileMount, HyperlightCodeActProvider, HyperlightExecuteCodeTool +from agent_framework_hyperlight import _execute_code_tool as execute_code_module +from agent_framework_hyperlight import _instructions as instructions_module from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from pytest import fixture -from agent_framework_hyperlight import AllowedDomain, FileMount, HyperlightCodeActProvider, HyperlightExecuteCodeTool -from agent_framework_hyperlight import _execute_code_tool as execute_code_module -from agent_framework_hyperlight import _instructions as instructions_module - def _hyperlight_integration_static_skip_reason() -> str | None: if sys.platform not in {"linux", "win32"}: @@ -1477,7 +1476,7 @@ def fail_post_cleanup(path: str | os.PathLike[str]) -> Any: recovered = await execute_code.invoke(arguments={"code": "fail"}) assert not any(item.type == "data" for item in recovered) - assert any(item.type == "error" and item.error_details == "sandbox boom" for item in recovered) + assert any(item.type == "error" and item.error_details is None for item in recovered) assert len(_FakeSandbox.instances) == 2 assert registry._entries == {} finally: @@ -2438,22 +2437,29 @@ def blocking_cleanup_temp_dirs(entry_to_clean: Any) -> None: second_result = await execute_code.invoke(arguments={"code": "fail"}) assert not any(item.type == "data" for item in second_result) - assert any(item.type == "error" and item.error_details == "sandbox boom" for item in second_result) + assert any(item.type == "error" and item.error_details is None for item in second_result) assert len(_FakeSandbox.instances) == 2 finally: release_cleanup.set() _close_execute_code_registry(execute_code) -async def test_execute_code_tool_failure_returns_error_content(monkeypatch: pytest.MonkeyPatch) -> None: +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_execute_code_tool_failure_returns_error_content( + monkeypatch: pytest.MonkeyPatch, include_detailed_errors: bool +) -> None: _FakeSandbox.instances.clear() monkeypatch.setattr(execute_code_module, "_load_sandbox_class", lambda: _FakeSandbox) - execute_code = HyperlightExecuteCodeTool() - result = await execute_code.invoke(arguments={"code": "fail"}) + execute_code = HyperlightExecuteCodeTool(include_detailed_errors=include_detailed_errors) + try: + result = await execute_code.invoke(arguments={"code": "fail"}) - assert result[0].type == "error" - assert result[0].error_details == "sandbox boom" + assert result[0].type == "error" + assert result[0].error_details == ("sandbox boom" if include_detailed_errors else None) + assert ("sandbox boom" in json.dumps([item.to_dict() for item in result])) is include_detailed_errors + finally: + _close_execute_code_registry(execute_code) async def test_execute_code_tool_retries_allowed_domains_with_urls_when_backend_rejects_host_targets( @@ -2829,7 +2835,7 @@ async def test_execute_code_tool_rejects_sparse_file_with_real_sandbox(tmp_path: errors = [item for item in contents if item.type == "error"] assert len(errors) == 1 assert errors[0].message == "Execution error" - assert errors[0].error_details + assert errors[0].message == "Execution error" @pytest.mark.flaky @@ -3258,11 +3264,88 @@ def boom(x: int) -> int: """Always fails.""" raise RuntimeError("nope") - callback = execute_code_module._make_sandbox_callback(boom) + callback = execute_code_module._make_sandbox_callback(boom, include_detailed_errors=True) with pytest.raises(RuntimeError, match="nope"): callback(x=1) +@pytest.mark.parametrize("failing_host_tool", [False, True], indirect=True) +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_sandbox_callback_redacts_errors_unless_explicitly_enabled( + failing_host_tool: FunctionTool, host_error_detail: str, include_detailed_errors: bool +) -> None: + callback = execute_code_module._make_sandbox_callback( + failing_host_tool, include_detailed_errors=include_detailed_errors + ) + + with pytest.raises(ValueError if include_detailed_errors else RuntimeError) as error: + callback() + + assert (host_error_detail in str(error.value)) is include_detailed_errors + if not include_detailed_errors: + assert str(error.value) == "Host tool execution failed." + assert error.value.__suppress_context__ + + +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_error_policy_preserves_callback_cancellation( + cancelled_host_tool: FunctionTool, include_detailed_errors: bool +) -> None: + callback = execute_code_module._make_sandbox_callback( + cancelled_host_tool, include_detailed_errors=include_detailed_errors + ) + + with pytest.raises(asyncio.CancelledError): + callback() + + +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +def test_error_policy_is_snapshotted_and_part_of_the_sandbox_cache_key(include_detailed_errors: bool) -> None: + registry = _FakeRuntime() + codeact = HyperlightExecuteCodeTool(include_detailed_errors=include_detailed_errors, _registry=registry) + provider = HyperlightCodeActProvider(include_detailed_errors=include_detailed_errors, _registry=registry) + opposite = HyperlightExecuteCodeTool(include_detailed_errors=not include_detailed_errors, _registry=registry) + + assert codeact.create_run_tool().build_serializable_state()["include_detailed_errors"] is include_detailed_errors + assert ( + provider._execute_code_tool.create_run_tool().build_serializable_state()["include_detailed_errors"] + is include_detailed_errors + ) + assert codeact._build_run_config().cache_key() != opposite._build_run_config().cache_key() + assert set(codeact.parameters()["properties"]) == {"code"} + + +@pytest.mark.parametrize("constructor", [HyperlightExecuteCodeTool, HyperlightCodeActProvider]) +@pytest.mark.parametrize("value", ["false", 1, None]) +def test_detailed_error_option_rejects_non_boolean_values(constructor: Any, value: Any) -> None: + with pytest.raises(TypeError, match="include_detailed_errors must be a bool"): + constructor(include_detailed_errors=value) + + +@pytest.mark.integration +@skip_if_hyperlight_integration_tests_disabled +@pytest.mark.parametrize("failing_host_tool", [False, True], indirect=True) +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_host_error_policy_with_real_sandbox( + failing_host_tool: FunctionTool, host_error_detail: str, include_detailed_errors: bool +) -> None: + _skip_if_hyperlight_integration_runtime_disabled() + execute_code = HyperlightExecuteCodeTool(tools=[failing_host_tool], include_detailed_errors=include_detailed_errors) + try: + result = await execute_code.invoke( + arguments={ + "code": "try:\n call_tool('failing_host_tool')\nexcept Exception as exc:\n print(str(exc))" + }, + skip_parsing=True, + ) + serialized = json.dumps([content.to_dict() for content in result]) + assert (host_error_detail in serialized) is include_detailed_errors + if not include_detailed_errors: + assert "Host tool execution failed." in serialized + finally: + _close_execute_code_registry(execute_code) + + class _OwnerThreadTrackedResult: """Fake sandbox.run() return value that mirrors a PyO3 ``unsendable`` object's Drop. diff --git a/python/packages/monty/AGENTS.md b/python/packages/monty/AGENTS.md index e366b78750b..390b7de63bb 100644 --- a/python/packages/monty/AGENTS.md +++ b/python/packages/monty/AGENTS.md @@ -36,6 +36,10 @@ from agent_framework.monty import ( - `file_mounts` — sequence of `FileMountInput` (str shorthand, `(host_path, mount_path)` tuple, or `FileMount`) - `resource_limits` — Monty `ResourceLimits` TypedDict +- `include_detailed_errors` — keyword-only Boolean, default `False`. Host failures become generic + `RuntimeError`s and execution errors omit raw details. Trusted opt-in preserves original diagnostics. + The flag is not a code input, is independent of outer-client error/logging settings, and is copied + into run snapshots and serializable state. - `tool_description_format` — keyword-only `Literal["compact", "json"] | Mapping[str, Literal["compact", "json"]] = "compact"`. Selects parameter documentation in both the description and instructions. diff --git a/python/packages/monty/README.md b/python/packages/monty/README.md index 6b7603acde6..b9ec3e9e82f 100644 --- a/python/packages/monty/README.md +++ b/python/packages/monty/README.md @@ -22,6 +22,24 @@ Monty-backed CodeAct integrations for Microsoft Agent Framework. > enforce authorization and destination restrictions inside host functions. > See the [FIDES guide](../../samples/02-agents/security/FIDES_DEVELOPER_GUIDE.md). +## Error diagnostics + +Host tool failures raise a generic `RuntimeError("Host tool execution failed.")` +inside generated code by default. Execution error contents omit raw diagnostic +details. This applies to both direct tool calls and `call_tool(...)`, including +failures that generated code catches and prints or returns. + +Both `MontyCodeActProvider` and `MontyExecuteCodeTool` accept the host-side option +`include_detailed_errors=True` to retain original host exception types/messages +and execution diagnostics. Enable it only for a trusted channel: exception +messages can contain credentials or other sensitive data. This option is not a +parameter of `execute_code` and is independent of the outer chat client's error +policy and sensitive-data logging. Run snapshots and serialized state retain it. + +Code that catches a specific host exception type, such as `ValueError`, must +instead handle the generic `RuntimeError` in the default mode. Failure signaling +and successful tool results are unchanged. + ## Installation ```bash diff --git a/python/packages/monty/agent_framework_monty/_execute_code_tool.py b/python/packages/monty/agent_framework_monty/_execute_code_tool.py index 38fe9db8d0e..fc2cbb6aa56 100644 --- a/python/packages/monty/agent_framework_monty/_execute_code_tool.py +++ b/python/packages/monty/agent_framework_monty/_execute_code_tool.py @@ -260,6 +260,11 @@ class MontyExecuteCodeTool(FunctionTool): All mutators (``add_tools``, ``add_file_mounts`` etc.) must be called from the same task/thread that owns the tool. Monty itself runs on the event loop, so no internal locking is needed. + + Keyword Args: + include_detailed_errors: Include host exception types/messages and execution error details. + Defaults to ``False``. Enable only for a trusted channel; diagnostics can contain secrets. + This setting is independent of the outer client's error and sensitive-logging settings. """ def __init__( @@ -271,8 +276,11 @@ def __init__( file_mounts: FileMountInput | Sequence[FileMountInput] | None = None, resource_limits: dict[str, Any] | None = None, tool_description_format: _ToolDescriptionFormat = "compact", + include_detailed_errors: bool = False, _fides_warned_metadata: list[tuple[FunctionTool, dict[str, str]]] | None = None, ) -> None: + if not isinstance(include_detailed_errors, bool): + raise TypeError("include_detailed_errors must be a bool.") super().__init__( name=EXECUTE_CODE_TOOL_NAME, description=EXECUTE_CODE_TOOL_DESCRIPTION, @@ -281,6 +289,7 @@ def __init__( input_model=EXECUTE_CODE_INPUT_SCHEMA, ) self._default_approval_mode: ApprovalMode = approval_mode or "never_require" + self._include_detailed_errors = include_detailed_errors self._tool_description_format: _NormalizedToolDescriptionFormat = _normalize_tool_description_format( tool_description_format ) @@ -413,6 +422,7 @@ def create_run_tool(self) -> MontyExecuteCodeTool: file_mounts=list(self._file_mounts.values()) or None, resource_limits=self._resource_limits, tool_description_format=self._tool_description_format, + include_detailed_errors=self._include_detailed_errors, ) def build_serializable_state(self) -> dict[str, Any]: @@ -425,6 +435,7 @@ def build_serializable_state(self) -> dict[str, Any]: return { "runtime": "monty", "approval_mode": approval_mode, + "include_detailed_errors": self._include_detailed_errors, "tool_names": [tool_obj.name for tool_obj in self._managed_tools], "tool_description_format": ( dict(self._tool_description_format) @@ -489,6 +500,7 @@ def _effective_mounts(self) -> list[FileMount]: return mounts async def _run_code(self, *, code: str) -> list[Content]: + include_detailed_errors = self._include_detailed_errors tools = list(self._managed_tools) mounts = self._effective_mounts() @@ -505,6 +517,7 @@ async def _run_code(self, *, code: str) -> list[Content]: type_stubs=type_stubs, mounts=[_to_monty_mount(mount) for mount in mounts] or None, resource_limits=self._resource_limits, + include_detailed_errors=include_detailed_errors, ) try: @@ -513,7 +526,7 @@ async def _run_code(self, *, code: str) -> list[Content]: return [ Content.from_error( message="Execution error", - error_details=f"{type(exc).__name__}: {exc}", + error_details=f"{type(exc).__name__}: {exc}" if include_detailed_errors else None, ), ] diff --git a/python/packages/monty/agent_framework_monty/_monty_bridge.py b/python/packages/monty/agent_framework_monty/_monty_bridge.py index a7f9291dd9b..008d3ed177b 100644 --- a/python/packages/monty/agent_framework_monty/_monty_bridge.py +++ b/python/packages/monty/agent_framework_monty/_monty_bridge.py @@ -39,8 +39,10 @@ def _ensure_json_value(value: Any) -> Any: raise ValueError(f"Value of type {type(value).__name__} is not JSON-safe.") -def _external_error(exc: Exception) -> dict[str, str]: - return {"exc_type": type(exc).__name__, "message": str(exc)} +def _external_error(exc: Exception, *, include_detailed_errors: bool = False) -> dict[str, str]: + if include_detailed_errors: + return {"exc_type": type(exc).__name__, "message": str(exc)} + return {"exc_type": "RuntimeError", "message": "Host tool execution failed."} def _parse_call_tool(args: tuple[Any, ...], kwargs: dict[str, Any]) -> tuple[str, dict[str, Any]]: @@ -210,11 +212,13 @@ def __init__( type_stubs: str | None = None, mounts: Sequence[Any] | None = None, resource_limits: dict[str, Any] | None = None, + include_detailed_errors: bool = False, ) -> None: self.tool_map: dict[str, Callable[..., Any]] = dict(tool_map) self.type_stubs: str | None = type_stubs self._mounts = tuple(mounts) if mounts else () self._resource_limits = resource_limits + self._include_detailed_errors = include_detailed_errors self._pending_calls: dict[int, tuple[str, dict[str, Any]]] = {} async def run(self, code: str) -> dict[str, Any]: @@ -301,7 +305,7 @@ def _schedule_call_tool(self, snapshot: Any) -> Any: raise ValueError(f"Tool {name!r} is not registered. Available tools: {allowed}") self._pending_calls[int(snapshot.call_id)] = (name, kwargs) except Exception as exc: - return snapshot.resume(_external_error(exc)) + return snapshot.resume(_external_error(exc, include_detailed_errors=True)) return snapshot.resume({"future": ...}) async def _handle_future(self, snapshot: Any) -> Any: @@ -331,4 +335,4 @@ async def _invoke_tool(self, cid: int, name: str, kwargs: dict[str, Any]) -> tup result = await self.tool_map[name](**kwargs) return cid, {"return_value": _ensure_json_value(result)} except Exception as exc: - return cid, _external_error(exc) + return cid, _external_error(exc, include_detailed_errors=self._include_detailed_errors) diff --git a/python/packages/monty/agent_framework_monty/_provider.py b/python/packages/monty/agent_framework_monty/_provider.py index 81192a038ee..4b4a79fb90d 100644 --- a/python/packages/monty/agent_framework_monty/_provider.py +++ b/python/packages/monty/agent_framework_monty/_provider.py @@ -31,6 +31,10 @@ class MontyCodeActProvider(ContextProvider): names to either format. Missing names use compact; rich schemas fall back to full JSON Schema. Mappings are copied on construction and for each run, retaining entries for tools registered later. + + Keyword Args: + include_detailed_errors: Expose host exception and execution diagnostics only for a trusted + channel. Defaults to ``False`` and is independent of outer-client error/logging settings. """ DEFAULT_SOURCE_ID = "monty_codeact" @@ -45,6 +49,7 @@ def __init__( file_mounts: FileMountInput | Sequence[FileMountInput] | None = None, resource_limits: dict[str, Any] | None = None, tool_description_format: _ToolDescriptionFormat = "compact", + include_detailed_errors: bool = False, ) -> None: super().__init__(source_id) self._execute_code_tool = MontyExecuteCodeTool( @@ -54,6 +59,7 @@ def __init__( file_mounts=file_mounts, resource_limits=resource_limits, tool_description_format=tool_description_format, + include_detailed_errors=include_detailed_errors, ) def add_tools( diff --git a/python/packages/monty/tests/monty/conftest.py b/python/packages/monty/tests/monty/conftest.py new file mode 100644 index 00000000000..d94bdb75583 --- /dev/null +++ b/python/packages/monty/tests/monty/conftest.py @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +import asyncio + +import pytest +from agent_framework import FunctionTool + + +@pytest.fixture +def host_error_detail() -> str: + return "synthetic backend diagnostic" + + +@pytest.fixture +def failing_host_tool(request: pytest.FixtureRequest, host_error_detail: str) -> FunctionTool: + def fail() -> str: + raise ValueError(host_error_detail) + + async def fail_async() -> str: + await asyncio.sleep(0) + raise ValueError(host_error_detail) + + return FunctionTool( + name="failing_host_tool", + description="Raise a synthetic diagnostic.", + func=fail_async if request.param else fail, + ) + + +@pytest.fixture +def cancelled_host_tool() -> FunctionTool: + async def cancel() -> str: + raise asyncio.CancelledError + + return FunctionTool(name="cancelled_host_tool", func=cancel) diff --git a/python/packages/monty/tests/monty/test_monty_codeact.py b/python/packages/monty/tests/monty/test_monty_codeact.py index 4f2cd10e1f8..d3e81887ede 100644 --- a/python/packages/monty/tests/monty/test_monty_codeact.py +++ b/python/packages/monty/tests/monty/test_monty_codeact.py @@ -28,7 +28,6 @@ from agent_framework import Content, FunctionTool, Message, tool from agent_framework._sessions import SessionContext from agent_framework.exceptions import ToolException - from agent_framework_monty import MontyCodeActProvider, MontyExecuteCodeTool from agent_framework_monty import _execute_code_tool as execute_code_module from agent_framework_monty import _instructions as instructions_module @@ -946,7 +945,10 @@ async def test_run_code_call_tool_with_unregistered_name_returns_error() -> None ) -async def test_run_code_returns_error_content_on_runtime_failure(monkeypatch: pytest.MonkeyPatch) -> None: +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_run_code_returns_error_content_on_runtime_failure( + monkeypatch: pytest.MonkeyPatch, include_detailed_errors: bool +) -> None: class _BoomBridge: def __init__(self, tool_map: Any, **_: Any) -> None: pass @@ -956,11 +958,32 @@ async def run(self, code: str) -> dict[str, Any]: monkeypatch.setattr(execute_code_module, "InlineCodeBridge", _BoomBridge) - monty_tool = MontyExecuteCodeTool() + monty_tool = MontyExecuteCodeTool(include_detailed_errors=include_detailed_errors) result = await monty_tool._run_code(code="x = 1") assert len(result) == 1 assert result[0].type == "error" - assert "boom" in (result[0].error_details or "") + assert ("boom" in (result[0].error_details or "")) is include_detailed_errors + assert ("boom" in json.dumps([item.to_dict() for item in result])) is include_detailed_errors + + +@pytest.mark.parametrize("constructor", [MontyExecuteCodeTool, MontyCodeActProvider]) +@pytest.mark.parametrize("value", ["false", 1, None]) +def test_detailed_error_option_rejects_non_boolean_values(constructor: Any, value: Any) -> None: + with pytest.raises(TypeError, match="include_detailed_errors must be a bool"): + constructor(include_detailed_errors=value) + + +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +def test_detailed_error_policy_is_snapshotted_and_not_a_code_parameter(include_detailed_errors: bool) -> None: + codeact = MontyExecuteCodeTool(include_detailed_errors=include_detailed_errors) + provider = MontyCodeActProvider(include_detailed_errors=include_detailed_errors) + + assert codeact.create_run_tool().build_serializable_state()["include_detailed_errors"] is include_detailed_errors + assert ( + provider._execute_code_tool.create_run_tool().build_serializable_state()["include_detailed_errors"] + is include_detailed_errors + ) + assert set(codeact.parameters()["properties"]) == {"code"} def test_build_execution_contents_handles_truncation_and_non_json_output() -> None: @@ -1836,7 +1859,10 @@ async def limited() -> int: results = await asyncio.gather(*(bridge._invoke_tool(index, "limited", {}) for index, bridge in enumerate(bridges))) assert sum(payload == {"return_value": 42} for _, payload in results) == 1 - assert sum(payload.get("exc_type") == "ToolException" for _, payload in results) == 7 + assert ( + sum(payload == {"exc_type": "RuntimeError", "message": "Host tool execution failed."} for _, payload in results) + == 7 + ) assert limited.invocation_count == 1 @@ -1893,8 +1919,7 @@ def limited() -> int: if run_index == 0: assert payload == {"return_value": 42} else: - assert payload["exc_type"] == "ToolException" - assert "maximum invocation limit" in payload["message"] + assert payload == {"exc_type": "RuntimeError", "message": "Host tool execution failed."} assert limited.invocation_count == 1 diff --git a/python/packages/monty/tests/monty/test_monty_codeact_integration.py b/python/packages/monty/tests/monty/test_monty_codeact_integration.py index c52fb0e00ee..4f851c5c32c 100644 --- a/python/packages/monty/tests/monty/test_monty_codeact_integration.py +++ b/python/packages/monty/tests/monty/test_monty_codeact_integration.py @@ -12,6 +12,7 @@ import asyncio import importlib.util +import json import time from typing import Annotated, Any from unittest.mock import MagicMock @@ -19,7 +20,6 @@ import pytest from agent_framework import Agent, Content, FunctionTool, Message, tool from agent_framework._sessions import SessionContext - from agent_framework_monty import MontyCodeActProvider, MontyExecuteCodeTool @@ -138,6 +138,62 @@ async def test_async_host_tool_is_awaited() -> None: assert any("ping" in text for text in texts) +@pytest.mark.parametrize("failing_host_tool", [False, True], indirect=True) +@pytest.mark.parametrize("call", ["await failing_host_tool()", "await call_tool('failing_host_tool')"]) +@pytest.mark.parametrize("handling", ["print", "value", "uncaught"]) +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_host_error_details_require_trusted_opt_in( + failing_host_tool: FunctionTool, + host_error_detail: str, + call: str, + handling: str, + include_detailed_errors: bool, +) -> None: + if handling == "print": + code = f"try:\n {call}\nexcept Exception as exc:\n print(str(exc))" + elif handling == "value": + code = f"message = ''\ntry:\n {call}\nexcept Exception as exc:\n message = str(exc)\nmessage" + else: + code = call + execute_code = MontyExecuteCodeTool(tools=[failing_host_tool], include_detailed_errors=include_detailed_errors) + + contents = await execute_code._run_code(code=code) + + serialized = json.dumps([content.to_dict() for content in contents]) + assert (host_error_detail in serialized) is include_detailed_errors + if handling == "uncaught": + assert contents[0].type == "error" + elif not include_detailed_errors: + assert "Host tool execution failed." in serialized + + +@pytest.mark.parametrize("failing_host_tool", [True], indirect=True) +async def test_concurrent_executions_keep_error_policies_separate( + failing_host_tool: FunctionTool, host_error_detail: str +) -> None: + redacted = MontyExecuteCodeTool(tools=[failing_host_tool]) + detailed = MontyExecuteCodeTool(tools=[failing_host_tool], include_detailed_errors=True) + code = "try:\n await failing_host_tool()\nexcept Exception as exc:\n print(str(exc))" + + redacted_result, detailed_result = await asyncio.gather( + redacted._run_code(code=code), detailed._run_code(code=code) + ) + + assert host_error_detail not in "\n".join(_text_outputs(redacted_result)) + assert host_error_detail in "\n".join(_text_outputs(detailed_result)) + assert failing_host_tool.invocation_count == 2 + + +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_error_policy_preserves_host_cancellation( + cancelled_host_tool: FunctionTool, include_detailed_errors: bool +) -> None: + execute_code = MontyExecuteCodeTool(tools=[cancelled_host_tool], include_detailed_errors=include_detailed_errors) + + with pytest.raises(asyncio.CancelledError): + await execute_code._run_code(code="await cancelled_host_tool()") + + @pytest.mark.parametrize("is_async", [False, True]) @pytest.mark.parametrize("call", ["await limited()", "await call_tool('limited')"]) async def test_host_tool_invocation_limit_across_code_executions(is_async: bool, call: str) -> None: diff --git a/skills/agent-framework-builder-dotnet/references/integrations.md b/skills/agent-framework-builder-dotnet/references/integrations.md index d9a9aec7eef..515214941e4 100644 --- a/skills/agent-framework-builder-dotnet/references/integrations.md +++ b/skills/agent-framework-builder-dotnet/references/integrations.md @@ -40,6 +40,11 @@ otherwise see [Extensibility](extensibility.md#custom-providers). - Durable agents and workflows run on Azure Functions and Durable Task; see [Hosting and deployment](hosting-and-deployment.md). +LocalCodeAct and Hyperlight redact host tool exception diagnostics by default. Enable +`IncludeDetailedErrors` on their provider options only for a trusted channel. This is separate from the +outer `FunctionInvokingChatClient` error policy and sensitive-data logging; outer redaction cannot repair +diagnostics already caught and returned by generated code. + For AG-UI, remember that thread and run IDs are correlation identifiers, not authentication boundaries. The application owns authorization, durable snapshots, scaling, and rendering policy. diff --git a/skills/agent-framework-builder-python/references/integrations.md b/skills/agent-framework-builder-python/references/integrations.md index 415d4b3336e..6690a7d41af 100644 --- a/skills/agent-framework-builder-python/references/integrations.md +++ b/skills/agent-framework-builder-python/references/integrations.md @@ -46,6 +46,11 @@ application owns authorization, durable snapshots, scaling, and rendering policy ## Code execution +Monty and Hyperlight redact host tool exception diagnostics by default. Enable +`include_detailed_errors=True` only for a trusted channel. This host-side setting +is separate from the outer chat client's error policy and sensitive-data logging; +outer redaction cannot repair a diagnostic already caught and returned by generated code. + For Monty, distinguish interpreter `resource_limits` from host-side attachment capture. Capture has per-call file-count and raw-byte limits shared across read-write mounts. Omitted attachments do not undo filesystem writes. Check the [Monty file-mount and resource-limit guidance](https://github.com/microsoft/agent-framework/blob/main/python/packages/monty/README.md#file-mounts-and-resource-limits) From 65b733a8de90b6adc44b30f97f22cfdc57c5669c Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Mon, 2 Nov 2026 14:17:34 +0100 Subject: [PATCH 2/3] Separate .NET CodeAct changes from Python PR --- .../HyperlightCodeActProviderOptions.cs | 10 -- .../Internal/SandboxExecutor.cs | 14 +-- .../Internal/ToolBridge.cs | 26 ++-- .../Microsoft.Agents.AI.Hyperlight/README.md | 14 --- .../Internal/CodeExecutor.cs | 8 +- .../Internal/ProcessBridge.cs | 9 +- .../LocalCodeActProvider.cs | 5 +- .../LocalCodeActProviderOptions.cs | 10 -- .../LocalExecuteCodeFunction.cs | 2 +- .../README.md | 13 -- .../CodeActEndToEndTests.cs | 39 ------ .../SandboxExecutorTests.cs | 6 - .../ToolBridgeTests.cs | 29 ++--- ...ocalExecuteCodeFunctionIntegrationTests.cs | 116 +----------------- .../references/integrations.md | 5 - 15 files changed, 28 insertions(+), 278 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs b/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs index d394d328259..93e5a09c395 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs @@ -81,16 +81,6 @@ public static HyperlightCodeActProviderOptions CreateForJavaScript() /// public CodeActApprovalMode ApprovalMode { get; set; } = CodeActApprovalMode.NeverRequire; - /// - /// Gets or sets whether host exception details are exposed to generated code and in execution errors. - /// Defaults to . - /// - /// - /// Enable only for a trusted channel: diagnostics can contain sensitive host data. - /// This is independent of the outer chat client's error policy and sensitive-data logging. - /// - public bool IncludeDetailedErrors { get; set; } - /// /// Gets or sets an optional host directory exposed to the sandbox as its /// /input directory. diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs index 6ae6dfbb1d0..4c13b10d60a 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs @@ -52,7 +52,6 @@ public RunSnapshot( this.HeapSize = options.HeapSize; this.StackSize = options.StackSize; this.HostInputDirectory = options.HostInputDirectory; - this.IncludeDetailedErrors = options.IncludeDetailedErrors; this.ToolRegistryVersion = toolRegistryVersion; this.ConfigFingerprint = ComputeFingerprint( this.Tools, @@ -63,8 +62,7 @@ public RunSnapshot( this.Backend, this.ModulePath, this.HeapSize, - this.StackSize, - this.IncludeDetailedErrors); + this.StackSize); } public IReadOnlyList Tools { get; } @@ -83,8 +81,6 @@ public RunSnapshot( public string? HostInputDirectory { get; } - public bool IncludeDetailedErrors { get; } - public Guid ToolRegistryVersion { get; } /// @@ -104,8 +100,7 @@ internal static string ComputeFingerprint( SandboxBackend backend = SandboxBackend.JavaScript, string? modulePath = null, string? heapSize = null, - string? stackSize = null, - bool includeDetailedErrors = false) + string? stackSize = null) { var buffer = new ArrayBufferWriter(); using (var writer = new Utf8JsonWriter(buffer)) @@ -116,7 +111,6 @@ internal static string ComputeFingerprint( writer.WriteString("heapSize", heapSize); writer.WriteString("stackSize", stackSize); writer.WriteString("hostInputDirectory", hostInputDirectory); - writer.WriteBoolean("includeDetailedErrors", includeDetailedErrors); writer.WriteString("toolRegistryVersion", toolRegistryVersion); writer.WritePropertyName("tools"); @@ -240,7 +234,7 @@ public async Task ExecuteAsync(RunSnapshot snapshot, string code, Cancel catch (Exception ex) #pragma warning restore CA1031 { - return BuildErrorResult(snapshot.IncludeDetailedErrors ? ex.Message : "Code execution failed."); + return BuildErrorResult(ex.Message); } return BuildResult(result); @@ -306,7 +300,7 @@ private void BuildAndWarmUp(RunSnapshot snapshot) var sandbox = builder.Build(); // Tools must be registered before the first Run() call. - ToolBridge.RegisterAll(sandbox, snapshot.Tools, snapshot.IncludeDetailedErrors); + ToolBridge.RegisterAll(sandbox, snapshot.Tools); foreach (var allowedDomain in snapshot.AllowedDomains) { diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs index aab8072b6ed..b2f735474e4 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs @@ -21,33 +21,24 @@ internal static class ToolBridge /// Registers every entry against the provided /// as a raw JSON-in / JSON-out async tool. /// - public static void RegisterAll(Sandbox sandbox, IReadOnlyList tools, bool includeDetailedErrors = false) + public static void RegisterAll(Sandbox sandbox, IReadOnlyList tools) { foreach (var tool in tools) { - RegisterOne(sandbox, tool, includeDetailedErrors); + RegisterOne(sandbox, tool); } } - private static void RegisterOne(Sandbox sandbox, AIFunction tool, bool includeDetailedErrors) + private static void RegisterOne(Sandbox sandbox, AIFunction tool) => sandbox.RegisterToolAsync( tool.Name, - async (string argsJson) => await InvokeAsync(tool, argsJson, includeDetailedErrors).ConfigureAwait(false)); + async (string argsJson) => await InvokeAsync(tool, argsJson).ConfigureAwait(false)); - internal static async Task InvokeAsync(AIFunction tool, string argsJson, bool includeDetailedErrors = false) + internal static async Task InvokeAsync(AIFunction tool, string argsJson) { try { - IDictionary arguments; - try - { - arguments = ParseArguments(argsJson); - } - catch (Exception ex) when (ex is JsonException or ArgumentException) - { - return SerializeError(ex.Message); - } - + var arguments = ParseArguments(argsJson); var result = await tool.InvokeAsync(new AIFunctionArguments(arguments)).ConfigureAwait(false); return SerializeResult(result); } @@ -55,13 +46,10 @@ internal static async Task InvokeAsync(AIFunction tool, string argsJson, catch (Exception ex) #pragma warning restore CA1031 { - return SerializeError(includeDetailedErrors ? ex.Message : "Host tool execution failed."); + return JsonSerializer.Serialize(new HyperlightToolError(ex.Message), HyperlightJsonContext.Default.HyperlightToolError); } } - private static string SerializeError(string message) => - JsonSerializer.Serialize(new HyperlightToolError(message), HyperlightJsonContext.Default.HyperlightToolError); - internal static IDictionary ParseArguments(string argsJson) { if (string.IsNullOrWhiteSpace(argsJson)) diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md b/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md index bdd7f3063bb..8596db97644 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md @@ -35,20 +35,6 @@ Both surfaces support: the meantime. * A Hyperlight Python guest module when using `SandboxBackend.Wasm`. -## Error diagnostics - -Host tool failures return a generic JSON error by default. Host execution -exceptions returned by the executor also use a generic message. -`HyperlightCodeActProviderOptions.IncludeDetailedErrors` defaults to `false` -for both the provider and standalone function. Set it to `true` only for a -trusted channel: host exception messages can contain sensitive data. - -This option is independent of the outer `FunctionInvokingChatClient` error policy -and sensitive-data logging. Outer error handling cannot redact diagnostics that -generated code has already caught and included in normal output. Run snapshots -capture the policy, and sandbox configuration fingerprints include it so cached -callbacks cannot retain a different disclosure policy. - ## Network access and guest packages `HyperlightCodeActProviderOptions.AllowedDomains` controls outbound requests diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/CodeExecutor.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/CodeExecutor.cs index abbf23dcfe2..51d6a7136d1 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/CodeExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/CodeExecutor.cs @@ -39,18 +39,15 @@ public CodeExecutor( /// Immutable snapshot of provider state captured at the start of an invocation. public sealed class RunSnapshot { - public RunSnapshot(IReadOnlyList tools, IReadOnlyList fileMounts, bool includeDetailedErrors = false) + public RunSnapshot(IReadOnlyList tools, IReadOnlyList fileMounts) { this.Tools = tools; this.FileMounts = fileMounts; - this.IncludeDetailedErrors = includeDetailedErrors; } public IReadOnlyList Tools { get; } public IReadOnlyList FileMounts { get; } - - public bool IncludeDetailedErrors { get; } } public async Task> ExecuteAsync(RunSnapshot snapshot, string code, CancellationToken cancellationToken) @@ -68,8 +65,7 @@ public async Task> ExecuteAsync(RunSnapshot snapshot, string cod snapshot.Tools, this._limits, this._environment, - this._workingDirectory, - snapshot.IncludeDetailedErrors); + this._workingDirectory); var result = await bridge.RunAsync(code, cancellationToken).ConfigureAwait(false); diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs index 6e234779b38..dee86154c38 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs @@ -31,7 +31,6 @@ internal sealed class ProcessBridge private readonly ProcessExecutionLimits _limits; private readonly IReadOnlyDictionary? _environment; private readonly string? _workingDirectory; - private readonly bool _includeDetailedErrors; public ProcessBridge( string pythonExecutable, @@ -39,8 +38,7 @@ public ProcessBridge( IReadOnlyList tools, ProcessExecutionLimits limits, IReadOnlyDictionary? environment, - string? workingDirectory, - bool includeDetailedErrors = false) + string? workingDirectory) { this._pythonExecutable = pythonExecutable; this._runnerScript = runnerScript; @@ -48,7 +46,6 @@ public ProcessBridge( this._limits = limits; this._environment = environment; this._workingDirectory = workingDirectory; - this._includeDetailedErrors = includeDetailedErrors; } /// Represents the parsed final result returned by the Python runner. @@ -267,9 +264,7 @@ await SendToolResponseAsync(process, callId, ok: false, result: null, #pragma warning restore CA1031 { await SendToolResponseAsync(process, callId, ok: false, result: null, - excType: this._includeDetailedErrors ? ex.GetType().Name : "RuntimeError", - excMessage: this._includeDetailedErrors ? ex.Message : "Host tool execution failed.", - cancellationToken).ConfigureAwait(false); + excType: ex.GetType().Name, excMessage: ex.Message, cancellationToken).ConfigureAwait(false); } } diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProvider.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProvider.cs index 4cdb3b38d86..33f9205f735 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProvider.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProvider.cs @@ -37,7 +37,6 @@ public sealed class LocalCodeActProvider : AIContextProvider, IDisposable private readonly CodeExecutor _executor; private readonly LocalCodeActApprovalMode _approvalMode; - private readonly bool _includeDetailedErrors; private readonly ConcurrentDictionary _tools = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _fileMounts = new(StringComparer.Ordinal); @@ -69,7 +68,6 @@ public LocalCodeActProvider(string pythonExecutablePath, LocalCodeActProviderOpt } this._approvalMode = options.ApprovalMode; - this._includeDetailedErrors = options.IncludeDetailedErrors; this._executor = new CodeExecutor( pythonExecutablePath, @@ -192,8 +190,7 @@ protected override ValueTask ProvideAIContextAsync(InvokingContext co this.ThrowIfDisposed(); snapshot = new CodeExecutor.RunSnapshot( this._tools.Values.ToList(), - this._fileMounts.Values.ToList(), - this._includeDetailedErrors); + this._fileMounts.Values.ToList()); FeatureUsageMarker.MarkUsed(); var approvalRequired = ComputeApprovalRequired(this._approvalMode, snapshot.Tools); diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs index 65c8a759dcd..51a161a20ea 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs @@ -31,16 +31,6 @@ public sealed class LocalCodeActProviderOptions /// public LocalCodeActApprovalMode ApprovalMode { get; set; } = LocalCodeActApprovalMode.NeverRequire; - /// - /// Gets or sets whether host exception details are exposed to generated code. - /// Defaults to . - /// - /// - /// Enable only for a trusted channel: diagnostics can contain sensitive host data. - /// This is independent of the outer chat client's error policy and sensitive-data logging. - /// - public bool IncludeDetailedErrors { get; set; } - /// /// Gets or sets the initial set of file mounts exposed to generated code. /// diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalExecuteCodeFunction.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalExecuteCodeFunction.cs index 5bb0804208d..a8b0fd34e22 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalExecuteCodeFunction.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalExecuteCodeFunction.cs @@ -72,7 +72,7 @@ public LocalExecuteCodeFunction(string pythonExecutablePath, LocalCodeActProvide options.Environment, options.WorkingDirectory); - this._snapshot = new CodeExecutor.RunSnapshot(tools, fileMounts, options.IncludeDetailedErrors); + this._snapshot = new CodeExecutor.RunSnapshot(tools, fileMounts); this._inner = AIFunctionFactory.Create( this.ExecuteCodeAsync, new AIFunctionFactoryOptions diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md index d8d3b5cd5cf..e4ae51298bf 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md @@ -37,19 +37,6 @@ using var provider = new LocalCodeActProvider("/usr/bin/python3", options); // Register provider with your AIAgent's context providers. ``` -## Error diagnostics - -Host tool failures send a generic failed response to generated Python code by -default. `LocalCodeActProviderOptions.IncludeDetailedErrors` defaults to `false` -for both the provider and standalone function. Set it to `true` only for a -trusted channel: original exception types and messages can contain sensitive data. - -The policy is captured at construction and passed through run snapshots. It is -independent of the outer `FunctionInvokingChatClient` error policy and -sensitive-data logging. Outer error handling still applies to uncaught failures, -but cannot redact diagnostics that generated code has already caught and printed -or returned. Successful results and failure signaling are unchanged. - ## What the Package Controls - **AST validation** (default on): Validates generated code against allow-lists diff --git a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs index 255e946a266..58b9ebceb6a 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs @@ -56,43 +56,4 @@ public async Task ExecuteCode_PythonPrint_ReturnsStdoutAsync() Assert.Contains("hi", doc.RootElement.GetProperty("stdout").GetString()!); Assert.Equal(0, doc.RootElement.GetProperty("exit_code").GetInt32()); } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task ExecuteCode_HostErrorDetailsRequireOptInAsync(bool includeDetailedErrors) - { - // Arrange - if (string.IsNullOrWhiteSpace(GuestPath)) - { - Assert.Skip(SkipReason); - return; - } - - const string Detail = "synthetic backend diagnostic"; - var options = HyperlightCodeActProviderOptions.CreateForWasm(GuestPath!); - options.IncludeDetailedErrors = includeDetailedErrors; - options.Tools = [AIFunctionFactory.Create( - (Func)(() => throw new InvalidOperationException(Detail)), name: "failing_tool")]; - using var function = new HyperlightExecuteCodeFunction(options); - - // Act - var result = await function.InvokeAsync(new AIFunctionArguments - { - ["code"] = """ - try: - print(call_tool('failing_tool')) - except Exception as exc: - print(str(exc)) - """, - }); - - // Assert - var json = Assert.IsType(result); - Assert.Equal(includeDetailedErrors, json.Contains(Detail, StringComparison.Ordinal)); - if (!includeDetailedErrors) - { - Assert.Contains("Host tool execution failed.", json, StringComparison.Ordinal); - } - } } diff --git a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs index 386a839e65d..5a528606b60 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs @@ -254,15 +254,12 @@ public void Fingerprint_DifferentSandboxOptions_DifferentFingerprints() [], [], [], hostInputDirectory: null, heapSize: "20Mi"); var differentStack = SandboxExecutor.RunSnapshot.ComputeFingerprint( [], [], [], hostInputDirectory: null, stackSize: "10Mi"); - var detailedErrors = SandboxExecutor.RunSnapshot.ComputeFingerprint( - [], [], [], hostInputDirectory: null, includeDetailedErrors: true); // Assert Assert.NotEqual(baseline, differentBackend); Assert.NotEqual(baseline, differentModule); Assert.NotEqual(baseline, differentHeap); Assert.NotEqual(baseline, differentStack); - Assert.NotEqual(baseline, detailedErrors); } [Fact] @@ -288,7 +285,6 @@ public void RunSnapshot_CapturesMutableInputs() HeapSize = "10Mi", StackSize = "5Mi", HostInputDirectory = "/host/input", - IncludeDetailedErrors = true, }; // Act @@ -300,7 +296,6 @@ public void RunSnapshot_CapturesMutableInputs() options.HeapSize = "20Mi"; options.StackSize = "10Mi"; options.HostInputDirectory = "/host/other"; - options.IncludeDetailedErrors = false; // Assert Assert.Single(snapshot.Tools); @@ -310,7 +305,6 @@ public void RunSnapshot_CapturesMutableInputs() Assert.Equal("10Mi", snapshot.HeapSize); Assert.Equal("5Mi", snapshot.StackSize); Assert.Equal("/host/input", snapshot.HostInputDirectory); - Assert.True(snapshot.IncludeDetailedErrors); } [Fact] diff --git a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs index d4bd084f3f8..9d94d719302 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs @@ -24,34 +24,20 @@ public async Task InvokeAsync_PassesArgumentsAndReturnsSerializedResultAsync() Assert.Equal("\"echo:hello\"", result); } - [Theory] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public async Task InvokeAsync_ReturnsErrorJsonOnExceptionAsync(bool includeDetailedErrors, bool isAsync) + [Fact] + public async Task InvokeAsync_ReturnsErrorJsonOnExceptionAsync() { // Arrange - const string Detail = "synthetic backend diagnostic"; - static int Boom() => throw new InvalidOperationException(Detail); - static async Task BoomAsync() - { - await Task.Yield(); - throw new InvalidOperationException(Detail); - } - - var tool = isAsync - ? AIFunctionFactory.Create(BoomAsync, name: "boom") - : AIFunctionFactory.Create(Boom, name: "boom"); + static int Boom() => throw new InvalidOperationException("nope"); + var tool = AIFunctionFactory.Create(Boom, name: "boom"); // Act - var result = await ToolBridge.InvokeAsync(tool, "{}", includeDetailedErrors); + var result = await ToolBridge.InvokeAsync(tool, "{}"); // Assert using var doc = JsonDocument.Parse(result); Assert.True(doc.RootElement.TryGetProperty("error", out var err)); - Assert.Equal(includeDetailedErrors ? Detail : "Host tool execution failed.", err.GetString()); - Assert.Equal(includeDetailedErrors, result.Contains(Detail, StringComparison.Ordinal)); + Assert.Contains("nope", err.GetString()!); } [Fact] @@ -80,7 +66,6 @@ public async Task InvokeAsync_NonObjectJson_ReturnsErrorAsync() // Assert using var doc = JsonDocument.Parse(result); - Assert.True(doc.RootElement.TryGetProperty("error", out var error)); - Assert.Contains("Tool arguments must be a JSON object.", error.GetString()!, StringComparison.Ordinal); + Assert.True(doc.RootElement.TryGetProperty("error", out _)); } } diff --git a/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs b/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs index 9db177a097c..c7118a452ab 100644 --- a/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs @@ -49,110 +49,6 @@ public async Task ExecuteCode_PrintsAndReturnsResultAsync() Assert.Contains("3", combined); } - [Theory] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public async Task ExecuteCode_HostErrorsRespectDetailedPolicyAsync(bool includeDetailedErrors, bool isAsync) - { - // Arrange - SkipIfNoPython(); - const string Detail = "synthetic backend diagnostic"; - static string Fail() => throw new InvalidOperationException(Detail); - static async Task FailAsync() - { - await Task.Yield(); - throw new InvalidOperationException(Detail); - } - - var tool = isAsync - ? AIFunctionFactory.Create(FailAsync, name: "failing_tool") - : AIFunctionFactory.Create(Fail, name: "failing_tool"); - var function = new LocalExecuteCodeFunction(s_python!, new LocalCodeActProviderOptions - { - Tools = [tool], - IncludeDetailedErrors = includeDetailedErrors, - }); - var args = new AIFunctionArguments - { - ["code"] = """ - try: - await failing_tool() - except Exception as exc: - print(str(exc)) - try: - await call_tool('failing_tool') - except Exception as exc: - print(str(exc)) - """, - }; - - // Act - var result = await function.InvokeAsync(args, CancellationToken.None); - - // Assert - var text = GetResultText(result); - Assert.Contains(includeDetailedErrors ? Detail : "Host tool execution failed.", text, StringComparison.Ordinal); - Assert.Equal(includeDetailedErrors, text.Contains(Detail, StringComparison.Ordinal)); - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task ExecuteCode_UncaughtHostErrorsPreserveFailureAsync(bool includeDetailedErrors) - { - // Arrange - SkipIfNoPython(); - const string Detail = "synthetic backend diagnostic"; - var tool = AIFunctionFactory.Create( - (Func)(() => throw new InvalidOperationException(Detail)), name: "failing_tool"); - var function = new LocalExecuteCodeFunction(s_python!, new LocalCodeActProviderOptions - { - Tools = [tool], - IncludeDetailedErrors = includeDetailedErrors, - }); - - // Act - var error = await Assert.ThrowsAsync(async () => - await function.InvokeAsync(new AIFunctionArguments { ["code"] = "await failing_tool()" })); - - // Assert - Assert.Contains(includeDetailedErrors ? Detail : "Host tool execution failed.", error.Message, StringComparison.Ordinal); - Assert.Equal(includeDetailedErrors, error.Message.Contains(Detail, StringComparison.Ordinal)); - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task Provider_HostErrorPolicyIsCapturedAtConstructionAsync(bool includeDetailedErrors) - { - // Arrange - SkipIfNoPython(); - const string Detail = "synthetic backend diagnostic"; - var tool = AIFunctionFactory.Create( - (Func)(() => throw new InvalidOperationException(Detail)), name: "failing_tool"); - var options = new LocalCodeActProviderOptions - { - Tools = [tool], - IncludeDetailedErrors = includeDetailedErrors, - }; - using var provider = new LocalCodeActProvider(s_python!, options); - options.IncludeDetailedErrors = !includeDetailedErrors; - var context = await provider.InvokingAsync( - new AIContextProvider.InvokingContext(new Moq.Mock().Object, session: null, new AIContext())); - var function = Assert.IsAssignableFrom(Assert.Single(context.Tools!)); - - // Act - var result = await function.InvokeAsync(new AIFunctionArguments - { - ["code"] = "try:\n await failing_tool()\nexcept Exception as exc:\n print(str(exc))", - }); - - // Assert - Assert.Equal(includeDetailedErrors, GetResultText(result).Contains(Detail, StringComparison.Ordinal)); - } - [Fact] public async Task ExecuteCode_AllowsAstValidatedFormattingAsync() { @@ -987,10 +883,8 @@ await call_tool('definitely_not_registered', x=1) Assert.DoesNotContain("NO_ERROR", combined); } - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task ExecuteCode_ToolThrowingExceptionPropagatesToGeneratedCodeAsync(bool includeDetailedErrors) + [Fact] + public async Task ExecuteCode_ToolThrowingExceptionPropagatesToGeneratedCodeAsync() { SkipIfNoPython(); @@ -1002,7 +896,6 @@ public async Task ExecuteCode_ToolThrowingExceptionPropagatesToGeneratedCodeAsyn var options = new LocalCodeActProviderOptions { Tools = new[] { faultyTool }, - IncludeDetailedErrors = includeDetailedErrors, }; var function = new LocalExecuteCodeFunction(s_python!, options); @@ -1019,9 +912,8 @@ await call_tool('faulty', message='boom') var result = await function.InvokeAsync(args, CancellationToken.None); var combined = GetResultText(result); Assert.Contains("GOT_ERROR", combined); - Assert.Contains(includeDetailedErrors ? "intentional: boom" : "Host tool execution failed.", combined); - Assert.Equal(includeDetailedErrors, combined.Contains("InvalidOperationException", StringComparison.Ordinal)); - Assert.Equal(includeDetailedErrors, combined.Contains("intentional: boom", StringComparison.Ordinal)); + Assert.Contains("InvalidOperationException", combined); + Assert.Contains("intentional: boom", combined); } [Fact] diff --git a/skills/agent-framework-builder-dotnet/references/integrations.md b/skills/agent-framework-builder-dotnet/references/integrations.md index 515214941e4..d9a9aec7eef 100644 --- a/skills/agent-framework-builder-dotnet/references/integrations.md +++ b/skills/agent-framework-builder-dotnet/references/integrations.md @@ -40,11 +40,6 @@ otherwise see [Extensibility](extensibility.md#custom-providers). - Durable agents and workflows run on Azure Functions and Durable Task; see [Hosting and deployment](hosting-and-deployment.md). -LocalCodeAct and Hyperlight redact host tool exception diagnostics by default. Enable -`IncludeDetailedErrors` on their provider options only for a trusted channel. This is separate from the -outer `FunctionInvokingChatClient` error policy and sensitive-data logging; outer redaction cannot repair -diagnostics already caught and returned by generated code. - For AG-UI, remember that thread and run IDs are correlation identifiers, not authentication boundaries. The application owns authorization, durable snapshots, scaling, and rendering policy. From f18141a01814976b441df8e2b922b497109e0b1e Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Mon, 2 Nov 2026 14:43:15 +0100 Subject: [PATCH 3/3] Redact Python Hyperlight lifecycle failures --- python/packages/hyperlight/README.md | 3 + .../_execute_code_tool.py | 42 +++++++++---- .../hyperlight/tests/hyperlight/conftest.py | 47 +++++++++++++++ .../hyperlight/test_hyperlight_codeact.py | 60 ++++++++++++++++++- 4 files changed, 137 insertions(+), 15 deletions(-) diff --git a/python/packages/hyperlight/README.md b/python/packages/hyperlight/README.md index 94bd4b9aeb5..ead73e45062 100644 --- a/python/packages/hyperlight/README.md +++ b/python/packages/hyperlight/README.md @@ -34,6 +34,9 @@ outer chat client's error policy and sensitive-data logging. The policy is captured in each run snapshot and serialized state. Sandbox cache keys include it, so detailed-error callbacks are not reused for redacted runs. +Initialization, snapshot restore, and execution failures follow the same policy. +Failed sandbox entries are retired; cancellation and system-exit signals still +propagate without becoming ordinary execution errors. ## Installation diff --git a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py index c0d88cb042b..645e0f2de45 100644 --- a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py +++ b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py @@ -175,7 +175,8 @@ class _SandboxWorker: are dropped on the wrong thread and PyO3 panics. To prevent this, every exception raised inside a worker closure is caught on the worker, the traceback is dropped while still on the worker thread, and a sanitized - copy (preserving message and exception type) is re-raised on the caller. + copy is re-raised on the caller. Ordinary failures expose their original + type and message only when detailed errors are enabled. """ __slots__ = ("_executor", "_initialized", "_reusable", "_sandbox", "_snapshot") @@ -188,20 +189,23 @@ def __init__(self, *, name: str = "hl-sandbox") -> None: self._initialized = False self._reusable = True - def _run_on_worker(self, fn: Callable[[], _T]) -> _T: + def _run_on_worker(self, fn: Callable[[], _T], *, include_detailed_errors: bool = False) -> _T: """Run ``fn`` on the worker thread; sanitize any exception's traceback there. If ``fn`` raises, the exception's ``__traceback__`` is dropped on the worker thread (so any PyO3 unsendable locals captured in frame locals are released - on the owner thread) and a fresh exception of the same type is raised on - the caller's thread carrying only the original message. + on the owner thread). Ordinary failures are replaced with a generic + exception unless detailed errors are enabled. Cancellation and other + system signals retain their type. """ def _wrapped() -> tuple[bool, Any]: try: return True, fn() except BaseException as exc: - exc_type = type(exc) + self._reusable = False + exc_type: type[BaseException] + exc_args: tuple[str, ...] # Capture args (usually (message,)) so the re-raised exception keeps the # original shape for types whose constructor doesn't accept a single str. # Coerce each arg to ``str`` on the worker thread: if a caller-supplied @@ -209,7 +213,12 @@ def _wrapped() -> tuple[bool, Any]: # unsendable object in args, forwarding it as-is would re-introduce the # same cross-thread Drop hazard the traceback nulling avoids. Strings # are always sendable. Fall back to the str() form if args is empty. - exc_args: tuple[str, ...] = tuple(str(a) for a in exc.args) if exc.args else (str(exc),) + if isinstance(exc, Exception) and not include_detailed_errors: + exc_type = RuntimeError + exc_args = ("Code execution failed.",) + else: + exc_type = type(exc) + exc_args = tuple(str(a) for a in exc.args) if exc.args else (str(exc),) # Drop the traceback on the worker thread so frame locals (which # may include PyO3 unsendable objects) are released here, not on # the caller thread that will receive the wrapped exception. @@ -226,11 +235,12 @@ def _wrapped() -> tuple[bool, Any]: # If the exception type's constructor rejects the captured args (rare), fall # back to a RuntimeError carrying the string form so we never lose the signal. try: - raise exc_type(*exc_args) + error = exc_type(*exc_args) except TypeError: raise RuntimeError(f"{exc_type.__name__}: {exc_args}") from None + raise error - def initialize(self, build_fn: Callable[[], tuple[Any, Any]]) -> None: + def initialize(self, build_fn: Callable[[], tuple[Any, Any]], *, include_detailed_errors: bool = False) -> None: """Build and install the sandbox+snapshot on the worker thread. ``build_fn`` is invoked with no arguments on the worker thread. It must @@ -246,7 +256,7 @@ def _init_on_worker() -> None: # Locals fall out of scope on the worker thread; the worker-local # attributes hold the only strong refs from now on. - self._run_on_worker(_init_on_worker) + self._run_on_worker(_init_on_worker, include_detailed_errors=include_detailed_errors) def execute( self, @@ -317,7 +327,7 @@ def _on_worker() -> list[Content]: if contents is not None and not any(item.type == "error" for item in contents): contents.append(_output_cleanup_error_content(exc)) - return self._run_on_worker(_on_worker) + return self._run_on_worker(_on_worker, include_detailed_errors=include_detailed_errors) def is_reusable(self) -> bool: return self._reusable @@ -1529,7 +1539,7 @@ def _build_sandbox() -> tuple[Any, Any]: worker = _SandboxWorker() try: - worker.initialize(_build_sandbox) + worker.initialize(_build_sandbox, include_detailed_errors=config.include_detailed_errors) except BaseException: worker.dispose() raise @@ -1904,5 +1914,11 @@ def _build_run_config(self) -> _RunConfig: ) async def _run_code(self, *, code: str) -> list[Content]: - config = self._build_run_config() - return await asyncio.to_thread(self._registry.execute, config=config, code=code) + include_detailed_errors = self._include_detailed_errors + try: + config = self._build_run_config() + return await asyncio.to_thread(self._registry.execute, config=config, code=code) + except Exception: + if include_detailed_errors: + raise + raise RuntimeError("Code execution failed.") from None diff --git a/python/packages/hyperlight/tests/hyperlight/conftest.py b/python/packages/hyperlight/tests/hyperlight/conftest.py index d94bdb75583..23d4bfa8515 100644 --- a/python/packages/hyperlight/tests/hyperlight/conftest.py +++ b/python/packages/hyperlight/tests/hyperlight/conftest.py @@ -3,6 +3,10 @@ from __future__ import annotations import asyncio +import threading +from collections.abc import Callable +from types import SimpleNamespace +from typing import Any import pytest from agent_framework import FunctionTool @@ -35,3 +39,46 @@ async def cancel() -> str: raise asyncio.CancelledError return FunctionTool(name="cancelled_host_tool", func=cancel) + + +@pytest.fixture +def lifecycle_sandbox_factory(host_error_detail: str) -> Callable[..., type[Any]]: + def factory(failure_stage: str, error_type: type[BaseException] = ValueError) -> type[Any]: + class LifecycleSandbox: + events: list[tuple[str, int]] = [] + + def __init__(self, **kwargs: Any) -> None: + self._owner_thread = threading.get_ident() + self._record("build") + + def _record(self, stage: str) -> None: + self.events.append((stage, threading.get_ident())) + if stage == failure_stage: + raise error_type(host_error_detail) + + def register_tool(self, name: str, callback: Any) -> None: + self._record("register_tool") + + def allow_domain(self, target: str, methods: Any = None) -> None: + self._record("allow_domain") + + def run(self, code: str) -> Any: + self._record("warmup" if code == "None" else "run") + return SimpleNamespace(success=True, stdout="ok\n", stderr="") + + def snapshot(self) -> object: + self._record("snapshot") + return object() + + def restore(self, snapshot: object) -> None: + self._record("restore") + + def close(self) -> None: + self._record("close") + + def __del__(self) -> None: + self.events.append(("drop", threading.get_ident())) + + return LifecycleSandbox + + return factory diff --git a/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py b/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py index 0255cb608e6..9c024d1b7ff 100644 --- a/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py +++ b/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py @@ -1539,13 +1539,15 @@ def blocking_cleanup_temp_dirs(entry_to_clean: Any) -> None: _close_execute_code_registry(execute_code) +@pytest.mark.parametrize("include_detailed_errors", [False, True]) async def test_execute_code_tool_preserves_exception_when_post_cleanup_fails( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + include_detailed_errors: bool, ) -> None: _FakeSandbox.instances.clear() monkeypatch.setattr(execute_code_module, "_load_sandbox_class", lambda: _FakeSandbox) - execute_code = HyperlightExecuteCodeTool(workspace_root=tmp_path) + execute_code = HyperlightExecuteCodeTool(workspace_root=tmp_path, include_detailed_errors=include_detailed_errors) config = execute_code._build_run_config() registry = cast(Any, execute_code._registry) output_root = Path(registry._get_or_create_entry(config).output_dir.name) @@ -1568,7 +1570,8 @@ def fail_build_contents(**kwargs: Any) -> list[Content]: monkeypatch.setattr(execute_code_module, "_build_execution_contents", fail_build_contents) try: - with pytest.raises(RuntimeError, match="primary build failure"): + expected = "primary build failure" if include_detailed_errors else "Code execution failed." + with pytest.raises(RuntimeError, match=expected): await execute_code.invoke(arguments={"code": "create-output"}) assert registry._entries == {} finally: @@ -3299,6 +3302,59 @@ async def test_error_policy_preserves_callback_cancellation( callback() +@pytest.mark.parametrize( + "failure_stage", ["build", "register_tool", "allow_domain", "warmup", "snapshot", "restore", "run"] +) +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_lifecycle_failures_obey_error_policy_and_retire_worker( + monkeypatch: pytest.MonkeyPatch, + lifecycle_sandbox_factory: Callable[..., type[Any]], + host_error_detail: str, + failure_stage: str, + include_detailed_errors: bool, +) -> None: + sandbox_cls = lifecycle_sandbox_factory(failure_stage) + monkeypatch.setattr(execute_code_module, "_load_sandbox_class", lambda: sandbox_cls) + execute_code = HyperlightExecuteCodeTool( + tools=[lambda: "ok"], allowed_domains=["example.com"], include_detailed_errors=include_detailed_errors + ) + registry = cast(Any, execute_code._registry) + try: + with pytest.raises(ValueError if include_detailed_errors else RuntimeError) as error: + await execute_code.invoke(arguments={"code": "print('ok')"}) + assert (host_error_detail in str(error.value)) is include_detailed_errors + if not include_detailed_errors: + assert str(error.value) == "Code execution failed." + assert registry._entries == {} + finally: + _close_execute_code_registry(execute_code) + gc.collect() + owner_threads = {ident for _, ident in sandbox_cls.events} + assert len(owner_threads) == 1 + assert threading.get_ident() not in owner_threads + + +@pytest.mark.parametrize("signal_type", [asyncio.CancelledError, SystemExit, KeyboardInterrupt]) +@pytest.mark.parametrize("failure_stage", ["warmup", "restore", "run"]) +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_lifecycle_error_policy_preserves_system_signals( + monkeypatch: pytest.MonkeyPatch, + lifecycle_sandbox_factory: Callable[..., type[Any]], + host_error_detail: str, + signal_type: type[BaseException], + failure_stage: str, + include_detailed_errors: bool, +) -> None: + sandbox_cls = lifecycle_sandbox_factory(failure_stage, signal_type) + monkeypatch.setattr(execute_code_module, "_load_sandbox_class", lambda: sandbox_cls) + execute_code = HyperlightExecuteCodeTool(include_detailed_errors=include_detailed_errors) + try: + with pytest.raises(signal_type, match=host_error_detail): + await execute_code.invoke(arguments={"code": "print('ok')"}) + finally: + _close_execute_code_registry(execute_code) + + @pytest.mark.parametrize("include_detailed_errors", [False, True]) def test_error_policy_is_snapshotted_and_part_of_the_sandbox_cache_key(include_detailed_errors: bool) -> None: registry = _FakeRuntime()