From 674b20a228e6bd267d8af6843694030d27ee656f Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:51:24 +0000 Subject: [PATCH 1/2] Change unicode encoding behavior for hosting. --- .../OutputConverter.cs | 13 +++- .../AgentResponseExtensions.cs | 2 +- .../OpenAIHostingJsonUtilities.cs | 5 ++ .../OutputConverterTests.cs | 78 +++++++++++++++++++ ...OpenAIChatCompletionsSerializationTests.cs | 20 +++++ .../OpenAIResponsesSerializationTests.cs | 32 ++++++++ 6 files changed, 145 insertions(+), 5 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs index 837f32f209c..e0157d94dee 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs @@ -7,7 +7,9 @@ using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Text; +using System.Text.Encodings.Web; using System.Text.Json; +using System.Text.Unicode; using System.Threading; using System.Threading.Tasks; using Azure.AI.AgentServer.Responses; @@ -46,6 +48,9 @@ public static implicit operator OutputConverterItem(ResponseStreamEvent value) = /// internal static class OutputConverter { + // Keep non-ASCII characters as-is while continuing to escape HTML-sensitive characters. + private static readonly JsonSerializerOptions s_jsonOptions = new() { Encoder = JavaScriptEncoder.Create(UnicodeRanges.All) }; + /// /// Converts a stream of into a stream of /// using the SDK builder pattern. @@ -209,7 +214,7 @@ public static async IAsyncEnumerable ConvertUpdatesToItemsA previousMessageId = null; var arguments = functionCall.Arguments is not null - ? JsonSerializer.Serialize(functionCall.Arguments) + ? JsonSerializer.Serialize(functionCall.Arguments, s_jsonOptions) : "{}"; var fcBuilder = stream.AddOutputItemFunctionCall(functionCall.Name, functionCall.CallId); @@ -328,7 +333,7 @@ public static async IAsyncEnumerable ConvertUpdatesToItemsA var wireId = ToolApprovalIdMap.ComputeWireId(approvalRequest.RequestId); var approvalArguments = approvalFunctionCall.Arguments is not null - ? JsonSerializer.Serialize(approvalFunctionCall.Arguments) + ? JsonSerializer.Serialize(approvalFunctionCall.Arguments, s_jsonOptions) : "{}"; ToolApprovalIdMap.Record( @@ -767,9 +772,9 @@ internal static string EncodeFunctionResultAsJsonStringPayload(object? result) JsonDocument jd => jd.RootElement.ValueKind == JsonValueKind.String ? (jd.RootElement.GetString() ?? string.Empty) : jd.RootElement.GetRawText(), - _ => JsonSerializer.Serialize(result), + _ => JsonSerializer.Serialize(result, s_jsonOptions), }; - return JsonSerializer.Serialize(innerText); + return JsonSerializer.Serialize(innerText, s_jsonOptions); } } diff --git a/dotnet/src/Microsoft.Agents.AI.Hosting.OpenAI/ChatCompletions/AgentResponseExtensions.cs b/dotnet/src/Microsoft.Agents.AI.Hosting.OpenAI/ChatCompletions/AgentResponseExtensions.cs index 823f0e7fef2..1e7ff4a2f97 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hosting.OpenAI/ChatCompletions/AgentResponseExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hosting.OpenAI/ChatCompletions/AgentResponseExtensions.cs @@ -200,7 +200,7 @@ public static ChoiceMessageToolCall ToChoiceMessageToolCall(this FunctionCallCon Function = new() { Name = functionCall.Name, - Arguments = JsonSerializer.Serialize(functionCall.Arguments, ChatCompletionsJsonContext.Default.DictionaryStringObject) + Arguments = JsonSerializer.Serialize(functionCall.Arguments, OpenAIHostingJsonUtilities.DefaultOptions.GetTypeInfo(typeof(IDictionary))) } }; } diff --git a/dotnet/src/Microsoft.Agents.AI.Hosting.OpenAI/OpenAIHostingJsonUtilities.cs b/dotnet/src/Microsoft.Agents.AI.Hosting.OpenAI/OpenAIHostingJsonUtilities.cs index 424e34de14f..08f54441547 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hosting.OpenAI/OpenAIHostingJsonUtilities.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hosting.OpenAI/OpenAIHostingJsonUtilities.cs @@ -2,8 +2,10 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; +using System.Text.Unicode; using Microsoft.Agents.AI.Hosting.OpenAI.Conversations.Models; using Microsoft.Agents.AI.Hosting.OpenAI.Models; using Microsoft.Agents.AI.Hosting.OpenAI.Responses.Models; @@ -31,6 +33,9 @@ private static JsonSerializerOptions CreateDefaultOptions() options.TypeInfoResolverChain.Add(AgentAbstractionsJsonUtilities.DefaultOptions.TypeInfoResolver!); options.TypeInfoResolverChain.Add(OpenAIHostingJsonContext.Default.Options.TypeInfoResolver!); + // Keep non-ASCII characters as-is while continuing to escape HTML-sensitive characters. + options.Encoder = JavaScriptEncoder.Create(UnicodeRanges.All); + options.MakeReadOnly(); return options; } diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/OutputConverterTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/OutputConverterTests.cs index 48982e6a518..d1f7b51db77 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/OutputConverterTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/OutputConverterTests.cs @@ -443,6 +443,84 @@ public async Task ConvertUpdatesToEventsAsync_FunctionCallWithNullArguments_Emit Assert.IsType(events[^1]); } + [Fact] + public async Task ConvertUpdatesToEventsAsync_FunctionCallNonAsciiArguments_PreservesCharactersAsync() + { + // Arrange + var (stream, _) = CreateTestStream(); + var update = new AgentResponseUpdate + { + Contents = [new FunctionCallContent("call_1", "get_weather", new Dictionary { ["location"] = "Ереван" })] + }; + + // Act + var events = new List(); + await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(new[] { update }), stream)) + { + events.Add(evt); + } + + // Assert + var done = Assert.Single(events.OfType()); + Assert.Equal("""{"location":"Ереван"}""", done.Arguments); + } + + [Fact] + public async Task ConvertUpdatesToEventsAsync_FunctionCallHtmlSensitiveArguments_EscapesCharactersAsync() + { + // Arrange + var (stream, _) = CreateTestStream(); + var update = new AgentResponseUpdate + { + Contents = [new FunctionCallContent("call_1", "search", new Dictionary { ["query"] = "&" })] + }; + + // Act + var events = new List(); + await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(new[] { update }), stream)) + { + events.Add(evt); + } + + // Assert + var done = Assert.Single(events.OfType()); + Assert.Equal("""{"query":"\u003Ctag\u003E\u0026"}""", done.Arguments); + } + + [Fact] + public async Task ConvertUpdatesToEventsAsync_ToolApprovalRequestNonAsciiArguments_PreservesCharactersAsync() + { + // Arrange + var (stream, _) = CreateTestStream(); + var functionCall = new FunctionCallContent("call_1", "get_weather", new Dictionary { ["location"] = "Ереван" }); + var update = new AgentResponseUpdate { Contents = [new ToolApprovalRequestContent("af_request_1", functionCall)] }; + + // Act + var events = new List(); + await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(new[] { update }), stream, new AgentSessionStateBag())) + { + events.Add(evt); + } + + // Assert + var added = Assert.Single(events.OfType()); + var item = Assert.IsType(added.Item); + Assert.Equal("""{"location":"Ереван"}""", item.Arguments); + } + + [Fact] + public void EncodeFunctionResultAsJsonStringPayload_NonAsciiObject_PreservesCharacters() + { + // Arrange + var result = new { city = "Ереван" }; + + // Act + string payload = OutputConverter.EncodeFunctionResultAsJsonStringPayload(result); + + // Assert + Assert.Equal("""{"city":"Ереван"}""", System.Text.Json.JsonSerializer.Deserialize(payload)); + } + // G-04 [Fact] public async Task ConvertUpdatesToEventsAsync_FunctionCallWithEmptyCallId_DoesNotEmitWireItemAsync() diff --git a/dotnet/tests/Microsoft.Agents.AI.Hosting.OpenAI.UnitTests/OpenAIChatCompletionsSerializationTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hosting.OpenAI.UnitTests/OpenAIChatCompletionsSerializationTests.cs index edf11fbf6bb..cd346f04662 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hosting.OpenAI.UnitTests/OpenAIChatCompletionsSerializationTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hosting.OpenAI.UnitTests/OpenAIChatCompletionsSerializationTests.cs @@ -4,8 +4,10 @@ using System.Collections.Generic; using System.Linq; using System.Text.Json; +using Microsoft.Agents.AI.Hosting.OpenAI.ChatCompletions; using Microsoft.Agents.AI.Hosting.OpenAI.ChatCompletions.Models; using Microsoft.Agents.AI.Hosting.OpenAI.Tests; +using Microsoft.Extensions.AI; namespace Microsoft.Agents.AI.Hosting.OpenAI.UnitTests; @@ -385,6 +387,24 @@ public void Deserialize_ResponseRoundTrip_PreservesData() #endregion + #region Function Call Conversion Tests + + [Fact] + public void ToChoiceMessageToolCall_NonAsciiArguments_PreservesCharacters() + { + // Arrange + var functionCall = new FunctionCallContent("call_1", "get_weather", new Dictionary { ["location"] = "Ереван" }); + + // Act + ChoiceMessageToolCall toolCall = functionCall.ToChoiceMessageToolCall(); + + // Assert + Assert.NotNull(toolCall.Function); + Assert.Equal("""{"location":"Ереван"}""", toolCall.Function.Arguments); + } + + #endregion + #region Streaming Chunk Deserialization Tests [Fact] diff --git a/dotnet/tests/Microsoft.Agents.AI.Hosting.OpenAI.UnitTests/OpenAIResponsesSerializationTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hosting.OpenAI.UnitTests/OpenAIResponsesSerializationTests.cs index 98e3fb43541..df2c28d7c98 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hosting.OpenAI.UnitTests/OpenAIResponsesSerializationTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hosting.OpenAI.UnitTests/OpenAIResponsesSerializationTests.cs @@ -4,8 +4,10 @@ using System.Collections.Generic; using System.Linq; using System.Text.Json; +using Microsoft.Agents.AI.Hosting.OpenAI.Responses; using Microsoft.Agents.AI.Hosting.OpenAI.Responses.Models; using Microsoft.Agents.AI.Hosting.OpenAI.Tests; +using Microsoft.Extensions.AI; namespace Microsoft.Agents.AI.Hosting.OpenAI.UnitTests; @@ -766,6 +768,36 @@ public void Deserialize_ResponseRoundTrip_PreservesData() #endregion + #region Function Call Conversion Tests + + [Fact] + public void ToFunctionToolCallItemResource_NonAsciiArguments_PreservesCharacters() + { + // Arrange + var functionCall = new FunctionCallContent("call_1", "get_weather", new Dictionary { ["location"] = "Ереван" }); + + // Act + FunctionToolCallItemResource resource = functionCall.ToFunctionToolCallItemResource("fc_1", OpenAIHostingJsonUtilities.DefaultOptions); + + // Assert + Assert.Equal("""{"location":"Ереван"}""", resource.Arguments); + } + + [Fact] + public void ToFunctionToolCallItemResource_HtmlSensitiveArguments_EscapesCharacters() + { + // Arrange + var functionCall = new FunctionCallContent("call_1", "search", new Dictionary { ["query"] = "&" }); + + // Act + FunctionToolCallItemResource resource = functionCall.ToFunctionToolCallItemResource("fc_1", OpenAIHostingJsonUtilities.DefaultOptions); + + // Assert + Assert.Equal("""{"query":"\u003Ctag\u003E\u0026"}""", resource.Arguments); + } + + #endregion + #region Streaming Event Deserialization Tests [Fact] From 9b3564d0c6c41a0193a902dadd6f2fb24cde45d7 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 4 Nov 2026 10:40:51 +0000 Subject: [PATCH 2/2] Address PR comments --- .../OutputConverter.cs | 4 +-- .../OutputConverterTests.cs | 26 +++++++++++++++++++ ...OpenAIChatCompletionsSerializationTests.cs | 4 +-- .../OpenAIResponsesSerializationTests.cs | 6 ++--- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs index e0157d94dee..3d373cad3b3 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs @@ -768,10 +768,10 @@ internal static string EncodeFunctionResultAsJsonStringPayload(object? result) string s => s, JsonElement je => je.ValueKind == JsonValueKind.String ? (je.GetString() ?? string.Empty) - : je.GetRawText(), + : JsonSerializer.Serialize(je, s_jsonOptions), JsonDocument jd => jd.RootElement.ValueKind == JsonValueKind.String ? (jd.RootElement.GetString() ?? string.Empty) - : jd.RootElement.GetRawText(), + : JsonSerializer.Serialize(jd.RootElement, s_jsonOptions), _ => JsonSerializer.Serialize(result, s_jsonOptions), }; diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/OutputConverterTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/OutputConverterTests.cs index d1f7b51db77..29eda0d197c 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/OutputConverterTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/OutputConverterTests.cs @@ -521,6 +521,32 @@ public void EncodeFunctionResultAsJsonStringPayload_NonAsciiObject_PreservesChar Assert.Equal("""{"city":"Ереван"}""", System.Text.Json.JsonSerializer.Deserialize(payload)); } + [Fact] + public void EncodeFunctionResultAsJsonStringPayload_JsonElementWithEscapedUnicode_DecodesCharacters() + { + // Arrange + using System.Text.Json.JsonDocument document = System.Text.Json.JsonDocument.Parse("""{"city":"\u0415"}"""); + + // Act + string payload = OutputConverter.EncodeFunctionResultAsJsonStringPayload(document.RootElement); + + // Assert + Assert.Equal("""{"city":"Е"}""", System.Text.Json.JsonSerializer.Deserialize(payload)); + } + + [Fact] + public void EncodeFunctionResultAsJsonStringPayload_JsonDocumentWithEscapedUnicode_DecodesCharacters() + { + // Arrange + using System.Text.Json.JsonDocument document = System.Text.Json.JsonDocument.Parse("""{"city":"\u0415"}"""); + + // Act + string payload = OutputConverter.EncodeFunctionResultAsJsonStringPayload(document); + + // Assert + Assert.Equal("""{"city":"Е"}""", System.Text.Json.JsonSerializer.Deserialize(payload)); + } + // G-04 [Fact] public async Task ConvertUpdatesToEventsAsync_FunctionCallWithEmptyCallId_DoesNotEmitWireItemAsync() diff --git a/dotnet/tests/Microsoft.Agents.AI.Hosting.OpenAI.UnitTests/OpenAIChatCompletionsSerializationTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hosting.OpenAI.UnitTests/OpenAIChatCompletionsSerializationTests.cs index cd346f04662..2692ee74ad6 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hosting.OpenAI.UnitTests/OpenAIChatCompletionsSerializationTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hosting.OpenAI.UnitTests/OpenAIChatCompletionsSerializationTests.cs @@ -4,10 +4,10 @@ using System.Collections.Generic; using System.Linq; using System.Text.Json; -using Microsoft.Agents.AI.Hosting.OpenAI.ChatCompletions; using Microsoft.Agents.AI.Hosting.OpenAI.ChatCompletions.Models; using Microsoft.Agents.AI.Hosting.OpenAI.Tests; using Microsoft.Extensions.AI; +using ChatCompletionsAgentResponseExtensions = Microsoft.Agents.AI.Hosting.OpenAI.ChatCompletions.AgentResponseExtensions; namespace Microsoft.Agents.AI.Hosting.OpenAI.UnitTests; @@ -396,7 +396,7 @@ public void ToChoiceMessageToolCall_NonAsciiArguments_PreservesCharacters() var functionCall = new FunctionCallContent("call_1", "get_weather", new Dictionary { ["location"] = "Ереван" }); // Act - ChoiceMessageToolCall toolCall = functionCall.ToChoiceMessageToolCall(); + ChoiceMessageToolCall toolCall = ChatCompletionsAgentResponseExtensions.ToChoiceMessageToolCall(functionCall); // Assert Assert.NotNull(toolCall.Function); diff --git a/dotnet/tests/Microsoft.Agents.AI.Hosting.OpenAI.UnitTests/OpenAIResponsesSerializationTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hosting.OpenAI.UnitTests/OpenAIResponsesSerializationTests.cs index df2c28d7c98..ef16215b0e2 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hosting.OpenAI.UnitTests/OpenAIResponsesSerializationTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hosting.OpenAI.UnitTests/OpenAIResponsesSerializationTests.cs @@ -4,10 +4,10 @@ using System.Collections.Generic; using System.Linq; using System.Text.Json; -using Microsoft.Agents.AI.Hosting.OpenAI.Responses; using Microsoft.Agents.AI.Hosting.OpenAI.Responses.Models; using Microsoft.Agents.AI.Hosting.OpenAI.Tests; using Microsoft.Extensions.AI; +using ResponsesAgentResponseExtensions = Microsoft.Agents.AI.Hosting.OpenAI.Responses.AgentResponseExtensions; namespace Microsoft.Agents.AI.Hosting.OpenAI.UnitTests; @@ -777,7 +777,7 @@ public void ToFunctionToolCallItemResource_NonAsciiArguments_PreservesCharacters var functionCall = new FunctionCallContent("call_1", "get_weather", new Dictionary { ["location"] = "Ереван" }); // Act - FunctionToolCallItemResource resource = functionCall.ToFunctionToolCallItemResource("fc_1", OpenAIHostingJsonUtilities.DefaultOptions); + FunctionToolCallItemResource resource = ResponsesAgentResponseExtensions.ToFunctionToolCallItemResource(functionCall, "fc_1", OpenAIHostingJsonUtilities.DefaultOptions); // Assert Assert.Equal("""{"location":"Ереван"}""", resource.Arguments); @@ -790,7 +790,7 @@ public void ToFunctionToolCallItemResource_HtmlSensitiveArguments_EscapesCharact var functionCall = new FunctionCallContent("call_1", "search", new Dictionary { ["query"] = "&" }); // Act - FunctionToolCallItemResource resource = functionCall.ToFunctionToolCallItemResource("fc_1", OpenAIHostingJsonUtilities.DefaultOptions); + FunctionToolCallItemResource resource = ResponsesAgentResponseExtensions.ToFunctionToolCallItemResource(functionCall, "fc_1", OpenAIHostingJsonUtilities.DefaultOptions); // Assert Assert.Equal("""{"query":"\u003Ctag\u003E\u0026"}""", resource.Arguments);