Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,7 @@ public async Task<RoleDialogModel> GetChatCompletionsStreamingAsync(Agent agent,
var contentBlock = choice.ContentBlock;
var delta = choice.Delta;

tokenUsage = delta?.Usage ?? startMsg?.Usage ?? choice.Usage;
tokenUsage = ProviderHelper.UpdateStreamedUsage(tokenUsage, choice);

if (delta != null)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,28 @@ public static AnthropicClient GetAnthropicClient(string provider, string model,
return client;
}

public static Usage? UpdateStreamedUsage(Usage? current, MessageResponse response)
{
if (response.StreamStartMessage?.Usage is { } initialUsage)
{
current = initialUsage;
}

var outputUsage = response.Delta?.Usage ?? response.Usage;
if (outputUsage != null)
{
if (current == null)
{
return outputUsage;
}

// Later usage contains cumulative output tokens, not the initial input/cache counts.
current.OutputTokens = outputUsage.OutputTokens;
}

return current;
}

public const string MaxTokensMessage = "AI response exceeded max output length";

/// <summary>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
using System.Text.Json;
using Anthropic.SDK.Messaging;
using BotSharp.Plugin.AnthropicAI.Providers;
using Xunit;

namespace BotSharp.Core.UnitTests.Plugins;

public class AnthropicStreamedUsageTests
{
[Fact]
public void StreamingUsage_PreservesInputAndCacheAcrossEvents()
{
string[] events =
[
"""{"message":{"usage":{"input_tokens":120,"cache_read_input_tokens":1000,"cache_creation_input_tokens":200,"output_tokens":1}}}""",
"""{"delta":{"text":"hello"}}""",
"""{"usage":{"output_tokens":8}}""",
"""{"delta":{"usage":{"output_tokens":12}}}""",
"""{"type":"message_stop"}"""
];
Usage? usage = null;
foreach (var json in events)
{
usage = ProviderHelper.UpdateStreamedUsage(usage, JsonSerializer.Deserialize<MessageResponse>(json)!);
}

Assert.NotNull(usage);
Assert.Equal(120, usage.InputTokens);
Assert.Equal(1000, usage.CacheReadInputTokens);
Assert.Equal(200, usage.CacheCreationInputTokens);
Assert.Equal(12, usage.OutputTokens);
}

[Fact]
public void StreamingUsage_WithoutUsage_RemainsNull()
{
Assert.Null(ProviderHelper.UpdateStreamedUsage(null, new MessageResponse()));
}

[Fact]
public void StreamingUsage_OutputOnly_IsRetained()
{
var response = JsonSerializer.Deserialize<MessageResponse>("""{"usage":{"output_tokens":9}}""")!;
var usage = ProviderHelper.UpdateStreamedUsage(null, response);

Assert.NotNull(usage);
Assert.Equal(0, usage.InputTokens);
Assert.Equal(9, usage.OutputTokens);
}
}
Loading