Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ internal interface ISuperStepRunner

string StartExecutorId { get; }

string? WorkflowName { get; }

WorkflowTelemetryContext TelemetryContext { get; }

bool HasUnservicedRequests { get; }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,9 @@ public async IAsyncEnumerable<WorkflowEvent> TakeEventStreamAsync(bool blockOnPe
using CancellationTokenSource linkedSource = CancellationTokenSource.CreateLinkedTokenSource(this._stopCancellation.Token, cancellationToken);

// Re-establish session as parent so the run activity nests correctly.
Activity.Current = this._sessionActivity;
Activity.Current = this._sessionActivity ?? Activity.Current;

// Not 'using' — must dispose explicitly in finally for deterministic export.
Activity? runActivity = this._stepRunner.TelemetryContext.StartWorkflowRunActivity();
using Activity? runActivity = this._stepRunner.TelemetryContext.StartWorkflowRunActivity(this._stepRunner.WorkflowName);
runActivity?.SetTag(Tags.WorkflowId, this._stepRunner.StartExecutorId).SetTag(Tags.SessionId, this._stepRunner.SessionId);

try
Expand Down Expand Up @@ -167,10 +166,6 @@ public async IAsyncEnumerable<WorkflowEvent> TakeEventStreamAsync(bool blockOnPe
finally
{
this.RunStatus = this._stepRunner.HasUnservicedRequests ? RunStatus.PendingRequests : RunStatus.Idle;

// Explicitly dispose the Activity so Activity.Stop fires deterministically,
// regardless of how the async iterator enumerator is disposed.
runActivity?.Dispose();
}

// If we are Idle or Ended, we should break out of the loop
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ private async Task RunLoopAsync(CancellationToken cancellationToken)
while (!linkedSource.Token.IsCancellationRequested)
{
// Start a new run-stage activity for this input→processing→halt cycle
runActivity = this._stepRunner.TelemetryContext.StartWorkflowRunActivity();
runActivity = this._stepRunner.TelemetryContext.StartWorkflowRunActivity(this._stepRunner.WorkflowName);
runActivity?.SetTag(Tags.WorkflowId, this._stepRunner.StartExecutorId)
.SetTag(Tags.SessionId, this._stepRunner.SessionId);
runActivity?.AddEvent(new ActivityEvent(EventNames.WorkflowStarted));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ private InProcessRunner(Workflow workflow, ICheckpointManager? checkpointManager
/// <inheritdoc cref="ISuperStepRunner.StartExecutorId"/>
public string StartExecutorId { get; }

/// <inheritdoc cref="ISuperStepRunner.WorkflowName"/>
public string? WorkflowName => this.Workflow.Name;

/// <summary>
/// Gating flag for deferred event republishing after checkpoint restore.
/// </summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ internal static class ActivityNames
{
public const string WorkflowBuild = "workflow.build";
public const string WorkflowSession = "workflow.session";
public const string WorkflowInvoke = "workflow_invoke";
public const string WorkflowInvoke = "invoke_workflow";
public const string MessageSend = "message.send";
public const string ExecutorProcess = "executor.process";
public const string EdgeGroupProcess = "edge_group.process";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@ namespace Microsoft.Agents.AI.Workflows.Observability;

internal static class Tags
{
public const string OperationName = "gen_ai.operation.name";
public const string ConversationId = "gen_ai.conversation.id";
public const string WorkflowId = "workflow.id";
public const string WorkflowName = "workflow.name";
public const string WorkflowName = "gen_ai.workflow.name";
public const string WorkflowDescription = "workflow.description";
public const string WorkflowDefinition = "workflow.definition";
public const string BuildErrorMessage = "build.error.message";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using OpenTelemetry;

namespace Microsoft.Agents.AI.Workflows.Observability;

Expand Down Expand Up @@ -109,15 +110,34 @@ public WorkflowTelemetryContext(WorkflowTelemetryOptions options, ActivitySource
/// Starts a workflow run activity if enabled. This represents a single
/// input-to-halt cycle within a workflow session.
/// </summary>
/// <param name="workflowName">The optional application-defined workflow name.</param>
/// <returns>An activity if workflow run telemetry is enabled, otherwise null.</returns>
public Activity? StartWorkflowRunActivity()
public Activity? StartWorkflowRunActivity(string? workflowName)
{
if (!this.IsEnabled || this.Options.DisableWorkflowRun)
{
return null;
}

return this.ActivitySource.StartActivity(ActivityNames.WorkflowInvoke);
ActivityTagsCollection tags = new() { { Tags.OperationName, ActivityNames.WorkflowInvoke } };
string activityName = ActivityNames.WorkflowInvoke;
if (!string.IsNullOrWhiteSpace(workflowName))
{
tags.Add(Tags.WorkflowName, workflowName);
activityName += " " + workflowName;
}

string? conversationId = Baggage.GetBaggage(Tags.ConversationId);
if (string.IsNullOrWhiteSpace(conversationId))
{
conversationId = Activity.Current?.GetBaggageItem(Tags.ConversationId);
}
if (!string.IsNullOrWhiteSpace(conversationId))
{
tags.Add(Tags.ConversationId, conversationId);
}

return this.ActivitySource.StartActivity(activityName, ActivityKind.Internal, parentContext: default, tags: tags);
}

/// <summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ public sealed class WorkflowTelemetryOptions
/// Gets or sets a value indicating whether workflow run activities should be disabled.
/// </summary>
/// <value>
/// <see langword="true"/> to disable <c>workflow_invoke</c> activities;
/// <see langword="true"/> to disable <c>invoke_workflow {gen_ai.workflow.name}</c> activities
/// (or <c>invoke_workflow</c> when unnamed);
/// <see langword="false"/> to enable them. The default value is <see langword="false"/>.
/// </value>
public bool DisableWorkflowRun { get; set; }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -643,7 +643,7 @@ private Workflow BuildInternal(bool validateOrphans, Activity? activity = null)

// Using the start executor ID as a proxy for the workflow ID
activity?.SetTag(Tags.WorkflowId, workflow.StartExecutorId);
if (workflow.Name is not null)
if (!string.IsNullOrWhiteSpace(workflow.Name))
{
activity?.SetTag(Tags.WorkflowName, workflow.Name);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public sealed class DeclarativeWorkflowOptionsTest : IDisposable
// These constants mirror Microsoft.Agents.AI.Workflows.Observability.ActivityNames
// which is internal and not accessible from this test project.
private const string WorkflowBuildActivityName = "workflow.build";
private const string WorkflowRunActivityName = "workflow_invoke";
private const string WorkflowRunActivityName = "invoke_workflow";

// The default activity source name used by the workflow telemetry context.
private const string DefaultTelemetrySourceName = "Microsoft.Agents.AI.Workflows";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using Microsoft.Agents.AI.Workflows.InProc;
using Microsoft.Agents.AI.Workflows.Observability;
using Microsoft.Extensions.AI;
using OpenTelemetry;

namespace Microsoft.Agents.AI.Workflows.UnitTests;

Expand Down Expand Up @@ -56,7 +57,7 @@ public ObservabilityTests()
/// ---- ActivityNames.MessageSend
/// </remarks>
/// <returns>The created workflow.</returns>
private static Workflow CreateWorkflow()
private static Workflow CreateWorkflow(string? name = null)
{
// Create the executors
Func<string, string> uppercaseFunc = s => s.ToUpperInvariant();
Expand All @@ -68,6 +69,10 @@ private static Workflow CreateWorkflow()
// Build the workflow by connecting executors sequentially
WorkflowBuilder builder = new(uppercase);
builder.AddEdge(uppercase, reverse).WithOutputFrom(reverse);
if (name is not null)
{
builder.WithName(name);
}

return builder.WithOpenTelemetry().Build();
}
Expand Down Expand Up @@ -158,6 +163,195 @@ public async Task CreatesWorkflowEndToEndActivities_WithCorrectName_LockstepAsyn
await this.TestWorkflowEndToEndActivitiesAsync("Lockstep");
}

[Theory]
[InlineData("Lockstep", false, null)]
[InlineData("Lockstep", true, null)]
[InlineData("OffThread", false, null)]
[InlineData("OffThread", true, null)]
[InlineData("Concurrent", false, null)]
[InlineData("Concurrent", true, null)]
[InlineData("Lockstep", false, "customer_support_pipeline")]
[InlineData("Lockstep", true, "customer_support_pipeline")]
[InlineData("OffThread", false, "customer_support_pipeline")]
[InlineData("OffThread", true, "customer_support_pipeline")]
[InlineData("Concurrent", false, "customer_support_pipeline")]
[InlineData("Concurrent", true, "customer_support_pipeline")]
public async Task WorkflowRunActivity_UsesSemanticConventionNameAsync(string executionEnvironmentName, bool streaming, string? workflowName)
{
// Arrange
using var testActivity = new Activity("WorkflowRunSpanNameTest").Start();
testActivity.AddBaggage("gen_ai.conversation.id", "application-thread");
var workflow = CreateWorkflow(workflowName);
var executionEnvironment = GetExecutionEnvironment(executionEnvironmentName);

// Act
if (streaming)
{
await using StreamingRun run = await executionEnvironment.RunStreamingAsync(workflow, "Hello, World!");
await foreach (WorkflowEvent evt in run.WatchStreamAsync())
{
// Drain the stream to complete the workflow invocation.
}
}
else
{
await using Run run = await executionEnvironment.RunAsync(workflow, "Hello, World!");
}

// Assert
var capturedActivities = this._capturedActivities.Where(a => a.RootId == testActivity.RootId).ToList();
var workflowRunActivity = Assert.Single(capturedActivities, a =>
a.Events.Any(e => e.Name == EventNames.WorkflowStarted));
string expectedSpanName = workflowName is null ? "invoke_workflow" : $"invoke_workflow {workflowName}";
Assert.Equal(expectedSpanName, workflowRunActivity.OperationName);
Assert.Equal(expectedSpanName, workflowRunActivity.DisplayName);
Assert.Equal(workflowName, workflowRunActivity.GetTagItem("gen_ai.workflow.name"));
Assert.Equal("invoke_workflow", workflowRunActivity.GetTagItem("gen_ai.operation.name"));
Assert.Equal("application-thread", workflowRunActivity.GetTagItem("gen_ai.conversation.id"));
Assert.IsType<string>(workflowRunActivity.GetTagItem("session.id"));
Assert.All(capturedActivities.Where(a => a != workflowRunActivity), a =>
Assert.DoesNotContain(a.TagObjects, tag => tag.Key is "gen_ai.operation.name" or "gen_ai.conversation.id"));

var workflowBuildActivity = Assert.Single(capturedActivities, a => a.OperationName == ActivityNames.WorkflowBuild);
Assert.Equal(workflowName, workflowBuildActivity.GetTagItem("gen_ai.workflow.name"));
Assert.All(capturedActivities, a => Assert.DoesNotContain(a.TagObjects, tag => tag.Key == "workflow.name"));
}

[Theory]
[InlineData(null, "invoke_workflow")]
[InlineData("", "invoke_workflow")]
[InlineData(" ", "invoke_workflow")]
[InlineData("customer_support_pipeline", "invoke_workflow customer_support_pipeline")]
public void WorkflowRunActivity_ProvidesSemanticConventionsAtCreation(string? workflowName, string expectedSpanName)
{
// Arrange
using var activitySource = new ActivitySource("WorkflowOperationNameSamplingTest");
string? spanNameAtCreation = null;
object? operationNameAtCreation = null;
using var listener = new ActivityListener
{
ShouldListenTo = source => source == activitySource,
Sample = (ref ActivityCreationOptions<ActivityContext> options) =>
{
spanNameAtCreation = options.Name;
operationNameAtCreation = options.Tags?.FirstOrDefault(tag => tag.Key == "gen_ai.operation.name").Value;
return ActivitySamplingResult.AllData;
},
};
ActivitySource.AddActivityListener(listener);
var telemetryContext = new WorkflowTelemetryContext(new WorkflowTelemetryOptions(), activitySource);

// Act
using Activity? activity = telemetryContext.StartWorkflowRunActivity(workflowName);

// Assert
Assert.NotNull(activity);
Assert.Equal(expectedSpanName, spanNameAtCreation);
Assert.Equal("invoke_workflow", operationNameAtCreation);
}

[Theory]
[InlineData(null, null, null)]
[InlineData("", null, null)]
[InlineData(" ", null, null)]
[InlineData("otel-thread", null, "otel-thread")]
[InlineData("otel-thread", "activity-thread", "otel-thread")]
[InlineData(null, "activity-thread", "activity-thread")]
[InlineData("", "activity-thread", "activity-thread")]
[InlineData(" ", "activity-thread", "activity-thread")]
public void WorkflowRunActivity_UsesConversationContextAtCreation(
string? otelConversationId, string? activityConversationId, string? expectedConversationId)
{
// Arrange
using var testActivity = new Activity("WorkflowConversationContextTest").Start();
testActivity.AddBaggage("gen_ai.conversation.id", activityConversationId);
using var activitySource = new ActivitySource("WorkflowConversationSamplingTest");
object? conversationIdAtCreation = null;
using var listener = new ActivityListener
{
ShouldListenTo = source => source == activitySource,
Sample = (ref ActivityCreationOptions<ActivityContext> options) =>
{
conversationIdAtCreation = options.Tags?.FirstOrDefault(tag => tag.Key == "gen_ai.conversation.id").Value;
return ActivitySamplingResult.AllData;
},
};
ActivitySource.AddActivityListener(listener);
var telemetryContext = new WorkflowTelemetryContext(new WorkflowTelemetryOptions(), activitySource);
Baggage previousBaggage = Baggage.Current;

try
{
Baggage.SetBaggage("gen_ai.conversation.id", otelConversationId);

// Act
using Activity? activity = telemetryContext.StartWorkflowRunActivity("customer_support_pipeline");

// Assert
Assert.NotNull(activity);
Assert.Equal(expectedConversationId, conversationIdAtCreation);
Assert.Equal(expectedConversationId, activity.GetTagItem("gen_ai.conversation.id"));
if (expectedConversationId is null)
{
Assert.DoesNotContain(activity.TagObjects, tag => tag.Key == "gen_ai.conversation.id");
}
}
finally
{
Baggage.Current = previousBaggage;
}
}

[Theory]
[InlineData("Lockstep", null)]
[InlineData("OffThread", null)]
[InlineData("Concurrent", null)]
[InlineData("Lockstep", "execution-only-session")]
[InlineData("OffThread", "execution-only-session")]
[InlineData("Concurrent", "execution-only-session")]
public async Task WorkflowRunActivity_WithoutConversationContext_DoesNotUseSessionIdAsync(string executionEnvironmentName, string? sessionId)
{
// Arrange
using var testActivity = new Activity("WorkflowConversationOmissionTest").Start();
var workflow = CreateWorkflow("customer_support_pipeline");
var executionEnvironment = GetExecutionEnvironment(executionEnvironmentName);

// Act
Run run = await executionEnvironment.RunAsync(workflow, "Hello, World!", sessionId);
await run.DisposeAsync();

// Assert
var workflowRunActivity = Assert.Single(this._capturedActivities, a =>
a.RootId == testActivity.RootId &&
a.Events.Any(e => e.Name == EventNames.WorkflowStarted));
Assert.DoesNotContain(workflowRunActivity.TagObjects, tag => tag.Key == "gen_ai.conversation.id");
Assert.IsType<string>(workflowRunActivity.GetTagItem("session.id"));
if (sessionId is not null)
{
Assert.Equal(sessionId, workflowRunActivity.GetTagItem("session.id"));
}
}

[Theory]
[InlineData("")]
[InlineData(" ")]
public async Task WorkflowActivities_WithoutMeaningfulName_OmitWorkflowNameAsync(string workflowName)
{
// Arrange
using var testActivity = new Activity("WorkflowNameOmissionTest").Start();
var workflow = CreateWorkflow(workflowName);

// Act
Run run = await InProcessExecution.Default.RunAsync(workflow, "Hello, World!");
await run.DisposeAsync();

// Assert
var capturedActivities = this._capturedActivities.Where(a => a.RootId == testActivity.RootId).ToList();
Assert.Contains(capturedActivities, a => a.OperationName == ActivityNames.WorkflowInvoke);
Assert.All(capturedActivities, a => Assert.DoesNotContain(a.TagObjects,
tag => tag.Key is "gen_ai.workflow.name" or "workflow.name"));
}

[Fact]
public async Task CreatesWorkflowActivities_WithCorrectNameAsync()
{
Expand Down
Loading
Loading