Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions otel-plugin/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,11 @@ This telemetry decision does not remove the SDK's failure notification: a presen
remains a `DurableOperationException` in operation hooks and snapshots. An absent container retains its existing
null metadata behavior. Persisted errors, caller failures, replay, and user-function error handling are unchanged.

In both trace views, operation exception events use a nonblank stored `ErrorObject.errorType` verbatim, including
remote-language error types. A missing, empty, or whitespace-only type retains the existing throwable-type reporting.
Blank stored types still count as supplied error details and do not suppress the exception event.
The throwable's message and stack diagnostics, attempt events, and Workflow/Invocation exception reporting are unchanged.

Execution-view operation spans with a known, nonzero backend end timestamp record the exception event at that
same timestamp. This path uses the public `exception` event with standard `exception.type`, `exception.message`
(when present), and `exception.stacktrace` attributes. Java's standard stack-trace text and the provider's ordinary
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ static void applyOperationStatus(Span span, OperationEndInfo info, Instant endTi
span.setStatus(StatusCode.ERROR, info.error().getMessage());
if (endTimestamp == null || endTimestamp.equals(Instant.EPOCH)) {
// The SDK treats epoch zero as its current-clock sentinel when ending a span.
span.recordException(info.error());
span.recordException(info.error(), operationExceptionAttributes(info.error()));
} else {
recordExceptionAt(span, info.error(), endTimestamp);
}
Expand Down Expand Up @@ -126,9 +126,25 @@ private static void recordExceptionAt(Span span, Throwable error, Instant timest
error.printStackTrace(writer);
}
attributes.put("exception.stacktrace", stacktrace.toString());
attributes.putAll(operationExceptionAttributes(error));
span.addEvent("exception", attributes.build(), timestamp);
}

/** Override only the operation event's type; retain the actual throwable's message and stack diagnostics. */
private static Attributes operationExceptionAttributes(Throwable error) {
if (error instanceof DurableOperationException failure) {
var details = failure.getErrorObject();
if (details != null
&& details.errorType() != null
&& !details.errorType().isBlank()) {
return Attributes.builder()
.put("exception.type", details.errorType())
.build();
}
}
return Attributes.empty();
}

/** Distinguishes operation telemetry details without changing the SDK's failure notification object. */
static boolean hasOperationErrorDetails(Throwable error) {
if (error == null) return false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,9 +155,7 @@ public void onOperationEnd(OperationEndInfo info) {
assertEquals(1, span.getEvents().size());
var diagnostic = span.getEvents().get(0);
var failure = ends.get(0).error();
assertEquals(
failure.getClass().getCanonicalName(),
diagnostic.getAttributes().get(AttributeKey.stringKey("exception.type")));
assertEquals(details.errorType(), diagnostic.getAttributes().get(AttributeKey.stringKey("exception.type")));
assertEquals(
failure.getMessage(), diagnostic.getAttributes().get(AttributeKey.stringKey("exception.message")));
assertTrue(diagnostic
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import static org.junit.jupiter.api.Assertions.*;
import static software.amazon.lambda.durable.otel.SpanAttributes.*;

import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.api.trace.StatusCode;
import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
import io.opentelemetry.sdk.trace.SdkTracerProvider;
Expand Down Expand Up @@ -51,6 +52,12 @@ static Stream<Arguments> errors() {
ErrorObject.builder().errorType("").build(),
true,
false),
Arguments.of(
executionView,
"blank-type",
ErrorObject.builder().errorType(" \t\n").build(),
true,
false),
Arguments.of(
executionView,
"message",
Expand Down Expand Up @@ -176,6 +183,16 @@ public void onOperationEnd(OperationEndInfo info) {
assertEquals(1, terminalSpans.size());
assertEquals(StatusCode.ERROR, terminalSpans.get(0).getStatus().getStatusCode());
assertEquals(hasDetails ? 1 : 0, terminalSpans.get(0).getEvents().size());
if (hasDetails) {
assertEquals(
"type".equals(label) ? "ExternalFailure" : DurableOperationException.class.getCanonicalName(),
terminalSpans
.get(0)
.getEvents()
.get(0)
.getAttributes()
.get(AttributeKey.stringKey("exception.type")));
}
var replay = runner.run("input");
assertEquals(ExecutionStatus.FAILED, replay.getStatus());
assertEquals(result.getError(), replay.getError());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.api.trace.StatusCode;
import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
import io.opentelemetry.sdk.testing.time.TestClock;
import io.opentelemetry.sdk.trace.SdkTracerProvider;
import io.opentelemetry.sdk.trace.SpanLimits;
import io.opentelemetry.sdk.trace.data.SpanData;
Expand All @@ -20,9 +21,14 @@
import java.util.List;
import java.util.concurrent.CompletionException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
import software.amazon.awssdk.services.lambda.model.ErrorObject;
import software.amazon.lambda.durable.exception.DurableOperationException;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.InvocationStatus;
Expand All @@ -34,6 +40,83 @@ class OperationExceptionEventTest {
private static final AttributeKey<String> MESSAGE = AttributeKey.stringKey("exception.message");
private static final AttributeKey<String> STACK = AttributeKey.stringKey("exception.stacktrace");

static Stream<Arguments> storedErrors() {
return Stream.of(null, Instant.EPOCH, Instant.ofEpochSecond(20))
.flatMap(end -> Stream.of(
Arguments.of(end, new RuntimeException("ordinary failure"), "java.lang.RuntimeException"),
Arguments.of(
end,
new DurableOperationException(null, null, "no stored error"),
DurableOperationException.class.getCanonicalName()),
Arguments.of(end, storedFailure(null), DurableOperationException.class.getCanonicalName()),
Arguments.of(end, storedFailure("java.lang.RuntimeException"), "java.lang.RuntimeException"),
Arguments.of(end, storedFailure("RejectedError"), "RejectedError"),
Arguments.of(end, storedFailure(" RejectedError "), " RejectedError "),
Arguments.of(end, storedFailure(""), DurableOperationException.class.getCanonicalName()),
Arguments.of(end, storedFailure(" "), DurableOperationException.class.getCanonicalName()),
Arguments.of(end, storedFailure("\t\r\n"), DurableOperationException.class.getCanonicalName()),
Arguments.of(
end, storedFailure("\u2003"), DurableOperationException.class.getCanonicalName())));
}

@ParameterizedTest
@MethodSource("storedErrors")
void storedTypeChangesOnlyTheOperationExceptionType(Instant end, Throwable error, String expectedType) {
try (var exporter = InMemorySpanExporter.create();
var provider = SdkTracerProvider.builder()
.setClock(TestClock.create(Instant.ofEpochSecond(100)))
.addSpanProcessor(SimpleSpanProcessor.create(exporter))
.build()) {
var tracer = provider.get("operation-exception-test");
var baseline = tracer.spanBuilder("baseline").startSpan();
baseline.recordException(error);
baseline.end();
var span = tracer.spanBuilder(NAME).startSpan();
OtelPluginSupport.applyOperationStatus(
span,
new OperationEndInfo(
"op", NAME, "CALLBACK", "Callback", null, null, end, "FAILED", null, false, error),
end);
span.end();
var spans = exporter.getFinishedSpanItems();
assertEquals(StatusCode.ERROR, spans.get(1).getStatus().getStatusCode());
assertEquals(error.getMessage(), spans.get(1).getStatus().getDescription());
assertExceptionDetails(
spans.get(0),
spans.get(1),
expectedType,
end == null || end.equals(Instant.EPOCH) ? 100_000_000_000L : 20_000_000_000L);
}
}

private static void assertExceptionDetails(
SpanData baseline, SpanData actual, String expectedType, long expectedTime) {
assertEquals(1, actual.getEvents().size());
var event = actual.getEvents().get(0);
assertEquals("exception", event.getName());
assertEquals(
baseline.getEvents().get(0).getAttributes().toBuilder()
.put(TYPE, expectedType)
.build()
.asMap(),
event.getAttributes().asMap());
assertEquals(expectedTime, event.getEpochNanos());
}

private static DurableOperationException storedFailure(String type) {
var error = new DurableOperationException(
null,
ErrorObject.builder()
.errorType(type)
.errorMessage("stored failure")
.stackTrace("Example|step|Workflow.java|42")
.build(),
"wrapper message",
new IllegalArgumentException("cause"));
error.addSuppressed(new IllegalStateException("suppressed cleanup"));
return error;
}

@ParameterizedTest
@ValueSource(booleans = {false, true})
void knownEndRetainsRealExceptionAndWrapperDiagnostics(boolean wrapped) {
Expand Down
Loading
Loading