diff --git a/otel-plugin/README.md b/otel-plugin/README.md index 149bd2cbb..4233e5fdd 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -411,6 +411,11 @@ This telemetry decision does not remove the SDK's failure notification: a presen remains a `DurableOperationException` in operation hooks and snapshots. An absent container retains its existing null metadata behavior. Persisted errors, caller failures, replay, and user-function error handling are unchanged. +In both trace views, operation exception events use a nonblank stored `ErrorObject.errorType` verbatim, including +remote-language error types. A missing, empty, or whitespace-only type retains the existing throwable-type reporting. +Blank stored types still count as supplied error details and do not suppress the exception event. +The throwable's message and stack diagnostics, attempt events, and Workflow/Invocation exception reporting are unchanged. + Execution-view operation spans with a known, nonzero backend end timestamp record the exception event at that same timestamp. This path uses the public `exception` event with standard `exception.type`, `exception.message` (when present), and `exception.stacktrace` attributes. Java's standard stack-trace text and the provider's ordinary diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 7687fc271..3a87dad20 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -98,7 +98,7 @@ static void applyOperationStatus(Span span, OperationEndInfo info, Instant endTi span.setStatus(StatusCode.ERROR, info.error().getMessage()); if (endTimestamp == null || endTimestamp.equals(Instant.EPOCH)) { // The SDK treats epoch zero as its current-clock sentinel when ending a span. - span.recordException(info.error()); + span.recordException(info.error(), operationExceptionAttributes(info.error())); } else { recordExceptionAt(span, info.error(), endTimestamp); } @@ -126,9 +126,25 @@ private static void recordExceptionAt(Span span, Throwable error, Instant timest error.printStackTrace(writer); } attributes.put("exception.stacktrace", stacktrace.toString()); + attributes.putAll(operationExceptionAttributes(error)); span.addEvent("exception", attributes.build(), timestamp); } + /** Override only the operation event's type; retain the actual throwable's message and stack diagnostics. */ + private static Attributes operationExceptionAttributes(Throwable error) { + if (error instanceof DurableOperationException failure) { + var details = failure.getErrorObject(); + if (details != null + && details.errorType() != null + && !details.errorType().isBlank()) { + return Attributes.builder() + .put("exception.type", details.errorType()) + .build(); + } + } + return Attributes.empty(); + } + /** Distinguishes operation telemetry details without changing the SDK's failure notification object. */ static boolean hasOperationErrorDetails(Throwable error) { if (error == null) return false; diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/BackendOperationTimingTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/BackendOperationTimingTest.java index 328dc35cf..d6c698ce3 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/BackendOperationTimingTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/BackendOperationTimingTest.java @@ -155,9 +155,7 @@ public void onOperationEnd(OperationEndInfo info) { assertEquals(1, span.getEvents().size()); var diagnostic = span.getEvents().get(0); var failure = ends.get(0).error(); - assertEquals( - failure.getClass().getCanonicalName(), - diagnostic.getAttributes().get(AttributeKey.stringKey("exception.type"))); + assertEquals(details.errorType(), diagnostic.getAttributes().get(AttributeKey.stringKey("exception.type"))); assertEquals( failure.getMessage(), diagnostic.getAttributes().get(AttributeKey.stringKey("exception.message"))); assertTrue(diagnostic diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CallbackErrorMetadataTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CallbackErrorMetadataTest.java index 546e55611..4cf1c2729 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CallbackErrorMetadataTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CallbackErrorMetadataTest.java @@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.*; import static software.amazon.lambda.durable.otel.SpanAttributes.*; +import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; @@ -51,6 +52,12 @@ static Stream errors() { ErrorObject.builder().errorType("").build(), true, false), + Arguments.of( + executionView, + "blank-type", + ErrorObject.builder().errorType(" \t\n").build(), + true, + false), Arguments.of( executionView, "message", @@ -176,6 +183,16 @@ public void onOperationEnd(OperationEndInfo info) { assertEquals(1, terminalSpans.size()); assertEquals(StatusCode.ERROR, terminalSpans.get(0).getStatus().getStatusCode()); assertEquals(hasDetails ? 1 : 0, terminalSpans.get(0).getEvents().size()); + if (hasDetails) { + assertEquals( + "type".equals(label) ? "ExternalFailure" : DurableOperationException.class.getCanonicalName(), + terminalSpans + .get(0) + .getEvents() + .get(0) + .getAttributes() + .get(AttributeKey.stringKey("exception.type"))); + } var replay = runner.run("input"); assertEquals(ExecutionStatus.FAILED, replay.getStatus()); assertEquals(result.getError(), replay.getError()); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OperationExceptionEventTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OperationExceptionEventTest.java index e5fc59f29..dae8711ee 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OperationExceptionEventTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OperationExceptionEventTest.java @@ -9,6 +9,7 @@ import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.testing.time.TestClock; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.SpanLimits; import io.opentelemetry.sdk.trace.data.SpanData; @@ -20,9 +21,14 @@ import java.util.List; import java.util.concurrent.CompletionException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.lambda.durable.exception.DurableOperationException; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; import software.amazon.lambda.durable.plugin.InvocationStatus; @@ -34,6 +40,83 @@ class OperationExceptionEventTest { private static final AttributeKey MESSAGE = AttributeKey.stringKey("exception.message"); private static final AttributeKey STACK = AttributeKey.stringKey("exception.stacktrace"); + static Stream storedErrors() { + return Stream.of(null, Instant.EPOCH, Instant.ofEpochSecond(20)) + .flatMap(end -> Stream.of( + Arguments.of(end, new RuntimeException("ordinary failure"), "java.lang.RuntimeException"), + Arguments.of( + end, + new DurableOperationException(null, null, "no stored error"), + DurableOperationException.class.getCanonicalName()), + Arguments.of(end, storedFailure(null), DurableOperationException.class.getCanonicalName()), + Arguments.of(end, storedFailure("java.lang.RuntimeException"), "java.lang.RuntimeException"), + Arguments.of(end, storedFailure("RejectedError"), "RejectedError"), + Arguments.of(end, storedFailure(" RejectedError "), " RejectedError "), + Arguments.of(end, storedFailure(""), DurableOperationException.class.getCanonicalName()), + Arguments.of(end, storedFailure(" "), DurableOperationException.class.getCanonicalName()), + Arguments.of(end, storedFailure("\t\r\n"), DurableOperationException.class.getCanonicalName()), + Arguments.of( + end, storedFailure("\u2003"), DurableOperationException.class.getCanonicalName()))); + } + + @ParameterizedTest + @MethodSource("storedErrors") + void storedTypeChangesOnlyTheOperationExceptionType(Instant end, Throwable error, String expectedType) { + try (var exporter = InMemorySpanExporter.create(); + var provider = SdkTracerProvider.builder() + .setClock(TestClock.create(Instant.ofEpochSecond(100))) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build()) { + var tracer = provider.get("operation-exception-test"); + var baseline = tracer.spanBuilder("baseline").startSpan(); + baseline.recordException(error); + baseline.end(); + var span = tracer.spanBuilder(NAME).startSpan(); + OtelPluginSupport.applyOperationStatus( + span, + new OperationEndInfo( + "op", NAME, "CALLBACK", "Callback", null, null, end, "FAILED", null, false, error), + end); + span.end(); + var spans = exporter.getFinishedSpanItems(); + assertEquals(StatusCode.ERROR, spans.get(1).getStatus().getStatusCode()); + assertEquals(error.getMessage(), spans.get(1).getStatus().getDescription()); + assertExceptionDetails( + spans.get(0), + spans.get(1), + expectedType, + end == null || end.equals(Instant.EPOCH) ? 100_000_000_000L : 20_000_000_000L); + } + } + + private static void assertExceptionDetails( + SpanData baseline, SpanData actual, String expectedType, long expectedTime) { + assertEquals(1, actual.getEvents().size()); + var event = actual.getEvents().get(0); + assertEquals("exception", event.getName()); + assertEquals( + baseline.getEvents().get(0).getAttributes().toBuilder() + .put(TYPE, expectedType) + .build() + .asMap(), + event.getAttributes().asMap()); + assertEquals(expectedTime, event.getEpochNanos()); + } + + private static DurableOperationException storedFailure(String type) { + var error = new DurableOperationException( + null, + ErrorObject.builder() + .errorType(type) + .errorMessage("stored failure") + .stackTrace("Example|step|Workflow.java|42") + .build(), + "wrapper message", + new IllegalArgumentException("cause")); + error.addSuppressed(new IllegalStateException("suppressed cleanup")); + return error; + } + @ParameterizedTest @ValueSource(booleans = {false, true}) void knownEndRetainsRealExceptionAndWrapperDiagnostics(boolean wrapped) { diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OperationExceptionTypeIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OperationExceptionTypeIntegrationTest.java new file mode 100644 index 000000000..343acca34 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OperationExceptionTypeIntegrationTest.java @@ -0,0 +1,209 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.exception.CallbackFailedException; +import software.amazon.lambda.durable.exception.InvokeFailedException; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +import software.amazon.lambda.durable.testing.TestResult; + +class OperationExceptionTypeIntegrationTest { + private static final String RUNTIME_EXCEPTION = RuntimeException.class.getName(); + private InMemorySpanExporter exporter; + + @BeforeEach + void setUp() { + resetSharedState(); + exporter = InMemorySpanExporter.create(); + } + + @AfterEach + void tearDown() { + exporter.close(); + resetSharedState(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void childContextReportsStoredTypeAndSkipsCompletedCodeOnReplay(boolean executionView) { + var calls = new AtomicInteger(); + var name = "otel-failed-child-context"; + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> context.runInChildContext(name, String.class, child -> { + calls.incrementAndGet(); + throw new RuntimeException("Intentional child-context failure"); + }), + config(executionView)); + + var result = runner.runUntilComplete("input"); + assertEquals(ExecutionStatus.FAILED, result.getStatus()); + var stored = result.getOperation(name).getContextDetails().error(); + assertEquals(RUNTIME_EXCEPTION, stored.errorType()); + assertLocalFailure(stored, result); + assertExceptionType(name, RUNTIME_EXCEPTION); + assertBoundaryExceptionTypes(RUNTIME_EXCEPTION); + assertReplayPreservesFailure(runner, result, calls); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void conditionReportsStoredTypeAndPreservesAttemptException(boolean executionView) { + var calls = new AtomicInteger(); + var name = "otel-failed-condition"; + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> context.waitForCondition(name, String.class, (state, step) -> { + calls.incrementAndGet(); + throw new RuntimeException("Intentional condition failure"); + }), + config(executionView)); + + var result = runner.runUntilComplete("input"); + assertEquals(ExecutionStatus.FAILED, result.getStatus()); + var stored = result.getOperation(name).getStepDetails().error(); + assertEquals(RUNTIME_EXCEPTION, stored.errorType()); + assertLocalFailure(stored, result); + assertExceptionType(name, RUNTIME_EXCEPTION); + assertExceptionType(name + " attempt 1", RUNTIME_EXCEPTION); + assertBoundaryExceptionTypes(RUNTIME_EXCEPTION); + assertReplayPreservesFailure(runner, result, calls); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void callbackReportsRemoteTypeAndPreservesBoundaryWrapper(boolean executionView) { + var submitterCalls = new AtomicInteger(); + var name = "otel-failed-callback"; + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> context.waitForCallback( + name, String.class, (callbackId, step) -> submitterCalls.incrementAndGet()), + config(executionView)); + assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus()); + var stored = remoteError("RejectedError"); + runner.failCallback(runner.getCallbackId(name + "-callback"), stored); + + var result = runner.runUntilComplete("input"); + assertEquals(ExecutionStatus.FAILED, result.getStatus()); + assertEquals( + stored, + result.getOperation(name + "-callback").getCallbackDetails().error()); + assertEquals(stored, result.getOperation(name).getContextDetails().error()); + assertEquals(stored, result.getError().orElseThrow()); + assertExceptionType(name, "RejectedError"); + assertExceptionType(name + "-callback", "RejectedError"); + assertBoundaryExceptionTypes(CallbackFailedException.class.getName()); + assertReplayPreservesFailure(runner, result, submitterCalls); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void invokeReportsStoredTypeAndPreservesBoundaryWrapper(boolean executionView) { + var stepCalls = new AtomicInteger(); + var name = "otel-failed-invoke"; + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + context.step("before-invoke", Integer.class, step -> stepCalls.incrementAndGet()); + return context.invoke(name, "target-fn", "{}", String.class); + }, + config(executionView)); + assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus()); + var stored = remoteError(RUNTIME_EXCEPTION); + runner.failChainedInvoke(name, stored); + + var result = runner.runUntilComplete("input"); + assertEquals(ExecutionStatus.FAILED, result.getStatus()); + assertEquals(stored, result.getOperation(name).getChainedInvokeDetails().error()); + assertEquals(stored, result.getError().orElseThrow()); + assertExceptionType(name, RUNTIME_EXCEPTION); + assertBoundaryExceptionTypes(InvokeFailedException.class.getName()); + assertReplayPreservesFailure(runner, result, stepCalls); + } + + private DurableConfig config(boolean executionView) { + var builder = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var settings = OtelPluginConfig.builder() + .contextExtractor(() -> null) + .enableMdc(false) + .build(); + DurableExecutionPluginFactory factory = executionView + ? ExecutionOtelPlugin.factory(builder, settings) + : InvocationOtelPlugin.factory(builder, settings); + return DurableConfig.builder().withPlugins(factory).build(); + } + + private void assertExceptionType(String name, String expectedType) { + var failedSpans = exporter.getFinishedSpanItems().stream() + .filter(span -> name.equals(span.getName())) + .filter(span -> span.getStatus().getStatusCode() == StatusCode.ERROR) + .toList(); + assertEquals(1, failedSpans.size(), name); + var events = failedSpans.get(0).getEvents(); + assertEquals(1, events.size(), name); + assertEquals("exception", events.get(0).getName()); + assertEquals(expectedType, events.get(0).getAttributes().get(AttributeKey.stringKey("exception.type"))); + } + + private void assertBoundaryExceptionTypes(String expectedType) { + assertExceptionType("Workflow", expectedType); + assertExceptionType("Invocation", expectedType); + } + + private static void assertLocalFailure(ErrorObject stored, TestResult result) { + var failure = result.getError().orElseThrow(); + assertEquals(stored.errorType(), failure.errorType()); + assertEquals(stored.errorMessage(), failure.errorMessage()); + assertEquals(stored.stackTrace(), failure.stackTrace()); + } + + private void assertReplayPreservesFailure( + LocalDurableTestRunner runner, TestResult result, AtomicInteger calls) { + exporter.reset(); + var replay = runner.run("input"); + assertEquals(ExecutionStatus.FAILED, replay.getStatus()); + assertEquals(result.getError(), replay.getError()); + assertEquals(1, calls.get(), "completed user code must not run again"); + // Without a remote parent, each invocation also re-exports the deterministic execution root. + assertEquals( + List.of("DurableExecutionRoot", "Invocation", "Workflow"), + exporter.getFinishedSpanItems().stream() + .map(SpanData::getName) + .sorted() + .toList(), + "replay must not emit operation or attempt spans"); + } + + private static ErrorObject remoteError(String type) { + return ErrorObject.builder() + .errorType(type) + .errorMessage("Intentional remote failure") + .stackTrace("Target|handle|Target.java|42") + .build(); + } + + private static void resetSharedState() { + DeterministicIdGenerator.clearSharedStateForTest(); + DurableSamplingDecision.clearSharedStateForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); + } +}