From 278f48f98e92724bf500ad2fe09d8578f50a27c3 Mon Sep 17 00:00:00 2001 From: Alex Wang Date: Tue, 1 Sep 2026 18:11:13 +0000 Subject: [PATCH 1/3] fix(insight): handle NaN sampling rates NaN compares False to everything, so a NaN sampling_rate flowed through _should_sample and sampled OUT every execution, silently disabling all instrumentation. _resolve_sampling_rate now fails open to 1.0 (full sampling), matching the JS plugin. Adds focused tests and a README note that on-change exporter calls run synchronously on the checkpoint path (async work tracked in #687). --- .../README.md | 5 +++++ .../plugin.py | 13 ++++++++++++ .../tests/test_plugin.py | 20 +++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/packages/aws-durable-execution-sdk-python-insight/README.md b/packages/aws-durable-execution-sdk-python-insight/README.md index 44196691..ed47c053 100644 --- a/packages/aws-durable-execution-sdk-python-insight/README.md +++ b/packages/aws-durable-execution-sdk-python-insight/README.md @@ -55,6 +55,11 @@ and `top-level` vs `full-tree` operation detail all mirror the JS plugin. Behavior is validated cross-SDK by the `insight` conformance suite (`aws-durable-execution-conformance-tests-insight`). +> **Note (`on-change` emission).** In `on-change` mode, exporter calls currently +> run synchronously on the SDK checkpoint path, so a slow exporter can delay +> workflow progress. Asynchronous scheduling/coalescing is deferred and tracked +> in [issue #687](https://github.com/aws/aws-durable-execution-sdk-python/issues/687). + ## Requirements - `aws-durable-execution-sdk-python` with the plugin invocation hooks that diff --git a/packages/aws-durable-execution-sdk-python-insight/src/aws_durable_execution_sdk_python_insight/plugin.py b/packages/aws-durable-execution-sdk-python-insight/src/aws_durable_execution_sdk_python_insight/plugin.py index d5d23be2..04c9a8ef 100644 --- a/packages/aws-durable-execution-sdk-python-insight/src/aws_durable_execution_sdk_python_insight/plugin.py +++ b/packages/aws-durable-execution-sdk-python-insight/src/aws_durable_execution_sdk_python_insight/plugin.py @@ -32,6 +32,7 @@ import datetime import json +import math import sys import threading from typing import Any, Callable @@ -107,6 +108,18 @@ def _resolve_sampling_rate(rate: float | None) -> float: return 1.0 if not isinstance(rate, (int, float)): return 1.0 + if isinstance(rate, float) and math.isnan(rate): + # Fail open. NaN compares False to everything, so without this guard it + # would flow through _should_sample (rate >= 1 -> False, rate <= 0 -> + # False, x < NaN -> False) and silently sample OUT every execution, + # disabling all instrumentation. Coerce to full sampling instead, which + # matches the JS plugin's treatment of non-finite/invalid rates. + print( + "[workflow-insight] sampling_rate is NaN; falling back to 1.0 " + "(full sampling)", + file=sys.stderr, + ) # noqa: T201 + return 1.0 if rate < 0 or rate > 1: return max(0.0, min(1.0, float(rate))) return float(rate) diff --git a/packages/aws-durable-execution-sdk-python-insight/tests/test_plugin.py b/packages/aws-durable-execution-sdk-python-insight/tests/test_plugin.py index 087ed0ca..54585a5b 100644 --- a/packages/aws-durable-execution-sdk-python-insight/tests/test_plugin.py +++ b/packages/aws-durable-execution-sdk-python-insight/tests/test_plugin.py @@ -38,6 +38,7 @@ WorkflowInsightConfig, workflow_insight, ) +from aws_durable_execution_sdk_python_insight.plugin import _resolve_sampling_rate ARN = "arn:aws:lambda:us-west-2:123456789012:function:my-fn:$LATEST/durable-execution/exec-1/inv-1" ARN_B = "arn:aws:lambda:us-west-2:123456789012:function:my-fn:$LATEST/durable-execution/exec-2/inv-1" @@ -196,6 +197,25 @@ def test_sampling_zero_emits_nothing(): assert exporter.records == [] +def test_resolve_sampling_rate_nan_fails_open_to_one(): + # NaN compares False to everything; without the guard this would sample OUT + # every execution. It must fail open to full sampling (JS parity). + assert _resolve_sampling_rate(float("nan")) == 1.0 + + +def test_nan_sampling_rate_emits_instead_of_silently_disabling(capsys): + exporter = CaptureExporter() + plugin = workflow_insight( + WorkflowInsightConfig(exporters=[exporter], sampling_rate=float("nan")) + ) + _run(plugin, ops=[_step("greet")]) + # A NaN rate must not disable instrumentation: the record is still emitted. + assert len(exporter.records) == 1 + # And a one-time warning is surfaced on stderr (library uses print/stderr, + # not the logging module). + assert "sampling_rate is NaN" in capsys.readouterr().err + + def test_content_omit_input_output_without_drop_flags(): exporter = CaptureExporter() plugin = workflow_insight( From 27f7bb04b42a18b08a27e122bdf59c589be9da92 Mon Sep 17 00:00:00 2001 From: Alex Wang Date: Tue, 1 Sep 2026 18:34:29 +0000 Subject: [PATCH 2/3] refactor(insight): log NaN fallback --- .../plugin.py | 10 +++++----- .../tests/test_plugin.py | 19 +++++++++++-------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-insight/src/aws_durable_execution_sdk_python_insight/plugin.py b/packages/aws-durable-execution-sdk-python-insight/src/aws_durable_execution_sdk_python_insight/plugin.py index 04c9a8ef..9a01318e 100644 --- a/packages/aws-durable-execution-sdk-python-insight/src/aws_durable_execution_sdk_python_insight/plugin.py +++ b/packages/aws-durable-execution-sdk-python-insight/src/aws_durable_execution_sdk_python_insight/plugin.py @@ -32,6 +32,7 @@ import datetime import json +import logging import math import sys import threading @@ -61,6 +62,9 @@ ) +logger = logging.getLogger(__name__) + + # Maps the SDK invocation status onto the record status. A durable execution # suspends (PENDING) while waiting; from the execution's point of view it is # still in flight, so surface it as RUNNING (mirrors the JS STATUS_MAP). @@ -114,11 +118,7 @@ def _resolve_sampling_rate(rate: float | None) -> float: # False, x < NaN -> False) and silently sample OUT every execution, # disabling all instrumentation. Coerce to full sampling instead, which # matches the JS plugin's treatment of non-finite/invalid rates. - print( - "[workflow-insight] sampling_rate is NaN; falling back to 1.0 " - "(full sampling)", - file=sys.stderr, - ) # noqa: T201 + logger.warning("sampling_rate is NaN; falling back to 1.0 (full sampling)") return 1.0 if rate < 0 or rate > 1: return max(0.0, min(1.0, float(rate))) diff --git a/packages/aws-durable-execution-sdk-python-insight/tests/test_plugin.py b/packages/aws-durable-execution-sdk-python-insight/tests/test_plugin.py index 54585a5b..1397b89e 100644 --- a/packages/aws-durable-execution-sdk-python-insight/tests/test_plugin.py +++ b/packages/aws-durable-execution-sdk-python-insight/tests/test_plugin.py @@ -13,6 +13,7 @@ from __future__ import annotations import datetime +import logging from typing import Any from aws_durable_execution_sdk_python.lambda_service import ( @@ -203,17 +204,19 @@ def test_resolve_sampling_rate_nan_fails_open_to_one(): assert _resolve_sampling_rate(float("nan")) == 1.0 -def test_nan_sampling_rate_emits_instead_of_silently_disabling(capsys): +def test_nan_sampling_rate_emits_instead_of_silently_disabling(caplog): exporter = CaptureExporter() - plugin = workflow_insight( - WorkflowInsightConfig(exporters=[exporter], sampling_rate=float("nan")) - ) - _run(plugin, ops=[_step("greet")]) + with caplog.at_level( + logging.WARNING, + logger="aws_durable_execution_sdk_python_insight.plugin", + ): + plugin = workflow_insight( + WorkflowInsightConfig(exporters=[exporter], sampling_rate=float("nan")) + ) + _run(plugin, ops=[_step("greet")]) # A NaN rate must not disable instrumentation: the record is still emitted. assert len(exporter.records) == 1 - # And a one-time warning is surfaced on stderr (library uses print/stderr, - # not the logging module). - assert "sampling_rate is NaN" in capsys.readouterr().err + assert "sampling_rate is NaN" in caplog.text def test_content_omit_input_output_without_drop_flags(): From 2d540e27d4d4ff0d1d91ad6816e501f6e6e9470b Mon Sep 17 00:00:00 2001 From: Alex Wang Date: Tue, 1 Sep 2026 18:38:25 +0000 Subject: [PATCH 3/3] refactor(insight): remove NaN warning --- .../plugin.py | 5 ----- .../tests/test_plugin.py | 16 +++++----------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-insight/src/aws_durable_execution_sdk_python_insight/plugin.py b/packages/aws-durable-execution-sdk-python-insight/src/aws_durable_execution_sdk_python_insight/plugin.py index 9a01318e..13f16a06 100644 --- a/packages/aws-durable-execution-sdk-python-insight/src/aws_durable_execution_sdk_python_insight/plugin.py +++ b/packages/aws-durable-execution-sdk-python-insight/src/aws_durable_execution_sdk_python_insight/plugin.py @@ -32,7 +32,6 @@ import datetime import json -import logging import math import sys import threading @@ -62,9 +61,6 @@ ) -logger = logging.getLogger(__name__) - - # Maps the SDK invocation status onto the record status. A durable execution # suspends (PENDING) while waiting; from the execution's point of view it is # still in flight, so surface it as RUNNING (mirrors the JS STATUS_MAP). @@ -118,7 +114,6 @@ def _resolve_sampling_rate(rate: float | None) -> float: # False, x < NaN -> False) and silently sample OUT every execution, # disabling all instrumentation. Coerce to full sampling instead, which # matches the JS plugin's treatment of non-finite/invalid rates. - logger.warning("sampling_rate is NaN; falling back to 1.0 (full sampling)") return 1.0 if rate < 0 or rate > 1: return max(0.0, min(1.0, float(rate))) diff --git a/packages/aws-durable-execution-sdk-python-insight/tests/test_plugin.py b/packages/aws-durable-execution-sdk-python-insight/tests/test_plugin.py index 1397b89e..485bcaab 100644 --- a/packages/aws-durable-execution-sdk-python-insight/tests/test_plugin.py +++ b/packages/aws-durable-execution-sdk-python-insight/tests/test_plugin.py @@ -13,7 +13,6 @@ from __future__ import annotations import datetime -import logging from typing import Any from aws_durable_execution_sdk_python.lambda_service import ( @@ -204,19 +203,14 @@ def test_resolve_sampling_rate_nan_fails_open_to_one(): assert _resolve_sampling_rate(float("nan")) == 1.0 -def test_nan_sampling_rate_emits_instead_of_silently_disabling(caplog): +def test_nan_sampling_rate_emits_instead_of_silently_disabling(): exporter = CaptureExporter() - with caplog.at_level( - logging.WARNING, - logger="aws_durable_execution_sdk_python_insight.plugin", - ): - plugin = workflow_insight( - WorkflowInsightConfig(exporters=[exporter], sampling_rate=float("nan")) - ) - _run(plugin, ops=[_step("greet")]) + plugin = workflow_insight( + WorkflowInsightConfig(exporters=[exporter], sampling_rate=float("nan")) + ) + _run(plugin, ops=[_step("greet")]) # A NaN rate must not disable instrumentation: the record is still emitted. assert len(exporter.records) == 1 - assert "sampling_rate is NaN" in caplog.text def test_content_omit_input_output_without_drop_flags():