fix(idempotency): serialize PydanticSerializer output in JSON mode (#8570)

* fix(idempotency): serialize PydanticSerializer output in JSON mode

PydanticSerializer.to_dict used model_dump() in Python mode, but
save_success stores the result with json.dumps and the shared Encoder,
which only handles Decimal, Pydantic models and dataclasses. A returned
model with a UUID, datetime, plain Enum, bytes or set field raised
TypeError after the function had already run. The record stayed
INPROGRESS, so a retry after the in-progress expiry ran the function
again.

Dump in JSON mode, as _prepare_data does for inputs since #8075.
from_dict validates the JSON values back into the model's types, and
models that already worked read back the same as before.

* docs(idempotency): warn about SecretStr and strict models with PydanticSerializer

---------

Co-authored-by: Leandro Damascena <lcdama@amazon.pt>

Author: AtulJoshi1206

aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py

@@ -28,7 +28,8 @@ def __init__(self, model: type[BaseModel]):
         self.__model: type[BaseModel] = model
 
     def to_dict(self, data: BaseModel) -> dict:
-        return data.model_dump()
+        # The result is stored as JSON, so dump JSON-compatible values; from_dict validates them back into the model.
+        return data.model_dump(mode="json")
 
     def from_dict(self, data: dict) -> BaseModel:
         return self.__model.model_validate(data)

docs/utilities/idempotency.md

@@ -243,6 +243,12 @@ The output serializer supports any JSON serializable data, **Python Dataclasses*
         --8<-- "examples/idempotency/src/working_with_pydantic_explicitly_output_serializer.py"
         ```
 
+    ???+ warning "Results are stored as JSON"
+        `PydanticSerializer` stores your model with `model_dump(mode="json")`, so fields such as `UUID`, `datetime`, `Enum`, `bytes` and `set` are saved as JSON values and validated back into the model on replay. Two cases need attention:
+
+        * `SecretStr` and `SecretBytes` fields are stored **masked** (`**********`), so a replayed result returns the masked value, not the secret. Keep secrets out of idempotent results.
+        * Models with `strict=True` and fields like `datetime` or `UUID` save correctly, but the replay fails validation because strict mode does not accept the stored strings. Use a non-strict model for the result, or a custom serializer.
+
 === "Dataclasses"
 
      Use `DataclassSerializer` to automatically serialize what's retrieved from the persistent storage based on the return type annotated.

tests/functional/idempotency/_pydantic/test_idempotency_with_pydantic.py

@@ -1,5 +1,11 @@
+import json
+from datetime import datetime, timezone
+from enum import Enum
+from uuid import UUID
+
 import pytest
 from pydantic import BaseModel
+from pytest_mock import MockerFixture
 
 from aws_lambda_powertools.utilities.idempotency import (
     IdempotencyConfig,
@@ -22,6 +28,7 @@
 from tests.functional.idempotency.utils import (
     hash_idempotency_key,
 )
+from tests.functional.utils import json_serialize
 
 TESTS_MODULE_PREFIX = "test-func.tests.functional.idempotency._pydantic.test_idempotency_with_pydantic"
 
@@ -263,3 +270,87 @@ def collect_payment(payment: PaymentInput) -> PaymentOutput | None:
     assert isinstance(second_call, PaymentOutput)
     assert second_call.customer_id == payment.customer_id
     assert second_call.transaction_id == payment.transaction_id
+
+
+class PaymentStatus(Enum):
+    PAID = "paid"
+
+
+class PaymentReceipt(BaseModel):
+    payment_id: UUID
+    paid_at: datetime
+    status: PaymentStatus
+    customer_id: str
+
+
+@pytest.fixture
+def payment_receipt() -> PaymentReceipt:
+    return PaymentReceipt(
+        payment_id=UUID("12345678-1234-5678-1234-567812345678"),
+        paid_at=datetime(2024, 3, 20, 14, 30, tzinfo=timezone.utc),
+        status=PaymentStatus.PAID,
+        customer_id="c-456",
+    )
+
+
+def test_pydantic_serializer_round_trips_non_json_types(payment_receipt: PaymentReceipt):
+    # GIVEN a serializer for a model with UUID, datetime and plain Enum fields
+    serializer = PydanticSerializer(model=PaymentReceipt)
+
+    # WHEN the model is stored as JSON, the same way the persistence layer saves a result
+    stored = json_serialize(serializer.to_dict(payment_receipt))
+
+    # THEN reading it back returns the same model
+    assert serializer.from_dict(json.loads(stored)) == payment_receipt
+
+
+@pytest.mark.parametrize("output_serializer_type", ["explicit", "deduced"])
+def test_idempotent_function_serialization_pydantic_with_non_json_types(
+    output_serializer_type: str,
+    payment_receipt: PaymentReceipt,
+    mocker: MockerFixture,
+):
+    # GIVEN a function that returns a model with UUID, datetime and plain Enum fields
+    config = IdempotencyConfig(use_local_cache=True)
+    mock_event = {"customer_id": "c-456", "transaction_id": "fake-id"}
+    idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_with_non_json_types.<locals>.collect_payment#{hash_idempotency_key(mock_event)}"  # noqa: E501
+    persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key)
+    update_record_spy = mocker.spy(persistence_layer, "_update_record")
+
+    class PaymentInput(BaseModel):
+        customer_id: str
+        transaction_id: str
+
+    if output_serializer_type == "explicit":
+        output_serializer = PydanticSerializer(
+            model=PaymentReceipt,
+        )
+    else:
+        output_serializer = PydanticSerializer
+
+    executions = []
+
+    @idempotent_function(
+        data_keyword_argument="payment",
+        persistence_store=persistence_layer,
+        config=config,
+        output_serializer=output_serializer,
+    )
+    def collect_payment(payment: PaymentInput) -> PaymentReceipt:
+        executions.append(payment)
+        return payment_receipt
+
+    # WHEN calling the function for the first time
+    payment = PaymentInput(**mock_event)
+    first_call = collect_payment(payment=payment)
+
+    # THEN the result is saved as a COMPLETED record
+    assert update_record_spy.call_args.kwargs["data_record"].status == "COMPLETED"
+
+    # WHEN calling the function again with the same payment
+    second_call = collect_payment(payment=payment)
+
+    # THEN the saved result is returned as the model, without running the function again
+    assert isinstance(second_call, PaymentReceipt)
+    assert second_call == first_call
+    assert executions == [payment]