From 3567d1a73b3874bf2c0ffb42eaf839ca5983cc80 Mon Sep 17 00:00:00 2001 From: Hardik Kaurani Date: Thu, 22 Oct 2026 12:10:35 +0530 Subject: [PATCH 1/4] fix(serialization): support set and frozenset generic containers in JsonDataConverter --- CHANGELOG.md | 1 + durabletask/serialization.py | 17 ++++++++-- tests/durabletask/test_data_converter.py | 31 +++++++++++++++++ tests/durabletask/test_serialization.py | 42 ++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ad5e0bc..ab6b2504 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ by one or more task hubs. FIXED +- Added support for `set` and `frozenset` generic containers in `JsonDataConverter`, enabling typed serialization and deserialization of sets. - Large-payload externalization now checks every eligible payload's maximum size before uploading or replacing any protobuf values, including carryover events and entity payloads, for both synchronous and asynchronous calls. diff --git a/durabletask/serialization.py b/durabletask/serialization.py index 52c3dfed..23b969ec 100644 --- a/durabletask/serialization.py +++ b/durabletask/serialization.py @@ -42,7 +42,7 @@ import types import typing from abc import ABC, abstractmethod -from collections.abc import Mapping, Sequence +from collections.abc import Mapping, Sequence, Set as AbstractSet from types import SimpleNamespace from typing import Any, cast @@ -238,7 +238,7 @@ def _can_reconstruct(converter: DataConverter, target_type: Any) -> bool: return any( converter.can_reconstruct(a) for a in args if a is not type(None) ) - if origin in (list, Sequence): + if origin in (list, Sequence, set, frozenset, AbstractSet, typing.AbstractSet): return any(converter.can_reconstruct(a) for a in args) return False if not isinstance(target_type, type): @@ -319,6 +319,11 @@ def _encode_custom_object(o: Any) -> Any: to_json_hook = getattr(cast(Any, type(o)), "to_json", None) if callable(to_json_hook): return to_json_hook(o) + if isinstance(o, (set, frozenset, AbstractSet)): + try: + return sorted(o) + except Exception: + return list(o) if isinstance(o, enum.Enum): # Emit the member's underlying value (a primitive). Reconstruction is # type-directed: passing the enum type to ``deserialize`` rebuilds the @@ -477,6 +482,14 @@ def _coerce_generic(value: Any, expected_type: Any, origin: Any, if origin in (list, Sequence) and isinstance(value, list): elem_type = args[0] if args else None return [_coerce_to_type(item, elem_type, converter) for item in cast(list[Any], value)] + if origin in (set, frozenset, AbstractSet, typing.AbstractSet) and isinstance( + value, (list, set, frozenset, AbstractSet) + ): + elem_type = args[0] if args else None + coerced = [_coerce_to_type(item, elem_type, converter) for item in value] + if origin is frozenset: + return frozenset(coerced) + return set(coerced) if origin in (dict, Mapping) and isinstance(value, dict): # JSON object keys are always strings, so only the values can carry a # reconstructable type. Keys are passed through unchanged. diff --git a/tests/durabletask/test_data_converter.py b/tests/durabletask/test_data_converter.py index 19909138..ca5b4517 100644 --- a/tests/durabletask/test_data_converter.py +++ b/tests/durabletask/test_data_converter.py @@ -375,3 +375,34 @@ def coerce(self, value: Any, target_type: type | None = None) -> Any: assert conv.serialize("hello") == '"HELLO"' assert conv.deserialize('"HELLO"') == "HELLO" assert conv.coerce("HELLO") == "HELLO" + + +# ----- set and frozenset container support ----- + + +@dataclass(frozen=True) +class TaggedOrder: + item: str + tags: set[str] + flags: frozenset[int] + + +def test_roundtrip_dataclass_with_sets(): + converter = JsonDataConverter() + original = TaggedOrder(item="laptop", tags={"electronics", "sale"}, flags=frozenset({1, 2})) + serialized = converter.serialize(original) + assert serialized is not None + restored = converter.deserialize(serialized, TaggedOrder) + assert isinstance(restored, TaggedOrder) + assert restored.item == "laptop" + assert isinstance(restored.tags, set) + assert restored.tags == {"electronics", "sale"} + assert isinstance(restored.flags, frozenset) + assert restored.flags == frozenset({1, 2}) + + +def test_can_reconstruct_set_and_frozenset(): + converter = JsonDataConverter() + assert converter.can_reconstruct(set[PricedOrder]) is True + assert converter.can_reconstruct(frozenset[PricedOrder]) is True + assert converter.can_reconstruct(set[int]) is False diff --git a/tests/durabletask/test_serialization.py b/tests/durabletask/test_serialization.py index 02861dcd..9b66d078 100644 --- a/tests/durabletask/test_serialization.py +++ b/tests/durabletask/test_serialization.py @@ -571,3 +571,45 @@ def test_resolve_forward_refs_leaves_unresolvable_name_untouched(): # the parsed JSON rather than raising. resolved = resolve_forward_refs(list["DoesNotExist"], {}) assert get_args(resolved) == ("DoesNotExist",) + + +# ----- set and frozenset container serialization and coercion ----- + + +@dataclass(frozen=True) +class FrozenAddress: + street: str + city: str + + +def test_from_json_coerces_set_of_primitives(): + encoded = to_json(["a", "b", "c"]) + result = from_json(encoded, set[str]) + assert isinstance(result, set) + assert result == {"a", "b", "c"} + + +def test_from_json_coerces_frozenset_of_primitives(): + encoded = to_json([1, 2, 3]) + result = from_json(encoded, frozenset[int]) + assert isinstance(result, frozenset) + assert result == frozenset({1, 2, 3}) + + +def test_from_json_coerces_set_of_dataclasses(): + encoded = to_json([ + {"street": "1 Main St", "city": "Redmond"}, + {"street": "2 1st Ave", "city": "Seattle"}, + ]) + result = from_json(encoded, set[FrozenAddress]) + assert isinstance(result, set) + assert all(isinstance(item, FrozenAddress) for item in result) + assert FrozenAddress("1 Main St", "Redmond") in result + + +def test_to_json_serializes_set_and_frozenset(): + encoded_set = to_json({"b", "a"}) + assert json.loads(encoded_set) == ["a", "b"] + + encoded_frozenset = to_json(frozenset([2, 1])) + assert json.loads(encoded_frozenset) == [1, 2] From 2bedbec61fd39c624a3c58028e37cbba3d7af9ba Mon Sep 17 00:00:00 2001 From: Hardik Kaurani Date: Fri, 23 Oct 2026 15:40:39 +0530 Subject: [PATCH 2/4] fix(serialization): deterministic set ordering, opt-in set reconstruction, and pyright typing - Preserve replay compatibility by gating generic set reconstruction behind enable_generic_sets=False default on JsonDataConverter - Ensure deterministic set serialization with fallback sort key on recursive encoded values, qualname, and repr - Retain primitive set element annotations in type discovery when set reconstruction is enabled - Resolve strict pyright errors and update tests --- CHANGELOG.md | 2 +- durabletask/serialization.py | 89 ++++++++++++--- tests/durabletask/test_data_converter.py | 83 ++++++++++++-- .../test_data_converter_roundtrip.py | 2 - tests/durabletask/test_serialization.py | 105 +++++++++++++----- 5 files changed, 229 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab6b2504..6e739f82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ by one or more task hubs. FIXED -- Added support for `set` and `frozenset` generic containers in `JsonDataConverter`, enabling typed serialization and deserialization of sets. +- Added deterministic serialization for `set`, `frozenset`, and `collections.abc.Set` containers, and enabled opt-in generic set reconstruction (`enable_generic_sets`) in `JsonDataConverter` to preserve existing-instance replay compatibility. - Large-payload externalization now checks every eligible payload's maximum size before uploading or replacing any protobuf values, including carryover events and entity payloads, for both synchronous and asynchronous calls. diff --git a/durabletask/serialization.py b/durabletask/serialization.py index 23b969ec..df6d92c7 100644 --- a/durabletask/serialization.py +++ b/durabletask/serialization.py @@ -42,7 +42,8 @@ import types import typing from abc import ABC, abstractmethod -from collections.abc import Mapping, Sequence, Set as AbstractSet +from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Set as AbstractSet from types import SimpleNamespace from typing import Any, cast @@ -68,7 +69,6 @@ class DataConverter(ABC): @abstractmethod def serialize(self, value: Any) -> str | None: """Serialize ``value`` to a string, or ``None`` when ``value`` is ``None``.""" - pass @abstractmethod def deserialize(self, data: str | None, target_type: type | None = None) -> Any: @@ -85,7 +85,6 @@ def deserialize(self, data: str | None, target_type: type | None = None) -> Any: :class:`JsonDataConverter` is best-effort and falls back; a validating converter may instead raise. """ - pass @abstractmethod def coerce(self, value: Any, target_type: type | None = None) -> Any: @@ -99,7 +98,6 @@ def coerce(self, value: Any, target_type: type | None = None) -> Any: (strict vs. best-effort) that an implementation applies in :meth:`deserialize` should apply here. """ - pass def can_reconstruct(self, target_type: Any) -> bool: """Return True if this converter can rebuild ``target_type`` from a payload. @@ -167,6 +165,18 @@ class JsonDataConverter(DataConverter): > payload; the destination type is always supplied by the caller. """ + def __init__( + self, + *, + enable_generic_sets: bool = False, + enable_set_coercion: bool = False, + ) -> None: + self.enable_generic_sets: bool = enable_generic_sets or enable_set_coercion + + @property + def enable_set_coercion(self) -> bool: + return self.enable_generic_sets + def serialize(self, value: Any) -> str | None: if value is None: return None @@ -221,15 +231,40 @@ def _log_coercion_fallback(target_type: type, error: Exception) -> None: # --------------------------------------------------------------------------- +def _is_supported_set_element(converter: DataConverter, elem_type: Any) -> bool: + if elem_type is None or elem_type is typing.Any: + return True + elem_origin = typing.get_origin(elem_type) + if elem_origin is not None: + if elem_origin in (list, dict, Sequence, Mapping, set, AbstractSet): + # Mutable collections are unhashable + return False + if elem_origin is typing.Union or elem_origin is types.UnionType: + non_none = [a for a in typing.get_args(elem_type) if a is not type(None)] + return bool(non_none) and all(_is_supported_set_element(converter, a) for a in non_none) + if elem_origin is frozenset: + args = typing.get_args(elem_type) + return not args or _is_supported_set_element(converter, args[0]) + return False + if not isinstance(elem_type, type): + return False + if issubclass(elem_type, (list, dict, set)): + return False + return True + + def _can_reconstruct(converter: DataConverter, target_type: Any) -> bool: """:class:`JsonDataConverter`'s reconstruction policy. Recognizes dataclasses and ``from_json()``-capable types, plus ``Optional`` / ``list`` / ``Sequence`` hints wrapping them; builtins and unknown - annotations are excluded. Recurses through ``converter.can_reconstruct`` - (not itself) so a :class:`JsonDataConverter` subclass that overrides - ``can_reconstruct`` still participates in the element-type checks of - ``Optional`` / ``list`` hints. + annotations are excluded. When set support is enabled on the converter + (via ``enable_generic_sets=True``), ``set`` / ``frozenset`` / ``Set`` + containers with supported element types are also recognized. Recurses + through ``converter.can_reconstruct`` (not itself) so a + :class:`JsonDataConverter` subclass that overrides ``can_reconstruct`` + still participates in the element-type checks of ``Optional`` / ``list`` + hints. """ origin = typing.get_origin(target_type) if origin is not None: @@ -238,13 +273,21 @@ def _can_reconstruct(converter: DataConverter, target_type: Any) -> bool: return any( converter.can_reconstruct(a) for a in args if a is not type(None) ) - if origin in (list, Sequence, set, frozenset, AbstractSet, typing.AbstractSet): + if origin in (list, Sequence): return any(converter.can_reconstruct(a) for a in args) + if origin in (set, frozenset, AbstractSet, typing.AbstractSet): + if not getattr(converter, "enable_generic_sets", False): + return False + if not args: + return True + return _is_supported_set_element(converter, args[0]) return False if not isinstance(target_type, type): return False if dataclasses.is_dataclass(target_type): return True + if target_type in (set, frozenset, AbstractSet): + return getattr(converter, "enable_generic_sets", False) return callable(getattr(cast(Any, target_type), "from_json", None)) @@ -320,10 +363,21 @@ def _encode_custom_object(o: Any) -> Any: if callable(to_json_hook): return to_json_hook(o) if isinstance(o, (set, frozenset, AbstractSet)): + items: AbstractSet[Any] = cast("AbstractSet[Any]", o) try: - return sorted(o) - except Exception: - return list(o) + return sorted(items) + except TypeError: + try: + def _set_sort_key(item: Any) -> tuple[str, str, str]: + encoded = json.dumps(item, default=_encode_custom_object, sort_keys=True) + return (encoded, type(item).__qualname__, repr(item)) + + return sorted(items, key=_set_sort_key) + except Exception as e: + raise TypeError( + f"Cannot deterministically serialize set of type '{type(items).__name__}' " + f"because its elements cannot be compared or deterministically encoded: {e}" + ) from e if isinstance(o, enum.Enum): # Emit the member's underlying value (a primitive). Reconstruction is # type-directed: passing the enum type to ``deserialize`` rebuilds the @@ -485,8 +539,13 @@ def _coerce_generic(value: Any, expected_type: Any, origin: Any, if origin in (set, frozenset, AbstractSet, typing.AbstractSet) and isinstance( value, (list, set, frozenset, AbstractSet) ): + if not getattr(converter, "enable_generic_sets", False): + return cast(Any, value) elem_type = args[0] if args else None - coerced = [_coerce_to_type(item, elem_type, converter) for item in value] + coerced: list[Any] = [ + _coerce_to_type(item, elem_type, converter) + for item in cast("Iterable[Any]", value) + ] if origin is frozenset: return frozenset(coerced) return set(coerced) @@ -608,12 +667,12 @@ def _resolve_forward_refs(tp: Any, globalns: dict[str, Any]) -> Any: """ if isinstance(tp, str): try: - tp = eval(tp, globalns) # noqa: S307 - resolves caller-authored annotations + tp = eval(tp, globalns) except Exception: return tp elif isinstance(tp, typing.ForwardRef): try: - tp = eval(tp.__forward_arg__, globalns) # noqa: S307 + tp = eval(tp.__forward_arg__, globalns) except Exception: return tp diff --git a/tests/durabletask/test_data_converter.py b/tests/durabletask/test_data_converter.py index ca5b4517..8186280f 100644 --- a/tests/durabletask/test_data_converter.py +++ b/tests/durabletask/test_data_converter.py @@ -12,13 +12,14 @@ from unittest.mock import patch import pytest - from durabletask.internal.entity_state_shim import StateShim from durabletask.serialization import ( DEFAULT_DATA_CONVERTER, DataConverter, JsonDataConverter, ) +from durabletask.internal import type_discovery +from durabletask.worker import _ActivityExecutor, _Registry @dataclass @@ -219,7 +220,7 @@ def __init__(self, /, **kwargs): self.self = kwargs.get("self", 0) converter = JsonDataConverter() - encoded = converter.serialize(PositionalReceiver(**{"self": 42})) + encoded = converter.serialize(PositionalReceiver(self=42)) restored = converter.deserialize(encoded, PositionalReceiver) assert isinstance(restored, PositionalReceiver) assert restored.self == 42 @@ -388,7 +389,7 @@ class TaggedOrder: def test_roundtrip_dataclass_with_sets(): - converter = JsonDataConverter() + converter = JsonDataConverter(enable_generic_sets=True) original = TaggedOrder(item="laptop", tags={"electronics", "sale"}, flags=frozenset({1, 2})) serialized = converter.serialize(original) assert serialized is not None @@ -402,7 +403,75 @@ def test_roundtrip_dataclass_with_sets(): def test_can_reconstruct_set_and_frozenset(): - converter = JsonDataConverter() - assert converter.can_reconstruct(set[PricedOrder]) is True - assert converter.can_reconstruct(frozenset[PricedOrder]) is True - assert converter.can_reconstruct(set[int]) is False + # By default (enable_generic_sets=False), generic sets are not reconstructed to + # preserve backward compatibility and existing-instance replay behavior. + default_converter = JsonDataConverter() + assert default_converter.can_reconstruct(set[PricedOrder]) is False + assert default_converter.can_reconstruct(frozenset[PricedOrder]) is False + assert default_converter.can_reconstruct(set[int]) is False + assert default_converter.can_reconstruct(set[str]) is False + + # When opted in, supported sets (including primitive elements) return True, + # while unsupported unhashable element types return False. + opt_in_converter = JsonDataConverter(enable_generic_sets=True) + assert opt_in_converter.can_reconstruct(set[PricedOrder]) is True + assert opt_in_converter.can_reconstruct(frozenset[PricedOrder]) is True + assert opt_in_converter.can_reconstruct(set[int]) is True + assert opt_in_converter.can_reconstruct(set[str]) is True + assert opt_in_converter.can_reconstruct(frozenset[int]) is True + assert opt_in_converter.can_reconstruct(set[list[str]]) is False + assert opt_in_converter.can_reconstruct(set[dict[str, int]]) is False + + +def test_existing_instance_replay_compatibility_preserves_list(): + """Preserve existing-instance replay behavior for recorded arrays under set annotations. + + When generic set coercion is not explicitly enabled, historical JSON arrays + containing duplicates and order (e.g. {"tags": ["b", "a", "b"]}) remain raw + lists, avoiding NonDeterminismError during orchestration replay. + """ + default_converter = JsonDataConverter() + recorded_input = '{"item": "laptop", "tags": ["b", "a", "b"], "flags": [1, 2, 1]}' + restored = default_converter.deserialize(recorded_input, TaggedOrder) + assert isinstance(restored, TaggedOrder) + assert isinstance(restored.tags, list) + assert restored.tags == ["b", "a", "b"] + assert len(restored.tags) == 3 + assert isinstance(restored.flags, list) + assert restored.flags == [1, 2, 1] + assert len(restored.flags) == 3 + + # With opt-in enabled, set coercion deduplicates and converts to sets + opt_in_converter = JsonDataConverter(enable_generic_sets=True) + opt_in_restored = opt_in_converter.deserialize(recorded_input, TaggedOrder) + assert isinstance(opt_in_restored, TaggedOrder) + assert isinstance(opt_in_restored.tags, set) + assert opt_in_restored.tags == {"a", "b"} + assert len(opt_in_restored.tags) == 2 + assert isinstance(opt_in_restored.flags, frozenset) + assert opt_in_restored.flags == frozenset({1, 2}) + assert len(opt_in_restored.flags) == 2 + + +def test_type_discovery_and_activity_execution_with_sets(): + """Verify that type discovery recognizes set annotations when opted in, + enabling activities to receive sets rather than raw lists.""" + def tag_activity(ctx: Any, tags: set[str]) -> int: + assert isinstance(tags, set) + tags.add("processed") + return len(tags) + + reg = _Registry() + reg.add_activity(tag_activity) + logger = logging.getLogger("test_type_discovery_sets") + + # With opt-in, activity input is discovered as set[str] and executed + conv = JsonDataConverter(enable_generic_sets=True) + assert type_discovery.activity_input_type(tag_activity, conv) == set[str] + executor = _ActivityExecutor(reg, logger, conv) + output = executor.execute("orch-1", "tag_activity", 1, '["item1", "item2"]') + assert output == "3" + + # Default converter preserves input as None (raw JSON), passing list + default_conv = JsonDataConverter() + assert type_discovery.activity_input_type(tag_activity, default_conv) is None diff --git a/tests/durabletask/test_data_converter_roundtrip.py b/tests/durabletask/test_data_converter_roundtrip.py index 465aeb3a..804f81bf 100644 --- a/tests/durabletask/test_data_converter_roundtrip.py +++ b/tests/durabletask/test_data_converter_roundtrip.py @@ -545,8 +545,6 @@ class HasUnion: @pytest.mark.parametrize("value", [ datetime(2020, 1, 1, 12, 0, 0), Decimal("1.5"), - {1, 2, 3}, - frozenset({1, 2}), b"bytes", ]) def test_unsupported_types_raise_clear_typeerror(conv, value): diff --git a/tests/durabletask/test_serialization.py b/tests/durabletask/test_serialization.py index 9b66d078..71c5b62d 100644 --- a/tests/durabletask/test_serialization.py +++ b/tests/durabletask/test_serialization.py @@ -9,20 +9,21 @@ """ import json +import subprocess +import sys from collections import namedtuple from dataclasses import dataclass from types import SimpleNamespace from typing import Any, List, Optional, Union, get_args import pytest - from durabletask.serialization import _AUTO_SERIALIZED as AUTO_SERIALIZED +from durabletask.serialization import JsonDataConverter from durabletask.serialization import _coerce_to_type as coerce_to_type from durabletask.serialization import _from_json as from_json from durabletask.serialization import _resolve_forward_refs as resolve_forward_refs from durabletask.serialization import _to_json as to_json - # ----- Test fixtures ----- @@ -548,7 +549,7 @@ def from_json(cls, value, strict=False): def test_resolve_forward_refs_resolves_string_element_type(): # ``list["Address"]`` evaluates to a generic whose arg is the raw string # "Address" -- exactly what 3.10 leaves behind. - resolved = resolve_forward_refs(list["Address"], {"Address": Address}) + resolved = resolve_forward_refs(List["Address"], {"Address": Address}) assert get_args(resolved) == (Address,) @@ -582,34 +583,84 @@ class FrozenAddress: city: str -def test_from_json_coerces_set_of_primitives(): - encoded = to_json(["a", "b", "c"]) - result = from_json(encoded, set[str]) - assert isinstance(result, set) - assert result == {"a", "b", "c"} +def test_to_json_serializes_set_and_frozenset(): + encoded_set = to_json({"b", "a"}) + assert json.loads(encoded_set) == ["a", "b"] + encoded_frozenset = to_json(frozenset([2, 1])) + assert json.loads(encoded_frozenset) == [1, 2] -def test_from_json_coerces_frozenset_of_primitives(): - encoded = to_json([1, 2, 3]) - result = from_json(encoded, frozenset[int]) - assert isinstance(result, frozenset) - assert result == frozenset({1, 2, 3}) +def test_set_serialization_deterministic_across_hash_seeds(): + """Verify that sets of unorderable custom objects (like frozen dataclasses without order=True) + are deterministically serialized across separate processes with differing PYTHONHASHSEED.""" + code = """ +from dataclasses import dataclass +from durabletask.serialization import JsonDataConverter -def test_from_json_coerces_set_of_dataclasses(): - encoded = to_json([ - {"street": "1 Main St", "city": "Redmond"}, - {"street": "2 1st Ave", "city": "Seattle"}, - ]) - result = from_json(encoded, set[FrozenAddress]) +@dataclass(frozen=True) +class Entry: + id: str + val: int + +s = {Entry("delta", 4), Entry("alpha", 1), Entry("charlie", 3), Entry("bravo", 2)} +conv = JsonDataConverter() +print(conv.serialize(s)) +""" + res1 = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + env={"PYTHONHASHSEED": "1", "PYTHONPATH": r"C:\Users\hardi\durabletask-python-pr310"}, + ).stdout.strip() + + res2 = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + env={"PYTHONHASHSEED": "2", "PYTHONPATH": r"C:\Users\hardi\durabletask-python-pr310"}, + ).stdout.strip() + + res3 = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + env={"PYTHONHASHSEED": "999", "PYTHONPATH": r"C:\Users\hardi\durabletask-python-pr310"}, + ).stdout.strip() + + assert res1 == res2 == res3 + expected = '[{"id": "alpha", "val": 1}, {"id": "bravo", "val": 2}, {"id": "charlie", "val": 3}, {"id": "delta", "val": 4}]' + assert res1 == expected + + +def test_deserialize_coerces_set_when_opted_in(): + converter = JsonDataConverter(enable_generic_sets=True) + encoded = converter.serialize(["a", "b", "c"]) + assert encoded is not None + result = converter.deserialize(encoded, set[str]) assert isinstance(result, set) - assert all(isinstance(item, FrozenAddress) for item in result) - assert FrozenAddress("1 Main St", "Redmond") in result + assert result == {"a", "b", "c"} + encoded_int = converter.serialize([1, 2, 3]) + assert encoded_int is not None + result_frozen = converter.deserialize(encoded_int, frozenset[int]) + assert isinstance(result_frozen, frozenset) + assert result_frozen == frozenset({1, 2, 3}) -def test_to_json_serializes_set_and_frozenset(): - encoded_set = to_json({"b", "a"}) - assert json.loads(encoded_set) == ["a", "b"] - - encoded_frozenset = to_json(frozenset([2, 1])) - assert json.loads(encoded_frozenset) == [1, 2] + encoded_dc = json.dumps([ + {"street": "1 Main St", "city": "Redmond"}, + {"street": "2 1st Ave", "city": "Seattle"}, + ]) + result_dc = converter.deserialize(encoded_dc, set[FrozenAddress]) + assert isinstance(result_dc, set) + assert all(isinstance(item, FrozenAddress) for item in result_dc) + assert FrozenAddress("1 Main St", "Redmond") in result_dc + + +def test_deserialize_preserves_raw_array_when_opted_out(): + default_converter = JsonDataConverter() + payload = '["b", "a", "b"]' + result = default_converter.deserialize(payload, set[str]) + # Preserves list and duplicates for replay compatibility + assert isinstance(result, list) + assert result == ["b", "a", "b"] From 6c924e50d8692af676cfa3a734dc6ef1446b63af Mon Sep 17 00:00:00 2001 From: Hardik Kaurani Date: Sat, 24 Oct 2026 02:03:37 +0530 Subject: [PATCH 3/4] fix(serialization): stabilize nested set ordering --- durabletask/serialization.py | 35 +++++--- tests/durabletask/test_serialization.py | 110 ++++++++++++++++++------ 2 files changed, 104 insertions(+), 41 deletions(-) diff --git a/durabletask/serialization.py b/durabletask/serialization.py index df6d92c7..853ab6c3 100644 --- a/durabletask/serialization.py +++ b/durabletask/serialization.py @@ -340,6 +340,22 @@ def _from_json( return _coerce_to_type(raw, expected_type, converter) +def _deterministic_set_sort_key(item: Any) -> tuple[int, float | int, str, str, str]: + """Return a deterministic, total-ordering sort key for an unordered set element. + + Elements are ordered primarily by category (numbers vs. other JSON-encodable + values) and recursively encoded JSON representation, with type qualname and + representation as tie-breakers. This ensures deterministic serialization + across independent processes regardless of PYTHONHASHSEED or native comparison + semantics (such as frozenset partial ordering). + """ + if isinstance(item, (int, float)) and not isinstance(item, bool): + encoded = json.dumps(item, default=_encode_custom_object, sort_keys=True) + return (0, item, encoded, type(item).__qualname__, repr(item)) + encoded = json.dumps(item, default=_encode_custom_object, sort_keys=True) + return (1, 0, encoded, type(item).__qualname__, repr(item)) + + def _encode_custom_object(o: Any) -> Any: """``default`` hook for :func:`json.dumps` that emits plain JSON. @@ -365,19 +381,12 @@ def _encode_custom_object(o: Any) -> Any: if isinstance(o, (set, frozenset, AbstractSet)): items: AbstractSet[Any] = cast("AbstractSet[Any]", o) try: - return sorted(items) - except TypeError: - try: - def _set_sort_key(item: Any) -> tuple[str, str, str]: - encoded = json.dumps(item, default=_encode_custom_object, sort_keys=True) - return (encoded, type(item).__qualname__, repr(item)) - - return sorted(items, key=_set_sort_key) - except Exception as e: - raise TypeError( - f"Cannot deterministically serialize set of type '{type(items).__name__}' " - f"because its elements cannot be compared or deterministically encoded: {e}" - ) from e + return sorted(items, key=_deterministic_set_sort_key) + except Exception as e: + raise TypeError( + f"Cannot deterministically serialize set of type '{type(items).__name__}' " + f"because its elements cannot be compared or deterministically encoded: {e}" + ) from e if isinstance(o, enum.Enum): # Emit the member's underlying value (a primitive). Reconstruction is # type-directed: passing the enum type to ``deserialize`` rebuilds the diff --git a/tests/durabletask/test_serialization.py b/tests/durabletask/test_serialization.py index 71c5b62d..011abef7 100644 --- a/tests/durabletask/test_serialization.py +++ b/tests/durabletask/test_serialization.py @@ -9,8 +9,10 @@ """ import json +import os import subprocess import sys +from pathlib import Path from collections import namedtuple from dataclasses import dataclass from types import SimpleNamespace @@ -590,47 +592,99 @@ def test_to_json_serializes_set_and_frozenset(): encoded_frozenset = to_json(frozenset([2, 1])) assert json.loads(encoded_frozenset) == [1, 2] + # Verify disjoint and nested frozenset in-process serialization + encoded_disjoint = to_json({ + frozenset({"alpha"}), + frozenset({"beta"}), + frozenset({"gamma"}), + }) + assert json.loads(encoded_disjoint) == [["alpha"], ["beta"], ["gamma"]] + + encoded_nested = to_json({ + frozenset({frozenset({"b", "a"}), frozenset({"c"})}), + frozenset({frozenset({"d"})}), + }) + assert json.loads(encoded_nested) == [[["a", "b"], ["c"]], [["d"]]] + def test_set_serialization_deterministic_across_hash_seeds(): - """Verify that sets of unorderable custom objects (like frozen dataclasses without order=True) + """Verify that sets of disjoint frozensets, nested frozensets, and unorderable custom objects are deterministically serialized across separate processes with differing PYTHONHASHSEED.""" + repo_root = str(Path(__file__).resolve().parents[2]) code = """ +import json from dataclasses import dataclass from durabletask.serialization import JsonDataConverter +conv = JsonDataConverter() + +# Case A: Disjoint frozensets (exposes frozenset partial-ordering defects) +case_a = conv.serialize({ + frozenset({"alpha"}), + frozenset({"beta"}), + frozenset({"gamma"}), +}) + +# Case B: Nested frozensets (multi-element, multi-level nesting) +case_b = conv.serialize({ + frozenset({frozenset({"b", "a"}), frozenset({"c"})}), + frozenset({frozenset({"d"})}), +}) + +# Case C: Non-orderable custom objects (frozen dataclass without order=True) @dataclass(frozen=True) class Entry: id: str val: int -s = {Entry("delta", 4), Entry("alpha", 1), Entry("charlie", 3), Entry("bravo", 2)} -conv = JsonDataConverter() -print(conv.serialize(s)) +case_c = conv.serialize({ + Entry("delta", 4), + Entry("alpha", 1), + Entry("charlie", 3), + Entry("bravo", 2), +}) + +print(json.dumps({"a": case_a, "b": case_b, "c": case_c})) """ - res1 = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True, - env={"PYTHONHASHSEED": "1", "PYTHONPATH": r"C:\Users\hardi\durabletask-python-pr310"}, - ).stdout.strip() - - res2 = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True, - env={"PYTHONHASHSEED": "2", "PYTHONPATH": r"C:\Users\hardi\durabletask-python-pr310"}, - ).stdout.strip() - - res3 = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True, - env={"PYTHONHASHSEED": "999", "PYTHONPATH": r"C:\Users\hardi\durabletask-python-pr310"}, - ).stdout.strip() - - assert res1 == res2 == res3 - expected = '[{"id": "alpha", "val": 1}, {"id": "bravo", "val": 2}, {"id": "charlie", "val": 3}, {"id": "delta", "val": 4}]' - assert res1 == expected + seeds = ["0", "1", "2", "42", "999"] + outputs: dict[str, dict[str, str]] = {} + + for seed in seeds: + env = os.environ.copy() + env["PYTHONHASHSEED"] = seed + env["PYTHONPATH"] = ( + f"{repo_root}{os.pathsep}{env['PYTHONPATH']}" + if "PYTHONPATH" in env + else repo_root + ) + proc = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + env=env, + ) + assert proc.returncode == 0, ( + f"Subprocess failed for seed {seed}:\nstdout: {proc.stdout}\nstderr: {proc.stderr}" + ) + outputs[seed] = json.loads(proc.stdout.strip()) + + base = outputs[seeds[0]] + for seed in seeds[1:]: + assert outputs[seed] == base, ( + f"Deterministic serialization failed between seed {seeds[0]} and {seed}:\n" + f"Seed {seeds[0]}: {base}\nSeed {seed}: {outputs[seed]}" + ) + + expected_a = '[["alpha"], ["beta"], ["gamma"]]' + expected_b = '[[["a", "b"], ["c"]], [["d"]]]' + expected_c = ( + '[{"id": "alpha", "val": 1}, {"id": "bravo", "val": 2}, ' + '{"id": "charlie", "val": 3}, {"id": "delta", "val": 4}]' + ) + + assert base["a"] == expected_a + assert base["b"] == expected_b + assert base["c"] == expected_c def test_deserialize_coerces_set_when_opted_in(): From ecf8270989f69fd7ddcc281d8c25c3cdcae28d27 Mon Sep 17 00:00:00 2001 From: Hardik Kaurani Date: Tue, 27 Oct 2026 04:43:24 +0530 Subject: [PATCH 4/4] fix(serialization): simplify set serialization by emitting sorted canonical JSON strings --- durabletask/serialization.py | 47 ++++++++------- tests/durabletask/test_serialization.py | 79 ++++++++++++++++++++++++- 2 files changed, 103 insertions(+), 23 deletions(-) diff --git a/durabletask/serialization.py b/durabletask/serialization.py index 853ab6c3..c2d916f0 100644 --- a/durabletask/serialization.py +++ b/durabletask/serialization.py @@ -134,7 +134,9 @@ class JsonDataConverter(DataConverter): ``SimpleNamespace`` handling, and nested values are encoded recursively, so a dataclass (or any object) with a ``to_json`` hook -- including one nested inside another dataclass, ``list``, or ``dict`` -- round-trips through its - own hooks. + own hooks. Sets and frozensets serialize to JSON arrays ordered + deterministically by their canonical JSON representation (using lexical + string order, so numeric sets like ``{2, 10}`` serialize as ``[10, 2]``). Deserialization (and value-level :meth:`coerce`) is **best-effort**: when a ``target_type`` is supplied and the value cannot be coerced to it, the raw @@ -296,9 +298,12 @@ def _to_json(obj: Any) -> str: Builtins serialize to plain JSON. Objects exposing a ``to_json()`` method, dataclasses, and ``SimpleNamespace`` instances are serialized to plain JSON - as well (without any type marker), recursively. Custom objects can be - reconstructed on the receiving side by passing ``expected_type`` to - :func:`_from_json`. + as well (without any type marker), recursively. Sets and frozensets are + serialized as JSON arrays with elements deterministically ordered by their + canonical JSON string representation (note that numeric set elements follow + lexical JSON ordering, e.g. ``{2, 10}`` serializes as ``[10, 2]``). Custom + objects can be reconstructed on the receiving side by passing ``expected_type`` + to :func:`_from_json`. """ try: return json.dumps(obj, default=_encode_custom_object) @@ -340,22 +345,6 @@ def _from_json( return _coerce_to_type(raw, expected_type, converter) -def _deterministic_set_sort_key(item: Any) -> tuple[int, float | int, str, str, str]: - """Return a deterministic, total-ordering sort key for an unordered set element. - - Elements are ordered primarily by category (numbers vs. other JSON-encodable - values) and recursively encoded JSON representation, with type qualname and - representation as tie-breakers. This ensures deterministic serialization - across independent processes regardless of PYTHONHASHSEED or native comparison - semantics (such as frozenset partial ordering). - """ - if isinstance(item, (int, float)) and not isinstance(item, bool): - encoded = json.dumps(item, default=_encode_custom_object, sort_keys=True) - return (0, item, encoded, type(item).__qualname__, repr(item)) - encoded = json.dumps(item, default=_encode_custom_object, sort_keys=True) - return (1, 0, encoded, type(item).__qualname__, repr(item)) - - def _encode_custom_object(o: Any) -> Any: """``default`` hook for :func:`json.dumps` that emits plain JSON. @@ -380,8 +369,24 @@ def _encode_custom_object(o: Any) -> Any: return to_json_hook(o) if isinstance(o, (set, frozenset, AbstractSet)): items: AbstractSet[Any] = cast("AbstractSet[Any]", o) + # Sets and frozensets are unordered collections. To ensure deterministic + # serialization across processes, worker replays, and PYTHONHASHSEED + # variations, elements are individually encoded to canonical JSON + # (with sort_keys=True), sorted lexically by their encoded JSON string, + # and emitted as normalized parsed values. + # + # Emitting the normalized parsed values rather than original objects + # guarantees identical order and eliminates leaks from dict insertion + # order variations, NaN/non-finite comparison anomalies, or partial + # ordering in frozensets. A consequence of sorting canonical JSON strings + # is lexical ordering for numeric sets (e.g., {2, 10} serializes as + # [10, 2]). try: - return sorted(items, key=_deterministic_set_sort_key) + encoded_items = sorted( + json.dumps(item, default=_encode_custom_object, sort_keys=True) + for item in items + ) + return [json.loads(encoded) for encoded in encoded_items] except Exception as e: raise TypeError( f"Cannot deterministically serialize set of type '{type(items).__name__}' " diff --git a/tests/durabletask/test_serialization.py b/tests/durabletask/test_serialization.py index 011abef7..5bd7851a 100644 --- a/tests/durabletask/test_serialization.py +++ b/tests/durabletask/test_serialization.py @@ -592,6 +592,10 @@ def test_to_json_serializes_set_and_frozenset(): encoded_frozenset = to_json(frozenset([2, 1])) assert json.loads(encoded_frozenset) == [1, 2] + # Numeric sets follow lexical JSON ordering (e.g. "10" < "2") + encoded_numeric_lexical = to_json({2, 10}) + assert json.loads(encoded_numeric_lexical) == [10, 2] + # Verify disjoint and nested frozenset in-process serialization encoded_disjoint = to_json({ frozenset({"alpha"}), @@ -607,13 +611,60 @@ def test_to_json_serializes_set_and_frozenset(): assert json.loads(encoded_nested) == [[["a", "b"], ["c"]], [["d"]]] +def test_set_serialization_non_finite_numbers(): + """Verify that sets with non-finite numbers (NaN, Infinity, -Infinity) + serialize deterministically with lexical JSON string sorting.""" + import math + + s = {float("nan"), float("inf"), float("-inf"), 2.0, 10.0} + encoded = to_json(s) + parsed = json.loads(encoded) + # Lexical order of canonical JSON strings: + # "-Infinity", "10.0", "2.0", "Infinity", "NaN" + assert parsed[0] == float("-inf") + assert parsed[1] == 10.0 + assert parsed[2] == 2.0 + assert parsed[3] == float("inf") + assert math.isnan(parsed[4]) + + +def test_set_serialization_equal_canonical_keys_differently_ordered_dicts(): + """Verify that custom objects with identical canonical JSON but differently ordered + dictionaries emit normalized values without order leaks.""" + class CustomDictObj: + def __init__(self, data: dict[str, Any], tag: str): + self.data = data + self.tag = tag + + def to_json(self) -> dict[str, Any]: + return self.data + + def __hash__(self) -> int: + return hash(self.tag) + + def __eq__(self, other: object) -> bool: + return isinstance(other, CustomDictObj) and self.tag == other.tag + + obj1 = CustomDictObj({"z": 100, "a": 1, "m": 50}, "obj1") + obj2 = CustomDictObj({"a": 1, "m": 50, "z": 100}, "obj2") + s = {obj1, obj2} + encoded = to_json(s) + parsed = json.loads(encoded) + # Both elements must be normalized to canonical sorted key order: {"a": 1, "m": 50, "z": 100} + assert len(parsed) == 2 + assert parsed[0] == {"a": 1, "m": 50, "z": 100} + assert parsed[1] == {"a": 1, "m": 50, "z": 100} + + def test_set_serialization_deterministic_across_hash_seeds(): - """Verify that sets of disjoint frozensets, nested frozensets, and unorderable custom objects + """Verify that sets of disjoint frozensets, nested frozensets, unorderable custom objects, + non-finite numbers, and equal canonical keys with differently ordered dictionaries are deterministically serialized across separate processes with differing PYTHONHASHSEED.""" repo_root = str(Path(__file__).resolve().parents[2]) code = """ import json from dataclasses import dataclass +from typing import Any from durabletask.serialization import JsonDataConverter conv = JsonDataConverter() @@ -644,7 +695,27 @@ class Entry: Entry("bravo", 2), }) -print(json.dumps({"a": case_a, "b": case_b, "c": case_c})) +# Case D: Numeric sets with lexical JSON ordering and non-finite floats +case_d = conv.serialize({float("nan"), float("inf"), float("-inf"), 2.0, 10.0}) + +# Case E: Equal canonical keys with differently ordered dictionaries +class DictItem: + def __init__(self, data: dict[str, Any], tag: str): + self.data = data + self.tag = tag + def to_json(self) -> dict[str, Any]: + return self.data + def __hash__(self) -> int: + return hash(self.tag) + def __eq__(self, other: object) -> bool: + return isinstance(other, DictItem) and self.tag == other.tag + +case_e = conv.serialize({ + DictItem({"z": 10, "a": 1}, "item1"), + DictItem({"a": 1, "z": 10}, "item2"), +}) + +print(json.dumps({"a": case_a, "b": case_b, "c": case_c, "d": case_d, "e": case_e})) """ seeds = ["0", "1", "2", "42", "999"] outputs: dict[str, dict[str, str]] = {} @@ -681,10 +752,14 @@ class Entry: '[{"id": "alpha", "val": 1}, {"id": "bravo", "val": 2}, ' '{"id": "charlie", "val": 3}, {"id": "delta", "val": 4}]' ) + expected_d = '[-Infinity, 10.0, 2.0, Infinity, NaN]' + expected_e = '[{"a": 1, "z": 10}, {"a": 1, "z": 10}]' assert base["a"] == expected_a assert base["b"] == expected_b assert base["c"] == expected_c + assert base["d"] == expected_d + assert base["e"] == expected_e def test_deserialize_coerces_set_when_opted_in():