diff --git a/HISTORY.md b/HISTORY.md index 942616e3..84088dc4 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -13,6 +13,8 @@ Our backwards-compatibility policy can be found [here](https://github.com/python ## NEXT +- Attributes of optional types (unions including `None`) without defaults are now structured as `None` when their key is missing from the input, instead of raising an error. + ([#249](https://github.com/python-attrs/cattrs/issues/249)) - Fix the msgspec converter passing typed deques directly to the encoder, which cannot serialize them. Deques are now unstructured into lists, including when nested in attrs classes or dataclasses. ([#818](https://github.com/python-attrs/cattrs/issues/818) [#819](https://github.com/python-attrs/cattrs/pull/819)) diff --git a/src/cattrs/_compat.py b/src/cattrs/_compat.py index 45ed2e52..fda9cefb 100644 --- a/src/cattrs/_compat.py +++ b/src/cattrs/_compat.py @@ -108,6 +108,16 @@ def is_optional(typ: Any) -> bool: return is_union_type(typ) and NoneType in typ.__args__ and len(typ.__args__) == 2 +def includes_none_type(typ: Any) -> bool: + """Whether *typ* is a union type which includes `None`, possibly annotated. + + Unlike `is_optional`, unions of any arity are accepted. + """ + if is_annotated(typ): + return includes_none_type(get_args(typ)[0]) + return is_union_type(typ) and NoneType in get_args(typ) + + def is_typeddict(cls: Any): """Thin wrapper around typing(_extensions).is_typeddict""" return _is_typeddict(getattr(cls, "__origin__", cls)) diff --git a/src/cattrs/converters.py b/src/cattrs/converters.py index 70c19b44..24b99364 100644 --- a/src/cattrs/converters.py +++ b/src/cattrs/converters.py @@ -4,14 +4,14 @@ from collections.abc import Callable, Iterable from collections.abc import Mapping as AbcMapping from collections.abc import MutableMapping as AbcMutableMapping -from dataclasses import Field +from dataclasses import MISSING, Field from enum import Enum from inspect import Signature from inspect import signature as inspect_signature from pathlib import Path from typing import Any, Optional, Tuple, TypeVar, overload -from attrs import Attribute, resolve_types +from attrs import NOTHING, Attribute, resolve_types from attrs import has as attrs_has from typing_extensions import Self, TypeForm @@ -39,6 +39,7 @@ get_origin, has, has_with_generic, + includes_none_type, is_annotated, is_bare, is_counter, @@ -185,6 +186,13 @@ def _is_extended_factory(factory: Callable) -> bool: ) +def _attribute_has_default(a: Attribute | Field) -> bool: + """Whether an attrs attribute or a dataclass field has a default.""" + if isinstance(a, Field): + return a.default is not MISSING or a.default_factory is not MISSING + return a.default is not NOTHING + + class BaseConverter: """Converts between structured and unstructured data.""" @@ -824,6 +832,17 @@ def structure_attrs_fromdict(self, obj: Mapping[str, Any], cl: type[T]) -> T: try: val = obj[a.name] except KeyError: + # Fields of optional types without defaults treat a + # missing key as an explicit None, instead of being + # skipped (and failing when instantiating). + if ( + getattr(a, "init", True) + and not _attribute_has_default(a) + and includes_none_type(a.type) + ): + conv_obj[getattr(a, "alias", a.name)] = self._structure_attribute( + a, None + ) continue # try .alias and .name because this code also supports dataclasses! diff --git a/src/cattrs/gen/__init__.py b/src/cattrs/gen/__init__.py index 79d318e1..d61af2b1 100644 --- a/src/cattrs/gen/__init__.py +++ b/src/cattrs/gen/__init__.py @@ -13,6 +13,7 @@ adapted_fields, get_args, get_origin, + includes_none_type, is_annotated, is_bare, is_bare_final, @@ -375,6 +376,10 @@ def make_dict_structure_fn_from_attrs( .. versionchanged:: 26.1.0 `typing.Annotated[T, override()]` is now recognized and can be used to customize unstructuring. + .. versionchanged:: 26.3.0 + Attributes of optional types (unions including `None`) without defaults + are structured as `None` when their key is missing from the input, + instead of raising an error. """ cl_name = cl.__name__ @@ -473,6 +478,13 @@ def make_dict_structure_fn_from_attrs( allowed_fields.add(kn) i = " " + # Optional fields without defaults treat a missing key as an + # explicit None, instead of raising a KeyError. + if a.default is NOTHING and includes_none_type(t): + access = f"o.get({kn!r})" + else: + access = f"o[{kn!r}]" + if not a.init: if a.default is not NOTHING: pi_lines.append(f"{i}if {kn!r} in o:") @@ -485,16 +497,16 @@ def make_dict_structure_fn_from_attrs( if handler == converter._structure_call: internal_arg_parts[struct_handler_name] = t pi_lines.append( - f"{i}instance.{an} = {struct_handler_name}(o[{kn!r}])" + f"{i}instance.{an} = {struct_handler_name}({access})" ) else: tn = f"__c_type_{an}" internal_arg_parts[tn] = t pi_lines.append( - f"{i}instance.{an} = {struct_handler_name}(o[{kn!r}], {tn})" + f"{i}instance.{an} = {struct_handler_name}({access}, {tn})" ) else: - pi_lines.append(f"{i}instance.{an} = o[{kn!r}]") + pi_lines.append(f"{i}instance.{an} = {access}") i = i[:-2] pi_lines.append(f"{i}except Exception as e:") i = f"{i} " @@ -515,14 +527,14 @@ def make_dict_structure_fn_from_attrs( if handler == converter._structure_call: internal_arg_parts[struct_handler_name] = t lines.append( - f"{i}res['{ian}'] = {struct_handler_name}(o[{kn!r}])" + f"{i}res['{ian}'] = {struct_handler_name}({access})" ) else: lines.append( - f"{i}res['{ian}'] = {struct_handler_name}(o[{kn!r}], {type_name})" + f"{i}res['{ian}'] = {struct_handler_name}({access}, {type_name})" ) else: - lines.append(f"{i}res['{ian}'] = o[{kn!r}]") + lines.append(f"{i}res['{ian}'] = {access}") i = i[:-2] lines.append(f"{i}except Exception as e:") i = f"{i} " @@ -610,21 +622,26 @@ def make_dict_structure_fn_from_attrs( kn = override.rename allowed_fields.add(kn) + # Optional fields treat a missing key as an explicit None, + # instead of raising a KeyError. (Only fields without + # defaults reach this loop.) + access = f"o.get({kn!r})" if includes_none_type(t) else f"o[{kn!r}]" + if not a.init: if handler is not None: struct_handler_name = f"__c_structure_{an}" internal_arg_parts[struct_handler_name] = handler if handler == converter._structure_call: internal_arg_parts[struct_handler_name] = t - pi_line = f" instance.{an} = {struct_handler_name}(o[{kn!r}])" + pi_line = f" instance.{an} = {struct_handler_name}({access})" else: tn = f"__c_type_{an}" internal_arg_parts[tn] = t pi_line = ( - f" instance.{an} = {struct_handler_name}(o[{kn!r}], {tn})" + f" instance.{an} = {struct_handler_name}({access}, {tn})" ) else: - pi_line = f" instance.{an} = o[{kn!r}]" + pi_line = f" instance.{an} = {access}" pi_lines.append(pi_line) else: @@ -633,13 +650,13 @@ def make_dict_structure_fn_from_attrs( internal_arg_parts[struct_handler_name] = handler if handler == converter._structure_call: internal_arg_parts[struct_handler_name] = t - invocation_line = f"{struct_handler_name}(o[{kn!r}])," + invocation_line = f"{struct_handler_name}({access})," else: tn = f"__c_type_{an}" internal_arg_parts[tn] = t - invocation_line = f"{struct_handler_name}(o[{kn!r}], {tn})," + invocation_line = f"{struct_handler_name}({access}, {tn})," else: - invocation_line = f"o[{kn!r}]," + invocation_line = f"{access}," if a.kw_only: invocation_line = f"{a.alias}={invocation_line}" @@ -805,6 +822,10 @@ def make_dict_structure_fn( .. versionchanged:: 26.1.0 `typing.Annotated[T, override()]` is now recognized and can be used to customize unstructuring. + .. versionchanged:: 26.3.0 + Attributes of optional types (unions including `None`) without defaults + are structured as `None` when their key is missing from the input, + instead of raising an error. """ mapping = {} diff --git a/tests/test_optionals.py b/tests/test_optionals.py index 724bd15a..42db1437 100644 --- a/tests/test_optionals.py +++ b/tests/test_optionals.py @@ -1,9 +1,12 @@ -from typing import Any, NewType, Optional +from dataclasses import dataclass +from typing import Annotated, Any, NewType, Optional, Union import pytest -from attrs import define +from attrs import define, field, frozen from cattrs import BaseConverter, Converter +from cattrs.errors import ClassValidationError +from cattrs.gen import make_dict_structure_fn, override from ._compat import is_py310_plus @@ -73,3 +76,167 @@ def _(val: Optional[int]) -> Any: assert converter.unstructure(0, Optional[int]) is None assert converter.unstructure(5, Optional[int]) == 5 + + +def test_missing_optional_without_default(converter): + """Optional attributes without defaults are structured as None when missing.""" + + @define + class Dog: + name: str + nickname: Optional[str] + + assert converter.structure({"name": "Rex"}, Dog) == Dog("Rex", None) + assert converter.structure({"name": "Spike", "nickname": "Carl"}, Dog) == Dog( + "Spike", "Carl" + ) + # An explicit None behaves the same as a missing key. + assert converter.structure({"name": "Rex", "nickname": None}, Dog) == Dog( + "Rex", None + ) + + +def test_missing_optional_between_required(converter): + """Optional attributes without defaults may sit between required ones.""" + + @frozen + class C: + a_float: float + maybe_int: Optional[int] + a_string: str + + assert converter.structure({"a_float": 0.1, "a_string": "hello"}, C) == C( + 0.1, None, "hello" + ) + + +def test_missing_optional_defaults_and_required(converter): + """Defaults still win for missing keys, and required attributes still fail.""" + + @define + class C: + req: str + opt_default: Optional[int] = None + opt_custom: Optional[str] = "custom default" + plain_default: int = 42 + + assert converter.structure({"req": "x"}, C) == C("x", None, "custom default", 42) + # The error type depends on the structuring path, as before. + with pytest.raises((ClassValidationError, KeyError, TypeError)): + converter.structure({}, C) + + +def test_missing_union_with_none(converter): + """Unions including None (of any arity) are structured as None when missing.""" + + @define + class C: + multi: Union[bool, str, None] + opt: Optional[int] + + converter.register_structure_hook( + Union[bool, str, None], + lambda v, _: v if v is None or isinstance(v, bool) else str(v), + ) + + assert converter.structure({}, C) == C(None, None) + assert converter.structure({"multi": 5, "opt": 3}, C) == C("5", 3) + + +@pytest.mark.skipif(not is_py310_plus, reason="3.10+ union syntax") +def test_missing_optional_pep604(genconverter): + """PEP 604 unions including None are structured as None when missing.""" + + @define + class C: + a: int | None + b: str + + assert genconverter.structure({"b": "x"}, C) == C(None, "x") + + +def test_missing_optional_kw_only_and_alias(genconverter): + """Missing optional kw-only and underscore attributes are structured as None.""" + + @define + class C: + _private_opt: Optional[int] + renamed: Optional[str] = field(kw_only=True) + + assert genconverter.structure({}, C) == C(None, renamed=None) + assert genconverter.structure({"_private_opt": 3, "renamed": "a"}, C) == C( + 3, renamed="a" + ) + + aliased = Converter(use_alias=True) + assert aliased.structure({}, C) == C(None, renamed=None) + assert aliased.structure({"private_opt": 3, "renamed": "a"}, C) == C(3, renamed="a") + + +def test_missing_optional_rename(): + """Missing renamed optional attributes are structured as None.""" + + @define + class C: + a: Optional[int] + b: str + + converter = Converter() + fn = make_dict_structure_fn(C, converter, a=override(rename="a_key")) + + assert fn({"b": "x"}, C) == C(None, "x") + assert fn({"a_key": 1, "b": "x"}, C) == C(1, "x") + + +def test_missing_optional_init_false(): + """Missing optional init=False attributes are set to None if included.""" + + @define + class C: + a: int + opt: Optional[str] = field(init=False) + + converter = Converter() + fn = make_dict_structure_fn(C, converter, _cattrs_include_init_false=True) + + assert fn({"a": 1}, C).opt is None + assert fn({"a": 1, "opt": "v"}, C).opt == "v" + # Without the flag, the attribute is not set at all, as before. + assert not hasattr(converter.structure({"a": 1}, C), "opt") + + +def test_missing_optional_annotated(genconverter): + """Missing annotated optional attributes are structured as None.""" + + @define + class C: + a: Annotated[Optional[int], "note"] + b: str + + assert genconverter.structure({"b": "x"}, C) == C(None, "x") + + +def test_missing_optional_dataclass(converter): + """Missing optional dataclass fields without defaults are structured as None.""" + + @dataclass + class C: + name: str + nickname: Optional[str] + + assert converter.structure({"name": "Rex"}, C) == C("Rex", None) + + +def test_missing_optional_not_an_error(): + """A missing optional attribute does not contribute to the collected errors.""" + + @define + class C: + opt: Optional[int] + num: int + + converter = Converter(detailed_validation=True) + with pytest.raises(ClassValidationError) as exc_info: + converter.structure({"num": "not an int"}, C) + + assert len(exc_info.value.exceptions) == 1