Structure missing optional attributes without defaults as None

HISTORY.md

@@ -13,6 +13,8 @@ Our backwards-compatibility policy can be found [here](https://github.com/python
 
 ## NEXT
 
+- Attributes of optional types (unions including `None`) without defaults are now structured as `None` when their key is missing from the input, instead of raising an error.
+  ([#249](https://github.com/python-attrs/cattrs/issues/249))
 - Fix the msgspec converter passing typed deques directly to the encoder, which cannot serialize them. Deques are now unstructured into lists, including when nested in attrs classes or dataclasses.
   ([#818](https://github.com/python-attrs/cattrs/issues/818) [#819](https://github.com/python-attrs/cattrs/pull/819))
 

src/cattrs/_compat.py

@@ -108,6 +108,16 @@ def is_optional(typ: Any) -> bool:
     return is_union_type(typ) and NoneType in typ.__args__ and len(typ.__args__) == 2
 
 
+def includes_none_type(typ: Any) -> bool:
+    """Whether *typ* is a union type which includes `None`, possibly annotated.
+
+    Unlike `is_optional`, unions of any arity are accepted.
+    """
+    if is_annotated(typ):
+        return includes_none_type(get_args(typ)[0])
+    return is_union_type(typ) and NoneType in get_args(typ)
+
+
 def is_typeddict(cls: Any):
     """Thin wrapper around typing(_extensions).is_typeddict"""
     return _is_typeddict(getattr(cls, "__origin__", cls))

src/cattrs/converters.py

@@ -4,14 +4,14 @@
 from collections.abc import Callable, Iterable
 from collections.abc import Mapping as AbcMapping
 from collections.abc import MutableMapping as AbcMutableMapping
-from dataclasses import Field
+from dataclasses import MISSING, Field
 from enum import Enum
 from inspect import Signature
 from inspect import signature as inspect_signature
 from pathlib import Path
 from typing import Any, Optional, Tuple, TypeVar, overload
 
-from attrs import Attribute, resolve_types
+from attrs import NOTHING, Attribute, resolve_types
 from attrs import has as attrs_has
 from typing_extensions import Self, TypeForm
 
@@ -39,6 +39,7 @@
     get_origin,
     has,
     has_with_generic,
+    includes_none_type,
     is_annotated,
     is_bare,
     is_counter,
@@ -185,6 +186,13 @@ def _is_extended_factory(factory: Callable) -> bool:
     )
 
 
+def _attribute_has_default(a: Attribute | Field) -> bool:
+    """Whether an attrs attribute or a dataclass field has a default."""
+    if isinstance(a, Field):
+        return a.default is not MISSING or a.default_factory is not MISSING
+    return a.default is not NOTHING
+
+
 class BaseConverter:
     """Converts between structured and unstructured data."""
 
@@ -824,6 +832,17 @@ def structure_attrs_fromdict(self, obj: Mapping[str, Any], cl: type[T]) -> T:
             try:
                 val = obj[a.name]
             except KeyError:
+                # Fields of optional types without defaults treat a
+                # missing key as an explicit None, instead of being
+                # skipped (and failing when instantiating).
+                if (
+                    getattr(a, "init", True)
+                    and not _attribute_has_default(a)
+                    and includes_none_type(a.type)
+                ):
+                    conv_obj[getattr(a, "alias", a.name)] = self._structure_attribute(
+                        a, None
+                    )
                 continue
 
             # try .alias and .name because this code also supports dataclasses!

src/cattrs/gen/__init__.py

@@ -13,6 +13,7 @@
     adapted_fields,
     get_args,
     get_origin,
+    includes_none_type,
     is_annotated,
     is_bare,
     is_bare_final,
@@ -375,6 +376,10 @@ def make_dict_structure_fn_from_attrs(
     .. versionchanged:: 26.1.0
         `typing.Annotated[T, override()]` is now recognized and can be used to customize
         unstructuring.
+    .. versionchanged:: 26.3.0
+        Attributes of optional types (unions including `None`) without defaults
+        are structured as `None` when their key is missing from the input,
+        instead of raising an error.
     """
 
     cl_name = cl.__name__
@@ -473,6 +478,13 @@ def make_dict_structure_fn_from_attrs(
             allowed_fields.add(kn)
             i = "  "
 
+            # Optional fields without defaults treat a missing key as an
+            # explicit None, instead of raising a KeyError.
+            if a.default is NOTHING and includes_none_type(t):
+                access = f"o.get({kn!r})"
+            else:
+                access = f"o[{kn!r}]"
+
             if not a.init:
                 if a.default is not NOTHING:
                     pi_lines.append(f"{i}if {kn!r} in o:")
@@ -485,16 +497,16 @@ def make_dict_structure_fn_from_attrs(
                     if handler == converter._structure_call:
                         internal_arg_parts[struct_handler_name] = t
                         pi_lines.append(
-                            f"{i}instance.{an} = {struct_handler_name}(o[{kn!r}])"
+                            f"{i}instance.{an} = {struct_handler_name}({access})"
                         )
                     else:
                         tn = f"__c_type_{an}"
                         internal_arg_parts[tn] = t
                         pi_lines.append(
-                            f"{i}instance.{an} = {struct_handler_name}(o[{kn!r}], {tn})"
+                            f"{i}instance.{an} = {struct_handler_name}({access}, {tn})"
                         )
                 else:
-                    pi_lines.append(f"{i}instance.{an} = o[{kn!r}]")
+                    pi_lines.append(f"{i}instance.{an} = {access}")
                 i = i[:-2]
                 pi_lines.append(f"{i}except Exception as e:")
                 i = f"{i}  "
@@ -515,14 +527,14 @@ def make_dict_structure_fn_from_attrs(
                     if handler == converter._structure_call:
                         internal_arg_parts[struct_handler_name] = t
                         lines.append(
-                            f"{i}res['{ian}'] = {struct_handler_name}(o[{kn!r}])"
+                            f"{i}res['{ian}'] = {struct_handler_name}({access})"
                         )
                     else:
                         lines.append(
-                            f"{i}res['{ian}'] = {struct_handler_name}(o[{kn!r}], {type_name})"
+                            f"{i}res['{ian}'] = {struct_handler_name}({access}, {type_name})"
                         )
                 else:
-                    lines.append(f"{i}res['{ian}'] = o[{kn!r}]")
+                    lines.append(f"{i}res['{ian}'] = {access}")
                 i = i[:-2]
                 lines.append(f"{i}except Exception as e:")
                 i = f"{i}  "
@@ -610,21 +622,26 @@ def make_dict_structure_fn_from_attrs(
                 kn = override.rename
             allowed_fields.add(kn)
 
+            # Optional fields treat a missing key as an explicit None,
+            # instead of raising a KeyError. (Only fields without
+            # defaults reach this loop.)
+            access = f"o.get({kn!r})" if includes_none_type(t) else f"o[{kn!r}]"
+
             if not a.init:
                 if handler is not None:
                     struct_handler_name = f"__c_structure_{an}"
                     internal_arg_parts[struct_handler_name] = handler
                     if handler == converter._structure_call:
                         internal_arg_parts[struct_handler_name] = t
-                        pi_line = f"  instance.{an} = {struct_handler_name}(o[{kn!r}])"
+                        pi_line = f"  instance.{an} = {struct_handler_name}({access})"
                     else:
                         tn = f"__c_type_{an}"
                         internal_arg_parts[tn] = t
                         pi_line = (
-                            f"  instance.{an} = {struct_handler_name}(o[{kn!r}], {tn})"
+                            f"  instance.{an} = {struct_handler_name}({access}, {tn})"
                         )
                 else:
-                    pi_line = f"  instance.{an} = o[{kn!r}]"
+                    pi_line = f"  instance.{an} = {access}"
 
                 pi_lines.append(pi_line)
             else:
@@ -633,13 +650,13 @@ def make_dict_structure_fn_from_attrs(
                     internal_arg_parts[struct_handler_name] = handler
                     if handler == converter._structure_call:
                         internal_arg_parts[struct_handler_name] = t
-                        invocation_line = f"{struct_handler_name}(o[{kn!r}]),"
+                        invocation_line = f"{struct_handler_name}({access}),"
                     else:
                         tn = f"__c_type_{an}"
                         internal_arg_parts[tn] = t
-                        invocation_line = f"{struct_handler_name}(o[{kn!r}], {tn}),"
+                        invocation_line = f"{struct_handler_name}({access}, {tn}),"
                 else:
-                    invocation_line = f"o[{kn!r}],"
+                    invocation_line = f"{access},"
 
                 if a.kw_only:
                     invocation_line = f"{a.alias}={invocation_line}"
@@ -805,6 +822,10 @@ def make_dict_structure_fn(
     .. versionchanged:: 26.1.0
         `typing.Annotated[T, override()]` is now recognized and can be used to customize
         unstructuring.
+    .. versionchanged:: 26.3.0
+        Attributes of optional types (unions including `None`) without defaults
+        are structured as `None` when their key is missing from the input,
+        instead of raising an error.
     """
 
     mapping = {}

tests/test_optionals.py

@@ -1,9 +1,12 @@
-from typing import Any, NewType, Optional
+from dataclasses import dataclass
+from typing import Annotated, Any, NewType, Optional, Union
 
 import pytest
-from attrs import define
+from attrs import define, field, frozen
 
 from cattrs import BaseConverter, Converter
+from cattrs.errors import ClassValidationError
+from cattrs.gen import make_dict_structure_fn, override
 
 from ._compat import is_py310_plus
 
@@ -73,3 +76,167 @@ def _(val: Optional[int]) -> Any:
 
     assert converter.unstructure(0, Optional[int]) is None
     assert converter.unstructure(5, Optional[int]) == 5
+
+
+def test_missing_optional_without_default(converter):
+    """Optional attributes without defaults are structured as None when missing."""
+
+    @define
+    class Dog:
+        name: str
+        nickname: Optional[str]
+
+    assert converter.structure({"name": "Rex"}, Dog) == Dog("Rex", None)
+    assert converter.structure({"name": "Spike", "nickname": "Carl"}, Dog) == Dog(
+        "Spike", "Carl"
+    )
+    # An explicit None behaves the same as a missing key.
+    assert converter.structure({"name": "Rex", "nickname": None}, Dog) == Dog(
+        "Rex", None
+    )
+
+
+def test_missing_optional_between_required(converter):
+    """Optional attributes without defaults may sit between required ones."""
+
+    @frozen
+    class C:
+        a_float: float
+        maybe_int: Optional[int]
+        a_string: str
+
+    assert converter.structure({"a_float": 0.1, "a_string": "hello"}, C) == C(
+        0.1, None, "hello"
+    )
+
+
+def test_missing_optional_defaults_and_required(converter):
+    """Defaults still win for missing keys, and required attributes still fail."""
+
+    @define
+    class C:
+        req: str
+        opt_default: Optional[int] = None
+        opt_custom: Optional[str] = "custom default"
+        plain_default: int = 42
+
+    assert converter.structure({"req": "x"}, C) == C("x", None, "custom default", 42)
+    # The error type depends on the structuring path, as before.
+    with pytest.raises((ClassValidationError, KeyError, TypeError)):
+        converter.structure({}, C)
+
+
+def test_missing_union_with_none(converter):
+    """Unions including None (of any arity) are structured as None when missing."""
+
+    @define
+    class C:
+        multi: Union[bool, str, None]
+        opt: Optional[int]
+
+    converter.register_structure_hook(
+        Union[bool, str, None],
+        lambda v, _: v if v is None or isinstance(v, bool) else str(v),
+    )
+
+    assert converter.structure({}, C) == C(None, None)
+    assert converter.structure({"multi": 5, "opt": 3}, C) == C("5", 3)
+
+
+@pytest.mark.skipif(not is_py310_plus, reason="3.10+ union syntax")
+def test_missing_optional_pep604(genconverter):
+    """PEP 604 unions including None are structured as None when missing."""
+
+    @define
+    class C:
+        a: int | None
+        b: str
+
+    assert genconverter.structure({"b": "x"}, C) == C(None, "x")
+
+
+def test_missing_optional_kw_only_and_alias(genconverter):
+    """Missing optional kw-only and underscore attributes are structured as None."""
+
+    @define
+    class C:
+        _private_opt: Optional[int]
+        renamed: Optional[str] = field(kw_only=True)
+
+    assert genconverter.structure({}, C) == C(None, renamed=None)
+    assert genconverter.structure({"_private_opt": 3, "renamed": "a"}, C) == C(
+        3, renamed="a"
+    )
+
+    aliased = Converter(use_alias=True)
+    assert aliased.structure({}, C) == C(None, renamed=None)
+    assert aliased.structure({"private_opt": 3, "renamed": "a"}, C) == C(3, renamed="a")
+
+
+def test_missing_optional_rename():
+    """Missing renamed optional attributes are structured as None."""
+
+    @define
+    class C:
+        a: Optional[int]
+        b: str
+
+    converter = Converter()
+    fn = make_dict_structure_fn(C, converter, a=override(rename="a_key"))
+
+    assert fn({"b": "x"}, C) == C(None, "x")
+    assert fn({"a_key": 1, "b": "x"}, C) == C(1, "x")
+
+
+def test_missing_optional_init_false():
+    """Missing optional init=False attributes are set to None if included."""
+
+    @define
+    class C:
+        a: int
+        opt: Optional[str] = field(init=False)
+
+    converter = Converter()
+    fn = make_dict_structure_fn(C, converter, _cattrs_include_init_false=True)
+
+    assert fn({"a": 1}, C).opt is None
+    assert fn({"a": 1, "opt": "v"}, C).opt == "v"
+    # Without the flag, the attribute is not set at all, as before.
+    assert not hasattr(converter.structure({"a": 1}, C), "opt")
+
+
+def test_missing_optional_annotated(genconverter):
+    """Missing annotated optional attributes are structured as None."""
+
+    @define
+    class C:
+        a: Annotated[Optional[int], "note"]
+        b: str
+
+    assert genconverter.structure({"b": "x"}, C) == C(None, "x")
+
+
+def test_missing_optional_dataclass(converter):
+    """Missing optional dataclass fields without defaults are structured as None."""
+
+    @dataclass
+    class C:
+        name: str
+        nickname: Optional[str]
+
+    assert converter.structure({"name": "Rex"}, C) == C("Rex", None)
+
+
+def test_missing_optional_not_an_error():
+    """A missing optional attribute does not contribute to the collected errors."""
+
+    @define
+    class C:
+        opt: Optional[int]
+        num: int
+
+    converter = Converter(detailed_validation=True)
+    with pytest.raises(ClassValidationError) as exc_info:
+        converter.structure({"num": "not an int"}, C)
+
+    assert len(exc_info.value.exceptions) == 1