diff --git a/HISTORY.md b/HISTORY.md index 942616e3..88c62bbc 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -13,6 +13,8 @@ Our backwards-compatibility policy can be found [here](https://github.com/python ## NEXT +- The standard library JSON and ujson converters now support `bytes_encoding="base64"` in `make_converter` and `configure_converter`, while retaining Base85 as the default. + ([#552](https://github.com/python-attrs/cattrs/issues/552)) - Fix the msgspec converter passing typed deques directly to the encoder, which cannot serialize them. Deques are now unstructured into lists, including when nested in attrs classes or dataclasses. ([#818](https://github.com/python-attrs/cattrs/issues/818) [#819](https://github.com/python-attrs/cattrs/pull/819)) diff --git a/README.md b/README.md index 3942b414..fc8df3a6 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,8 @@ _cattrs_ comes with pre-configured converters for a number of serialization libr For details, see the [cattrs.preconf package](https://catt.rs/en/stable/preconf.html). +The standard library JSON and UltraJSON converters support optional Base64 encoding for bytes with `make_converter(bytes_encoding="base64")`. + ## Design Decisions diff --git a/docs/preconf.md b/docs/preconf.md index 022426a3..2a34a84a 100644 --- a/docs/preconf.md +++ b/docs/preconf.md @@ -72,6 +72,23 @@ Found at {mod}`cattrs.preconf.json`. Bytes are serialized as base 85 strings. Counters are serialized as dictionaries. Sets are serialized as lists, and deserialized back into sets. `datetime` s and `date` s are serialized as ISO 8601 strings. +Pass `bytes_encoding="base64"` to {func}`cattrs.preconf.json.make_converter` or +{func}`cattrs.preconf.json.configure_converter` to use standard Base64 instead. +Use the same encoding when serializing and deserializing. + +```{doctest} +>>> from cattrs.preconf.json import make_converter +>>> converter = make_converter(bytes_encoding="base64") +>>> converter.dumps(b"hello") +'"aGVsbG8="' +>>> converter.loads('"aGVsbG8="', bytes) +b'hello' +``` + +```{versionadded} NEXT +The `bytes_encoding` option accepts `"base85"` (the default) and `"base64"`. +``` + ## _orjson_ @@ -156,6 +173,14 @@ Found at {mod}`cattrs.preconf.ujson`. Bytes are serialized as base 85 strings. Sets are serialized as lists, and deserialized back into sets. `datetime` s and `date` s are serialized as ISO 8601 strings. +Pass `bytes_encoding="base64"` to {func}`cattrs.preconf.ujson.make_converter` or +{func}`cattrs.preconf.ujson.configure_converter` to use standard Base64 instead, +as with the standard library JSON converter. Use the same encoding when serializing and deserializing. + +```{versionadded} NEXT +The `bytes_encoding` option accepts `"base85"` (the default) and `"base64"`. +``` + _ujson_ doesn't support integers less than -9223372036854775808, and greater than 9223372036854775807, nor does it support `float('inf')`. diff --git a/src/cattrs/preconf/json.py b/src/cattrs/preconf/json.py index 4a556597..6088943f 100644 --- a/src/cattrs/preconf/json.py +++ b/src/cattrs/preconf/json.py @@ -1,19 +1,21 @@ """Preconfigured converters for the stdlib json.""" -from base64 import b85decode, b85encode -from collections.abc import Set +from base64 import b64decode, b64encode, b85decode, b85encode +from collections.abc import Callable, Mapping, Set from datetime import date, datetime from json import dumps, loads -from typing import Any, TypeVar, Union +from typing import Any, Literal, TypeVar, Union from typing_extensions import TypeForm from .._compat import Counter -from ..converters import BaseConverter, Converter -from ..fns import identity +from ..converters import BaseConverter, Converter, UnstructureStrategy +from ..dispatch import HookFactory, StructureHook, UnstructureHook +from ..fns import identity, raise_error +from ..gen._consts import AttributeOverride from ..literals import is_literal_containing_enums from ..strategies import configure_union_passthrough -from . import is_primitive_enum, literals_with_enums_unstructure_factory, wrap +from . import is_primitive_enum, literals_with_enums_unstructure_factory __all__ = ["JsonConverter", "configure_converter", "make_converter"] @@ -34,11 +36,13 @@ def loads(self, data: Union[bytes, str], cl: TypeForm[T], **kwargs: Any) -> T: return self.structure(loads(data, **kwargs), cl) -def configure_converter(converter: BaseConverter) -> None: +def configure_converter( + converter: BaseConverter, *, bytes_encoding: Literal["base85", "base64"] = "base85" +) -> None: """ Configure the converter for use with the stdlib json module. - * bytes are serialized as base85 strings + * bytes are serialized as base85 strings, or base64 if requested * datetimes are serialized as ISO 8601 * counters are serialized as dicts * sets are serialized as lists @@ -48,11 +52,20 @@ def configure_converter(converter: BaseConverter) -> None: .. versionchanged:: 24.2.0 Enums are left to the library to unstructure, speeding them up. + + .. versionchanged:: NEXT + *bytes_encoding* selects base85 (the default) or base64 for bytes. """ - converter.register_unstructure_hook( - bytes, lambda v: (b85encode(v) if v else b"").decode("utf8") - ) - converter.register_structure_hook(bytes, lambda v, _: b85decode(v)) + encode: Callable[[bytes], bytes] + decode: Callable[[Union[str, bytes]], bytes] + if bytes_encoding == "base85": + encode, decode = b85encode, b85decode + elif bytes_encoding == "base64": + encode, decode = b64encode, b64decode + else: + raise ValueError("bytes_encoding must be 'base85' or 'base64'") + converter.register_unstructure_hook(bytes, lambda v: encode(v).decode("ascii")) + converter.register_structure_hook(bytes, lambda v, _: decode(v)) converter.register_unstructure_hook(datetime, lambda v: v.isoformat()) converter.register_structure_hook(datetime, lambda v, _: datetime.fromisoformat(v)) converter.register_unstructure_hook(date, lambda v: v.isoformat()) @@ -64,14 +77,47 @@ def configure_converter(converter: BaseConverter) -> None: configure_union_passthrough(Union[str, bool, int, float, None], converter) -@wrap(JsonConverter) -def make_converter(*args: Any, **kwargs: Any) -> JsonConverter: - kwargs["unstruct_collection_overrides"] = { - Set: list, - Counter: dict, - **kwargs.get("unstruct_collection_overrides", {}), - } - res = JsonConverter(*args, **kwargs) - configure_converter(res) +def make_converter( + dict_factory: Callable[[], Any] = dict, + unstruct_strat: UnstructureStrategy = UnstructureStrategy.AS_DICT, + omit_if_default: bool = False, + forbid_extra_keys: bool = False, + type_overrides: Mapping[type, AttributeOverride] = {}, + unstruct_collection_overrides: Mapping[type, UnstructureHook] = {}, + prefer_attrib_converters: bool = False, + detailed_validation: bool = True, + unstructure_fallback_factory: HookFactory[UnstructureHook] = lambda _: identity, + structure_fallback_factory: HookFactory[StructureHook] = lambda t: raise_error( + None, t + ), + use_alias: bool = False, + *, + bytes_encoding: Literal["base85", "base64"] = "base85", +) -> JsonConverter: + """Create a JSON converter, optionally using base64 for bytes. + + All other parameters are passed to :class:`cattrs.Converter`. + + .. versionchanged:: NEXT + *bytes_encoding* selects base85 (the default) or base64 for bytes. + """ + res = JsonConverter( + dict_factory=dict_factory, + unstruct_strat=unstruct_strat, + omit_if_default=omit_if_default, + forbid_extra_keys=forbid_extra_keys, + type_overrides=type_overrides, + unstruct_collection_overrides={ + Set: list, + Counter: dict, + **unstruct_collection_overrides, + }, + prefer_attrib_converters=prefer_attrib_converters, + detailed_validation=detailed_validation, + unstructure_fallback_factory=unstructure_fallback_factory, + structure_fallback_factory=structure_fallback_factory, + use_alias=use_alias, + ) + configure_converter(res, bytes_encoding=bytes_encoding) return res diff --git a/src/cattrs/preconf/ujson.py b/src/cattrs/preconf/ujson.py index 2ba5682f..42664867 100644 --- a/src/cattrs/preconf/ujson.py +++ b/src/cattrs/preconf/ujson.py @@ -1,18 +1,20 @@ """Preconfigured converters for ujson.""" -from base64 import b85decode, b85encode -from collections.abc import Set +from base64 import b64decode, b64encode, b85decode, b85encode +from collections.abc import Callable, Mapping, Set from datetime import date, datetime -from typing import Any, AnyStr, TypeVar, Union +from typing import Any, AnyStr, Literal, TypeVar, Union from typing_extensions import TypeForm from ujson import dumps, loads -from ..converters import BaseConverter, Converter -from ..fns import identity +from ..converters import BaseConverter, Converter, UnstructureStrategy +from ..dispatch import HookFactory, StructureHook, UnstructureHook +from ..fns import identity, raise_error +from ..gen._consts import AttributeOverride from ..literals import is_literal_containing_enums from ..strategies import configure_union_passthrough -from . import is_primitive_enum, literals_with_enums_unstructure_factory, wrap +from . import is_primitive_enum, literals_with_enums_unstructure_factory __all__ = ["UjsonConverter", "configure_converter", "make_converter"] @@ -33,22 +35,33 @@ def loads(self, data: AnyStr, cl: TypeForm[T], **kwargs: Any) -> T: return self.structure(loads(data, **kwargs), cl) -def configure_converter(converter: BaseConverter): +def configure_converter( + converter: BaseConverter, *, bytes_encoding: Literal["base85", "base64"] = "base85" +) -> None: """ Configure the converter for use with the ujson library. - * bytes are serialized as base64 strings + * bytes are serialized as base85 strings, or base64 if requested * datetimes are serialized as ISO 8601 * sets are serialized as lists * string and int enums are passed through when unstructuring .. versionchanged:: 24.2.0 Enums are left to the library to unstructure, speeding them up. + + .. versionchanged:: NEXT + *bytes_encoding* selects base85 (the default) or base64 for bytes. """ - converter.register_unstructure_hook( - bytes, lambda v: (b85encode(v) if v else b"").decode("utf8") - ) - converter.register_structure_hook(bytes, lambda v, _: b85decode(v)) + encode: Callable[[bytes], bytes] + decode: Callable[[Union[str, bytes]], bytes] + if bytes_encoding == "base85": + encode, decode = b85encode, b85decode + elif bytes_encoding == "base64": + encode, decode = b64encode, b64decode + else: + raise ValueError("bytes_encoding must be 'base85' or 'base64'") + converter.register_unstructure_hook(bytes, lambda v: encode(v).decode("ascii")) + converter.register_structure_hook(bytes, lambda v, _: decode(v)) converter.register_unstructure_hook(datetime, lambda v: v.isoformat()) converter.register_structure_hook(datetime, lambda v, _: datetime.fromisoformat(v)) @@ -61,13 +74,43 @@ def configure_converter(converter: BaseConverter): configure_union_passthrough(Union[str, bool, int, float, None], converter) -@wrap(UjsonConverter) -def make_converter(*args: Any, **kwargs: Any) -> UjsonConverter: - kwargs["unstruct_collection_overrides"] = { - Set: list, - **kwargs.get("unstruct_collection_overrides", {}), - } - res = UjsonConverter(*args, **kwargs) - configure_converter(res) +def make_converter( + dict_factory: Callable[[], Any] = dict, + unstruct_strat: UnstructureStrategy = UnstructureStrategy.AS_DICT, + omit_if_default: bool = False, + forbid_extra_keys: bool = False, + type_overrides: Mapping[type, AttributeOverride] = {}, + unstruct_collection_overrides: Mapping[type, UnstructureHook] = {}, + prefer_attrib_converters: bool = False, + detailed_validation: bool = True, + unstructure_fallback_factory: HookFactory[UnstructureHook] = lambda _: identity, + structure_fallback_factory: HookFactory[StructureHook] = lambda t: raise_error( + None, t + ), + use_alias: bool = False, + *, + bytes_encoding: Literal["base85", "base64"] = "base85", +) -> UjsonConverter: + """Create a ujson converter, optionally using base64 for bytes. + + All other parameters are passed to :class:`cattrs.Converter`. + + .. versionchanged:: NEXT + *bytes_encoding* selects base85 (the default) or base64 for bytes. + """ + res = UjsonConverter( + dict_factory=dict_factory, + unstruct_strat=unstruct_strat, + omit_if_default=omit_if_default, + forbid_extra_keys=forbid_extra_keys, + type_overrides=type_overrides, + unstruct_collection_overrides={Set: list, **unstruct_collection_overrides}, + prefer_attrib_converters=prefer_attrib_converters, + detailed_validation=detailed_validation, + unstructure_fallback_factory=unstructure_fallback_factory, + structure_fallback_factory=structure_fallback_factory, + use_alias=use_alias, + ) + configure_converter(res, bytes_encoding=bytes_encoding) return res diff --git a/tests/test_preconf.py b/tests/test_preconf.py index 18258502..08d7b785 100644 --- a/tests/test_preconf.py +++ b/tests/test_preconf.py @@ -1,6 +1,7 @@ # ruff: noqa: PLC0415 import sys import time +from base64 import b64encode, b85encode from collections.abc import Callable, Iterator, Set from datetime import date, datetime, timezone from enum import Enum, IntEnum, unique @@ -34,7 +35,7 @@ text, ) -from cattrs import Converter +from cattrs import BaseConverter, Converter from cattrs._compat import ( Counter, FrozenSet, @@ -47,11 +48,13 @@ ) from cattrs.fns import identity from cattrs.preconf.bson import make_converter as bson_make_converter +from cattrs.preconf.json import configure_converter as json_configure_converter from cattrs.preconf.json import make_converter as json_make_converter from cattrs.preconf.msgpack import make_converter as msgpack_make_converter from cattrs.preconf.pyyaml import make_converter as pyyaml_make_converter from cattrs.preconf.tomlkit import make_converter as tomlkit_make_converter from cattrs.preconf.tomllib import make_converter as tomllib_make_converter +from cattrs.preconf.ujson import configure_converter as ujson_configure_converter from cattrs.preconf.ujson import make_converter as ujson_make_converter NO_MSGSPEC: Final = python_implementation() == "PyPy" @@ -365,9 +368,10 @@ def native_unions( return Union[tuple(chosen_types)], draw(one_of(*[strats[t] for t in chosen_types])) -@given(everythings()) -def test_stdlib_json(everything: Everything): - converter = json_make_converter() +@pytest.mark.parametrize("bytes_encoding", ["base85", "base64"]) +@given(everything=everythings()) +def test_stdlib_json(everything: Everything, bytes_encoding): + converter = json_make_converter(bytes_encoding=bytes_encoding) assert ( converter.structure( json_loads(json_dumps(converter.unstructure(everything))), Everything @@ -376,6 +380,79 @@ def test_stdlib_json(everything: Everything): ) +@pytest.mark.parametrize("make_converter", [json_make_converter, ujson_make_converter]) +@pytest.mark.parametrize("bytes_encoding", [None, "base85", "base64"]) +@pytest.mark.parametrize("value", [b"", b"f", b"fo", b"foo", bytes(range(256))]) +def test_json_bytes_encoding(make_converter, bytes_encoding, value): + """Bytes use the selected wire format, including padding and empty inputs.""" + converter = ( + make_converter() + if bytes_encoding is None + else make_converter(bytes_encoding=bytes_encoding) + ) + encode = b64encode if bytes_encoding == "base64" else b85encode + expected = encode(value).decode("ascii") + assert converter.unstructure(value) == expected + assert converter.structure(expected, bytes) == value + assert json_loads(converter.dumps(value)) == expected + assert converter.loads(json_dumps(expected), bytes) == value + + +@pytest.mark.parametrize( + "configure_converter", [json_configure_converter, ujson_configure_converter] +) +@pytest.mark.parametrize("converter_cls", [BaseConverter, Converter]) +def test_configure_json_bytes_encoding(configure_converter, converter_cls): + """An existing converter can select base64 and return to the default base85.""" + converter = converter_cls() + configure_converter(converter, bytes_encoding="base64") + assert converter.unstructure(b"hello") == "aGVsbG8=" + assert converter.structure("aGVsbG8=", bytes) == b"hello" + configure_converter(converter) + assert converter.unstructure(b"hello") == "Xk~0{Zv" + assert converter.structure("Xk~0{Zv", bytes) == b"hello" + + +@pytest.mark.parametrize( + "configure_converter", [json_configure_converter, ujson_configure_converter] +) +def test_configure_json_invalid_bytes_encoding(configure_converter): + """Invalid encodings fail before overwriting existing byte hooks.""" + converter = Converter() + converter.register_unstructure_hook(bytes, lambda _: "custom") + with pytest.raises(ValueError, match="bytes_encoding must be 'base85' or 'base64'"): + configure_converter(converter, bytes_encoding="invalid") + assert converter.unstructure(b"hello") == "custom" + + +@pytest.mark.parametrize("make_converter", [json_make_converter, ujson_make_converter]) +def test_make_json_invalid_bytes_encoding(make_converter): + with pytest.raises(ValueError, match="bytes_encoding must be 'base85' or 'base64'"): + make_converter(bytes_encoding="invalid") + + +@pytest.mark.parametrize("make_converter", [json_make_converter, ujson_make_converter]) +def test_json_base64_converter_options(make_converter): + """Base64 coexists with converter options and collection overrides.""" + + @define + class Payload: + data: bytes + items: set[int] + optional: int = 0 + + converter = make_converter( + dict, + omit_if_default=True, + unstruct_collection_overrides={Set: sorted}, + bytes_encoding="base64", + ) + original = Payload(b"hello", {3, 1, 2}) + expected = {"data": "aGVsbG8=", "items": [1, 2, 3]} + assert json_loads(converter.dumps(original)) == expected + assert converter.loads(json_dumps(expected), Payload) == original + + @given(everythings()) def test_stdlib_json_converter(everything: Everything): converter = json_make_converter() @@ -447,16 +524,17 @@ def test_stdlib_json_efficient_enum(): assert converter.get_unstructure_hook(fields(Everything).a_literal) == identity +@pytest.mark.parametrize("bytes_encoding", ["base85", "base64"]) @given( - everythings( + everything=everythings( min_int=-9223372036854775808, max_int=9223372036854775807, allow_inf=False ) ) -def test_ujson(everything: Everything): +def test_ujson(everything: Everything, bytes_encoding): from ujson import dumps as ujson_dumps from ujson import loads as ujson_loads - converter = ujson_make_converter() + converter = ujson_make_converter(bytes_encoding=bytes_encoding) raw = ujson_dumps(converter.unstructure(everything)) assert converter.structure(ujson_loads(raw), Everything) == everything diff --git a/tests/test_typing_structure.md b/tests/test_typing_structure.md index 11859dd3..545524b1 100644 --- a/tests/test_typing_structure.md +++ b/tests/test_typing_structure.md @@ -2,6 +2,37 @@ These tests exercise the static return types for the structuring APIs. +## JSON converter factories accept the bytes encoding option + +```python +from typing_extensions import assert_type + +from cattrs.preconf.json import JsonConverter, make_converter as json_make_converter +from cattrs.preconf.ujson import UjsonConverter, make_converter as ujson_make_converter + + +json_converter = json_make_converter(bytes_encoding="base64", omit_if_default=True) +assert_type(json_converter, JsonConverter) +assert_type(json_converter.loads('"aGVsbG8="', bytes), bytes) + +ujson_converter = ujson_make_converter(bytes_encoding="base64", detailed_validation=False) +assert_type(ujson_converter, UjsonConverter) +assert_type(ujson_converter.loads('"aGVsbG8="', bytes), bytes) + +# mypy-error: [arg-type] +# pyrefly-error: [bad-argument-type] +json_make_converter(bytes_encoding="hex") +# mypy-error: [arg-type] +# pyrefly-error: [bad-argument-type] +ujson_make_converter(bytes_encoding="hex") +# mypy-error: [arg-type] +# pyrefly-error: [bad-argument-type] +json_make_converter(bytes_encoding="base64", omit_if_default="yes") +# mypy-error: [arg-type] +# pyrefly-error: [bad-argument-type] +ujson_make_converter(bytes_encoding="base64", detailed_validation="yes") +``` + ## Global structure returns attrs instances ```python