From 73e00eefc97cdebd30abe4dec41d845cc83d074c Mon Sep 17 00:00:00 2001 From: chivector <200636136+chivector@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:10:04 +0800 Subject: [PATCH 1/2] fix: route S3 output options to the client --- CHANGELOG.md | 3 ++ README.md | 9 ++++ benedict/dicts/io/io_util.py | 6 ++- tests/dicts/io/test_io_dict_s3.py | 88 +++++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 tests/dicts/io/test_io_dict_s3.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 89d478b7..9d856701 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased +- Fix S3 outputs forwarding `s3_options` to serializers and failing when client options are omitted. + ## [0.40.1](https://github.com/fabiocaccamo/python-benedict/releases/tag/0.40.1) - 2026-10-20 - Fix `items` and `values` returning generators instead of views (no `len`, single iteration), eg. with `toml-rs`. #636 (by [@Gabrieljr42](https://github.com/Gabrieljr42) in #637) - Honor freezing when deleting or popping a keypath and when merging into a frozen `benedict` instance. (by [@helo060228](https://github.com/helo060228) in #635) diff --git a/README.md b/README.md index 84549d4a..7ff2a5f7 100644 --- a/README.md +++ b/README.md @@ -278,6 +278,15 @@ d = benedict.from_json( - All *output* methods can be accessed as instance methods and are prefixed by `to_*` followed by the format name. - In all *output* methods, if `filepath="..."` kwarg is specified, the output will be also **saved** at the specified filepath. +- S3 outputs accept the same `s3_options` client settings as inputs. Omit them to use the standard AWS credentials and configuration: + +```python +d.to_json( + filepath="s3://my-bucket/data.json", + s3_options={"region_name": "us-east-1"}, + indent=2, +) +``` #### Supported formats diff --git a/benedict/dicts/io/io_util.py b/benedict/dicts/io/io_util.py index c2ee5eea..c36b4f43 100644 --- a/benedict/dicts/io/io_util.py +++ b/benedict/dicts/io/io_util.py @@ -83,10 +83,11 @@ def encode(d: Any, format: str, filepath: str | None = None, **kwargs: Any) -> A schema = options.pop("schema", None) if schema is not None: d = schema_util.apply_schema(d, schema) + s3_options = options.pop("s3_options", None) content = serializer.encode(d, **options) if filepath: filepath = str(filepath) - write_content(filepath, content, **options) + write_content(filepath, content, s3_options=s3_options, **options) return content @@ -201,7 +202,8 @@ def read_content_from_url( def write_content(filepath: str, content: str, **options: Any) -> None: if is_s3(filepath): - write_content_to_s3(filepath, content, **options) + s3_options = options.pop("s3_options", None) or {} + write_content_to_s3(filepath, content, s3_options=s3_options, **options) else: write_content_to_file(filepath, content, **options) diff --git a/tests/dicts/io/test_io_dict_s3.py b/tests/dicts/io/test_io_dict_s3.py new file mode 100644 index 00000000..ab2efd51 --- /dev/null +++ b/tests/dicts/io/test_io_dict_s3.py @@ -0,0 +1,88 @@ +import json +import os +import tempfile +import unittest +from pathlib import Path +from typing import Any +from unittest.mock import patch + +from benedict import benedict + + +class io_dict_s3_test_case(unittest.TestCase): + def test_to_json_s3_client_and_serializer_options(self) -> None: + for options in ( + {}, + {"s3_options": None}, + {"s3_options": {}}, + {"s3_options": {"region_name": "us-east-1"}}, + ): + with self.subTest(options=options): + data = {"z": 1, "a": "example"} + expected = json.dumps(data, indent=2, sort_keys=True) + with tempfile.TemporaryDirectory() as directory: + temporary_path = os.path.join(directory, "output.json") + + def upload( + filepath: str, + bucket: str, + key: str, + expected_path: str = temporary_path, + expected_content: str = expected, + ) -> None: + self.assertEqual(filepath, expected_path) + self.assertEqual(bucket, "test-bucket") + self.assertEqual(key, "folder/output.json") + self.assertEqual(Path(filepath).read_text(), expected_content) + + with ( + patch( + "benedict.dicts.io.io_util.tempfile.gettempdir", + return_value=directory, + ), + patch("benedict.dicts.io.io_util.boto3.client") as client, + ): + client.return_value.upload_file.side_effect = upload + result = benedict(data).to_json( + filepath="s3://test-bucket/folder/output.json", + indent=2, + sort_keys=True, + **options, + ) + self.assertEqual(result, expected) + client.assert_called_once_with( + "s3", **(options.get("s3_options") or {}) + ) + client.return_value.upload_file.assert_called_once() + client.return_value.close.assert_called_once_with() + self.assertFalse(Path(temporary_path).exists()) + + def test_output_formats_accept_s3_options(self) -> None: + data = benedict({"root": {"name": "example", "count": 3}}) + options: dict[str, Any] = {"region_name": "us-east-1"} + for method in ( + "to_base64", + "to_ini", + "to_json", + "to_pickle", + "to_plist", + "to_query_string", + "to_toml", + "to_xml", + "to_yaml", + ): + with self.subTest(method=method): + encode = getattr(data, method) + expected = encode() + with patch("benedict.dicts.io.io_util.write_content") as write: + actual = encode( + filepath="s3://test-bucket/output", + s3_options=options, + ) + self.assertEqual(actual, expected) + write.assert_called_once() + self.assertEqual( + write.call_args.args, ("s3://test-bucket/output", expected) + ) + self.assertEqual(write.call_args.kwargs["s3_options"], options) + self.assertEqual(options, {"region_name": "us-east-1"}) From c98fd4fb96e68c605e6c17ea2af7b207bd62b733 Mon Sep 17 00:00:00 2001 From: chivector <200636136+chivector@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:12:52 +0800 Subject: [PATCH 2/2] test: use postponed annotations in S3 regressions --- tests/dicts/io/test_io_dict_s3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/dicts/io/test_io_dict_s3.py b/tests/dicts/io/test_io_dict_s3.py index ab2efd51..cf717a65 100644 --- a/tests/dicts/io/test_io_dict_s3.py +++ b/tests/dicts/io/test_io_dict_s3.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import os import tempfile