From 7854b34dfa547f8d0dda1afcd4ef2733ef1fd5a6 Mon Sep 17 00:00:00 2001 From: HocVoNgThai <161827816+HocVoNgThai@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:49:01 +0700 Subject: [PATCH 01/20] Preserve BoxList comments during dotted deletion and sorting --- AUTHORS.rst | 1 + CHANGES.rst | 5 +++++ box/box_list.py | 6 +++--- test/test_comments.py | 12 ++++++++++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 2b5dbf4a..9f167759 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -121,3 +121,4 @@ Suggestions and bug reporting: - bwanshoom - tan_unique (vinson0522) - lin (theman6660) +- HocVoNgThai diff --git a/CHANGES.rst b/CHANGES.rst index e3801a3f..5f647084 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,11 @@ Changelog ========= +Unreleased +---------- + +* Preserve BoxList item comments after dotted-index deletion, and evaluate sort keys only once when comments are present. + Version 8.0.0 ------------- diff --git a/box/box_list.py b/box/box_list.py index 8de6b2f0..1470aacd 100644 --- a/box/box_list.py +++ b/box/box_list.py @@ -101,7 +101,7 @@ def __delitem__(self, key): raise BoxTypeError(f"Invalid list index: {key!r}") pos = int(list_pos.groups()[0]) if len(list_pos.group()) == len(key): - return super().__delitem__(pos) + return self.__delitem__(pos) if hasattr(self[pos], "__delitem__"): return self[pos].__delitem__(key[len(list_pos.group()) :].lstrip(".")) # type: ignore length = len(self) @@ -214,10 +214,10 @@ def sort(self, *, key=None, reverse=False): if not self.__dict__.get("_box_comments"): return super().sort(key=key, reverse=reverse) # Work out where every item goes (same stable order as list.sort) so comments follow them - sort_key = key or (lambda x: x) + sort_key = (lambda x: x) if key is None else key order = sorted(range(len(self)), key=lambda i: sort_key(list.__getitem__(self, i)), reverse=reverse) new_position = {old: new for new, old in enumerate(order)} - super().sort(key=key, reverse=reverse) + super().__setitem__(slice(None), [list.__getitem__(self, i) for i in order]) self._comment_reindex(new_position.get) # ---- Comments (#184), by item index ---- diff --git a/test/test_comments.py b/test/test_comments.py index e685c436..0e6fbc36 100644 --- a/test/test_comments.py +++ b/test/test_comments.py @@ -370,6 +370,18 @@ def test_box_list_without_comments_keeps_working(self): del bl[0:1] assert bl == [2, 1] + def test_dotted_list_deletion_keeps_surviving_comments(self): + values = BoxList.from_yaml("- first # first comment\n- second # second comment\n", box_dots=True) + del values["[0]"] + assert values.to_yaml() == "- second # second comment\n" + + def test_sort_with_comments_evaluates_keys_once(self): + values = BoxList.from_yaml("- z # Z\n- a # A\n") + ranks = {"z": 2, "a": 1} + assert values.sort(key=ranks.pop) is None + assert ranks == {} + assert values.to_yaml() == "- a # A\n- z # Z\n" + def test_slice_delete_drops_item_comments(self): bl = BoxList([1, 2, 3]) bl.set_comment(2, "x") From fa7ca67d16c5619f1f3f2c542284100abd3c1383 Mon Sep 17 00:00:00 2001 From: Chris Griffith Date: Thu, 5 Nov 2026 20:57:28 -0500 Subject: [PATCH 02/20] Version 8.0.1 Co-Authored-By: Claude Opus 5.5 --- CHANGES.rst | 7 ++++--- box/__init__.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 5f647084..237e0d98 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,10 +1,11 @@ Changelog ========= -Unreleased ----------- +Version 8.0.1 +------------- -* Preserve BoxList item comments after dotted-index deletion, and evaluate sort keys only once when comments are present. +* Fixing #349 BoxList item comments moving to the wrong items after a box_dots deletion, e.g. del bl["[0]"] (thanks to HocVoNgThai) +* Fixing #349 BoxList.sort() calling the key function twice when the list has comments (thanks to HocVoNgThai) Version 8.0.0 ------------- diff --git a/box/__init__.py b/box/__init__.py index 0f14c44b..783a5d0c 100644 --- a/box/__init__.py +++ b/box/__init__.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- __author__ = "Chris Griffith" -__version__ = "8.0.0" +__version__ = "8.0.1" from box.box import Box from box.box_list import BoxList From c66ae2792268f61a29dc8e1a1477677b43959675 Mon Sep 17 00:00:00 2001 From: Mrunmayee Daware Date: Sat, 7 Nov 2026 22:35:22 +0530 Subject: [PATCH 03/20] Clear removed BoxList item comments after in-place repetition --- AUTHORS.rst | 2 ++ CHANGES.rst | 2 ++ box/box_list.py | 5 ++++- test/test_comments.py | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 9f167759..a43e670b 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -122,3 +122,5 @@ Suggestions and bug reporting: - tan_unique (vinson0522) - lin (theman6660) - HocVoNgThai + +- Mrunmayee Daware (Mrun25) diff --git a/CHANGES.rst b/CHANGES.rst index 237e0d98..75dc240c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,8 @@ Changelog Version 8.0.1 ------------- +* Clear removed BoxList item comments after in-place repetition by zero or a negative count, so new items do not inherit stale YAML comments (thanks to Mrunmayee Daware) + * Fixing #349 BoxList item comments moving to the wrong items after a box_dots deletion, e.g. del bl["[0]"] (thanks to HocVoNgThai) * Fixing #349 BoxList.sort() calling the key function twice when the list has comments (thanks to HocVoNgThai) diff --git a/box/box_list.py b/box/box_list.py index 1470aacd..2bd853a5 100644 --- a/box/box_list.py +++ b/box/box_list.py @@ -154,7 +154,10 @@ def __iadd__(self, value): # type: ignore[misc] # list accepts any iterable fo def __imul__(self, value): # type: ignore[misc] if self.box_options.get("frozen_box"): raise BoxError("BoxList is frozen") - return super().__imul__(value) + result = super().__imul__(value) + if len(self) == 0: + self.__dict__.pop("_box_comments", None) + return result def _is_intact_type(self, obj): if self.box_options.get("box_intact_types") and isinstance(obj, self.box_options["box_intact_types"]): diff --git a/test/test_comments.py b/test/test_comments.py index 0e6fbc36..51be0aff 100644 --- a/test/test_comments.py +++ b/test/test_comments.py @@ -434,3 +434,37 @@ def test_tabs_are_fine(self): bx = Box(a=1) bx.set_comment("a", "x\ty") assert Box.from_yaml(bx.to_yaml()).get_comment("a") == {"eol": "x\ty"} + + +@pytest.mark.parametrize("count", [0, -1, -3]) +def test_inplace_repetition_drops_comments_of_removed_items(count): + values = BoxList.from_yaml("# list heading\n- old # obsolete explanation\n") + original = values + values *= count + assert values is original + assert values == [] + values.append("new") + assert values.get_comment(0) is None + assert values.get_comment() == {"start": "list heading"} + assert values.to_yaml() == "# list heading\n- new\n" + + +@pytest.mark.parametrize("count", [1, 2]) +def test_inplace_repetition_keeps_surviving_item_comments(count): + values = BoxList.from_yaml("- old # explanation\n") + original = values + values *= count + assert values is original + assert values == ["old"] * count + assert values.get_comment(0) == {"eol": "explanation"} + + +def test_inplace_repetition_keeps_comments_on_false_valued_subclass(): + class FalseBoxList(BoxList): + def __bool__(self): + return False + + values = FalseBoxList(["old"]) + values.set_comment(0, "explanation") + values *= 1 + assert values.get_comment(0) == {"eol": "explanation"} From 8183d430951da61d1382a96307438530b0e5ffed Mon Sep 17 00:00:00 2001 From: Alexandre <46300952+alexbertho@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:58:48 +0100 Subject: [PATCH 04/20] Preserve one-shot pairs in ConfigBox.merge_update --- AUTHORS.rst | 1 + CHANGES.rst | 1 + box/config_box.py | 3 ++- test/test_config_box_case.py | 9 +++++++++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 9f167759..bb07826b 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -5,6 +5,7 @@ reporting, all input is greatly appreciated! Code contributions: +- Alexandre Bertho (alexbertho) - Alexandre Decan (AlexandreDecan) - dhilipsiva (dhilipsiva) - MAA (FooBarQuaxx) diff --git a/CHANGES.rst b/CHANGES.rst index 237e0d98..cf117388 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,7 @@ Changelog Version 8.0.1 ------------- +* Fix ConfigBox.merge_update consuming one-shot key/value pairs before merging them * Fixing #349 BoxList item comments moving to the wrong items after a box_dots deletion, e.g. del bl["[0]"] (thanks to HocVoNgThai) * Fixing #349 BoxList.sort() calling the key function twice when the list has comments (thanks to HocVoNgThai) diff --git a/box/config_box.py b/box/config_box.py index 5d5ed876..9f7b37e8 100644 --- a/box/config_box.py +++ b/box/config_box.py @@ -203,7 +203,8 @@ def merge_update(self, *args, **kwargs): if hasattr(args[0], "keys"): incoming = list(args[0].keys()) else: - args = (list(args[0]),) # may be a one-shot iterator, it is read twice + # Both the outer iterable and its pairs may be one-shot iterators. + args = ([tuple(pair) for pair in args[0]],) incoming = [k for k, _ in args[0]] else: incoming = [] diff --git a/test/test_config_box_case.py b/test/test_config_box_case.py index 95f5f9c5..63530cc1 100644 --- a/test/test_config_box_case.py +++ b/test/test_config_box_case.py @@ -12,6 +12,15 @@ class TestConfigBoxCaseInsensitive: + @pytest.mark.parametrize("outer", [list, iter]) + def test_merge_update_one_shot_pairs(self, outer): + cb = ConfigBox({"Host": {"port": 80}}) + pairs = outer([iter(("host", {"name": "localhost"})), iter(("timeout", 30))]) + cb.merge_update(pairs) + assert cb.to_dict() == {"host": {"port": 80, "name": "localhost"}, "timeout": 30} + assert list(cb) == ["host", "timeout"] + assert cb.HOST.port == 80 + def test_lookup_any_case(self): cb = ConfigBox({"HOST": "localhost", "ServerPort": 8080}) assert cb["host"] == cb["Host"] == cb.host == cb.HOST == "localhost" From 40edc1fa98b3e53acf4d739d15c7539581713f8c Mon Sep 17 00:00:00 2001 From: parijat1222q <98056935+parijat1222q@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:00:36 +0530 Subject: [PATCH 05/20] Preserve BoxList comments with index protocol objects --- AUTHORS.rst | 2 + CHANGES.rst | 2 + box/box_list.py | 5 +++ test/test_box_list.py | 96 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index 9f167759..ff270fad 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -5,6 +5,8 @@ reporting, all input is greatly appreciated! Code contributions: +- parijat1222q + - Alexandre Decan (AlexandreDecan) - dhilipsiva (dhilipsiva) - MAA (FooBarQuaxx) diff --git a/CHANGES.rst b/CHANGES.rst index 237e0d98..4f103e79 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,8 @@ Changelog Version 8.0.1 ------------- +* Preserve BoxList item comments when insert, pop and deletion use objects with __index__ (thanks to parijat1222q) + * Fixing #349 BoxList item comments moving to the wrong items after a box_dots deletion, e.g. del bl["[0]"] (thanks to HocVoNgThai) * Fixing #349 BoxList.sort() calling the key function twice when the list has comments (thanks to HocVoNgThai) diff --git a/box/box_list.py b/box/box_list.py index 1470aacd..f6091494 100644 --- a/box/box_list.py +++ b/box/box_list.py @@ -6,6 +6,7 @@ import copy import copyreg +import operator import re from collections.abc import Iterable from os import PathLike @@ -104,6 +105,8 @@ def __delitem__(self, key): return self.__delitem__(pos) if hasattr(self[pos], "__delitem__"): return self[pos].__delitem__(key[len(list_pos.group()) :].lstrip(".")) # type: ignore + if not isinstance(key, slice) and hasattr(type(key), "__index__"): + key = operator.index(key) length = len(self) super().__delitem__(key) self._comment_after_delete(key, length) @@ -186,6 +189,7 @@ def extend(self, iterable): self.append(item) def insert(self, index, p_object): + index = operator.index(index) length = len(self) super().insert(index, self._convert(p_object)) if self.__dict__.get("_box_comments"): @@ -193,6 +197,7 @@ def insert(self, index, p_object): self._comment_reindex(lambda k: k + 1 if k >= at else k) def pop(self, index=-1): + index = operator.index(index) length = len(self) value = super().pop(index) self._comment_after_delete(index, length) diff --git a/test/test_box_list.py b/test/test_box_list.py index 469c3e98..4291ac63 100644 --- a/test/test_box_list.py +++ b/test/test_box_list.py @@ -666,3 +666,99 @@ def test_iadd_frozen_does_not_consume_source(): data += source assert next(source) == {"value": 1} assert data == [0] + + +class _IndexOnce: + def __init__(self, value): + self.value = value + self.calls = 0 + + def __index__(self): + self.calls += 1 + if self.calls > 1: + raise AssertionError("index was evaluated more than once") + return self.value + + +@pytest.mark.parametrize("operation", ["insert", "pop", "delete"]) +@pytest.mark.parametrize("index", [0, 1, -1, -3]) +@pytest.mark.parametrize("comments", [False, True]) +def test_index_protocol_keeps_item_comments(operation, index, comments): + values = BoxList(["a", "b", "c"]) + if comments: + for position, item in enumerate(values): + values.set_comment(position, item) + expected = list(values) + key = _IndexOnce(index) + if operation == "insert": + expected.insert(index, "new") + values.insert(key, "new") + elif operation == "pop": + assert values.pop(key) == expected.pop(index) + else: + del expected[index] + del values[key] + assert key.calls == 1 + assert values == expected + for position, item in enumerate(values): + comment = {"eol": item} if comments and item != "new" else None + assert values.get_comment(position) == comment + + +@pytest.mark.parametrize("operation", ["insert", "pop", "delete"]) +@pytest.mark.parametrize("comments", [False, True]) +def test_invalid_index_does_not_change_items_or_comments(operation, comments): + class BadIndex: + def __index__(self): + raise ValueError("invalid index") + + values = BoxList(["a", "b", "c"]) + if comments: + for position, item in enumerate(values): + values.set_comment(position, item) + with pytest.raises(ValueError, match="invalid index"): + if operation == "insert": + values.insert(BadIndex(), "new") + elif operation == "pop": + values.pop(BadIndex()) + else: + del values[BadIndex()] + assert values == ["a", "b", "c"] + for position, item in enumerate(values): + assert values.get_comment(position) == ({"eol": item} if comments else None) + + +@pytest.mark.parametrize("index", [-100, 100]) +@pytest.mark.parametrize("comments", [False, True]) +def test_insert_index_protocol_clamps_and_moves_comments(index, comments): + values = BoxList(["a", "b"]) + if comments: + values.set_comment(0, "a") + values.set_comment(1, "b") + expected = list(values) + expected.insert(index, "new") + key = _IndexOnce(index) + values.insert(key, "new") + assert key.calls == 1 + assert values == expected + for position, item in enumerate(values): + expected_comment = {"eol": item} if comments and item != "new" else None + assert values.get_comment(position) == expected_comment + + +@pytest.mark.parametrize("operation", ["pop", "delete"]) +@pytest.mark.parametrize("index", [-100, 100]) +def test_out_of_bounds_index_preserves_comments(operation, index): + values = BoxList(["a", "b"]) + values.set_comment(0, "a") + values.set_comment(1, "b") + key = _IndexOnce(index) + with pytest.raises(IndexError): + if operation == "pop": + values.pop(key) + else: + del values[key] + assert key.calls == 1 + assert values == ["a", "b"] + assert values.get_comment(0) == {"eol": "a"} + assert values.get_comment(1) == {"eol": "b"} From 5c19bca734cb2c5392d57ec141676a97bf0bf6ab Mon Sep 17 00:00:00 2001 From: Rahul Ranjan Date: Mon, 9 Nov 2026 09:25:13 +0000 Subject: [PATCH 06/20] Fix update() storing dotted keys literally in box_dots mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit update() wrote through __convert_and_store directly, skipping the _parse_box_dots traversal that __setitem__ applies, so a key such as 'x.z' was stored as a literal key in parallel to the nested x path — even when x already existed. merge_update already stores through __setitem__ and nests correctly; Box + Box merges through update() and was affected too. update() now routes keys that box_dots would treat as paths through __setitem__; everything else (non-dotted keys, or keys matched by box_dots_exclude) keeps the previous behavior. Fixes #356 Co-authored-by: rahul05ranjan <11793455+rahul05ranjan@users.noreply.github.com> --- box/box.py | 15 ++++++++++++--- test/test_box_keys_8.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/box/box.py b/box/box.py index e1ce2047..020ba875 100644 --- a/box/box.py +++ b/box/box.py @@ -1058,12 +1058,21 @@ def update(self, *args, **kwargs): if single_arg: if hasattr(single_arg, "keys"): for k in single_arg: - self.__convert_and_store(k, single_arg[k]) + self._update_store(k, single_arg[k]) else: for k, v in single_arg: - self.__convert_and_store(k, v) + self._update_store(k, v) for k in kwargs: - self.__convert_and_store(k, kwargs[k]) + self._update_store(k, kwargs[k]) + + def _update_store(self, key, value): + # In box_dots mode a dotted key is a path, the same as in + # __setitem__ and merge_update; storing it literally would create + # a parallel key that dots lookups cannot reach consistently. + if self.__process_dotted_key(key): + self.__setitem__(key, value) + else: + self.__convert_and_store(key, value) def merge_update(self, *args, **kwargs): """ diff --git a/test/test_box_keys_8.py b/test/test_box_keys_8.py index 690dc424..0b39778f 100644 --- a/test/test_box_keys_8.py +++ b/test/test_box_keys_8.py @@ -117,3 +117,38 @@ def test_config_box(self): cb = ConfigBox({"maxConnections": "5"}, camel_killer_box=True, camel_killer_box_keep_keys=True) assert cb.int("max_connections") == 5 assert cb.to_dict() == {"maxConnections": "5"} + + +class TestUpdateDottedPaths: + """ update() honors box_dots path semantics, like __setitem__ and merge_update. """ + + def test_update_dotted_key_nests(self): + b = Box(default_box=True, box_dots=True) + b.update({"x.y": 5}) + assert b["x"]["y"] == 5 + + def test_update_dotted_key_extends_existing_path(self): + b = Box({"x": {"y": 1}}, box_dots=True) + b.update({"x.z": 5}) + assert b.to_dict() == {"x": {"y": 1, "z": 5}} + + def test_update_dotted_key_with_dots_exclude_stays_literal(self): + b = Box(box_dots=True, box_dots_exclude=r"^[a-z]+\.[a-z]+$") + b.update({"keep.this": 1}) + assert b.to_dict() == {"keep.this": 1} + + def test_update_kwargs_dotted_key_nests(self): + b = Box(default_box=True, box_dots=True) + b.update(**{"k.k": 9}) + assert b["k"]["k"] == 9 + + def test_update_non_dotted_unchanged(self): + b = Box({"a": 1}) + b.update({"b": 2}, c=3) + assert b.to_dict() == {"a": 1, "b": 2, "c": 3} + + def test_box_addition_merges_dotted_paths(self): + b = Box({"x": {"y": 1}}, default_box=True, box_dots=True) + Box( + {"x.z": 2}, default_box=True, box_dots=True + ) + assert b.to_dict() == {"x": {"y": 1, "z": 2}} From d3f0d31ded4094c9300b385e173cb16f25aff6f5 Mon Sep 17 00:00:00 2001 From: Rahul Ranjan Date: Mon, 9 Nov 2026 09:29:37 +0000 Subject: [PATCH 07/20] Make BoxList + and * return a BoxList The out-of-place operators fell through to plain list, dropping the class, the box options and the conversion of appended items: BoxList([{'a': 1}]) + [{'b': 2}] -> plain list, second item a raw dict BoxList([{'a': 1}]) * 2 -> plain list __add__ and __mul__ now return a BoxList with the same box options; appended items are converted the same way append/extend convert them, and a frozen operand produces a frozen result. The in-place variants were already correct. Fixes #358 Co-authored-by: rahul05ranjan <11793455+rahul05ranjan@users.noreply.github.com> --- box/box_list.py | 10 ++++++++++ test/test_box_list.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/box/box_list.py b/box/box_list.py index 8de6b2f0..15ba282d 100644 --- a/box/box_list.py +++ b/box/box_list.py @@ -145,6 +145,16 @@ def converted_items(): return super().__setitem__(key, self._convert(value)) + def __add__(self, other): + if not isinstance(other, list): + raise BoxTypeError("BoxList can only be added to another list.") + return box.BoxList(list(self) + list(other), **self.box_options) + + def __mul__(self, count): + if not isinstance(count, int): + raise BoxTypeError("BoxList can only be multiplied by an integer.") + return box.BoxList(list(self) * count, **self.box_options) + def __iadd__(self, value): # type: ignore[misc] # list accepts any iterable for +=, not only lists if self.box_options.get("frozen_box"): raise BoxError("BoxList is frozen") diff --git a/test/test_box_list.py b/test/test_box_list.py index 469c3e98..15df57b0 100644 --- a/test/test_box_list.py +++ b/test/test_box_list.py @@ -14,6 +14,7 @@ from ruamel.yaml import YAML from box import Box, BoxError, BoxList +from box.exceptions import BoxTypeError from box.converters import toml_read_library, toml_write_library from test.common import test_root, tmp_dir @@ -666,3 +667,41 @@ def test_iadd_frozen_does_not_consume_source(): data += source assert next(source) == {"value": 1} assert data == [0] + + +class TestBoxListOperators: + def test_add_keeps_boxlist_and_converts(self): + bl = BoxList([{"a": 1}], box_dots=True) + result = bl + [{"b": 2}] + assert isinstance(result, BoxList) + assert isinstance(result[0], Box) + assert isinstance(result[1], Box) + assert result.box_options["box_dots"] is True + + def test_add_two_boxlists(self): + result = BoxList([{"a": 1}]) + BoxList([{"b": 2}]) + assert isinstance(result, BoxList) + assert result.to_list() == [{"a": 1}, {"b": 2}] + + def test_add_non_list_raises(self): + with pytest.raises(BoxTypeError): + BoxList() + 5 + + def test_mul_keeps_boxlist(self): + bl = BoxList([{"a": 1}], box_dots=True) + result = bl * 3 + assert isinstance(result, BoxList) + assert len(result) == 3 + assert result.box_options["box_dots"] is True + assert result.to_list() == [{"a": 1}] * 3 + + def test_mul_non_int_raises(self): + with pytest.raises(BoxTypeError): + BoxList() * "x" + + def test_frozen_add_result_is_frozen(self): + frozen = BoxList([{"a": 1}], frozen_box=True) + result = frozen + [{"b": 2}] + assert isinstance(result, BoxList) + with pytest.raises(BoxError): + result.append({"c": 3}) From a12d27f16d8078794749f0eae0a1aaff5ec271ab Mon Sep 17 00:00:00 2001 From: Anish Pentyala Date: Wed, 11 Nov 2026 16:55:20 -0800 Subject: [PATCH 08/20] Normalize in-place BoxList repetition counts before mutation --- box/box_list.py | 1 + 1 file changed, 1 insertion(+) diff --git a/box/box_list.py b/box/box_list.py index c4ba9464..7bf6c70f 100644 --- a/box/box_list.py +++ b/box/box_list.py @@ -175,6 +175,7 @@ def __iadd__(self, value): # type: ignore[misc] # list accepts any iterable fo def __imul__(self, value): # type: ignore[misc] if self.box_options.get("frozen_box"): raise BoxError("BoxList is frozen") + value = operator.index(value) length = len(self) result = super().__imul__(value) if len(self) == 0: From b6743c342be70b955aa42cfc3e18522807bba2e6 Mon Sep 17 00:00:00 2001 From: Chris Griffith Date: Wed, 11 Nov 2026 11:57:36 -0600 Subject: [PATCH 09/20] Add changelog for #356/#358; BoxList * accepts __index__ and supports reflected multiply Co-Authored-By: Claude Sonnet 5.5 --- CHANGES.rst | 2 ++ box/box_list.py | 8 ++++++-- test/test_box_list.py | 10 ++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b0a04c05..bad8e4f0 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,8 @@ Version 8.0.1 * Fix ConfigBox.merge_update consuming one-shot key/value pairs before merging them * Preserve BoxList item comments when insert, pop and deletion use objects with __index__ (thanks to parijat1222q) +* Fixing #356 update() storing dotted keys literally with box_dots instead of treating them as paths (thanks to rahul05ranjan) +* Fixing #358 BoxList + and * returning a plain list instead of a BoxList (thanks to rahul05ranjan) * Fixing #349 BoxList item comments moving to the wrong items after a box_dots deletion, e.g. del bl["[0]"] (thanks to HocVoNgThai) * Fixing #349 BoxList.sort() calling the key function twice when the list has comments (thanks to HocVoNgThai) diff --git a/box/box_list.py b/box/box_list.py index 77c1549f..62a963f0 100644 --- a/box/box_list.py +++ b/box/box_list.py @@ -154,10 +154,14 @@ def __add__(self, other): return box.BoxList(list(self) + list(other), **self.box_options) def __mul__(self, count): - if not isinstance(count, int): - raise BoxTypeError("BoxList can only be multiplied by an integer.") + try: + count = operator.index(count) + except TypeError: + raise BoxTypeError("BoxList can only be multiplied by an integer.") from None return box.BoxList(list(self) * count, **self.box_options) + __rmul__ = __mul__ + def __iadd__(self, value): # type: ignore[misc] # list accepts any iterable for +=, not only lists if self.box_options.get("frozen_box"): raise BoxError("BoxList is frozen") diff --git a/test/test_box_list.py b/test/test_box_list.py index f6de2183..e42e9680 100644 --- a/test/test_box_list.py +++ b/test/test_box_list.py @@ -789,6 +789,16 @@ def test_mul_keeps_boxlist(self): assert result.box_options["box_dots"] is True assert result.to_list() == [{"a": 1}] * 3 + def test_rmul_and_index_protocol(self): + class Two: + def __index__(self): + return 2 + + bl = BoxList([{"a": 1}]) + for result in (2 * bl, bl * Two()): + assert isinstance(result, BoxList) + assert result.to_list() == [{"a": 1}] * 2 + def test_mul_non_int_raises(self): with pytest.raises(BoxTypeError): BoxList() * "x" From 2ed8e6baad068e5aa9eec89d29a4de6d5e5d9a2e Mon Sep 17 00:00:00 2001 From: Anish Pentyala Date: Wed, 11 Nov 2026 16:55:37 -0800 Subject: [PATCH 10/20] Document in-place repetition count fix --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 29f807be..dc749bbc 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,6 +3,8 @@ Changelog Version 8.0.1 ------------- +* Preserve BoxList item comments when in-place repetition uses an object with __index__, without raising after changing the list + * Clear removed BoxList item comments after in-place repetition by zero or a negative count, so new items do not inherit stale YAML comments (thanks to Mrunmayee Daware) * Fix ConfigBox.merge_update consuming one-shot key/value pairs before merging them From b3effff60dd51ddcc56dd9f735cbd4c76cd63dd3 Mon Sep 17 00:00:00 2001 From: Chris Griffith Date: Wed, 11 Nov 2026 11:57:43 -0600 Subject: [PATCH 11/20] Sort imports in test_box_list Co-Authored-By: Claude Sonnet 5.5 --- test/test_box_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_box_list.py b/test/test_box_list.py index e42e9680..6faabe2a 100644 --- a/test/test_box_list.py +++ b/test/test_box_list.py @@ -14,8 +14,8 @@ from ruamel.yaml import YAML from box import Box, BoxError, BoxList -from box.exceptions import BoxTypeError from box.converters import toml_read_library, toml_write_library +from box.exceptions import BoxTypeError from test.common import test_root, tmp_dir From cde977073c411e819ad5395521d24047601da95b Mon Sep 17 00:00:00 2001 From: Anish Pentyala Date: Wed, 11 Nov 2026 16:55:55 -0800 Subject: [PATCH 12/20] Add contributor credit for repetition fix --- AUTHORS.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index 43c5149d..ea7e27c4 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -5,6 +5,8 @@ reporting, all input is greatly appreciated! Code contributions: +- Anish Pentyala + - Alexandre Bertho (alexbertho) - parijat1222q From a0d5939eb077800391e0899610ef55dcd3bc9837 Mon Sep 17 00:00:00 2001 From: Chris Griffith Date: Wed, 11 Nov 2026 12:07:38 -0600 Subject: [PATCH 13/20] Add audit tests for BoxList comment sync, box_dots consistency and round trips Parametrized matrix over every BoxList mutator (plain, box_dots, frozen), box_dots accessor consistency, and copy/deepcopy/pickle round trips of Box, BoxList, ConfigBox, SBox and DDBox with comments. Known bugs are reproduced as strict xfails (run with --runxfail to see them fail). Co-Authored-By: Claude Opus 5.5 --- test/test_audit_sync.py | 727 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 727 insertions(+) create mode 100644 test/test_audit_sync.py diff --git a/test/test_audit_sync.py b/test/test_audit_sync.py new file mode 100644 index 00000000..94e7f52f --- /dev/null +++ b/test/test_audit_sync.py @@ -0,0 +1,727 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Audit of the places where BoxList item comments and Box options are kept in sync by hand. + +1. Every BoxList mutator, with item comments present, plain / box_dots / frozen: comments must stay + on the item they were attached to, and new items must be converted to Box / BoxList. +2. box_dots paths must behave the same through every access method. +3. copy / deepcopy / pickle round trips of every Box flavour, with comments. + +Tests marked ``xfail(strict=True)`` reproduce a known bug: run with ``--runxfail`` to see them fail. +Once a fix lands, the strict xfail turns into a failure, so the marker must be removed with the fix. +""" + +import copy +import pickle + +import pytest + +from box import Box, BoxError, BoxList, ConfigBox, SBox +from box.shorthand_box import DDBox + +# -------------------------------------------------------------------------------------------------- +# 1. BoxList mutators +# -------------------------------------------------------------------------------------------------- + +N = 4 # items 0..3, item i is {"id": i} carrying the comment "c" +NEW = 100 # ids >= NEW are items added by the operation, they never carry a comment + + +def annotated(**options): + bl = BoxList([{"id": i} for i in range(N)], **options) + for i in range(N): + bl.set_comment(i, f"c{i}") + return bl + + +def frozen_annotated(**options): + return BoxList(annotated(**options), frozen_box=True, **options) + + +def check_comments(bl, kept): + """ + A comment, wherever it is, must be the one of the item now at that position, and every + item id in `kept` must still have its comment. + """ + seen = set() + for i, item in enumerate(bl): + comment = bl.get_comment(i) + if comment is None: + continue + assert isinstance(item, Box), f"comment {comment} on non item {item!r} at {i}" + assert comment == {"eol": f"c{item.id}"}, f"position {i} holds item {item.id} but has comment {comment}" + seen.add(item.id) + missing = set(kept) - seen + assert not missing, f"comments lost for items {sorted(missing)}" + + +def check_converted(bl, **options): + assert isinstance(bl, BoxList) + for item in bl: + assert not (isinstance(item, dict) and not isinstance(item, Box)), f"{item!r} not converted" + assert not (isinstance(item, list) and not isinstance(item, BoxList)), f"{item!r} not converted" + if isinstance(item, Box): + for k, v in options.items(): + assert item._box_config[k] == v, f"option {k} not passed down to {item!r}" + if isinstance(item, BoxList): + for k, v in options.items(): + assert item.box_options.get(k) == v, f"option {k} not passed down to {item!r}" + + +def _setitem(bl, key, value): + bl[key] = value + + +def _delitem(bl, key): + del bl[key] + + +def _iadd(bl, value): + bl += value + + +def _imul(bl, value): + bl *= value + + +ALL = set(range(N)) + +# (id, operation, ids whose comment must survive) +IN_PLACE_OPS = [ + ("append", lambda bl: bl.append({"id": NEW}), ALL), + ("append_list", lambda bl: bl.append([{"id": NEW}]), ALL), + ("extend", lambda bl: bl.extend([{"id": NEW}, [{"id": NEW + 1}]]), ALL), + ("extend_generator", lambda bl: bl.extend({"id": NEW + i} for i in range(2)), ALL), + ("extend_self", lambda bl: bl.extend(bl), ALL), + ("iadd", lambda bl: _iadd(bl, [{"id": NEW}]), ALL), + ("iadd_tuple", lambda bl: _iadd(bl, ({"id": NEW},)), ALL), + ("insert_0", lambda bl: bl.insert(0, {"id": NEW}), ALL), + ("insert_mid", lambda bl: bl.insert(2, {"id": NEW}), ALL), + ("insert_neg", lambda bl: bl.insert(-1, {"id": NEW}), ALL), + ("insert_past_end", lambda bl: bl.insert(99, {"id": NEW}), ALL), + ("insert_before_start", lambda bl: bl.insert(-99, {"id": NEW}), ALL), + ("pop_last", lambda bl: bl.pop(), {0, 1, 2}), + ("pop_0", lambda bl: bl.pop(0), {1, 2, 3}), + ("pop_neg", lambda bl: bl.pop(-3), {0, 2, 3}), + ("del_1", lambda bl: _delitem(bl, 1), {0, 2, 3}), + ("del_neg", lambda bl: _delitem(bl, -1), {0, 1, 2}), + ("remove", lambda bl: bl.remove({"id": 2}), {0, 1, 3}), + ("clear", lambda bl: bl.clear(), set()), + ("reverse", lambda bl: bl.reverse(), ALL), + ("sort_desc", lambda bl: bl.sort(key=lambda x: x.id, reverse=True), ALL), + ("sort_mixed", lambda bl: bl.sort(key=lambda x: x.id % 2), ALL), + # A replaced item keeps the comment of its position, like a Box key; the replacement reuses the id + ("setitem", lambda bl: _setitem(bl, 1, {"id": 1, "new": True}), ALL), + ("setitem_neg", lambda bl: _setitem(bl, -1, {"id": 3, "new": True}), ALL), + ("imul_1", lambda bl: _imul(bl, 1), ALL), + ("imul_2", lambda bl: _imul(bl, 2), ALL), + ("imul_0", lambda bl: _imul(bl, 0), set()), + ("imul_neg", lambda bl: _imul(bl, -1), set()), +] + +# Slices: the items outside the slice never move (or only shift), their comments should survive +SLICE_OPS = [ + ("del_slice_tail", lambda bl: _delitem(bl, slice(2, None)), {0, 1}), + ("del_slice_head", lambda bl: _delitem(bl, slice(0, 1)), {1, 2, 3}), + ("del_slice_step", lambda bl: _delitem(bl, slice(None, None, 2)), {1, 3}), + ("setitem_slice_same_len", lambda bl: _setitem(bl, slice(3, 4), [{"id": NEW}]), {0, 1, 2}), + ("setitem_slice_grow", lambda bl: _setitem(bl, slice(1, 2), [{"id": NEW}, {"id": NEW + 1}]), {0, 2, 3}), + ("setitem_slice_shrink", lambda bl: _setitem(bl, slice(1, 3), []), {0, 3}), + ("setitem_slice_append", lambda bl: _setitem(bl, slice(N, N), [{"id": NEW}]), ALL), + ("setitem_ext_slice", lambda bl: _setitem(bl, slice(None, None, 2), [{"id": NEW}, {"id": NEW + 1}]), {1, 3}), +] + +DOTS_OPS = [ + ("dots_setitem", lambda bl: _setitem(bl, "[1]", {"id": 1, "new": True}), ALL), + ("dots_setitem_child", lambda bl: _setitem(bl, "[1].extra", {"x": 1}), ALL), + ("dots_del", lambda bl: _delitem(bl, "[1]"), {0, 2, 3}), + ("dots_del_child", lambda bl: _delitem(bl, "[1].id") or bl[1].__setitem__("id", 1), ALL), +] + +OPTION_SETS = [pytest.param({}, id="plain"), pytest.param({"box_dots": True}, id="box_dots")] + + +def _ids(ops): + return [pytest.param(fn, kept, id=name) for name, fn, kept in ops] + + +@pytest.mark.parametrize("options", OPTION_SETS) +@pytest.mark.parametrize("operation, kept", _ids(IN_PLACE_OPS)) +def test_mutator_keeps_comments_and_converts(options, operation, kept): + bl = annotated(**options) + operation(bl) + check_comments(bl, kept) + check_converted(bl, **options) + + +@pytest.mark.parametrize("options", OPTION_SETS) +@pytest.mark.parametrize("operation, kept", _ids(SLICE_OPS)) +@pytest.mark.xfail( + strict=True, + reason="BUG: slice assignment / deletion drops the comments of every item, also the ones the slice " + "does not touch (box_list.py BoxList.__setitem__ slice branch and _comment_after_delete)", +) +def test_slice_mutator_keeps_comments_of_untouched_items(options, operation, kept): + bl = annotated(**options) + operation(bl) + check_comments(bl, kept) + + +@pytest.mark.parametrize("options", OPTION_SETS) +@pytest.mark.parametrize("operation, kept", _ids(SLICE_OPS)) +def test_slice_mutator_never_misplaces_comments(options, operation, kept): + bl = annotated(**options) + operation(bl) + check_comments(bl, set()) + check_converted(bl, **options) + + +@pytest.mark.parametrize("operation, kept", _ids(DOTS_OPS)) +def test_box_dots_mutator(operation, kept): + bl = annotated(box_dots=True) + operation(bl) + check_comments(bl, kept) + check_converted(bl, box_dots=True) + + +def test_box_dots_mutators_through_parent_box(): + b = Box({"l": [{"id": i} for i in range(N)]}, box_dots=True) + for i in range(N): + b.l.set_comment(i, f"c{i}") + b["l[1]"] = {"id": 1, "new": True} + check_comments(b.l, ALL) + del b["l[0]"] + check_comments(b.l, {1, 2, 3}) + assert b.pop("l[0]").id == 1 + check_comments(b.l, {2, 3}) + assert b.get_comment("l[0]") == {"eol": "c2"} + assert b.get_comment("l[1]") == {"eol": "c3"} + check_converted(b.l, box_dots=True) + + +# ---- non mutating operations: the result carries the comments, the original is untouched ---- + +COPY_OPS = [ + ("copy", copy.copy), + ("deepcopy", copy.deepcopy), + ("pickle", lambda bl: pickle.loads(pickle.dumps(bl))), + ("pickle_p0", lambda bl: pickle.loads(pickle.dumps(bl, protocol=0))), + ("BoxList()", lambda bl: BoxList(bl, **bl.box_options)), +] + + +@pytest.mark.parametrize("options", OPTION_SETS + [pytest.param({"frozen_box": True}, id="frozen")]) +@pytest.mark.parametrize("operation", [pytest.param(fn, id=name) for name, fn in COPY_OPS]) +def test_copies_keep_comments_options_and_conversion(options, operation): + bl = frozen_annotated() if options.get("frozen_box") else annotated(**options) + bl.set_comment(None, "document") if not options.get("frozen_box") else None + out = operation(bl) + assert out is not bl + assert out == bl + check_comments(out, ALL) + check_converted(out, **options) + for k, v in options.items(): + assert out.box_options.get(k) == v + if not options.get("frozen_box"): + assert out.get_comment(None) == {"start": "document"} + out.set_comment(0, "changed") # comments are not shared with the original + assert bl.get_comment(0) == {"eol": "c0"} + + +@pytest.mark.parametrize( + "operation", + [ + pytest.param(lambda bl: bl + [{"id": NEW}], id="add"), + pytest.param(lambda bl: bl * 2, id="mul"), + pytest.param(lambda bl: 2 * bl, id="rmul"), + ], +) +def test_binary_operators_convert(operation): + out = operation(annotated(box_dots=True)) + check_comments(out, set()) + check_converted(out, box_dots=True) + + +@pytest.mark.parametrize( + "operation", + [ + pytest.param(lambda bl: bl + [{"id": NEW}], id="add"), + pytest.param(lambda bl: bl * 2, id="mul"), + pytest.param(lambda bl: 2 * bl, id="rmul"), + ], +) +@pytest.mark.xfail( + strict=True, + reason="INCONSISTENT: + and * drop every item comment while copy(), BoxList(bl), += and *= (n>0) keep " + "them on the original items (box_list.py BoxList.__add__ / __mul__)", +) +def test_binary_operators_keep_comments_of_left_operand(operation): + out = operation(annotated()) + check_comments(out, ALL) + + +# ---- frozen ---- + +FROZEN_MUTATORS = [ + ("append", lambda bl: bl.append({"id": NEW})), + ("extend", lambda bl: bl.extend([{"id": NEW}])), + ("insert", lambda bl: bl.insert(0, {"id": NEW})), + ("pop", lambda bl: bl.pop()), + ("remove", lambda bl: bl.remove(bl[0])), + ("clear", lambda bl: bl.clear()), + ("reverse", lambda bl: bl.reverse()), + ("sort", lambda bl: bl.sort(key=lambda x: -x.id)), + ("setitem", lambda bl: _setitem(bl, 0, {"id": NEW})), + ("setitem_slice", lambda bl: _setitem(bl, slice(0, 1), [{"id": NEW}])), + ("delitem", lambda bl: _delitem(bl, 0)), + ("delitem_slice", lambda bl: _delitem(bl, slice(0, 1))), + ("iadd", lambda bl: _iadd(bl, [{"id": NEW}])), + ("imul", lambda bl: _imul(bl, 2)), + ("imul_0", lambda bl: _imul(bl, 0)), + ("set_comment", lambda bl: bl.set_comment(0, "x")), + ("del_comment", lambda bl: bl.del_comment(0)), +] + + +def _frozen_direct(): + return frozen_annotated() + + +def _frozen_through_box(): + # A frozen Box adopting an existing BoxList (for example Box(other_box, frozen_box=True)) + return Box({"l": annotated()}, frozen_box=True).l + + +def _frozen_through_box_copy(): + return Box(Box({"l": annotated()}), frozen_box=True).l + + +FROZEN_SOURCES = [ + pytest.param(_frozen_direct, id="BoxList(frozen_box)"), + pytest.param(lambda: copy.deepcopy(_frozen_direct()), id="deepcopy"), + pytest.param(lambda: pickle.loads(pickle.dumps(_frozen_direct())), id="pickle"), + pytest.param(lambda: copy.copy(_frozen_direct()), id="copy"), +] + +HALF_FROZEN_SOURCES = [ + pytest.param(_frozen_through_box, id="Box(frozen_box)-adopts-BoxList"), + pytest.param(_frozen_through_box_copy, id="Box(Box, frozen_box)"), +] + +HALF_FROZEN_LEAKS = {"append", "extend", "insert", "pop", "remove", "clear", "reverse", "sort"} + + +def _frozen_check(source, operation): + bl = source() + before = bl.to_list() + with pytest.raises(BoxError): + operation(bl) + assert bl.to_list() == before + check_comments(bl, ALL) + + +@pytest.mark.parametrize("source", FROZEN_SOURCES) +@pytest.mark.parametrize("operation", [pytest.param(fn, id=name) for name, fn in FROZEN_MUTATORS]) +def test_frozen_boxlist_rejects_every_mutator(source, operation): + _frozen_check(source, operation) + + +@pytest.mark.parametrize("source", HALF_FROZEN_SOURCES) +@pytest.mark.parametrize( + "operation", + [ + pytest.param( + fn, + id=name, + marks=pytest.mark.xfail( + strict=True, + reason="BUG: a BoxList adopted by a frozen Box only gets frozen_box in box_options; the " + "method overrides are installed by BoxList.__init__ only, so append/pop/sort/... still " + "mutate (box.py Box.__convert_and_store BoxList branch, box_list.py BoxList.__init__)", + ) + if name in HALF_FROZEN_LEAKS + else (), + ) + for name, fn in FROZEN_MUTATORS + ], +) +def test_boxlist_frozen_by_parent_box_rejects_every_mutator(source, operation): + _frozen_check(source, operation) + + +@pytest.mark.xfail( + strict=True, + reason="BUG: Box(src, frozen_box=True) shares src's BoxList and writes frozen_box into its options, " + "so the unfrozen source list can no longer be assigned to (box.py Box.__convert_and_store)", +) +def test_freezing_a_copy_does_not_freeze_the_source_list(): + src = Box({"l": [1, 2]}) + Box(src, frozen_box=True) + src.l[0] = 9 + src.l.set_comment(0, "still mutable") + assert src.l == [9, 2] + + +@pytest.mark.xfail( + strict=True, + reason="BUG: Box(box_with_list, frozen_box=True) adopts the BoxList without re-converting its items, " + "so the items stay unfrozen: they can be modified and hash() of the frozen Box raises " + "(box.py Box.__convert_and_store BoxList branch)", +) +def test_freezing_a_box_freezes_list_items(): + frozen = Box(Box({"lst": [{"i": 0}]}), frozen_box=True) + assert frozen.lst[0]._box_config["frozen_box"] + with pytest.raises(BoxError): + frozen.lst[0].i = 5 + hash(frozen) + + +@pytest.mark.xfail( + strict=True, + reason="BUG: frozen_box turns lists into tuples, which cannot carry comments, so from_yaml(frozen_box=True) " + "silently drops every comment on list items and on keys nested under a list (box.py " + "Box.__convert_and_store tuple branch, comments.py apply_comments / commented_tree skip tuples)", +) +def test_frozen_from_yaml_keeps_list_comments(): + pytest.importorskip("ruamel.yaml") + text = "# doc\ntop: 1 # top-c\nlst:\n- i: 0 # i0\n- 2 # l1\n" + assert Box.from_yaml(text).to_yaml() == text + assert Box.from_yaml(text, frozen_box=True).to_yaml() == text + + +def test_frozen_boxlist_items_are_frozen_and_hash_is_stable(): + bl = frozen_annotated(box_dots=True) + assert all(item._box_config["frozen_box"] for item in bl) + with pytest.raises(BoxError): + bl[0].id = 5 + with pytest.raises(BoxError): + bl["[0].id"] = 5 + assert bl["[0].id"] == 0 + assert hash(bl) == hash(pickle.loads(pickle.dumps(bl))) == hash(copy.deepcopy(bl)) + + +# -------------------------------------------------------------------------------------------------- +# 2. box_dots consistency +# -------------------------------------------------------------------------------------------------- + + +def dots_box(**options): + return Box({"a": {"b": 1}, "l": [{"x": 1}, [10, 11]], "s": "str"}, box_dots=True, **options) + + +EXISTING_PATHS = [("a.b", 1), ("l[0].x", 1), ("l[1][0]", 10), ("l[1]", [10, 11])] +MISSING_PATHS = ["a.zz", "zz.yy", "l[0].zz", "l[9]", "l[9].x", "l[1][9]"] + + +@pytest.mark.parametrize("path, value", EXISTING_PATHS) +def test_dots_existing_path_same_through_every_accessor(path, value): + b = dots_box() + assert path in b + assert b[path] == value + assert b.get(path) == value + assert b.get(path, "default") == value + assert b.setdefault(path, "default") == value + assert b.pop(path) == value + + +@pytest.mark.parametrize("path, value", EXISTING_PATHS) +def test_dots_existing_path_writes(path, value): + for write in ( + lambda b: b.__setitem__(path, "new"), + lambda b: b.update({path: "new"}), + lambda b: b.update(**{path: "new"}), + lambda b: b.merge_update({path: "new"}), + lambda b: b.__ior__({path: "new"}), + ): + b = dots_box() + write(b) + assert b[path] == "new" + assert path not in dict.keys(b), "dotted key stored literally" + + +@pytest.mark.parametrize("path", MISSING_PATHS) +def test_dots_missing_path_reads_agree(path): + b = dots_box() + assert path not in b + assert b.get(path) is None + assert b.get(path, "default") == "default" + + +@pytest.mark.parametrize( + "path", + [ + pytest.param(p, marks=pytest.mark.xfail(strict=True, reason="BUG: see test docstring")) if "[9]" in p else p + for p in MISSING_PATHS + ], +) +def test_dots_missing_path_pop_default_agrees_with_get(path): + """ + BUG: Box.pop(key, default) only catches KeyError (box.py Box.pop, the args branch), so a box_dots path + with an out of range list index raises IndexError instead of returning the default, while get() and + `in` (which catch IndexError in __contains__) treat the same path as missing. + """ + b = dots_box() + before = b.to_dict() + assert b.pop(path, "default") == "default" + assert b.to_dict() == before + + +@pytest.mark.xfail( + strict=True, + reason="BUG: Box.__delitem__ looks the dotted parent up with self[first_item], which with " + "default_box_create_on_get stores an empty Box before the KeyError (box.py Box.__delitem__)", +) +def test_dots_failed_delete_has_no_side_effect(): + b = Box(box_dots=True, default_box=True, default_box_create_on_get=True) + with pytest.raises(KeyError): + del b["zz.yy"] + assert b.to_dict() == {} + + +def test_dots_missing_reads_have_no_side_effect_with_create_on_get(): + b = Box(box_dots=True, default_box=True, default_box_create_on_get=True) + assert "zz.yy" not in b + assert b.get("zz.yy", 1) == 1 + assert b.pop("zz.yy", 2) == 2 + assert b.to_dict() == {} + + +@pytest.mark.xfail( + strict=True, + reason="BUG: frozen_box stores lists as tuples, which box_dots list paths cannot traverse: " + "b['l[0].x'] raises a bare TypeError while 'l[0].x' in b is False (box.py Box.__getitem__)", +) +def test_dots_list_paths_in_frozen_box(): + b = dots_box(frozen_box=True) + assert b["l[0].x"] == 1 + assert "l[0].x" in b + + +def test_dots_dict_paths_in_frozen_box(): + b = dots_box(frozen_box=True) + before = b.to_dict() + assert b["a.b"] == 1 + assert "a.b" in b + for write in ( + lambda: b.__setitem__("a.b", 2), + lambda: b.update({"a.b": 2}), + lambda: b.merge_update({"a.b": 2}), + lambda: b.__delitem__("a.b"), + lambda: b.pop("a.b"), + lambda: b.setdefault("a.new", 1), + ): + with pytest.raises(BoxError): + write() + assert b.to_dict() == before + + +def test_dots_comments_do_not_resurrect(): + b = Box({"a": {"b": 1, "c": 2}, "l": [1, 2]}, box_dots=True) + b.set_comment("a.b", "ab") + del b["a.b"] + b["a.b"] = 5 + assert b.get_comment("a.b") is None + b.set_comment("a.c", "ac") + b.pop("a.c") + b.update({"a.c": 5}) + assert b.get_comment("a.c") is None + b.l.set_comment(1, "l1") + b.pop("l[1]") + b.l.append(9) + assert b.get_comment("l[1]") is None + + +def test_dots_boxlist_index_forms(): + bl = BoxList([{"a": [{"b": 1}]}, [[2]]], box_dots=True) + assert bl["[0].a[0].b"] == 1 + assert bl["[1][0][0]"] == 2 + bl["[0].a[0].b"] = 3 + bl["[1][0][0]"] = 4 + assert bl.to_list() == [{"a": [{"b": 3}]}, [[4]]] + del bl["[0].a[0]"] + del bl["[1][0][0]"] + assert bl.to_list() == [{"a": []}, [[]]] + + +@pytest.mark.xfail( + strict=True, + reason="BUG: BoxList dotted indexes are found with _list_pos_re.search, not match, and the rest of the " + "key is cut at len(match) from the start, so '[x][1]' reads / deletes item [1][1] / [0][0] instead " + "of being rejected (box_list.py BoxList.__getitem__ / __setitem__ / __delitem__)", +) +def test_dots_boxlist_rejects_garbage_index(): + bl = BoxList([["a", "b"], ["c", "d"]], box_dots=True) + with pytest.raises(BoxError): + bl["[x][1]"] + with pytest.raises(BoxError): + del bl["[x][0]"] + assert bl.to_list() == [["a", "b"], ["c", "d"]] + + +@pytest.mark.xfail( + strict=True, + reason="INCONSISTENT: the comment API accepts negative box_dots indexes (b.set_comment('l[-1]')) but " + "data access does not (b['l[-1]'] raises BoxTypeError) (box_list.py _list_pos_re vs _comment_index_re)", +) +def test_dots_negative_index_matches_comment_api(): + b = dots_box() + b.set_comment("l[-1]", "last") + assert b.get_comment("l[1]") == {"eol": "last"} + assert b["l[-1]"] == [10, 11] + + +def test_dots_configbox_case_insensitive_paths(): + c = ConfigBox({"Sect": {"Key": 1, "Other": 2}}, box_dots=True) + c.Sect.set_comment("Other", "oc") + assert c["sect.key"] == 1 + assert "SECT.KEY" in c + assert c.get("sect.key") == 1 + assert c.setdefault("sect.key", 5) == 1 + c.update({"sect.key": 2}) + assert c.Sect.to_dict() == {"key": 2, "Other": 2} + c.merge_update({"sect.key": 3}) + assert c["Sect.Key"] == 3 + assert c.pop("SECT.key") == 3 + assert c.get_comment("sect.other") == {"eol": "oc"} + del c["sect.other"] + assert c.to_dict() == {"Sect": {}} + + +# -------------------------------------------------------------------------------------------------- +# 3. copy / deepcopy / pickle round trips +# -------------------------------------------------------------------------------------------------- + +ROUND_TRIPS = [ + pytest.param(copy.copy, id="copy"), + pytest.param(copy.deepcopy, id="deepcopy"), + pytest.param(lambda x: x.copy(), id="method_copy"), +] + [pytest.param(lambda x, p=p: pickle.loads(pickle.dumps(x, protocol=p)), id=f"pickle_p{p}") for p in (0, 2, 5)] + +BOX_CLASSES = [Box, ConfigBox, SBox, DDBox] + +BOX_OPTIONS = [ + pytest.param({}, id="plain"), + pytest.param({"box_dots": True}, id="box_dots"), + pytest.param({"frozen_box": True}, id="frozen"), + pytest.param({"box_dots": True, "frozen_box": True}, id="box_dots-frozen"), + pytest.param({"default_box": True}, id="default_box"), + pytest.param({"camel_killer_box": True}, id="camel_killer"), + pytest.param({"box_dots": True, "box_dots_exclude": r"x\.y"}, id="box_dots_exclude"), +] + + +def _annotated_box(cls, options): + data = {"Top": 1, "sub": {"k": 2}, "lst": [{"i": 0}, {"i": 1}]} + build = {k: v for k, v in options.items() if k != "frozen_box"} + b = cls(data, **build) + b.set_comment("Top", "top-c", "before") + b.set_comment(None, "doc") + b.sub.set_comment("k", "k-c") + b.lst.set_comment(1, "l1-c") + b.lst[0].set_comment("i", "i0-c") + b.lst.set_comment(None, "ldoc", "end") + if options.get("frozen_box"): + b = cls(b, **options) + return b + + +def _box_state(b): + return ( + type(b), + b.to_dict(), + b.get_comment("Top"), + b.get_comment(None), + b.sub.get_comment("k"), + b.lst.get_comment(1), + b.lst[0].get_comment("i"), + b.lst.get_comment(None), + type(b.sub), + type(b.lst), + type(b.lst[0]), + {k: v for k, v in b._box_config.items() if not k.startswith("__") and k != "box_dots_exclude"}, + b._box_config["box_dots_exclude"] and b._box_config["box_dots_exclude"].pattern, + b.sub._box_config["box_dots"], + b.sub._box_config["frozen_box"], + b.lst.box_options.get("box_dots"), + b.lst[0]._box_config["box_dots"], + # lst[0] frozen_box is left out: Box(b, frozen_box=True) leaves it unfrozen and a pickle / + # deepcopy round trip then freezes it, see test_freezing_a_box_freezes_list_items + ) + + +DEEPCOPY_FROZEN_BUG = ( + "BUG: Box.__deepcopy__ builds the copy unfrozen and re-freezes only the top level; the nested Boxes are " + "re-created by __setitem__ with the unfrozen config, so they stay mutable and hash() raises " + "(box.py Box.__deepcopy__)" +) + + +@pytest.mark.xfail(strict=True, reason=DEEPCOPY_FROZEN_BUG) +def test_deepcopy_of_frozen_box_is_frozen_all_the_way_down(): + frozen = Box({"sub": {"k": 1}}, frozen_box=True) + out = copy.deepcopy(frozen) + with pytest.raises(BoxError): + out.sub.k = 2 + assert hash(out) == hash(frozen) + + +@pytest.mark.parametrize("cls", BOX_CLASSES, ids=lambda c: c.__name__) +@pytest.mark.parametrize("options", BOX_OPTIONS) +@pytest.mark.parametrize("round_trip", ROUND_TRIPS) +def test_box_round_trip(cls, options, round_trip, request): + if options.get("frozen_box") and "deepcopy" in request.node.callspec.id: + request.applymarker(pytest.mark.xfail(strict=True, reason=DEEPCOPY_FROZEN_BUG)) + b = _annotated_box(cls, options) + out = round_trip(b) + assert out is not b + assert _box_state(out) == _box_state(b) + if options.get("frozen_box"): + with pytest.raises(BoxError): + out.Top = 5 + with pytest.raises(BoxError): + out.sub.k = 5 + else: + out.set_comment("Top", "changed") + assert b.get_comment("Top") == {"before": "top-c"} + if options.get("box_dots"): + assert out["lst[0].i"] == 0 + assert out.get_comment("lst[1]") == {"eol": "l1-c"} + + +@pytest.mark.parametrize("round_trip", ROUND_TRIPS[:2] + ROUND_TRIPS[3:]) +def test_nested_boxlist_round_trip(round_trip): + bl = BoxList([[1, 2], [{"a": [3]}]], box_dots=True) + bl.set_comment(1, "one") + bl[0].set_comment(1, "inner") + bl[1][0].a.set_comment(0, "deep") + out = round_trip(bl) + assert out == bl + assert out.get_comment(1) == {"eol": "one"} + assert out.get_comment("[0][1]") == {"eol": "inner"} + assert out.get_comment("[1][0].a[0]") == {"eol": "deep"} + assert out["[1][0].a[0]"] == 3 + check_converted(out, box_dots=True) + + +@pytest.mark.parametrize("round_trip", ROUND_TRIPS[:2] + ROUND_TRIPS[3:]) +def test_self_referencing_boxlist_round_trip(round_trip): + bl = BoxList([1]) + bl.append(bl) + bl.set_comment(0, "c") + out = round_trip(bl) + assert out.get_comment(0) == {"eol": "c"} + if round_trip is not ROUND_TRIPS[0].values[0]: # copy.copy re-wraps items + assert out[1] is out + + +@pytest.mark.parametrize("round_trip", ROUND_TRIPS[:2] + ROUND_TRIPS[3:]) +def test_configbox_case_index_after_round_trip(round_trip): + c = ConfigBox({"Host": "h", "Port": 1}) + _ = c["host"] # builds the case index that is stored in the box config + c.set_comment("host", "hc") + out = round_trip(c) + out["HOST"] = "z" + assert out.to_dict() == {"Port": 1, "HOST": "z"} + assert out.get_comment("host") == {"eol": "hc"} + assert c.to_dict() == {"Host": "h", "Port": 1} From 0ef1a6af7e655e1ecf5b4580872b11e034c00c40 Mon Sep 17 00:00:00 2001 From: Anish Pentyala Date: Wed, 11 Nov 2026 16:56:13 -0800 Subject: [PATCH 14/20] Test index-protocol repetition comment consistency --- test/test/test_imul_index_comments.py | 60 +++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 test/test/test_imul_index_comments.py diff --git a/test/test/test_imul_index_comments.py b/test/test/test_imul_index_comments.py new file mode 100644 index 00000000..fddfbf7e --- /dev/null +++ b/test/test/test_imul_index_comments.py @@ -0,0 +1,60 @@ +"""In-place repetition must normalize index-protocol counts before mutation.""" + +import operator + +import pytest + +from box import BoxError, BoxList + + +class IndexOnce: + def __init__(self, value): + self.value = value + self.calls = 0 + + def __index__(self): + self.calls += 1 + if self.calls > 1: + raise AssertionError("count evaluated twice") + return self.value + + +@pytest.mark.parametrize("count", [2, 1, 0, -2]) +@pytest.mark.parametrize("comments", [True, False]) +def test_imul_index_count_normalized_once(count, comments): + values = BoxList(["a", "b"]) + if comments: + values.set_comment(0, "first") + values.set_comment(1, "second") + key = IndexOnce(count) + result = operator.imul(values, key) + assert result is values + assert list(values) == ["a", "b"] * count + assert key.calls == 1 + if comments: + for index in range(max(count, 0) * 2): + assert values.get_comment(index) == {"eol": "first" if index % 2 == 0 else "second"} + if count <= 0: + assert not values.__dict__.get("_box_comments") + + +def test_imul_invalid_index_preserves_data_and_comments(): + class InvalidIndex: + def __index__(self): + raise ValueError("bad count") + + values = BoxList(["a", "b"]) + values.set_comment(1, "second") + with pytest.raises(ValueError, match="bad count"): + operator.imul(values, InvalidIndex()) + assert list(values) == ["a", "b"] + assert values.get_comment(1) == {"eol": "second"} + + +def test_imul_frozen_refuses_before_index_conversion(): + values = BoxList(["a"], frozen_box=True) + key = IndexOnce(2) + with pytest.raises(BoxError): + operator.imul(values, key) + assert key.calls == 0 + assert list(values) == ["a"] From db1a175f52ace7b0f90057ed6b5f7beb9ff8fdf0 Mon Sep 17 00:00:00 2001 From: Chris Griffith Date: Wed, 11 Nov 2026 12:19:41 -0600 Subject: [PATCH 15/20] Fix the 8.0.1 audit findings: frozen lists, slices, dotted paths, copied comments - BoxList checks frozen_box on every change; a frozen Box gets its own frozen copy of a BoxList - deepcopy of a frozen Box keeps the nested Boxes frozen - frozen lists (tuples) carry comments and follow box_dots list paths - slice assignment / deletion keep the comments of the other items - BoxList +, *, n *, *= copy item comments - box_dots: pop() default for out of range index, no parent created by failed del, garbage BoxList index rejected, negative index accepted - audit test matrix added; ruff format Co-Authored-By: Claude Sonnet 5.5 --- CHANGES.rst | 9 ++ box/box.py | 99 +++++++++++++++++-- box/box_list.py | 90 +++++++++++++---- box/comments.py | 19 +++- test/test_audit_sync.py | 209 +++++++++++++++++++++++----------------- test/test_box_keys_8.py | 6 +- test/test_box_list.py | 2 + test/test_comments.py | 13 ++- test/test_edge_cases.py | 5 +- 9 files changed, 318 insertions(+), 134 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index bad8e4f0..03561b47 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,6 +10,15 @@ Version 8.0.1 * Fixing #356 update() storing dotted keys literally with box_dots instead of treating them as paths (thanks to rahul05ranjan) * Fixing #358 BoxList + and * returning a plain list instead of a BoxList (thanks to rahul05ranjan) +* Fixing a BoxList in a frozen Box only being half frozen: append, extend, insert, pop, remove, clear, sort and reverse still changed it, and it was shared with the Box it came from. A frozen Box now gets its own frozen copy, and a BoxList checks frozen_box on every change +* Fixing copy.deepcopy of a frozen Box leaving the nested Boxes mutable +* Fixing BoxList slice assignment and slice deletion dropping the item comments of every item, comments now stay with the items they were on +* Fixing from_yaml(frozen_box=True) dropping the comments on list items, frozen lists (tuples) now keep comments, and box_dots list paths such as "l[0].x" now work in a frozen Box +* Fixing Box.pop(path, default) raising IndexError for an out of range list index in a box_dots path, it returns the default like get() +* Fixing a failed dotted del creating the parent key with default_box_create_on_get +* Fixing BoxList box_dots indexes such as "[x][1]" being accepted and acting on the wrong item, they now raise BoxTypeError +* BoxList box_dots indexes can be negative, "[-1]", like the comment API already allowed +* BoxList + and * (and n * BoxList) keep the item comments, also those of the added BoxList, and *= repeats them * Fixing #349 BoxList item comments moving to the wrong items after a box_dots deletion, e.g. del bl["[0]"] (thanks to HocVoNgThai) * Fixing #349 BoxList.sort() calling the key function twice when the list has comments (thanks to HocVoNgThai) diff --git a/box/box.py b/box/box.py index 020ba875..e3d1283f 100644 --- a/box/box.py +++ b/box/box.py @@ -85,6 +85,76 @@ def _camel_killer(attr): return re.sub(" *_+", "_", s2.lower()) +_tuple_pos_re = re.compile(r"\[(-?\d+)\]") + + +class _BoxTuple(tuple): + """ + The tuple ``frozen_box`` stores a list as. Unlike a plain tuple it can carry comments (#184) and + follows ``box_dots`` list paths such as ``"[0].key"``. + """ + + def __getitem__(self, item, _ignore_default=False): + if isinstance(item, str) and item.startswith("["): + list_pos = _tuple_pos_re.match(item) + if list_pos is None: + raise BoxTypeError(f"Invalid list index: {item!r}") + value = tuple.__getitem__(self, int(list_pos.group(1))) + children = item[list_pos.end() :].lstrip(".") + if not children: + return value + if _ignore_default and isinstance(value, (Box, box.BoxList, _BoxTuple)): + return value.__getitem__(children, _ignore_default=True) + return value[children] + return tuple.__getitem__(self, item) + + def _comment_store(self, create: bool = True) -> Any: + store = self.__dict__.get("_box_comments") + if store is None and create: + store = self.__dict__["_box_comments"] = {} + return store + + def _comment_document(self, create: bool = True) -> Any: + document = self.__dict__.get("_box_comments_document") + if document is None and create: + document = self.__dict__["_box_comments_document"] = {} + return document + + def _comment_check_frozen(self): + raise BoxError("Box is frozen") + + def _comment_owner(self, key): + if key is None: + return self, None + if isinstance(key, int) and not isinstance(key, bool): + index = key + len(self) if key < 0 else key + if 0 <= index < len(self): + return self, index + elif isinstance(key, str) and key.startswith("["): + list_pos = _tuple_pos_re.match(key) + if list_pos is not None: + index = int(list_pos.group(1)) + index = index + len(self) if index < 0 else index + rest = key[list_pos.end() :].lstrip(".") + if 0 <= index < len(self): + if not rest: + return self, index + child = tuple.__getitem__(self, index) + if hasattr(child, "_comment_owner"): + return child._comment_owner(rest) + raise BoxKeyError(f"{key!r} is not in the frozen list, comments can only be attached to existing items") + + def get_comment(self, key: Any = None) -> dict | None: + """The comments of an item (see ``BoxList.get_comment``), ``None`` for the document comments.""" + return _comments.get_comment(self, key) + + def set_comment(self, key: Any, text: Any, position: str | None = None) -> None: + _comments.set_comment(self, key, text, position) + + def del_comment(self, key: Any = None) -> None: + _comments.del_comment(self, key) + + def _recursive_tuples(iterable, box_class, recreate_tuples=False, **kwargs): out_list = [] for i in iterable: @@ -94,7 +164,10 @@ def _recursive_tuples(iterable, box_class, recreate_tuples=False, **kwargs): out_list.append(_recursive_tuples(i, box_class, recreate_tuples, **kwargs)) else: out_list.append(i) - return tuple(out_list) + out = _BoxTuple(out_list) + if isinstance(iterable, box.BoxList): + iterable._comment_copy_to(out) + return out def _to_native_tuple(iterable): @@ -449,7 +522,7 @@ def __contains__(self, item): it = self[first_item] if isinstance(it, Box): return it.__dots_int_key(children) in it - if isinstance(it, box.BoxList): + if isinstance(it, (box.BoxList, _BoxTuple)): # children is the rest of the path starting with "[", e.g. "[0].x" try: it.__getitem__(children, _ignore_default=True) @@ -523,16 +596,18 @@ def __copy__(self): return self.copy() def __deepcopy__(self, memodict=None): - frozen = self._box_config["frozen_box"] config = self.__box_config() - config["frozen_box"] = False out = self.__class__(**config) memodict = memodict if memodict is not None else {} memodict[id(self)] = out + # Fill it like __init__ does, still "being created", so nested Boxes get the frozen config too + frozen = self._box_config["frozen_box"] + if frozen: + out._box_config["__created"] = False for k, v in self.items(): out[copy.deepcopy(k, memodict)] = copy.deepcopy(v, memodict) + out._box_config["__created"] = True self.__copy_internals(out, deep=True, memo=memodict) - out._box_config["frozen_box"] = frozen return out def __setstate__(self, state): @@ -693,7 +768,11 @@ def __convert_and_store(self, item, value): else: value = box.BoxList(value, **self.__box_config(extra_namespace=item)) elif isinstance(value, box.BoxList): - value.box_options.update(self.__box_config(extra_namespace=item)) + if self._box_config["frozen_box"]: + # A frozen Box gets its own frozen copy, the list it was given stays as it was + value = box.BoxList(value, **self.__box_config(extra_namespace=item)) + else: + value.box_options.update(self.__box_config(extra_namespace=item)) elif self._box_config["modify_tuples_box"] and isinstance(value, tuple): value = _recursive_tuples(value, recreate_tuples=True, **self.__box_config(extra_namespace=item)) super().__setitem__(item, value) @@ -721,7 +800,7 @@ def __getitem__(self, item, _ignore_default=False): if isinstance(child, Box): children = child.__dots_int_key(children) if hasattr(child, "__getitem__"): - if _ignore_default and isinstance(child, (Box, box.BoxList)): + if _ignore_default and isinstance(child, (Box, box.BoxList, _BoxTuple)): return child.__getitem__(children, _ignore_default=True) return child[children] if self.__camel_destructive() and isinstance(item, str): @@ -822,7 +901,7 @@ def _comment_owner(self, key): child = dict.__getitem__(self, first) if isinstance(child, Box): return child._comment_owner(child.__dots_int_key(children)) - if isinstance(child, box.BoxList): + if isinstance(child, (box.BoxList, _BoxTuple)): return child._comment_owner(children) raise BoxKeyError(f"{key!r} is not in the Box, comments can only be attached to existing keys") @@ -923,7 +1002,7 @@ def __delitem__(self, key): if key not in self.keys() and self.__process_dotted_key(key): first_item, children = _parse_box_dots(self, key) first_item = self.__dots_int_key(first_item) - child = self[first_item] + child = self.__getitem__(first_item, _ignore_default=True) if isinstance(child, Box): children = child.__dots_int_key(children) if hasattr(child, "__delitem__"): @@ -986,7 +1065,7 @@ def pop(self, key, *args): raise BoxError('pop() takes only one optional argument "default"') try: item = self.__getitem__(key, _ignore_default=True) - except KeyError: + except (KeyError, IndexError): return args[0] else: del self[key] diff --git a/box/box_list.py b/box/box_list.py index 62a963f0..074e72e6 100644 --- a/box/box_list.py +++ b/box/box_list.py @@ -4,6 +4,7 @@ # Copyright (c) 2017-2026 - Chris Griffith - MIT License from __future__ import annotations +import bisect import copy import copyreg import operator @@ -34,12 +35,7 @@ ) from box.exceptions import BoxError, BoxKeyError, BoxTypeError -_list_pos_re = re.compile(r"\[(\d+)\]") -_comment_index_re = re.compile(r"\[(-?\d+)\]") # the comment API also takes negative indexes - - -def _frozen_box_list_method(*args, **kwargs): - raise BoxError("BoxList is frozen") +_list_pos_re = re.compile(r"\[(-?\d+)\]") # negative indexes count from the end, like a list class BoxList(list): @@ -63,17 +59,14 @@ def __init__(self, iterable: Iterable | None = None, box_class: type[box.Box] = # 2026-10-10: Iteration does not require a truth value. if iterable is not None: for x in iterable: - self.append(x) + list.append(self, self._convert(x)) if isinstance(iterable, BoxList) and iterable is not self: iterable._comment_copy_to(self) self.box_org_ref = None - if box_options.get("frozen_box"): - for method in ["append", "clear", "extend", "insert", "pop", "remove", "reverse", "sort"]: - self.__setattr__(method, _frozen_box_list_method) def __getitem__(self, item, _ignore_default=False): if self.box_options.get("box_dots") and isinstance(item, str) and item.startswith("["): - list_pos = _list_pos_re.search(item) + list_pos = _list_pos_re.match(item) if list_pos is None: raise BoxTypeError(f"Invalid list index: {item!r}") value = super().__getitem__(int(list_pos.groups()[0])) @@ -97,7 +90,7 @@ def __delitem__(self, key): if self.box_options.get("frozen_box"): raise BoxError("BoxList is frozen") if self.box_options.get("box_dots") and isinstance(key, str) and key.startswith("["): - list_pos = _list_pos_re.search(key) + list_pos = _list_pos_re.match(key) if list_pos is None: raise BoxTypeError(f"Invalid list index: {key!r}") pos = int(list_pos.groups()[0]) @@ -115,7 +108,7 @@ def __setitem__(self, key, value): if self.box_options.get("frozen_box"): raise BoxError("BoxList is frozen") if self.box_options.get("box_dots") and isinstance(key, str) and key.startswith("["): - list_pos = _list_pos_re.search(key) + list_pos = _list_pos_re.match(key) if list_pos is None: raise BoxTypeError(f"Invalid list index: {key!r}") pos = int(list_pos.groups()[0]) @@ -142,23 +135,33 @@ def converted_items(): for item in value: yield self._convert(item) + length = len(self) # Let list validate the slice before consuming replacement values. super().__setitem__(key, converted_items()) - self.__dict__.pop("_box_comments", None) # items moved, their comments can't follow + self._comment_after_slice_set(key, length) return super().__setitem__(key, self._convert(value)) def __add__(self, other): if not isinstance(other, list): raise BoxTypeError("BoxList can only be added to another list.") - return box.BoxList(list(self) + list(other), **self.box_options) + out = box.BoxList(list(self) + list(other), **self.box_options) + self._comment_copy_to(out) + if isinstance(other, BoxList): + other._comment_copy_to(out, offset=len(self), document=False) + return out def __mul__(self, count): try: count = operator.index(count) except TypeError: raise BoxTypeError("BoxList can only be multiplied by an integer.") from None - return box.BoxList(list(self) * count, **self.box_options) + out = box.BoxList(list(self) * count, **self.box_options) + self._comment_repeat(out, count, len(self)) + document = self.__dict__.get("_box_comments_document") + if document: + out._comment_document().update(copy.deepcopy(document)) + return out __rmul__ = __mul__ @@ -171,9 +174,12 @@ def __iadd__(self, value): # type: ignore[misc] # list accepts any iterable fo def __imul__(self, value): # type: ignore[misc] if self.box_options.get("frozen_box"): raise BoxError("BoxList is frozen") + length = len(self) result = super().__imul__(value) if len(self) == 0: self.__dict__.pop("_box_comments", None) + else: + self._comment_repeat(self, value, length, first=1) return result def _is_intact_type(self, obj): @@ -197,15 +203,18 @@ def _convert(self, p_object): return p_object def append(self, p_object): + self._comment_check_frozen() super().append(self._convert(p_object)) def extend(self, iterable): + self._comment_check_frozen() if iterable is self: iterable = list(self) for item in iterable: self.append(item) def insert(self, index, p_object): + self._comment_check_frozen() index = operator.index(index) length = len(self) super().insert(index, self._convert(p_object)) @@ -214,6 +223,7 @@ def insert(self, index, p_object): self._comment_reindex(lambda k: k + 1 if k >= at else k) def pop(self, index=-1): + self._comment_check_frozen() index = operator.index(index) length = len(self) value = super().pop(index) @@ -224,15 +234,18 @@ def remove(self, value): self.pop(self.index(value)) def clear(self): + self._comment_check_frozen() super().clear() self.__dict__.pop("_box_comments", None) def reverse(self): + self._comment_check_frozen() super().reverse() last = len(self) - 1 self._comment_reindex(lambda k: last - k) def sort(self, *, key=None, reverse=False): + self._comment_check_frozen() if not self.__dict__.get("_box_comments"): return super().sort(key=key, reverse=reverse) # Work out where every item goes (same stable order as list.sort) so comments follow them @@ -274,11 +287,44 @@ def _comment_after_delete(self, key, length): if not self.__dict__.get("_box_comments"): return if isinstance(key, slice): - self.__dict__.pop("_box_comments", None) + removed = sorted(range(*key.indices(length))) + if removed: + gone_set = set(removed) + self._comment_reindex(lambda k: None if k in gone_set else k - bisect.bisect_left(removed, k)) return gone = key + length if key < 0 else key self._comment_reindex(lambda k: None if k == gone else (k - 1 if k > gone else k)) + def _comment_after_slice_set(self, key, length): + """Move the item comments after ``self[key] = values``, `length` is the size before it.""" + if not self.__dict__.get("_box_comments"): + return + start, stop, step = key.indices(length) + if step != 1: + # An extended slice replaces items one for one, the replacements start without a comment + replaced = set(range(start, stop, step)) + self._comment_reindex(lambda k: None if k in replaced else k) + return + stop = max(start, stop) + shift = len(self) - length + + def mapping(k): + if k < start: + return k + return k + shift if k >= stop else None # replaced items are new, they start without a comment + + self._comment_reindex(mapping) + + def _comment_repeat(self, out, count, length, first=0): + """Copy the comments of the first `length` items to repetitions `first`..`count` of them in `out`.""" + store = self.__dict__.get("_box_comments") + if store: + target = out._comment_store() + for repeat in range(first, max(count, 0)): + for index, comment in list(store.items()): + if index < length: + target[index + repeat * length] = copy.deepcopy(comment) + def _comment_owner(self, key): """(container, index) that holds the comment for `key`, following box_dots paths.""" if key is None: @@ -288,7 +334,7 @@ def _comment_owner(self, key): if 0 <= index < len(self): return self, index elif isinstance(key, str) and key.startswith("["): - list_pos = _comment_index_re.match(key) + list_pos = _list_pos_re.match(key) if list_pos is not None: index = int(list_pos.groups()[0]) index = index + len(self) if index < 0 else index @@ -301,11 +347,13 @@ def _comment_owner(self, key): return child._comment_owner(rest) raise BoxKeyError(f"{key!r} is not in the BoxList, comments can only be attached to existing items") - def _comment_copy_to(self, other): + def _comment_copy_to(self, other, offset=0, document=True): store = self.__dict__.get("_box_comments") if store: - other._comment_store().update({k: copy.deepcopy(v) for k, v in store.items() if k < len(other)}) - document = self.__dict__.get("_box_comments_document") + other._comment_store().update( + {k + offset: copy.deepcopy(v) for k, v in store.items() if k + offset < len(other)} + ) + document = self.__dict__.get("_box_comments_document") if document else None if document: other._comment_document().update(copy.deepcopy(document)) diff --git a/box/comments.py b/box/comments.py index 8f2cf632..c1485ba6 100644 --- a/box/comments.py +++ b/box/comments.py @@ -197,7 +197,7 @@ def walk(node, path): def _resolve_key(container, key): """The key in `container` the YAML key ended up as, or a sentinel when it isn't there.""" - if isinstance(container, list): + if isinstance(container, (list, tuple)): return key if isinstance(key, int) and 0 <= key < len(container) else _MISSING if dict.__contains__(container, key): return key @@ -222,7 +222,12 @@ def apply_comments(root, model: dict | None) -> None: if found is _MISSING: node = None break - node = list.__getitem__(node, found) if isinstance(node, list) else dict.__getitem__(node, found) + if isinstance(node, list): + node = list.__getitem__(node, found) + elif isinstance(node, tuple): + node = tuple.__getitem__(node, found) + else: + node = dict.__getitem__(node, found) if not hasattr(node, "_comment_store"): node = None break @@ -251,7 +256,7 @@ def has_comments(obj, _seen=None) -> bool: if id(obj) in seen: return False seen.add(id(obj)) - values = obj.values() if isinstance(obj, dict) else obj + values = obj.values() if isinstance(obj, dict) else obj # a list or a frozen list (tuple) return any(has_comments(value, seen) for value in values) @@ -286,6 +291,8 @@ def commented_tree(obj, plain, column=0, _seen=None): from ruamel.yaml.comments import CommentedMap, CommentedSeq seen = _seen if _seen is not None else set() + if isinstance(plain, tuple) and hasattr(obj, "_comment_store"): + plain = list(plain) # a frozen list: a tuple that can carry comments if isinstance(plain, (dict, list)): if id(plain) in seen: return plain # self references are written as they are @@ -299,7 +306,11 @@ def commented_tree(obj, plain, column=0, _seen=None): elif isinstance(plain, list): out = CommentedSeq() for index, value in enumerate(plain): - child = list.__getitem__(obj, index) if isinstance(obj, list) and index < len(obj) else None + child = ( + (list if isinstance(obj, list) else tuple).__getitem__(obj, index) + if isinstance(obj, (list, tuple)) and index < len(obj) + else None + ) out.append(commented_tree(child, value, column + 2, seen)) keys = list(range(len(out))) else: diff --git a/test/test_audit_sync.py b/test/test_audit_sync.py index 94e7f52f..d9fb224b 100644 --- a/test/test_audit_sync.py +++ b/test/test_audit_sync.py @@ -8,8 +8,7 @@ 2. box_dots paths must behave the same through every access method. 3. copy / deepcopy / pickle round trips of every Box flavour, with comments. -Tests marked ``xfail(strict=True)`` reproduce a known bug: run with ``--runxfail`` to see them fail. -Once a fix lands, the strict xfail turns into a failure, so the marker must be removed with the fix. +The matrix was written to reproduce the bugs found in the 8.0.1 audit; they are fixed, these tests keep them so. """ import copy @@ -18,6 +17,7 @@ import pytest from box import Box, BoxError, BoxList, ConfigBox, SBox +from box.exceptions import BoxKeyError, BoxTypeError from box.shorthand_box import DDBox # -------------------------------------------------------------------------------------------------- @@ -157,11 +157,6 @@ def test_mutator_keeps_comments_and_converts(options, operation, kept): @pytest.mark.parametrize("options", OPTION_SETS) @pytest.mark.parametrize("operation, kept", _ids(SLICE_OPS)) -@pytest.mark.xfail( - strict=True, - reason="BUG: slice assignment / deletion drops the comments of every item, also the ones the slice " - "does not touch (box_list.py BoxList.__setitem__ slice branch and _comment_after_delete)", -) def test_slice_mutator_keeps_comments_of_untouched_items(options, operation, kept): bl = annotated(**options) operation(bl) @@ -251,11 +246,6 @@ def test_binary_operators_convert(operation): pytest.param(lambda bl: 2 * bl, id="rmul"), ], ) -@pytest.mark.xfail( - strict=True, - reason="INCONSISTENT: + and * drop every item comment while copy(), BoxList(bl), += and *= (n>0) keep " - "them on the original items (box_list.py BoxList.__add__ / __mul__)", -) def test_binary_operators_keep_comments_of_left_operand(operation): out = operation(annotated()) check_comments(out, ALL) @@ -309,8 +299,6 @@ def _frozen_through_box_copy(): pytest.param(_frozen_through_box_copy, id="Box(Box, frozen_box)"), ] -HALF_FROZEN_LEAKS = {"append", "extend", "insert", "pop", "remove", "clear", "reverse", "sort"} - def _frozen_check(source, operation): bl = source() @@ -330,31 +318,12 @@ def test_frozen_boxlist_rejects_every_mutator(source, operation): @pytest.mark.parametrize("source", HALF_FROZEN_SOURCES) @pytest.mark.parametrize( "operation", - [ - pytest.param( - fn, - id=name, - marks=pytest.mark.xfail( - strict=True, - reason="BUG: a BoxList adopted by a frozen Box only gets frozen_box in box_options; the " - "method overrides are installed by BoxList.__init__ only, so append/pop/sort/... still " - "mutate (box.py Box.__convert_and_store BoxList branch, box_list.py BoxList.__init__)", - ) - if name in HALF_FROZEN_LEAKS - else (), - ) - for name, fn in FROZEN_MUTATORS - ], + [pytest.param(fn, id=name) for name, fn in FROZEN_MUTATORS], ) def test_boxlist_frozen_by_parent_box_rejects_every_mutator(source, operation): _frozen_check(source, operation) -@pytest.mark.xfail( - strict=True, - reason="BUG: Box(src, frozen_box=True) shares src's BoxList and writes frozen_box into its options, " - "so the unfrozen source list can no longer be assigned to (box.py Box.__convert_and_store)", -) def test_freezing_a_copy_does_not_freeze_the_source_list(): src = Box({"l": [1, 2]}) Box(src, frozen_box=True) @@ -363,12 +332,6 @@ def test_freezing_a_copy_does_not_freeze_the_source_list(): assert src.l == [9, 2] -@pytest.mark.xfail( - strict=True, - reason="BUG: Box(box_with_list, frozen_box=True) adopts the BoxList without re-converting its items, " - "so the items stay unfrozen: they can be modified and hash() of the frozen Box raises " - "(box.py Box.__convert_and_store BoxList branch)", -) def test_freezing_a_box_freezes_list_items(): frozen = Box(Box({"lst": [{"i": 0}]}), frozen_box=True) assert frozen.lst[0]._box_config["frozen_box"] @@ -377,12 +340,6 @@ def test_freezing_a_box_freezes_list_items(): hash(frozen) -@pytest.mark.xfail( - strict=True, - reason="BUG: frozen_box turns lists into tuples, which cannot carry comments, so from_yaml(frozen_box=True) " - "silently drops every comment on list items and on keys nested under a list (box.py " - "Box.__convert_and_store tuple branch, comments.py apply_comments / commented_tree skip tuples)", -) def test_frozen_from_yaml_keeps_list_comments(): pytest.importorskip("ruamel.yaml") text = "# doc\ntop: 1 # top-c\nlst:\n- i: 0 # i0\n- 2 # l1\n" @@ -450,28 +407,16 @@ def test_dots_missing_path_reads_agree(path): @pytest.mark.parametrize( "path", - [ - pytest.param(p, marks=pytest.mark.xfail(strict=True, reason="BUG: see test docstring")) if "[9]" in p else p - for p in MISSING_PATHS - ], + MISSING_PATHS, ) def test_dots_missing_path_pop_default_agrees_with_get(path): - """ - BUG: Box.pop(key, default) only catches KeyError (box.py Box.pop, the args branch), so a box_dots path - with an out of range list index raises IndexError instead of returning the default, while get() and - `in` (which catch IndexError in __contains__) treat the same path as missing. - """ + """Box.pop(key, default) treats an out of range list index like get() and `in` do: as missing.""" b = dots_box() before = b.to_dict() assert b.pop(path, "default") == "default" assert b.to_dict() == before -@pytest.mark.xfail( - strict=True, - reason="BUG: Box.__delitem__ looks the dotted parent up with self[first_item], which with " - "default_box_create_on_get stores an empty Box before the KeyError (box.py Box.__delitem__)", -) def test_dots_failed_delete_has_no_side_effect(): b = Box(box_dots=True, default_box=True, default_box_create_on_get=True) with pytest.raises(KeyError): @@ -487,11 +432,6 @@ def test_dots_missing_reads_have_no_side_effect_with_create_on_get(): assert b.to_dict() == {} -@pytest.mark.xfail( - strict=True, - reason="BUG: frozen_box stores lists as tuples, which box_dots list paths cannot traverse: " - "b['l[0].x'] raises a bare TypeError while 'l[0].x' in b is False (box.py Box.__getitem__)", -) def test_dots_list_paths_in_frozen_box(): b = dots_box(frozen_box=True) assert b["l[0].x"] == 1 @@ -544,12 +484,6 @@ def test_dots_boxlist_index_forms(): assert bl.to_list() == [{"a": []}, [[]]] -@pytest.mark.xfail( - strict=True, - reason="BUG: BoxList dotted indexes are found with _list_pos_re.search, not match, and the rest of the " - "key is cut at len(match) from the start, so '[x][1]' reads / deletes item [1][1] / [0][0] instead " - "of being rejected (box_list.py BoxList.__getitem__ / __setitem__ / __delitem__)", -) def test_dots_boxlist_rejects_garbage_index(): bl = BoxList([["a", "b"], ["c", "d"]], box_dots=True) with pytest.raises(BoxError): @@ -559,11 +493,6 @@ def test_dots_boxlist_rejects_garbage_index(): assert bl.to_list() == [["a", "b"], ["c", "d"]] -@pytest.mark.xfail( - strict=True, - reason="INCONSISTENT: the comment API accepts negative box_dots indexes (b.set_comment('l[-1]')) but " - "data access does not (b['l[-1]'] raises BoxTypeError) (box_list.py _list_pos_re vs _comment_index_re)", -) def test_dots_negative_index_matches_comment_api(): b = dots_box() b.set_comment("l[-1]", "last") @@ -645,19 +574,9 @@ def _box_state(b): b.sub._box_config["frozen_box"], b.lst.box_options.get("box_dots"), b.lst[0]._box_config["box_dots"], - # lst[0] frozen_box is left out: Box(b, frozen_box=True) leaves it unfrozen and a pickle / - # deepcopy round trip then freezes it, see test_freezing_a_box_freezes_list_items ) -DEEPCOPY_FROZEN_BUG = ( - "BUG: Box.__deepcopy__ builds the copy unfrozen and re-freezes only the top level; the nested Boxes are " - "re-created by __setitem__ with the unfrozen config, so they stay mutable and hash() raises " - "(box.py Box.__deepcopy__)" -) - - -@pytest.mark.xfail(strict=True, reason=DEEPCOPY_FROZEN_BUG) def test_deepcopy_of_frozen_box_is_frozen_all_the_way_down(): frozen = Box({"sub": {"k": 1}}, frozen_box=True) out = copy.deepcopy(frozen) @@ -669,9 +588,7 @@ def test_deepcopy_of_frozen_box_is_frozen_all_the_way_down(): @pytest.mark.parametrize("cls", BOX_CLASSES, ids=lambda c: c.__name__) @pytest.mark.parametrize("options", BOX_OPTIONS) @pytest.mark.parametrize("round_trip", ROUND_TRIPS) -def test_box_round_trip(cls, options, round_trip, request): - if options.get("frozen_box") and "deepcopy" in request.node.callspec.id: - request.applymarker(pytest.mark.xfail(strict=True, reason=DEEPCOPY_FROZEN_BUG)) +def test_box_round_trip(cls, options, round_trip): b = _annotated_box(cls, options) out = round_trip(b) assert out is not b @@ -725,3 +642,117 @@ def test_configbox_case_index_after_round_trip(round_trip): assert out.to_dict() == {"Port": 1, "HOST": "z"} assert out.get_comment("host") == {"eol": "hc"} assert c.to_dict() == {"Host": "h", "Port": 1} + + +# ---- fixes of the 8.0.1 audit: frozen lists (tuples), operators ---- + + +def _frozen_yaml_box(): + pytest.importorskip("ruamel.yaml") + return Box.from_yaml("l:\n- a: 1 # a\n- - 5 # deep\n- 3 # three\n", frozen_box=True) + + +def test_frozen_list_tuple_comment_api(): + bx = _frozen_yaml_box() + assert type(bx.l) is not tuple and isinstance(bx.l, tuple) + assert bx.l.get_comment(2) == {"eol": "three"} + assert bx.l.get_comment("[-1]") == {"eol": "three"} + assert bx.l.get_comment("[0].a") == {"eol": "a"} + assert bx.l.get_comment("[1][0]") == {"eol": "deep"} + dots = Box.from_yaml("l:\n- - 5 # deep\n", frozen_box=True, box_dots=True) + assert dots.get_comment("l[0][0]") == {"eol": "deep"} + assert bx.l.get_comment(None) is None + for call in (lambda: bx.l.set_comment(0, "x"), lambda: bx.l.del_comment(0)): + with pytest.raises(BoxError): + call() + for bad in (5, "[9]", "x", "[x]", "[2].nope"): + with pytest.raises(BoxKeyError): + bx.l.get_comment(bad) + bx.l._comment_document()["start"] = ["doc"] + assert bx.l.get_comment() == {"start": "doc"} + + +def test_frozen_list_tuple_dotted_reads(): + bx = Box({"l": [{"a": 1}, [7, {"b": 2}]]}, box_dots=True, frozen_box=True) + assert bx["l[0].a"] == 1 + assert bx["l[-1][1].b"] == 2 + assert bx.l["[1]"] == (7, {"b": 2}) + assert bx.l["[0]"].a == 1 + with pytest.raises(BoxTypeError): + bx.l["[x]"] + assert bx.l[0].a == 1 + assert "l[1][1].b" in bx + assert "l[1][1].zz" not in bx + assert bx.get("l[9].a", "d") == "d" + + +def test_frozen_list_tuple_copies_keep_comments(): + bx = _frozen_yaml_box() + for out in (copy.deepcopy(bx), pickle.loads(pickle.dumps(bx)), copy.copy(bx)): + assert out.l.get_comment(2) == {"eol": "three"} + assert out.to_yaml() == bx.to_yaml() + + +def test_frozen_list_comments_follow_box_list_adoption(): + bl = BoxList([{"a": 1}, 2]) + bl.set_comment(1, "two") + bx = Box({"l": bl}, frozen_box=True) + assert bx.l.get_comment(1) == {"eol": "two"} + assert bl.get_comment(1) == {"eol": "two"} # the source list is left as it was + + +def test_dots_negative_index_set_and_delete(): + bl = BoxList([1, 2, 3], box_dots=True) + bl["[-1]"] = 9 + del bl["[-2]"] + assert bl == [1, 9] + + +@pytest.mark.parametrize("repeat", [2, 3]) +def test_repeat_copies_comments_to_every_repetition(repeat): + bl = annotated() + original = len(bl) + for out in (bl * repeat, repeat * bl): + assert len(out) == original * repeat + for index in range(len(out)): + assert out.get_comment(index) == {"eol": f"c{index % original}"} + bl *= repeat + for index in range(len(bl)): + assert bl.get_comment(index) == {"eol": f"c{index % original}"} + + +def test_add_copies_comments_of_both_lists_and_document(): + left = annotated() + left.set_comment(None, "doc") + right = annotated() + out = left + right + assert out.get_comment() == {"start": "doc"} + for index in range(len(out)): + assert out.get_comment(index) == {"eol": f"c{index % len(left)}"} + assert (left + [{"id": NEW}]).get_comment(len(left)) is None + + +def test_mul_copies_document_comments(): + bl = annotated() + bl.set_comment(None, "doc") + assert (bl * 2).get_comment() == {"start": "doc"} + assert (bl * 0).to_list() == [] + + +def test_frozen_box_converts_a_box_list_inside_a_plain_list(): + bl = BoxList([1]) + bl.set_comment(0, "one") + bx = Box({"l": [bl]}, frozen_box=True) + assert bx.l[0].get_comment(0) == {"eol": "one"} + + +def test_empty_slice_delete_keeps_comments(): + bl = annotated() + del bl[1:1] + check_comments(bl, ALL) + + +def test_from_yaml_comments_inside_an_intact_dict_are_skipped(): + pytest.importorskip("ruamel.yaml") + bx = Box.from_yaml("a:\n b: 1 # c\nz: 2 # z\n", box_intact_types=(dict,)) + assert bx.get_comment("z") == {"eol": "z"} diff --git a/test/test_box_keys_8.py b/test/test_box_keys_8.py index 0b39778f..fea8ab81 100644 --- a/test/test_box_keys_8.py +++ b/test/test_box_keys_8.py @@ -120,7 +120,7 @@ def test_config_box(self): class TestUpdateDottedPaths: - """ update() honors box_dots path semantics, like __setitem__ and merge_update. """ + """update() honors box_dots path semantics, like __setitem__ and merge_update.""" def test_update_dotted_key_nests(self): b = Box(default_box=True, box_dots=True) @@ -148,7 +148,5 @@ def test_update_non_dotted_unchanged(self): assert b.to_dict() == {"a": 1, "b": 2, "c": 3} def test_box_addition_merges_dotted_paths(self): - b = Box({"x": {"y": 1}}, default_box=True, box_dots=True) + Box( - {"x.z": 2}, default_box=True, box_dots=True - ) + b = Box({"x": {"y": 1}}, default_box=True, box_dots=True) + Box({"x.z": 2}, default_box=True, box_dots=True) assert b.to_dict() == {"x": {"y": 1, "z": 2}} diff --git a/test/test_box_list.py b/test/test_box_list.py index 6faabe2a..d13a8370 100644 --- a/test/test_box_list.py +++ b/test/test_box_list.py @@ -763,6 +763,8 @@ def test_out_of_bounds_index_preserves_comments(operation, index): assert values == ["a", "b"] assert values.get_comment(0) == {"eol": "a"} assert values.get_comment(1) == {"eol": "b"} + + class TestBoxListOperators: def test_add_keeps_boxlist_and_converts(self): bl = BoxList([{"a": 1}], box_dots=True) diff --git a/test/test_comments.py b/test/test_comments.py index 51be0aff..feb1d067 100644 --- a/test/test_comments.py +++ b/test/test_comments.py @@ -382,11 +382,13 @@ def test_sort_with_comments_evaluates_keys_once(self): assert ranks == {} assert values.to_yaml() == "- a # A\n- z # Z\n" - def test_slice_delete_drops_item_comments(self): + def test_slice_delete_drops_removed_comments_and_moves_the_rest(self): bl = BoxList([1, 2, 3]) + bl.set_comment(0, "gone") bl.set_comment(2, "x") del bl[0:1] - assert bl.get_comment(1) is None + assert bl.get_comment(0) is None + assert bl.get_comment(1) == {"eol": "x"} def test_empty_comment_and_missing_final_newline(self): bx = Box.from_yaml("a: 1 #\nb: 2 # two\n# end") @@ -394,9 +396,12 @@ def test_empty_comment_and_missing_final_newline(self): assert bx.get_comment("b") == {"eol": "two"} assert bx.get_comment(None) == {"end": "end"} - def test_frozen_lists_skip_item_comments(self): - bx = Box.from_yaml("l:\n- a: 1 # in a frozen list\n", frozen_box=True) + def test_frozen_lists_keep_item_comments(self): + text = "l:\n- a: 1 # in a frozen list\n" + bx = Box.from_yaml(text, frozen_box=True) assert bx.to_dict() == {"l": ({"a": 1},)} # frozen boxes store lists as tuples + assert bx.l[0].get_comment("a") == {"eol": "in a frozen list"} + assert bx.to_yaml() == text def test_box_dots_split_parent_key(self): bx = Box.from_yaml("a.b:\n c: 1 # c\n", box_dots=True, default_box=True) diff --git a/test/test_edge_cases.py b/test/test_edge_cases.py index c414b193..67f771a7 100644 --- a/test/test_edge_cases.py +++ b/test/test_edge_cases.py @@ -258,8 +258,9 @@ def test_negative_and_out_of_range_list_indexes(self): assert bx["a[2]"] == 3 with pytest.raises(IndexError): bx["a[10]"] - with pytest.raises(BoxTypeError): - bx["a[-1]"] + assert bx["a[-1]"] == 3 # counts from the end, like the comment API and a list + with pytest.raises(IndexError): + bx["a[-10]"] with pytest.raises(BoxTypeError): bx["a[x]"] From 6558b11efe895cbd4b537030d5a3b78c404dedbb Mon Sep 17 00:00:00 2001 From: Anish Pentyala Date: Wed, 11 Nov 2026 16:56:40 -0800 Subject: [PATCH 16/20] Place repetition regression tests in the existing test directory --- test/{test => }/test_imul_index_comments.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{test => }/test_imul_index_comments.py (100%) diff --git a/test/test/test_imul_index_comments.py b/test/test_imul_index_comments.py similarity index 100% rename from test/test/test_imul_index_comments.py rename to test/test_imul_index_comments.py From d14b81f3bd837bed851f4bf6591603d31fba8050 Mon Sep 17 00:00:00 2001 From: Chris Griffith Date: Wed, 11 Nov 2026 12:21:00 -0600 Subject: [PATCH 17/20] Revert negative dotted index data access; keep it a comment API convenience test_negative_and_out_of_range_list_indexes asserted that a[-1] raises, which is intended behaviour, not a bug. Co-Authored-By: Claude Sonnet 5.5 --- CHANGES.rst | 1 - box/box.py | 5 +++-- box/box_list.py | 5 +++-- test/test_audit_sync.py | 15 +++++---------- test/test_edge_cases.py | 5 ++--- 5 files changed, 13 insertions(+), 18 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 03561b47..29f807be 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,7 +17,6 @@ Version 8.0.1 * Fixing Box.pop(path, default) raising IndexError for an out of range list index in a box_dots path, it returns the default like get() * Fixing a failed dotted del creating the parent key with default_box_create_on_get * Fixing BoxList box_dots indexes such as "[x][1]" being accepted and acting on the wrong item, they now raise BoxTypeError -* BoxList box_dots indexes can be negative, "[-1]", like the comment API already allowed * BoxList + and * (and n * BoxList) keep the item comments, also those of the added BoxList, and *= repeats them * Fixing #349 BoxList item comments moving to the wrong items after a box_dots deletion, e.g. del bl["[0]"] (thanks to HocVoNgThai) * Fixing #349 BoxList.sort() calling the key function twice when the list has comments (thanks to HocVoNgThai) diff --git a/box/box.py b/box/box.py index e3d1283f..edd7fbe1 100644 --- a/box/box.py +++ b/box/box.py @@ -85,7 +85,8 @@ def _camel_killer(attr): return re.sub(" *_+", "_", s2.lower()) -_tuple_pos_re = re.compile(r"\[(-?\d+)\]") +_tuple_pos_re = re.compile(r"\[(\d+)\]") +_tuple_comment_index_re = re.compile(r"\[(-?\d+)\]") class _BoxTuple(tuple): @@ -131,7 +132,7 @@ def _comment_owner(self, key): if 0 <= index < len(self): return self, index elif isinstance(key, str) and key.startswith("["): - list_pos = _tuple_pos_re.match(key) + list_pos = _tuple_comment_index_re.match(key) if list_pos is not None: index = int(list_pos.group(1)) index = index + len(self) if index < 0 else index diff --git a/box/box_list.py b/box/box_list.py index 074e72e6..c4ba9464 100644 --- a/box/box_list.py +++ b/box/box_list.py @@ -35,7 +35,8 @@ ) from box.exceptions import BoxError, BoxKeyError, BoxTypeError -_list_pos_re = re.compile(r"\[(-?\d+)\]") # negative indexes count from the end, like a list +_list_pos_re = re.compile(r"\[(\d+)\]") +_comment_index_re = re.compile(r"\[(-?\d+)\]") # the comment API also takes negative indexes class BoxList(list): @@ -334,7 +335,7 @@ def _comment_owner(self, key): if 0 <= index < len(self): return self, index elif isinstance(key, str) and key.startswith("["): - list_pos = _list_pos_re.match(key) + list_pos = _comment_index_re.match(key) if list_pos is not None: index = int(list_pos.groups()[0]) index = index + len(self) if index < 0 else index diff --git a/test/test_audit_sync.py b/test/test_audit_sync.py index d9fb224b..e003d9dd 100644 --- a/test/test_audit_sync.py +++ b/test/test_audit_sync.py @@ -493,11 +493,13 @@ def test_dots_boxlist_rejects_garbage_index(): assert bl.to_list() == [["a", "b"], ["c", "d"]] -def test_dots_negative_index_matches_comment_api(): +def test_dots_negative_index_is_comment_api_only(): + """Negative dotted indexes are a comment API convenience, data access rejects them (by design).""" b = dots_box() b.set_comment("l[-1]", "last") assert b.get_comment("l[1]") == {"eol": "last"} - assert b["l[-1]"] == [10, 11] + with pytest.raises(BoxTypeError): + b["l[-1]"] def test_dots_configbox_case_insensitive_paths(): @@ -675,7 +677,7 @@ def test_frozen_list_tuple_comment_api(): def test_frozen_list_tuple_dotted_reads(): bx = Box({"l": [{"a": 1}, [7, {"b": 2}]]}, box_dots=True, frozen_box=True) assert bx["l[0].a"] == 1 - assert bx["l[-1][1].b"] == 2 + assert bx["l[1][1].b"] == 2 assert bx.l["[1]"] == (7, {"b": 2}) assert bx.l["[0]"].a == 1 with pytest.raises(BoxTypeError): @@ -701,13 +703,6 @@ def test_frozen_list_comments_follow_box_list_adoption(): assert bl.get_comment(1) == {"eol": "two"} # the source list is left as it was -def test_dots_negative_index_set_and_delete(): - bl = BoxList([1, 2, 3], box_dots=True) - bl["[-1]"] = 9 - del bl["[-2]"] - assert bl == [1, 9] - - @pytest.mark.parametrize("repeat", [2, 3]) def test_repeat_copies_comments_to_every_repetition(repeat): bl = annotated() diff --git a/test/test_edge_cases.py b/test/test_edge_cases.py index 67f771a7..c414b193 100644 --- a/test/test_edge_cases.py +++ b/test/test_edge_cases.py @@ -258,9 +258,8 @@ def test_negative_and_out_of_range_list_indexes(self): assert bx["a[2]"] == 3 with pytest.raises(IndexError): bx["a[10]"] - assert bx["a[-1]"] == 3 # counts from the end, like the comment API and a list - with pytest.raises(IndexError): - bx["a[-10]"] + with pytest.raises(BoxTypeError): + bx["a[-1]"] with pytest.raises(BoxTypeError): bx["a[x]"] From c786e25339af6068f451460142540dc18d7b6d9b Mon Sep 17 00:00:00 2001 From: peterolkhov Date: Wed, 11 Nov 2026 15:45:09 -0500 Subject: [PATCH 18/20] Apply configured insertion rules in Box.update --- AUTHORS.rst | 1 + CHANGES.rst | 2 + box/box.py | 8 +--- test/test_box.py | 3 +- test/test_update_insertion.py | 88 +++++++++++++++++++++++++++++++++++ 5 files changed, 94 insertions(+), 8 deletions(-) create mode 100644 test/test_update_insertion.py diff --git a/AUTHORS.rst b/AUTHORS.rst index 43c5149d..f7eaa97a 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -127,3 +127,4 @@ Suggestions and bug reporting: - HocVoNgThai - Mrunmayee Daware (Mrun25) +- Peter Olkhovets (peterolkhov) diff --git a/CHANGES.rst b/CHANGES.rst index 29f807be..3f80f893 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,6 +3,8 @@ Changelog Version 8.0.1 ------------- +* Apply configured value recasting, key conversion and duplicate checks to update and union insertions + * Clear removed BoxList item comments after in-place repetition by zero or a negative count, so new items do not inherit stale YAML comments (thanks to Mrunmayee Daware) * Fix ConfigBox.merge_update consuming one-shot key/value pairs before merging them diff --git a/box/box.py b/box/box.py index edd7fbe1..f2b539b2 100644 --- a/box/box.py +++ b/box/box.py @@ -1146,13 +1146,7 @@ def update(self, *args, **kwargs): self._update_store(k, kwargs[k]) def _update_store(self, key, value): - # In box_dots mode a dotted key is a path, the same as in - # __setitem__ and merge_update; storing it literally would create - # a parallel key that dots lookups cannot reach consistently. - if self.__process_dotted_key(key): - self.__setitem__(key, value) - else: - self.__convert_and_store(key, value) + self.__setitem__(key, value) def merge_update(self, *args, **kwargs): """ diff --git a/test/test_box.py b/test/test_box.py index 143f4b0c..31315f6f 100644 --- a/test/test_box.py +++ b/test/test_box.py @@ -730,7 +730,8 @@ def test_camel_killer_box_dotted_lookup_disabled(self, camel_killer_box, box_dot def test_camel_killer_box_dotted_lookup_exact_key(self): bx = Box({"someKey": {"innerKey": "converted"}}, camel_killer_box=True, box_dots=True) - bx.update({"someKey": {"innerKey": "exact"}}) + # Deliberately retain an unnormalized key to test exact-key lookup precedence. + dict.__setitem__(bx, "someKey", Box({"innerKey": "exact"}, camel_killer_box=True, box_dots=True)) assert bx["someKey.innerKey"] == "exact" assert bx["some_key.innerKey"] == "converted" diff --git a/test/test_update_insertion.py b/test/test_update_insertion.py new file mode 100644 index 00000000..6b7d4508 --- /dev/null +++ b/test/test_update_insertion.py @@ -0,0 +1,88 @@ +import pytest + +from box import Box, BoxError, ConfigBox +from box.exceptions import BoxWarning + + +@pytest.mark.parametrize("box_class", [Box, ConfigBox]) +@pytest.mark.parametrize("source_type", [dict, list, iter]) +def test_update_recasts_each_incoming_value_once(box_class, source_type): + calls = [] + + def cast(value): + calls.append(value) + return int(value) + + bx = box_class(box_recast={"batch_size": cast}) + pairs = [("batch_size", "12")] + assert bx.update(source_type(pairs), batch_size="13") is None + assert bx.batch_size == 13 + assert calls == ["12", "13"] + assert pairs == [("batch_size", "12")] + + +@pytest.mark.parametrize("method", ["__or__", "__ior__"]) +def test_union_uses_configured_insertions(method): + bx = Box({"TimeOut": 10}, camel_killer_box=True, box_recast={"batch_size": int}) + source = {"TimeOut": 30, "batch_size": "12"} + result = getattr(bx, method)(source) + assert result.to_dict() == {"time_out": 30, "batch_size": 12} + assert result.batch_size * 2 == 24 + assert source == {"TimeOut": 30, "batch_size": "12"} + + +def test_update_rejects_invalid_recast_without_replacing_value(): + bx = Box(batch_size=10, box_recast={"batch_size": int}) + with pytest.raises(ValueError, match="Cannot convert"): + bx.update(batch_size="invalid") + assert bx.batch_size == 10 + + +@pytest.mark.parametrize("keep_keys", [False, True]) +def test_update_respects_camel_key_policy(keep_keys): + bx = Box({"TimeOut": 10}, camel_killer_box=True, camel_killer_box_keep_keys=keep_keys) + bx.update({"TimeOut": 30}) + key = "TimeOut" if keep_keys else "time_out" + assert bx.to_dict() == {key: 30} + assert bx.time_out == 30 + + +@pytest.mark.parametrize("policy", ["error", "warn", "ignore"]) +def test_update_respects_duplicate_policy(policy): + bx = Box({"a b": 1}, box_duplicates=policy) + bx.update({"a b": 2}) + if policy == "error": + with pytest.raises(BoxError, match="Duplicate"): + bx.update({"a-b": 3}) + assert bx.to_dict() == {"a b": 2} + elif policy == "warn": + with pytest.warns(BoxWarning, match="Duplicate"): + bx.update({"a-b": 3}) + assert bx.to_dict() == {"a b": 2, "a-b": 3} + else: + bx.update({"a-b": 3}) + assert bx.to_dict() == {"a b": 2, "a-b": 3} + + +def test_update_dispatches_subclass_assignment_and_replaces_nested_values(): + calls = [] + + class RecordingBox(Box): + def __setitem__(self, key, value): + calls.append(key) + super().__setitem__(key, value) + + bx = RecordingBox(service={"keep": 1}, box_dots=True) + calls.clear() + source = {"service": {"replace": 2}} + bx.update(source) + assert calls == ["service", "replace"] + assert bx.to_dict() == source + assert bx.service.to_dict() == {"replace": 2} + + +def test_config_box_update_keeps_latest_case_spelling_and_recasts(): + bx = ConfigBox({"batch_size": 10}, box_recast={"BATCH_SIZE": int}) + bx.update({"BATCH_SIZE": "12"}) + assert bx.to_dict() == {"BATCH_SIZE": 12} + assert bx.batch_size == 12 From df55c71e43780b323c87cebf183e969a094d207f Mon Sep 17 00:00:00 2001 From: Chris Griffith Date: Fri, 13 Nov 2026 11:00:34 -0600 Subject: [PATCH 19/20] Document the 8.0.1 update() changes as breaking changes in the changelog Co-Authored-By: Claude Opus 5.5 --- CHANGES.rst | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 845746e4..c31442ef 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,16 +3,30 @@ Changelog Version 8.0.1 ------------- -* Apply configured value recasting, key conversion and duplicate checks to update and union insertions -* Preserve BoxList item comments when in-place repetition uses an object with __index__, without raising after changing the list +Breaking changes + +These change how Box.update (and the ``|`` and ``|=`` operators, which use it) store values. Code that relied on the old behavior may see different data after upgrading. See the "Box 8.0.1" section of the Major Version Breaking Changes wiki page for before / after examples. + +* Fixing #356 update() storing dotted keys literally with box_dots, they are now paths like box["a.b"] = 1 and the Box constructor. Every earlier version (6.x, 7.x, 8.0.0) stored them literally: + + * Box({"a": {"x": 1}}, box_dots=True).update({"a.b": 2}) now gives {"a": {"x": 1, "b": 2}}, previously {"a": {"x": 1}, "a.b": 2} + * Box(box_dots=True).update({"a.b": 1}) now raises BoxKeyError because "a" does not exist (as box["a.b"] = 1 already did), previously it stored {"a.b": 1}. Add default_box=True to create the missing parents -* Clear removed BoxList item comments after in-place repetition by zero or a negative count, so new items do not inherit stale YAML comments (thanks to Mrunmayee Daware) + (thanks to rahul05ranjan) -* Fix ConfigBox.merge_update consuming one-shot key/value pairs before merging them -* Preserve BoxList item comments when insert, pop and deletion use objects with __index__ (thanks to parijat1222q) +* update() now applies the same rules as box[key] = value and the Box constructor, previously it skipped them (thanks to peterolkhov): + + * box_recast converts updated values, and raises BoxValueError when a value cannot be converted, e.g. update(n="12") with box_recast={"n": int} now stores 12, previously "12" + * camel_killer_box converts updated keys, e.g. update(TimeOut=2) now replaces time_out, previously it added a second TimeOut key + * box_duplicates checks updated keys, so "error" now raises BoxError and "warn" now warns for keys that collide on attribute access + * Box subclasses that override __setitem__ now have it called for every updated key + +Changes -* Fixing #356 update() storing dotted keys literally with box_dots instead of treating them as paths (thanks to rahul05ranjan) * Fixing #358 BoxList + and * returning a plain list instead of a BoxList (thanks to rahul05ranjan) +* Fixing #354 BoxList insert, pop, deletion and ``*=`` with an object with __index__ raising TypeError after changing the list when the list has item comments (thanks to parijat1222q and Anish Pentyala) +* Fixing ConfigBox.merge_update consuming one-shot key/value pairs before merging them +* Fixing BoxList in-place repetition by zero or a negative count leaving the removed items' comments behind for new items to inherit (thanks to Mrunmayee Daware) * Fixing a BoxList in a frozen Box only being half frozen: append, extend, insert, pop, remove, clear, sort and reverse still changed it, and it was shared with the Box it came from. A frozen Box now gets its own frozen copy, and a BoxList checks frozen_box on every change * Fixing copy.deepcopy of a frozen Box leaving the nested Boxes mutable * Fixing BoxList slice assignment and slice deletion dropping the item comments of every item, comments now stay with the items they were on @@ -20,7 +34,7 @@ Version 8.0.1 * Fixing Box.pop(path, default) raising IndexError for an out of range list index in a box_dots path, it returns the default like get() * Fixing a failed dotted del creating the parent key with default_box_create_on_get * Fixing BoxList box_dots indexes such as "[x][1]" being accepted and acting on the wrong item, they now raise BoxTypeError -* BoxList + and * (and n * BoxList) keep the item comments, also those of the added BoxList, and *= repeats them +* BoxList ``+`` and ``*`` (and ``n * BoxList``) keep the item comments, also those of the added BoxList, and ``*=`` repeats them * Fixing #349 BoxList item comments moving to the wrong items after a box_dots deletion, e.g. del bl["[0]"] (thanks to HocVoNgThai) * Fixing #349 BoxList.sort() calling the key function twice when the list has comments (thanks to HocVoNgThai) From 1a355209564030dc5b2bd0161dccfbb604e62604 Mon Sep 17 00:00:00 2001 From: Chris Griffith Date: Fri, 13 Nov 2026 11:51:19 -0600 Subject: [PATCH 20/20] Credit all 8.0.1 contributors in AUTHORS and the changelog Co-Authored-By: Claude Opus 5.5 --- AUTHORS.rst | 6 ++---- CHANGES.rst | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index fce381c2..b2f08ec3 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -5,11 +5,8 @@ reporting, all input is greatly appreciated! Code contributions: -- Anish Pentyala - - Alexandre Bertho (alexbertho) - parijat1222q - - Alexandre Decan (AlexandreDecan) - dhilipsiva (dhilipsiva) - MAA (FooBarQuaxx) @@ -62,7 +59,8 @@ Code contributions: - machines-6657 (machines-6657) - Chris (ChrisJr404) - gyanu2507 (gyanu2507) - +- Rahul (rahul05ranjan) +- Anish Pentyala (anishpentyala) Suggestions and bug reporting: diff --git a/CHANGES.rst b/CHANGES.rst index c31442ef..826d7485 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -25,7 +25,7 @@ Changes * Fixing #358 BoxList + and * returning a plain list instead of a BoxList (thanks to rahul05ranjan) * Fixing #354 BoxList insert, pop, deletion and ``*=`` with an object with __index__ raising TypeError after changing the list when the list has item comments (thanks to parijat1222q and Anish Pentyala) -* Fixing ConfigBox.merge_update consuming one-shot key/value pairs before merging them +* Fixing ConfigBox.merge_update consuming one-shot key/value pairs before merging them (thanks to alexbertho) * Fixing BoxList in-place repetition by zero or a negative count leaving the removed items' comments behind for new items to inherit (thanks to Mrunmayee Daware) * Fixing a BoxList in a frozen Box only being half frozen: append, extend, insert, pop, remove, clear, sort and reverse still changed it, and it was shared with the Box it came from. A frozen Box now gets its own frozen copy, and a BoxList checks frozen_box on every change * Fixing copy.deepcopy of a frozen Box leaving the nested Boxes mutable