diff --git a/AUTHORS.rst b/AUTHORS.rst index 2b5dbf4a..b2f08ec3 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -5,6 +5,8 @@ reporting, all input is greatly appreciated! Code contributions: +- Alexandre Bertho (alexbertho) +- parijat1222q - Alexandre Decan (AlexandreDecan) - dhilipsiva (dhilipsiva) - MAA (FooBarQuaxx) @@ -57,7 +59,8 @@ Code contributions: - machines-6657 (machines-6657) - Chris (ChrisJr404) - gyanu2507 (gyanu2507) - +- Rahul (rahul05ranjan) +- Anish Pentyala (anishpentyala) Suggestions and bug reporting: @@ -121,3 +124,7 @@ Suggestions and bug reporting: - bwanshoom - tan_unique (vinson0522) - lin (theman6660) +- HocVoNgThai + +- Mrunmayee Daware (Mrun25) +- Peter Olkhovets (peterolkhov) diff --git a/CHANGES.rst b/CHANGES.rst index e3801a3f..826d7485 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,42 @@ Changelog -========= + +Version 8.0.1 +------------- + +Breaking changes + +These change how Box.update (and the ``|`` and ``|=`` operators, which use it) store values. Code that relied on the old behavior may see different data after upgrading. See the "Box 8.0.1" section of the Major Version Breaking Changes wiki page for before / after examples. + +* Fixing #356 update() storing dotted keys literally with box_dots, they are now paths like box["a.b"] = 1 and the Box constructor. Every earlier version (6.x, 7.x, 8.0.0) stored them literally: + + * Box({"a": {"x": 1}}, box_dots=True).update({"a.b": 2}) now gives {"a": {"x": 1, "b": 2}}, previously {"a": {"x": 1}, "a.b": 2} + * Box(box_dots=True).update({"a.b": 1}) now raises BoxKeyError because "a" does not exist (as box["a.b"] = 1 already did), previously it stored {"a.b": 1}. Add default_box=True to create the missing parents + + (thanks to rahul05ranjan) + +* update() now applies the same rules as box[key] = value and the Box constructor, previously it skipped them (thanks to peterolkhov): + + * box_recast converts updated values, and raises BoxValueError when a value cannot be converted, e.g. update(n="12") with box_recast={"n": int} now stores 12, previously "12" + * camel_killer_box converts updated keys, e.g. update(TimeOut=2) now replaces time_out, previously it added a second TimeOut key + * box_duplicates checks updated keys, so "error" now raises BoxError and "warn" now warns for keys that collide on attribute access + * Box subclasses that override __setitem__ now have it called for every updated key + +Changes + +* Fixing #358 BoxList + and * returning a plain list instead of a BoxList (thanks to rahul05ranjan) +* Fixing #354 BoxList insert, pop, deletion and ``*=`` with an object with __index__ raising TypeError after changing the list when the list has item comments (thanks to parijat1222q and Anish Pentyala) +* Fixing ConfigBox.merge_update consuming one-shot key/value pairs before merging them (thanks to alexbertho) +* Fixing BoxList in-place repetition by zero or a negative count leaving the removed items' comments behind for new items to inherit (thanks to Mrunmayee Daware) +* Fixing a BoxList in a frozen Box only being half frozen: append, extend, insert, pop, remove, clear, sort and reverse still changed it, and it was shared with the Box it came from. A frozen Box now gets its own frozen copy, and a BoxList checks frozen_box on every change +* Fixing copy.deepcopy of a frozen Box leaving the nested Boxes mutable +* Fixing BoxList slice assignment and slice deletion dropping the item comments of every item, comments now stay with the items they were on +* Fixing from_yaml(frozen_box=True) dropping the comments on list items, frozen lists (tuples) now keep comments, and box_dots list paths such as "l[0].x" now work in a frozen Box +* Fixing Box.pop(path, default) raising IndexError for an out of range list index in a box_dots path, it returns the default like get() +* Fixing a failed dotted del creating the parent key with default_box_create_on_get +* Fixing BoxList box_dots indexes such as "[x][1]" being accepted and acting on the wrong item, they now raise BoxTypeError +* BoxList ``+`` and ``*`` (and ``n * BoxList``) keep the item comments, also those of the added BoxList, and ``*=`` repeats them +* Fixing #349 BoxList item comments moving to the wrong items after a box_dots deletion, e.g. del bl["[0]"] (thanks to HocVoNgThai) +* Fixing #349 BoxList.sort() calling the key function twice when the list has comments (thanks to HocVoNgThai) Version 8.0.0 ------------- diff --git a/box/__init__.py b/box/__init__.py index 0f14c44b..783a5d0c 100644 --- a/box/__init__.py +++ b/box/__init__.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- __author__ = "Chris Griffith" -__version__ = "8.0.0" +__version__ = "8.0.1" from box.box import Box from box.box_list import BoxList diff --git a/box/box.py b/box/box.py index e1ce2047..f2b539b2 100644 --- a/box/box.py +++ b/box/box.py @@ -85,6 +85,77 @@ def _camel_killer(attr): return re.sub(" *_+", "_", s2.lower()) +_tuple_pos_re = re.compile(r"\[(\d+)\]") +_tuple_comment_index_re = re.compile(r"\[(-?\d+)\]") + + +class _BoxTuple(tuple): + """ + The tuple ``frozen_box`` stores a list as. Unlike a plain tuple it can carry comments (#184) and + follows ``box_dots`` list paths such as ``"[0].key"``. + """ + + def __getitem__(self, item, _ignore_default=False): + if isinstance(item, str) and item.startswith("["): + list_pos = _tuple_pos_re.match(item) + if list_pos is None: + raise BoxTypeError(f"Invalid list index: {item!r}") + value = tuple.__getitem__(self, int(list_pos.group(1))) + children = item[list_pos.end() :].lstrip(".") + if not children: + return value + if _ignore_default and isinstance(value, (Box, box.BoxList, _BoxTuple)): + return value.__getitem__(children, _ignore_default=True) + return value[children] + return tuple.__getitem__(self, item) + + def _comment_store(self, create: bool = True) -> Any: + store = self.__dict__.get("_box_comments") + if store is None and create: + store = self.__dict__["_box_comments"] = {} + return store + + def _comment_document(self, create: bool = True) -> Any: + document = self.__dict__.get("_box_comments_document") + if document is None and create: + document = self.__dict__["_box_comments_document"] = {} + return document + + def _comment_check_frozen(self): + raise BoxError("Box is frozen") + + def _comment_owner(self, key): + if key is None: + return self, None + if isinstance(key, int) and not isinstance(key, bool): + index = key + len(self) if key < 0 else key + if 0 <= index < len(self): + return self, index + elif isinstance(key, str) and key.startswith("["): + list_pos = _tuple_comment_index_re.match(key) + if list_pos is not None: + index = int(list_pos.group(1)) + index = index + len(self) if index < 0 else index + rest = key[list_pos.end() :].lstrip(".") + if 0 <= index < len(self): + if not rest: + return self, index + child = tuple.__getitem__(self, index) + if hasattr(child, "_comment_owner"): + return child._comment_owner(rest) + raise BoxKeyError(f"{key!r} is not in the frozen list, comments can only be attached to existing items") + + def get_comment(self, key: Any = None) -> dict | None: + """The comments of an item (see ``BoxList.get_comment``), ``None`` for the document comments.""" + return _comments.get_comment(self, key) + + def set_comment(self, key: Any, text: Any, position: str | None = None) -> None: + _comments.set_comment(self, key, text, position) + + def del_comment(self, key: Any = None) -> None: + _comments.del_comment(self, key) + + def _recursive_tuples(iterable, box_class, recreate_tuples=False, **kwargs): out_list = [] for i in iterable: @@ -94,7 +165,10 @@ def _recursive_tuples(iterable, box_class, recreate_tuples=False, **kwargs): out_list.append(_recursive_tuples(i, box_class, recreate_tuples, **kwargs)) else: out_list.append(i) - return tuple(out_list) + out = _BoxTuple(out_list) + if isinstance(iterable, box.BoxList): + iterable._comment_copy_to(out) + return out def _to_native_tuple(iterable): @@ -449,7 +523,7 @@ def __contains__(self, item): it = self[first_item] if isinstance(it, Box): return it.__dots_int_key(children) in it - if isinstance(it, box.BoxList): + if isinstance(it, (box.BoxList, _BoxTuple)): # children is the rest of the path starting with "[", e.g. "[0].x" try: it.__getitem__(children, _ignore_default=True) @@ -523,16 +597,18 @@ def __copy__(self): return self.copy() def __deepcopy__(self, memodict=None): - frozen = self._box_config["frozen_box"] config = self.__box_config() - config["frozen_box"] = False out = self.__class__(**config) memodict = memodict if memodict is not None else {} memodict[id(self)] = out + # Fill it like __init__ does, still "being created", so nested Boxes get the frozen config too + frozen = self._box_config["frozen_box"] + if frozen: + out._box_config["__created"] = False for k, v in self.items(): out[copy.deepcopy(k, memodict)] = copy.deepcopy(v, memodict) + out._box_config["__created"] = True self.__copy_internals(out, deep=True, memo=memodict) - out._box_config["frozen_box"] = frozen return out def __setstate__(self, state): @@ -693,7 +769,11 @@ def __convert_and_store(self, item, value): else: value = box.BoxList(value, **self.__box_config(extra_namespace=item)) elif isinstance(value, box.BoxList): - value.box_options.update(self.__box_config(extra_namespace=item)) + if self._box_config["frozen_box"]: + # A frozen Box gets its own frozen copy, the list it was given stays as it was + value = box.BoxList(value, **self.__box_config(extra_namespace=item)) + else: + value.box_options.update(self.__box_config(extra_namespace=item)) elif self._box_config["modify_tuples_box"] and isinstance(value, tuple): value = _recursive_tuples(value, recreate_tuples=True, **self.__box_config(extra_namespace=item)) super().__setitem__(item, value) @@ -721,7 +801,7 @@ def __getitem__(self, item, _ignore_default=False): if isinstance(child, Box): children = child.__dots_int_key(children) if hasattr(child, "__getitem__"): - if _ignore_default and isinstance(child, (Box, box.BoxList)): + if _ignore_default and isinstance(child, (Box, box.BoxList, _BoxTuple)): return child.__getitem__(children, _ignore_default=True) return child[children] if self.__camel_destructive() and isinstance(item, str): @@ -822,7 +902,7 @@ def _comment_owner(self, key): child = dict.__getitem__(self, first) if isinstance(child, Box): return child._comment_owner(child.__dots_int_key(children)) - if isinstance(child, box.BoxList): + if isinstance(child, (box.BoxList, _BoxTuple)): return child._comment_owner(children) raise BoxKeyError(f"{key!r} is not in the Box, comments can only be attached to existing keys") @@ -923,7 +1003,7 @@ def __delitem__(self, key): if key not in self.keys() and self.__process_dotted_key(key): first_item, children = _parse_box_dots(self, key) first_item = self.__dots_int_key(first_item) - child = self[first_item] + child = self.__getitem__(first_item, _ignore_default=True) if isinstance(child, Box): children = child.__dots_int_key(children) if hasattr(child, "__delitem__"): @@ -986,7 +1066,7 @@ def pop(self, key, *args): raise BoxError('pop() takes only one optional argument "default"') try: item = self.__getitem__(key, _ignore_default=True) - except KeyError: + except (KeyError, IndexError): return args[0] else: del self[key] @@ -1058,12 +1138,15 @@ def update(self, *args, **kwargs): if single_arg: if hasattr(single_arg, "keys"): for k in single_arg: - self.__convert_and_store(k, single_arg[k]) + self._update_store(k, single_arg[k]) else: for k, v in single_arg: - self.__convert_and_store(k, v) + self._update_store(k, v) for k in kwargs: - self.__convert_and_store(k, kwargs[k]) + self._update_store(k, kwargs[k]) + + def _update_store(self, key, value): + self.__setitem__(key, value) def merge_update(self, *args, **kwargs): """ diff --git a/box/box_list.py b/box/box_list.py index 8de6b2f0..7bf6c70f 100644 --- a/box/box_list.py +++ b/box/box_list.py @@ -4,8 +4,10 @@ # Copyright (c) 2017-2026 - Chris Griffith - MIT License from __future__ import annotations +import bisect import copy import copyreg +import operator import re from collections.abc import Iterable from os import PathLike @@ -37,10 +39,6 @@ _comment_index_re = re.compile(r"\[(-?\d+)\]") # the comment API also takes negative indexes -def _frozen_box_list_method(*args, **kwargs): - raise BoxError("BoxList is frozen") - - class BoxList(list): """ Drop in replacement of list, that converts added objects to Box or BoxList @@ -62,17 +60,14 @@ def __init__(self, iterable: Iterable | None = None, box_class: type[box.Box] = # 2026-10-10: Iteration does not require a truth value. if iterable is not None: for x in iterable: - self.append(x) + list.append(self, self._convert(x)) if isinstance(iterable, BoxList) and iterable is not self: iterable._comment_copy_to(self) self.box_org_ref = None - if box_options.get("frozen_box"): - for method in ["append", "clear", "extend", "insert", "pop", "remove", "reverse", "sort"]: - self.__setattr__(method, _frozen_box_list_method) def __getitem__(self, item, _ignore_default=False): if self.box_options.get("box_dots") and isinstance(item, str) and item.startswith("["): - list_pos = _list_pos_re.search(item) + list_pos = _list_pos_re.match(item) if list_pos is None: raise BoxTypeError(f"Invalid list index: {item!r}") value = super().__getitem__(int(list_pos.groups()[0])) @@ -96,14 +91,16 @@ def __delitem__(self, key): if self.box_options.get("frozen_box"): raise BoxError("BoxList is frozen") if self.box_options.get("box_dots") and isinstance(key, str) and key.startswith("["): - list_pos = _list_pos_re.search(key) + list_pos = _list_pos_re.match(key) if list_pos is None: raise BoxTypeError(f"Invalid list index: {key!r}") pos = int(list_pos.groups()[0]) if len(list_pos.group()) == len(key): - return super().__delitem__(pos) + return self.__delitem__(pos) if hasattr(self[pos], "__delitem__"): return self[pos].__delitem__(key[len(list_pos.group()) :].lstrip(".")) # type: ignore + if not isinstance(key, slice) and hasattr(type(key), "__index__"): + key = operator.index(key) length = len(self) super().__delitem__(key) self._comment_after_delete(key, length) @@ -112,7 +109,7 @@ def __setitem__(self, key, value): if self.box_options.get("frozen_box"): raise BoxError("BoxList is frozen") if self.box_options.get("box_dots") and isinstance(key, str) and key.startswith("["): - list_pos = _list_pos_re.search(key) + list_pos = _list_pos_re.match(key) if list_pos is None: raise BoxTypeError(f"Invalid list index: {key!r}") pos = int(list_pos.groups()[0]) @@ -139,12 +136,36 @@ def converted_items(): for item in value: yield self._convert(item) + length = len(self) # Let list validate the slice before consuming replacement values. super().__setitem__(key, converted_items()) - self.__dict__.pop("_box_comments", None) # items moved, their comments can't follow + self._comment_after_slice_set(key, length) return super().__setitem__(key, self._convert(value)) + def __add__(self, other): + if not isinstance(other, list): + raise BoxTypeError("BoxList can only be added to another list.") + out = box.BoxList(list(self) + list(other), **self.box_options) + self._comment_copy_to(out) + if isinstance(other, BoxList): + other._comment_copy_to(out, offset=len(self), document=False) + return out + + def __mul__(self, count): + try: + count = operator.index(count) + except TypeError: + raise BoxTypeError("BoxList can only be multiplied by an integer.") from None + out = box.BoxList(list(self) * count, **self.box_options) + self._comment_repeat(out, count, len(self)) + document = self.__dict__.get("_box_comments_document") + if document: + out._comment_document().update(copy.deepcopy(document)) + return out + + __rmul__ = __mul__ + def __iadd__(self, value): # type: ignore[misc] # list accepts any iterable for +=, not only lists if self.box_options.get("frozen_box"): raise BoxError("BoxList is frozen") @@ -154,7 +175,14 @@ def __iadd__(self, value): # type: ignore[misc] # list accepts any iterable fo def __imul__(self, value): # type: ignore[misc] if self.box_options.get("frozen_box"): raise BoxError("BoxList is frozen") - return super().__imul__(value) + value = operator.index(value) + length = len(self) + result = super().__imul__(value) + if len(self) == 0: + self.__dict__.pop("_box_comments", None) + else: + self._comment_repeat(self, value, length, first=1) + return result def _is_intact_type(self, obj): if self.box_options.get("box_intact_types") and isinstance(obj, self.box_options["box_intact_types"]): @@ -177,15 +205,19 @@ def _convert(self, p_object): return p_object def append(self, p_object): + self._comment_check_frozen() super().append(self._convert(p_object)) def extend(self, iterable): + self._comment_check_frozen() if iterable is self: iterable = list(self) for item in iterable: self.append(item) def insert(self, index, p_object): + self._comment_check_frozen() + index = operator.index(index) length = len(self) super().insert(index, self._convert(p_object)) if self.__dict__.get("_box_comments"): @@ -193,6 +225,8 @@ def insert(self, index, p_object): self._comment_reindex(lambda k: k + 1 if k >= at else k) def pop(self, index=-1): + self._comment_check_frozen() + index = operator.index(index) length = len(self) value = super().pop(index) self._comment_after_delete(index, length) @@ -202,22 +236,25 @@ def remove(self, value): self.pop(self.index(value)) def clear(self): + self._comment_check_frozen() super().clear() self.__dict__.pop("_box_comments", None) def reverse(self): + self._comment_check_frozen() super().reverse() last = len(self) - 1 self._comment_reindex(lambda k: last - k) def sort(self, *, key=None, reverse=False): + self._comment_check_frozen() if not self.__dict__.get("_box_comments"): return super().sort(key=key, reverse=reverse) # Work out where every item goes (same stable order as list.sort) so comments follow them - sort_key = key or (lambda x: x) + sort_key = (lambda x: x) if key is None else key order = sorted(range(len(self)), key=lambda i: sort_key(list.__getitem__(self, i)), reverse=reverse) new_position = {old: new for new, old in enumerate(order)} - super().sort(key=key, reverse=reverse) + super().__setitem__(slice(None), [list.__getitem__(self, i) for i in order]) self._comment_reindex(new_position.get) # ---- Comments (#184), by item index ---- @@ -252,11 +289,44 @@ def _comment_after_delete(self, key, length): if not self.__dict__.get("_box_comments"): return if isinstance(key, slice): - self.__dict__.pop("_box_comments", None) + removed = sorted(range(*key.indices(length))) + if removed: + gone_set = set(removed) + self._comment_reindex(lambda k: None if k in gone_set else k - bisect.bisect_left(removed, k)) return gone = key + length if key < 0 else key self._comment_reindex(lambda k: None if k == gone else (k - 1 if k > gone else k)) + def _comment_after_slice_set(self, key, length): + """Move the item comments after ``self[key] = values``, `length` is the size before it.""" + if not self.__dict__.get("_box_comments"): + return + start, stop, step = key.indices(length) + if step != 1: + # An extended slice replaces items one for one, the replacements start without a comment + replaced = set(range(start, stop, step)) + self._comment_reindex(lambda k: None if k in replaced else k) + return + stop = max(start, stop) + shift = len(self) - length + + def mapping(k): + if k < start: + return k + return k + shift if k >= stop else None # replaced items are new, they start without a comment + + self._comment_reindex(mapping) + + def _comment_repeat(self, out, count, length, first=0): + """Copy the comments of the first `length` items to repetitions `first`..`count` of them in `out`.""" + store = self.__dict__.get("_box_comments") + if store: + target = out._comment_store() + for repeat in range(first, max(count, 0)): + for index, comment in list(store.items()): + if index < length: + target[index + repeat * length] = copy.deepcopy(comment) + def _comment_owner(self, key): """(container, index) that holds the comment for `key`, following box_dots paths.""" if key is None: @@ -279,11 +349,13 @@ def _comment_owner(self, key): return child._comment_owner(rest) raise BoxKeyError(f"{key!r} is not in the BoxList, comments can only be attached to existing items") - def _comment_copy_to(self, other): + def _comment_copy_to(self, other, offset=0, document=True): store = self.__dict__.get("_box_comments") if store: - other._comment_store().update({k: copy.deepcopy(v) for k, v in store.items() if k < len(other)}) - document = self.__dict__.get("_box_comments_document") + other._comment_store().update( + {k + offset: copy.deepcopy(v) for k, v in store.items() if k + offset < len(other)} + ) + document = self.__dict__.get("_box_comments_document") if document else None if document: other._comment_document().update(copy.deepcopy(document)) diff --git a/box/comments.py b/box/comments.py index 8f2cf632..c1485ba6 100644 --- a/box/comments.py +++ b/box/comments.py @@ -197,7 +197,7 @@ def walk(node, path): def _resolve_key(container, key): """The key in `container` the YAML key ended up as, or a sentinel when it isn't there.""" - if isinstance(container, list): + if isinstance(container, (list, tuple)): return key if isinstance(key, int) and 0 <= key < len(container) else _MISSING if dict.__contains__(container, key): return key @@ -222,7 +222,12 @@ def apply_comments(root, model: dict | None) -> None: if found is _MISSING: node = None break - node = list.__getitem__(node, found) if isinstance(node, list) else dict.__getitem__(node, found) + if isinstance(node, list): + node = list.__getitem__(node, found) + elif isinstance(node, tuple): + node = tuple.__getitem__(node, found) + else: + node = dict.__getitem__(node, found) if not hasattr(node, "_comment_store"): node = None break @@ -251,7 +256,7 @@ def has_comments(obj, _seen=None) -> bool: if id(obj) in seen: return False seen.add(id(obj)) - values = obj.values() if isinstance(obj, dict) else obj + values = obj.values() if isinstance(obj, dict) else obj # a list or a frozen list (tuple) return any(has_comments(value, seen) for value in values) @@ -286,6 +291,8 @@ def commented_tree(obj, plain, column=0, _seen=None): from ruamel.yaml.comments import CommentedMap, CommentedSeq seen = _seen if _seen is not None else set() + if isinstance(plain, tuple) and hasattr(obj, "_comment_store"): + plain = list(plain) # a frozen list: a tuple that can carry comments if isinstance(plain, (dict, list)): if id(plain) in seen: return plain # self references are written as they are @@ -299,7 +306,11 @@ def commented_tree(obj, plain, column=0, _seen=None): elif isinstance(plain, list): out = CommentedSeq() for index, value in enumerate(plain): - child = list.__getitem__(obj, index) if isinstance(obj, list) and index < len(obj) else None + child = ( + (list if isinstance(obj, list) else tuple).__getitem__(obj, index) + if isinstance(obj, (list, tuple)) and index < len(obj) + else None + ) out.append(commented_tree(child, value, column + 2, seen)) keys = list(range(len(out))) else: diff --git a/box/config_box.py b/box/config_box.py index 5d5ed876..9f7b37e8 100644 --- a/box/config_box.py +++ b/box/config_box.py @@ -203,7 +203,8 @@ def merge_update(self, *args, **kwargs): if hasattr(args[0], "keys"): incoming = list(args[0].keys()) else: - args = (list(args[0]),) # may be a one-shot iterator, it is read twice + # Both the outer iterable and its pairs may be one-shot iterators. + args = ([tuple(pair) for pair in args[0]],) incoming = [k for k, _ in args[0]] else: incoming = [] diff --git a/test/test_audit_sync.py b/test/test_audit_sync.py new file mode 100644 index 00000000..e003d9dd --- /dev/null +++ b/test/test_audit_sync.py @@ -0,0 +1,753 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Audit of the places where BoxList item comments and Box options are kept in sync by hand. + +1. Every BoxList mutator, with item comments present, plain / box_dots / frozen: comments must stay + on the item they were attached to, and new items must be converted to Box / BoxList. +2. box_dots paths must behave the same through every access method. +3. copy / deepcopy / pickle round trips of every Box flavour, with comments. + +The matrix was written to reproduce the bugs found in the 8.0.1 audit; they are fixed, these tests keep them so. +""" + +import copy +import pickle + +import pytest + +from box import Box, BoxError, BoxList, ConfigBox, SBox +from box.exceptions import BoxKeyError, BoxTypeError +from box.shorthand_box import DDBox + +# -------------------------------------------------------------------------------------------------- +# 1. BoxList mutators +# -------------------------------------------------------------------------------------------------- + +N = 4 # items 0..3, item i is {"id": i} carrying the comment "c" +NEW = 100 # ids >= NEW are items added by the operation, they never carry a comment + + +def annotated(**options): + bl = BoxList([{"id": i} for i in range(N)], **options) + for i in range(N): + bl.set_comment(i, f"c{i}") + return bl + + +def frozen_annotated(**options): + return BoxList(annotated(**options), frozen_box=True, **options) + + +def check_comments(bl, kept): + """ + A comment, wherever it is, must be the one of the item now at that position, and every + item id in `kept` must still have its comment. + """ + seen = set() + for i, item in enumerate(bl): + comment = bl.get_comment(i) + if comment is None: + continue + assert isinstance(item, Box), f"comment {comment} on non item {item!r} at {i}" + assert comment == {"eol": f"c{item.id}"}, f"position {i} holds item {item.id} but has comment {comment}" + seen.add(item.id) + missing = set(kept) - seen + assert not missing, f"comments lost for items {sorted(missing)}" + + +def check_converted(bl, **options): + assert isinstance(bl, BoxList) + for item in bl: + assert not (isinstance(item, dict) and not isinstance(item, Box)), f"{item!r} not converted" + assert not (isinstance(item, list) and not isinstance(item, BoxList)), f"{item!r} not converted" + if isinstance(item, Box): + for k, v in options.items(): + assert item._box_config[k] == v, f"option {k} not passed down to {item!r}" + if isinstance(item, BoxList): + for k, v in options.items(): + assert item.box_options.get(k) == v, f"option {k} not passed down to {item!r}" + + +def _setitem(bl, key, value): + bl[key] = value + + +def _delitem(bl, key): + del bl[key] + + +def _iadd(bl, value): + bl += value + + +def _imul(bl, value): + bl *= value + + +ALL = set(range(N)) + +# (id, operation, ids whose comment must survive) +IN_PLACE_OPS = [ + ("append", lambda bl: bl.append({"id": NEW}), ALL), + ("append_list", lambda bl: bl.append([{"id": NEW}]), ALL), + ("extend", lambda bl: bl.extend([{"id": NEW}, [{"id": NEW + 1}]]), ALL), + ("extend_generator", lambda bl: bl.extend({"id": NEW + i} for i in range(2)), ALL), + ("extend_self", lambda bl: bl.extend(bl), ALL), + ("iadd", lambda bl: _iadd(bl, [{"id": NEW}]), ALL), + ("iadd_tuple", lambda bl: _iadd(bl, ({"id": NEW},)), ALL), + ("insert_0", lambda bl: bl.insert(0, {"id": NEW}), ALL), + ("insert_mid", lambda bl: bl.insert(2, {"id": NEW}), ALL), + ("insert_neg", lambda bl: bl.insert(-1, {"id": NEW}), ALL), + ("insert_past_end", lambda bl: bl.insert(99, {"id": NEW}), ALL), + ("insert_before_start", lambda bl: bl.insert(-99, {"id": NEW}), ALL), + ("pop_last", lambda bl: bl.pop(), {0, 1, 2}), + ("pop_0", lambda bl: bl.pop(0), {1, 2, 3}), + ("pop_neg", lambda bl: bl.pop(-3), {0, 2, 3}), + ("del_1", lambda bl: _delitem(bl, 1), {0, 2, 3}), + ("del_neg", lambda bl: _delitem(bl, -1), {0, 1, 2}), + ("remove", lambda bl: bl.remove({"id": 2}), {0, 1, 3}), + ("clear", lambda bl: bl.clear(), set()), + ("reverse", lambda bl: bl.reverse(), ALL), + ("sort_desc", lambda bl: bl.sort(key=lambda x: x.id, reverse=True), ALL), + ("sort_mixed", lambda bl: bl.sort(key=lambda x: x.id % 2), ALL), + # A replaced item keeps the comment of its position, like a Box key; the replacement reuses the id + ("setitem", lambda bl: _setitem(bl, 1, {"id": 1, "new": True}), ALL), + ("setitem_neg", lambda bl: _setitem(bl, -1, {"id": 3, "new": True}), ALL), + ("imul_1", lambda bl: _imul(bl, 1), ALL), + ("imul_2", lambda bl: _imul(bl, 2), ALL), + ("imul_0", lambda bl: _imul(bl, 0), set()), + ("imul_neg", lambda bl: _imul(bl, -1), set()), +] + +# Slices: the items outside the slice never move (or only shift), their comments should survive +SLICE_OPS = [ + ("del_slice_tail", lambda bl: _delitem(bl, slice(2, None)), {0, 1}), + ("del_slice_head", lambda bl: _delitem(bl, slice(0, 1)), {1, 2, 3}), + ("del_slice_step", lambda bl: _delitem(bl, slice(None, None, 2)), {1, 3}), + ("setitem_slice_same_len", lambda bl: _setitem(bl, slice(3, 4), [{"id": NEW}]), {0, 1, 2}), + ("setitem_slice_grow", lambda bl: _setitem(bl, slice(1, 2), [{"id": NEW}, {"id": NEW + 1}]), {0, 2, 3}), + ("setitem_slice_shrink", lambda bl: _setitem(bl, slice(1, 3), []), {0, 3}), + ("setitem_slice_append", lambda bl: _setitem(bl, slice(N, N), [{"id": NEW}]), ALL), + ("setitem_ext_slice", lambda bl: _setitem(bl, slice(None, None, 2), [{"id": NEW}, {"id": NEW + 1}]), {1, 3}), +] + +DOTS_OPS = [ + ("dots_setitem", lambda bl: _setitem(bl, "[1]", {"id": 1, "new": True}), ALL), + ("dots_setitem_child", lambda bl: _setitem(bl, "[1].extra", {"x": 1}), ALL), + ("dots_del", lambda bl: _delitem(bl, "[1]"), {0, 2, 3}), + ("dots_del_child", lambda bl: _delitem(bl, "[1].id") or bl[1].__setitem__("id", 1), ALL), +] + +OPTION_SETS = [pytest.param({}, id="plain"), pytest.param({"box_dots": True}, id="box_dots")] + + +def _ids(ops): + return [pytest.param(fn, kept, id=name) for name, fn, kept in ops] + + +@pytest.mark.parametrize("options", OPTION_SETS) +@pytest.mark.parametrize("operation, kept", _ids(IN_PLACE_OPS)) +def test_mutator_keeps_comments_and_converts(options, operation, kept): + bl = annotated(**options) + operation(bl) + check_comments(bl, kept) + check_converted(bl, **options) + + +@pytest.mark.parametrize("options", OPTION_SETS) +@pytest.mark.parametrize("operation, kept", _ids(SLICE_OPS)) +def test_slice_mutator_keeps_comments_of_untouched_items(options, operation, kept): + bl = annotated(**options) + operation(bl) + check_comments(bl, kept) + + +@pytest.mark.parametrize("options", OPTION_SETS) +@pytest.mark.parametrize("operation, kept", _ids(SLICE_OPS)) +def test_slice_mutator_never_misplaces_comments(options, operation, kept): + bl = annotated(**options) + operation(bl) + check_comments(bl, set()) + check_converted(bl, **options) + + +@pytest.mark.parametrize("operation, kept", _ids(DOTS_OPS)) +def test_box_dots_mutator(operation, kept): + bl = annotated(box_dots=True) + operation(bl) + check_comments(bl, kept) + check_converted(bl, box_dots=True) + + +def test_box_dots_mutators_through_parent_box(): + b = Box({"l": [{"id": i} for i in range(N)]}, box_dots=True) + for i in range(N): + b.l.set_comment(i, f"c{i}") + b["l[1]"] = {"id": 1, "new": True} + check_comments(b.l, ALL) + del b["l[0]"] + check_comments(b.l, {1, 2, 3}) + assert b.pop("l[0]").id == 1 + check_comments(b.l, {2, 3}) + assert b.get_comment("l[0]") == {"eol": "c2"} + assert b.get_comment("l[1]") == {"eol": "c3"} + check_converted(b.l, box_dots=True) + + +# ---- non mutating operations: the result carries the comments, the original is untouched ---- + +COPY_OPS = [ + ("copy", copy.copy), + ("deepcopy", copy.deepcopy), + ("pickle", lambda bl: pickle.loads(pickle.dumps(bl))), + ("pickle_p0", lambda bl: pickle.loads(pickle.dumps(bl, protocol=0))), + ("BoxList()", lambda bl: BoxList(bl, **bl.box_options)), +] + + +@pytest.mark.parametrize("options", OPTION_SETS + [pytest.param({"frozen_box": True}, id="frozen")]) +@pytest.mark.parametrize("operation", [pytest.param(fn, id=name) for name, fn in COPY_OPS]) +def test_copies_keep_comments_options_and_conversion(options, operation): + bl = frozen_annotated() if options.get("frozen_box") else annotated(**options) + bl.set_comment(None, "document") if not options.get("frozen_box") else None + out = operation(bl) + assert out is not bl + assert out == bl + check_comments(out, ALL) + check_converted(out, **options) + for k, v in options.items(): + assert out.box_options.get(k) == v + if not options.get("frozen_box"): + assert out.get_comment(None) == {"start": "document"} + out.set_comment(0, "changed") # comments are not shared with the original + assert bl.get_comment(0) == {"eol": "c0"} + + +@pytest.mark.parametrize( + "operation", + [ + pytest.param(lambda bl: bl + [{"id": NEW}], id="add"), + pytest.param(lambda bl: bl * 2, id="mul"), + pytest.param(lambda bl: 2 * bl, id="rmul"), + ], +) +def test_binary_operators_convert(operation): + out = operation(annotated(box_dots=True)) + check_comments(out, set()) + check_converted(out, box_dots=True) + + +@pytest.mark.parametrize( + "operation", + [ + pytest.param(lambda bl: bl + [{"id": NEW}], id="add"), + pytest.param(lambda bl: bl * 2, id="mul"), + pytest.param(lambda bl: 2 * bl, id="rmul"), + ], +) +def test_binary_operators_keep_comments_of_left_operand(operation): + out = operation(annotated()) + check_comments(out, ALL) + + +# ---- frozen ---- + +FROZEN_MUTATORS = [ + ("append", lambda bl: bl.append({"id": NEW})), + ("extend", lambda bl: bl.extend([{"id": NEW}])), + ("insert", lambda bl: bl.insert(0, {"id": NEW})), + ("pop", lambda bl: bl.pop()), + ("remove", lambda bl: bl.remove(bl[0])), + ("clear", lambda bl: bl.clear()), + ("reverse", lambda bl: bl.reverse()), + ("sort", lambda bl: bl.sort(key=lambda x: -x.id)), + ("setitem", lambda bl: _setitem(bl, 0, {"id": NEW})), + ("setitem_slice", lambda bl: _setitem(bl, slice(0, 1), [{"id": NEW}])), + ("delitem", lambda bl: _delitem(bl, 0)), + ("delitem_slice", lambda bl: _delitem(bl, slice(0, 1))), + ("iadd", lambda bl: _iadd(bl, [{"id": NEW}])), + ("imul", lambda bl: _imul(bl, 2)), + ("imul_0", lambda bl: _imul(bl, 0)), + ("set_comment", lambda bl: bl.set_comment(0, "x")), + ("del_comment", lambda bl: bl.del_comment(0)), +] + + +def _frozen_direct(): + return frozen_annotated() + + +def _frozen_through_box(): + # A frozen Box adopting an existing BoxList (for example Box(other_box, frozen_box=True)) + return Box({"l": annotated()}, frozen_box=True).l + + +def _frozen_through_box_copy(): + return Box(Box({"l": annotated()}), frozen_box=True).l + + +FROZEN_SOURCES = [ + pytest.param(_frozen_direct, id="BoxList(frozen_box)"), + pytest.param(lambda: copy.deepcopy(_frozen_direct()), id="deepcopy"), + pytest.param(lambda: pickle.loads(pickle.dumps(_frozen_direct())), id="pickle"), + pytest.param(lambda: copy.copy(_frozen_direct()), id="copy"), +] + +HALF_FROZEN_SOURCES = [ + pytest.param(_frozen_through_box, id="Box(frozen_box)-adopts-BoxList"), + pytest.param(_frozen_through_box_copy, id="Box(Box, frozen_box)"), +] + + +def _frozen_check(source, operation): + bl = source() + before = bl.to_list() + with pytest.raises(BoxError): + operation(bl) + assert bl.to_list() == before + check_comments(bl, ALL) + + +@pytest.mark.parametrize("source", FROZEN_SOURCES) +@pytest.mark.parametrize("operation", [pytest.param(fn, id=name) for name, fn in FROZEN_MUTATORS]) +def test_frozen_boxlist_rejects_every_mutator(source, operation): + _frozen_check(source, operation) + + +@pytest.mark.parametrize("source", HALF_FROZEN_SOURCES) +@pytest.mark.parametrize( + "operation", + [pytest.param(fn, id=name) for name, fn in FROZEN_MUTATORS], +) +def test_boxlist_frozen_by_parent_box_rejects_every_mutator(source, operation): + _frozen_check(source, operation) + + +def test_freezing_a_copy_does_not_freeze_the_source_list(): + src = Box({"l": [1, 2]}) + Box(src, frozen_box=True) + src.l[0] = 9 + src.l.set_comment(0, "still mutable") + assert src.l == [9, 2] + + +def test_freezing_a_box_freezes_list_items(): + frozen = Box(Box({"lst": [{"i": 0}]}), frozen_box=True) + assert frozen.lst[0]._box_config["frozen_box"] + with pytest.raises(BoxError): + frozen.lst[0].i = 5 + hash(frozen) + + +def test_frozen_from_yaml_keeps_list_comments(): + pytest.importorskip("ruamel.yaml") + text = "# doc\ntop: 1 # top-c\nlst:\n- i: 0 # i0\n- 2 # l1\n" + assert Box.from_yaml(text).to_yaml() == text + assert Box.from_yaml(text, frozen_box=True).to_yaml() == text + + +def test_frozen_boxlist_items_are_frozen_and_hash_is_stable(): + bl = frozen_annotated(box_dots=True) + assert all(item._box_config["frozen_box"] for item in bl) + with pytest.raises(BoxError): + bl[0].id = 5 + with pytest.raises(BoxError): + bl["[0].id"] = 5 + assert bl["[0].id"] == 0 + assert hash(bl) == hash(pickle.loads(pickle.dumps(bl))) == hash(copy.deepcopy(bl)) + + +# -------------------------------------------------------------------------------------------------- +# 2. box_dots consistency +# -------------------------------------------------------------------------------------------------- + + +def dots_box(**options): + return Box({"a": {"b": 1}, "l": [{"x": 1}, [10, 11]], "s": "str"}, box_dots=True, **options) + + +EXISTING_PATHS = [("a.b", 1), ("l[0].x", 1), ("l[1][0]", 10), ("l[1]", [10, 11])] +MISSING_PATHS = ["a.zz", "zz.yy", "l[0].zz", "l[9]", "l[9].x", "l[1][9]"] + + +@pytest.mark.parametrize("path, value", EXISTING_PATHS) +def test_dots_existing_path_same_through_every_accessor(path, value): + b = dots_box() + assert path in b + assert b[path] == value + assert b.get(path) == value + assert b.get(path, "default") == value + assert b.setdefault(path, "default") == value + assert b.pop(path) == value + + +@pytest.mark.parametrize("path, value", EXISTING_PATHS) +def test_dots_existing_path_writes(path, value): + for write in ( + lambda b: b.__setitem__(path, "new"), + lambda b: b.update({path: "new"}), + lambda b: b.update(**{path: "new"}), + lambda b: b.merge_update({path: "new"}), + lambda b: b.__ior__({path: "new"}), + ): + b = dots_box() + write(b) + assert b[path] == "new" + assert path not in dict.keys(b), "dotted key stored literally" + + +@pytest.mark.parametrize("path", MISSING_PATHS) +def test_dots_missing_path_reads_agree(path): + b = dots_box() + assert path not in b + assert b.get(path) is None + assert b.get(path, "default") == "default" + + +@pytest.mark.parametrize( + "path", + MISSING_PATHS, +) +def test_dots_missing_path_pop_default_agrees_with_get(path): + """Box.pop(key, default) treats an out of range list index like get() and `in` do: as missing.""" + b = dots_box() + before = b.to_dict() + assert b.pop(path, "default") == "default" + assert b.to_dict() == before + + +def test_dots_failed_delete_has_no_side_effect(): + b = Box(box_dots=True, default_box=True, default_box_create_on_get=True) + with pytest.raises(KeyError): + del b["zz.yy"] + assert b.to_dict() == {} + + +def test_dots_missing_reads_have_no_side_effect_with_create_on_get(): + b = Box(box_dots=True, default_box=True, default_box_create_on_get=True) + assert "zz.yy" not in b + assert b.get("zz.yy", 1) == 1 + assert b.pop("zz.yy", 2) == 2 + assert b.to_dict() == {} + + +def test_dots_list_paths_in_frozen_box(): + b = dots_box(frozen_box=True) + assert b["l[0].x"] == 1 + assert "l[0].x" in b + + +def test_dots_dict_paths_in_frozen_box(): + b = dots_box(frozen_box=True) + before = b.to_dict() + assert b["a.b"] == 1 + assert "a.b" in b + for write in ( + lambda: b.__setitem__("a.b", 2), + lambda: b.update({"a.b": 2}), + lambda: b.merge_update({"a.b": 2}), + lambda: b.__delitem__("a.b"), + lambda: b.pop("a.b"), + lambda: b.setdefault("a.new", 1), + ): + with pytest.raises(BoxError): + write() + assert b.to_dict() == before + + +def test_dots_comments_do_not_resurrect(): + b = Box({"a": {"b": 1, "c": 2}, "l": [1, 2]}, box_dots=True) + b.set_comment("a.b", "ab") + del b["a.b"] + b["a.b"] = 5 + assert b.get_comment("a.b") is None + b.set_comment("a.c", "ac") + b.pop("a.c") + b.update({"a.c": 5}) + assert b.get_comment("a.c") is None + b.l.set_comment(1, "l1") + b.pop("l[1]") + b.l.append(9) + assert b.get_comment("l[1]") is None + + +def test_dots_boxlist_index_forms(): + bl = BoxList([{"a": [{"b": 1}]}, [[2]]], box_dots=True) + assert bl["[0].a[0].b"] == 1 + assert bl["[1][0][0]"] == 2 + bl["[0].a[0].b"] = 3 + bl["[1][0][0]"] = 4 + assert bl.to_list() == [{"a": [{"b": 3}]}, [[4]]] + del bl["[0].a[0]"] + del bl["[1][0][0]"] + assert bl.to_list() == [{"a": []}, [[]]] + + +def test_dots_boxlist_rejects_garbage_index(): + bl = BoxList([["a", "b"], ["c", "d"]], box_dots=True) + with pytest.raises(BoxError): + bl["[x][1]"] + with pytest.raises(BoxError): + del bl["[x][0]"] + assert bl.to_list() == [["a", "b"], ["c", "d"]] + + +def test_dots_negative_index_is_comment_api_only(): + """Negative dotted indexes are a comment API convenience, data access rejects them (by design).""" + b = dots_box() + b.set_comment("l[-1]", "last") + assert b.get_comment("l[1]") == {"eol": "last"} + with pytest.raises(BoxTypeError): + b["l[-1]"] + + +def test_dots_configbox_case_insensitive_paths(): + c = ConfigBox({"Sect": {"Key": 1, "Other": 2}}, box_dots=True) + c.Sect.set_comment("Other", "oc") + assert c["sect.key"] == 1 + assert "SECT.KEY" in c + assert c.get("sect.key") == 1 + assert c.setdefault("sect.key", 5) == 1 + c.update({"sect.key": 2}) + assert c.Sect.to_dict() == {"key": 2, "Other": 2} + c.merge_update({"sect.key": 3}) + assert c["Sect.Key"] == 3 + assert c.pop("SECT.key") == 3 + assert c.get_comment("sect.other") == {"eol": "oc"} + del c["sect.other"] + assert c.to_dict() == {"Sect": {}} + + +# -------------------------------------------------------------------------------------------------- +# 3. copy / deepcopy / pickle round trips +# -------------------------------------------------------------------------------------------------- + +ROUND_TRIPS = [ + pytest.param(copy.copy, id="copy"), + pytest.param(copy.deepcopy, id="deepcopy"), + pytest.param(lambda x: x.copy(), id="method_copy"), +] + [pytest.param(lambda x, p=p: pickle.loads(pickle.dumps(x, protocol=p)), id=f"pickle_p{p}") for p in (0, 2, 5)] + +BOX_CLASSES = [Box, ConfigBox, SBox, DDBox] + +BOX_OPTIONS = [ + pytest.param({}, id="plain"), + pytest.param({"box_dots": True}, id="box_dots"), + pytest.param({"frozen_box": True}, id="frozen"), + pytest.param({"box_dots": True, "frozen_box": True}, id="box_dots-frozen"), + pytest.param({"default_box": True}, id="default_box"), + pytest.param({"camel_killer_box": True}, id="camel_killer"), + pytest.param({"box_dots": True, "box_dots_exclude": r"x\.y"}, id="box_dots_exclude"), +] + + +def _annotated_box(cls, options): + data = {"Top": 1, "sub": {"k": 2}, "lst": [{"i": 0}, {"i": 1}]} + build = {k: v for k, v in options.items() if k != "frozen_box"} + b = cls(data, **build) + b.set_comment("Top", "top-c", "before") + b.set_comment(None, "doc") + b.sub.set_comment("k", "k-c") + b.lst.set_comment(1, "l1-c") + b.lst[0].set_comment("i", "i0-c") + b.lst.set_comment(None, "ldoc", "end") + if options.get("frozen_box"): + b = cls(b, **options) + return b + + +def _box_state(b): + return ( + type(b), + b.to_dict(), + b.get_comment("Top"), + b.get_comment(None), + b.sub.get_comment("k"), + b.lst.get_comment(1), + b.lst[0].get_comment("i"), + b.lst.get_comment(None), + type(b.sub), + type(b.lst), + type(b.lst[0]), + {k: v for k, v in b._box_config.items() if not k.startswith("__") and k != "box_dots_exclude"}, + b._box_config["box_dots_exclude"] and b._box_config["box_dots_exclude"].pattern, + b.sub._box_config["box_dots"], + b.sub._box_config["frozen_box"], + b.lst.box_options.get("box_dots"), + b.lst[0]._box_config["box_dots"], + ) + + +def test_deepcopy_of_frozen_box_is_frozen_all_the_way_down(): + frozen = Box({"sub": {"k": 1}}, frozen_box=True) + out = copy.deepcopy(frozen) + with pytest.raises(BoxError): + out.sub.k = 2 + assert hash(out) == hash(frozen) + + +@pytest.mark.parametrize("cls", BOX_CLASSES, ids=lambda c: c.__name__) +@pytest.mark.parametrize("options", BOX_OPTIONS) +@pytest.mark.parametrize("round_trip", ROUND_TRIPS) +def test_box_round_trip(cls, options, round_trip): + b = _annotated_box(cls, options) + out = round_trip(b) + assert out is not b + assert _box_state(out) == _box_state(b) + if options.get("frozen_box"): + with pytest.raises(BoxError): + out.Top = 5 + with pytest.raises(BoxError): + out.sub.k = 5 + else: + out.set_comment("Top", "changed") + assert b.get_comment("Top") == {"before": "top-c"} + if options.get("box_dots"): + assert out["lst[0].i"] == 0 + assert out.get_comment("lst[1]") == {"eol": "l1-c"} + + +@pytest.mark.parametrize("round_trip", ROUND_TRIPS[:2] + ROUND_TRIPS[3:]) +def test_nested_boxlist_round_trip(round_trip): + bl = BoxList([[1, 2], [{"a": [3]}]], box_dots=True) + bl.set_comment(1, "one") + bl[0].set_comment(1, "inner") + bl[1][0].a.set_comment(0, "deep") + out = round_trip(bl) + assert out == bl + assert out.get_comment(1) == {"eol": "one"} + assert out.get_comment("[0][1]") == {"eol": "inner"} + assert out.get_comment("[1][0].a[0]") == {"eol": "deep"} + assert out["[1][0].a[0]"] == 3 + check_converted(out, box_dots=True) + + +@pytest.mark.parametrize("round_trip", ROUND_TRIPS[:2] + ROUND_TRIPS[3:]) +def test_self_referencing_boxlist_round_trip(round_trip): + bl = BoxList([1]) + bl.append(bl) + bl.set_comment(0, "c") + out = round_trip(bl) + assert out.get_comment(0) == {"eol": "c"} + if round_trip is not ROUND_TRIPS[0].values[0]: # copy.copy re-wraps items + assert out[1] is out + + +@pytest.mark.parametrize("round_trip", ROUND_TRIPS[:2] + ROUND_TRIPS[3:]) +def test_configbox_case_index_after_round_trip(round_trip): + c = ConfigBox({"Host": "h", "Port": 1}) + _ = c["host"] # builds the case index that is stored in the box config + c.set_comment("host", "hc") + out = round_trip(c) + out["HOST"] = "z" + assert out.to_dict() == {"Port": 1, "HOST": "z"} + assert out.get_comment("host") == {"eol": "hc"} + assert c.to_dict() == {"Host": "h", "Port": 1} + + +# ---- fixes of the 8.0.1 audit: frozen lists (tuples), operators ---- + + +def _frozen_yaml_box(): + pytest.importorskip("ruamel.yaml") + return Box.from_yaml("l:\n- a: 1 # a\n- - 5 # deep\n- 3 # three\n", frozen_box=True) + + +def test_frozen_list_tuple_comment_api(): + bx = _frozen_yaml_box() + assert type(bx.l) is not tuple and isinstance(bx.l, tuple) + assert bx.l.get_comment(2) == {"eol": "three"} + assert bx.l.get_comment("[-1]") == {"eol": "three"} + assert bx.l.get_comment("[0].a") == {"eol": "a"} + assert bx.l.get_comment("[1][0]") == {"eol": "deep"} + dots = Box.from_yaml("l:\n- - 5 # deep\n", frozen_box=True, box_dots=True) + assert dots.get_comment("l[0][0]") == {"eol": "deep"} + assert bx.l.get_comment(None) is None + for call in (lambda: bx.l.set_comment(0, "x"), lambda: bx.l.del_comment(0)): + with pytest.raises(BoxError): + call() + for bad in (5, "[9]", "x", "[x]", "[2].nope"): + with pytest.raises(BoxKeyError): + bx.l.get_comment(bad) + bx.l._comment_document()["start"] = ["doc"] + assert bx.l.get_comment() == {"start": "doc"} + + +def test_frozen_list_tuple_dotted_reads(): + bx = Box({"l": [{"a": 1}, [7, {"b": 2}]]}, box_dots=True, frozen_box=True) + assert bx["l[0].a"] == 1 + assert bx["l[1][1].b"] == 2 + assert bx.l["[1]"] == (7, {"b": 2}) + assert bx.l["[0]"].a == 1 + with pytest.raises(BoxTypeError): + bx.l["[x]"] + assert bx.l[0].a == 1 + assert "l[1][1].b" in bx + assert "l[1][1].zz" not in bx + assert bx.get("l[9].a", "d") == "d" + + +def test_frozen_list_tuple_copies_keep_comments(): + bx = _frozen_yaml_box() + for out in (copy.deepcopy(bx), pickle.loads(pickle.dumps(bx)), copy.copy(bx)): + assert out.l.get_comment(2) == {"eol": "three"} + assert out.to_yaml() == bx.to_yaml() + + +def test_frozen_list_comments_follow_box_list_adoption(): + bl = BoxList([{"a": 1}, 2]) + bl.set_comment(1, "two") + bx = Box({"l": bl}, frozen_box=True) + assert bx.l.get_comment(1) == {"eol": "two"} + assert bl.get_comment(1) == {"eol": "two"} # the source list is left as it was + + +@pytest.mark.parametrize("repeat", [2, 3]) +def test_repeat_copies_comments_to_every_repetition(repeat): + bl = annotated() + original = len(bl) + for out in (bl * repeat, repeat * bl): + assert len(out) == original * repeat + for index in range(len(out)): + assert out.get_comment(index) == {"eol": f"c{index % original}"} + bl *= repeat + for index in range(len(bl)): + assert bl.get_comment(index) == {"eol": f"c{index % original}"} + + +def test_add_copies_comments_of_both_lists_and_document(): + left = annotated() + left.set_comment(None, "doc") + right = annotated() + out = left + right + assert out.get_comment() == {"start": "doc"} + for index in range(len(out)): + assert out.get_comment(index) == {"eol": f"c{index % len(left)}"} + assert (left + [{"id": NEW}]).get_comment(len(left)) is None + + +def test_mul_copies_document_comments(): + bl = annotated() + bl.set_comment(None, "doc") + assert (bl * 2).get_comment() == {"start": "doc"} + assert (bl * 0).to_list() == [] + + +def test_frozen_box_converts_a_box_list_inside_a_plain_list(): + bl = BoxList([1]) + bl.set_comment(0, "one") + bx = Box({"l": [bl]}, frozen_box=True) + assert bx.l[0].get_comment(0) == {"eol": "one"} + + +def test_empty_slice_delete_keeps_comments(): + bl = annotated() + del bl[1:1] + check_comments(bl, ALL) + + +def test_from_yaml_comments_inside_an_intact_dict_are_skipped(): + pytest.importorskip("ruamel.yaml") + bx = Box.from_yaml("a:\n b: 1 # c\nz: 2 # z\n", box_intact_types=(dict,)) + assert bx.get_comment("z") == {"eol": "z"} diff --git a/test/test_box.py b/test/test_box.py index 143f4b0c..31315f6f 100644 --- a/test/test_box.py +++ b/test/test_box.py @@ -730,7 +730,8 @@ def test_camel_killer_box_dotted_lookup_disabled(self, camel_killer_box, box_dot def test_camel_killer_box_dotted_lookup_exact_key(self): bx = Box({"someKey": {"innerKey": "converted"}}, camel_killer_box=True, box_dots=True) - bx.update({"someKey": {"innerKey": "exact"}}) + # Deliberately retain an unnormalized key to test exact-key lookup precedence. + dict.__setitem__(bx, "someKey", Box({"innerKey": "exact"}, camel_killer_box=True, box_dots=True)) assert bx["someKey.innerKey"] == "exact" assert bx["some_key.innerKey"] == "converted" diff --git a/test/test_box_keys_8.py b/test/test_box_keys_8.py index 690dc424..fea8ab81 100644 --- a/test/test_box_keys_8.py +++ b/test/test_box_keys_8.py @@ -117,3 +117,36 @@ def test_config_box(self): cb = ConfigBox({"maxConnections": "5"}, camel_killer_box=True, camel_killer_box_keep_keys=True) assert cb.int("max_connections") == 5 assert cb.to_dict() == {"maxConnections": "5"} + + +class TestUpdateDottedPaths: + """update() honors box_dots path semantics, like __setitem__ and merge_update.""" + + def test_update_dotted_key_nests(self): + b = Box(default_box=True, box_dots=True) + b.update({"x.y": 5}) + assert b["x"]["y"] == 5 + + def test_update_dotted_key_extends_existing_path(self): + b = Box({"x": {"y": 1}}, box_dots=True) + b.update({"x.z": 5}) + assert b.to_dict() == {"x": {"y": 1, "z": 5}} + + def test_update_dotted_key_with_dots_exclude_stays_literal(self): + b = Box(box_dots=True, box_dots_exclude=r"^[a-z]+\.[a-z]+$") + b.update({"keep.this": 1}) + assert b.to_dict() == {"keep.this": 1} + + def test_update_kwargs_dotted_key_nests(self): + b = Box(default_box=True, box_dots=True) + b.update(**{"k.k": 9}) + assert b["k"]["k"] == 9 + + def test_update_non_dotted_unchanged(self): + b = Box({"a": 1}) + b.update({"b": 2}, c=3) + assert b.to_dict() == {"a": 1, "b": 2, "c": 3} + + def test_box_addition_merges_dotted_paths(self): + b = Box({"x": {"y": 1}}, default_box=True, box_dots=True) + Box({"x.z": 2}, default_box=True, box_dots=True) + assert b.to_dict() == {"x": {"y": 1, "z": 2}} diff --git a/test/test_box_list.py b/test/test_box_list.py index 469c3e98..d13a8370 100644 --- a/test/test_box_list.py +++ b/test/test_box_list.py @@ -15,6 +15,7 @@ from box import Box, BoxError, BoxList from box.converters import toml_read_library, toml_write_library +from box.exceptions import BoxTypeError from test.common import test_root, tmp_dir @@ -666,3 +667,147 @@ def test_iadd_frozen_does_not_consume_source(): data += source assert next(source) == {"value": 1} assert data == [0] + + +class _IndexOnce: + def __init__(self, value): + self.value = value + self.calls = 0 + + def __index__(self): + self.calls += 1 + if self.calls > 1: + raise AssertionError("index was evaluated more than once") + return self.value + + +@pytest.mark.parametrize("operation", ["insert", "pop", "delete"]) +@pytest.mark.parametrize("index", [0, 1, -1, -3]) +@pytest.mark.parametrize("comments", [False, True]) +def test_index_protocol_keeps_item_comments(operation, index, comments): + values = BoxList(["a", "b", "c"]) + if comments: + for position, item in enumerate(values): + values.set_comment(position, item) + expected = list(values) + key = _IndexOnce(index) + if operation == "insert": + expected.insert(index, "new") + values.insert(key, "new") + elif operation == "pop": + assert values.pop(key) == expected.pop(index) + else: + del expected[index] + del values[key] + assert key.calls == 1 + assert values == expected + for position, item in enumerate(values): + comment = {"eol": item} if comments and item != "new" else None + assert values.get_comment(position) == comment + + +@pytest.mark.parametrize("operation", ["insert", "pop", "delete"]) +@pytest.mark.parametrize("comments", [False, True]) +def test_invalid_index_does_not_change_items_or_comments(operation, comments): + class BadIndex: + def __index__(self): + raise ValueError("invalid index") + + values = BoxList(["a", "b", "c"]) + if comments: + for position, item in enumerate(values): + values.set_comment(position, item) + with pytest.raises(ValueError, match="invalid index"): + if operation == "insert": + values.insert(BadIndex(), "new") + elif operation == "pop": + values.pop(BadIndex()) + else: + del values[BadIndex()] + assert values == ["a", "b", "c"] + for position, item in enumerate(values): + assert values.get_comment(position) == ({"eol": item} if comments else None) + + +@pytest.mark.parametrize("index", [-100, 100]) +@pytest.mark.parametrize("comments", [False, True]) +def test_insert_index_protocol_clamps_and_moves_comments(index, comments): + values = BoxList(["a", "b"]) + if comments: + values.set_comment(0, "a") + values.set_comment(1, "b") + expected = list(values) + expected.insert(index, "new") + key = _IndexOnce(index) + values.insert(key, "new") + assert key.calls == 1 + assert values == expected + for position, item in enumerate(values): + expected_comment = {"eol": item} if comments and item != "new" else None + assert values.get_comment(position) == expected_comment + + +@pytest.mark.parametrize("operation", ["pop", "delete"]) +@pytest.mark.parametrize("index", [-100, 100]) +def test_out_of_bounds_index_preserves_comments(operation, index): + values = BoxList(["a", "b"]) + values.set_comment(0, "a") + values.set_comment(1, "b") + key = _IndexOnce(index) + with pytest.raises(IndexError): + if operation == "pop": + values.pop(key) + else: + del values[key] + assert key.calls == 1 + assert values == ["a", "b"] + assert values.get_comment(0) == {"eol": "a"} + assert values.get_comment(1) == {"eol": "b"} + + +class TestBoxListOperators: + def test_add_keeps_boxlist_and_converts(self): + bl = BoxList([{"a": 1}], box_dots=True) + result = bl + [{"b": 2}] + assert isinstance(result, BoxList) + assert isinstance(result[0], Box) + assert isinstance(result[1], Box) + assert result.box_options["box_dots"] is True + + def test_add_two_boxlists(self): + result = BoxList([{"a": 1}]) + BoxList([{"b": 2}]) + assert isinstance(result, BoxList) + assert result.to_list() == [{"a": 1}, {"b": 2}] + + def test_add_non_list_raises(self): + with pytest.raises(BoxTypeError): + BoxList() + 5 + + def test_mul_keeps_boxlist(self): + bl = BoxList([{"a": 1}], box_dots=True) + result = bl * 3 + assert isinstance(result, BoxList) + assert len(result) == 3 + assert result.box_options["box_dots"] is True + assert result.to_list() == [{"a": 1}] * 3 + + def test_rmul_and_index_protocol(self): + class Two: + def __index__(self): + return 2 + + bl = BoxList([{"a": 1}]) + for result in (2 * bl, bl * Two()): + assert isinstance(result, BoxList) + assert result.to_list() == [{"a": 1}] * 2 + + def test_mul_non_int_raises(self): + with pytest.raises(BoxTypeError): + BoxList() * "x" + + def test_frozen_add_result_is_frozen(self): + frozen = BoxList([{"a": 1}], frozen_box=True) + result = frozen + [{"b": 2}] + assert isinstance(result, BoxList) + with pytest.raises(BoxError): + result.append({"c": 3}) diff --git a/test/test_comments.py b/test/test_comments.py index e685c436..feb1d067 100644 --- a/test/test_comments.py +++ b/test/test_comments.py @@ -370,11 +370,25 @@ def test_box_list_without_comments_keeps_working(self): del bl[0:1] assert bl == [2, 1] - def test_slice_delete_drops_item_comments(self): + def test_dotted_list_deletion_keeps_surviving_comments(self): + values = BoxList.from_yaml("- first # first comment\n- second # second comment\n", box_dots=True) + del values["[0]"] + assert values.to_yaml() == "- second # second comment\n" + + def test_sort_with_comments_evaluates_keys_once(self): + values = BoxList.from_yaml("- z # Z\n- a # A\n") + ranks = {"z": 2, "a": 1} + assert values.sort(key=ranks.pop) is None + assert ranks == {} + assert values.to_yaml() == "- a # A\n- z # Z\n" + + def test_slice_delete_drops_removed_comments_and_moves_the_rest(self): bl = BoxList([1, 2, 3]) + bl.set_comment(0, "gone") bl.set_comment(2, "x") del bl[0:1] - assert bl.get_comment(1) is None + assert bl.get_comment(0) is None + assert bl.get_comment(1) == {"eol": "x"} def test_empty_comment_and_missing_final_newline(self): bx = Box.from_yaml("a: 1 #\nb: 2 # two\n# end") @@ -382,9 +396,12 @@ def test_empty_comment_and_missing_final_newline(self): assert bx.get_comment("b") == {"eol": "two"} assert bx.get_comment(None) == {"end": "end"} - def test_frozen_lists_skip_item_comments(self): - bx = Box.from_yaml("l:\n- a: 1 # in a frozen list\n", frozen_box=True) + def test_frozen_lists_keep_item_comments(self): + text = "l:\n- a: 1 # in a frozen list\n" + bx = Box.from_yaml(text, frozen_box=True) assert bx.to_dict() == {"l": ({"a": 1},)} # frozen boxes store lists as tuples + assert bx.l[0].get_comment("a") == {"eol": "in a frozen list"} + assert bx.to_yaml() == text def test_box_dots_split_parent_key(self): bx = Box.from_yaml("a.b:\n c: 1 # c\n", box_dots=True, default_box=True) @@ -422,3 +439,37 @@ def test_tabs_are_fine(self): bx = Box(a=1) bx.set_comment("a", "x\ty") assert Box.from_yaml(bx.to_yaml()).get_comment("a") == {"eol": "x\ty"} + + +@pytest.mark.parametrize("count", [0, -1, -3]) +def test_inplace_repetition_drops_comments_of_removed_items(count): + values = BoxList.from_yaml("# list heading\n- old # obsolete explanation\n") + original = values + values *= count + assert values is original + assert values == [] + values.append("new") + assert values.get_comment(0) is None + assert values.get_comment() == {"start": "list heading"} + assert values.to_yaml() == "# list heading\n- new\n" + + +@pytest.mark.parametrize("count", [1, 2]) +def test_inplace_repetition_keeps_surviving_item_comments(count): + values = BoxList.from_yaml("- old # explanation\n") + original = values + values *= count + assert values is original + assert values == ["old"] * count + assert values.get_comment(0) == {"eol": "explanation"} + + +def test_inplace_repetition_keeps_comments_on_false_valued_subclass(): + class FalseBoxList(BoxList): + def __bool__(self): + return False + + values = FalseBoxList(["old"]) + values.set_comment(0, "explanation") + values *= 1 + assert values.get_comment(0) == {"eol": "explanation"} diff --git a/test/test_config_box_case.py b/test/test_config_box_case.py index 95f5f9c5..63530cc1 100644 --- a/test/test_config_box_case.py +++ b/test/test_config_box_case.py @@ -12,6 +12,15 @@ class TestConfigBoxCaseInsensitive: + @pytest.mark.parametrize("outer", [list, iter]) + def test_merge_update_one_shot_pairs(self, outer): + cb = ConfigBox({"Host": {"port": 80}}) + pairs = outer([iter(("host", {"name": "localhost"})), iter(("timeout", 30))]) + cb.merge_update(pairs) + assert cb.to_dict() == {"host": {"port": 80, "name": "localhost"}, "timeout": 30} + assert list(cb) == ["host", "timeout"] + assert cb.HOST.port == 80 + def test_lookup_any_case(self): cb = ConfigBox({"HOST": "localhost", "ServerPort": 8080}) assert cb["host"] == cb["Host"] == cb.host == cb.HOST == "localhost" diff --git a/test/test_imul_index_comments.py b/test/test_imul_index_comments.py new file mode 100644 index 00000000..fddfbf7e --- /dev/null +++ b/test/test_imul_index_comments.py @@ -0,0 +1,60 @@ +"""In-place repetition must normalize index-protocol counts before mutation.""" + +import operator + +import pytest + +from box import BoxError, BoxList + + +class IndexOnce: + def __init__(self, value): + self.value = value + self.calls = 0 + + def __index__(self): + self.calls += 1 + if self.calls > 1: + raise AssertionError("count evaluated twice") + return self.value + + +@pytest.mark.parametrize("count", [2, 1, 0, -2]) +@pytest.mark.parametrize("comments", [True, False]) +def test_imul_index_count_normalized_once(count, comments): + values = BoxList(["a", "b"]) + if comments: + values.set_comment(0, "first") + values.set_comment(1, "second") + key = IndexOnce(count) + result = operator.imul(values, key) + assert result is values + assert list(values) == ["a", "b"] * count + assert key.calls == 1 + if comments: + for index in range(max(count, 0) * 2): + assert values.get_comment(index) == {"eol": "first" if index % 2 == 0 else "second"} + if count <= 0: + assert not values.__dict__.get("_box_comments") + + +def test_imul_invalid_index_preserves_data_and_comments(): + class InvalidIndex: + def __index__(self): + raise ValueError("bad count") + + values = BoxList(["a", "b"]) + values.set_comment(1, "second") + with pytest.raises(ValueError, match="bad count"): + operator.imul(values, InvalidIndex()) + assert list(values) == ["a", "b"] + assert values.get_comment(1) == {"eol": "second"} + + +def test_imul_frozen_refuses_before_index_conversion(): + values = BoxList(["a"], frozen_box=True) + key = IndexOnce(2) + with pytest.raises(BoxError): + operator.imul(values, key) + assert key.calls == 0 + assert list(values) == ["a"] diff --git a/test/test_update_insertion.py b/test/test_update_insertion.py new file mode 100644 index 00000000..6b7d4508 --- /dev/null +++ b/test/test_update_insertion.py @@ -0,0 +1,88 @@ +import pytest + +from box import Box, BoxError, ConfigBox +from box.exceptions import BoxWarning + + +@pytest.mark.parametrize("box_class", [Box, ConfigBox]) +@pytest.mark.parametrize("source_type", [dict, list, iter]) +def test_update_recasts_each_incoming_value_once(box_class, source_type): + calls = [] + + def cast(value): + calls.append(value) + return int(value) + + bx = box_class(box_recast={"batch_size": cast}) + pairs = [("batch_size", "12")] + assert bx.update(source_type(pairs), batch_size="13") is None + assert bx.batch_size == 13 + assert calls == ["12", "13"] + assert pairs == [("batch_size", "12")] + + +@pytest.mark.parametrize("method", ["__or__", "__ior__"]) +def test_union_uses_configured_insertions(method): + bx = Box({"TimeOut": 10}, camel_killer_box=True, box_recast={"batch_size": int}) + source = {"TimeOut": 30, "batch_size": "12"} + result = getattr(bx, method)(source) + assert result.to_dict() == {"time_out": 30, "batch_size": 12} + assert result.batch_size * 2 == 24 + assert source == {"TimeOut": 30, "batch_size": "12"} + + +def test_update_rejects_invalid_recast_without_replacing_value(): + bx = Box(batch_size=10, box_recast={"batch_size": int}) + with pytest.raises(ValueError, match="Cannot convert"): + bx.update(batch_size="invalid") + assert bx.batch_size == 10 + + +@pytest.mark.parametrize("keep_keys", [False, True]) +def test_update_respects_camel_key_policy(keep_keys): + bx = Box({"TimeOut": 10}, camel_killer_box=True, camel_killer_box_keep_keys=keep_keys) + bx.update({"TimeOut": 30}) + key = "TimeOut" if keep_keys else "time_out" + assert bx.to_dict() == {key: 30} + assert bx.time_out == 30 + + +@pytest.mark.parametrize("policy", ["error", "warn", "ignore"]) +def test_update_respects_duplicate_policy(policy): + bx = Box({"a b": 1}, box_duplicates=policy) + bx.update({"a b": 2}) + if policy == "error": + with pytest.raises(BoxError, match="Duplicate"): + bx.update({"a-b": 3}) + assert bx.to_dict() == {"a b": 2} + elif policy == "warn": + with pytest.warns(BoxWarning, match="Duplicate"): + bx.update({"a-b": 3}) + assert bx.to_dict() == {"a b": 2, "a-b": 3} + else: + bx.update({"a-b": 3}) + assert bx.to_dict() == {"a b": 2, "a-b": 3} + + +def test_update_dispatches_subclass_assignment_and_replaces_nested_values(): + calls = [] + + class RecordingBox(Box): + def __setitem__(self, key, value): + calls.append(key) + super().__setitem__(key, value) + + bx = RecordingBox(service={"keep": 1}, box_dots=True) + calls.clear() + source = {"service": {"replace": 2}} + bx.update(source) + assert calls == ["service", "replace"] + assert bx.to_dict() == source + assert bx.service.to_dict() == {"replace": 2} + + +def test_config_box_update_keeps_latest_case_spelling_and_recasts(): + bx = ConfigBox({"batch_size": 10}, box_recast={"BATCH_SIZE": int}) + bx.update({"BATCH_SIZE": "12"}) + assert bx.to_dict() == {"BATCH_SIZE": 12} + assert bx.batch_size == 12