From c5e4a8cdb8ce2ea04091bad31485fe803193b111 Mon Sep 17 00:00:00 2001 From: Deirdre Wang Date: Wed, 9 Sep 2026 14:44:03 -0700 Subject: [PATCH 1/3] Report container nesting past the recursion limit as IonException --- src-python/amazon/ion/simpleion.py | 63 ++++++++++----- src/ion-c | 2 +- src/ioncmodule.c | 77 +++++++++++++++--- tests/test_simpleion.py | 121 ++++++++++++++++++++++++++++- 4 files changed, 228 insertions(+), 35 deletions(-) diff --git a/src-python/amazon/ion/simpleion.py b/src-python/amazon/ion/simpleion.py index f2d98c512..1857e199e 100644 --- a/src-python/amazon/ion/simpleion.py +++ b/src-python/amazon/ion/simpleion.py @@ -56,6 +56,7 @@ """ import io import warnings +from contextlib import contextmanager from datetime import datetime from decimal import Decimal from enum import IntFlag @@ -306,13 +307,14 @@ def dump_python(obj, fp, imports=None, binary=True, sequence_as_stream=False, from_type = _FROM_TYPE_TUPLE_AS_SEXP if tuple_as_sexp else _FROM_TYPE if binary or not omit_version_marker: writer.send(ION_VERSION_MARKER_EVENT) # The IVM is emitted automatically in binary; it's optional in text. - if sequence_as_stream and isinstance(obj, (list, tuple)) or isinstance(obj, GeneratorType): - # Treat this top-level sequence as a stream; serialize its elements as top-level values, but don't serialize the - # sequence itself. - for top_level in obj: - _dump(top_level, writer, from_type) - else: - _dump(obj, writer, from_type) + with _translate_recursion_error(): + if sequence_as_stream and isinstance(obj, (list, tuple)) or isinstance(obj, GeneratorType): + # Treat this top-level sequence as a stream; serialize its elements as top-level values, but don't + # serialize the sequence itself. + for top_level in obj: + _dump(top_level, writer, from_type) + else: + _dump(obj, writer, from_type) writer.send(ION_STREAM_END_EVENT) @@ -387,6 +389,16 @@ def _dump(obj, writer, from_type, field=None, in_struct=False, depth=0): writer.send(event) +@contextmanager +def _translate_recursion_error(): + """Reports container nesting too deep to process as an IonException, like other malformed + input, keeping the RecursionError as the cause.""" + try: + yield + except RecursionError as e: + raise IonException('Container nesting exceeded the maximum supported recursion depth.') from e + + def load_python(fp, catalog=None, single_value=True, parse_eagerly=True): """'pure' Python implementation. Users should prefer to call ``load``.""" if isinstance(fp, _TEXT_TYPES): @@ -402,7 +414,8 @@ def load_python(fp, catalog=None, single_value=True, parse_eagerly=True): reader = blocking_reader(managed_reader(raw_reader, catalog), fp) if parse_eagerly: out = [] # top-level - _load(out, reader) + with _translate_recursion_error(): + _load(out, reader) if single_value: if len(out) != 1: raise IonException('Stream contained %d values; expected a single value.' % (len(out),)) @@ -411,7 +424,8 @@ def load_python(fp, catalog=None, single_value=True, parse_eagerly=True): else: out = _load_iteratively(reader) if single_value: - result = next(out) + with _translate_recursion_error(): + result = next(out) try: next(out) raise IonException('Stream contained more than 1 values; expected a single value.') @@ -443,7 +457,8 @@ def _load_iteratively(reader, end_type=IonEventType.STREAM_END): ion_type = event.ion_type if event.event_type is IonEventType.CONTAINER_START: container = _FROM_ION_TYPE[ion_type].from_event(event) - _load(container, reader, IonEventType.CONTAINER_END, ion_type is IonType.STRUCT) + with _translate_recursion_error(): + _load(container, reader, IonEventType.CONTAINER_END, ion_type is IonType.STRUCT) yield container elif event.event_type is IonEventType.SCALAR: if event.value is None or ion_type is IonType.NULL or ion_type.is_container: @@ -480,7 +495,8 @@ def add(obj): def dump_extension(obj, fp, binary=True, sequence_as_stream=False, tuple_as_sexp=False, omit_version_marker=False): """C-extension implementation. Users should prefer to call ``dump``.""" - res = ionc.ionc_write(obj, binary, sequence_as_stream, tuple_as_sexp) + with _translate_recursion_error(): + res = ionc.ionc_write(obj, binary, sequence_as_stream, tuple_as_sexp) # TODO: support "omit_version_marker" rather than hacking. # TODO: support "trailing_commas" (support is not included in the C code). @@ -494,16 +510,21 @@ def load_extension(fp, single_value=True, parse_eagerly=True, """C-extension implementation. Users should prefer to call ``load``.""" iterator = ionc.ionc_read(fp, value_model=value_model.value, text_buffer_size_limit=text_buffer_size_limit) if single_value: - try: - value = next(iterator) - except StopIteration: - return None - try: - next(iterator) - raise IonException('Stream contained more than 1 values; expected a single value.') - except StopIteration: - pass + with _translate_recursion_error(): + try: + value = next(iterator) + except StopIteration: + return None + with _translate_recursion_error(): + try: + next(iterator) + raise IonException('Stream contained more than 1 values; expected a single value.') + except StopIteration: + pass return value if parse_eagerly: - return list(iterator) + with _translate_recursion_error(): + return list(iterator) + # The raw iterator is returned so that ionc_write can recognise its type and re-serialize the + # stream directly. A caller advancing it past the recursion limit sees a RecursionError. return iterator diff --git a/src/ion-c b/src/ion-c index 3132373af..493c12251 160000 --- a/src/ion-c +++ b/src/ion-c @@ -1 +1 @@ -Subproject commit 3132373af6e326bb9e32dbbc2310c98b70559316 +Subproject commit 493c12251be6ad93b924abb5cf8621e69f1dd2d5 diff --git a/src/ioncmodule.c b/src/ioncmodule.c index 45155f5e1..0385992f9 100644 --- a/src/ioncmodule.c +++ b/src/ioncmodule.c @@ -27,6 +27,10 @@ static char _err_msg[ERR_MSG_MAX_LEN]; #define _FAILWITHMSG(x, msg) { err = x; snprintf(_err_msg, ERR_MSG_MAX_LEN, msg); goto fail; } +// Unwinds when a Python C-API call returns NULL, having already set an exception. Yields a +// non-zero iERR to stop the IONCHECK chain while leaving that exception intact for the caller. +#define _FAILWITH_PENDING_PYERR() { err = IERR_INTERNAL_ERROR; goto fail; } + #define IONC_BYTES_FORMAT "y#" #define IONC_READ_ARGS_FORMAT "ObO" @@ -355,7 +359,11 @@ static iERR ionc_write_sequence(hWRITER writer, PyObject* sequence, PyObject* tu child_obj = PySequence_Fast_GET_ITEM(sequence, i); Py_INCREF(child_obj); - IONCHECK(Py_EnterRecursiveCall(" while writing an Ion sequence")); + // Returns a plain int, not an iERR; see ionc_read_into_container. + if (Py_EnterRecursiveCall(" while writing an Ion sequence")) { + err = IERR_STACK_OVERFLOW; + goto fail; + } err = ionc_write_value(writer, child_obj, tuple_as_sexp); Py_LeaveRecursiveCall(); IONCHECK(err); @@ -378,7 +386,7 @@ static iERR ionc_write_sequence(hWRITER writer, PyObject* sequence, PyObject* tu */ static iERR write_struct_field(hWRITER writer, PyObject* key, PyObject* val, PyObject* tuple_as_sexp) { - iERR err; + iENTER; if (PyUnicode_Check(key)) { ION_STRING field_name; ion_string_from_py(key, &field_name); @@ -386,7 +394,11 @@ static iERR write_struct_field(hWRITER writer, PyObject* key, PyObject* val, PyO } else if (key == Py_None) { IONCHECK(_ion_writer_write_field_sid_helper(writer, 0)); } - IONCHECK(Py_EnterRecursiveCall(" while writing an Ion struct")); + // Returns a plain int, not an iERR; see ionc_read_into_container. + if (Py_EnterRecursiveCall(" while writing an Ion struct")) { + err = IERR_STACK_OVERFLOW; + goto fail; + } err = ionc_write_value(writer, val, tuple_as_sexp); Py_LeaveRecursiveCall(); IONCHECK(err); @@ -863,6 +875,10 @@ static PyObject* ionc_write(PyObject *self, PyObject *args, PyObject *kwds) { Py_XDECREF(sequence_as_stream); Py_XDECREF(tuple_as_sexp); + // Reported as-is and translated in simpleion.py; see ionc_read_iter_next. + if (PyErr_ExceptionMatches(PyExc_RecursionError)) { + return NULL; + } PyObject* exception = NULL; if (err == IERR_INVALID_STATE) { exception = PyErr_Format(PyExc_TypeError, "%s", _err_msg); @@ -870,7 +886,6 @@ static PyObject* ionc_write(PyObject *self, PyObject *args, PyObject *kwds) { else { exception = PyErr_Format(_ion_exception_cls, "%s %s", ion_error_to_str(err), _err_msg); } - _err_msg[0] = '\0'; return exception; } @@ -1005,7 +1020,12 @@ static iERR ionc_read_timestamp(hREADER hreader, PyObject** timestamp_out) { static iERR ionc_read_into_container(hREADER hreader, PyObject* container, enum ContainerType parent_type, uint8_t value_model) { iENTER; IONCHECK(ion_reader_step_in(hreader)); - IONCHECK(Py_EnterRecursiveCall(" while reading an Ion container")); + // Py_EnterRecursiveCall reports failure with a plain non-zero int and sets a RecursionError. + // That value is not an iERR, so it is checked here rather than handed to IONCHECK. + if (Py_EnterRecursiveCall(" while reading an Ion container")) { + err = IERR_STACK_OVERFLOW; + goto fail; + } err = ionc_read_all(hreader, container, parent_type, value_model); Py_LeaveRecursiveCall(); IONCHECK(err); @@ -1022,32 +1042,49 @@ static iERR ionc_read_into_container(hREADER hreader, PyObject* container, enum * container_type: Type of container to add to. * field_name: The field name of the element if it is inside a struct */ -static void ionc_add_to_container(PyObject* pyContainer, PyObject* element, enum ContainerType container_type, PyObject* field_name) { +static iERR ionc_add_to_container(PyObject* pyContainer, PyObject* element, enum ContainerType container_type, PyObject* field_name) { + iENTER; + // A NULL element means its construction already failed and set an exception. Reporting that + // exception is more useful than the SystemError PyList_Append/PyDict_SetItem would raise. + if (element == NULL) { + _FAILWITH_PENDING_PYERR(); + } switch (container_type) { case MULTIMAP: { // this builds the "hash-map of lists" structure that the IonPyDict object // expects for its __store PyObject* empty = PyList_New(0); + if (empty == NULL) { + _FAILWITH_PENDING_PYERR(); + } // SetDefault performs get|set with a single hash of the key PyObject* found = PyDict_SetDefault(pyContainer, field_name, empty); - PyList_Append(found, element); - + if (found == NULL || PyList_Append(found, element) < 0) { + Py_DECREF(empty); + _FAILWITH_PENDING_PYERR(); + } Py_DECREF(empty); break; } case STD_DICT: { - PyDict_SetItem(pyContainer, field_name, element); + if (PyDict_SetItem(pyContainer, field_name, element) < 0) { + _FAILWITH_PENDING_PYERR(); + } break; } case LIST: { - PyList_Append(pyContainer, (PyObject*)element); + if (PyList_Append(pyContainer, (PyObject*)element) < 0) { + _FAILWITH_PENDING_PYERR(); + } break; } } +fail: Py_XDECREF(element); + cRETURN; } /* @@ -1285,6 +1322,10 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, enum Cont wrap_py_value = TRUE; container_type = MULTIMAP; } + // Runs Python code once per nesting level, so it can fail with a RecursionError set. + if (py_value == NULL) { + _FAILWITH_PENDING_PYERR(); + } IONCHECK(ionc_read_into_container(hreader, py_value, container_type, value_model)); break; @@ -1309,6 +1350,10 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, enum Cont } else { py_value = PyList_New(0); } + // Runs Python code once per nesting level, so it can fail with a RecursionError set. + if (py_value == NULL) { + _FAILWITH_PENDING_PYERR(); + } IONCHECK(ionc_read_into_container(hreader, py_value, LIST, value_model)); ion_nature_constructor = _ionpylist_fromvalue; break; @@ -1328,9 +1373,14 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, enum Cont NULL ); Py_XDECREF(py_value); + py_value = NULL; + // Wrapping runs Python code too, so it can fail with a RecursionError set. + if (final_py_value == NULL) { + _FAILWITH_PENDING_PYERR(); + } } - ionc_add_to_container(container, final_py_value, parent_type, py_field_name); + IONCHECK(ionc_add_to_container(container, final_py_value, parent_type, py_field_name)); fail: Py_XDECREF(py_annotations); @@ -1456,6 +1506,11 @@ PyObject* ionc_read_iter_next(PyObject *self) { fail: Py_XDECREF(container); + // A RecursionError is reported as-is; simpleion.py translates it into an IonException. Any + // other pending exception is superseded by the IonException built below. + if (PyErr_ExceptionMatches(PyExc_RecursionError)) { + return NULL; + } PyObject* exception = PyErr_Format(_ion_exception_cls, "%s %s", ion_error_to_str(err), _err_msg); _err_msg[0] = '\0'; return exception; diff --git a/tests/test_simpleion.py b/tests/test_simpleion.py index 972c3c8c0..3a37b9d73 100644 --- a/tests/test_simpleion.py +++ b/tests/test_simpleion.py @@ -20,13 +20,14 @@ from math import isnan import re +import sys from typing import NamedTuple, Any, Sequence, Optional -from pytest import raises +from pytest import raises, mark from amazon.ion import simpleion from amazon.ion.exceptions import IonException -from amazon.ion.symbols import SymbolToken, SYSTEM_SYMBOL_TABLE +from amazon.ion.symbols import SymbolToken, SYSTEM_SYMBOL_TABLE, SymbolTableCatalog from amazon.ion.writer_binary import _IVM from amazon.ion.core import IonType, IonEvent, IonEventType, OffsetTZInfo, Multimap, TimestampPrecision, Timestamp from amazon.ion.simple_types import IonPyDict, IonPyText, IonPyList, IonPyNull, IonPyBool, IonPyInt, IonPyFloat, \ @@ -911,3 +912,119 @@ def test_ion_py_objects_construction(v): So, we compare if they represent the same instant; in other words, we set timestamps_instants_only to True. """ assert True is ion_equals(v[0], v[1], timestamps_instants_only=True) + +def test_loads_deeply_nested_list_text_raises_ion_exception(): + text = "[" * 5000 + "]" * 5000 + with raises(IonException): + loads(text, catalog=SymbolTableCatalog()) + + +def _build_nested_annotation_binary(depth): + """Builds `depth` nested annotation wrappers around a null.null, to exercise + reader_binary.py's _annotation_handler -> _tlv_parser recursion.""" + def varuint(n): + bs = [n & 0x7f] + n >>= 7 + while n: + bs.append(n & 0x7f) + n >>= 7 + bs.reverse() + bs[-1] |= 0x80 + return bytes(bs) + + def wrap_annot(content, annot_bytes): + inner = varuint(len(annot_bytes)) + annot_bytes + content + length = len(inner) + if length < 14: + return bytes([0xE0 | length]) + inner + return bytes([0xEE]) + varuint(length) + inner + + annot_sym = varuint(10) # annotate with symbol id 10 (arbitrary) + content = bytes([0x0F]) # innermost value: null.null + for _ in range(depth): + content = wrap_annot(content, annot_sym) + + ivm = bytes([0xE0, 0x01, 0x00, 0xEA]) # Ion 1.0 version marker + return ivm + content + + +def test_load_deeply_nested_annotations_binary_raises_ion_exception(): + data = _build_nested_annotation_binary(3000) + buf = BytesIO(data) + with raises(IonException): + load(buf, single_value=True, catalog=SymbolTableCatalog()) + + +def test_deeply_nested_recursion_is_recoverable_and_wraps_recursion_error(): + text = "[" * 5000 + "]" * 5000 + with raises(IonException) as exc_info: + loads(text, catalog=SymbolTableCatalog()) + assert isinstance(exc_info.value.__cause__, RecursionError) + # Rejecting the input must leave the interpreter able to handle well-formed values. + assert loads("[0]", catalog=SymbolTableCatalog()) == [0] + + +def test_load_iteratively_deeply_nested_list_raises_ion_exception(): + text = "[" * 5000 + "]" * 5000 + it = loads(text, catalog=SymbolTableCatalog(), parse_eagerly=False, single_value=False) + with raises(IonException): + next(it) + + +def _build_nested_list(depth): + """Builds a `depth`-deep nested list, for use as input to dump/dumps.""" + obj = [] + cur = obj + for _ in range(depth - 1): + child = [] + cur.append(child) + cur = child + return obj + + +@parametrize(True, False) +def test_dumps_deeply_nested_list_pure_python_raises_ion_exception(binary): + # imports=[] selects dump_python even when the C extension is available. + with raises(IonException) as exc_info: + dumps(_build_nested_list(5000), binary=binary, imports=[]) + assert isinstance(exc_info.value.__cause__, RecursionError) + # Rejecting the input must leave the interpreter able to handle well-formed values. + assert dumps([0], binary=False, imports=[]) == '$ion_1_0 [0]' + + +@mark.skipif(not c_ext, reason="C extension is not available in this environment.") +def test_c_extension_loads_deeply_nested_list_text_raises_ion_exception(): + # Omitting catalog selects the C-extension read path. + text = "[" * 5000 + "]" * 5000 + with raises(IonException): + loads(text) + + +@mark.skipif(not c_ext, reason="C extension is not available in this environment.") +def test_c_extension_dumps_deeply_nested_list_raises_ion_exception(): + # Omitting imports and indent selects the C-extension write path. + with raises(IonException): + dumps(_build_nested_list(5000), binary=False) + + +@mark.skipif(not c_ext, reason="C extension is not available in this environment.") +def test_c_extension_loads_at_recursion_limit_boundary_raises_ion_exception(): + """Just under the recursion limit, the Python calls the C extension makes per container level + are what fail, rather than ion-c's own max_container_depth check rejecting the input.""" + depth = sys.getrecursionlimit() - 1 + text = "[" * depth + "]" * depth + with raises(IonException) as exc_info: + loads(text) + assert isinstance(exc_info.value.__cause__, RecursionError) + + +@mark.skipif(not c_ext, reason="C extension is not available in this environment.") +def test_c_extension_recursion_is_recoverable_and_wraps_recursion_error(): + """The RecursionError is preserved as the cause, and the interpreter survives to keep + serializing well-formed values.""" + with raises(IonException) as exc_info: + dumps(_build_nested_list(5000), binary=False) + assert isinstance(exc_info.value.__cause__, RecursionError) + # Rejecting the input must leave the interpreter able to handle well-formed values. + assert loads("[0]") == [0] + assert dumps([0], binary=False) == '$ion_1_0 [0]' From ccbd283ad5e9640b2f422bef9b68f7621b0951c3 Mon Sep 17 00:00:00 2001 From: Deirdre Wang Date: Wed, 9 Sep 2026 16:15:39 -0700 Subject: [PATCH 2/3] test: Make recursion tests version-independent --- src-python/amazon/ion/simpleion.py | 3 ++- tests/test_simpleion.py | 19 +++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src-python/amazon/ion/simpleion.py b/src-python/amazon/ion/simpleion.py index 1857e199e..7412d08c3 100644 --- a/src-python/amazon/ion/simpleion.py +++ b/src-python/amazon/ion/simpleion.py @@ -526,5 +526,6 @@ def load_extension(fp, single_value=True, parse_eagerly=True, with _translate_recursion_error(): return list(iterator) # The raw iterator is returned so that ionc_write can recognise its type and re-serialize the - # stream directly. A caller advancing it past the recursion limit sees a RecursionError. + # stream directly. A caller advancing it past the recursion limit may see a RecursionError + # instead of an IonException, depending on which depth limit the runtime reaches first. return iterator diff --git a/tests/test_simpleion.py b/tests/test_simpleion.py index 3a37b9d73..611262e6c 100644 --- a/tests/test_simpleion.py +++ b/tests/test_simpleion.py @@ -1008,9 +1008,13 @@ def test_c_extension_dumps_deeply_nested_list_raises_ion_exception(): @mark.skipif(not c_ext, reason="C extension is not available in this environment.") -def test_c_extension_loads_at_recursion_limit_boundary_raises_ion_exception(): - """Just under the recursion limit, the Python calls the C extension makes per container level - are what fail, rather than ion-c's own max_container_depth check rejecting the input.""" +@mark.skipif(sys.version_info >= (3, 12), + reason="Before 3.12 the C extension's per-level Python calls draw on the same limit " + "as Python frames, so they fail just under it. From 3.12 they have their own, " + "much higher limit and ion-c's max_container_depth is always reached first.") +def test_c_extension_loads_just_under_recursion_limit_raises_ion_exception(): + """The per-level Python calls the C extension makes are what fail here, rather than ion-c's + max_container_depth check rejecting the input.""" depth = sys.getrecursionlimit() - 1 text = "[" * depth + "]" * depth with raises(IonException) as exc_info: @@ -1019,12 +1023,11 @@ def test_c_extension_loads_at_recursion_limit_boundary_raises_ion_exception(): @mark.skipif(not c_ext, reason="C extension is not available in this environment.") -def test_c_extension_recursion_is_recoverable_and_wraps_recursion_error(): - """The RecursionError is preserved as the cause, and the interpreter survives to keep - serializing well-formed values.""" - with raises(IonException) as exc_info: +def test_c_extension_recursion_is_recoverable(): + """Whichever depth limit rejects the input, the interpreter survives to keep serializing + well-formed values.""" + with raises(IonException): dumps(_build_nested_list(5000), binary=False) - assert isinstance(exc_info.value.__cause__, RecursionError) # Rejecting the input must leave the interpreter able to handle well-formed values. assert loads("[0]") == [0] assert dumps([0], binary=False) == '$ion_1_0 [0]' From 2d292e0d7ec3bd7f730ab7ec148059d8c307b6c3 Mon Sep 17 00:00:00 2001 From: Deirdre Wang Date: Thu, 10 Sep 2026 01:19:43 -0700 Subject: [PATCH 3/3] Raise IonException for deep nesting on every read and write path --- src-python/amazon/ion/simpleion.py | 23 ++++++++++++++++++----- src/ioncmodule.c | 4 +++- tests/test_simpleion.py | 9 +++++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src-python/amazon/ion/simpleion.py b/src-python/amazon/ion/simpleion.py index 7412d08c3..12336f186 100644 --- a/src-python/amazon/ion/simpleion.py +++ b/src-python/amazon/ion/simpleion.py @@ -363,8 +363,11 @@ def _dump(obj, writer, from_type, field=None, in_struct=False, depth=0): ion_type = obj.ion_type ion_nature = True except AttributeError: - ion_type = _ion_type(obj, from_type) ion_nature = False + if not ion_nature: + # Resolved outside the handler above: a RecursionError raised here while the AttributeError + # is still in flight has to record it as its context, which PyPy cannot do out of stack. + ion_type = _ion_type(obj, from_type) if ion_type is None: raise IonException('Value must have a non-None ion_type: %s, depth: %d, field: %s' % (repr(obj), depth, field)) if not null and ion_type.is_container: @@ -505,6 +508,17 @@ def dump_extension(obj, fp, binary=True, sequence_as_stream=False, tuple_as_sexp fp.write(res) +def _translate_recursion_error_iter(iterator): + """Yields from ``iterator``, reporting nesting too deep to process as an IonException.""" + while True: + with _translate_recursion_error(): + try: + value = next(iterator) + except StopIteration: + return + yield value + + def load_extension(fp, single_value=True, parse_eagerly=True, text_buffer_size_limit=None, value_model=IonPyValueModel.ION_PY): """C-extension implementation. Users should prefer to call ``load``.""" @@ -525,7 +539,6 @@ def load_extension(fp, single_value=True, parse_eagerly=True, if parse_eagerly: with _translate_recursion_error(): return list(iterator) - # The raw iterator is returned so that ionc_write can recognise its type and re-serialize the - # stream directly. A caller advancing it past the recursion limit may see a RecursionError - # instead of an IonException, depending on which depth limit the runtime reaches first. - return iterator + # Wrapped so that a caller advancing the iterator themselves also gets an IonException. The + # wrapper is a generator, which ionc_write writes as a stream just like the raw iterator. + return _translate_recursion_error_iter(iterator) diff --git a/src/ioncmodule.c b/src/ioncmodule.c index 0385992f9..b53258d78 100644 --- a/src/ioncmodule.c +++ b/src/ioncmodule.c @@ -810,7 +810,9 @@ static PyObject* ionc_write(PyObject *self, PyObject *args, PyObject *kwds) { options.max_annotation_count = ANNOTATION_MAX_LEN; IONCHECK(ion_writer_open(&writer, ion_stream, &options)); - if (Py_TYPE(obj) == &ionc_read_IteratorType) { + // Generators are written as a stream of top-level values, matching dump_python, and covering + // the read iterator once simpleion has wrapped it to translate errors. + if (Py_TYPE(obj) == &ionc_read_IteratorType || PyGen_Check(obj)) { PyObject *item; while ((item = PyIter_Next(obj)) != NULL) { err = ionc_write_value(writer, item, tuple_as_sexp); diff --git a/tests/test_simpleion.py b/tests/test_simpleion.py index 611262e6c..3cf264143 100644 --- a/tests/test_simpleion.py +++ b/tests/test_simpleion.py @@ -1000,6 +1000,15 @@ def test_c_extension_loads_deeply_nested_list_text_raises_ion_exception(): loads(text) +@mark.skipif(not c_ext, reason="C extension is not available in this environment.") +def test_c_extension_load_iteratively_deeply_nested_list_raises_ion_exception(): + # The caller advances this iterator themselves, outside any of load_extension's own calls. + text = "[" * 5000 + "]" * 5000 + it = loads(text, parse_eagerly=False, single_value=False) + with raises(IonException): + next(it) + + @mark.skipif(not c_ext, reason="C extension is not available in this environment.") def test_c_extension_dumps_deeply_nested_list_raises_ion_exception(): # Omitting imports and indent selects the C-extension write path.