Skip to content

Commit 6a9fa39

Browse files
kumaraditya303akx
andauthored
[3.15] gh-156310: Make the iter() sequence fallback iterator safe in free-threaded build (GH-156311) (#162006)
(cherry picked from commit b707ce6) Co-authored-by: Aarni Koskela <akx@iki.fi>
1 parent fc152df commit 6a9fa39

4 files changed

Lines changed: 165 additions & 15 deletions

File tree

‎Lib/test/test_free_threading/test_iteration.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1+
import sys
12
import threading
23
import unittest
34
from test import support
5+
from test.support import threading_helper
46

57
# The race conditions these tests were written for only happen every now and
68
# then, even with the current numbers. To find rare race conditions, bumping
@@ -112,6 +114,62 @@ def worker():
112114
self.assert_iterator_results(results, list(seq))
113115

114116

117+
class ContendedSeqIterExhaustionTest(unittest.TestCase):
118+
"""Test draining a shared iter() fallback iterator (PySeqIter_Type).
119+
120+
Sequences implementing __getitem__ but not __iter__ iterate through
121+
PySeqIter_Type. Unlike the other tests in this file, this uses a
122+
tiny sequence and many rounds so that many threads reach the racy
123+
exhaustion path simultaneously (see gh-156310, where this
124+
use-after-freed the sequence).
125+
"""
126+
127+
class Seq:
128+
def __init__(self, n):
129+
self.n = n
130+
131+
def __getitem__(self, i):
132+
if i >= self.n:
133+
raise IndexError(i)
134+
return i
135+
136+
@support.refcount_test
137+
def test_shared_iterator_exhaustion(self):
138+
nthreads = 8
139+
nrounds = 20 if support.check_sanitizer(thread=True) else 100
140+
seq = self.Seq(4)
141+
expected = set(range(seq.n))
142+
refcount_before = sys.getrefcount(seq)
143+
144+
def drain(it, barrier, results):
145+
items = []
146+
barrier.wait()
147+
for item in it:
148+
items.append(item)
149+
results.extend(items)
150+
151+
for _ in range(nrounds):
152+
it = iter(seq)
153+
barrier = threading.Barrier(nthreads)
154+
results = []
155+
threads = [
156+
threading.Thread(target=drain, args=(it, barrier, results))
157+
for _ in range(nthreads)
158+
]
159+
with threading_helper.catch_threading_exception() as cm:
160+
with threading_helper.start_threads(threads):
161+
pass
162+
self.assertIsNone(cm.exc_value)
163+
del it
164+
# Threads may see duplicate or missing items, but never
165+
# invented ones.
166+
self.assertEqual(set(results) - expected, set())
167+
168+
# A double-DECREF of the sequence does not always crash; it
169+
# reliably shows up as a sagging reference count.
170+
self.assertEqual(sys.getrefcount(seq), refcount_before)
171+
172+
115173
class ContendedRangeIterationTest(ContendedTupleIterationTest):
116174
def make_testdata(self, n):
117175
return range(n)

‎Lib/test/test_iter.py‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import sys
44
import unittest
5+
from test import support
56
from test.support import cpython_only
67
from test.support.os_helper import TESTFN, unlink
78
from test.support import check_free_after_iterating, ALWAYS_EQ, NEVER_EQ
@@ -249,6 +250,71 @@ def test_mutating_seq_class_exhausted_iter(self):
249250
self.assertEqual(list(empit), [5, 6])
250251
self.assertEqual(list(a), [0, 1, 2, 3, 4, 5, 6])
251252

253+
@support.refcount_test
254+
def test_seq_class_reentrant_exhaustion(self):
255+
# gh-156310: a re-entrant next() from inside __getitem__ (or from
256+
# __del__ of the IndexError instance) that exhausts the iterator
257+
# used to make the outer next() DECREF the sequence a second time.
258+
it = None
259+
260+
class ReentrantGetItem:
261+
def __init__(self):
262+
self.calls = 0
263+
264+
def __getitem__(self, i):
265+
self.calls += 1
266+
if self.calls == 1:
267+
for _ in it:
268+
pass
269+
raise IndexError(i)
270+
271+
seq = ReentrantGetItem()
272+
refcount = sys.getrefcount(seq)
273+
it = iter(seq)
274+
self.assertEqual(list(it), [])
275+
del it
276+
support.gc_collect()
277+
self.assertEqual(sys.getrefcount(seq), refcount)
278+
279+
class ReentrantIndexError(IndexError):
280+
def __del__(self):
281+
try:
282+
next(it)
283+
except StopIteration:
284+
pass
285+
286+
class RaiseReentrant:
287+
def __getitem__(self, i):
288+
raise ReentrantIndexError(i)
289+
290+
seq = RaiseReentrant()
291+
refcount = sys.getrefcount(seq)
292+
it = iter(seq)
293+
self.assertEqual(list(it), [])
294+
del it
295+
support.gc_collect()
296+
self.assertEqual(sys.getrefcount(seq), refcount)
297+
298+
# An outer __getitem__ that succeeds after a re-entrant next()
299+
# exhausted the iterator must not revive it.
300+
class ReviveGetItem:
301+
def __init__(self):
302+
self.calls = 0
303+
304+
def __getitem__(self, i):
305+
self.calls += 1
306+
if self.calls == 1:
307+
for _ in it:
308+
pass
309+
if i >= 3:
310+
raise IndexError(i)
311+
return i
312+
313+
it = iter(ReviveGetItem())
314+
self.assertEqual(next(it), 0)
315+
self.assertEqual(list(it), [])
316+
self.assertEqual(it.__length_hint__(), 0)
317+
252318
def test_reduce_mutating_builtins_iter(self):
253319
# This is a reproducer of issue #101765
254320
# where iter `__reduce__` calls could lead to a segfault or SystemError
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Fix memory safety issues in the :func:`iter` fallback for objects that
2+
implement :meth:`~object.__getitem__` without :meth:`~object.__iter__`
3+
(``PySeqIter_Type``). Sharing an iterator between threads in the
4+
free-threaded build could use the underlying sequence after it was freed,
5+
and re-entrant exhaustion in the default build could decrement the sequence's
6+
reference count twice. Concurrent iteration may still see duplicate or
7+
missing items, but it no longer corrupts the interpreter state.

‎Objects/iterobject.c‎

Lines changed: 34 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,14 @@
66
#include "pycore_ceval.h" // _PyEval_GetBuiltin()
77
#include "pycore_genobject.h" // _PyCoro_GetAwaitableIter()
88
#include "pycore_object.h" // _PyObject_GC_TRACK()
9+
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_RELAXED()
910

1011

1112
typedef struct {
1213
PyObject_HEAD
13-
Py_ssize_t it_index;
14-
PyObject *it_seq; /* Set to NULL when iterator is exhausted */
14+
Py_ssize_t it_index; /* -1 when iterator is exhausted */
15+
PyObject *it_seq; /* Set to NULL when iterator is exhausted
16+
(in the default build) */
1517
} seqiterobject;
1618

1719
PyObject *
@@ -58,26 +60,41 @@ iter_iternext(PyObject *iterator)
5860

5961
assert(PySeqIter_Check(iterator));
6062
it = (seqiterobject *)iterator;
63+
Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index);
64+
if (index < 0)
65+
return NULL;
6166
seq = it->it_seq;
67+
#ifndef Py_GIL_DISABLED
6268
if (seq == NULL)
6369
return NULL;
64-
if (it->it_index == PY_SSIZE_T_MAX) {
70+
#endif
71+
if (index == PY_SSIZE_T_MAX) {
6572
PyErr_SetString(PyExc_OverflowError,
6673
"iter index too large");
6774
return NULL;
6875
}
6976

70-
result = PySequence_GetItem(seq, it->it_index);
77+
result = PySequence_GetItem(seq, index);
7178
if (result != NULL) {
72-
it->it_index++;
79+
/* PySequence_GetItem() can exhaust the iterator re-entrantly.
80+
* Preserve the exhaustion sentinel if it is observed. Concurrent
81+
* exhaustion can still race with the store, but remains memory-safe
82+
* because the sequence stays alive. */
83+
if (FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index) >= 0) {
84+
FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, index + 1);
85+
}
7386
return result;
7487
}
7588
if (PyErr_ExceptionMatches(PyExc_IndexError) ||
7689
PyErr_ExceptionMatches(PyExc_StopIteration))
7790
{
91+
/* Mark the iterator exhausted before anything that can run
92+
* arbitrary code. */
93+
FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, -1);
94+
#ifndef Py_GIL_DISABLED
95+
Py_CLEAR(it->it_seq);
96+
#endif
7897
PyErr_Clear();
79-
it->it_seq = NULL;
80-
Py_DECREF(seq);
8198
}
8299
return NULL;
83100
}
@@ -88,7 +105,8 @@ iter_len(PyObject *op, PyObject *Py_UNUSED(ignored))
88105
seqiterobject *it = (seqiterobject*)op;
89106
Py_ssize_t seqsize, len;
90107

91-
if (it->it_seq) {
108+
Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index);
109+
if (index >= 0 && it->it_seq != NULL) {
92110
if (_PyObject_HasLen(it->it_seq)) {
93111
seqsize = PySequence_Size(it->it_seq);
94112
if (seqsize == -1)
@@ -97,7 +115,7 @@ iter_len(PyObject *op, PyObject *Py_UNUSED(ignored))
97115
else {
98116
Py_RETURN_NOTIMPLEMENTED;
99117
}
100-
len = seqsize - it->it_index;
118+
len = seqsize - index;
101119
if (len >= 0)
102120
return PyLong_FromSsize_t(len);
103121
}
@@ -116,8 +134,9 @@ iter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored))
116134
* call must be before access of iterator pointers.
117135
* see issue #101765 */
118136

119-
if (it->it_seq != NULL)
120-
return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index);
137+
Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index);
138+
if (index >= 0 && it->it_seq != NULL)
139+
return Py_BuildValue("N(O)n", iter, it->it_seq, index);
121140
else
122141
return Py_BuildValue("N(())", iter);
123142
}
@@ -131,10 +150,10 @@ iter_setstate(PyObject *op, PyObject *state)
131150
Py_ssize_t index = PyLong_AsSsize_t(state);
132151
if (index == -1 && PyErr_Occurred())
133152
return NULL;
134-
if (it->it_seq != NULL) {
135-
if (index < 0)
136-
index = 0;
137-
it->it_index = index;
153+
if (index < 0)
154+
index = 0;
155+
if (it->it_seq && FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index) >= 0) {
156+
FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, index);
138157
}
139158
Py_RETURN_NONE;
140159
}

0 commit comments

Comments
 (0)