Skip to content

Commit 66da5fc

Browse files
eendebakptclaude
andcommitted
pythongh-149557: Raise RuntimeError on re-entrant itertools.pairwise.__next__
Add a per-iterator `running` flag, set/cleared via atomic test-and-set, so a recursive or concurrent call to `pairwise.__next__` raises RuntimeError instead of crashing or producing garbled output. Matches the long-standing behavior of `itertools.tee` and generators. With re-entry blocked, the defensive re-read guard and incref/decref pair added by pythongh-109788 are no longer needed and have been removed. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
1 parent ef2246f commit 66da5fc

4 files changed

Lines changed: 130 additions & 72 deletions

File tree

‎Lib/test/test_free_threading/test_itertools.py‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1+
import threading
12
import unittest
2-
from itertools import accumulate, batched, chain, combinations_with_replacement, cycle, permutations, zip_longest
3+
from itertools import accumulate, batched, chain, combinations_with_replacement, cycle, pairwise, permutations, zip_longest
34
from test.support import threading_helper
45

56

@@ -69,6 +70,64 @@ def test_zip_longest(self):
6970
it = zip_longest(list(range(4)), list(range(8)), fillvalue=0)
7071
threading_helper.run_concurrently(work_iterator, nthreads=10, args=[it])
7172

73+
@threading_helper.reap_threads
74+
def test_pairwise(self):
75+
# gh-149557: stress concurrent iteration over a shared pairwise.
76+
# The running-flag guard makes the loser thread raise RuntimeError
77+
# rather than serialize, so collected pairs may be a sparse subset
78+
# of the full sequence, but every pair must be (k, k+1) and no
79+
# pair may appear twice.
80+
N = 1000
81+
for _ in range(6):
82+
it = pairwise(iter(range(N)))
83+
results = []
84+
results_lock = threading.Lock()
85+
86+
def worker(it):
87+
while True:
88+
try:
89+
v = next(it)
90+
except StopIteration:
91+
return
92+
except RuntimeError:
93+
continue
94+
with results_lock:
95+
results.append(v)
96+
97+
threading_helper.run_concurrently(worker, nthreads=6, args=[it])
98+
99+
self.assertEqual(len(results), len(set(results)))
100+
for a, b in results:
101+
self.assertEqual(b, a + 1)
102+
self.assertGreaterEqual(a, 0)
103+
self.assertLess(b, N)
104+
105+
@threading_helper.reap_threads
106+
def test_pairwise_concurrent_raises(self):
107+
# gh-149557: a second concurrent caller observes the running flag
108+
# already set and must raise RuntimeError rather than crash or
109+
# deadlock. Mirrors test_tee_concurrent in test_itertools.
110+
start = threading.Event()
111+
finish = threading.Event()
112+
113+
class I:
114+
def __iter__(self):
115+
return self
116+
def __next__(self):
117+
start.set()
118+
finish.wait()
119+
120+
it = pairwise(I())
121+
thread = threading.Thread(target=next, args=[it])
122+
thread.start()
123+
try:
124+
start.wait()
125+
with self.assertRaisesRegex(RuntimeError, "pairwise"):
126+
next(it)
127+
finally:
128+
finish.set()
129+
thread.join()
130+
72131

73132
if __name__ == "__main__":
74133
unittest.main()

‎Lib/test/test_itertools.py‎

Lines changed: 37 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -968,76 +968,55 @@ def test_pairwise(self):
968968
pairwise(None) # non-iterable argument
969969

970970
def test_pairwise_reenter(self):
971-
def check(reenter_at, expected):
971+
# gh-149557: re-entering pairwise.__next__ from within the input
972+
# iterator's __next__ now raises RuntimeError, matching the behavior
973+
# of itertools.tee and of generators. Previously, depending on the
974+
# ref-count topology of the input iterator, this could either
975+
# silently produce nested-tuple output or crash via use-after-free
976+
# (see gh-149557 for the crash reproducer).
977+
def check(reenter_at):
972978
class I:
973979
count = 0
974980
def __iter__(self):
975981
return self
976982
def __next__(self):
977-
self.count +=1
983+
self.count += 1
978984
if self.count in reenter_at:
979985
return next(it)
980986
return [self.count] # new object
981987

982988
it = pairwise(I())
983-
for item in expected:
984-
self.assertEqual(next(it), item)
985-
986-
check({1}, [
987-
(([2], [3]), [4]),
988-
([4], [5]),
989-
])
990-
check({2}, [
991-
([1], ([1], [3])),
992-
(([1], [3]), [4]),
993-
([4], [5]),
994-
])
995-
check({3}, [
996-
([1], [2]),
997-
([2], ([2], [4])),
998-
(([2], [4]), [5]),
999-
([5], [6]),
1000-
])
1001-
check({1, 2}, [
1002-
((([3], [4]), [5]), [6]),
1003-
([6], [7]),
1004-
])
1005-
check({1, 3}, [
1006-
(([2], ([2], [4])), [5]),
1007-
([5], [6]),
1008-
])
1009-
check({1, 4}, [
1010-
(([2], [3]), (([2], [3]), [5])),
1011-
((([2], [3]), [5]), [6]),
1012-
([6], [7]),
1013-
])
1014-
check({2, 3}, [
1015-
([1], ([1], ([1], [4]))),
1016-
(([1], ([1], [4])), [5]),
1017-
([5], [6]),
1018-
])
989+
with self.assertRaisesRegex(RuntimeError, "pairwise"):
990+
# Iterate to exhaustion or error. Wrapping in list() lets
991+
# the assertion fire whether the error surfaces on the
992+
# first next() or only after a successful pair.
993+
list(it)
1019994

1020-
def test_pairwise_reenter2(self):
1021-
def check(maxcount, expected):
1022-
class I:
1023-
count = 0
1024-
def __iter__(self):
1025-
return self
1026-
def __next__(self):
1027-
if self.count >= maxcount:
1028-
raise StopIteration
1029-
self.count +=1
1030-
if self.count == 1:
1031-
return next(it, None)
1032-
return [self.count] # new object
995+
# Re-entry on any of the first few __next__ calls must raise.
996+
for reenter_at in [{1}, {2}, {3}, {1, 2}, {1, 3}, {1, 4}, {2, 3}]:
997+
with self.subTest(reenter_at=reenter_at):
998+
check(reenter_at)
1033999

1034-
it = pairwise(I())
1035-
self.assertEqual(list(it), expected)
1036-
1037-
check(1, [])
1038-
check(2, [])
1039-
check(3, [])
1040-
check(4, [(([2], [3]), [4])])
1000+
def test_pairwise_reenter2(self):
1001+
# gh-149557: variant of test_pairwise_reenter where the re-entrant
1002+
# call uses next(it, None) (i.e. supplies a default). Even with a
1003+
# default that suppresses StopIteration, the re-entry itself must
1004+
# raise RuntimeError, propagating out of the outer pairwise call.
1005+
class I:
1006+
count = 0
1007+
def __iter__(self):
1008+
return self
1009+
def __next__(self):
1010+
if self.count >= 4:
1011+
raise StopIteration
1012+
self.count += 1
1013+
if self.count == 1:
1014+
return next(it, None)
1015+
return [self.count] # new object
1016+
1017+
it = pairwise(I())
1018+
with self.assertRaisesRegex(RuntimeError, "pairwise"):
1019+
list(it)
10411020

10421021
def test_product(self):
10431022
for args, result in [
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
:class:`itertools.pairwise` now raises :exc:`RuntimeError` when its
2+
``__next__`` method is called recursively while it is already executing,
3+
matching the behavior of generators and :class:`itertools.tee`. This
4+
fixes a use-after-free crash that could occur when the input iterator
5+
re-entered the :class:`!pairwise` object. Patch by Pieter Eendebak.

‎Modules/itertoolsmodule.c‎

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -278,6 +278,7 @@ typedef struct {
278278
PyObject *it;
279279
PyObject *old;
280280
PyObject *result;
281+
uint8_t running;
281282
} pairwiseobject;
282283

283284
#define pairwiseobject_CAST(op) ((pairwiseobject *)(op))
@@ -347,33 +348,45 @@ static PyObject *
347348
pairwise_next(PyObject *op)
348349
{
349350
pairwiseobject *po = pairwiseobject_CAST(op);
351+
352+
// Re-entrancy / concurrency guard. The atomic exchange both detects
353+
// same-thread re-entry (e.g. via the input iterator's __next__ calling
354+
// next() on this pairwise object) and prevents two threads from racing
355+
// through the body in the free-threaded build. The losing caller raises
356+
// instead of waiting, matching the running-flag pattern already used by
357+
// itertools.tee (see teedataobject_getitem) and by generators. Because
358+
// re-entry is blocked, po->old and po->it cannot change underneath us,
359+
// so we can use po->old as a borrowed reference for the duration of the
360+
// call.
361+
if (_Py_atomic_exchange_uint8(&po->running, 1)) {
362+
PyErr_SetString(PyExc_RuntimeError,
363+
"pairwise() iterator already executing");
364+
return NULL;
365+
}
366+
350367
PyObject *it = po->it;
351368
PyObject *old = po->old;
352369
PyObject *new, *result;
353370

354371
if (it == NULL) {
355-
return NULL;
372+
result = NULL;
373+
goto done;
356374
}
357375
if (old == NULL) {
358376
old = (*Py_TYPE(it)->tp_iternext)(it);
359-
Py_XSETREF(po->old, old);
360377
if (old == NULL) {
361378
Py_CLEAR(po->it);
362-
return NULL;
363-
}
364-
it = po->it;
365-
if (it == NULL) {
366-
Py_CLEAR(po->old);
367-
return NULL;
379+
result = NULL;
380+
goto done;
368381
}
382+
po->old = old; // po->old was NULL; no decref needed
369383
}
370-
Py_INCREF(old);
371384
new = (*Py_TYPE(it)->tp_iternext)(it);
372385
if (new == NULL) {
373386
Py_CLEAR(po->it);
374387
Py_CLEAR(po->old);
375-
Py_DECREF(old);
376-
return NULL;
388+
result = NULL;
389+
goto done;
377390
}
378391

379392
result = po->result;
@@ -393,8 +406,10 @@ pairwise_next(PyObject *op)
393406
result = _PyTuple_FromPair(old, new);
394407
}
395408

396-
Py_XSETREF(po->old, new);
397-
Py_DECREF(old);
409+
Py_SETREF(po->old, new); // po->old == old, known non-NULL
410+
411+
done:
412+
_Py_atomic_store_uint8(&po->running, 0);
398413
return result;
399414
}
400415

0 commit comments

Comments
 (0)