gh-160573: Drop the set once when a shared iterator is exhausted (#160619) Co-authored-by: Kumar Aditya <kumaraditya@python.org>
diff --git a/Lib/test/test_free_threading/test_set.py b/Lib/test/test_free_threading/test_set.py index 9dd3d68..639bc72 100644 --- a/Lib/test/test_free_threading/test_set.py +++ b/Lib/test/test_free_threading/test_set.py
@@ -159,5 +159,72 @@ class LargeSetTest(RaceTestBase, unittest.TestCase): SET_SIZE = 20 # larger than PySet_MINSIZE +class TestSetIteratorExhaustion(unittest.TestCase): + def test_racing_shared_iterator_exhaustion(self): + # gh-160573: exhausting a set iterator shared between threads must + # not release the iterator's reference to the set more than once. + for tp in (set, frozenset): + cell = [iter(tp(range(8)))] + + def worker(): + for _ in range(5000): + it = cell[0] + try: + next(it) + except StopIteration: + cell[0] = iter(tp(range(8))) + + with self.subTest(tp=tp): + threading_helper.run_concurrently(worker, nthreads=4) + + def test_racing_shared_iterator_exhaustion_contended(self): + # Same as above, but the set's lock is contended: other threads + # keep mutating and iterating the set while its iterator is shared. + s = set(range(8)) + cell = [iter(s)] + done = Event() + + def exhaust(): + for _ in range(5000): + it = cell[0] + try: + next(it) + except StopIteration: + cell[0] = iter(s) + except RuntimeError: + # The set changed size during iteration. + cell[0] = iter(s) + + def mutate(): + while not done.is_set(): + s.add(100) + s.discard(100) + + def iterate(): + while not done.is_set(): + try: + for _ in s: + pass + except RuntimeError: + pass + + def run_exhaust(): + try: + threading_helper.run_concurrently(exhaust, nthreads=4) + finally: + done.set() + + threading_helper.run_concurrently([run_exhaust, mutate, iterate]) + + def test_exhausted_iterator_stays_exhausted(self): + s = {1} + it = iter(s) + self.assertEqual(list(it), [1]) + s.add(2) + self.assertRaises(StopIteration, next, it) + self.assertEqual(it.__length_hint__(), 0) + self.assertEqual(list(it.__reduce__()[1][0]), []) + + if __name__ == "__main__": unittest.main()
diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-40-00.gh-issue-160573.sT9kQe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-40-00.gh-issue-160573.sT9kQe.rst new file mode 100644 index 0000000..7a69a47 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-40-00.gh-issue-160573.sT9kQe.rst
@@ -0,0 +1,4 @@ +Fix a crash when a :class:`set` iterator that is shared between threads reaches +its end in a free-threaded build: every thread that walked off the end of the +table released the iterator's reference to the set, so the set was over-released. +The iterator now keeps its reference to the set until it is deallocated.
diff --git a/Objects/setobject.c b/Objects/setobject.c index c6ceebe..6ba6e61 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c
@@ -1027,7 +1027,10 @@ frozenset_hash(PyObject *self) typedef struct { PyObject_HEAD - PySetObject *si_set; /* Set to NULL when iterator is exhausted */ + /* Set to NULL when the iterator is exhausted, except in the free-threaded + build, where the reference is kept until the iterator is deallocated + (see setiter_set_exhausted()). */ + PySetObject *si_set; Py_ssize_t si_used; Py_ssize_t si_pos; Py_ssize_t len; @@ -1056,8 +1059,14 @@ setiter_len(PyObject *op, PyObject *Py_UNUSED(ignored)) { setiterobject *si = (setiterobject*)op; Py_ssize_t len = 0; - if (si->si_set != NULL && si->si_used == si->si_set->used) - len = si->len; + PySetObject *so = si->si_set; + if (so != NULL) { + Py_ssize_t so_used = FT_ATOMIC_LOAD_SSIZE_RELAXED(so->used); + Py_ssize_t si_used = FT_ATOMIC_LOAD_SSIZE_RELAXED(si->si_used); + if (si_used == so_used) { + len = FT_ATOMIC_LOAD_SSIZE_RELAXED(si->len); + } + } return PyLong_FromSsize_t(len); } @@ -1089,6 +1098,25 @@ static PyMethodDef setiter_methods[] = { {NULL, NULL} /* sentinel */ }; +/* Mark the iterator as exhausted. In the free-threaded build an iterator + may be advanced by several threads at once, each holding a borrowed + pointer to si_set, so the iterator keeps its reference to the set until + it is deallocated (as list and tuple iterators do) and records exhaustion + by setting si_pos to SETITER_EXHAUSTED. */ +#define SETITER_EXHAUSTED (-1) + +static void +setiter_set_exhausted(setiterobject *si, PySetObject *so) +{ +#ifdef Py_GIL_DISABLED + FT_ATOMIC_STORE_SSIZE_RELAXED(si->si_pos, SETITER_EXHAUSTED); + FT_ATOMIC_STORE_SSIZE_RELAXED(si->len, 0); +#else + si->si_set = NULL; + Py_DECREF(so); +#endif +} + static PyObject *setiter_iternext(PyObject *self) { setiterobject *si = (setiterobject*)self; @@ -1101,34 +1129,43 @@ static PyObject *setiter_iternext(PyObject *self) return NULL; assert (PyAnySet_Check(so)); +#ifdef Py_GIL_DISABLED + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(si->si_pos) == SETITER_EXHAUSTED) { + return NULL; + } +#endif + Py_ssize_t so_used = FT_ATOMIC_LOAD_SSIZE_RELAXED(so->used); Py_ssize_t si_used = FT_ATOMIC_LOAD_SSIZE_RELAXED(si->si_used); if (si_used != so_used) { PyErr_SetString(PyExc_RuntimeError, "Set changed size during iteration"); - si->si_used = -1; /* Make this state sticky */ + /* Make this state sticky */ + FT_ATOMIC_STORE_SSIZE_RELAXED(si->si_used, -1); return NULL; } + /* si_pos and len are only written with the set locked, so that an + iterator marked as exhausted by another thread stays exhausted. */ Py_BEGIN_CRITICAL_SECTION(so); i = si->si_pos; - assert(i>=0); - entry = so->table; - mask = so->mask; - while (i <= mask && (entry[i].key == NULL || entry[i].key == dummy)) { - i++; - } - if (i <= mask) { - key = Py_NewRef(entry[i].key); + if (i != SETITER_EXHAUSTED) { + assert(i >= 0); + entry = so->table; + mask = so->mask; + while (i <= mask && (entry[i].key == NULL || entry[i].key == dummy)) { + i++; + } + if (i <= mask) { + key = Py_NewRef(entry[i].key); + FT_ATOMIC_STORE_SSIZE_RELAXED(si->si_pos, i + 1); + FT_ATOMIC_STORE_SSIZE_RELAXED(si->len, si->len - 1); + } + else { + setiter_set_exhausted(si, so); + } } Py_END_CRITICAL_SECTION(); - si->si_pos = i+1; - if (key == NULL) { - si->si_set = NULL; - Py_DECREF(so); - return NULL; - } - si->len--; return key; }