[3.15] gh-149816: #96 Fix a race condition in invoke_gc_callback with free threading (GH-150029) (GH-156119)

(cherry picked from commit 91d71dd)

Co-authored-by: dzaramelcone <134235821+dzaramelcone@users.noreply.github.com>
Co-authored-by: Petr Viktorin <encukou@gmail.com>

Author: 3 people

Lib/test/test_free_threading/test_gc.py

@@ -2,6 +2,7 @@
 
 import threading
 from threading import Thread
+import time
 from unittest import TestCase
 import gc
 import weakref
@@ -96,6 +97,27 @@ def evil():
         thread.start()
         thread.join()
 
+    def test_gc_callbacks_race_with_mutation(self):
+        def collect():
+            b.wait()
+            while not stop.is_set():
+                gc.collect()
+
+        def mutate():
+            b.wait()
+            while not stop.is_set():
+                gc.callbacks[:] = [lambda *_: _ for _ in range(16)]
+                time.sleep(0)
+                gc.callbacks.clear()
+
+        threads = [threading.Thread(target=f) for f in (collect, mutate) * 4]
+        b = threading.Barrier(len(threads) + 1)
+        stop = threading.Event()
+
+        with threading_helper.start_threads(threads, stop.set):
+            b.wait()
+            time.sleep(0.2)
+
     def test_merge_brc_queue_of_detached_thread(self):
         # GH-157838: objects queued for merging by a thread that is detached
         # (blocked in a lock acquire, sleep, etc.) are merged and freed on its

Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-12-32-33.gh-issue-149816.v18Ypf.rst

@@ -0,0 +1,2 @@
+Fix race conditions in ``invoke_gc_callback`` iterating ``gc.callbacks``
+in free-threading mode.

Python/gc_free_threading.c

@@ -9,6 +9,7 @@
 #include "pycore_initconfig.h"    // _PyStatus_NO_MEMORY()
 #include "pycore_interp.h"        // PyInterpreterState.gc
 #include "pycore_interpframe.h"   // _PyFrame_GetLocalsArray()
+#include "pycore_list.h"          // _PyList_GetItemRef()
 #include "pycore_object_alloc.h"  // _PyObject_MallocWithType()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "pycore_tstate.h"        // _PyThreadStateImpl
@@ -1940,33 +1941,36 @@ invoke_gc_callback(PyThreadState *tstate, const char *phase,
 
     /* The local variable cannot be rebound, check it for sanity */
     assert(PyList_CheckExact(gcstate->callbacks));
-    PyObject *info = NULL;
-    if (PyList_GET_SIZE(gcstate->callbacks) != 0) {
-        info = Py_BuildValue("{sisnsnsnsd}",
-            "generation", generation,
-            "collected", collected,
-            "uncollectable", uncollectable,
-            "candidates", candidates,
-            "duration", duration);
-        if (info == NULL) {
-            PyErr_FormatUnraisable("Exception ignored while "
-                                   "invoking gc callbacks");
-            return;
-        }
+    if (PyList_GET_SIZE(gcstate->callbacks) == 0) {
+        return;
+    }
+
+    PyObject *info = Py_BuildValue("{sisnsnsnsd}",
+        "generation", generation,
+        "collected", collected,
+        "uncollectable", uncollectable,
+        "candidates", candidates,
+        "duration", duration);
+    if (info == NULL) {
+        PyErr_FormatUnraisable("Exception ignored while "
+                               "invoking gc callbacks");
+        return;
     }
 
     PyObject *phase_obj = PyUnicode_FromString(phase);
     if (phase_obj == NULL) {
-        Py_XDECREF(info);
+        Py_DECREF(info);
         PyErr_FormatUnraisable("Exception ignored while "
                                "invoking gc callbacks");
         return;
     }
 
     PyObject *stack[] = {phase_obj, info};
     for (Py_ssize_t i=0; i<PyList_GET_SIZE(gcstate->callbacks); i++) {
-        PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i);
-        Py_INCREF(cb); /* make sure cb doesn't go away */
+        PyObject *r, *cb = _PyList_GetItemRef((PyListObject *)gcstate->callbacks, i);
+        if (cb == NULL) {
+            break;
+        }
         r = PyObject_Vectorcall(cb, stack, 2, NULL);
         if (r == NULL) {
             PyErr_FormatUnraisable("Exception ignored while "
@@ -1978,7 +1982,7 @@ invoke_gc_callback(PyThreadState *tstate, const char *phase,
         Py_DECREF(cb);
     }
     Py_DECREF(phase_obj);
-    Py_XDECREF(info);
+    Py_DECREF(info);
     assert(!_PyErr_Occurred(tstate));
 }