gh-159041: Strengthen asyncio task cleanup checks

Author: ngoldbaum

Lib/test/test_asyncio/test_tasks.py

@@ -3102,6 +3102,7 @@ def test_task_created_during_thread_state_clear(self):
         loop = self.loop
         task = None
         var = contextvars.ContextVar('var')
+        # Avoid recreating the thread state's context after it is cleared.
         task_context = contextvars.Context()
 
         async def noop():
@@ -3113,11 +3114,15 @@ def __del__(self):
                 task = loop.create_task(noop(), context=task_context)
 
         def create_finalizer():
+            # gh-159041: Run the finalizer when PyThreadState_Clear()
+            # clears the thread's context.
             var.set(CreatesTaskOnClear())
 
+        # threading.Thread owns its Context, so use _thread to make the
+        # Context lifetime match the thread state's lifetime.
         handle = _thread.start_joinable_thread(create_finalizer)
         handle.join(support.SHORT_TIMEOUT)
-        self.assertTrue(handle.is_done())
+        self.assertTrue(handle.is_done(), 'thread failed to exit')
         self.assertIsNotNone(task)
 
         try:

Python/pystate.c

@@ -815,9 +815,14 @@ common_constants_clear(PyInterpreterState *interp)
 static void
 move_asyncio_tasks_to_interpreter(PyThreadState *tstate)
 {
+    _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;
+    if (llist_empty(&tstate_impl->asyncio_tasks_head)) {
+        return;
+    }
+
     PyMutex_Lock(&tstate->interp->asyncio_tasks_lock);
     llist_concat(&tstate->interp->asyncio_tasks_head,
-                 &((_PyThreadStateImpl *)tstate)->asyncio_tasks_head);
+                 &tstate_impl->asyncio_tasks_head);
     PyMutex_Unlock(&tstate->interp->asyncio_tasks_lock);
 }
 
@@ -961,12 +966,11 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)
 #endif
 
     if (tstate->interp == interp) {
+        // gh-159041: Finalizers above may have registered new asyncio tasks.
+        move_asyncio_tasks_to_interpreter(tstate);
+
         /* We are now safe to fix tstate->_status.cleared. */
         // XXX Do this (much) earlier?
-        // Finalizers above may have registered new asyncio tasks.
-        move_asyncio_tasks_to_interpreter(tstate);
-        assert(llist_empty(
-            &((_PyThreadStateImpl *)tstate)->asyncio_tasks_head));
         tstate->_status.cleared = 1;
     }
 
@@ -1936,6 +1940,8 @@ static void
 tstate_delete_common(PyThreadState *tstate, int release_gil)
 {
     assert(tstate->_status.cleared && !tstate->_status.finalized);
+    assert(llist_empty(
+        &((_PyThreadStateImpl *)tstate)->asyncio_tasks_head));
     tstate_verify_not_active(tstate);
     assert(!_PyThreadState_IsRunningMain(tstate));