gh-158539: Fix exception mode missing handlers in generators/coroutines (#158581)

* gh-158539: Fix exception mode missing handlers in generators/coroutines

The sampling profiler's exception mode decided whether a thread was
handling an exception by reading the embedded PyThreadState.exc_state.
Generators, coroutines and async generators repoint tstate->exc_info
at their own _PyErr_StackItem while they run, so an except block
running in one of them (or in a function they call) stored the
exception in that item instead, and was never sampled.

Follow tstate->exc_info and its previous_item chain, mirroring
_PyErr_GetTopmostException(), and export the two debug offsets needed
to walk the chain from remote memory.  The common case where exc_info
points at the embedded exc_state keeps the existing zero-extra-read
fast path.

* gh-158539: Handle deep exception chains without changing debug offsets

* gh-158539: Use portable static assertion messages

* gh-158539: Keep layout assertions with debug-offset validation

* gh-158539: Use the platform guard for in-process inspection tests

---------

Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>

Author: hetaozdh

Lib/test/test_external_inspection.py

@@ -3293,6 +3293,242 @@ def test_finally_no_exception_no_flag(self):
             self._check_exception_status(p, thread_tid, expect_exception=False)
 
 
+@skip_if_not_supported
+class TestExceptionDetectionInProcess(RemoteInspectionTestBase):
+    """gh-158539: HAS_EXCEPTION for handlers running in generators/coroutines.
+
+    ``TestExceptionDetectionScenarios`` samples a child process and therefore
+    needs subprocess debugging permissions. These tests inspect the current
+    process with ``RemoteUnwinder`` and only need self-inspection, so they also
+    run on macOS without special entitlements.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        try:
+            RemoteUnwinder(os.getpid(), all_threads=True).get_stack_trace()
+        except PermissionError as exc:
+            raise unittest.SkipTest(f"self-inspection is unavailable: {exc}")
+
+    def _check_running_handler(
+        self, target, expect_exception, *, mode=PROFILING_MODE_ALL,
+        skip_non_matching_threads=False,
+    ):
+        """Run *target* in a thread and check its HAS_EXCEPTION flag.
+
+        *target* receives ``(ready, stop)`` events and must signal ``ready``
+        only once it is executing inside the code region under test, then keep
+        running until ``stop`` is set.
+        """
+        stop = threading.Event()
+        ready = threading.Event()
+        failure = []
+
+        def runner():
+            try:
+                target(ready, stop)
+            except BaseException as exc:
+                failure.append(exc)
+                ready.set()
+
+        thread = threading.Thread(target=runner, daemon=True)
+        thread.start()
+        try:
+            self.assertTrue(ready.wait(SHORT_TIMEOUT), "handler never started")
+            self.assertFalse(failure, f"handler raised {failure!r}")
+
+            unwinder = RemoteUnwinder(
+                os.getpid(),
+                all_threads=True,
+                mode=mode,
+                skip_non_matching_threads=skip_non_matching_threads,
+            )
+            observed = []
+            for _ in busy_retry(SHORT_TIMEOUT):
+                with contextlib.suppress(*TRANSIENT_ERRORS):
+                    statuses = self._get_thread_statuses(unwinder.get_stack_trace())
+                    status = statuses.get(thread.native_id)
+                    if status is None:
+                        continue
+                    has_exception = bool(status & THREAD_STATUS_HAS_EXCEPTION)
+                    observed.append(has_exception)
+                    if has_exception == expect_exception:
+                        break
+            self.assertTrue(
+                observed, "target thread status was never observed"
+            )
+            self.assertIn(
+                expect_exception,
+                observed,
+                f"HAS_EXCEPTION was never {expect_exception} while the "
+                f"handler was running (observed {observed})",
+            )
+        finally:
+            stop.set()
+            thread.join(SHORT_TIMEOUT)
+
+    def _busy_until_stopped(self, ready, stop):
+        ready.set()
+        while not stop.is_set():
+            time.sleep(0.001)
+
+    def test_handler_in_function(self):
+        def target(ready, stop):
+            try:
+                raise ValueError("test")
+            except ValueError:
+                self._busy_until_stopped(ready, stop)
+
+        self._check_running_handler(target, expect_exception=True)
+
+    def test_handler_in_generator(self):
+        def target(ready, stop):
+            def gen():
+                try:
+                    raise ValueError("test")
+                except ValueError:
+                    self._busy_until_stopped(ready, stop)
+                yield
+
+            for _ in gen():
+                pass
+
+        self._check_running_handler(target, expect_exception=True)
+
+    def test_handler_in_genexpr_callee(self):
+        def target(ready, stop):
+            def callee():
+                try:
+                    raise ValueError("test")
+                except ValueError:
+                    self._busy_until_stopped(ready, stop)
+
+            list(callee() for _ in range(1))
+
+        self._check_running_handler(target, expect_exception=True)
+
+    def test_handler_in_coroutine(self):
+        async def coro(ready, stop):
+            try:
+                raise ValueError("test")
+            except ValueError:
+                self._busy_until_stopped(ready, stop)
+
+        def target(ready, stop):
+            asyncio.run(coro(ready, stop))
+
+        self._check_running_handler(target, expect_exception=True)
+
+    def test_handler_in_callee_from_coroutine(self):
+        def callee(ready, stop):
+            try:
+                raise ValueError("test")
+            except ValueError:
+                self._busy_until_stopped(ready, stop)
+
+        async def coro(ready, stop):
+            callee(ready, stop)
+
+        def target(ready, stop):
+            asyncio.run(coro(ready, stop))
+
+        self._check_running_handler(target, expect_exception=True)
+
+    def test_outer_handler_while_generator_runs(self):
+        """A generator with no handler of its own must not hide the outer one.
+
+        ``exc_info`` points at the generator's empty ``_PyErr_StackItem`` whose
+        ``previous_item`` is the thread's ``exc_state``, so the profiler has to
+        walk the chain to find the exception ``sys.exception()`` reports.
+        """
+        def target(ready, stop):
+            def gen():
+                self._busy_until_stopped(ready, stop)
+                yield
+
+            try:
+                raise ValueError("outer")
+            except ValueError:
+                for _ in gen():
+                    pass
+
+        self._check_running_handler(target, expect_exception=True)
+
+    def test_generator_without_exception(self):
+        def target(ready, stop):
+            def gen():
+                self._busy_until_stopped(ready, stop)
+                yield
+
+            for _ in gen():
+                pass
+
+        self._check_running_handler(target, expect_exception=False)
+
+    def test_outer_handler_while_nested_generators_run(self):
+        def target(ready, stop):
+            def gen(depth):
+                if depth:
+                    yield from gen(depth - 1)
+                else:
+                    self._busy_until_stopped(ready, stop)
+                yield
+
+            try:
+                raise ValueError("outer")
+            except ValueError:
+                for _ in gen(32):
+                    pass
+
+        self._check_running_handler(
+            target,
+            expect_exception=True,
+            mode=PROFILING_MODE_EXCEPTION,
+            skip_non_matching_threads=True,
+        )
+
+    def test_generator_finally_after_except(self):
+        """The handled exception is cleared before the generator's finally."""
+        def target(ready, stop):
+            def gen():
+                try:
+                    raise ValueError("test")
+                except ValueError:
+                    pass
+                finally:
+                    self._busy_until_stopped(ready, stop)
+                yield
+
+            for _ in gen():
+                pass
+
+        self._check_running_handler(target, expect_exception=False)
+
+    def test_exception_mode_filter_keeps_generator_handler(self):
+        """The exception-mode thread filter must not drop a generator handler.
+
+        This mirrors what ``--mode=exception`` actually does: threads without
+        HAS_EXCEPTION are skipped before their stack is unwound.
+        """
+        def target(ready, stop):
+            def gen():
+                try:
+                    raise ValueError("test")
+                except ValueError:
+                    self._busy_until_stopped(ready, stop)
+                yield
+
+            for _ in gen():
+                pass
+
+        self._check_running_handler(
+            target,
+            expect_exception=True,
+            mode=PROFILING_MODE_EXCEPTION,
+            skip_non_matching_threads=True,
+        )
+
+
 @requires_remote_subprocess_debugging()
 class TestFrameCaching(RemoteInspectionTestBase):
     """Test that frame caching produces correct results.

Misc/NEWS.d/next/Library/2026-10-01-14-00-00.gh-issue-158539.q1w2e3.rst

@@ -0,0 +1,5 @@
+Fix :mod:`profiling.sampling` exception mode discarding samples for code
+running inside an ``except`` block in a generator or coroutine. The remote
+debugger now follows ``tstate->exc_info`` and its ``previous_item`` chain
+instead of only reading the embedded ``exc_state``, matching the exception
+that :func:`sys.exception` reports.

Modules/_remote_debugging/debug_offsets_validation.h

@@ -48,6 +48,15 @@ static_assert(
         PY_REMOTE_ASYNC_DEBUG_OFFSETS_TOTAL_SIZE,
     "Update _remote_debugging validation for _Py_AsyncioModuleDebugOffsets");
 
+/* Derive unexported offsets from adjacent fields to keep the debug-offset
+ * table compatible across patch releases. */
+static_assert(offsetof(PyThreadState, exc_info) ==
+              offsetof(PyThreadState, current_exception) + sizeof(uintptr_t),
+              "exc_info must immediately follow current_exception");
+static_assert(offsetof(_PyErr_StackItem, previous_item) ==
+              offsetof(_PyErr_StackItem, exc_value) + sizeof(uintptr_t),
+              "previous_item must immediately follow exc_value");
+
 /*
  * This logic lives in a private header because it is shared by module.c and
  * asyncio.c. Keep the helpers static inline so they stay local to those users
@@ -249,14 +258,15 @@ validate_fixed_field(
 #define PY_REMOTE_DEBUG_RUNTIME_STATE_FIELDS(APPLY, buffer_size) \
     APPLY(runtime_state, interpreters_head, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size)
 
+/* current_exception also covers the adjacent exc_info pointer. */
 #define PY_REMOTE_DEBUG_THREAD_STATE_FIELDS(APPLY, buffer_size) \
     APPLY(thread_state, native_thread_id, sizeof(unsigned long), _Alignof(long), buffer_size); \
     APPLY(thread_state, interp, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \
     APPLY(thread_state, datastack_chunk, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \
     APPLY(thread_state, status, FIELD_SIZE(PyThreadState, _status), _Alignof(unsigned int), buffer_size); \
     APPLY(thread_state, holds_gil, sizeof(int), _Alignof(int), buffer_size); \
     APPLY(thread_state, gil_requested, sizeof(int), _Alignof(int), buffer_size); \
-    APPLY(thread_state, current_exception, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \
+    APPLY(thread_state, current_exception, 2 * sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \
     APPLY(thread_state, thread_id, sizeof(unsigned long), _Alignof(long), buffer_size); \
     APPLY(thread_state, next, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \
     APPLY(thread_state, current_frame, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \
@@ -351,10 +361,11 @@ _PyRemoteDebug_ValidateDebugOffsetsLayout(struct _Py_DebugOffsets *debug_offsets
     PY_REMOTE_DEBUG_THREAD_STATE_FIELDS(
         PY_REMOTE_DEBUG_VALIDATE_FIELD,
         SIZEOF_THREAD_STATE);
+    /* exc_value also covers the adjacent previous_item pointer. */
     PY_REMOTE_DEBUG_VALIDATE_FIXED_FIELD(
         err_stackitem,
         exc_value,
-        sizeof(uintptr_t),
+        2 * sizeof(uintptr_t),
         _Alignof(uintptr_t),
         sizeof(_PyErr_StackItem));
     PY_REMOTE_DEBUG_VALIDATE_NESTED_FIELD(

Modules/_remote_debugging/threads.c

@@ -17,6 +17,9 @@
 #include <sys/wait.h>
 #endif
 
+/* Bound traversal of corrupted remote exception chains. */
+#define MAX_EXCEPTION_CHAIN_DEPTH (2 << 15)
+
 /* ============================================================================
  * THREAD ITERATION FUNCTIONS
  * ============================================================================ */
@@ -436,16 +439,49 @@ unwind_stack_for_thread(
         has_exception = 1;
     }
 
-    // Check exc_state.exc_value (exception being handled in except block)
-    // exc_state is embedded in PyThreadState, so we read it directly from
-    // the thread state buffer. This catches most cases; nested exception
-    // handlers where exc_info points elsewhere are rare.
+    // Generators and coroutines use their own exception stack items.
+    // Follow exc_info to find the innermost handler, as sys.exception() does.
     if (!has_exception) {
-        uintptr_t exc_value = GET_MEMBER(uintptr_t, ts,
-            unwinder->debug_offsets.thread_state.exc_state +
-            unwinder->debug_offsets.err_stackitem.exc_value);
-        if (exc_value != 0) {
-            has_exception = 1;
+        uintptr_t exc_info = GET_MEMBER(uintptr_t, ts,
+            unwinder->debug_offsets.thread_state.current_exception +
+            sizeof(uintptr_t));
+        uintptr_t exc_state_addr =
+            *current_tstate + unwinder->debug_offsets.thread_state.exc_state;
+        uintptr_t exc_value_offset =
+            unwinder->debug_offsets.err_stackitem.exc_value;
+        uintptr_t previous_item_offset =
+            exc_value_offset + sizeof(uintptr_t);
+
+        for (int depth = 0; exc_info != 0 && depth < MAX_EXCEPTION_CHAIN_DEPTH;
+             depth++)
+        {
+            if (exc_info == exc_state_addr) {
+                // Bottom of the chain: the stack item embedded in the thread
+                // state, which is already in the local thread state buffer.
+                uintptr_t exc_value = GET_MEMBER(uintptr_t, ts,
+                    unwinder->debug_offsets.thread_state.exc_state +
+                    exc_value_offset);
+                if (exc_value != 0) {
+                    has_exception = 1;
+                }
+                break;
+            }
+            uintptr_t exc_value = 0;
+            if (read_ptr(unwinder, exc_info + exc_value_offset, &exc_value) < 0) {
+                PyErr_Clear();  // Best effort: treat as no active exception
+                break;
+            }
+            if (exc_value != 0) {
+                has_exception = 1;
+                break;
+            }
+            uintptr_t previous_item = 0;
+            if (read_ptr(unwinder, exc_info + previous_item_offset,
+                         &previous_item) < 0) {
+                PyErr_Clear();
+                break;
+            }
+            exc_info = previous_item;
         }
     }