gh-156545, gh-158539: Fix deep flamegraph export on small C stacks and macOS runtime lookup (#158874)

(cherry picked from commit 114de19)

Include the C-stack test helper from main, introduced by
ce5ae29, which the regression test
requires but 3.15 does not yet provide.

Author: pablogsal

Lib/profiling/sampling/stack_collector.py

@@ -68,8 +68,8 @@ def export(self, filename):
         return True
 
 
-# Bounded by the unwinder's maximum captured stack depth (MAX_FRAMES).
-_FLAMEGRAPH_RECURSION_MARGIN = 2000
+# Allow for tree conversion and the dict/list frames in the Python JSON encoder.
+_FLAMEGRAPH_RECURSION_MARGIN = 6000
 
 
 class FlamegraphCollector(StackTraceCollector):
@@ -498,7 +498,12 @@ def _get_source_lines(self, func):
             return None
 
     def _create_flamegraph_html(self, data):
-        data_json = json.dumps(data)
+        try:
+            data_json = json.dumps(data)
+        except RecursionError:
+            # The C encoder can exhaust the C stack independently of the
+            # Python recursion limit. iterencode() uses the Python encoder.
+            data_json = "".join(json.JSONEncoder().iterencode(data))
 
         template_dir = importlib.resources.files(__package__)
         vendor_dir = template_dir / "_vendor"

Lib/test/support/__init__.py

@@ -46,6 +46,7 @@
     "check_disallow_instantiation", "check_sanitizer", "skip_if_sanitizer",
     "requires_limited_api", "requires_specialization", "thread_unsafe",
     "skip_if_unlimited_stack_size", "skip_if_huge_c_stack",
+    "run_with_limited_c_stack",
     # sys
     "MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi",
     "is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval",
@@ -2830,30 +2831,94 @@ def exceeds_recursion_limit():
     return 150_000
 
 
+def _has_huge_c_stack(depth):
+    """Check that *depth* recursive calls cannot exhaust the C stack."""
+    try:
+        from _testinternalcapi import get_c_recursion_remaining
+    except ImportError:
+        # Fall back to checking for an unlimited stack size.
+        if is_emscripten or is_wasi or os.name == "nt":
+            return False
+        import resource
+        soft, hard = resource.getrlimit(resource.RLIMIT_STACK)
+        return soft == hard and soft in (-1, 0xFFFF_FFFF_FFFF_FFFF)
+    else:
+        remaining = get_c_recursion_remaining()
+        # A negative value means integer overflow in the estimate
+        # (e.g. with an unlimited RLIMIT_STACK).  The estimate is based on
+        # the size of the interpreter loop frame, so it is only a lower
+        # bound for recursion with smaller C frames.
+        return remaining >= depth or remaining < 0
+
+
 def skip_if_huge_c_stack(depth=150_000):
     """Skip decorator for tests which cannot overflow the C stack.
 
     Tests exhausting the C stack with *depth* recursive calls cannot
     trigger the recursion protection if the C stack is too large (e.g.
     with a large or unlimited RLIMIT_STACK), and either fail, or run
     for a very long time, or crash, or consume all memory.
+
+    Prefer run_with_limited_c_stack() for tests recursing to a fixed depth.
     """
-    try:
-        from _testinternalcapi import get_c_recursion_remaining
-    except ImportError:
-        # Fall back to checking for an unlimited stack size.
-        huge = False
-        if not (is_emscripten or is_wasi) and os.name != "nt":
-            import resource
-            soft, hard = resource.getrlimit(resource.RLIMIT_STACK)
-            huge = soft == hard and soft in (-1, 0xFFFF_FFFF_FFFF_FFFF)
-    else:
-        remaining = get_c_recursion_remaining()
-        # A negative value means integer overflow in the estimate
-        # (e.g. with an unlimited RLIMIT_STACK).
-        huge = remaining >= depth or remaining < 0
     return unittest.skipIf(
-        huge, f"the C stack is large enough for {depth} recursive calls")
+        _has_huge_c_stack(depth),
+        f"the C stack is large enough for {depth} recursive calls")
+
+
+# Small enough to be exhausted by tens of thousands of recursive calls,
+# but not smaller than Py_C_STACK_SIZE (4 MiB) which the interpreter
+# assumes if it cannot query the thread stack size.
+C_STACK_SIZE = 8 * 1024 * 1024
+
+
+def run_with_limited_c_stack(depth=150_000, size=C_STACK_SIZE):
+    """Decorator for tests exhausting the C stack with *depth* recursive calls.
+
+    Run the test in a separate thread with the C stack of *size* bytes, so
+    that the outcome does not depend on the C stack size of the main thread
+    (which can be large or unlimited, see RLIMIT_STACK).
+
+    If a thread with the limited C stack cannot be created, run the test in
+    the current thread, but skip it if the C stack is too large.
+    """
+    reason = f"the C stack is large enough for {depth} recursive calls"
+    def decorator(test):
+        @functools.wraps(test)
+        def wrapper(*args, **kwargs):
+            def run_test():
+                # The C stack can still be too large if limiting it failed.
+                if _has_huge_c_stack(depth):
+                    raise unittest.SkipTest(reason)
+                test(*args, **kwargs)
+
+            try:
+                import threading
+                old_size = threading.stack_size(size)
+            except (ImportError, ValueError, RuntimeError):
+                # Setting the thread stack size is not supported.
+                return run_test()
+
+            exceptions = []
+            def run():
+                try:
+                    run_test()
+                except BaseException as exc:
+                    exceptions.append(exc)
+
+            thread = threading.Thread(target=run)
+            try:
+                thread.start()
+            except RuntimeError:
+                # Threads are not supported.
+                return run_test()
+            finally:
+                threading.stack_size(old_size)
+            thread.join()
+            if exceptions:
+                raise exceptions[0]
+        return wrapper
+    return decorator
 
 
 # Windows doesn't have os.uname() but it doesn't support s390x.

Lib/test/test_profiling/test_sampling_profiler/test_collectors.py

@@ -8,7 +8,7 @@
 import tempfile
 import unittest
 
-from test.support import is_emscripten, set_recursion_limit
+from test.support import is_emscripten, run_with_limited_c_stack, set_recursion_limit
 
 try:
     import _remote_debugging  # noqa: F401
@@ -611,6 +611,7 @@ def test_flamegraph_collector_empty_export_fails(self):
         self.assertFalse(export_ok)
         self.assertEqual(os.path.getsize(flamegraph_out.name), 0)
 
+    @run_with_limited_c_stack(size=1024 * 1024)
     def test_flamegraph_deep_stack_export(self):
         flamegraph_out = tempfile.NamedTemporaryFile(
             suffix=".html", delete=False

Misc/NEWS.d/next/Library/2026-10-05-18-00-00.gh-issue-156545.z1mQp7.rst

@@ -0,0 +1,2 @@
+Fix deep flamegraph exports on platforms with small C stacks by falling back
+to the Python JSON encoder when the C encoder runs out of stack space.

Misc/NEWS.d/next/Library/2026-10-05-18-00-01.gh-issue-158539.k8Lv2a.rst

@@ -0,0 +1,2 @@
+Fix macOS runtime section lookup for remote debugging when virtual memory
+region sizes differ from Mach-O segment file sizes.

Python/remote_debug.h

@@ -297,7 +297,6 @@ _Py_RemoteDebug_CleanupProcHandle(proc_handle_t *handle) {
 static uintptr_t
 return_section_address64(
     const char* section,
-    mach_port_t proc_ref,
     uintptr_t base,
     void* map
 ) {
@@ -307,37 +306,15 @@ return_section_address64(
     int cmd_cnt = 0;
     struct segment_command_64* cmd = map + sizeof(struct mach_header_64);
 
-    mach_vm_size_t size = 0;
-    mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t);
-    mach_vm_address_t address = (mach_vm_address_t)base;
-    vm_region_basic_info_data_64_t r_info;
-    mach_port_t object_name;
     uintptr_t vmaddr = 0;
 
     for (int i = 0; cmd_cnt < 2 && i < ncmds; i++) {
         if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__TEXT") == 0) {
             vmaddr = cmd->vmaddr;
         }
         if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__DATA") == 0) {
-            while (cmd->filesize != size) {
-                address += size;
-                kern_return_t ret = mach_vm_region(
-                    proc_ref,
-                    &address,
-                    &size,
-                    VM_REGION_BASIC_INFO_64,
-                    (vm_region_info_t)&r_info,  // cppcheck-suppress [uninitvar]
-                    &count,
-                    &object_name
-                );
-                if (ret != KERN_SUCCESS) {
-                    PyErr_Format(PyExc_RuntimeError,
-                        "mach_vm_region failed while parsing 64-bit Mach-O binary "
-                        "at base address 0x%lx (kern_return_t: %d)",
-                        base, ret);
-                    return 0;
-                }
-            }
+            // The section address only needs the image's ASLR slide.
+            // VM regions need not match the segment's on-disk size.
 
             int nsects = cmd->nsects;
             struct section_64* sec = (struct section_64*)(
@@ -360,7 +337,6 @@ return_section_address64(
 static uintptr_t
 return_section_address32(
     const char* section,
-    mach_port_t proc_ref,
     uintptr_t base,
     void* map
 ) {
@@ -370,37 +346,15 @@ return_section_address32(
     int cmd_cnt = 0;
     struct segment_command* cmd = map + sizeof(struct mach_header);
 
-    mach_vm_size_t size = 0;
-    mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_t);
-    mach_vm_address_t address = (mach_vm_address_t)base;
-    vm_region_basic_info_data_t r_info;
-    mach_port_t object_name;
     uintptr_t vmaddr = 0;
 
     for (int i = 0; cmd_cnt < 2 && i < ncmds; i++) {
         if (cmd->cmd == LC_SEGMENT && strcmp(cmd->segname, "__TEXT") == 0) {
             vmaddr = cmd->vmaddr;
         }
         if (cmd->cmd == LC_SEGMENT && strcmp(cmd->segname, "__DATA") == 0) {
-            while (cmd->filesize != size) {
-                address += size;
-                kern_return_t ret = mach_vm_region(
-                    proc_ref,
-                    &address,
-                    &size,
-                    VM_REGION_BASIC_INFO,
-                    (vm_region_info_t)&r_info,  // cppcheck-suppress [uninitvar]
-                    &count,
-                    &object_name
-                );
-                if (ret != KERN_SUCCESS) {
-                    PyErr_Format(PyExc_RuntimeError,
-                        "mach_vm_region failed while parsing 32-bit Mach-O binary "
-                        "at base address 0x%lx (kern_return_t: %d)",
-                        base, ret);
-                    return 0;
-                }
-            }
+            // The section address only needs the image's ASLR slide.
+            // VM regions need not match the segment's on-disk size.
 
             int nsects = cmd->nsects;
             struct section* sec = (struct section*)(
@@ -423,7 +377,6 @@ return_section_address32(
 static uintptr_t
 return_section_address_fat(
     const char* section,
-    mach_port_t proc_ref,
     uintptr_t base,
     void* map
 ) {
@@ -473,11 +426,11 @@ return_section_address_fat(
             switch (hdr->magic) {
                 case MH_MAGIC:
                 case MH_CIGAM:
-                    return return_section_address32(section, proc_ref, base, (void*)hdr);
+                    return return_section_address32(section, base, (void*)hdr);
 
                 case MH_MAGIC_64:
                 case MH_CIGAM_64:
-                    return return_section_address64(section, proc_ref, base, (void*)hdr);
+                    return return_section_address64(section, base, (void*)hdr);
 
                 default:
                     PyErr_Format(PyExc_RuntimeError,
@@ -496,7 +449,7 @@ return_section_address_fat(
 }
 
 static uintptr_t
-search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_size_t size, mach_port_t proc_ref)
+search_section_in_file(const char* secname, char* path, uintptr_t base)
 {
     int fd = open(path, O_RDONLY);
     if (fd == -1) {
@@ -533,15 +486,15 @@ search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_
     switch (magic) {
     case MH_MAGIC:
     case MH_CIGAM:
-        result = return_section_address32(secname, proc_ref, base, map);
+        result = return_section_address32(secname, base, map);
         break;
     case MH_MAGIC_64:
     case MH_CIGAM_64:
-        result = return_section_address64(secname, proc_ref, base, map);
+        result = return_section_address64(secname, base, map);
         break;
     case FAT_MAGIC:
     case FAT_CIGAM:
-        result = return_section_address_fat(secname, proc_ref, base, map);
+        result = return_section_address_fat(secname, base, map);
         break;
     default:
         PyErr_Format(PyExc_RuntimeError,
@@ -644,7 +597,7 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s
         if (strncmp(filename, substr, strlen(substr)) == 0) {
             PyErr_Clear();
             uintptr_t result = search_section_in_file(
-                secname, map_filename, address, size, proc_ref);
+                secname, map_filename, address);
             if (result != 0) {
                 if (validator == NULL || validator(handle, result)) {
                     return result;