Re-apply CPython-specific changes to mimalloc v2.5.2

Most of the previous downstream changes were upstreamed in mimalloc
v2.1.3 - v2.1.7 (heap tags, _mi_heap_init/_mi_tld_init, MI_PRIM_THREAD_ID,
sub-processes, heap walk optimizations, platform fixes) and are dropped.

Remaining CPython-specific changes, all marked with a "CPython" comment:

- QSBR delayed page freeing for free-threaded builds (gh-115103,
  gh-145615): mi_page_t/mi_heap_t fields and _PyMem_mi_* hooks in
  page.c, heap.c and segment.c.
- debug_offset: preserve the first bytes of a PyObject (ob_tid, refcount
  fields) when filling freed or uninitialized memory in debug builds
  (gh-115491).
- Atomic mi_block_t.next loads/stores to avoid TSAN reports (gh-129748).
- Relax the owning thread assertion in _mi_free_delayed_block for
  free-threaded builds, since the GC traverses heaps of other threads.
- _mi_abandoned_visit_blocks_unlocked(): visit abandoned segments of a
  sub-process without locks or mi_option_visit_abandoned. Only safe
  while the world is stopped.
- Call _mi_heap_delayed_free_partial() in mi_heap_visit_blocks()
  (gh-112532).
- Only count reclaimed pages up to MI_MEDIUM_OBJ_SIZE_MAX as a
  "right page" in mi_segment_reclaim (gh-115103).
- Rename the weak _ZSt15get_new_handlerv symbol and disable the
  mi_is_in_heap_region debug warning.
- Zero-initialize _mi_page_empty instead of a positional initializer.

Author: corona10

Include/internal/mimalloc/mimalloc/internal.h

@@ -249,6 +249,7 @@ void       _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld);
 void       _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld);
 bool       _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment);
 bool       _mi_segment_visit_blocks(mi_segment_t* segment, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
+bool       _mi_abandoned_visit_blocks_unlocked(mi_subproc_t* subproc, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);  // CPython
 
 // "page.c"
 void*       _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage)  mi_attr_noexcept mi_attr_malloc;
@@ -846,10 +847,10 @@ static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* bl
   mi_track_mem_defined(block,sizeof(mi_block_t));
   mi_block_t* next;
   #ifdef MI_ENCODE_FREELIST
-  next = (mi_block_t*)mi_ptr_decode(null, block->next, keys);
+  next = (mi_block_t*)mi_ptr_decode(null, mi_atomic_load_relaxed((_Atomic(mi_encoded_t)*)&block->next), keys);
   #else
   MI_UNUSED(keys); MI_UNUSED(null);
-  next = (mi_block_t*)block->next;
+  next = (mi_block_t*)mi_atomic_load_relaxed((_Atomic(mi_encoded_t)*)&block->next);
   #endif
   mi_track_mem_noaccess(block,sizeof(mi_block_t));
   return next;
@@ -858,10 +859,10 @@ static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* bl
 static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) {
   mi_track_mem_undefined(block,sizeof(mi_block_t));
   #ifdef MI_ENCODE_FREELIST
-  block->next = mi_ptr_encode(null, next, keys);
+  mi_atomic_store_relaxed(&block->next, mi_ptr_encode(null, next, keys));
   #else
   MI_UNUSED(keys); MI_UNUSED(null);
-  block->next = (mi_encoded_t)next;
+  mi_atomic_store_relaxed(&block->next, (mi_encoded_t)next);
   #endif
   mi_track_mem_noaccess(block,sizeof(mi_block_t));
 }

Include/internal/mimalloc/mimalloc/types.h

@@ -248,7 +248,7 @@ typedef size_t     mi_threadid_t;
 
 // free lists contain blocks
 typedef struct mi_block_s {
-  mi_encoded_t next;
+  _Atomic(mi_encoded_t) next;   // CPython: atomic to avoid TSAN reports from concurrent heap visits (gh-129748)
 } mi_block_t;
 
 #if MI_GUARDED
@@ -341,6 +341,8 @@ typedef struct mi_page_s {
   uint16_t              used;              // number of blocks in use (including blocks in `thread_free`)
   uint8_t               block_size_shift;  // if not zero, then `(1 << block_size_shift) == block_size` (only used for fast path in `free.c:_mi_page_ptr_unalign`)
   uint8_t               heap_tag;          // tag of the owning heap, used to separate heaps by object type
+  uint8_t               use_qsbr:1;        // CPython: delay page freeing using QSBR
+  uint8_t               debug_offset;      // CPython: number of bytes to preserve when filling freed or uninitialized memory
                                            // padding
   size_t                block_size;        // size available in each block (always `>0`)
   uint8_t*              page_start;        // start of the page area containing the blocks
@@ -355,8 +357,13 @@ typedef struct mi_page_s {
   struct mi_page_s*     next;              // next page owned by this thread with the same `block_size`
   struct mi_page_s*     prev;              // previous page owned by this thread with the same `block_size`
 
+#ifdef Py_GIL_DISABLED
+  struct llist_node     qsbr_node;         // CPython: list of pages waiting for a QSBR goal before being freed
+  uint64_t              qsbr_goal;         // CPython: QSBR goal to reach before the page can be freed (0 if none)
+#else
   // 64-bit 11 words, 32-bit 13 words, (+2 for secure)
   void* padding[1];
+#endif
 } mi_page_t;
 
 
@@ -573,6 +580,8 @@ struct mi_heap_s {
   mi_heap_t*            next;                                // list of heaps per thread
   bool                  no_reclaim;                          // `true` if this heap should not reclaim abandoned pages
   uint8_t               tag;                                 // custom tag, can be used for separating heaps based on the object types
+  uint8_t               debug_offset;                        // CPython: number of bytes to preserve when filling freed or uninitialized memory
+  bool                  page_use_qsbr;                       // CPython: should freeing pages be delayed using QSBR
   #if MI_GUARDED
   size_t                guarded_size_min;                    // minimal size for guarded objects
   size_t                guarded_size_max;                    // maximal size for guarded objects

Objects/mimalloc/alloc.c

@@ -16,6 +16,20 @@ terms of the MIT license. A copy of the license can be found in the file
 #include <string.h>      // memset, strlen (for mi_strdup)
 #include <stdlib.h>      // malloc, abort
 
+// CPython: don't export the weak C++ `std::get_new_handler` symbol from libpython
+#define _ZSt15get_new_handlerv _Py__ZSt15get_new_handlerv
+
+#if (MI_DEBUG>0)
+// CPython: fill freed or uninitialized memory but preserve the first `page->debug_offset`
+// bytes (e.g. `ob_tid` and the reference count fields that lock-free readers may still access)
+static inline void mi_debug_fill(mi_page_t* page, mi_block_t* block, int c, size_t size) {
+  size_t offset = (size_t)page->debug_offset;
+  if (offset < size) {
+    memset((char*)block + offset, c, size - offset);
+  }
+}
+#endif
+
 #define MI_IN_ALLOC_C
 #include "alloc-override.c"
 #include "free.c"
@@ -74,7 +88,7 @@ extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_
 
   #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN
   if (!zero && !mi_page_is_huge(page)) {
-    memset(block, MI_DEBUG_UNINIT, mi_page_usable_block_size(page));
+    mi_debug_fill(page, block, MI_DEBUG_UNINIT, mi_page_usable_block_size(page));
   }
   #elif (MI_SECURE!=0)
   if (!zero) { block->next = 0; } // don't leak internal data

Objects/mimalloc/arena-abandon.c

@@ -344,3 +344,42 @@ bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool vi
   _mi_arena_field_cursor_done(&current);
   return ok;
 }
+
+
+/* -----------------------------------------------------------
+  CPython: visit all abandoned segments of a sub-process without
+  taking any locks or claiming the segments.
+
+  Unlike `mi_abandoned_visit_blocks`, this does not require
+  `mi_option_visit_abandoned` to be enabled. The caller must
+  guarantee that no other thread abandons or reclaims segments
+  concurrently (in CPython: a stop-the-world pause), and the
+  visitor must not allocate or free mimalloc managed memory.
+----------------------------------------------------------- */
+
+bool _mi_abandoned_visit_blocks_unlocked(mi_subproc_t* subproc, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {
+  // segments in arenas are tracked in the `blocks_abandoned` bitmaps
+  const size_t max_arena = mi_arena_get_count();
+  for (size_t arena_idx = 0; arena_idx < max_arena; arena_idx++) {
+    mi_arena_t* arena = mi_arena_from_index(arena_idx);
+    if (arena == NULL) continue;
+    for (size_t field_idx = 0; field_idx < arena->field_count; field_idx++) {
+      size_t field = mi_atomic_load_relaxed(&arena->blocks_abandoned[field_idx]);
+      while (field != 0) {
+        const size_t bit_idx = mi_ctz(field);
+        field &= ~((size_t)1 << bit_idx);
+        const mi_bitmap_index_t bitmap_idx = mi_bitmap_index_create(field_idx, bit_idx);
+        mi_segment_t* const segment = (mi_segment_t*)mi_arena_block_start(arena, bitmap_idx);
+        mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id) == 0);
+        if (segment->subproc != subproc) continue;
+        if (!_mi_segment_visit_blocks(segment, heap_tag, visit_blocks, visitor, arg)) return false;
+      }
+    }
+  }
+  // segments outside arenas (OS allocated) are kept in a per sub-process list
+  for (mi_segment_t* segment = subproc->abandoned_os_list; segment != NULL; segment = segment->abandoned_os_next) {
+    mi_assert_internal(segment->subproc == subproc);
+    if (!_mi_segment_visit_blocks(segment, heap_tag, visit_blocks, visitor, arg)) return false;
+  }
+  return true;
+}

Objects/mimalloc/free.c

@@ -38,7 +38,7 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool
   #if (MI_DEBUG>0) && !MI_TRACK_ENABLED  && !MI_TSAN
   if (!mi_page_is_huge(page)) {   // huge page content may be already decommitted
     size_t dbgsize = mi_page_block_size(page); if (dbgsize > 1*MI_MiB) { dbgsize = 1*MI_MiB; }
-    memset(block, MI_DEBUG_FREED, dbgsize);
+    mi_debug_fill(page, block, MI_DEBUG_FREED, dbgsize);
   }
   #endif
   if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); } // faster then mi_usable_size as we already know the page and that p is unaligned
@@ -147,7 +147,7 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms
   mi_segment_t* const segment = _mi_ptr_segment(p);
   if mi_unlikely(segment==NULL) return segment;
 
-  #if (MI_DEBUG>0)
+  #if 0 && (MI_DEBUG>0)  // CPython: disabled, the warning is noisy and not actionable for Python allocations
   if mi_unlikely(!mi_is_in_heap_region(p)) {
   #if (MI_INTPTR_SIZE == 8 && defined(__linux__))
     if (((uintptr_t)p >> 40) != 0x7F) { // linux tends to align large blocks above 0x7F000000000 (issue #640)
@@ -222,7 +222,10 @@ bool _mi_free_delayed_block(mi_block_t* block) {
   mi_assert_internal(block!=NULL);
   const mi_segment_t* const segment = _mi_ptr_segment(block);
   mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie);
+#ifndef Py_GIL_DISABLED
+  // CPython: the GC traverses heaps of other threads, which can trigger this assert.
   mi_assert_internal(_mi_thread_id() == segment->thread_id);
+#endif
   mi_page_t* const page = _mi_segment_page_of(segment, block);
 
   // Clear the no-delayed flag so delayed freeing is used again for this page.
@@ -340,7 +343,7 @@ static void mi_decl_noinline mi_free_block_mt(mi_page_t* page, mi_segment_t* seg
   else {
     #if (MI_DEBUG>0) && !MI_TRACK_ENABLED  && !MI_TSAN       // note: when tracking, cannot use mi_usable_size with multi-threading
     size_t dbgsize = mi_usable_size(block); if (dbgsize > 1*MI_MiB) { dbgsize = 1*MI_MiB; }
-    memset(block, MI_DEBUG_FREED, dbgsize);
+    mi_debug_fill(page, block, MI_DEBUG_FREED, dbgsize);
     #endif
   }
 

Objects/mimalloc/heap.c

@@ -104,7 +104,13 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t
   if (mi_page_all_free(page)) {
     // no more used blocks, free the page.
     // note: this will free retired pages as well.
-    _mi_page_free(page, pq, collect >= MI_FORCE);
+    bool freed = _PyMem_mi_page_maybe_free(page, pq, collect >= MI_FORCE);
+    if (!freed && collect == MI_ABANDON) {
+      // CPython: _PyMem_mi_page_maybe_free may have moved the page to a different
+      // page queue, so we need to re-fetch the correct queue.
+      size_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : _mi_bin(mi_page_block_size(page)));
+      _mi_page_abandon(page, &heap->pages[bin]);
+    }
   }
   else if (collect == MI_ABANDON) {
     // still used blocks but the thread is done; abandon the page
@@ -164,6 +170,9 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
   // collect retired pages
   _mi_heap_collect_retired(heap, force);
 
+  // CPython: free pages that were delayed with QSBR
+  _PyMem_mi_heap_collect_qsbr(heap);
+
   // collect all pages owned by this thread
   mi_heap_visit_pages(heap, &mi_heap_page_collect, (collect!=MI_NORMAL), &collect, NULL); // normally don't visit full pages, issue #1220
   mi_assert_internal( collect != MI_ABANDON || mi_atomic_load_ptr_acquire(mi_block_t,&heap->thread_delayed_free) == NULL );
@@ -741,6 +750,7 @@ static bool mi_heap_area_visitor(const mi_heap_t* heap, const mi_heap_area_ex_t*
 // Visit all blocks in a heap
 bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {
   mi_visit_blocks_args_t args = { visit_blocks, visitor, arg };
+  _mi_heap_delayed_free_partial((mi_heap_t *)heap);  // CPython: gh-112532
   return mi_heap_visit_areas(heap, &mi_heap_area_visitor, &args);
 }
 

Objects/mimalloc/init.c

@@ -13,29 +13,7 @@ terms of the MIT license. A copy of the license can be found in the file
 
 
 // Empty page used to initialize the small free pages array
-const mi_page_t _mi_page_empty = {
-  0,
-  false, false, false, false,
-  0,       // capacity
-  0,       // reserved capacity
-  { 0 },   // flags
-  false,   // is_zero
-  0,       // retire_expire
-  NULL,    // free
-  NULL,    // local_free
-  0,       // used
-  0,       // block size shift
-  0,       // heap tag
-  0,       // block_size
-  NULL,    // page_start
-  #if (MI_PADDING || MI_ENCODE_FREELIST)
-  { 0, 0 },
-  #endif
-  MI_ATOMIC_VAR_INIT(0), // xthread_free
-  MI_ATOMIC_VAR_INIT(0), // xheap
-  NULL, NULL
-  , { 0 }  // padding
-};
+const mi_page_t _mi_page_empty;  // CPython: zero-initialized since mi_page_t has extra fields
 
 #define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty)
 
@@ -124,6 +102,8 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = {
   NULL,             // next
   false,            // can reclaim
   0,                // tag
+  0,                // debug_offset (CPython)
+  false,            // page_use_qsbr (CPython)
   #if MI_GUARDED
   0, 0, 0, 1,       // rate is 0 and count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`)
   #endif
@@ -176,6 +156,8 @@ mi_decl_cache_align mi_heap_t _mi_heap_main = {
   NULL,             // next heap
   false,            // can reclaim
   0,                // tag
+  0,                // debug_offset (CPython)
+  false,            // page_use_qsbr (CPython)
   #if MI_GUARDED
   0, 0, 0, 0,
   #endif