From 77fbeb5bfd770a4f6295c42e1971fdcee539f0f9 Mon Sep 17 00:00:00 2001 From: Ben Reisner Date: Wed, 12 Aug 2026 15:06:12 -0700 Subject: [PATCH] Enable TCMALLOC_USE_PAGEMAP3 unconditionally. Currently, we use a two level PageMap for all but small_but_slow configurations. This change swaps all other use cases to PageMap3, aligned with small_but_slow PiperOrigin-RevId: 963681940 --- tcmalloc/central_freelist_benchmark.cc | 5 - tcmalloc/common.h | 1 - tcmalloc/pagemap.cc | 21 -- tcmalloc/pagemap.h | 219 ------------------ tcmalloc/pagemap_test.cc | 5 +- tcmalloc/static_vars.cc | 1 - tcmalloc/testing/fast_path.aarch64+opt.golden | 2 +- tcmalloc/testing/fast_path.insecure.golden | 2 +- tcmalloc/testing/fast_path.opt.golden | 2 +- .../testing/fast_path.release+insecure.golden | 4 +- tcmalloc/testing/fast_path.release.golden | 4 +- 11 files changed, 10 insertions(+), 256 deletions(-) diff --git a/tcmalloc/central_freelist_benchmark.cc b/tcmalloc/central_freelist_benchmark.cc index 552ae84af..598fc1c43 100644 --- a/tcmalloc/central_freelist_benchmark.cc +++ b/tcmalloc/central_freelist_benchmark.cc @@ -50,13 +50,8 @@ namespace { void* BenchmarkMetaDataAlloc(size_t bytes) { return ::operator new(bytes); } -#ifdef TCMALLOC_USE_PAGEMAP3 using BenchmarkPageMap = PageMap3; -#else -using BenchmarkPageMap = - PageMap2; -#endif // BenchmarkStaticForwarder provides a wrapper around ordinary TCMalloc and a // PageMap to allow us to carve up memory we obtained into our own objects. diff --git a/tcmalloc/common.h b/tcmalloc/common.h index d17e94b78..936e6e19c 100644 --- a/tcmalloc/common.h +++ b/tcmalloc/common.h @@ -99,7 +99,6 @@ static_assert(sizeof(void*) == 8); #ifndef TCMALLOC_PAGE_SHIFT #ifdef TCMALLOC_INTERNAL_SMALL_BUT_SLOW #define TCMALLOC_PAGE_SHIFT 12 -#define TCMALLOC_USE_PAGEMAP3 #elif defined(TCMALLOC_INTERNAL_256K_PAGES) #define TCMALLOC_PAGE_SHIFT 18 #elif defined(TCMALLOC_INTERNAL_32K_PAGES) diff --git a/tcmalloc/pagemap.cc b/tcmalloc/pagemap.cc index 763282435..2a5152046 100644 --- a/tcmalloc/pagemap.cc +++ b/tcmalloc/pagemap.cc @@ -50,27 +50,6 @@ void PageMap::UnregisterSizeClass(Span* span) { } } -void PageMap::MapRootWithSmallPages() { - constexpr size_t kHugePageMask = ~(kHugePageSize - 1); - uintptr_t begin = reinterpret_cast(map_.RootAddress()); - // Round begin up to the nearest hugepage, this avoids causing memory before - // the start of the pagemap to become mapped onto small pages. - uintptr_t rbegin = (begin + kHugePageSize - 1) & kHugePageMask; - size_t length = map_.RootSize(); - // Round end down to the nearest hugepage, this avoids causing memory after - // the end of the pagemap becoming mapped onto small pages. - size_t rend = (begin + length) & kHugePageMask; - // Since we have rounded the start up, and the end down, we also want to - // confirm that there is something left between them for us to modify. - // For small but slow, the root pagemap is less than a hugepage in size, - // so we will not end up forcing it to be small pages. - if (rend > rbegin) { - size_t rlength = rend - rbegin; - ErrnoRestorer errno_restorer; - madvise(reinterpret_cast(rbegin), rlength, MADV_NOHUGEPAGE); - } -} - void* MetaDataAlloc(size_t bytes) { return tc_globals.arena().Alloc(bytes); } } // namespace tcmalloc_internal diff --git a/tcmalloc/pagemap.h b/tcmalloc/pagemap.h index 602d4de34..db97fac52 100644 --- a/tcmalloc/pagemap.h +++ b/tcmalloc/pagemap.h @@ -76,214 +76,6 @@ class PackedSpanAndSizeclass { static constexpr uintptr_t kSpanMask = (uintptr_t{1} << kSizeclassShift) - 1; }; -template -class PageMap2 { - private: - // The leaf node (regardless of pointer size) always maps 2^15 entries; - // with 8K pages, this gives us 256MB mapped per leaf node. - static constexpr int kLeafBits = 15; - static constexpr int kLeafLength = 1 << kLeafBits; - static constexpr int kRootBits = (BITS >= kLeafBits) ? (BITS - kLeafBits) : 0; - // (1<= kHugePageSize, "leaf too small"); - static constexpr size_t kLeafHugeBits = - (kLeafBits + kPageShift - kHugePageShift); - static constexpr size_t kLeafHugepages = kLeafCoveredBytes / kHugePageSize; - static_assert(kLeafHugepages == 1 << kLeafHugeBits, "sanity"); - struct Leaf { - // We keep parallel arrays indexed by page number. One keeps the - // size class; another span pointers; the last hugepage-related - // information. The size class information is kept segregated - // since small object deallocations are so frequent and do not - // need the other information kept in a Span. - CompactSizeClass sizeclass[kLeafLength]; - // Span pointers, with the top two most significant bytes used to also - // store a redundant copy of the sizeclass. This allows us to avoid two - // separate memory loads when fetching both the span and the sizeclass. - PackedSpanAndSizeclass span_and_sizeclass[kLeafLength]; - void* hugepage[kLeafHugepages]; - - Span* absl_nullable span(int i) const { - return span_and_sizeclass[i].span(); - } - }; - - Leaf* absl_nullable root_[kRootLength]; // Top-level node - size_t bytes_used_; - - public: - typedef uintptr_t Number; - - constexpr PageMap2() : root_{}, bytes_used_(0) {} - - // No locks required. See SYNCHRONIZATION explanation at top of tcmalloc.cc. - Span* absl_nullable get(Number k) const ABSL_NO_THREAD_SAFETY_ANALYSIS { - const Number i1 = k >> kLeafBits; - const Number i2 = k & (kLeafLength - 1); - if (ABSL_PREDICT_FALSE((k >> BITS) > 0) || - ABSL_PREDICT_FALSE(root_[i1] == nullptr)) { - return nullptr; - } - return root_[i1]->span(i2); - } - - // No locks required. See SYNCHRONIZATION explanation at top of tcmalloc.cc. - std::optional get_next_set_page(Number k) const { - Number next_k = k + 1; - Number i1 = next_k >> kLeafBits; - Number i2 = next_k & (kLeafLength - 1); - for (; i1 < kRootLength; ++i1, i2 = 0) { - if (root_[i1] == nullptr) continue; - for (; i2 < kLeafLength; ++i2) { - if (root_[i1]->span(i2) != nullptr) return (i1 << kLeafBits) | i2; - } - } - return std::nullopt; - } - - // No locks required. See SYNCHRONIZATION explanation at top of tcmalloc.cc. - // Requires that the span is known to already exist. - // - // ABSL_ATTRIBUTE_NO_SANITIZE_UNDEFINED is to disable array-bounds sanitizer. - // This function is hot, and we can manually prove the array accesses. - // - // TODO(b/406313446): Remove ABSL_ATTRIBUTE_NO_SANITIZE_UNDEFINED once clang - // optimizes out the array bounds check. - template - std::pair get_existing_with_sizeclass( - Number k) const ABSL_NO_THREAD_SAFETY_ANALYSIS -#ifdef __clang__ - ABSL_ATTRIBUTE_NO_SANITIZE_UNDEFINED -#endif // __clang__ - { - const Number i1 = k >> kLeafBits; - const Number i2 = k & (kLeafLength - 1); - if constexpr (check_bounds) { - if (ABSL_PREDICT_FALSE((k >> BITS > 0)) || - ABSL_PREDICT_FALSE(root_[i1] == nullptr)) { - return std::make_pair(nullptr, 0); - } - } - TC_ASSERT_EQ(k >> BITS, 0); - TC_ASSERT_NE(root_[i1], nullptr); - // This is a static_assert to ensure that the index into root_ is within - // bounds. The index into span_and_sizeclass is trivially within bounds, - // because kLeafLength = 1 << kLeafBits, and i2 masks to kLeafLength - 1 - // bits. - static_assert((((Number(1) << BITS) - 1) >> kLeafBits) < kRootLength); - PackedSpanAndSizeclass span_and_sizeclass = - root_[i1]->span_and_sizeclass[i2]; - return std::make_pair(span_and_sizeclass.span(), - span_and_sizeclass.sizeclass()); - } - - // No locks required. See SYNCHRONIZATION explanation at top of tcmalloc.cc. - // Requires that the span is known to already exist. - Span* absl_nullable get_existing(Number k) const - ABSL_NO_THREAD_SAFETY_ANALYSIS { - const Number i1 = k >> kLeafBits; - const Number i2 = k & (kLeafLength - 1); - TC_ASSERT_EQ(k >> BITS, 0); - TC_ASSERT_NE(root_[i1], nullptr); - return root_[i1]->span(i2); - } - - // No locks required. See SYNCHRONIZATION explanation at top of tcmalloc.cc. - // REQUIRES: Must be a valid page number previously Ensure()d. - CompactSizeClass ABSL_ATTRIBUTE_ALWAYS_INLINE - sizeclass(Number k) const ABSL_NO_THREAD_SAFETY_ANALYSIS { - const Number i1 = k >> kLeafBits; - if (ABSL_PREDICT_FALSE((k >> BITS) > 0) || - ABSL_PREDICT_FALSE(root_[i1] == nullptr)) { - return 0; - } - const Number i2 = k & (kLeafLength - 1); - auto ret = root_[i1]->sizeclass[i2]; - TC_ASSERT_EQ(ret, root_[i1]->span_and_sizeclass[i2].sizeclass()); - return ret; - } - - void set(Number k, Span* s) { - TC_ASSERT_EQ(k >> BITS, 0); - const Number i1 = k >> kLeafBits; - const Number i2 = k & (kLeafLength - 1); - Leaf* leaf = root_[i1]; - // This function should be used just after allocating a new Span; - // in that case, the sizeclass should have been left at zero when the - // old span was deallocated/unregistered (or it would have been zero - // at initialization time.) - TC_ASSERT_EQ(leaf->sizeclass[i2], 0); - leaf->span_and_sizeclass[i2].set(s, 0); - } - - void set_with_sizeclass(Number k, Span* s, CompactSizeClass sc) { - TC_ASSERT_EQ(k >> BITS, 0); - const Number i1 = k >> kLeafBits; - const Number i2 = k & (kLeafLength - 1); - Leaf* leaf = root_[i1]; - leaf->span_and_sizeclass[i2].set(s, sc); - leaf->sizeclass[i2] = sc; - } - - void clear_sizeclass(Number k) { - TC_ASSERT_EQ(k >> BITS, 0); - const Number i1 = k >> kLeafBits; - const Number i2 = k & (kLeafLength - 1); - root_[i1]->sizeclass[i2] = 0; - } - - void* get_hugepage(Number k) { - TC_ASSERT_EQ(k >> BITS, 0); - const Number i1 = k >> kLeafBits; - const Number i2 = k & (kLeafLength - 1); - const Leaf* leaf = root_[i1]; - TC_ASSERT_NE(leaf, nullptr); - return leaf->hugepage[i2 >> (kLeafBits - kLeafHugeBits)]; - } - - void set_hugepage(Number k, void* v) { - TC_ASSERT_EQ(k >> BITS, 0); - const Number i1 = k >> kLeafBits; - const Number i2 = k & (kLeafLength - 1); - root_[i1]->hugepage[i2 >> (kLeafBits - kLeafHugeBits)] = v; - } - - bool Ensure(Number start, size_t n) { - TC_ASSERT_GT(n, 0); - for (Number key = start; key <= start + n - 1;) { - const Number i1 = key >> kLeafBits; - - // Check for overflow - if (i1 >= kRootLength) return false; - - // Make 2nd level node if necessary - if (root_[i1] == nullptr) { - Leaf* leaf = reinterpret_cast(Allocator(sizeof(Leaf))); - if (leaf == nullptr) return false; - bytes_used_ += sizeof(Leaf); - memset(leaf, 0, sizeof(*leaf)); - root_[i1] = leaf; - } - - // Advance key past whatever is covered by this leaf node - key = ((key >> kLeafBits) + 1) << kLeafBits; - } - return true; - } - - size_t bytes_used() const { - // Account for size of root node, etc. - return bytes_used_ + sizeof(*this); - } - - constexpr size_t RootSize() const { return sizeof(root_); } - const void* RootAddress() { return root_; } -}; - // Three-level radix tree // Currently only used for TCMALLOC_INTERNAL_SMALL_BUT_SLOW template @@ -529,7 +321,6 @@ class PageMap3 { size_t bytes_used() const { return bytes_used_ + sizeof(*this); } constexpr size_t RootSize() const { return sizeof(root_); } - const void* RootAddress() { return root_; } }; class PageMap { @@ -608,12 +399,6 @@ class PageMap { void SetHugepage(PageId p, void* v) { map_.set_hugepage(p.index(), v); } - // The PageMap root node can be quite large and sparsely used. If this - // gets mapped with hugepages we potentially end up holding a large - // amount of unused memory. So it is better to map the root node with - // small pages to minimise the amount of unused memory. - void MapRootWithSmallPages(); - // Returns the count of the currently allocated Spans and also adds details // of such Spans in the provided allocated_spans vector. This routine avoids // allocation events since we hold the pageheap_lock, so no more elements will @@ -650,11 +435,7 @@ class PageMap { } private: -#ifdef TCMALLOC_USE_PAGEMAP3 PageMap3 map_; -#else - PageMap2 map_; -#endif }; } // namespace tcmalloc_internal diff --git a/tcmalloc/pagemap_test.cc b/tcmalloc/pagemap_test.cc index db566b120..252f51483 100644 --- a/tcmalloc/pagemap_test.cc +++ b/tcmalloc/pagemap_test.cc @@ -73,7 +73,8 @@ class PageMapTest : public ::testing::TestWithParam { } public: - using Map = PageMap2<20, alloc>; + static constexpr int kTestBits = kAddressBits - kPageShift; + using Map = PageMap3; Map* map; private: @@ -117,7 +118,7 @@ TEST_P(PageMapTest, Bulk) { } TEST_P(PageMapTest, Overflow) { - const intptr_t kLimit = 1 << 20; + const uintptr_t kLimit = uintptr_t{1} << kTestBits; ASSERT_FALSE(map->Ensure(kLimit, kLimit + 1)); } diff --git a/tcmalloc/static_vars.cc b/tcmalloc/static_vars.cc index cfb1b008e..1b92aa63f 100644 --- a/tcmalloc/static_vars.cc +++ b/tcmalloc/static_vars.cc @@ -237,7 +237,6 @@ ABSL_ATTRIBUTE_COLD ABSL_ATTRIBUTE_NOINLINE void Static::SlowInitIfNecessary() { // state. sharded_transfer_cache_.Init(); new (page_allocator_.memory) PageAllocator; - pagemap_.MapRootWithSmallPages(); guardedpage_allocator_.Init(/*max_allocated_pages=*/64, /*total_pages=*/128); diff --git a/tcmalloc/testing/fast_path.aarch64+opt.golden b/tcmalloc/testing/fast_path.aarch64+opt.golden index 9a4e0394f..d161832af 100644 --- a/tcmalloc/testing/fast_path.aarch64+opt.golden +++ b/tcmalloc/testing/fast_path.aarch64+opt.golden @@ -1,7 +1,7 @@ func call cmov cmp div jcc jmp load mul pop push rmw rsp store aligned_alloc - - 4 - 7 - 8 - - - - 3 4 calloc - - 2 - 6 - 9 2 - - - 5 5 -delete - - 2 - 5 - 6 - - - - - 3 +delete - - 2 - 6 - 7 - - - - - 3 delete(size) - - 3 - 5 - 5 - - - - - 3 delete(size,align) - - 5 - 7 - 6 - - - - - 3 malloc - - 2 - 5 - 7 - - - - - 3 diff --git a/tcmalloc/testing/fast_path.insecure.golden b/tcmalloc/testing/fast_path.insecure.golden index 8244992d8..07b48a8f4 100644 --- a/tcmalloc/testing/fast_path.insecure.golden +++ b/tcmalloc/testing/fast_path.insecure.golden @@ -1,7 +1,7 @@ func call cmov cmp div jcc jmp load mul pop push rmw rsp store aligned_alloc - - 6 - 7 - 6 - 2 1 1 1 2 calloc - - 4 - 6 - 6 1 2 3 1 2 2 -delete - - 5 - 5 - 5 - - - - - 3 +delete - - 6 - 6 - 6 - - - - - 3 delete(size) - - 5 - 5 - 4 - - - - - 3 delete(size,align) - - 7 - 7 - 5 - - - - - 3 malloc - - 4 - 5 - 6 - - - 1 - 2 diff --git a/tcmalloc/testing/fast_path.opt.golden b/tcmalloc/testing/fast_path.opt.golden index 8af92e101..6fc69a987 100644 --- a/tcmalloc/testing/fast_path.opt.golden +++ b/tcmalloc/testing/fast_path.opt.golden @@ -1,7 +1,7 @@ func call cmov cmp div jcc jmp load mul pop push rmw rsp store aligned_alloc - - 6 - 7 - 6 - 2 1 1 1 2 calloc - - 4 - 6 - 6 1 2 3 1 2 2 -delete - - 5 - 5 - 5 - - - - - 3 +delete - - 6 - 6 - 6 - - - - - 3 delete(size) - - 5 - 5 - 4 - - - - - 3 delete(size,align) - - 7 - 7 - 5 - - - - - 3 malloc - - 4 - 5 - 6 - - - 1 - 2 diff --git a/tcmalloc/testing/fast_path.release+insecure.golden b/tcmalloc/testing/fast_path.release+insecure.golden index 028517776..07b48a8f4 100644 --- a/tcmalloc/testing/fast_path.release+insecure.golden +++ b/tcmalloc/testing/fast_path.release+insecure.golden @@ -1,7 +1,7 @@ func call cmov cmp div jcc jmp load mul pop push rmw rsp store aligned_alloc - - 6 - 7 - 6 - 2 1 1 1 2 calloc - - 4 - 6 - 6 1 2 3 1 2 2 -delete - - 5 - 5 - 5 - - - - - 3 +delete - - 6 - 6 - 6 - - - - - 3 delete(size) - - 5 - 5 - 4 - - - - - 3 delete(size,align) - - 7 - 7 - 5 - - - - - 3 malloc - - 4 - 5 - 6 - - - 1 - 2 @@ -12,7 +12,7 @@ new(align)->size - - 5 - 6 - 7 - - - 1 - new(align,cold)->size - 1 6 - 6 - 8 - - - 1 - 2 new(align,nothrow) - - 5 - 6 - 6 - - - 1 - 2 new(cold)->size - 1 4 - 4 - 7 - - - 1 - 2 -new(cold,token_1)->size - - 4 - 5 - 7 - - - 1 - 2 +new(cold,token_1)->size - 1 4 - 4 - 7 - - - 1 - 2 new(nothrow) - - 3 - 4 - 5 - - - 1 - 2 new(token_1) - - 3 - 4 - 5 - - - 1 - 2 new(token_1)->size - - 3 - 4 - 6 - - - 1 - 2 diff --git a/tcmalloc/testing/fast_path.release.golden b/tcmalloc/testing/fast_path.release.golden index 028517776..07b48a8f4 100644 --- a/tcmalloc/testing/fast_path.release.golden +++ b/tcmalloc/testing/fast_path.release.golden @@ -1,7 +1,7 @@ func call cmov cmp div jcc jmp load mul pop push rmw rsp store aligned_alloc - - 6 - 7 - 6 - 2 1 1 1 2 calloc - - 4 - 6 - 6 1 2 3 1 2 2 -delete - - 5 - 5 - 5 - - - - - 3 +delete - - 6 - 6 - 6 - - - - - 3 delete(size) - - 5 - 5 - 4 - - - - - 3 delete(size,align) - - 7 - 7 - 5 - - - - - 3 malloc - - 4 - 5 - 6 - - - 1 - 2 @@ -12,7 +12,7 @@ new(align)->size - - 5 - 6 - 7 - - - 1 - new(align,cold)->size - 1 6 - 6 - 8 - - - 1 - 2 new(align,nothrow) - - 5 - 6 - 6 - - - 1 - 2 new(cold)->size - 1 4 - 4 - 7 - - - 1 - 2 -new(cold,token_1)->size - - 4 - 5 - 7 - - - 1 - 2 +new(cold,token_1)->size - 1 4 - 4 - 7 - - - 1 - 2 new(nothrow) - - 3 - 4 - 5 - - - 1 - 2 new(token_1) - - 3 - 4 - 5 - - - 1 - 2 new(token_1)->size - - 3 - 4 - 6 - - - 1 - 2