Skip to content

Enable TCMALLOC_USE_PAGEMAP3 unconditionally. #466

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Aug 12, 2026
+10 −256
Merged

Enable TCMALLOC_USE_PAGEMAP3 unconditionally. #466

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions tcmalloc/central_freelist_benchmark.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,13 +50,8 @@ namespace {

void* BenchmarkMetaDataAlloc(size_t bytes) { return ::operator new(bytes); }

#ifdef TCMALLOC_USE_PAGEMAP3
using BenchmarkPageMap =
PageMap3<kAddressBits - kPageShift, BenchmarkMetaDataAlloc>;
#else
using BenchmarkPageMap =
PageMap2<kAddressBits - kPageShift, BenchmarkMetaDataAlloc>;
#endif

// BenchmarkStaticForwarder provides a wrapper around ordinary TCMalloc and a
// PageMap to allow us to carve up memory we obtained into our own objects.
Expand Down
1 change: 0 additions & 1 deletion tcmalloc/common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,6 @@ static_assert(sizeof(void*) == 8);
#ifndef TCMALLOC_PAGE_SHIFT
#ifdef TCMALLOC_INTERNAL_SMALL_BUT_SLOW
#define TCMALLOC_PAGE_SHIFT 12
#define TCMALLOC_USE_PAGEMAP3
#elif defined(TCMALLOC_INTERNAL_256K_PAGES)
#define TCMALLOC_PAGE_SHIFT 18
#elif defined(TCMALLOC_INTERNAL_32K_PAGES)
Expand Down
21 changes: 0 additions & 21 deletions tcmalloc/pagemap.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,27 +50,6 @@ void PageMap::UnregisterSizeClass(Span* span) {
}
}

void PageMap::MapRootWithSmallPages() {
constexpr size_t kHugePageMask = ~(kHugePageSize - 1);
uintptr_t begin = reinterpret_cast<uintptr_t>(map_.RootAddress());
// Round begin up to the nearest hugepage, this avoids causing memory before
// the start of the pagemap to become mapped onto small pages.
uintptr_t rbegin = (begin + kHugePageSize - 1) & kHugePageMask;
size_t length = map_.RootSize();
// Round end down to the nearest hugepage, this avoids causing memory after
// the end of the pagemap becoming mapped onto small pages.
size_t rend = (begin + length) & kHugePageMask;
// Since we have rounded the start up, and the end down, we also want to
// confirm that there is something left between them for us to modify.
// For small but slow, the root pagemap is less than a hugepage in size,
// so we will not end up forcing it to be small pages.
if (rend > rbegin) {
size_t rlength = rend - rbegin;
ErrnoRestorer errno_restorer;
madvise(reinterpret_cast<void*>(rbegin), rlength, MADV_NOHUGEPAGE);
}
}

void* MetaDataAlloc(size_t bytes) { return tc_globals.arena().Alloc(bytes); }

} // namespace tcmalloc_internal
Expand Down
219 changes: 0 additions & 219 deletions tcmalloc/pagemap.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,214 +76,6 @@ class PackedSpanAndSizeclass {
static constexpr uintptr_t kSpanMask = (uintptr_t{1} << kSizeclassShift) - 1;
};

template <int BITS, PagemapAllocator Allocator>
class PageMap2 {
private:
// The leaf node (regardless of pointer size) always maps 2^15 entries;
// with 8K pages, this gives us 256MB mapped per leaf node.
static constexpr int kLeafBits = 15;
static constexpr int kLeafLength = 1 << kLeafBits;
static constexpr int kRootBits = (BITS >= kLeafBits) ? (BITS - kLeafBits) : 0;
// (1<<kRootBits) must not overflow an "int"
static_assert(kRootBits < sizeof(int) * 8 - 1, "kRootBits is too large");
static constexpr int kRootLength = 1 << kRootBits;

static constexpr size_t kLeafCoveredBytes = 1ul << (kLeafBits + kPageShift);
static_assert(kLeafCoveredBytes >= kHugePageSize, "leaf too small");
static constexpr size_t kLeafHugeBits =
(kLeafBits + kPageShift - kHugePageShift);
static constexpr size_t kLeafHugepages = kLeafCoveredBytes / kHugePageSize;
static_assert(kLeafHugepages == 1 << kLeafHugeBits, "sanity");
struct Leaf {
// We keep parallel arrays indexed by page number. One keeps the
// size class; another span pointers; the last hugepage-related
// information. The size class information is kept segregated
// since small object deallocations are so frequent and do not
// need the other information kept in a Span.
CompactSizeClass sizeclass[kLeafLength];
// Span pointers, with the top two most significant bytes used to also
// store a redundant copy of the sizeclass. This allows us to avoid two
// separate memory loads when fetching both the span and the sizeclass.
PackedSpanAndSizeclass span_and_sizeclass[kLeafLength];
void* hugepage[kLeafHugepages];

Span* absl_nullable span(int i) const {
return span_and_sizeclass[i].span();
}
};

Leaf* absl_nullable root_[kRootLength]; // Top-level node
size_t bytes_used_;

public:
typedef uintptr_t Number;

constexpr PageMap2() : root_{}, bytes_used_(0) {}

// No locks required. See SYNCHRONIZATION explanation at top of tcmalloc.cc.
Span* absl_nullable get(Number k) const ABSL_NO_THREAD_SAFETY_ANALYSIS {
const Number i1 = k >> kLeafBits;
const Number i2 = k & (kLeafLength - 1);
if (ABSL_PREDICT_FALSE((k >> BITS) > 0) ||
ABSL_PREDICT_FALSE(root_[i1] == nullptr)) {
return nullptr;
}
return root_[i1]->span(i2);
}

// No locks required. See SYNCHRONIZATION explanation at top of tcmalloc.cc.
std::optional<Number> get_next_set_page(Number k) const {
Number next_k = k + 1;
Number i1 = next_k >> kLeafBits;
Number i2 = next_k & (kLeafLength - 1);
for (; i1 < kRootLength; ++i1, i2 = 0) {
if (root_[i1] == nullptr) continue;
for (; i2 < kLeafLength; ++i2) {
if (root_[i1]->span(i2) != nullptr) return (i1 << kLeafBits) | i2;
}
}
return std::nullopt;
}

// No locks required. See SYNCHRONIZATION explanation at top of tcmalloc.cc.
// Requires that the span is known to already exist.
//
// ABSL_ATTRIBUTE_NO_SANITIZE_UNDEFINED is to disable array-bounds sanitizer.
// This function is hot, and we can manually prove the array accesses.
//
// TODO(b/406313446): Remove ABSL_ATTRIBUTE_NO_SANITIZE_UNDEFINED once clang
// optimizes out the array bounds check.
template <bool check_bounds>
std::pair<Span* absl_nullable, int> get_existing_with_sizeclass(
Number k) const ABSL_NO_THREAD_SAFETY_ANALYSIS
#ifdef __clang__
ABSL_ATTRIBUTE_NO_SANITIZE_UNDEFINED
#endif // __clang__
{
const Number i1 = k >> kLeafBits;
const Number i2 = k & (kLeafLength - 1);
if constexpr (check_bounds) {
if (ABSL_PREDICT_FALSE((k >> BITS > 0)) ||
ABSL_PREDICT_FALSE(root_[i1] == nullptr)) {
return std::make_pair(nullptr, 0);
}
}
TC_ASSERT_EQ(k >> BITS, 0);
TC_ASSERT_NE(root_[i1], nullptr);
// This is a static_assert to ensure that the index into root_ is within
// bounds. The index into span_and_sizeclass is trivially within bounds,
// because kLeafLength = 1 << kLeafBits, and i2 masks to kLeafLength - 1
// bits.
static_assert((((Number(1) << BITS) - 1) >> kLeafBits) < kRootLength);
PackedSpanAndSizeclass span_and_sizeclass =
root_[i1]->span_and_sizeclass[i2];
return std::make_pair(span_and_sizeclass.span(),
span_and_sizeclass.sizeclass());
}

// No locks required. See SYNCHRONIZATION explanation at top of tcmalloc.cc.
// Requires that the span is known to already exist.
Span* absl_nullable get_existing(Number k) const
ABSL_NO_THREAD_SAFETY_ANALYSIS {
const Number i1 = k >> kLeafBits;
const Number i2 = k & (kLeafLength - 1);
TC_ASSERT_EQ(k >> BITS, 0);
TC_ASSERT_NE(root_[i1], nullptr);
return root_[i1]->span(i2);
}

// No locks required. See SYNCHRONIZATION explanation at top of tcmalloc.cc.
// REQUIRES: Must be a valid page number previously Ensure()d.
CompactSizeClass ABSL_ATTRIBUTE_ALWAYS_INLINE
sizeclass(Number k) const ABSL_NO_THREAD_SAFETY_ANALYSIS {
const Number i1 = k >> kLeafBits;
if (ABSL_PREDICT_FALSE((k >> BITS) > 0) ||
ABSL_PREDICT_FALSE(root_[i1] == nullptr)) {
return 0;
}
const Number i2 = k & (kLeafLength - 1);
auto ret = root_[i1]->sizeclass[i2];
TC_ASSERT_EQ(ret, root_[i1]->span_and_sizeclass[i2].sizeclass());
return ret;
}

void set(Number k, Span* s) {
TC_ASSERT_EQ(k >> BITS, 0);
const Number i1 = k >> kLeafBits;
const Number i2 = k & (kLeafLength - 1);
Leaf* leaf = root_[i1];
// This function should be used just after allocating a new Span;
// in that case, the sizeclass should have been left at zero when the
// old span was deallocated/unregistered (or it would have been zero
// at initialization time.)
TC_ASSERT_EQ(leaf->sizeclass[i2], 0);
leaf->span_and_sizeclass[i2].set(s, 0);
}

void set_with_sizeclass(Number k, Span* s, CompactSizeClass sc) {
TC_ASSERT_EQ(k >> BITS, 0);
const Number i1 = k >> kLeafBits;
const Number i2 = k & (kLeafLength - 1);
Leaf* leaf = root_[i1];
leaf->span_and_sizeclass[i2].set(s, sc);
leaf->sizeclass[i2] = sc;
}

void clear_sizeclass(Number k) {
TC_ASSERT_EQ(k >> BITS, 0);
const Number i1 = k >> kLeafBits;
const Number i2 = k & (kLeafLength - 1);
root_[i1]->sizeclass[i2] = 0;
}

void* get_hugepage(Number k) {
TC_ASSERT_EQ(k >> BITS, 0);
const Number i1 = k >> kLeafBits;
const Number i2 = k & (kLeafLength - 1);
const Leaf* leaf = root_[i1];
TC_ASSERT_NE(leaf, nullptr);
return leaf->hugepage[i2 >> (kLeafBits - kLeafHugeBits)];
}

void set_hugepage(Number k, void* v) {
TC_ASSERT_EQ(k >> BITS, 0);
const Number i1 = k >> kLeafBits;
const Number i2 = k & (kLeafLength - 1);
root_[i1]->hugepage[i2 >> (kLeafBits - kLeafHugeBits)] = v;
}

bool Ensure(Number start, size_t n) {
TC_ASSERT_GT(n, 0);
for (Number key = start; key <= start + n - 1;) {
const Number i1 = key >> kLeafBits;

// Check for overflow
if (i1 >= kRootLength) return false;

// Make 2nd level node if necessary
if (root_[i1] == nullptr) {
Leaf* leaf = reinterpret_cast<Leaf*>(Allocator(sizeof(Leaf)));
if (leaf == nullptr) return false;
bytes_used_ += sizeof(Leaf);
memset(leaf, 0, sizeof(*leaf));
root_[i1] = leaf;
}

// Advance key past whatever is covered by this leaf node
key = ((key >> kLeafBits) + 1) << kLeafBits;
}
return true;
}

size_t bytes_used() const {
// Account for size of root node, etc.
return bytes_used_ + sizeof(*this);
}

constexpr size_t RootSize() const { return sizeof(root_); }
const void* RootAddress() { return root_; }
};

// Three-level radix tree
// Currently only used for TCMALLOC_INTERNAL_SMALL_BUT_SLOW
template <int BITS, PagemapAllocator Allocator>
Expand Down Expand Up @@ -529,7 +321,6 @@ class PageMap3 {
size_t bytes_used() const { return bytes_used_ + sizeof(*this); }

constexpr size_t RootSize() const { return sizeof(root_); }
const void* RootAddress() { return root_; }
};

class PageMap {
Expand Down Expand Up @@ -608,12 +399,6 @@ class PageMap {

void SetHugepage(PageId p, void* v) { map_.set_hugepage(p.index(), v); }

// The PageMap root node can be quite large and sparsely used. If this
// gets mapped with hugepages we potentially end up holding a large
// amount of unused memory. So it is better to map the root node with
// small pages to minimise the amount of unused memory.
void MapRootWithSmallPages();

// Returns the count of the currently allocated Spans and also adds details
// of such Spans in the provided allocated_spans vector. This routine avoids
// allocation events since we hold the pageheap_lock, so no more elements will
Expand Down Expand Up @@ -650,11 +435,7 @@ class PageMap {
}

private:
#ifdef TCMALLOC_USE_PAGEMAP3
PageMap3<kAddressBits - kPageShift, MetaDataAlloc> map_;
#else
PageMap2<kAddressBits - kPageShift, MetaDataAlloc> map_;
#endif
};

} // namespace tcmalloc_internal
Expand Down
5 changes: 3 additions & 2 deletions tcmalloc/pagemap_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ class PageMapTest : public ::testing::TestWithParam<int> {
}

public:
using Map = PageMap2<20, alloc>;
static constexpr int kTestBits = kAddressBits - kPageShift;
using Map = PageMap3<kTestBits, alloc>;
Map* map;

private:
Expand Down Expand Up @@ -117,7 +118,7 @@ TEST_P(PageMapTest, Bulk) {
}

TEST_P(PageMapTest, Overflow) {
const intptr_t kLimit = 1 << 20;
const uintptr_t kLimit = uintptr_t{1} << kTestBits;
ASSERT_FALSE(map->Ensure(kLimit, kLimit + 1));
}

Expand Down
1 change: 0 additions & 1 deletion tcmalloc/static_vars.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,6 @@ ABSL_ATTRIBUTE_COLD ABSL_ATTRIBUTE_NOINLINE void Static::SlowInitIfNecessary() {
// state.
sharded_transfer_cache_.Init();
new (page_allocator_.memory) PageAllocator;
pagemap_.MapRootWithSmallPages();
guardedpage_allocator_.Init(/*max_allocated_pages=*/64,
/*total_pages=*/128);

Expand Down
2 changes: 1 addition & 1 deletion tcmalloc/testing/fast_path.aarch64+opt.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
func call cmov cmp div jcc jmp load mul pop push rmw rsp store
aligned_alloc - - 4 - 7 - 8 - - - - 3 4
calloc - - 2 - 6 - 9 2 - - - 5 5
delete - - 2 - 5 - 6 - - - - - 3
delete - - 2 - 6 - 7 - - - - - 3
delete(size) - - 3 - 5 - 5 - - - - - 3
delete(size,align) - - 5 - 7 - 6 - - - - - 3
malloc - - 2 - 5 - 7 - - - - - 3
Expand Down
2 changes: 1 addition & 1 deletion tcmalloc/testing/fast_path.insecure.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
func call cmov cmp div jcc jmp load mul pop push rmw rsp store
aligned_alloc - - 6 - 7 - 6 - 2 1 1 1 2
calloc - - 4 - 6 - 6 1 2 3 1 2 2
delete - - 5 - 5 - 5 - - - - - 3
delete - - 6 - 6 - 6 - - - - - 3
delete(size) - - 5 - 5 - 4 - - - - - 3
delete(size,align) - - 7 - 7 - 5 - - - - - 3
malloc - - 4 - 5 - 6 - - - 1 - 2
Expand Down
2 changes: 1 addition & 1 deletion tcmalloc/testing/fast_path.opt.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
func call cmov cmp div jcc jmp load mul pop push rmw rsp store
aligned_alloc - - 6 - 7 - 6 - 2 1 1 1 2
calloc - - 4 - 6 - 6 1 2 3 1 2 2
delete - - 5 - 5 - 5 - - - - - 3
delete - - 6 - 6 - 6 - - - - - 3
delete(size) - - 5 - 5 - 4 - - - - - 3
delete(size,align) - - 7 - 7 - 5 - - - - - 3
malloc - - 4 - 5 - 6 - - - 1 - 2
Expand Down
4 changes: 2 additions & 2 deletions tcmalloc/testing/fast_path.release+insecure.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
func call cmov cmp div jcc jmp load mul pop push rmw rsp store
aligned_alloc - - 6 - 7 - 6 - 2 1 1 1 2
calloc - - 4 - 6 - 6 1 2 3 1 2 2
delete - - 5 - 5 - 5 - - - - - 3
delete - - 6 - 6 - 6 - - - - - 3
delete(size) - - 5 - 5 - 4 - - - - - 3
delete(size,align) - - 7 - 7 - 5 - - - - - 3
malloc - - 4 - 5 - 6 - - - 1 - 2
Expand All @@ -12,7 +12,7 @@ new(align)->size - - 5 - 6 - 7 - - - 1 -
new(align,cold)->size - 1 6 - 6 - 8 - - - 1 - 2
new(align,nothrow) - - 5 - 6 - 6 - - - 1 - 2
new(cold)->size - 1 4 - 4 - 7 - - - 1 - 2
new(cold,token_1)->size - - 4 - 5 - 7 - - - 1 - 2
new(cold,token_1)->size - 1 4 - 4 - 7 - - - 1 - 2
new(nothrow) - - 3 - 4 - 5 - - - 1 - 2
new(token_1) - - 3 - 4 - 5 - - - 1 - 2
new(token_1)->size - - 3 - 4 - 6 - - - 1 - 2
Expand Down
4 changes: 2 additions & 2 deletions tcmalloc/testing/fast_path.release.golden
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
func call cmov cmp div jcc jmp load mul pop push rmw rsp store
aligned_alloc - - 6 - 7 - 6 - 2 1 1 1 2
calloc - - 4 - 6 - 6 1 2 3 1 2 2
delete - - 5 - 5 - 5 - - - - - 3
delete - - 6 - 6 - 6 - - - - - 3
delete(size) - - 5 - 5 - 4 - - - - - 3
delete(size,align) - - 7 - 7 - 5 - - - - - 3
malloc - - 4 - 5 - 6 - - - 1 - 2
Expand All @@ -12,7 +12,7 @@ new(align)->size - - 5 - 6 - 7 - - - 1 -
new(align,cold)->size - 1 6 - 6 - 8 - - - 1 - 2
new(align,nothrow) - - 5 - 6 - 6 - - - 1 - 2
new(cold)->size - 1 4 - 4 - 7 - - - 1 - 2
new(cold,token_1)->size - - 4 - 5 - 7 - - - 1 - 2
new(cold,token_1)->size - 1 4 - 4 - 7 - - - 1 - 2
new(nothrow) - - 3 - 4 - 5 - - - 1 - 2
new(token_1) - - 3 - 4 - 5 - - - 1 - 2
new(token_1)->size - - 3 - 4 - 6 - - - 1 - 2
Expand Down
Loading