Google Git
Sign in
chromium / external / github.com / gperftools / gperftools / refs/heads/master / . / src / tcmalloc.cc
blob: a860f2ec88f57ae11299e23b2f6b62113e210d49 [file] [log] [blame] [edit]
// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
// Copyright (c) 2005, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// ---
// Author: Sanjay Ghemawat <opensource@google.com>
//
// A malloc that uses a per-thread cache to satisfy small malloc requests.
// (The time for malloc/free of a small object drops from 300 ns to 50 ns.)
//
// See docs/tcmalloc.html for a high-level
// description of how this malloc works.
//
// SYNCHRONIZATION
// 1. The thread-specific lists are accessed without acquiring any locks.
// This is safe because each such list is only accessed by one thread.
// 2. We have a lock per central free-list, and hold it while manipulating
// the central free list for a particular size.
// 3. The central page allocator is protected by "pageheap_lock".
// 4. The pagemap (which maps from page-number to descriptor),
// can be read without holding any locks, and written while holding
// the "pageheap_lock".
// 5. To improve performance, a subset of the information one can get
// from the pagemap is cached in a data structure, pagemap_cache_,
// that atomically reads and writes its entries. This cache can be
// read and written without locking.
//
// This multi-threaded access to the pagemap is safe for fairly
// subtle reasons. We basically assume that when an object X is
// allocated by thread A and deallocated by thread B, there must
// have been appropriate synchronization in the handoff of object
// X from thread A to thread B. The same logic applies to pagemap_cache_.
//
// THE PAGEID-TO-SIZECLASS CACHE
// Hot PageID-to-sizeclass mappings are held by pagemap_cache_. If this cache
// returns 0 for a particular PageID then that means "no information," not that
// the sizeclass is 0. The cache may have stale information for pages that do
// not hold the beginning of any free()'able object. Staleness is eliminated
// in Populate() for pages with sizeclass > 0 objects, and in do_malloc() and
// do_memalign() for all other relevant pages.
//
// PAGEMAP
// -------
// Page map contains a mapping from page id to Span.
//
// If Span s occupies pages [p..q],
// pagemap[p] == s
// pagemap[q] == s
// pagemap[p+1..q-1] are undefined
// pagemap[p-1] and pagemap[q+1] are defined:
// nullptr if the corresponding page is not yet in the address space.
// Otherwise it points to a Span. This span may be free
// or allocated. If free, it is in one of pageheap's freelist.
//
// TODO: Bias reclamation to larger addresses
// TODO: implement mallinfo/mallopt
// TODO: Better testing
//
// 9/28/2003 (new page-level allocator replaces ptmalloc2):
// * malloc/free of small objects goes from ~300 ns to ~50 ns.
// * allocation of a reasonably complicated struct
// goes from about 1100 ns to about 300 ns.
#include "config.h"
// At least for gcc on Linux/i386 and Linux/amd64 not adding throw()
// to tc_xxx functions actually ends up generating better code.
#define PERFTOOLS_NOTHROW
#include <gperftools/tcmalloc.h>
#include <errno.h> // for ENOMEM, EINVAL, errno
#include <stdint.h>
#include <stddef.h> // for size_t
#include <stdlib.h> // for getenv
#include <string.h> // for strcmp, memset, strlen, etc
#ifdef HAVE_UNISTD_H
#include <unistd.h> // for getpagesize, write, etc
#endif
#include <algorithm> // for max, min
#include <limits> // for numeric_limits
#include <new> // for nothrow_t (ptr only), etc
#include <vector> // for vector
#include <gperftools/malloc_extension.h>
#include <gperftools/malloc_hook.h> // for MallocHook
#include <gperftools/nallocx.h>
#include "base/basictypes.h" // for int64
#include "base/commandlineflags.h" // for RegisterFlagValidator, etc
#include "base/dynamic_annotations.h" // for RunningOnValgrind
#include "base/spinlock.h" // for SpinLockHolder
#include "central_freelist.h"
#include "common.h" // for StackTrace, kPageShift, etc
#include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc
#include "linked_list.h" // for SLL_SetNext
#include "malloc_hook-inl.h" // for tcmalloc::InvokeNewHook, etc
#include "page_heap.h" // for PageHeap, PageHeap::Stats
#include "page_heap_allocator.h" // for PageHeapAllocator
#include "span.h" // for Span, DLL_Prepend, etc
#include "stack_trace_table.h" // for StackTraceTable
#include "static_vars.h" // for Static
#include "system-alloc.h" // for DumpSystemAllocatorStats, etc
#include "tcmalloc_guard.h" // for TCMallocGuard
#include "thread_cache.h" // for ThreadCache
#include "thread_cache_ptr.h"
#include "malloc_backtrace.h"
#include "maybe_emergency_malloc.h"
#include "testing_portal.h"
#if (defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)) && !defined(WIN32_OVERRIDE_ALLOCATORS)
# define WIN32_DO_PATCHING 1
#endif
// Some windows file somewhere (at least on cygwin) #define's small (!)
#undef small
#include "libc_override.h"
using tcmalloc::kLog;
using tcmalloc::kCrash;
using tcmalloc::Log;
using tcmalloc::PageHeap;
using tcmalloc::PageHeapAllocator;
using tcmalloc::SizeMap;
using tcmalloc::Span;
using tcmalloc::StackTrace;
using tcmalloc::Static;
using tcmalloc::ThreadCache;
using tcmalloc::ThreadCachePtr;
using tcmalloc::TestingPortal;
DECLARE_double(tcmalloc_release_rate);
DECLARE_int64(tcmalloc_heap_limit_mb);
// Those common architectures are known to be safe w.r.t. aliasing function
// with "extra" unused args to function with fewer arguments (e.g.
// tc_delete_nothrow being aliased to tc_delete).
//
// Benefit of aliasing is relatively moderate. It reduces instruction
// cache pressure a bit (not relevant for largely unused
// tc_delete_nothrow, but is potentially relevant for
// tc_delete_aligned (or sized)). It also used to be the case that gcc
// 5+ optimization for merging identical functions kicked in and
// "screwed" one of the otherwise identical functions with extra
// jump. I am not able to reproduce that anymore.
#if !defined(__i386__) && !defined(__x86_64__) && \
!defined(__ppc__) && !defined(__PPC__) && \
!defined(__aarch64__) && !defined(__mips__) && !defined(__arm__) && !defined(__loongarch64)
#undef TCMALLOC_NO_ALIASES
#define TCMALLOC_NO_ALIASES
#endif
#if defined(__GNUC__) && defined(__ELF__) && !defined(TCMALLOC_NO_ALIASES)
#define TC_ALIAS(name) __attribute__((alias(#name)))
#endif
// We already declared these functions in tcmalloc.h, but we have to
// declare them again to give them an ATTRIBUTE_SECTION: we want to
// put all callers of tcmalloc::Invoke* in this module into
// ATTRIBUTE_SECTION(google_malloc) section, so that
// MallocHook::GetCallerStackTrace can function accurately.
#ifndef _WIN32 // windows doesn't have attribute_section, so don't bother
extern "C" {
void* tc_malloc(size_t size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void tc_free(void* ptr) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void tc_free_sized(void* ptr, size_t size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void* tc_realloc(void* ptr, size_t size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void* tc_calloc(size_t nmemb, size_t size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void tc_cfree(void* ptr) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void* tc_memalign(size_t __alignment, size_t __size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
int tc_posix_memalign(void** ptr, size_t align, size_t size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void* tc_valloc(size_t __size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void* tc_pvalloc(size_t __size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void tc_malloc_stats(void) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
int tc_mallopt(int cmd, int value) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
#ifdef HAVE_STRUCT_MALLINFO
struct mallinfo tc_mallinfo(void) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
#endif
#ifdef HAVE_STRUCT_MALLINFO2
struct mallinfo2 tc_mallinfo2(void) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
#endif
void* tc_new(size_t size)
ATTRIBUTE_SECTION(google_malloc);
void tc_delete(void* p) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void tc_delete_sized(void* p, size_t size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void* tc_newarray(size_t size)
ATTRIBUTE_SECTION(google_malloc);
void tc_deletearray(void* p) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void tc_deletearray_sized(void* p, size_t size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
// And the nothrow variants of these:
void* tc_new_nothrow(size_t size, const std::nothrow_t&) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void* tc_newarray_nothrow(size_t size, const std::nothrow_t&) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
// Surprisingly, standard C++ library implementations use a
// nothrow-delete internally. See, eg:
// http://www.dinkumware.com/manuals/?manual=compleat&page=new.html
void tc_delete_nothrow(void* ptr, const std::nothrow_t&) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void tc_deletearray_nothrow(void* ptr, const std::nothrow_t&) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void* tc_new_aligned(size_t size, std::align_val_t al)
ATTRIBUTE_SECTION(google_malloc);
void tc_delete_aligned(void* p, std::align_val_t al) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void tc_delete_sized_aligned(void* p, size_t size, std::align_val_t al) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void* tc_newarray_aligned(size_t size, std::align_val_t al)
ATTRIBUTE_SECTION(google_malloc);
void tc_deletearray_aligned(void* p, std::align_val_t al) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void tc_deletearray_sized_aligned(void* p, size_t size, std::align_val_t al) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
// And the nothrow variants of these:
void* tc_new_aligned_nothrow(size_t size, std::align_val_t al, const std::nothrow_t&) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void* tc_newarray_aligned_nothrow(size_t size, std::align_val_t al, const std::nothrow_t&) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void tc_delete_aligned_nothrow(void* ptr, std::align_val_t al, const std::nothrow_t&) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void tc_deletearray_aligned_nothrow(void* ptr, std::align_val_t al, const std::nothrow_t&) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
// Some non-standard extensions that we support.
// This is equivalent to
// OS X: malloc_size()
// glibc: malloc_usable_size()
// Windows: _msize()
size_t tc_malloc_size(void* p) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
} // extern "C"
#endif // #ifndef _WIN32
// ----------------------- IMPLEMENTATION -------------------------------
static int tc_new_mode = 0; // See tc_set_new_mode().
// Routines such as free() and realloc() catch some erroneous pointers
// passed to them, and invoke the below when they do. (An erroneous pointer
// won't be caught if it's within a valid span or a stale span for which
// the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing
// required) kind of exception handling for these routines.
namespace {
ATTRIBUTE_NOINLINE void InvalidFree(void* ptr) {
if (tcmalloc::IsEmergencyPtr(ptr)) {
tcmalloc::EmergencyFree(ptr);
return;
}
Log(kCrash, __FILE__, __LINE__, "Attempt to free invalid pointer", ptr);
}
size_t InvalidGetAllocatedSize(const void* ptr) {
if (tcmalloc::IsEmergencyPtr(ptr)) {
return tcmalloc::EmergencyAllocatedSize(ptr);
}
Log(kCrash, __FILE__, __LINE__,
"Attempt to get the size of an invalid pointer", ptr);
return 0;
}
} // unnamed namespace
// Extract interesting stats
struct TCMallocStats {
uint64_t thread_bytes; // Bytes in thread caches
uint64_t central_bytes; // Bytes in central cache
uint64_t transfer_bytes; // Bytes in central transfer cache
uint64_t metadata_bytes; // Bytes alloced for metadata
PageHeap::Stats pageheap; // Stats from page heap
};
// Get stats into "r". Also, if class_count != nullptr, class_count[k]
// will be set to the total number of objects of size class k in the
// central cache, transfer cache, and per-thread caches. If small_spans
// is non-nullptr, it is filled. Same for large_spans.
static void ExtractStats(TCMallocStats* r, uint64_t* class_count,
PageHeap::SmallSpanStats* small_spans,
PageHeap::LargeSpanStats* large_spans) {
r->central_bytes = 0;
r->transfer_bytes = 0;
for (int cl = 0; cl < Static::num_size_classes(); ++cl) {
const int length = Static::central_cache()[cl].length();
const int tc_length = Static::central_cache()[cl].tc_length();
const size_t cache_overhead = Static::central_cache()[cl].OverheadBytes();
const size_t size = static_cast<uint64_t>(
Static::sizemap()->ByteSizeForClass(cl));
r->central_bytes += (size * length) + cache_overhead;
r->transfer_bytes += (size * tc_length);
if (class_count) {
// Sum the lengths of all per-class freelists, except the per-thread
// freelists, which get counted when we call GetThreadStats(), below.
class_count[cl] = length + tc_length;
}
}
// Add stats from per-thread heaps
r->thread_bytes = 0;
{ // scope
SpinLockHolder h(Static::pageheap_lock());
ThreadCache::GetThreadStats(&r->thread_bytes, class_count);
r->metadata_bytes = tcmalloc::metadata_system_bytes();
r->pageheap = Static::pageheap()->StatsLocked();
if (small_spans != nullptr) {
Static::pageheap()->GetSmallSpanStatsLocked(small_spans);
}
if (large_spans != nullptr) {
Static::pageheap()->GetLargeSpanStatsLocked(large_spans);
}
}
}
static double PagesToMiB(uint64_t pages) {
return (pages << kPageShift) / 1048576.0;
}
// WRITE stats to "out"
static void DumpStats(TCMalloc_Printer* out, int level) {
TCMallocStats stats;
uint64_t class_count[kClassSizesMax];
PageHeap::SmallSpanStats small;
PageHeap::LargeSpanStats large;
if (level >= 2) {
ExtractStats(&stats, class_count, &small, &large);
} else {
ExtractStats(&stats, nullptr, nullptr, nullptr);
}
static const double MiB = 1048576.0;
const uint64_t virtual_memory_used = (stats.pageheap.system_bytes
+ stats.metadata_bytes);
const uint64_t physical_memory_used = (virtual_memory_used
- stats.pageheap.unmapped_bytes);
const uint64_t bytes_in_use_by_app = (physical_memory_used
- stats.metadata_bytes
- stats.pageheap.free_bytes
- stats.central_bytes
- stats.transfer_bytes
- stats.thread_bytes);
#ifdef TCMALLOC_SMALL_BUT_SLOW
out->printf(
"NOTE: SMALL MEMORY MODEL IS IN USE, PERFORMANCE MAY SUFFER.\n");
#endif
out->printf(
"------------------------------------------------\n"
"MALLOC: %12" PRIu64 " (%7.1f MiB) Bytes in use by application\n"
"MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in page heap freelist\n"
"MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in central cache freelist\n"
"MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in transfer cache freelist\n"
"MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in thread cache freelists\n"
"MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in malloc metadata\n"
"MALLOC: ------------\n"
"MALLOC: = %12" PRIu64 " (%7.1f MiB) Actual memory used (physical + swap)\n"
"MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes released to OS (aka unmapped)\n"
"MALLOC: ------------\n"
"MALLOC: = %12" PRIu64 " (%7.1f MiB) Virtual address space used\n"
"MALLOC:\n"
"MALLOC: %12" PRIu64 " Spans in use\n"
"MALLOC: %12" PRIu64 " Thread heaps in use\n"
"MALLOC: %12" PRIu64 " Tcmalloc page size\n"
"------------------------------------------------\n"
"Call ReleaseFreeMemory() to release freelist memory to the OS"
" (via madvise()).\n"
"Bytes released to the OS take up virtual address space"
" but no physical memory.\n",
bytes_in_use_by_app, bytes_in_use_by_app / MiB,
stats.pageheap.free_bytes, stats.pageheap.free_bytes / MiB,
stats.central_bytes, stats.central_bytes / MiB,
stats.transfer_bytes, stats.transfer_bytes / MiB,
stats.thread_bytes, stats.thread_bytes / MiB,
stats.metadata_bytes, stats.metadata_bytes / MiB,
physical_memory_used, physical_memory_used / MiB,
stats.pageheap.unmapped_bytes, stats.pageheap.unmapped_bytes / MiB,
virtual_memory_used, virtual_memory_used / MiB,
uint64_t(Static::span_allocator()->inuse()),
uint64_t(ThreadCache::HeapsInUse()),
uint64_t(kPageSize));
if (level >= 2) {
out->printf("------------------------------------------------\n");
out->printf("Total size of freelists for per-thread caches,\n");
out->printf("transfer cache, and central cache, by size class\n");
out->printf("------------------------------------------------\n");
uint64_t cumulative_bytes = 0;
uint64_t cumulative_overhead = 0;
for (uint32_t cl = 0; cl < Static::num_size_classes(); ++cl) {
if (class_count[cl] > 0) {
size_t cl_size = Static::sizemap()->ByteSizeForClass(cl);
const uint64_t class_bytes = class_count[cl] * cl_size;
cumulative_bytes += class_bytes;
const uint64_t class_overhead =
Static::central_cache()[cl].OverheadBytes();
cumulative_overhead += class_overhead;
out->printf("class %3d [ %8zu bytes ] : "
"%8" PRIu64 " objs; %5.1f MiB; %5.1f cum MiB; "
"%8.3f overhead MiB; %8.3f cum overhead MiB\n",
cl, cl_size,
class_count[cl],
class_bytes / MiB,
cumulative_bytes / MiB,
class_overhead / MiB,
cumulative_overhead / MiB);
}
}
// append page heap info
int nonempty_sizes = 0;
for (int s = 0; s < kMaxPages; s++) {
if (small.normal_length[s] + small.returned_length[s] > 0) {
nonempty_sizes++;
}
}
out->printf("------------------------------------------------\n");
out->printf("PageHeap: %d sizes; %6.1f MiB free; %6.1f MiB unmapped\n",
nonempty_sizes, stats.pageheap.free_bytes / MiB,
stats.pageheap.unmapped_bytes / MiB);
out->printf("------------------------------------------------\n");
uint64_t total_normal = 0;
uint64_t total_returned = 0;
for (int s = 1; s <= kMaxPages; s++) {
const int n_length = small.normal_length[s - 1];
const int r_length = small.returned_length[s - 1];
if (n_length + r_length > 0) {
uint64_t n_pages = s * n_length;
uint64_t r_pages = s * r_length;
total_normal += n_pages;
total_returned += r_pages;
out->printf("%6u pages * %6u spans ~ %6.1f MiB; %6.1f MiB cum"
"; unmapped: %6.1f MiB; %6.1f MiB cum\n",
s,
(n_length + r_length),
PagesToMiB(n_pages + r_pages),
PagesToMiB(total_normal + total_returned),
PagesToMiB(r_pages),
PagesToMiB(total_returned));
}
}
total_normal += large.normal_pages;
total_returned += large.returned_pages;
out->printf(">%-5u large * %6u spans ~ %6.1f MiB; %6.1f MiB cum"
"; unmapped: %6.1f MiB; %6.1f MiB cum\n",
static_cast<unsigned int>(kMaxPages),
static_cast<unsigned int>(large.spans),
PagesToMiB(large.normal_pages + large.returned_pages),
PagesToMiB(total_normal + total_returned),
PagesToMiB(large.returned_pages),
PagesToMiB(total_returned));
}
}
static void PrintStats(int level) {
const int kBufferSize = 16 << 10;
char* buffer = new char[kBufferSize];
TCMalloc_Printer printer(buffer, kBufferSize);
DumpStats(&printer, level);
auto unused = write(STDERR_FILENO, buffer, strlen(buffer));
(void)unused;
delete[] buffer;
}
static void IterateOverRanges(void* arg, MallocExtension::RangeFunction func) {
PageID page = 1; // Some code may assume that page==0 is never used
bool done = false;
while (!done) {
// Accumulate a small number of ranges in a local buffer
static const int kNumRanges = 16;
static base::MallocRange ranges[kNumRanges];
int n = 0;
{
SpinLockHolder h(Static::pageheap_lock());
while (n < kNumRanges) {
if (!Static::pageheap()->GetNextRange(page, &ranges[n])) {
done = true;
break;
} else {
uintptr_t limit = ranges[n].address + ranges[n].length;
page = (limit + kPageSize - 1) >> kPageShift;
n++;
}
}
}
for (int i = 0; i < n; i++) {
(*func)(arg, &ranges[i]);
}
}
}
namespace tcmalloc {
TestingPortal::~TestingPortal() = default;
class ATTRIBUTE_HIDDEN TestingPortalImpl : public TestingPortal {
public:
~TestingPortalImpl() override = default;
bool HaveSystemRelease() override {
static bool have = ([] () {
size_t actual;
auto ptr = TCMalloc_SystemAlloc(kPageSize, &actual, 0);
return TCMalloc_SystemRelease(ptr, actual);
})();
return have;
}
bool IsDebuggingMalloc() override {
return false;
}
size_t GetPageSize() override {
return kPageSize;
}
size_t GetMinAlign() override {
return kMinAlign;
}
size_t GetMaxSize() override {
return kMaxSize;
}
int64_t& GetSampleParameter() override {
return FLAGS_tcmalloc_sample_parameter;
}
double& GetReleaseRate() override {
return FLAGS_tcmalloc_release_rate;
}
int32_t& GetMaxFreeQueueSize() override {
abort();
}
bool HasEmergencyMalloc() override {
return kUseEmergencyMalloc;
}
bool IsEmergencyPtr(void* ptr) override {
return tcmalloc::IsEmergencyPtr(ptr);
}
void WithEmergencyMallocEnabled(FunctionRef<void()> body) override {
auto body_adaptor = [body] (bool stacktrace_allowed) {
CHECK(stacktrace_allowed);
body();
};
FunctionRef<void(bool)> ref{body_adaptor};
ThreadCachePtr::WithStacktraceScope(ref.fn, ref.data);
}
static TestingPortalImpl* Get() {
static TestingPortalImpl* ptr = ([] () {
static StaticStorage<TestingPortalImpl> storage;
return storage.Construct();
}());
return ptr;
}
};
} // namespace tcmalloc
using tcmalloc::TestingPortalImpl;
// TCMalloc's support for extra malloc interfaces
class TCMallocImplementation : public MallocExtension {
private:
// ReleaseToSystem() might release more than the requested bytes because
// the page heap releases at the span granularity, and spans are of wildly
// different sizes. This member keeps track of the extra bytes bytes
// released so that the app can periodically call ReleaseToSystem() to
// release memory at a constant rate.
// NOTE: Protected by Static::pageheap_lock().
size_t extra_bytes_released_;
public:
TCMallocImplementation()
: extra_bytes_released_(0) {
}
virtual void GetStats(char* buffer, int buffer_length) {
ASSERT(buffer_length > 0);
TCMalloc_Printer printer(buffer, buffer_length);
// Print level one stats unless lots of space is available
if (buffer_length < 10000) {
DumpStats(&printer, 1);
} else {
DumpStats(&printer, 2);
}
}
// We may print an extra, tcmalloc-specific warning message here.
virtual void GetHeapSample(MallocExtensionWriter* writer) {
if (FLAGS_tcmalloc_sample_parameter == 0) {
const char* const kWarningMsg =
"%warn\n"
"%warn This heap profile does not have any data in it, because\n"
"%warn the application was run with heap sampling turned off.\n"
"%warn To get useful data from GetHeapSample(), you must\n"
"%warn set the environment variable TCMALLOC_SAMPLE_PARAMETER to\n"
"%warn a positive sampling period, such as 524288.\n"
"%warn\n";
writer->append(kWarningMsg, strlen(kWarningMsg));
}
MallocExtension::GetHeapSample(writer);
}
virtual void** ReadStackTraces(int* sample_period) {
tcmalloc::StackTraceTable table;
{
SpinLockHolder h(Static::pageheap_lock());
Span* sampled = Static::sampled_objects();
for (Span* s = sampled->next; s != sampled; s = s->next) {
table.AddTrace(*reinterpret_cast<StackTrace*>(s->objects));
}
}
*sample_period = ThreadCachePtr::Grab()->GetSamplePeriod();
return table.ReadStackTracesAndClear(); // grabs and releases pageheap_lock
}
virtual void** ReadHeapGrowthStackTraces() {
// Note: growth stacks are append only, and updated atomically. So
// we can just read them without any locks. And use arbitrarily long
// (since they're never cleared/deleted).
const StackTrace* head = Static::growth_stacks();
return ProduceStackTracesDump(
+[] (const void** current_head) {
const StackTrace* current = static_cast<const StackTrace*>(*current_head);
*current_head = current->stack[tcmalloc::kMaxStackDepth-1];
return current;
}, head).release();
}
virtual size_t GetThreadCacheSize() {
ThreadCache* tc = ThreadCachePtr::GetIfPresent();
if (!tc)
return 0;
return tc->Size();
}
virtual void Ranges(void* arg, RangeFunction func) {
IterateOverRanges(arg, func);
}
virtual bool GetNumericProperty(const char* name, size_t* value) {
ASSERT(name != nullptr);
if (strcmp(name, "generic.current_allocated_bytes") == 0) {
TCMallocStats stats;
ExtractStats(&stats, nullptr, nullptr, nullptr);
*value = stats.pageheap.system_bytes
- stats.thread_bytes
- stats.central_bytes
- stats.transfer_bytes
- stats.pageheap.free_bytes
- stats.pageheap.unmapped_bytes;
return true;
}
if (strcmp(name, "generic.heap_size") == 0) {
TCMallocStats stats;
ExtractStats(&stats, nullptr, nullptr, nullptr);
*value = stats.pageheap.system_bytes;
return true;
}
if (strcmp(name, "generic.total_physical_bytes") == 0) {
TCMallocStats stats;
ExtractStats(&stats, nullptr, nullptr, nullptr);
*value = stats.pageheap.system_bytes + stats.metadata_bytes -
stats.pageheap.unmapped_bytes;
return true;
}
if (strcmp(name, "tcmalloc.slack_bytes") == 0) {
// Kept for backwards compatibility. Now defined externally as:
// pageheap_free_bytes + pageheap_unmapped_bytes.
SpinLockHolder l(Static::pageheap_lock());
PageHeap::Stats stats = Static::pageheap()->StatsLocked();
*value = stats.free_bytes + stats.unmapped_bytes;
return true;
}
if (strcmp(name, "tcmalloc.central_cache_free_bytes") == 0) {
TCMallocStats stats;
ExtractStats(&stats, nullptr, nullptr, nullptr);
*value = stats.central_bytes;
return true;
}
if (strcmp(name, "tcmalloc.transfer_cache_free_bytes") == 0) {
TCMallocStats stats;
ExtractStats(&stats, nullptr, nullptr, nullptr);
*value = stats.transfer_bytes;
return true;
}
if (strcmp(name, "tcmalloc.thread_cache_free_bytes") == 0) {
TCMallocStats stats;
ExtractStats(&stats, nullptr, nullptr, nullptr);
*value = stats.thread_bytes;
return true;
}
if (strcmp(name, "tcmalloc.pageheap_free_bytes") == 0) {
SpinLockHolder l(Static::pageheap_lock());
*value = Static::pageheap()->StatsLocked().free_bytes;
return true;
}
if (strcmp(name, "tcmalloc.pageheap_unmapped_bytes") == 0) {
SpinLockHolder l(Static::pageheap_lock());
*value = Static::pageheap()->StatsLocked().unmapped_bytes;
return true;
}
if (strcmp(name, "tcmalloc.pageheap_committed_bytes") == 0) {
SpinLockHolder l(Static::pageheap_lock());
*value = Static::pageheap()->StatsLocked().committed_bytes;
return true;
}
if (strcmp(name, "tcmalloc.pageheap_scavenge_count") == 0) {
SpinLockHolder l(Static::pageheap_lock());
*value = Static::pageheap()->StatsLocked().scavenge_count;
return true;
}
if (strcmp(name, "tcmalloc.pageheap_commit_count") == 0) {
SpinLockHolder l(Static::pageheap_lock());
*value = Static::pageheap()->StatsLocked().commit_count;
return true;
}
if (strcmp(name, "tcmalloc.pageheap_total_commit_bytes") == 0) {
SpinLockHolder l(Static::pageheap_lock());
*value = Static::pageheap()->StatsLocked().total_commit_bytes;
return true;
}
if (strcmp(name, "tcmalloc.pageheap_decommit_count") == 0) {
SpinLockHolder l(Static::pageheap_lock());
*value = Static::pageheap()->StatsLocked().decommit_count;
return true;
}
if (strcmp(name, "tcmalloc.pageheap_total_decommit_bytes") == 0) {
SpinLockHolder l(Static::pageheap_lock());
*value = Static::pageheap()->StatsLocked().total_decommit_bytes;
return true;
}
if (strcmp(name, "tcmalloc.pageheap_reserve_count") == 0) {
SpinLockHolder l(Static::pageheap_lock());
*value = Static::pageheap()->StatsLocked().reserve_count;
return true;
}
if (strcmp(name, "tcmalloc.pageheap_total_reserve_bytes") == 0) {
SpinLockHolder l(Static::pageheap_lock());
*value = Static::pageheap()->StatsLocked().total_reserve_bytes;
return true;
}
if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) {
SpinLockHolder l(Static::pageheap_lock());
*value = ThreadCache::overall_thread_cache_size();
return true;
}
if (strcmp(name, "tcmalloc.min_per_thread_cache_bytes") == 0) {
*value = ThreadCache::min_per_thread_cache_size();
return true;
}
if (strcmp(name, "tcmalloc.current_total_thread_cache_bytes") == 0) {
TCMallocStats stats;
ExtractStats(&stats, nullptr, nullptr, nullptr);
*value = stats.thread_bytes;
return true;
}
if (strcmp(name, "tcmalloc.aggressive_memory_decommit") == 0) {
SpinLockHolder l(Static::pageheap_lock());
*value = size_t(Static::pageheap()->GetAggressiveDecommit());
return true;
}
if (strcmp(name, "tcmalloc.heap_limit_mb") == 0) {
SpinLockHolder l(Static::pageheap_lock());
*value = FLAGS_tcmalloc_heap_limit_mb;
return true;
}
if (strcmp(name, "tcmalloc.impl.thread_cache_count") == 0) {
SpinLockHolder h(Static::pageheap_lock());
*value = ThreadCache::thread_heap_count();
return true;
}
if (TestingPortal** portal = TestingPortal::CheckGetPortal(name, value); portal) {
*portal = TestingPortalImpl::Get();
*value = 1;
return true;
}
return false;
}
virtual bool SetNumericProperty(const char* name, size_t value) {
ASSERT(name != nullptr);
if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) {
SpinLockHolder l(Static::pageheap_lock());
ThreadCache::set_overall_thread_cache_size(value);
return true;
}
if (strcmp(name, "tcmalloc.min_per_thread_cache_bytes") == 0) {
ThreadCache::set_min_per_thread_cache_size(value);
return true;
}
if (strcmp(name, "tcmalloc.aggressive_memory_decommit") == 0) {
SpinLockHolder l(Static::pageheap_lock());
Static::pageheap()->SetAggressiveDecommit(value != 0);
return true;
}
if (strcmp(name, "tcmalloc.heap_limit_mb") == 0) {
SpinLockHolder l(Static::pageheap_lock());
FLAGS_tcmalloc_heap_limit_mb = value;
return true;
}
return false;
}
// Note, in gperftools 2.5 we introduced this 'ligher-weight'
// equivalent of MarkThreadIdle, but as of now mongo folk tells us
// they don't use it anymore. And there was indeed possible issue
// with this approach since it didn't 'return' thread's share of
// total thread cache back to common pool. But that was almost
// exactly the difference between regular mark idle and mark
// "temporarily" idle. So we now go back to original mark idle, but
// keep API for ABI and API compat sake.
virtual void MarkThreadTemporarilyIdle() {
MarkThreadIdle();
}
virtual void MarkThreadIdle() {
ThreadCache* cache = ThreadCachePtr::ReleaseAndClear();
if (cache) {
// When our thread had cache, lets delete it
ThreadCache::DeleteCache(cache);
}
}
virtual void MarkThreadBusy(); // Implemented below
virtual SysAllocator* GetSystemAllocator() {
SpinLockHolder h(Static::pageheap_lock());
return tcmalloc_sys_alloc;
}
virtual void SetSystemAllocator(SysAllocator* alloc) {
SpinLockHolder h(Static::pageheap_lock());
tcmalloc_sys_alloc = alloc;
}
virtual void ReleaseToSystem(size_t num_bytes) {
SpinLockHolder h(Static::pageheap_lock());
if (num_bytes <= extra_bytes_released_) {
// We released too much on a prior call, so don't release any
// more this time.
extra_bytes_released_ = extra_bytes_released_ - num_bytes;
return;
}
num_bytes = num_bytes - extra_bytes_released_;
// num_bytes might be less than one page. If we pass zero to
// ReleaseAtLeastNPages, it won't do anything, so we release a whole
// page now and let extra_bytes_released_ smooth it out over time.
Length num_pages = std::max<Length>(num_bytes >> kPageShift, 1);
size_t bytes_released = Static::pageheap()->ReleaseAtLeastNPages(
num_pages) << kPageShift;
if (bytes_released > num_bytes) {
extra_bytes_released_ = bytes_released - num_bytes;
} else {
// The PageHeap wasn't able to release num_bytes. Don't try to
// compensate with a big release next time. Specifically,
// ReleaseFreeMemory() calls ReleaseToSystem(LONG_MAX).
extra_bytes_released_ = 0;
}
}
virtual void SetMemoryReleaseRate(double rate) {
FLAGS_tcmalloc_release_rate = rate;
}
virtual double GetMemoryReleaseRate() {
return FLAGS_tcmalloc_release_rate;
}
virtual size_t GetEstimatedAllocatedSize(size_t size);
// This just calls GetSizeWithCallback, but because that's in an
// unnamed namespace, we need to move the definition below it in the
// file.
virtual size_t GetAllocatedSize(const void* ptr);
// This duplicates some of the logic in GetSizeWithCallback, but is
// faster. This is important on OS X, where this function is called
// on every allocation operation.
virtual Ownership GetOwnership(const void* ptr) {
const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
// The rest of tcmalloc assumes that all allocated pointers use at
// most kAddressBits bits. If ptr doesn't, then it definitely
// wasn't alloacted by tcmalloc.
if ((p >> (kAddressBits - kPageShift)) > 0) {
return kNotOwned;
}
uint32_t cl;
if (Static::pageheap()->TryGetSizeClass(p, &cl)) {
return kOwned;
}
const Span *span = Static::pageheap()->GetDescriptor(p);
if (span) {
return kOwned;
}
return tcmalloc::IsEmergencyPtr(ptr) ? kOwned : kNotOwned;
}
virtual void GetFreeListSizes(std::vector<MallocExtension::FreeListInfo>* v) {
static const char kCentralCacheType[] = "tcmalloc.central";
static const char kTransferCacheType[] = "tcmalloc.transfer";
static const char kThreadCacheType[] = "tcmalloc.thread";
static const char kPageHeapType[] = "tcmalloc.page";
static const char kPageHeapUnmappedType[] = "tcmalloc.page_unmapped";
static const char kLargeSpanType[] = "tcmalloc.large";
static const char kLargeUnmappedSpanType[] = "tcmalloc.large_unmapped";
v->clear();
// central class information
int64_t prev_class_size = 0;
for (int cl = 1; cl < Static::num_size_classes(); ++cl) {
size_t class_size = Static::sizemap()->ByteSizeForClass(cl);
MallocExtension::FreeListInfo i;
i.min_object_size = prev_class_size + 1;
i.max_object_size = class_size;
i.total_bytes_free =
Static::central_cache()[cl].length() * class_size;
i.type = kCentralCacheType;
v->push_back(i);
// transfer cache
i.total_bytes_free =
Static::central_cache()[cl].tc_length() * class_size;
i.type = kTransferCacheType;
v->push_back(i);
prev_class_size = Static::sizemap()->ByteSizeForClass(cl);
}
// Add stats from per-thread heaps
uint64_t class_count[kClassSizesMax];
memset(class_count, 0, sizeof(class_count));
{
SpinLockHolder h(Static::pageheap_lock());
uint64_t thread_bytes = 0;
ThreadCache::GetThreadStats(&thread_bytes, class_count);
}
prev_class_size = 0;
for (int cl = 1; cl < Static::num_size_classes(); ++cl) {
MallocExtension::FreeListInfo i;
i.min_object_size = prev_class_size + 1;
i.max_object_size = Static::sizemap()->ByteSizeForClass(cl);
i.total_bytes_free =
class_count[cl] * Static::sizemap()->ByteSizeForClass(cl);
i.type = kThreadCacheType;
v->push_back(i);
prev_class_size = Static::sizemap()->ByteSizeForClass(cl);
}
// append page heap info
PageHeap::SmallSpanStats small;
PageHeap::LargeSpanStats large;
{
SpinLockHolder h(Static::pageheap_lock());
Static::pageheap()->GetSmallSpanStatsLocked(&small);
Static::pageheap()->GetLargeSpanStatsLocked(&large);
}
// large spans: mapped
MallocExtension::FreeListInfo span_info;
span_info.type = kLargeSpanType;
span_info.max_object_size = (std::numeric_limits<size_t>::max)();
span_info.min_object_size = kMaxPages << kPageShift;
span_info.total_bytes_free = large.normal_pages << kPageShift;
v->push_back(span_info);
// large spans: unmapped
span_info.type = kLargeUnmappedSpanType;
span_info.total_bytes_free = large.returned_pages << kPageShift;
v->push_back(span_info);
// small spans
for (int s = 1; s <= kMaxPages; s++) {
MallocExtension::FreeListInfo i;
i.max_object_size = (s << kPageShift);
i.min_object_size = ((s - 1) << kPageShift);
i.type = kPageHeapType;
i.total_bytes_free = (s << kPageShift) * small.normal_length[s - 1];
v->push_back(i);
i.type = kPageHeapUnmappedType;
i.total_bytes_free = (s << kPageShift) * small.returned_length[s - 1];
v->push_back(i);
}
}
};
static ALWAYS_INLINE
size_t align_size_up(size_t size, size_t align) {
ASSERT(align <= kPageSize);
size_t new_size = (size + align - 1) & ~(align - 1);
if (PREDICT_FALSE(new_size == 0)) {
// Note, new_size == 0 catches both integer overflow and size
// being 0.
if (size == 0) {
new_size = align;
} else {
new_size = size;
}
}
return new_size;
}
// Puts in *cl size class that is suitable for allocation of size bytes with
// align alignment. Returns true if such size class exists and false otherwise.
static bool size_class_with_alignment(size_t size, size_t align, uint32_t* cl) {
if (PREDICT_FALSE(align > kPageSize)) {
return false;
}
size = align_size_up(size, align);
if (PREDICT_FALSE(!Static::sizemap()->GetSizeClass(size, cl))) {
return false;
}
ASSERT((Static::sizemap()->class_to_size(*cl) & (align - 1)) == 0);
return true;
}
// nallocx slow path. Moved to a separate function because
// ThreadCache::InitModule is not inlined which would cause nallocx to
// become non-leaf function with stack frame and stack spills.
static ATTRIBUTE_NOINLINE size_t nallocx_slow(size_t size, int flags) {
if (PREDICT_FALSE(!Static::IsInited())) ThreadCache::InitModule();
size_t align = static_cast<size_t>(1ull << (flags & 0x3f));
uint32_t cl;
bool ok = size_class_with_alignment(size, align, &cl);
if (ok) {
return Static::sizemap()->ByteSizeForClass(cl);
} else {
Length pages = tcmalloc::pages(size);
pages = Static::pageheap()->RoundUpSize(pages);
return pages << kPageShift;
}
}
// The nallocx function allocates no memory, but it performs the same size
// computation as the malloc function, and returns the real size of the
// allocation that would result from the equivalent malloc function call.
// nallocx is a malloc extension originally implemented by jemalloc:
// http://www.unix.com/man-page/freebsd/3/nallocx/
extern "C" PERFTOOLS_DLL_DECL
size_t tc_nallocx(size_t size, int flags) {
if (PREDICT_FALSE(flags != 0)) {
return nallocx_slow(size, flags);
}
uint32_t cl;
// size class 0 is only possible if malloc is not yet initialized
if (Static::sizemap()->GetSizeClass(size, &cl) && cl != 0) {
return Static::sizemap()->ByteSizeForClass(cl);
} else {
return nallocx_slow(size, 0);
}
}
extern "C" PERFTOOLS_DLL_DECL
size_t nallocx(size_t size, int flags)
#ifdef TC_ALIAS
TC_ALIAS(tc_nallocx);
#else
{
return nallocx_slow(size, flags);
}
#endif
size_t TCMallocImplementation::GetEstimatedAllocatedSize(size_t size) {
return tc_nallocx(size, 0);
}
// The constructor allocates an object to ensure that initialization
// runs before main(), and therefore we do not have a chance to become
// multi-threaded before initialization. We also create the TSD key
// here. Presumably by the time this constructor runs, runtime is in
// good enough shape to handle tcmalloc::CreateTlsKey().
//
// The destructor prints stats when the program exits.
static int tcmallocguard_refcount;
TCMallocGuard::TCMallocGuard() {
if (tcmallocguard_refcount++ > 0) {
return;
}
#ifndef WIN32_OVERRIDE_ALLOCATORS
ReplaceSystemAlloc(); // defined in libc_override_*.h
(void)MallocExtension::instance(); // make sure malloc extension is constructed
tc_free(tc_malloc(1));
#endif // !WIN32_OVERRIDE_ALLOCATORS
ThreadCachePtr::InitThreadCachePtrLate();
tc_free(tc_malloc(1));
}
TCMallocGuard::~TCMallocGuard() {
if (--tcmallocguard_refcount == 0) {
const char* env = nullptr;
if (!RunningOnValgrind()) {
// Valgrind uses it's own malloc so we cannot do MALLOCSTATS
env = getenv("MALLOCSTATS");
}
if (env != nullptr) {
int level = atoi(env);
if (level < 1) level = 1;
PrintStats(level);
}
}
}
static TCMallocGuard module_enter_exit_hook;
#ifndef TCMALLOC_USING_DEBUGALLOCATION
static tcmalloc::StaticStorage<TCMallocImplementation> malloc_impl_storage;
void SetupMallocExtension() {
MallocExtension::Register(malloc_impl_storage.Construct());
}
#endif // TCMALLOC_USING_DEBUGALLOCATION
//-------------------------------------------------------------------
// Helpers for the exported routines below
//-------------------------------------------------------------------
static ATTRIBUTE_UNUSED bool CheckCachedSizeClass(void *ptr) {
PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
uint32_t cached_value;
if (!Static::pageheap()->TryGetSizeClass(p, &cached_value)) {
return true;
}
return cached_value == Static::pageheap()->GetDescriptor(p)->sizeclass;
}
static ALWAYS_INLINE void* CheckedMallocResult(void *result) {
ASSERT(result == nullptr || CheckCachedSizeClass(result));
return result;
}
static ALWAYS_INLINE void* SpanToMallocResult(Span *span) {
return
CheckedMallocResult(reinterpret_cast<void*>(span->start << kPageShift));
}
static void* DoSampledAllocation(size_t size) {
#ifndef NO_TCMALLOC_SAMPLES
// Grab the stack trace outside the heap lock
StackTrace tmp;
tmp.depth = tcmalloc::GrabBacktrace(tmp.stack, tcmalloc::kMaxStackDepth, 1);
tmp.size = size;
// Allocate span
auto pages = tcmalloc::pages(size == 0 ? 1 : size);
Span *span = Static::pageheap()->New(pages);
if (PREDICT_FALSE(span == nullptr)) {
return nullptr;
}
SpinLockHolder h(Static::pageheap_lock());
// Allocate stack trace
StackTrace *stack = Static::stacktrace_allocator()->New();
if (PREDICT_TRUE(stack != nullptr)) {
*stack = tmp;
span->sample = 1;
span->objects = stack;
tcmalloc::DLL_Prepend(Static::sampled_objects(), span);
}
return SpanToMallocResult(span);
#else
abort();
#endif
}
namespace {
typedef void* (*malloc_fn)(void *arg);
void* handle_oom(malloc_fn retry_fn,
void* retry_arg,
bool from_operator,
bool nothrow) {
// we hit out of memory condition, usually if it happens we've
// called sbrk or mmap and failed, and thus errno is set. But there
// is support for setting up custom system allocator or setting up
// page heap size limit, in which cases errno may remain
// untouched.
//
// So we set errno here. C++ operator new doesn't require ENOMEM to
// be set, but doesn't forbid it too (and often C++ oom does happen
// with ENOMEM set).
errno = ENOMEM;
if (!from_operator && !tc_new_mode) {
// we're out of memory in C library function (malloc etc) and no
// "new mode" forced on us. Just return nullptr
return nullptr;
}
// we're OOM in operator new or "new mode" is set. We might have to
// call new_handler and maybe retry allocation.
for (;;) {
// Get the current new handler. NB: this function is not
// thread-safe. We make a feeble stab at making it so here, but
// this lock only protects against tcmalloc interfering with
// itself, not with other libraries calling set_new_handler.
std::new_handler nh = std::get_new_handler();
#if __cpp_exceptions
// If no new_handler is established, the allocation failed.
if (!nh) {
if (nothrow) {
return nullptr;
}
throw std::bad_alloc();
}
// Otherwise, try the new_handler. If it returns, retry the
// allocation. If it throws std::bad_alloc, fail the allocation.
// if it throws something else, don't interfere.
try {
(*nh)();
} catch (const std::bad_alloc&) {
if (!nothrow) throw;
return nullptr;
}
#else
if (!nh) {
if (nothrow) {
return nullptr;
}
Log(kCrash, __FILE__, __LINE__, "C++ OOM in -fno-exceptions case");
}
// Since exceptions are disabled, we don't really know if new_handler
// failed. Assume it will abort if it fails.
(*nh)();
#endif // !__cpp_exceptions
// we get here if new_handler returns successfully. So we retry
// allocation.
void* rv = retry_fn(retry_arg);
if (rv != nullptr) {
return rv;
}
// if allocation failed again we go to next loop iteration
}
}
static void ReportLargeAlloc(Length num_pages, void* result) {
StackTrace stack;
stack.depth = tcmalloc::GrabBacktrace(stack.stack, tcmalloc::kMaxStackDepth, 1);
static const int N = 1000;
char buffer[N];
TCMalloc_Printer printer(buffer, N);
printer.printf("tcmalloc: large alloc %" PRIu64 " bytes == %p @ ",
static_cast<uint64_t>(num_pages) << kPageShift,
result);
for (int i = 0; i < stack.depth; i++) {
printer.printf(" %p", stack.stack[i]);
}
printer.printf("\n");
auto unused = write(STDERR_FILENO, buffer, strlen(buffer));
(void)unused;
}
static bool should_report_large(Length num_pages) {
#ifdef ENABLE_LARGE_ALLOC_REPORT
// For windows, the printf we use to report large allocs is
// potentially dangerous: it could cause a malloc that would cause an
// infinite loop. So by default we set the threshold to a huge number
// on windows, so this bad situation will never trigger. You can
// always set TCMALLOC_LARGE_ALLOC_REPORT_THRESHOLD manually if you
// want this functionality.
#ifdef _WIN32
constexpr auto kDefaultLargeAllocReportThreshold = int64_t{1} << 62;
#else
constexpr auto kDefaultLargeAllocReportThreshold = int64_t{1} << 30;
#endif
// Note, our 'reporting threshold setting' is 64-bit, but we can
// only afford size_t threshold variable. I.e. some 32-bit machines
// don't support 64-bit atomics. So some care is taken to cast etc.
static std::atomic<size_t> large_alloc_threshold;
size_t threshold = large_alloc_threshold.load(std::memory_order_relaxed);
if (threshold == 0) {
int64_t value = tcmalloc::commandlineflags::StringToLongLong(
TCMallocGetenvSafe("TCMALLOC_LARGE_ALLOC_REPORT_THRESHOLD"),
kDefaultLargeAllocReportThreshold);
if (value < 0) {
// Negative limit means disable reporting
value = std::numeric_limits<size_t>::max();
}
value = std::max<int64_t>(kPageSize, value);
if (sizeof(size_t) < sizeof(int64_t)) {
// On 32-bit machines size_t is narrower than int64_t. So lets
// make limits larger than size_t's max (i.e. overflowing 32-bit
// unsigned int) to be infinity.
value = std::min<int64_t>(value, std::numeric_limits<size_t>::max());
}
threshold = static_cast<ssize_t>(value);
large_alloc_threshold.store(threshold); // harmless to race
}
do {
if (PREDICT_TRUE(num_pages < (threshold >> kPageShift))) {
return false;
}
// Increase the threshold by 1/8 every time we generate a report.
size_t new_threshold = threshold + threshold / 8;
if (new_threshold < threshold) {
new_threshold = std::numeric_limits<size_t>::max();
}
// Also make new threshold at least as big as the allocation that
// triggered the reporting.
new_threshold = std::max<size_t>(new_threshold,
num_pages << kPageShift);
if (large_alloc_threshold.compare_exchange_strong(
threshold, new_threshold,
std::memory_order_relaxed, std::memory_order_relaxed)) {
return true;
}
} while (true);
#endif
return false;
}
// Helper for do_malloc().
static void* do_malloc_pages(ThreadCache* heap, size_t size) {
void* result;
Length num_pages = tcmalloc::pages(size);
// NOTE: we're passing original size here as opposed to rounded-up
// size as we do in do_malloc_small. The difference is small here
// (at most 4k out of at least 256k). And not rounding up saves us
// from possibility of overflow, which rounding up could produce.
//
// See https://github.com/gperftools/gperftools/issues/723
if (heap->SampleAllocation(size)) {
result = DoSampledAllocation(size);
} else {
Span* span = Static::pageheap()->New(num_pages);
result = (PREDICT_FALSE(span == nullptr) ? nullptr : SpanToMallocResult(span));
}
if (should_report_large(num_pages)) {
ReportLargeAlloc(num_pages, result);
}
return result;
}
static void *nop_oom_handler(size_t size) {
return nullptr;
}
ALWAYS_INLINE void* do_malloc(size_t size) {
// note: it will force initialization of malloc if necessary
ThreadCachePtr cache_ptr = ThreadCachePtr::Grab();
if (PREDICT_FALSE(cache_ptr.IsEmergencyMallocEnabled())) {
return tcmalloc::EmergencyMalloc(size);
}
uint32_t cl;
ASSERT(Static::IsInited());
ASSERT(cache_ptr.get() != nullptr);
if (PREDICT_FALSE(!Static::sizemap()->GetSizeClass(size, &cl))) {
return do_malloc_pages(cache_ptr.get(), size);
}
size_t allocated_size = Static::sizemap()->class_to_size(cl);
if (PREDICT_FALSE(cache_ptr->SampleAllocation(allocated_size))) {
return DoSampledAllocation(size);
}
// The common case, and also the simplest. This just pops the
// size-appropriate freelist, after replenishing it if it's empty.
return CheckedMallocResult(
cache_ptr->Allocate(allocated_size, cl, nop_oom_handler));
}
static void *retry_malloc(void* size) {
return do_malloc(reinterpret_cast<size_t>(size));
}
ALWAYS_INLINE void* do_malloc_or_cpp_alloc(size_t size) {
void *rv = do_malloc(size);
if (PREDICT_TRUE(rv != nullptr)) {
return rv;
}
return handle_oom(retry_malloc, reinterpret_cast<void *>(size),
false, true);
}
ALWAYS_INLINE void* do_calloc(size_t n, size_t elem_size) {
// Overflow check
const size_t size = n * elem_size;
if (elem_size != 0 && size / elem_size != n) return nullptr;
void* result = do_malloc_or_cpp_alloc(size);
if (result != nullptr) {
size_t total_size = size;
if (!tcmalloc::IsEmergencyPtr(result)) {
// On windows we support recalloc (which was apparently
// originally introduced in Irix). In order for recalloc to work
// we need to zero-out not just the size we were asked for, but
// entire usable size. See also
// https://github.com/gperftools/gperftools/pull/994.
//
// But we can do it only when not dealing with emergency
// malloc-ed memory.
total_size = tc_nallocx(size, 0);
}
memset(result, 0, total_size);
}
return result;
}
// If ptr is nullptr, do nothing. Otherwise invoke the given function.
inline void free_null_or_invalid(void* ptr, void (*invalid_free_fn)(void*)) {
if (ptr != nullptr) {
(*invalid_free_fn)(ptr);
}
}
static ATTRIBUTE_NOINLINE void do_free_pages(Span* span, void* ptr) {
// Check to see if the object is in use.
CHECK_CONDITION_PRINT(span->location == Span::IN_USE,
"Object was not in-use");
CHECK_CONDITION_PRINT(
span->start << kPageShift == reinterpret_cast<uintptr_t>(ptr),
"Pointer is not pointing to the start of a span");
Static::pageheap()->PrepareAndDelete(span, [&] () {
if (span->sample) {
StackTrace* st = reinterpret_cast<StackTrace*>(span->objects);
tcmalloc::DLL_Remove(span);
Static::stacktrace_allocator()->Delete(st);
span->objects = nullptr;
}
});
}
#ifndef NDEBUG
// note, with sized deletions we have no means to support win32
// behavior where we detect "not ours" points and delegate them native
// memory management. This is because nature of sized deletes
// bypassing addr -> size class checks. So in this validation code we
// also assume that sized delete is always used with "our" pointers.
bool ValidateSizeHint(void* ptr, size_t size_hint) {
const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
Span* span = Static::pageheap()->GetDescriptor(p);
uint32_t cl = 0;
Static::sizemap()->GetSizeClass(size_hint, &cl);
return (span->sizeclass == cl);
}
#endif
// Helper for the object deletion (free, delete, etc.). Inputs:
// ptr is object to be freed
// invalid_free_fn is a function that gets invoked on certain "bad frees"
//
// We can usually detect the case where ptr is not pointing to a page that
// tcmalloc is using, and in those cases we invoke invalid_free_fn.
ALWAYS_INLINE
void do_free_with_callback(void* ptr,
void (*invalid_free_fn)(void*),
bool use_hint, size_t size_hint) {
ThreadCache* heap = ThreadCachePtr::GetIfPresent();
const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
uint32_t cl;
ASSERT(!use_hint || ValidateSizeHint(ptr, size_hint));
if (!use_hint || PREDICT_FALSE(!Static::sizemap()->GetSizeClass(size_hint, &cl))) {
// if we're in sized delete, but size is too large, no need to
// probe size cache
bool cache_hit = !use_hint && Static::pageheap()->TryGetSizeClass(p, &cl);
if (PREDICT_FALSE(!cache_hit)) {
Span* span = Static::pageheap()->GetDescriptor(p);
if (PREDICT_FALSE(!span)) {
// span can be nullptr because the pointer passed in is nullptr or invalid
// (not something returned by malloc or friends), or because the
// pointer was allocated with some other allocator besides
// tcmalloc. The latter can happen if tcmalloc is linked in via
// a dynamic library, but is not listed last on the link line.
// In that case, libraries after it on the link line will
// allocate with libc malloc, but free with tcmalloc's free.
free_null_or_invalid(ptr, invalid_free_fn);
return;
}
cl = span->sizeclass;
if (PREDICT_FALSE(cl == 0)) {
ASSERT(reinterpret_cast<uintptr_t>(ptr) % kPageSize == 0);
ASSERT(span != nullptr && span->start == p);
do_free_pages(span, ptr);
return;
}
if (!use_hint) {
Static::pageheap()->SetCachedSizeClass(p, cl);
}
}
}
if (PREDICT_TRUE(heap != nullptr)) {
ASSERT(Static::IsInited());
// If we've hit initialized thread cache, so we're done.
heap->Deallocate(ptr, cl);
return;
}
if (PREDICT_FALSE(!Static::IsInited())) {
// if free was called very early we've could have missed the case
// of invalid or nullptr free. I.e. because probing size classes
// cache could return bogus result (cl = 0 as of this
// writing). But since there is no way we could be dealing with
// ptr we've allocated, since successfull malloc implies IsInited,
// we can just call "invalid free" handling code.
free_null_or_invalid(ptr, invalid_free_fn);
return;
}
// Otherwise, delete directly into central cache
tcmalloc::SLL_SetNext(ptr, nullptr);
Static::central_cache()[cl].InsertRange(ptr, ptr, 1);
}
// The default "do_free" that uses the default callback.
ALWAYS_INLINE void do_free(void* ptr) {
return do_free_with_callback(ptr, &InvalidFree, false, 0);
}
// NOTE: some logic here is duplicated in GetOwnership (above), for
// speed. If you change this function, look at that one too.
inline size_t GetSizeWithCallback(const void* ptr,
size_t (*invalid_getsize_fn)(const void*)) {
if (ptr == nullptr)
return 0;
const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
uint32_t cl;
if (Static::pageheap()->TryGetSizeClass(p, &cl)) {
return Static::sizemap()->ByteSizeForClass(cl);
}
const Span *span = Static::pageheap()->GetDescriptor(p);
if (PREDICT_FALSE(span == nullptr)) { // means we do not own this memory
return (*invalid_getsize_fn)(ptr);
}
if (span->sizeclass != 0) {
return Static::sizemap()->ByteSizeForClass(span->sizeclass);
}
if (span->sample) {
size_t orig_size = reinterpret_cast<StackTrace*>(span->objects)->size;
return tc_nallocx(orig_size, 0);
}
return span->length << kPageShift;
}
// This lets you call back to a given function pointer if ptr is invalid.
// It is used primarily by windows code which wants a specialized callback.
ALWAYS_INLINE void* do_realloc_with_callback(
void* old_ptr, size_t new_size,
void (*invalid_free_fn)(void*),
size_t (*invalid_get_size_fn)(const void*)) {
// Get the size of the old entry
const size_t old_size = GetSizeWithCallback(old_ptr, invalid_get_size_fn);
// Reallocate if the new size is larger than the old size,
// or if the new size is significantly smaller than the old size.
// We do hysteresis to avoid resizing ping-pongs:
// . If we need to grow, grow to max(new_size, old_size * 1.X)
// . Don't shrink unless new_size < old_size * 0.Y
// X and Y trade-off time for wasted space. For now we do 1.25 and 0.5.
const size_t min_growth = std::min(old_size / 4,
(std::numeric_limits<size_t>::max)() - old_size); // Avoid overflow.
const size_t lower_bound_to_grow = old_size + min_growth;
const size_t upper_bound_to_shrink = old_size / 2ul;
if ((new_size > old_size) || (new_size < upper_bound_to_shrink)) {
// Need to reallocate.
void* new_ptr = nullptr;
if (new_size > old_size && new_size < lower_bound_to_grow) {
new_ptr = do_malloc_or_cpp_alloc(lower_bound_to_grow);
}
if (new_ptr == nullptr) {
// Either new_size is not a tiny increment, or last do_malloc failed.
new_ptr = do_malloc_or_cpp_alloc(new_size);
}
if (PREDICT_FALSE(new_ptr == nullptr)) {
return nullptr;
}
tcmalloc::InvokeNewHook(new_ptr, new_size);
memcpy(new_ptr, old_ptr, ((old_size < new_size) ? old_size : new_size));
tcmalloc::InvokeDeleteHook(old_ptr);
// We could use a variant of do_free() that leverages the fact
// that we already know the sizeclass of old_ptr. The benefit
// would be small, so don't bother.
do_free_with_callback(old_ptr, invalid_free_fn, false, 0);
return new_ptr;
} else {
// We still need to call hooks to report the updated size:
tcmalloc::InvokeDeleteHook(old_ptr);
tcmalloc::InvokeNewHook(old_ptr, new_size);
return old_ptr;
}
}
static ALWAYS_INLINE
void* do_memalign_pages(size_t align, size_t size) {
ASSERT((align & (align - 1)) == 0);
ASSERT(align > kPageSize);
if (size + align < size) return nullptr; // Overflow
if (PREDICT_FALSE(Static::pageheap() == nullptr)) ThreadCache::InitModule();
// Allocate at least one byte to avoid boundary conditions below
if (size == 0) size = 1;
// We will allocate directly from the page heap
Span* span = Static::pageheap()->NewAligned(tcmalloc::pages(size),
tcmalloc::pages(align));
if (span == nullptr) {
// errno was set inside page heap as necessary.
return nullptr;
}
return SpanToMallocResult(span);
}
// Helpers for use by exported routines below:
inline void do_malloc_stats() {
PrintStats(1);
}
inline int do_mallopt(int cmd, int value) {
return 1; // Indicates error
}
#if defined(HAVE_STRUCT_MALLINFO) || defined(HAVE_STRUCT_MALLINFO2)
template <typename Mallinfo>
inline Mallinfo do_mallinfo() {
TCMallocStats stats;
ExtractStats(&stats, nullptr, nullptr, nullptr);
// Just some of the fields are filled in.
Mallinfo info;
memset(&info, 0, sizeof(info));
// Note, struct mallinfo contains "int" fields, so some of the size
// values will be truncated. But thankfully we also have
// mallinfo2. We're able to produce code for both of those variants.
using inttp = decltype(info.arena); // int or size_t in practice
info.arena = static_cast<inttp>(stats.pageheap.system_bytes);
info.fsmblks = static_cast<inttp>(stats.thread_bytes
+ stats.central_bytes
+ stats.transfer_bytes);
info.fordblks = static_cast<inttp>(stats.pageheap.free_bytes +
stats.pageheap.unmapped_bytes);
info.uordblks = static_cast<inttp>(stats.pageheap.system_bytes
- stats.thread_bytes
- stats.central_bytes
- stats.transfer_bytes
- stats.pageheap.free_bytes
- stats.pageheap.unmapped_bytes);
return info;
}
#endif // HAVE_STRUCT_MALLINFO || HAVE_STRUCT_MALLINFO2
} // end unnamed namespace
// As promised, the definition of this function, declared above.
size_t TCMallocImplementation::GetAllocatedSize(const void* ptr) {
if (ptr == nullptr)
return 0;
ASSERT(TCMallocImplementation::GetOwnership(ptr)
!= TCMallocImplementation::kNotOwned);
return GetSizeWithCallback(ptr, &InvalidGetAllocatedSize);
}
void TCMallocImplementation::MarkThreadBusy() {
// Allocate to force the creation of a thread cache, but avoid
// invoking any hooks.
do_free(do_malloc(0));
}
//-------------------------------------------------------------------
// Exported routines
//-------------------------------------------------------------------
extern "C" PERFTOOLS_DLL_DECL const char* tc_version(
int* major, int* minor, const char** patch) PERFTOOLS_NOTHROW {
if (major) *major = TC_VERSION_MAJOR;
if (minor) *minor = TC_VERSION_MINOR;
if (patch) *patch = TC_VERSION_PATCH;
return TC_VERSION_STRING;
}
// This function behaves similarly to MSVC's _set_new_mode.
// If flag is 0 (default), calls to malloc will behave normally.
// If flag is 1, calls to malloc will behave like calls to new,
// and the std_new_handler will be invoked on failure.
// Returns the previous mode.
extern "C" PERFTOOLS_DLL_DECL int tc_set_new_mode(int flag) PERFTOOLS_NOTHROW {
int old_mode = tc_new_mode;
tc_new_mode = flag;
return old_mode;
}
extern "C" PERFTOOLS_DLL_DECL int tc_query_new_mode() PERFTOOLS_NOTHROW {
return tc_new_mode;
}
#ifndef TCMALLOC_USING_DEBUGALLOCATION // debugallocation.cc defines its own
// CAVEAT: The code structure below ensures that MallocHook methods are always
// called from the stack frame of the invoked allocation function.
// heap-checker.cc depends on this to start a stack trace from
// the call to the (de)allocation function.
namespace tcmalloc {
static ATTRIBUTE_SECTION(google_malloc)
void invoke_hooks_and_free(void *ptr) {
tcmalloc::InvokeDeleteHook(ptr);
do_free(ptr);
}
ATTRIBUTE_SECTION(google_malloc)
void* cpp_throw_oom(size_t size) {
return handle_oom(retry_malloc, reinterpret_cast<void *>(size),
true, false);
}
ATTRIBUTE_SECTION(google_malloc)
void* cpp_nothrow_oom(size_t size) {
return handle_oom(retry_malloc, reinterpret_cast<void *>(size),
true, true);
}
ATTRIBUTE_SECTION(google_malloc)
void* malloc_oom(size_t size) {
return handle_oom(retry_malloc, reinterpret_cast<void *>(size),
false, true);
}
// tcmalloc::allocate_full_XXX is called by fast-path malloc when some
// complex handling is needed (such as fetching object from central
// freelist or malloc sampling). It contains all 'operator new' logic,
// as opposed to malloc_fast_path which only deals with important
// subset of cases.
//
// Note that this is under tcmalloc namespace so that pprof
// can automatically filter it out of growthz/heapz profiles.
//
// We have slightly fancy setup because we need to call hooks from
// function in 'google_malloc' section and we cannot place template
// into this section. Thus 3 separate functions 'built' by macros.
//
// Also note that we're carefully orchestrating for
// MallocHook::GetCallerStackTrace to work even if compiler isn't
// optimizing tail calls (e.g. -O0 is given). We still require
// ALWAYS_INLINE to work for that case, but it was seen to
// work for -O0 -fno-inline across both GCC and clang. I.e. in this
// case we'll get stack frame for tc_new, followed by stack frame for
// allocate_full_cpp_throw_oom, followed by hooks machinery and user
// code's stack frames. So GetCallerStackTrace will find 2
// subsequent stack frames in google_malloc section and correctly
// 'cut' stack trace just before tc_new.
template <void* OOMHandler(size_t)>
ALWAYS_INLINE
static void* do_allocate_full(size_t size) {
void* p = do_malloc(size);
if (PREDICT_FALSE(p == nullptr)) {
p = OOMHandler(size);
}
tcmalloc::InvokeNewHook(p, size);
return CheckedMallocResult(p);
}
#define AF(oom) \
ATTRIBUTE_SECTION(google_malloc) \
void* allocate_full_##oom(size_t size) { \
return do_allocate_full<oom>(size); \
}
AF(cpp_throw_oom)
AF(cpp_nothrow_oom)
AF(malloc_oom)
#undef AF
template <void* OOMHandler(size_t)>
static ALWAYS_INLINE void* dispatch_allocate_full(size_t size) {
if (OOMHandler == cpp_throw_oom) {
return allocate_full_cpp_throw_oom(size);
}
if (OOMHandler == cpp_nothrow_oom) {
return allocate_full_cpp_nothrow_oom(size);
}
ASSERT(OOMHandler == malloc_oom);
return allocate_full_malloc_oom(size);
}
struct retry_memalign_data {
size_t align;
size_t size;
};
static void *retry_do_memalign(void *arg) {
retry_memalign_data *data = static_cast<retry_memalign_data *>(arg);
return do_memalign_pages(data->align, data->size);
}
static ATTRIBUTE_SECTION(google_malloc)
void* memalign_pages(size_t align, size_t size,
bool from_operator, bool nothrow) {
void *rv = do_memalign_pages(align, size);
if (PREDICT_FALSE(rv == nullptr)) {
retry_memalign_data data;
data.align = align;
data.size = size;
rv = handle_oom(retry_do_memalign, &data,
from_operator, nothrow);
}
tcmalloc::InvokeNewHook(rv, size);
return CheckedMallocResult(rv);
}
} // namespace tcmalloc
// This is quick, fast-path-only implementation of malloc/new. It is
// designed to only have support for fast-path. It checks if more
// complex handling is needed (such as a pageheap allocation or
// sampling) and only performs allocation if none of those uncommon
// conditions hold. When we have one of those odd cases it simply
// tail-calls to one of tcmalloc::allocate_full_XXX defined above.
//
// Such approach was found to be quite effective. Generated code for
// tc_{new,malloc} either succeeds quickly or tail-calls to
// allocate_full. Terseness of the source and lack of
// non-tail calls enables compiler to produce better code. Also
// produced code is short enough to enable effort-less human
// comprehension. Which itself led to elimination of various checks
// that were not necessary for fast-path.
template <void* OOMHandler(size_t)>
ALWAYS_INLINE
static void * malloc_fast_path(size_t size) {
if (PREDICT_FALSE(!base::internal::new_hooks_.empty())) {
return tcmalloc::dispatch_allocate_full<OOMHandler>(size);
}
ThreadCache *cache = ThreadCachePtr::GetIfPresent();
if (PREDICT_FALSE(cache == nullptr)) {
return tcmalloc::dispatch_allocate_full<OOMHandler>(size);
}
uint32_t cl;
if (PREDICT_FALSE(!Static::sizemap()->GetSizeClass(size, &cl))) {
return tcmalloc::dispatch_allocate_full<OOMHandler>(size);
}
size_t allocated_size = Static::sizemap()->ByteSizeForClass(cl);
if (PREDICT_FALSE(!cache->TryRecordAllocationFast(allocated_size))) {
return tcmalloc::dispatch_allocate_full<OOMHandler>(size);
}
return CheckedMallocResult(cache->Allocate(allocated_size, cl, OOMHandler));
}
template <void* OOMHandler(size_t)>
ALWAYS_INLINE
static void* memalign_fast_path(size_t align, size_t size) {
if (PREDICT_FALSE(align > kPageSize)) {
if (OOMHandler == tcmalloc::cpp_throw_oom) {
return tcmalloc::memalign_pages(align, size, true, false);
} else if (OOMHandler == tcmalloc::cpp_nothrow_oom) {
return tcmalloc::memalign_pages(align, size, true, true);
} else {
ASSERT(OOMHandler == tcmalloc::malloc_oom);
return tcmalloc::memalign_pages(align, size, false, true);
}
}
// Everything with alignment <= kPageSize we can easily delegate to
// regular malloc
return malloc_fast_path<OOMHandler>(align_size_up(size, align));
}
extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
void* tc_malloc(size_t size) PERFTOOLS_NOTHROW {
return malloc_fast_path<tcmalloc::malloc_oom>(size);
}
static ALWAYS_INLINE
void free_fast_path(void *ptr) {
if (PREDICT_FALSE(!base::internal::delete_hooks_.empty())) {
tcmalloc::invoke_hooks_and_free(ptr);
return;
}
do_free(ptr);
}
extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
void tc_free(void* ptr) PERFTOOLS_NOTHROW {
free_fast_path(ptr);
}
extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
void tc_free_sized(void *ptr, size_t size) PERFTOOLS_NOTHROW {
if (PREDICT_FALSE(!base::internal::delete_hooks_.empty())) {
tcmalloc::invoke_hooks_and_free(ptr);
return;
}
#ifndef NO_TCMALLOC_SAMPLES
// if ptr is kPageSize-aligned, then it could be sampled allocation,
// thus we don't trust hint and just do plain free. It also handles
// nullptr for us.
if (PREDICT_FALSE((reinterpret_cast<uintptr_t>(ptr) & (kPageSize-1)) == 0)) {
tc_free(ptr);
return;
}
#else
if (!ptr) {
return;
}
#endif
do_free_with_callback(ptr, &InvalidFree, true, size);
}
#ifdef TC_ALIAS
extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized(void *p, size_t size) PERFTOOLS_NOTHROW
TC_ALIAS(tc_free_sized);
extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized(void *p, size_t size) PERFTOOLS_NOTHROW
TC_ALIAS(tc_free_sized);
#else
extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized(void *p, size_t size) PERFTOOLS_NOTHROW {
tc_free_sized(p, size);
}
extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized(void *p, size_t size) PERFTOOLS_NOTHROW {
tc_free_sized(p, size);
}
#endif
extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t n,
size_t elem_size) PERFTOOLS_NOTHROW {
void* result = do_calloc(n, elem_size);
tcmalloc::InvokeNewHook(result, n * elem_size);
return result;
}
extern "C" PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) PERFTOOLS_NOTHROW
#ifdef TC_ALIAS
TC_ALIAS(tc_free);
#else
{
free_fast_path(ptr);
}
#endif
extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* old_ptr,
size_t new_size) PERFTOOLS_NOTHROW {
if (old_ptr == nullptr) {
void* result = do_malloc_or_cpp_alloc(new_size);
tcmalloc::InvokeNewHook(result, new_size);
return result;
}
if (new_size == 0) {
tcmalloc::InvokeDeleteHook(old_ptr);
do_free(old_ptr);
return nullptr;
}
if (PREDICT_FALSE(tcmalloc::IsEmergencyPtr(old_ptr))) {
return tcmalloc::EmergencyRealloc(old_ptr, new_size);
}
auto invalid_get_size = +[] (const void* old_ptr) -> size_t {
Log(kCrash, __FILE__, __LINE__,
"Attempt to realloc invalid pointer", old_ptr);
return 0;
};
return do_realloc_with_callback(old_ptr, new_size,
&InvalidFree, invalid_get_size);
}
extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
void* tc_new(size_t size) {
return malloc_fast_path<tcmalloc::cpp_throw_oom>(size);
}
extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
void* tc_new_nothrow(size_t size, const std::nothrow_t&) PERFTOOLS_NOTHROW {
return malloc_fast_path<tcmalloc::cpp_nothrow_oom>(size);
}
extern "C" PERFTOOLS_DLL_DECL void tc_delete(void* p) PERFTOOLS_NOTHROW
#ifdef TC_ALIAS
TC_ALIAS(tc_free);
#else
{
free_fast_path(p);
}
#endif
// Standard C++ library implementations define and use this
// (via ::operator delete(ptr, nothrow)).
// But it's really the same as normal delete, so we just do the same thing.
extern "C" PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, const std::nothrow_t&) PERFTOOLS_NOTHROW
{
if (PREDICT_FALSE(!base::internal::delete_hooks_.empty())) {
tcmalloc::invoke_hooks_and_free(p);
return;
}
do_free(p);
}
extern "C" PERFTOOLS_DLL_DECL void* tc_newarray(size_t size)
#ifdef TC_ALIAS
TC_ALIAS(tc_new);
#else
{
return malloc_fast_path<tcmalloc::cpp_throw_oom>(size);
}
#endif
extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, const std::nothrow_t&)
PERFTOOLS_NOTHROW
#ifdef TC_ALIAS
TC_ALIAS(tc_new_nothrow);
#else
{
return malloc_fast_path<tcmalloc::cpp_nothrow_oom>(size);
}
#endif
extern "C" PERFTOOLS_DLL_DECL void tc_deletearray(void* p) PERFTOOLS_NOTHROW
#ifdef TC_ALIAS
TC_ALIAS(tc_free);
#else
{
free_fast_path(p);
}
#endif
extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, const std::nothrow_t&) PERFTOOLS_NOTHROW
#ifdef TC_ALIAS
TC_ALIAS(tc_delete_nothrow);
#else
{
free_fast_path(p);
}
#endif
extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
void* tc_memalign(size_t align, size_t size) PERFTOOLS_NOTHROW {
return memalign_fast_path<tcmalloc::malloc_oom>(align, size);
}
extern "C" PERFTOOLS_DLL_DECL int tc_posix_memalign(
void** result_ptr, size_t align, size_t size) PERFTOOLS_NOTHROW {
if (((align % sizeof(void*)) != 0) ||
((align & (align - 1)) != 0) ||
(align == 0)) {
return EINVAL;
}
void* result = tc_memalign(align, size);
if (PREDICT_FALSE(result == nullptr)) {
return ENOMEM;
} else {
*result_ptr = result;
return 0;
}
}
extern "C" PERFTOOLS_DLL_DECL void* tc_new_aligned(size_t size, std::align_val_t align) {
return memalign_fast_path<tcmalloc::cpp_throw_oom>(static_cast<size_t>(align), size);
}
extern "C" PERFTOOLS_DLL_DECL void* tc_new_aligned_nothrow(size_t size, std::align_val_t align, const std::nothrow_t&) PERFTOOLS_NOTHROW {
return memalign_fast_path<tcmalloc::cpp_nothrow_oom>(static_cast<size_t>(align), size);
}
extern "C" PERFTOOLS_DLL_DECL void tc_delete_aligned(void* p, std::align_val_t) PERFTOOLS_NOTHROW
{
free_fast_path(p);
}
// There is no easy way to obtain the actual size used by do_memalign to allocate aligned storage, so for now
// just ignore the size. It might get useful in the future.
extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized_aligned(void* p, size_t size, std::align_val_t align) PERFTOOLS_NOTHROW
{
free_fast_path(p);
}
extern "C" PERFTOOLS_DLL_DECL void tc_delete_aligned_nothrow(void* p, std::align_val_t, const std::nothrow_t&) PERFTOOLS_NOTHROW
{
free_fast_path(p);
}
extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_aligned(size_t size, std::align_val_t align)
#ifdef TC_ALIAS
TC_ALIAS(tc_new_aligned);
#else
{
return memalign_fast_path<tcmalloc::cpp_throw_oom>(static_cast<size_t>(align), size);
}
#endif
extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_aligned_nothrow(size_t size, std::align_val_t align, const std::nothrow_t& nt) PERFTOOLS_NOTHROW
#ifdef TC_ALIAS
TC_ALIAS(tc_new_aligned_nothrow);
#else
{
return memalign_fast_path<tcmalloc::cpp_nothrow_oom>(static_cast<size_t>(align), size);
}
#endif
extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_aligned(void* p, std::align_val_t) PERFTOOLS_NOTHROW
#ifdef TC_ALIAS
TC_ALIAS(tc_delete_aligned);
#else
{
free_fast_path(p);
}
#endif
// There is no easy way to obtain the actual size used by do_memalign to allocate aligned storage, so for now
// just ignore the size. It might get useful in the future.
extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized_aligned(void* p, size_t size, std::align_val_t align) PERFTOOLS_NOTHROW
#ifdef TC_ALIAS
TC_ALIAS(tc_delete_sized_aligned);
#else
{
free_fast_path(p);
}
#endif
extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_aligned_nothrow(void* p, std::align_val_t, const std::nothrow_t&) PERFTOOLS_NOTHROW
#ifdef TC_ALIAS
TC_ALIAS(tc_delete_aligned_nothrow);
#else
{
free_fast_path(p);
}
#endif
static size_t pagesize = 0;
extern "C" PERFTOOLS_DLL_DECL void* tc_valloc(size_t size) PERFTOOLS_NOTHROW {
// Allocate page-aligned object of length >= size bytes
if (pagesize == 0) pagesize = getpagesize();
return tc_memalign(pagesize, size);
}
extern "C" PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t size) PERFTOOLS_NOTHROW {
// Round up size to a multiple of pagesize
if (pagesize == 0) pagesize = getpagesize();
if (size == 0) { // pvalloc(0) should allocate one page, according to
size = pagesize; // http://man.free4web.biz/man3/libmpatrol.3.html
}
size = (size + pagesize - 1) & ~(pagesize - 1);
return tc_memalign(pagesize, size);
}
extern "C" PERFTOOLS_DLL_DECL void tc_malloc_stats(void) PERFTOOLS_NOTHROW {
do_malloc_stats();
}
extern "C" PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) PERFTOOLS_NOTHROW {
return do_mallopt(cmd, value);
}
#ifdef HAVE_STRUCT_MALLINFO
extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) PERFTOOLS_NOTHROW {
return do_mallinfo<struct mallinfo>();
}
#endif
#ifdef HAVE_STRUCT_MALLINFO2
extern "C" PERFTOOLS_DLL_DECL struct mallinfo2 tc_mallinfo2(void) PERFTOOLS_NOTHROW {
return do_mallinfo<struct mallinfo2>();
}
#endif
extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) PERFTOOLS_NOTHROW {
return MallocExtension::instance()->GetAllocatedSize(ptr);
}
extern "C" PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) PERFTOOLS_NOTHROW {
void* result = do_malloc(size);
tcmalloc::InvokeNewHook(result, size);
return result;
}
#endif // TCMALLOC_USING_DEBUGALLOCATION