Google Git
Sign in
chromium / chromium / src / dac55f7b3ba2471df01f6bfa414f7d73423254fc / . / third_party / tcmalloc / chromium / src / debugallocation.cc
blob: 6ac0ea4f8cdd3a89f0f9c5abcd501ceade0497de [file] [log] [blame]
// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
// Copyright (c) 2000, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// ---
// Author: Urs Holzle <opensource@google.com>
#include "config.h"
#include <errno.h>
#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif
#ifdef HAVE_INTTYPES_H
#include <inttypes.h>
#endif
// We only need malloc.h for struct mallinfo.
#ifdef HAVE_STRUCT_MALLINFO
// Malloc can be in several places on older versions of OS X.
# if defined(HAVE_MALLOC_H)
# include <malloc.h>
# elif defined(HAVE_MALLOC_MALLOC_H)
# include <malloc/malloc.h>
# elif defined(HAVE_SYS_MALLOC_H)
# include <sys/malloc.h>
# endif
#endif
#ifdef HAVE_PTHREAD
#include <pthread.h>
#endif
#include <stdarg.h>
#include <stdio.h>
#include <string.h>
#ifdef HAVE_MMAP
#include <sys/mman.h>
#endif
#include <sys/stat.h>
#include <sys/types.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <gperftools/malloc_extension.h>
#include <gperftools/malloc_hook.h>
#include <gperftools/stacktrace.h>
#include "addressmap-inl.h"
#include "base/abort.h"
#include "base/commandlineflags.h"
#include "base/googleinit.h"
#include "base/logging.h"
#include "base/spinlock.h"
#include "malloc_hook-inl.h"
#include "symbolize.h"
// NOTE: due to #define below, tcmalloc.cc will omit tc_XXX
// definitions. So that debug implementations can be defined
// instead. We're going to use do_malloc, do_free and other do_XXX
// functions that are defined in tcmalloc.cc for actual memory
// management
#define TCMALLOC_USING_DEBUGALLOCATION
#include "tcmalloc.cc"
// __THROW is defined in glibc systems. It means, counter-intuitively,
// "This function will never throw an exception." It's an optional
// optimization tool, but we may need to use it to match glibc prototypes.
#ifndef __THROW // I guess we're not on a glibc system
# define __THROW // __THROW is just an optimization, so ok to make it ""
#endif
// On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old
// form of the name instead.
#ifndef MAP_ANONYMOUS
# define MAP_ANONYMOUS MAP_ANON
#endif
// ========================================================================= //
DEFINE_bool(malloctrace,
EnvToBool("TCMALLOC_TRACE", false),
"Enables memory (de)allocation tracing to /tmp/google.alloc.");
#ifdef HAVE_MMAP
DEFINE_bool(malloc_page_fence,
EnvToBool("TCMALLOC_PAGE_FENCE", false),
"Enables putting of memory allocations at page boundaries "
"with a guard page following the allocation (to catch buffer "
"overruns right when they happen).");
DEFINE_bool(malloc_page_fence_never_reclaim,
EnvToBool("TCMALLOC_PAGE_FENCE_NEVER_RECLAIM", false),
"Enables making the virtual address space inaccessible "
"upon a deallocation instead of returning it and reusing later.");
#else
DEFINE_bool(malloc_page_fence, false, "Not usable (requires mmap)");
DEFINE_bool(malloc_page_fence_never_reclaim, false, "Not usable (required mmap)");
#endif
DEFINE_bool(malloc_reclaim_memory,
EnvToBool("TCMALLOC_RECLAIM_MEMORY", true),
"If set to false, we never return memory to malloc "
"when an object is deallocated. This ensures that all "
"heap object addresses are unique.");
DEFINE_int32(max_free_queue_size,
EnvToInt("TCMALLOC_MAX_FREE_QUEUE_SIZE", 10*1024*1024),
"If greater than 0, keep freed blocks in a queue instead of "
"releasing them to the allocator immediately. Release them when "
"the total size of all blocks in the queue would otherwise exceed "
"this limit.");
DEFINE_bool(symbolize_stacktrace,
EnvToBool("TCMALLOC_SYMBOLIZE_STACKTRACE", true),
"Symbolize the stack trace when provided (on some error exits)");
// If we are LD_PRELOAD-ed against a non-pthreads app, then
// pthread_once won't be defined. We declare it here, for that
// case (with weak linkage) which will cause the non-definition to
// resolve to NULL. We can then check for NULL or not in Instance.
extern "C" int pthread_once(pthread_once_t *, void (*)(void))
ATTRIBUTE_WEAK;
// ========================================================================= //
// A safe version of printf() that does not do any allocation and
// uses very little stack space.
static void TracePrintf(int fd, const char *fmt, ...)
__attribute__ ((__format__ (__printf__, 2, 3)));
// Round "value" up to next "alignment" boundary.
// Requires that "alignment" be a power of two.
static intptr_t RoundUp(intptr_t value, intptr_t alignment) {
return (value + alignment - 1) & ~(alignment - 1);
}
// ========================================================================= //
class MallocBlock;
// A circular buffer to hold freed blocks of memory. MallocBlock::Deallocate
// (below) pushes blocks into this queue instead of returning them to the
// underlying allocator immediately. See MallocBlock::Deallocate for more
// information.
//
// We can't use an STL class for this because we need to be careful not to
// perform any heap de-allocations in any of the code in this class, since the
// code in MallocBlock::Deallocate is not re-entrant.
template <typename QueueEntry>
class FreeQueue {
public:
FreeQueue() : q_front_(0), q_back_(0) {}
bool Full() {
return (q_front_ + 1) % kFreeQueueSize == q_back_;
}
void Push(const QueueEntry& block) {
q_[q_front_] = block;
q_front_ = (q_front_ + 1) % kFreeQueueSize;
}
QueueEntry Pop() {
RAW_CHECK(q_back_ != q_front_, "Queue is empty");
const QueueEntry& ret = q_[q_back_];
q_back_ = (q_back_ + 1) % kFreeQueueSize;
return ret;
}
size_t size() const {
return (q_front_ - q_back_ + kFreeQueueSize) % kFreeQueueSize;
}
private:
// Maximum number of blocks kept in the free queue before being freed.
static const int kFreeQueueSize = 1024;
QueueEntry q_[kFreeQueueSize];
int q_front_;
int q_back_;
};
struct MallocBlockQueueEntry {
MallocBlockQueueEntry() : block(NULL), size(0),
num_deleter_pcs(0), deleter_threadid(0) {}
MallocBlockQueueEntry(MallocBlock* b, size_t s) : block(b), size(s) {
if (FLAGS_max_free_queue_size != 0 && b != NULL) {
// Adjust the number of frames to skip (4) if you change the
// location of this call.
num_deleter_pcs =
MallocHook::GetCallerStackTrace(
deleter_pcs,
sizeof(deleter_pcs) / sizeof(deleter_pcs[0]),
4);
deleter_threadid = pthread_self();
} else {
num_deleter_pcs = 0;
// Zero is an illegal pthread id by my reading of the pthread
// implementation:
deleter_threadid = 0;
}
}
MallocBlock* block;
size_t size;
// When deleted and put in the free queue, we (flag-controlled)
// record the stack so that if corruption is later found, we can
// print the deleter's stack. (These three vars add 144 bytes of
// overhead under the LP64 data model.)
void* deleter_pcs[16];
int num_deleter_pcs;
pthread_t deleter_threadid;
};
class MallocBlock {
public: // allocation type constants
// Different allocation types we distinguish.
// Note: The lower 4 bits are not random: we index kAllocName array
// by these values masked with kAllocTypeMask;
// the rest are "random" magic bits to help catch memory corruption.
static const int kMallocType = 0xEFCDAB90;
static const int kNewType = 0xFEBADC81;
static const int kArrayNewType = 0xBCEADF72;
private: // constants
// A mask used on alloc types above to get to 0, 1, 2
static const int kAllocTypeMask = 0x3;
// An additional bit to set in AllocType constants
// to mark now deallocated regions.
static const int kDeallocatedTypeBit = 0x4;
// For better memory debugging, we initialize all storage to known
// values, and overwrite the storage when it's deallocated:
// Byte that fills uninitialized storage.
static const int kMagicUninitializedByte = 0xAB;
// Byte that fills deallocated storage.
// NOTE: tcmalloc.cc depends on the value of kMagicDeletedByte
// to work around a bug in the pthread library.
static const int kMagicDeletedByte = 0xCD;
// A size_t (type of alloc_type_ below) in a deallocated storage
// filled with kMagicDeletedByte.
static const size_t kMagicDeletedSizeT =
0xCDCDCDCD | (((size_t)0xCDCDCDCD << 16) << 16);
// Initializer works for 32 and 64 bit size_ts;
// "<< 16 << 16" is to fool gcc from issuing a warning
// when size_ts are 32 bits.
// NOTE: on Linux, you can enable malloc debugging support in libc by
// setting the environment variable MALLOC_CHECK_ to 1 before you
// start the program (see man malloc).
// We use either do_malloc or mmap to make the actual allocation. In
// order to remember which one of the two was used for any block, we store an
// appropriate magic word next to the block.
static const size_t kMagicMalloc = 0xDEADBEEF;
static const size_t kMagicMMap = 0xABCDEFAB;
// This array will be filled with 0xCD, for use with memcmp.
static unsigned char kMagicDeletedBuffer[1024];
static pthread_once_t deleted_buffer_initialized_;
static bool deleted_buffer_initialized_no_pthreads_;
private: // data layout
// The four fields size1_,offset_,magic1_,alloc_type_
// should together occupy a multiple of 16 bytes. (At the
// moment, sizeof(size_t) == 4 or 8 depending on piii vs
// k8, and 4 of those sum to 16 or 32 bytes).
// This, combined with do_malloc's alignment guarantees,
// ensures that SSE types can be stored into the returned
// block, at &size2_.
size_t size1_;
size_t offset_; // normally 0 unless memaligned memory
// see comments in memalign() and FromRawPointer().
size_t magic1_;
size_t alloc_type_;
// here comes the actual data (variable length)
// ...
// then come the size2_ and magic2_, or a full page of mprotect-ed memory
// if the malloc_page_fence feature is enabled.
size_t size2_;
size_t magic2_;
private: // static data and helpers
// Allocation map: stores the allocation type for each allocated object,
// or the type or'ed with kDeallocatedTypeBit
// for each formerly allocated object.
typedef AddressMap<int> AllocMap;
static AllocMap* alloc_map_;
// This protects alloc_map_ and consistent state of metadata
// for each still-allocated object in it.
// We use spin locks instead of pthread_mutex_t locks
// to prevent crashes via calls to pthread_mutex_(un)lock
// for the (de)allocations coming from pthreads initialization itself.
static SpinLock alloc_map_lock_;
// A queue of freed blocks. Instead of releasing blocks to the allocator
// immediately, we put them in a queue, freeing them only when necessary
// to keep the total size of all the freed blocks below the limit set by
// FLAGS_max_free_queue_size.
static FreeQueue<MallocBlockQueueEntry>* free_queue_;
static size_t free_queue_size_; // total size of blocks in free_queue_
// protects free_queue_ and free_queue_size_
static SpinLock free_queue_lock_;
// Names of allocation types (kMallocType, kNewType, kArrayNewType)
static const char* const kAllocName[];
// Names of corresponding deallocation types
static const char* const kDeallocName[];
static const char* AllocName(int type) {
return kAllocName[type & kAllocTypeMask];
}
static const char* DeallocName(int type) {
return kDeallocName[type & kAllocTypeMask];
}
private: // helper accessors
bool IsMMapped() const { return kMagicMMap == magic1_; }
bool IsValidMagicValue(size_t value) const {
return kMagicMMap == value || kMagicMalloc == value;
}
static size_t real_malloced_size(size_t size) {
return size + sizeof(MallocBlock);
}
/*
* Here we assume size of page is kMinAlign aligned,
* so if size is MALLOC_ALIGNMENT aligned too, then we could
* guarantee return address is also kMinAlign aligned, because
* mmap return address at nearby page boundary on Linux.
*/
static size_t real_mmapped_size(size_t size) {
size_t tmp = size + MallocBlock::data_offset();
tmp = RoundUp(tmp, kMinAlign);
return tmp;
}
size_t real_size() {
return IsMMapped() ? real_mmapped_size(size1_) : real_malloced_size(size1_);
}
// NOTE: if the block is mmapped (that is, we're using the
// malloc_page_fence option) then there's no size2 or magic2
// (instead, the guard page begins where size2 would be).
size_t* size2_addr() { return (size_t*)((char*)&size2_ + size1_); }
const size_t* size2_addr() const {
return (const size_t*)((char*)&size2_ + size1_);
}
size_t* magic2_addr() { return (size_t*)(size2_addr() + 1); }
const size_t* magic2_addr() const { return (const size_t*)(size2_addr() + 1); }
private: // other helpers
void Initialize(size_t size, int type) {
RAW_CHECK(IsValidMagicValue(magic1_), "");
// record us as allocated in the map
alloc_map_lock_.Lock();
if (!alloc_map_) {
void* p = do_malloc(sizeof(AllocMap));
alloc_map_ = new(p) AllocMap(do_malloc, do_free);
}
alloc_map_->Insert(data_addr(), type);
// initialize us
size1_ = size;
offset_ = 0;
alloc_type_ = type;
if (!IsMMapped()) {
bit_store(magic2_addr(), &magic1_);
bit_store(size2_addr(), &size);
}
alloc_map_lock_.Unlock();
memset(data_addr(), kMagicUninitializedByte, size);
if (!IsMMapped()) {
RAW_CHECK(memcmp(&size1_, size2_addr(), sizeof(size1_)) == 0, "should hold");
RAW_CHECK(memcmp(&magic1_, magic2_addr(), sizeof(magic1_)) == 0, "should hold");
}
}
size_t CheckAndClear(int type, size_t given_size) {
alloc_map_lock_.Lock();
CheckLocked(type);
if (!IsMMapped()) {
RAW_CHECK(memcmp(&size1_, size2_addr(), sizeof(size1_)) == 0, "should hold");
}
// record us as deallocated in the map
alloc_map_->Insert(data_addr(), type | kDeallocatedTypeBit);
alloc_map_lock_.Unlock();
// clear us
const size_t size = real_size();
RAW_CHECK(!given_size || given_size == size1_,
"right size must be passed to sized delete");
memset(this, kMagicDeletedByte, size);
return size;
}
void CheckLocked(int type) const {
int map_type = 0;
const int* found_type =
alloc_map_ != NULL ? alloc_map_->Find(data_addr()) : NULL;
if (found_type == NULL) {
RAW_LOG(FATAL, "memory allocation bug: object at %p "
"has never been allocated", data_addr());
} else {
map_type = *found_type;
}
if ((map_type & kDeallocatedTypeBit) != 0) {
RAW_LOG(FATAL, "memory allocation bug: object at %p "
"has been already deallocated (it was allocated with %s)",
data_addr(), AllocName(map_type & ~kDeallocatedTypeBit));
}
if (alloc_type_ == kMagicDeletedSizeT) {
RAW_LOG(FATAL, "memory stomping bug: a word before object at %p "
"has been corrupted; or else the object has been already "
"deallocated and our memory map has been corrupted",
data_addr());
}
if (!IsValidMagicValue(magic1_)) {
RAW_LOG(FATAL, "memory stomping bug: a word before object at %p "
"has been corrupted; "
"or else our memory map has been corrupted and this is a "
"deallocation for not (currently) heap-allocated object",
data_addr());
}
if (!IsMMapped()) {
if (memcmp(&size1_, size2_addr(), sizeof(size1_))) {
RAW_LOG(FATAL, "memory stomping bug: a word after object at %p "
"has been corrupted", data_addr());
}
size_t addr;
bit_store(&addr, magic2_addr());
if (!IsValidMagicValue(addr)) {
RAW_LOG(FATAL, "memory stomping bug: a word after object at %p "
"has been corrupted", data_addr());
}
}
if (alloc_type_ != type) {
if ((alloc_type_ != MallocBlock::kMallocType) &&
(alloc_type_ != MallocBlock::kNewType) &&
(alloc_type_ != MallocBlock::kArrayNewType)) {
RAW_LOG(FATAL, "memory stomping bug: a word before object at %p "
"has been corrupted", data_addr());
}
RAW_LOG(FATAL, "memory allocation/deallocation mismatch at %p: "
"allocated with %s being deallocated with %s",
data_addr(), AllocName(alloc_type_), DeallocName(type));
}
if (alloc_type_ != map_type) {
RAW_LOG(FATAL, "memory stomping bug: our memory map has been corrupted : "
"allocation at %p made with %s "
"is recorded in the map to be made with %s",
data_addr(), AllocName(alloc_type_), AllocName(map_type));
}
}
public: // public accessors
void* data_addr() { return (void*)&size2_; }
const void* data_addr() const { return (const void*)&size2_; }
static size_t data_offset() { return OFFSETOF_MEMBER(MallocBlock, size2_); }
size_t data_size() const { return size1_; }
void set_offset(int offset) { this->offset_ = offset; }
public: // our main interface
static MallocBlock* Allocate(size_t size, int type) {
// Prevent an integer overflow / crash with large allocation sizes.
// TODO - Note that for a e.g. 64-bit size_t, max_size_t may not actually
// be the maximum value, depending on how the compiler treats ~0. The worst
// practical effect is that allocations are limited to 4Gb or so, even if
// the address space could take more.
static size_t max_size_t = ~0;
if (size > max_size_t - sizeof(MallocBlock)) {
RAW_LOG(ERROR, "Massive size passed to malloc: %" PRIuS "", size);
return NULL;
}
MallocBlock* b = NULL;
const bool use_malloc_page_fence = FLAGS_malloc_page_fence;
#ifdef HAVE_MMAP
if (use_malloc_page_fence) {
// Put the block towards the end of the page and make the next page
// inaccessible. This will catch buffer overrun right when it happens.
size_t sz = real_mmapped_size(size);
int pagesize = getpagesize();
int num_pages = (sz + pagesize - 1) / pagesize + 1;
char* p = (char*) mmap(NULL, num_pages * pagesize, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) {
// If the allocation fails, abort rather than returning NULL to
// malloc. This is because in most cases, the program will run out
// of memory in this mode due to tremendous amount of wastage. There
// is no point in propagating the error elsewhere.
RAW_LOG(FATAL, "Out of memory: possibly due to page fence overhead: %s",
strerror(errno));
}
// Mark the page after the block inaccessible
if (mprotect(p + (num_pages - 1) * pagesize, pagesize, PROT_NONE)) {
RAW_LOG(FATAL, "Guard page setup failed: %s", strerror(errno));
}
b = (MallocBlock*) (p + (num_pages - 1) * pagesize - sz);
} else {
b = (MallocBlock*) do_malloc(real_malloced_size(size));
}
#else
b = (MallocBlock*) do_malloc(real_malloced_size(size));
#endif
// It would be nice to output a diagnostic on allocation failure
// here, but logging (other than FATAL) requires allocating
// memory, which could trigger a nasty recursion. Instead, preserve
// malloc semantics and return NULL on failure.
if (b != NULL) {
b->magic1_ = use_malloc_page_fence ? kMagicMMap : kMagicMalloc;
b->Initialize(size, type);
}
return b;
}
void Deallocate(int type, size_t given_size) {
if (IsMMapped()) { // have to do this before CheckAndClear
#ifdef HAVE_MMAP
int size = CheckAndClear(type, given_size);
int pagesize = getpagesize();
int num_pages = (size + pagesize - 1) / pagesize + 1;
char* p = (char*) this;
if (FLAGS_malloc_page_fence_never_reclaim ||
!FLAGS_malloc_reclaim_memory) {
mprotect(p - (num_pages - 1) * pagesize + size,
num_pages * pagesize, PROT_NONE);
} else {
munmap(p - (num_pages - 1) * pagesize + size, num_pages * pagesize);
}
#endif
} else {
const size_t size = CheckAndClear(type, given_size);
if (FLAGS_malloc_reclaim_memory) {
// Instead of freeing the block immediately, push it onto a queue of
// recently freed blocks. Free only enough blocks to keep from
// exceeding the capacity of the queue or causing the total amount of
// un-released memory in the queue from exceeding
// FLAGS_max_free_queue_size.
ProcessFreeQueue(this, size, FLAGS_max_free_queue_size);
}
}
}
static size_t FreeQueueSize() {
SpinLockHolder l(&free_queue_lock_);
return free_queue_size_;
}
static void ProcessFreeQueue(MallocBlock* b, size_t size,
int max_free_queue_size) {
// MallocBlockQueueEntry are about 144 in size, so we can only
// use a small array of them on the stack.
MallocBlockQueueEntry entries[4];
int num_entries = 0;
MallocBlockQueueEntry new_entry(b, size);
free_queue_lock_.Lock();
if (free_queue_ == NULL)
free_queue_ = new FreeQueue<MallocBlockQueueEntry>;
RAW_CHECK(!free_queue_->Full(), "Free queue mustn't be full!");
if (b != NULL) {
free_queue_size_ += size + sizeof(MallocBlockQueueEntry);
free_queue_->Push(new_entry);
}
// Free blocks until the total size of unfreed blocks no longer exceeds
// max_free_queue_size, and the free queue has at least one free
// space in it.
while (free_queue_size_ > max_free_queue_size || free_queue_->Full()) {
RAW_CHECK(num_entries < arraysize(entries), "entries array overflow");
entries[num_entries] = free_queue_->Pop();
free_queue_size_ -=
entries[num_entries].size + sizeof(MallocBlockQueueEntry);
num_entries++;
if (num_entries == arraysize(entries)) {
// The queue will not be full at this point, so it is ok to
// release the lock. The queue may still contain more than
// max_free_queue_size, but this is not a strict invariant.
free_queue_lock_.Unlock();
for (int i = 0; i < num_entries; i++) {
CheckForDanglingWrites(entries[i]);
do_free(entries[i].block);
}
num_entries = 0;
free_queue_lock_.Lock();
}
}
free_queue_lock_.Unlock();
for (int i = 0; i < num_entries; i++) {
CheckForDanglingWrites(entries[i]);
do_free(entries[i].block);
}
}
static void InitDeletedBuffer() {
memset(kMagicDeletedBuffer, kMagicDeletedByte, sizeof(kMagicDeletedBuffer));
deleted_buffer_initialized_no_pthreads_ = true;
}
static void CheckForDanglingWrites(const MallocBlockQueueEntry& queue_entry) {
// Initialize the buffer if necessary.
if (pthread_once)
pthread_once(&deleted_buffer_initialized_, &InitDeletedBuffer);
if (!deleted_buffer_initialized_no_pthreads_) {
// This will be the case on systems that don't link in pthreads,
// including on FreeBSD where pthread_once has a non-zero address
// (but doesn't do anything) even when pthreads isn't linked in.
InitDeletedBuffer();
}
const unsigned char* p =
reinterpret_cast<unsigned char*>(queue_entry.block);
static const size_t size_of_buffer = sizeof(kMagicDeletedBuffer);
const size_t size = queue_entry.size;
const size_t buffers = size / size_of_buffer;
const size_t remainder = size % size_of_buffer;
size_t buffer_idx;
for (buffer_idx = 0; buffer_idx < buffers; ++buffer_idx) {
CheckForCorruptedBuffer(queue_entry, buffer_idx, p, size_of_buffer);
p += size_of_buffer;
}
CheckForCorruptedBuffer(queue_entry, buffer_idx, p, remainder);
}
static void CheckForCorruptedBuffer(const MallocBlockQueueEntry& queue_entry,
size_t buffer_idx,
const unsigned char* buffer,
size_t size_of_buffer) {
if (memcmp(buffer, kMagicDeletedBuffer, size_of_buffer) == 0) {
return;
}
RAW_LOG(ERROR,
"Found a corrupted memory buffer in MallocBlock (may be offset "
"from user ptr): buffer index: %zd, buffer ptr: %p, size of "
"buffer: %zd", buffer_idx, buffer, size_of_buffer);
// The magic deleted buffer should only be 1024 bytes, but in case
// this changes, let's put an upper limit on the number of debug
// lines we'll output:
if (size_of_buffer <= 1024) {
for (int i = 0; i < size_of_buffer; ++i) {
if (buffer[i] != kMagicDeletedByte) {
RAW_LOG(ERROR, "Buffer byte %d is 0x%02x (should be 0x%02x).",
i, buffer[i], kMagicDeletedByte);
}
}
} else {
RAW_LOG(ERROR, "Buffer too large to print corruption.");
}
const MallocBlock* b = queue_entry.block;
const size_t size = queue_entry.size;
if (queue_entry.num_deleter_pcs > 0) {
TracePrintf(STDERR_FILENO, "Deleted by thread %p\n",
reinterpret_cast<void*>(
PRINTABLE_PTHREAD(queue_entry.deleter_threadid)));
// We don't want to allocate or deallocate memory here, so we use
// placement-new. It's ok that we don't destroy this, since we're
// just going to error-exit below anyway. Union is for alignment.
union { void* alignment; char buf[sizeof(SymbolTable)]; } tablebuf;
SymbolTable* symbolization_table = new (tablebuf.buf) SymbolTable;
for (int i = 0; i < queue_entry.num_deleter_pcs; i++) {
// Symbolizes the previous address of pc because pc may be in the
// next function. This may happen when the function ends with
// a call to a function annotated noreturn (e.g. CHECK).
char *pc = reinterpret_cast<char*>(queue_entry.deleter_pcs[i]);
symbolization_table->Add(pc - 1);
}
if (FLAGS_symbolize_stacktrace)
symbolization_table->Symbolize();
for (int i = 0; i < queue_entry.num_deleter_pcs; i++) {
char *pc = reinterpret_cast<char*>(queue_entry.deleter_pcs[i]);
TracePrintf(STDERR_FILENO, " @ %p %s\n",
pc, symbolization_table->GetSymbol(pc - 1));
}
} else {
RAW_LOG(ERROR,
"Skipping the printing of the deleter's stack! Its stack was "
"not found; either the corruption occurred too early in "
"execution to obtain a stack trace or --max_free_queue_size was "
"set to 0.");
}
RAW_LOG(FATAL,
"Memory was written to after being freed. MallocBlock: %p, user "
"ptr: %p, size: %zd. If you can't find the source of the error, "
"try using ASan (http://code.google.com/p/address-sanitizer/), "
"Valgrind, or Purify, or study the "
"output of the deleter's stack printed above.",
b, b->data_addr(), size);
}
static MallocBlock* FromRawPointer(void* p) {
const size_t data_offset = MallocBlock::data_offset();
// Find the header just before client's memory.
MallocBlock *mb = reinterpret_cast<MallocBlock *>(
reinterpret_cast<char *>(p) - data_offset);
// If mb->alloc_type_ is kMagicDeletedSizeT, we're not an ok pointer.
if (mb->alloc_type_ == kMagicDeletedSizeT) {
RAW_LOG(FATAL, "memory allocation bug: object at %p has been already"
" deallocated; or else a word before the object has been"
" corrupted (memory stomping bug)", p);
}
// If mb->offset_ is zero (common case), mb is the real header.
// If mb->offset_ is non-zero, this block was allocated by debug
// memallign implementation, and mb->offset_ is the distance
// backwards to the real header from mb, which is a fake header.
if (mb->offset_ == 0) {
return mb;
}
MallocBlock *main_block = reinterpret_cast<MallocBlock *>(
reinterpret_cast<char *>(mb) - mb->offset_);
if (main_block->offset_ != 0) {
RAW_LOG(FATAL, "memory corruption bug: offset_ field is corrupted."
" Need 0 but got %x",
(unsigned)(main_block->offset_));
}
if (main_block >= p) {
RAW_LOG(FATAL, "memory corruption bug: offset_ field is corrupted."
" Detected main_block address overflow: %x",
(unsigned)(mb->offset_));
}
if (main_block->size2_addr() < p) {
RAW_LOG(FATAL, "memory corruption bug: offset_ field is corrupted."
" It points below it's own main_block: %x",
(unsigned)(mb->offset_));
}
return main_block;
}
static const MallocBlock* FromRawPointer(const void* p) {
// const-safe version: we just cast about
return FromRawPointer(const_cast<void*>(p));
}
// Return whether p points to memory returned by memalign.
// Requires that p be non-zero and has been checked for sanity with
// FromRawPointer().
static bool IsMemaligned(const void* p) {
const MallocBlock* mb = reinterpret_cast<const MallocBlock*>(
reinterpret_cast<const char*>(p) - MallocBlock::data_offset());
// If the offset is non-zero, the block was allocated by memalign
// (see FromRawPointer above).
return mb->offset_ != 0;
}
void Check(int type) const {
alloc_map_lock_.Lock();
CheckLocked(type);
alloc_map_lock_.Unlock();
}
static bool CheckEverything() {
alloc_map_lock_.Lock();
if (alloc_map_ != NULL) alloc_map_->Iterate(CheckCallback, 0);
alloc_map_lock_.Unlock();
return true; // if we get here, we're okay
}
static bool MemoryStats(int* blocks, size_t* total,
int histogram[kMallocHistogramSize]) {
memset(histogram, 0, kMallocHistogramSize * sizeof(int));
alloc_map_lock_.Lock();
stats_blocks_ = 0;
stats_total_ = 0;
stats_histogram_ = histogram;
if (alloc_map_ != NULL) alloc_map_->Iterate(StatsCallback, 0);
*blocks = stats_blocks_;
*total = stats_total_;
alloc_map_lock_.Unlock();
return true;
}
private: // helpers for CheckEverything and MemoryStats
static void CheckCallback(const void* ptr, int* type, int dummy) {
if ((*type & kDeallocatedTypeBit) == 0) {
FromRawPointer(ptr)->CheckLocked(*type);
}
}
// Accumulation variables for StatsCallback protected by alloc_map_lock_
static int stats_blocks_;
static size_t stats_total_;
static int* stats_histogram_;
static void StatsCallback(const void* ptr, int* type, int dummy) {
if ((*type & kDeallocatedTypeBit) == 0) {
const MallocBlock* b = FromRawPointer(ptr);
b->CheckLocked(*type);
++stats_blocks_;
size_t mysize = b->size1_;
int entry = 0;
stats_total_ += mysize;
while (mysize) {
++entry;
mysize >>= 1;
}
RAW_CHECK(entry < kMallocHistogramSize,
"kMallocHistogramSize should be at least as large as log2 "
"of the maximum process memory size");
stats_histogram_[entry] += 1;
}
}
};
void DanglingWriteChecker() {
// Clear out the remaining free queue to check for dangling writes.
MallocBlock::ProcessFreeQueue(NULL, 0, 0);
}
// ========================================================================= //
const size_t MallocBlock::kMagicMalloc;
const size_t MallocBlock::kMagicMMap;
MallocBlock::AllocMap* MallocBlock::alloc_map_ = NULL;
SpinLock MallocBlock::alloc_map_lock_(SpinLock::LINKER_INITIALIZED);
FreeQueue<MallocBlockQueueEntry>* MallocBlock::free_queue_ = NULL;
size_t MallocBlock::free_queue_size_ = 0;
SpinLock MallocBlock::free_queue_lock_(SpinLock::LINKER_INITIALIZED);
unsigned char MallocBlock::kMagicDeletedBuffer[1024];
pthread_once_t MallocBlock::deleted_buffer_initialized_ = PTHREAD_ONCE_INIT;
bool MallocBlock::deleted_buffer_initialized_no_pthreads_ = false;
const char* const MallocBlock::kAllocName[] = {
"malloc",
"new",
"new []",
NULL,
};
const char* const MallocBlock::kDeallocName[] = {
"free",
"delete",
"delete []",
NULL,
};
int MallocBlock::stats_blocks_;
size_t MallocBlock::stats_total_;
int* MallocBlock::stats_histogram_;
// ========================================================================= //
// The following cut-down version of printf() avoids
// using stdio or ostreams.
// This is to guarantee no recursive calls into
// the allocator and to bound the stack space consumed. (The pthread
// manager thread in linuxthreads has a very small stack,
// so fprintf can't be called.)
static void TracePrintf(int fd, const char *fmt, ...) {
char buf[64];
int i = 0;
va_list ap;
va_start(ap, fmt);
const char *p = fmt;
char numbuf[25];
if (fd < 0) {
va_end(ap);
return;
}
numbuf[sizeof(numbuf)-1] = 0;
while (*p != '\0') { // until end of format string
char *s = &numbuf[sizeof(numbuf)-1];
if (p[0] == '%' && p[1] != 0) { // handle % formats
int64 l = 0;
unsigned long base = 0;
if (*++p == 's') { // %s
s = va_arg(ap, char *);
} else if (*p == 'l' && p[1] == 'd') { // %ld
l = va_arg(ap, long);
base = 10;
p++;
} else if (*p == 'l' && p[1] == 'u') { // %lu
l = va_arg(ap, unsigned long);
base = 10;
p++;
} else if (*p == 'z' && p[1] == 'u') { // %zu
l = va_arg(ap, size_t);
base = 10;
p++;
} else if (*p == 'u') { // %u
l = va_arg(ap, unsigned int);
base = 10;
} else if (*p == 'd') { // %d
l = va_arg(ap, int);
base = 10;
} else if (*p == 'p') { // %p
l = va_arg(ap, intptr_t);
base = 16;
} else {
write(STDERR_FILENO, "Unimplemented TracePrintf format\n", 33);
write(STDERR_FILENO, p, 2);
write(STDERR_FILENO, "\n", 1);
tcmalloc::Abort();
}
p++;
if (base != 0) {
bool minus = (l < 0 && base == 10);
uint64 ul = minus? -l : l;
do {
*--s = "0123456789abcdef"[ul % base];
ul /= base;
} while (ul != 0);
if (base == 16) {
*--s = 'x';
*--s = '0';
} else if (minus) {
*--s = '-';
}
}
} else { // handle normal characters
*--s = *p++;
}
while (*s != 0) {
if (i == sizeof(buf)) {
write(fd, buf, i);
i = 0;
}
buf[i++] = *s++;
}
}
if (i != 0) {
write(fd, buf, i);
}
va_end(ap);
}
// Return the file descriptor we're writing a log to
static int TraceFd() {
static int trace_fd = -1;
if (trace_fd == -1) { // Open the trace file on the first call
const char *val = getenv("TCMALLOC_TRACE_FILE");
bool fallback_to_stderr = false;
if (!val) {
val = "/tmp/google.alloc";
fallback_to_stderr = true;
}
trace_fd = open(val, O_CREAT|O_TRUNC|O_WRONLY, 0666);
if (trace_fd == -1) {
if (fallback_to_stderr) {
trace_fd = 2;
TracePrintf(trace_fd, "Can't open %s. Logging to stderr.\n", val);
} else {
TracePrintf(2, "Can't open %s. Logging disabled.\n", val);
}
}
// Add a header to the log.
TracePrintf(trace_fd, "Trace started: %lu\n",
static_cast<unsigned long>(time(NULL)));
TracePrintf(trace_fd,
"func\tsize\tptr\tthread_id\tstack pcs for tools/symbolize\n");
}
return trace_fd;
}
// Print the hex stack dump on a single line. PCs are separated by tabs.
static void TraceStack(void) {
void *pcs[16];
int n = GetStackTrace(pcs, sizeof(pcs)/sizeof(pcs[0]), 0);
for (int i = 0; i != n; i++) {
TracePrintf(TraceFd(), "\t%p", pcs[i]);
}
}
// This protects MALLOC_TRACE, to make sure its info is atomically written.
static SpinLock malloc_trace_lock(SpinLock::LINKER_INITIALIZED);
#define MALLOC_TRACE(name, size, addr) \
do { \
if (FLAGS_malloctrace) { \
SpinLockHolder l(&malloc_trace_lock); \
TracePrintf(TraceFd(), "%s\t%" PRIuS "\t%p\t%" GPRIuPTHREAD, \
name, size, addr, PRINTABLE_PTHREAD(pthread_self())); \
TraceStack(); \
TracePrintf(TraceFd(), "\n"); \
} \
} while (0)
// ========================================================================= //
// Write the characters buf[0, ..., size-1] to
// the malloc trace buffer.
// This function is intended for debugging,
// and is not declared in any header file.
// You must insert a declaration of it by hand when you need
// to use it.
void __malloctrace_write(const char *buf, size_t size) {
if (FLAGS_malloctrace) {
write(TraceFd(), buf, size);
}
}
// ========================================================================= //
// General debug allocation/deallocation
static inline void* DebugAllocate(size_t size, int type) {
MallocBlock* ptr = MallocBlock::Allocate(size, type);
if (ptr == NULL) return NULL;
MALLOC_TRACE("malloc", size, ptr->data_addr());
return ptr->data_addr();
}
static inline void DebugDeallocate(void* ptr, int type, size_t given_size) {
MALLOC_TRACE("free",
(ptr != 0 ? MallocBlock::FromRawPointer(ptr)->data_size() : 0),
ptr);
if (ptr) MallocBlock::FromRawPointer(ptr)->Deallocate(type, given_size);
}
// ========================================================================= //
// The following functions may be called via MallocExtension::instance()
// for memory verification and statistics.
class DebugMallocImplementation : public TCMallocImplementation {
public:
virtual bool GetNumericProperty(const char* name, size_t* value) {
bool result = TCMallocImplementation::GetNumericProperty(name, value);
if (result && (strcmp(name, "generic.current_allocated_bytes") == 0)) {
// Subtract bytes kept in the free queue
size_t qsize = MallocBlock::FreeQueueSize();
if (*value >= qsize) {
*value -= qsize;
}
}
return result;
}
virtual bool VerifyNewMemory(const void* p) {
if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kNewType);
return true;
}
virtual bool VerifyArrayNewMemory(const void* p) {
if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kArrayNewType);
return true;
}
virtual bool VerifyMallocMemory(const void* p) {
if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kMallocType);
return true;
}
virtual bool VerifyAllMemory() {
return MallocBlock::CheckEverything();
}
virtual bool MallocMemoryStats(int* blocks, size_t* total,
int histogram[kMallocHistogramSize]) {
return MallocBlock::MemoryStats(blocks, total, histogram);
}
virtual size_t GetEstimatedAllocatedSize(size_t size) {
return size;
}
virtual size_t GetAllocatedSize(const void* p) {
if (p) {
RAW_CHECK(GetOwnership(p) != MallocExtension::kNotOwned,
"ptr not allocated by tcmalloc");
return MallocBlock::FromRawPointer(p)->data_size();
}
return 0;
}
virtual MallocExtension::Ownership GetOwnership(const void* p) {
if (!p) {
// nobody owns NULL
return MallocExtension::kNotOwned;
}
// FIXME: note that correct GetOwnership should not touch memory
// that is not owned by tcmalloc. Main implementation is using
// pagemap to discover if page in question is owned by us or
// not. But pagemap only has marks for first and last page of
// spans. Note that if p was returned out of our memalign with
// big alignment, then it will point outside of marked pages. Also
// note that FromRawPointer call below requires touching memory
// before pointer in order to handle memalign-ed chunks
// (offset_). This leaves us with two options:
//
// * do FromRawPointer first and have possibility of crashing if
// we're given not owned pointer
//
// * return incorrect ownership for those large memalign chunks
//
// I've decided to choose later, which appears to happen rarer and
// therefore is arguably a lesser evil
MallocExtension::Ownership rv = TCMallocImplementation::GetOwnership(p);
if (rv != MallocExtension::kOwned) {
return rv;
}
const MallocBlock* mb = MallocBlock::FromRawPointer(p);
return TCMallocImplementation::GetOwnership(mb);
}
virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) {
static const char* kDebugFreeQueue = "debug.free_queue";
TCMallocImplementation::GetFreeListSizes(v);
MallocExtension::FreeListInfo i;
i.type = kDebugFreeQueue;
i.min_object_size = 0;
i.max_object_size = numeric_limits<size_t>::max();
i.total_bytes_free = MallocBlock::FreeQueueSize();
v->push_back(i);
}
};
static union {
char chars[sizeof(DebugMallocImplementation)];
void *ptr;
} debug_malloc_implementation_space;
REGISTER_MODULE_INITIALIZER(debugallocation, {
#if (__cplusplus >= 201103L)
static_assert(
alignof(decltype(debug_malloc_implementation_space)) >=
alignof(DebugMallocImplementation),
"DebugMallocImplementation is expected to need just word alignment");
#endif
// Either we or valgrind will control memory management. We
// register our extension if we're the winner. Otherwise let
// Valgrind use its own malloc (so don't register our extension).
if (!RunningOnValgrind()) {
DebugMallocImplementation *impl = new (debug_malloc_implementation_space.chars) DebugMallocImplementation();
MallocExtension::Register(impl);
}
});
REGISTER_MODULE_DESTRUCTOR(debugallocation, {
if (!RunningOnValgrind()) {
// When the program exits, check all blocks still in the free
// queue for corruption.
DanglingWriteChecker();
}
});
// ========================================================================= //
struct debug_alloc_retry_data {
size_t size;
int new_type;
};
static void *retry_debug_allocate(void *arg) {
debug_alloc_retry_data *data = static_cast<debug_alloc_retry_data *>(arg);
return DebugAllocate(data->size, data->new_type);
}
// This is mostly the same a cpp_alloc in tcmalloc.cc.
// TODO(csilvers): change Allocate() above to call cpp_alloc, so we
// don't have to reproduce the logic here. To make tc_new_mode work
// properly, I think we'll need to separate out the logic of throwing
// from the logic of calling the new-handler.
inline void* debug_cpp_alloc(size_t size, int new_type, bool nothrow) {
void* p = DebugAllocate(size, new_type);
if (p != NULL) {
return p;
}
struct debug_alloc_retry_data data;
data.size = size;
data.new_type = new_type;
return handle_oom(retry_debug_allocate, &data,
true, nothrow);
}
inline void* do_debug_malloc_or_debug_cpp_alloc(size_t size) {
void* p = DebugAllocate(size, MallocBlock::kMallocType);
if (p != NULL) {
return p;
}
struct debug_alloc_retry_data data;
data.size = size;
data.new_type = MallocBlock::kMallocType;
return handle_oom(retry_debug_allocate, &data,
false, true);
}
// Exported routines
// frame forcer and force_frame exist only to prevent tail calls to
// DebugDeallocate to be actually implemented as tail calls. This is
// important because stack trace capturing in MallocBlockQueueEntry
// relies on google_malloc section being on stack and tc_XXX functions
// are in that section. So they must not jump to DebugDeallocate but
// have to do call. frame_forcer call at the end of such functions
// prevents tail calls to DebugDeallocate.
static int frame_forcer;
static void force_frame() {
int dummy = *(int volatile *)&frame_forcer;
(void)dummy;
}
extern "C" PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) PERFTOOLS_NOTHROW {
if (ThreadCache::IsUseEmergencyMalloc()) {
return tcmalloc::EmergencyMalloc(size);
}
void* ptr = do_debug_malloc_or_debug_cpp_alloc(size);
MallocHook::InvokeNewHook(ptr, size);
return ptr;
}
extern "C" PERFTOOLS_DLL_DECL void tc_free(void* ptr) PERFTOOLS_NOTHROW {
if (tcmalloc::IsEmergencyPtr(ptr)) {
return tcmalloc::EmergencyFree(ptr);
}
MallocHook::InvokeDeleteHook(ptr);
DebugDeallocate(ptr, MallocBlock::kMallocType, 0);
force_frame();
}
extern "C" PERFTOOLS_DLL_DECL void tc_free_sized(void *ptr, size_t size) PERFTOOLS_NOTHROW {
MallocHook::InvokeDeleteHook(ptr);
DebugDeallocate(ptr, MallocBlock::kMallocType, size);
force_frame();
}
extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t count, size_t size) PERFTOOLS_NOTHROW {
if (ThreadCache::IsUseEmergencyMalloc()) {
return tcmalloc::EmergencyCalloc(count, size);
}
// Overflow check
const size_t total_size = count * size;
if (size != 0 && total_size / size != count) return NULL;
void* block = do_debug_malloc_or_debug_cpp_alloc(total_size);
MallocHook::InvokeNewHook(block, total_size);
if (block) memset(block, 0, total_size);
return block;
}
extern "C" PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) PERFTOOLS_NOTHROW {
if (tcmalloc::IsEmergencyPtr(ptr)) {
return tcmalloc::EmergencyFree(ptr);
}
MallocHook::InvokeDeleteHook(ptr);
DebugDeallocate(ptr, MallocBlock::kMallocType, 0);
force_frame();
}
extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) PERFTOOLS_NOTHROW {
if (tcmalloc::IsEmergencyPtr(ptr)) {
return tcmalloc::EmergencyRealloc(ptr, size);
}
if (ptr == NULL) {
ptr = do_debug_malloc_or_debug_cpp_alloc(size);
MallocHook::InvokeNewHook(ptr, size);
return ptr;
}
MallocBlock* old = MallocBlock::FromRawPointer(ptr);
old->Check(MallocBlock::kMallocType);
if (MallocBlock::IsMemaligned(ptr)) {
RAW_LOG(FATAL,
"realloc/memalign mismatch at %p: "
"non-NULL pointers passed to realloc must be obtained "
"from malloc, calloc, or realloc",
ptr);
}
if (size == 0) {
MallocHook::InvokeDeleteHook(ptr);
DebugDeallocate(ptr, MallocBlock::kMallocType, 0);
return NULL;
}
MallocBlock* p = MallocBlock::Allocate(size, MallocBlock::kMallocType);
// If realloc fails we are to leave the old block untouched and
// return null
if (p == NULL) return NULL;
// if ptr was allocated via memalign, then old->data_size() is not
// start of user data. So we must be careful to copy only user-data
char *old_begin = (char *)old->data_addr();
char *old_end = old_begin + old->data_size();
ssize_t old_ssize = old_end - (char *)ptr;
CHECK_CONDITION(old_ssize >= 0);
size_t old_size = (size_t)old_ssize;
CHECK_CONDITION(old_size <= old->data_size());
memcpy(p->data_addr(), ptr, (old_size < size) ? old_size : size);
MallocHook::InvokeDeleteHook(ptr);
MallocHook::InvokeNewHook(p->data_addr(), size);
DebugDeallocate(ptr, MallocBlock::kMallocType, 0);
MALLOC_TRACE("realloc", p->data_size(), p->data_addr());
return p->data_addr();
}
extern "C" PERFTOOLS_DLL_DECL void* tc_new(size_t size) {
void* ptr = debug_cpp_alloc(size, MallocBlock::kNewType, false);
MallocHook::InvokeNewHook(ptr, size);
if (ptr == NULL) {
RAW_LOG(FATAL, "Unable to allocate %" PRIuS " bytes: new failed.", size);
}
return ptr;
}
extern "C" PERFTOOLS_DLL_DECL void* tc_new_nothrow(size_t size, const std::nothrow_t&) PERFTOOLS_NOTHROW {
void* ptr = debug_cpp_alloc(size, MallocBlock::kNewType, true);
MallocHook::InvokeNewHook(ptr, size);
return ptr;
}
extern "C" PERFTOOLS_DLL_DECL void tc_delete(void* p) PERFTOOLS_NOTHROW {
MallocHook::InvokeDeleteHook(p);
DebugDeallocate(p, MallocBlock::kNewType, 0);
force_frame();
}
extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized(void* p, size_t size) PERFTOOLS_NOTHROW {
MallocHook::InvokeDeleteHook(p);
DebugDeallocate(p, MallocBlock::kNewType, size);
force_frame();
}
// Some STL implementations explicitly invoke this.
// It is completely equivalent to a normal delete (delete never throws).
extern "C" PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, const std::nothrow_t&) PERFTOOLS_NOTHROW {
MallocHook::InvokeDeleteHook(p);
DebugDeallocate(p, MallocBlock::kNewType, 0);
force_frame();
}
extern "C" PERFTOOLS_DLL_DECL void* tc_newarray(size_t size) {
void* ptr = debug_cpp_alloc(size, MallocBlock::kArrayNewType, false);
MallocHook::InvokeNewHook(ptr, size);
if (ptr == NULL) {
RAW_LOG(FATAL, "Unable to allocate %" PRIuS " bytes: new[] failed.", size);
}
return ptr;
}
extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, const std::nothrow_t&)
PERFTOOLS_NOTHROW {
void* ptr = debug_cpp_alloc(size, MallocBlock::kArrayNewType, true);
MallocHook::InvokeNewHook(ptr, size);
return ptr;
}
extern "C" PERFTOOLS_DLL_DECL void tc_deletearray(void* p) PERFTOOLS_NOTHROW {
MallocHook::InvokeDeleteHook(p);
DebugDeallocate(p, MallocBlock::kArrayNewType, 0);
force_frame();
}
extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized(void* p, size_t size) PERFTOOLS_NOTHROW {
MallocHook::InvokeDeleteHook(p);
DebugDeallocate(p, MallocBlock::kArrayNewType, size);
force_frame();
}
// Some STL implementations explicitly invoke this.
// It is completely equivalent to a normal delete (delete never throws).
extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, const std::nothrow_t&) PERFTOOLS_NOTHROW {
MallocHook::InvokeDeleteHook(p);
DebugDeallocate(p, MallocBlock::kArrayNewType, 0);
force_frame();
}
// This is mostly the same as do_memalign in tcmalloc.cc.
static void *do_debug_memalign(size_t alignment, size_t size, int type) {
// Allocate >= size bytes aligned on "alignment" boundary
// "alignment" is a power of two.
void *p = 0;
RAW_CHECK((alignment & (alignment-1)) == 0, "must be power of two");
const size_t data_offset = MallocBlock::data_offset();
// Allocate "alignment-1" extra bytes to ensure alignment is possible, and
// a further data_offset bytes for an additional fake header.
size_t extra_bytes = data_offset + alignment - 1;
if (size + extra_bytes < size) return NULL; // Overflow
p = DebugAllocate(size + extra_bytes, type);
if (p != 0) {
intptr_t orig_p = reinterpret_cast<intptr_t>(p);
// Leave data_offset bytes for fake header, and round up to meet
// alignment.
p = reinterpret_cast<void *>(RoundUp(orig_p + data_offset, alignment));
// Create a fake header block with an offset_ that points back to the
// real header. FromRawPointer uses this value.
MallocBlock *fake_hdr = reinterpret_cast<MallocBlock *>(
reinterpret_cast<char *>(p) - data_offset);
// offset_ is distance between real and fake headers.
// p is now end of fake header (beginning of client area),
// and orig_p is the end of the real header, so offset_
// is their difference.
//
// Note that other fields of fake_hdr are initialized with
// kMagicUninitializedByte
fake_hdr->set_offset(reinterpret_cast<intptr_t>(p) - orig_p);
}
return p;
}
struct memalign_retry_data {
size_t align;
size_t size;
int type;
};
static void *retry_debug_memalign(void *arg) {
memalign_retry_data *data = static_cast<memalign_retry_data *>(arg);
return do_debug_memalign(data->align, data->size, data->type);
}
ATTRIBUTE_ALWAYS_INLINE
inline void* do_debug_memalign_or_debug_cpp_memalign(size_t align,
size_t size,
int type,
bool from_operator,
bool nothrow) {
void* p = do_debug_memalign(align, size, type);
if (p != NULL) {
return p;
}
struct memalign_retry_data data;
data.align = align;
data.size = size;
data.type = type;
return handle_oom(retry_debug_memalign, &data,
from_operator, nothrow);
}
extern "C" PERFTOOLS_DLL_DECL void* tc_memalign(size_t align, size_t size) PERFTOOLS_NOTHROW {
void *p = do_debug_memalign_or_debug_cpp_memalign(align, size, MallocBlock::kMallocType, false, true);
MallocHook::InvokeNewHook(p, size);
return p;
}
// Implementation taken from tcmalloc/tcmalloc.cc
extern "C" PERFTOOLS_DLL_DECL int tc_posix_memalign(void** result_ptr, size_t align, size_t size)
PERFTOOLS_NOTHROW {
if (((align % sizeof(void*)) != 0) ||
((align & (align - 1)) != 0) ||
(align == 0)) {
return EINVAL;
}
void* result = do_debug_memalign_or_debug_cpp_memalign(align, size, MallocBlock::kMallocType, false, true);
MallocHook::InvokeNewHook(result, size);
if (result == NULL) {
return ENOMEM;
} else {
*result_ptr = result;
return 0;
}
}
extern "C" PERFTOOLS_DLL_DECL void* tc_valloc(size_t size) PERFTOOLS_NOTHROW {
// Allocate >= size bytes starting on a page boundary
void *p = do_debug_memalign_or_debug_cpp_memalign(getpagesize(), size, MallocBlock::kMallocType, false, true);
MallocHook::InvokeNewHook(p, size);
return p;
}
extern "C" PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t size) PERFTOOLS_NOTHROW {
// Round size up to a multiple of pages
// then allocate memory on a page boundary
int pagesize = getpagesize();
size = RoundUp(size, pagesize);
if (size == 0) { // pvalloc(0) should allocate one page, according to
size = pagesize; // http://man.free4web.biz/man3/libmpatrol.3.html
}
void *p = do_debug_memalign_or_debug_cpp_memalign(pagesize, size, MallocBlock::kMallocType, false, true);
MallocHook::InvokeNewHook(p, size);
return p;
}
#if defined(ENABLE_ALIGNED_NEW_DELETE)
extern "C" PERFTOOLS_DLL_DECL void* tc_new_aligned(size_t size, std::align_val_t align) {
void* result = do_debug_memalign_or_debug_cpp_memalign(static_cast<size_t>(align), size, MallocBlock::kNewType, true, false);
MallocHook::InvokeNewHook(result, size);
return result;
}
extern "C" PERFTOOLS_DLL_DECL void* tc_new_aligned_nothrow(size_t size, std::align_val_t align, const std::nothrow_t&) PERFTOOLS_NOTHROW {
void* result = do_debug_memalign_or_debug_cpp_memalign(static_cast<size_t>(align), size, MallocBlock::kNewType, true, true);
MallocHook::InvokeNewHook(result, size);
return result;
}
extern "C" PERFTOOLS_DLL_DECL void tc_delete_aligned(void* p, std::align_val_t) PERFTOOLS_NOTHROW {
tc_delete(p);
}
extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized_aligned(void* p, size_t size, std::align_val_t align) PERFTOOLS_NOTHROW {
// Reproduce actual size calculation done by do_debug_memalign
const size_t alignment = static_cast<size_t>(align);
const size_t data_offset = MallocBlock::data_offset();
const size_t extra_bytes = data_offset + alignment - 1;
tc_delete_sized(p, size + extra_bytes);
}
extern "C" PERFTOOLS_DLL_DECL void tc_delete_aligned_nothrow(void* p, std::align_val_t, const std::nothrow_t&) PERFTOOLS_NOTHROW {
tc_delete(p);
}
extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_aligned(size_t size, std::align_val_t align) {
void* result = do_debug_memalign_or_debug_cpp_memalign(static_cast<size_t>(align), size, MallocBlock::kArrayNewType, true, false);
MallocHook::InvokeNewHook(result, size);
return result;
}
extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_aligned_nothrow(size_t size, std::align_val_t align, const std::nothrow_t& nt) PERFTOOLS_NOTHROW {
void* result = do_debug_memalign_or_debug_cpp_memalign(static_cast<size_t>(align), size, MallocBlock::kArrayNewType, true, true);
MallocHook::InvokeNewHook(result, size);
return result;
}
extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_aligned(void* p, std::align_val_t) PERFTOOLS_NOTHROW {
tc_deletearray(p);
}
extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized_aligned(void* p, size_t size, std::align_val_t align) PERFTOOLS_NOTHROW {
// Reproduce actual size calculation done by do_debug_memalign
const size_t alignment = static_cast<size_t>(align);
const size_t data_offset = MallocBlock::data_offset();
const size_t extra_bytes = data_offset + alignment - 1;
tc_deletearray_sized(p, size + extra_bytes);
}
extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_aligned_nothrow(void* p, std::align_val_t, const std::nothrow_t&) PERFTOOLS_NOTHROW {
tc_deletearray(p);
}
#endif // defined(ENABLE_ALIGNED_NEW_DELETE)
// malloc_stats just falls through to the base implementation.
extern "C" PERFTOOLS_DLL_DECL void tc_malloc_stats(void) PERFTOOLS_NOTHROW {
do_malloc_stats();
}
extern "C" PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) PERFTOOLS_NOTHROW {
return do_mallopt(cmd, value);
}
#ifdef HAVE_STRUCT_MALLINFO
extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) PERFTOOLS_NOTHROW {
return do_mallinfo();
}
#endif
extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) PERFTOOLS_NOTHROW {
return MallocExtension::instance()->GetAllocatedSize(ptr);
}
extern "C" PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) PERFTOOLS_NOTHROW {
void* result = DebugAllocate(size, MallocBlock::kMallocType);
MallocHook::InvokeNewHook(result, size);
return result;
}