blob: f5d96a5204ad6c12243bf1037914b2d18f9321d2 [file] [log] [blame]
// Copyright 2014 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "base/memory/discardable_shared_memory.h"
#include <stdint.h>
#include <algorithm>
#include "base/atomicops.h"
#include "base/bits.h"
#include "base/feature_list.h"
#include "base/logging.h"
#include "base/memory/discardable_memory.h"
#include "base/memory/discardable_memory_internal.h"
#include "base/memory/page_size.h"
#include "base/memory/shared_memory_tracker.h"
#include "base/numerics/safe_math.h"
#include "base/tracing_buildflags.h"
#include "build/build_config.h"
#include "partition_alloc/page_allocator.h"
#if BUILDFLAG(IS_POSIX) && !BUILDFLAG(IS_NACL)
// For madvise() which is available on all POSIX compatible systems.
#include <sys/mman.h>
#endif
#if BUILDFLAG(IS_ANDROID)
#include "third_party/ashmem/ashmem.h"
#endif
#if BUILDFLAG(IS_WIN)
#include <windows.h>
#include "base/win/windows_version.h"
#endif
#if BUILDFLAG(IS_FUCHSIA)
#include <lib/zx/vmar.h>
#include <zircon/types.h>
#include "base/fuchsia/fuchsia_logging.h"
#endif
#if BUILDFLAG(ENABLE_BASE_TRACING)
#include "base/trace_event/memory_allocator_dump.h" // no-presubmit-check
#include "base/trace_event/process_memory_dump.h" // no-presubmit-check
#endif // BUILDFLAG(ENABLE_BASE_TRACING)
namespace base {
namespace {
// Use a machine-sized pointer as atomic type. It will use the Atomic32 or
// Atomic64 routines, depending on the architecture.
typedef intptr_t AtomicType;
typedef uintptr_t UAtomicType;
// Template specialization for timestamp serialization/deserialization. This
// is used to serialize timestamps using Unix time on systems where AtomicType
// does not have enough precision to contain a timestamp in the standard
// serialized format.
template <int>
Time TimeFromWireFormat(int64_t value);
template <int>
int64_t TimeToWireFormat(Time time);
// Serialize to Unix time when using 4-byte wire format.
// Note: 19 January 2038, this will cease to work.
template <>
[[maybe_unused]] Time TimeFromWireFormat<4>(int64_t value) {
return value ? Time::UnixEpoch() + Seconds(value) : Time();
}
template <>
[[maybe_unused]] int64_t TimeToWireFormat<4>(Time time) {
return time > Time::UnixEpoch() ? (time - Time::UnixEpoch()).InSeconds() : 0;
}
// Standard serialization format when using 8-byte wire format.
template <>
[[maybe_unused]] Time TimeFromWireFormat<8>(int64_t value) {
return Time::FromInternalValue(value);
}
template <>
[[maybe_unused]] int64_t TimeToWireFormat<8>(Time time) {
return time.ToInternalValue();
}
struct SharedState {
enum LockState { UNLOCKED = 0, LOCKED = 1 };
explicit SharedState(AtomicType ivalue) { value.i = ivalue; }
SharedState(LockState lock_state, Time timestamp) {
int64_t wire_timestamp = TimeToWireFormat<sizeof(AtomicType)>(timestamp);
DCHECK_GE(wire_timestamp, 0);
DCHECK_EQ(lock_state & ~1, 0);
value.u = (static_cast<UAtomicType>(wire_timestamp) << 1) | lock_state;
}
LockState GetLockState() const { return static_cast<LockState>(value.u & 1); }
Time GetTimestamp() const {
return TimeFromWireFormat<sizeof(AtomicType)>(value.u >> 1);
}
// Bit 1: Lock state. Bit is set when locked.
// Bit 2..sizeof(AtomicType)*8: Usage timestamp. NULL time when locked or
// purged.
union {
AtomicType i;
UAtomicType u;
} value;
};
// Shared state is stored at offset 0 in shared memory segments.
SharedState* SharedStateFromSharedMemory(
const WritableSharedMemoryMapping& shared_memory) {
DCHECK(shared_memory.IsValid());
return static_cast<SharedState*>(shared_memory.memory());
}
// Round up |size| to a multiple of page size.
size_t AlignToPageSize(size_t size) {
return bits::AlignUp(size, base::GetPageSize());
}
#if BUILDFLAG(IS_ANDROID)
bool UseAshmemUnpinningForDiscardableMemory() {
if (!ashmem_device_is_supported())
return false;
// If we are participating in the discardable memory backing trial, only
// enable ashmem unpinning when we are in the corresponding trial group.
if (base::DiscardableMemoryBackingFieldTrialIsEnabled()) {
return base::GetDiscardableMemoryBackingFieldTrialGroup() ==
base::DiscardableMemoryTrialGroup::kAshmem;
}
return true;
}
#endif // BUILDFLAG(IS_ANDROID)
} // namespace
DiscardableSharedMemory::DiscardableSharedMemory() = default;
DiscardableSharedMemory::DiscardableSharedMemory(
UnsafeSharedMemoryRegion shared_memory_region)
: shared_memory_region_(std::move(shared_memory_region)) {}
DiscardableSharedMemory::~DiscardableSharedMemory() = default;
bool DiscardableSharedMemory::CreateAndMap(size_t size) {
CheckedNumeric<size_t> checked_size = size;
checked_size += AlignToPageSize(sizeof(SharedState));
if (!checked_size.IsValid())
return false;
shared_memory_region_ =
UnsafeSharedMemoryRegion::Create(checked_size.ValueOrDie());
if (!shared_memory_region_.IsValid())
return false;
shared_memory_mapping_ = shared_memory_region_.Map();
if (!shared_memory_mapping_.IsValid())
return false;
locked_page_count_ =
AlignToPageSize(mapped_memory().size()) / base::GetPageSize();
#if DCHECK_IS_ON()
for (size_t page = 0; page < locked_page_count_; ++page)
locked_pages_.insert(page);
#endif
DCHECK(last_known_usage_.is_null());
SharedState new_state(SharedState::LOCKED, Time());
subtle::Release_Store(
&SharedStateFromSharedMemory(shared_memory_mapping_)->value.i,
new_state.value.i);
return true;
}
bool DiscardableSharedMemory::Map(size_t size) {
DCHECK(!shared_memory_mapping_.IsValid());
if (shared_memory_mapping_.IsValid())
return false;
shared_memory_mapping_ = shared_memory_region_.MapAt(
0, AlignToPageSize(sizeof(SharedState)) + size);
if (!shared_memory_mapping_.IsValid())
return false;
locked_page_count_ =
AlignToPageSize(mapped_memory().size()) / base::GetPageSize();
#if DCHECK_IS_ON()
for (size_t page = 0; page < locked_page_count_; ++page)
locked_pages_.insert(page);
#endif
return true;
}
bool DiscardableSharedMemory::Unmap() {
if (!shared_memory_mapping_.IsValid())
return false;
shared_memory_mapping_ = WritableSharedMemoryMapping();
locked_page_count_ = 0;
#if DCHECK_IS_ON()
locked_pages_.clear();
#endif
return true;
}
DiscardableSharedMemory::LockResult DiscardableSharedMemory::Lock(
size_t offset, size_t length) {
DCHECK_EQ(AlignToPageSize(offset), offset);
DCHECK_EQ(AlignToPageSize(length), length);
// Calls to this function must be synchronized properly.
DFAKE_SCOPED_LOCK(thread_collision_warner_);
DCHECK(shared_memory_mapping_.IsValid());
// We need to successfully acquire the platform independent lock before
// individual pages can be locked.
if (!locked_page_count_) {
// Return false when instance has been purged or not initialized properly
// by checking if |last_known_usage_| is NULL.
if (last_known_usage_.is_null())
return FAILED;
SharedState old_state(SharedState::UNLOCKED, last_known_usage_);
SharedState new_state(SharedState::LOCKED, Time());
SharedState result(subtle::Acquire_CompareAndSwap(
&SharedStateFromSharedMemory(shared_memory_mapping_)->value.i,
old_state.value.i, new_state.value.i));
if (result.value.u != old_state.value.u) {
// Update |last_known_usage_| in case the above CAS failed because of
// an incorrect timestamp.
last_known_usage_ = result.GetTimestamp();
return FAILED;
}
}
// Zero for length means "everything onward".
if (!length)
length = AlignToPageSize(mapped_memory().size()) - offset;
size_t start = offset / base::GetPageSize();
size_t end = start + length / base::GetPageSize();
DCHECK_LE(start, end);
DCHECK_LE(end, AlignToPageSize(mapped_memory().size()) / base::GetPageSize());
// Add pages to |locked_page_count_|.
// Note: Locking a page that is already locked is an error.
locked_page_count_ += end - start;
#if DCHECK_IS_ON()
// Detect incorrect usage by keeping track of exactly what pages are locked.
for (auto page = start; page < end; ++page) {
auto result = locked_pages_.insert(page);
DCHECK(result.second);
}
DCHECK_EQ(locked_pages_.size(), locked_page_count_);
#endif
// Always behave as if memory was purged when trying to lock a 0 byte segment.
if (!length)
return PURGED;
#if BUILDFLAG(IS_ANDROID)
// Ensure that the platform won't discard the required pages.
return LockPages(shared_memory_region_,
AlignToPageSize(sizeof(SharedState)) + offset, length);
#elif BUILDFLAG(IS_APPLE)
// On macOS, there is no mechanism to lock pages. However, we do need to call
// madvise(MADV_FREE_REUSE) in order to correctly update accounting for memory
// footprint via task_info().
//
// Note that calling madvise(MADV_FREE_REUSE) on regions that haven't had
// madvise(MADV_FREE_REUSABLE) called on them has no effect.
//
// Note that the corresponding call to MADV_FREE_REUSABLE is in Purge(), since
// that's where the memory is actually released, rather than Unlock(), which
// is a no-op on macOS.
//
// For more information, see
// https://bugs.chromium.org/p/chromium/issues/detail?id=823915.
base::span<uint8_t> mapped = mapped_memory();
madvise(mapped.data(), AlignToPageSize(mapped.size()), MADV_FREE_REUSE);
return DiscardableSharedMemory::SUCCESS;
#else
return DiscardableSharedMemory::SUCCESS;
#endif
}
void DiscardableSharedMemory::Unlock(size_t offset, size_t length) {
DCHECK_EQ(AlignToPageSize(offset), offset);
DCHECK_EQ(AlignToPageSize(length), length);
// Calls to this function must be synchronized properly.
DFAKE_SCOPED_LOCK(thread_collision_warner_);
// Passing zero for |length| means "everything onward". Note that |length| may
// still be zero after this calculation, e.g. if the mapped size is zero.
if (!length)
length = AlignToPageSize(mapped_memory().size()) - offset;
DCHECK(shared_memory_mapping_.IsValid());
// Allow the pages to be discarded by the platform, if supported.
UnlockPages(shared_memory_region_,
AlignToPageSize(sizeof(SharedState)) + offset, length);
size_t start = offset / base::GetPageSize();
size_t end = start + length / base::GetPageSize();
DCHECK_LE(start, end);
DCHECK_LE(end, AlignToPageSize(mapped_memory().size()) / base::GetPageSize());
// Remove pages from |locked_page_count_|.
// Note: Unlocking a page that is not locked is an error.
DCHECK_GE(locked_page_count_, end - start);
locked_page_count_ -= end - start;
#if DCHECK_IS_ON()
// Detect incorrect usage by keeping track of exactly what pages are locked.
for (auto page = start; page < end; ++page) {
auto erased_count = locked_pages_.erase(page);
DCHECK_EQ(1u, erased_count);
}
DCHECK_EQ(locked_pages_.size(), locked_page_count_);
#endif
// Early out and avoid releasing the platform independent lock if some pages
// are still locked.
if (locked_page_count_)
return;
Time current_time = Now();
DCHECK(!current_time.is_null());
SharedState old_state(SharedState::LOCKED, Time());
SharedState new_state(SharedState::UNLOCKED, current_time);
// Note: timestamp cannot be NULL as that is a unique value used when
// locked or purged.
DCHECK(!new_state.GetTimestamp().is_null());
// Timestamp precision should at least be accurate to the second.
DCHECK_EQ((new_state.GetTimestamp() - Time::UnixEpoch()).InSeconds(),
(current_time - Time::UnixEpoch()).InSeconds());
SharedState result(subtle::Release_CompareAndSwap(
&SharedStateFromSharedMemory(shared_memory_mapping_)->value.i,
old_state.value.i, new_state.value.i));
DCHECK_EQ(old_state.value.u, result.value.u);
last_known_usage_ = current_time;
}
span<uint8_t> DiscardableSharedMemory::memory() const {
return shared_memory_mapping_.GetMemoryAsSpan<uint8_t>().subspan(
AlignToPageSize(sizeof(SharedState)));
}
span<uint8_t> DiscardableSharedMemory::mapped_memory() const {
return shared_memory_mapping_.mapped_memory().subspan(
AlignToPageSize(sizeof(SharedState)));
}
bool DiscardableSharedMemory::Purge(Time current_time) {
// Calls to this function must be synchronized properly.
DFAKE_SCOPED_LOCK(thread_collision_warner_);
DCHECK(shared_memory_mapping_.IsValid());
SharedState old_state(SharedState::UNLOCKED, last_known_usage_);
SharedState new_state(SharedState::UNLOCKED, Time());
SharedState result(subtle::Acquire_CompareAndSwap(
&SharedStateFromSharedMemory(shared_memory_mapping_)->value.i,
old_state.value.i, new_state.value.i));
// Update |last_known_usage_| to |current_time| if the memory is locked. This
// allows the caller to determine if purging failed because last known usage
// was incorrect or memory was locked. In the second case, the caller should
// most likely wait for some amount of time before attempting to purge the
// the memory again.
if (result.value.u != old_state.value.u) {
last_known_usage_ = result.GetLockState() == SharedState::LOCKED
? current_time
: result.GetTimestamp();
return false;
}
// The next section will release as much resource as can be done
// from the purging process, until the client process notices the
// purge and releases its own references.
// Note: this memory will not be accessed again. The segment will be
// freed asynchronously at a later time, so just do the best
// immediately.
#if BUILDFLAG(IS_POSIX) && !BUILDFLAG(IS_NACL)
// Linux and Android provide MADV_REMOVE which is preferred as it has a
// behavior that can be verified in tests. Other POSIX flavors (MacOSX, BSDs),
// provide MADV_FREE which has the same result but memory is purged lazily.
#if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS) || BUILDFLAG(IS_ANDROID)
#define MADV_PURGE_ARGUMENT MADV_REMOVE
#elif BUILDFLAG(IS_APPLE)
// MADV_FREE_REUSABLE is similar to MADV_FREE, but also marks the pages with the
// reusable bit, which allows both Activity Monitor and memory-infra to
// correctly track the pages.
#define MADV_PURGE_ARGUMENT MADV_FREE_REUSABLE
#else
#define MADV_PURGE_ARGUMENT MADV_FREE
#endif
// Advise the kernel to remove resources associated with purged pages.
// Subsequent accesses of memory pages will succeed, but might result in
// zero-fill-on-demand pages.
base::span<uint8_t> map = mapped_memory();
if (madvise(map.data(), AlignToPageSize(map.size()), MADV_PURGE_ARGUMENT)) {
DPLOG(ERROR) << "madvise() failed";
}
#elif BUILDFLAG(IS_WIN)
// On Windows, discarded pages are not returned to the system immediately and
// not guaranteed to be zeroed when returned to the application.
base::span<uint8_t> mapped = mapped_memory();
uint8_t* address = mapped.data();
size_t length = AlignToPageSize(mapped.size());
DWORD ret = DiscardVirtualMemory(address, length);
// DiscardVirtualMemory is buggy in Win10 SP0, so fall back to MEM_RESET on
// failure.
if (ret != ERROR_SUCCESS) {
void* ptr = VirtualAlloc(address, length, MEM_RESET, PAGE_READWRITE);
CHECK(ptr);
}
#elif BUILDFLAG(IS_FUCHSIA)
// De-commit via our VMAR, rather than relying on the VMO handle, since the
// handle may have been closed after the memory was mapped into this process.
base::span<uint8_t> mapped = mapped_memory();
uint64_t address_int = reinterpret_cast<uint64_t>(mapped.data());
zx_status_t status = zx::vmar::root_self()->op_range(
ZX_VMO_OP_DECOMMIT, address_int, AlignToPageSize(mapped.size()), nullptr,
0);
ZX_DCHECK(status == ZX_OK, status) << "zx_vmo_op_range(ZX_VMO_OP_DECOMMIT)";
#endif // BUILDFLAG(IS_FUCHSIA)
last_known_usage_ = Time();
return true;
}
bool DiscardableSharedMemory::IsMemoryResident() const {
DCHECK(shared_memory_mapping_.IsValid());
SharedState result(subtle::NoBarrier_Load(
&SharedStateFromSharedMemory(shared_memory_mapping_)->value.i));
return result.GetLockState() == SharedState::LOCKED ||
!result.GetTimestamp().is_null();
}
bool DiscardableSharedMemory::IsMemoryLocked() const {
DCHECK(shared_memory_mapping_.IsValid());
SharedState result(subtle::NoBarrier_Load(
&SharedStateFromSharedMemory(shared_memory_mapping_)->value.i));
return result.GetLockState() == SharedState::LOCKED;
}
void DiscardableSharedMemory::Close() {
shared_memory_region_ = UnsafeSharedMemoryRegion();
}
void DiscardableSharedMemory::CreateSharedMemoryOwnershipEdge(
trace_event::MemoryAllocatorDump* local_segment_dump,
trace_event::ProcessMemoryDump* pmd,
bool is_owned) const {
// Memory dumps are only supported when tracing support is enabled,.
#if BUILDFLAG(ENABLE_BASE_TRACING)
auto* shared_memory_dump = SharedMemoryTracker::GetOrCreateSharedMemoryDump(
shared_memory_mapping_, pmd);
// TODO(ssid): Clean this by a new api to inherit size of parent dump once the
// we send the full PMD and calculate sizes inside chrome, crbug.com/704203.
uint64_t resident_size = shared_memory_dump->GetSizeInternal();
local_segment_dump->AddScalar(trace_event::MemoryAllocatorDump::kNameSize,
trace_event::MemoryAllocatorDump::kUnitsBytes,
resident_size);
// By creating an edge with a higher |importance| (w.r.t non-owned dumps)
// the tracing UI will account the effective size of the segment to the
// client instead of manager.
// TODO(ssid): Define better constants in MemoryAllocatorDump for importance
// values, crbug.com/754793.
const int kImportance = is_owned ? 2 : 0;
auto shared_memory_guid = shared_memory_mapping_.guid();
local_segment_dump->AddString("id", "hash", shared_memory_guid.ToString());
// Owned discardable segments which are allocated by client process, could
// have been cleared by the discardable manager. So, the segment need not
// exist in memory and weak dumps are created to indicate the UI that the dump
// should exist only if the manager also created the global dump edge.
if (is_owned) {
pmd->CreateWeakSharedMemoryOwnershipEdge(local_segment_dump->guid(),
shared_memory_guid, kImportance);
} else {
pmd->CreateSharedMemoryOwnershipEdge(local_segment_dump->guid(),
shared_memory_guid, kImportance);
}
#endif // BUILDFLAG(ENABLE_BASE_TRACING)
}
// static
DiscardableSharedMemory::LockResult DiscardableSharedMemory::LockPages(
const UnsafeSharedMemoryRegion& region,
size_t offset,
size_t length) {
#if BUILDFLAG(IS_ANDROID)
if (region.IsValid()) {
if (UseAshmemUnpinningForDiscardableMemory()) {
int pin_result =
ashmem_pin_region(region.GetPlatformHandle(), offset, length);
if (pin_result == ASHMEM_WAS_PURGED)
return PURGED;
if (pin_result < 0)
return FAILED;
}
}
#endif
return SUCCESS;
}
// static
void DiscardableSharedMemory::UnlockPages(
const UnsafeSharedMemoryRegion& region,
size_t offset,
size_t length) {
#if BUILDFLAG(IS_ANDROID)
if (region.IsValid()) {
if (UseAshmemUnpinningForDiscardableMemory()) {
int unpin_result =
ashmem_unpin_region(region.GetPlatformHandle(), offset, length);
DCHECK_EQ(0, unpin_result);
}
}
#endif
}
Time DiscardableSharedMemory::Now() const {
return Time::Now();
}
#if BUILDFLAG(IS_ANDROID)
// static
bool DiscardableSharedMemory::IsAshmemDeviceSupportedForTesting() {
return UseAshmemUnpinningForDiscardableMemory();
}
#endif
} // namespace base