| // Copyright 2013 The Chromium Authors | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| #include "base/allocator/partition_allocator/page_allocator.h" | |
| #include <atomic> | |
| #include <cstdint> | |
| #include "base/allocator/partition_allocator/address_space_randomization.h" | |
| #include "base/allocator/partition_allocator/page_allocator_internal.h" | |
| #include "base/allocator/partition_allocator/partition_alloc_base/bits.h" | |
| #include "base/allocator/partition_allocator/partition_alloc_base/thread_annotations.h" | |
| #include "base/allocator/partition_allocator/partition_alloc_check.h" | |
| #include "base/allocator/partition_allocator/partition_lock.h" | |
| #include "build/build_config.h" | |
| #if BUILDFLAG(IS_WIN) | |
| #include <windows.h> | |
| #endif | |
| #if BUILDFLAG(IS_WIN) | |
| #include "base/allocator/partition_allocator/page_allocator_internals_win.h" | |
| #elif BUILDFLAG(IS_POSIX) | |
| #include "base/allocator/partition_allocator/page_allocator_internals_posix.h" | |
| #elif BUILDFLAG(IS_FUCHSIA) | |
| #include "base/allocator/partition_allocator/page_allocator_internals_fuchsia.h" | |
| #else | |
| #error Platform not supported. | |
| #endif | |
| namespace partition_alloc { | |
| namespace { | |
| internal::Lock g_reserve_lock; | |
| // We may reserve/release address space on different threads. | |
| internal::Lock& GetReserveLock() { | |
| return g_reserve_lock; | |
| } | |
| std::atomic<size_t> g_total_mapped_address_space; | |
| // We only support a single block of reserved address space. | |
| uintptr_t s_reservation_address PA_GUARDED_BY(GetReserveLock()) = 0; | |
| size_t s_reservation_size PA_GUARDED_BY(GetReserveLock()) = 0; | |
| uintptr_t AllocPagesIncludingReserved( | |
| uintptr_t address, | |
| size_t length, | |
| PageAccessibilityConfiguration accessibility, | |
| PageTag page_tag) { | |
| uintptr_t ret = | |
| internal::SystemAllocPages(address, length, accessibility, page_tag); | |
| if (!ret) { | |
| const bool cant_alloc_length = internal::kHintIsAdvisory || !address; | |
| if (cant_alloc_length) { | |
| // The system cannot allocate |length| bytes. Release any reserved address | |
| // space and try once more. | |
| ReleaseReservation(); | |
| ret = | |
| internal::SystemAllocPages(address, length, accessibility, page_tag); | |
| } | |
| } | |
| return ret; | |
| } | |
| // Trims memory at |base_address| to given |trim_length| and |alignment|. | |
| // | |
| // On failure, on Windows, this function returns 0 and frees memory at | |
| // |base_address|. | |
| uintptr_t TrimMapping(uintptr_t base_address, | |
| size_t base_length, | |
| size_t trim_length, | |
| uintptr_t alignment, | |
| uintptr_t alignment_offset, | |
| PageAccessibilityConfiguration accessibility) { | |
| PA_DCHECK(base_length >= trim_length); | |
| PA_DCHECK(internal::base::bits::IsPowerOfTwo(alignment)); | |
| PA_DCHECK(alignment_offset < alignment); | |
| uintptr_t new_base = | |
| NextAlignedWithOffset(base_address, alignment, alignment_offset); | |
| PA_DCHECK(new_base >= base_address); | |
| size_t pre_slack = new_base - base_address; | |
| size_t post_slack = base_length - pre_slack - trim_length; | |
| PA_DCHECK(base_length == trim_length || pre_slack || post_slack); | |
| PA_DCHECK(pre_slack < base_length); | |
| PA_DCHECK(post_slack < base_length); | |
| return internal::TrimMappingInternal(base_address, base_length, trim_length, | |
| accessibility, pre_slack, post_slack); | |
| } | |
| } // namespace | |
| // Align |address| up to the closest, non-smaller address, that gives | |
| // |requested_offset| remainder modulo |alignment|. | |
| // | |
| // Examples for alignment=1024 and requested_offset=64: | |
| // 64 -> 64 | |
| // 65 -> 1088 | |
| // 1024 -> 1088 | |
| // 1088 -> 1088 | |
| // 1089 -> 2112 | |
| // 2048 -> 2112 | |
| uintptr_t NextAlignedWithOffset(uintptr_t address, | |
| uintptr_t alignment, | |
| uintptr_t requested_offset) { | |
| PA_DCHECK(internal::base::bits::IsPowerOfTwo(alignment)); | |
| PA_DCHECK(requested_offset < alignment); | |
| uintptr_t actual_offset = address & (alignment - 1); | |
| uintptr_t new_address; | |
| if (actual_offset <= requested_offset) | |
| new_address = address + requested_offset - actual_offset; | |
| else | |
| new_address = address + alignment + requested_offset - actual_offset; | |
| PA_DCHECK(new_address >= address); | |
| PA_DCHECK(new_address - address < alignment); | |
| PA_DCHECK(new_address % alignment == requested_offset); | |
| return new_address; | |
| } | |
| namespace internal { | |
| uintptr_t SystemAllocPages(uintptr_t hint, | |
| size_t length, | |
| PageAccessibilityConfiguration accessibility, | |
| PageTag page_tag) { | |
| PA_DCHECK(!(length & internal::PageAllocationGranularityOffsetMask())); | |
| PA_DCHECK(!(hint & internal::PageAllocationGranularityOffsetMask())); | |
| uintptr_t ret = | |
| internal::SystemAllocPagesInternal(hint, length, accessibility, page_tag); | |
| if (ret) | |
| g_total_mapped_address_space.fetch_add(length, std::memory_order_relaxed); | |
| return ret; | |
| } | |
| } // namespace internal | |
| uintptr_t AllocPages(size_t length, | |
| size_t align, | |
| PageAccessibilityConfiguration accessibility, | |
| PageTag page_tag) { | |
| return AllocPagesWithAlignOffset(0, length, align, 0, accessibility, | |
| page_tag); | |
| } | |
| uintptr_t AllocPages(uintptr_t address, | |
| size_t length, | |
| size_t align, | |
| PageAccessibilityConfiguration accessibility, | |
| PageTag page_tag) { | |
| return AllocPagesWithAlignOffset(address, length, align, 0, accessibility, | |
| page_tag); | |
| } | |
| void* AllocPages(void* address, | |
| size_t length, | |
| size_t align, | |
| PageAccessibilityConfiguration accessibility, | |
| PageTag page_tag) { | |
| return reinterpret_cast<void*>( | |
| AllocPages(reinterpret_cast<uintptr_t>(address), length, align, | |
| accessibility, page_tag)); | |
| } | |
| uintptr_t AllocPagesWithAlignOffset( | |
| uintptr_t address, | |
| size_t length, | |
| size_t align, | |
| size_t align_offset, | |
| PageAccessibilityConfiguration accessibility, | |
| PageTag page_tag) { | |
| PA_DCHECK(length >= internal::PageAllocationGranularity()); | |
| PA_DCHECK(!(length & internal::PageAllocationGranularityOffsetMask())); | |
| PA_DCHECK(align >= internal::PageAllocationGranularity()); | |
| // Alignment must be power of 2 for masking math to work. | |
| PA_DCHECK(internal::base::bits::IsPowerOfTwo(align)); | |
| PA_DCHECK(align_offset < align); | |
| PA_DCHECK(!(align_offset & internal::PageAllocationGranularityOffsetMask())); | |
| PA_DCHECK(!(address & internal::PageAllocationGranularityOffsetMask())); | |
| uintptr_t align_offset_mask = align - 1; | |
| uintptr_t align_base_mask = ~align_offset_mask; | |
| PA_DCHECK(!address || (address & align_offset_mask) == align_offset); | |
| // If the client passed null as the address, choose a good one. | |
| if (!address) { | |
| address = (GetRandomPageBase() & align_base_mask) + align_offset; | |
| } | |
| // First try to force an exact-size, aligned allocation from our random base. | |
| #if defined(ARCH_CPU_32_BITS) | |
| // On 32 bit systems, first try one random aligned address, and then try an | |
| // aligned address derived from the value of |ret|. | |
| constexpr int kExactSizeTries = 2; | |
| #else | |
| // On 64 bit systems, try 3 random aligned addresses. | |
| constexpr int kExactSizeTries = 3; | |
| #endif | |
| for (int i = 0; i < kExactSizeTries; ++i) { | |
| uintptr_t ret = | |
| AllocPagesIncludingReserved(address, length, accessibility, page_tag); | |
| if (ret) { | |
| // If the alignment is to our liking, we're done. | |
| if ((ret & align_offset_mask) == align_offset) | |
| return ret; | |
| // Free the memory and try again. | |
| FreePages(ret, length); | |
| } else { | |
| // |ret| is null; if this try was unhinted, we're OOM. | |
| if (internal::kHintIsAdvisory || !address) | |
| return 0; | |
| } | |
| #if defined(ARCH_CPU_32_BITS) | |
| // For small address spaces, try the first aligned address >= |ret|. Note | |
| // |ret| may be null, in which case |address| becomes null. If | |
| // |align_offset| is non-zero, this calculation may get us not the first, | |
| // but the next matching address. | |
| address = ((ret + align_offset_mask) & align_base_mask) + align_offset; | |
| #else // defined(ARCH_CPU_64_BITS) | |
| // Keep trying random addresses on systems that have a large address space. | |
| address = NextAlignedWithOffset(GetRandomPageBase(), align, align_offset); | |
| #endif | |
| } | |
| // Make a larger allocation so we can force alignment. | |
| size_t try_length = length + (align - internal::PageAllocationGranularity()); | |
| PA_CHECK(try_length >= length); | |
| uintptr_t ret; | |
| do { | |
| // Continue randomizing only on POSIX. | |
| address = internal::kHintIsAdvisory ? GetRandomPageBase() : 0; | |
| ret = AllocPagesIncludingReserved(address, try_length, accessibility, | |
| page_tag); | |
| // The retries are for Windows, where a race can steal our mapping on | |
| // resize. | |
| } while (ret && (ret = TrimMapping(ret, try_length, length, align, | |
| align_offset, accessibility)) == 0); | |
| return ret; | |
| } | |
| void FreePages(uintptr_t address, size_t length) { | |
| PA_DCHECK(!(address & internal::PageAllocationGranularityOffsetMask())); | |
| PA_DCHECK(!(length & internal::PageAllocationGranularityOffsetMask())); | |
| internal::FreePagesInternal(address, length); | |
| PA_DCHECK(g_total_mapped_address_space.load(std::memory_order_relaxed) > 0); | |
| g_total_mapped_address_space.fetch_sub(length, std::memory_order_relaxed); | |
| } | |
| void FreePages(void* address, size_t length) { | |
| FreePages(reinterpret_cast<uintptr_t>(address), length); | |
| } | |
| bool TrySetSystemPagesAccess(uintptr_t address, | |
| size_t length, | |
| PageAccessibilityConfiguration accessibility) { | |
| PA_DCHECK(!(length & internal::SystemPageOffsetMask())); | |
| return internal::TrySetSystemPagesAccessInternal(address, length, | |
| accessibility); | |
| } | |
| bool TrySetSystemPagesAccess(void* address, | |
| size_t length, | |
| PageAccessibilityConfiguration accessibility) { | |
| return TrySetSystemPagesAccess(reinterpret_cast<uintptr_t>(address), length, | |
| accessibility); | |
| } | |
| void SetSystemPagesAccess(uintptr_t address, | |
| size_t length, | |
| PageAccessibilityConfiguration accessibility) { | |
| PA_DCHECK(!(length & internal::SystemPageOffsetMask())); | |
| internal::SetSystemPagesAccessInternal(address, length, accessibility); | |
| } | |
| void DecommitSystemPages( | |
| uintptr_t address, | |
| size_t length, | |
| PageAccessibilityDisposition accessibility_disposition) { | |
| PA_DCHECK(!(address & internal::SystemPageOffsetMask())); | |
| PA_DCHECK(!(length & internal::SystemPageOffsetMask())); | |
| internal::DecommitSystemPagesInternal(address, length, | |
| accessibility_disposition); | |
| } | |
| void DecommitSystemPages( | |
| void* address, | |
| size_t length, | |
| PageAccessibilityDisposition accessibility_disposition) { | |
| DecommitSystemPages(reinterpret_cast<uintptr_t>(address), length, | |
| accessibility_disposition); | |
| } | |
| void DecommitAndZeroSystemPages(uintptr_t address, size_t length) { | |
| PA_DCHECK(!(address & internal::SystemPageOffsetMask())); | |
| PA_DCHECK(!(length & internal::SystemPageOffsetMask())); | |
| internal::DecommitAndZeroSystemPagesInternal(address, length); | |
| } | |
| void DecommitAndZeroSystemPages(void* address, size_t length) { | |
| DecommitAndZeroSystemPages(reinterpret_cast<uintptr_t>(address), length); | |
| } | |
| void RecommitSystemPages( | |
| uintptr_t address, | |
| size_t length, | |
| PageAccessibilityConfiguration accessibility, | |
| PageAccessibilityDisposition accessibility_disposition) { | |
| PA_DCHECK(!(address & internal::SystemPageOffsetMask())); | |
| PA_DCHECK(!(length & internal::SystemPageOffsetMask())); | |
| PA_DCHECK(accessibility != PageAccessibilityConfiguration::kInaccessible); | |
| internal::RecommitSystemPagesInternal(address, length, accessibility, | |
| accessibility_disposition); | |
| } | |
| bool TryRecommitSystemPages( | |
| uintptr_t address, | |
| size_t length, | |
| PageAccessibilityConfiguration accessibility, | |
| PageAccessibilityDisposition accessibility_disposition) { | |
| // Duplicated because we want errors to be reported at a lower level in the | |
| // crashing case. | |
| PA_DCHECK(!(address & internal::SystemPageOffsetMask())); | |
| PA_DCHECK(!(length & internal::SystemPageOffsetMask())); | |
| PA_DCHECK(accessibility != PageAccessibilityConfiguration::kInaccessible); | |
| return internal::TryRecommitSystemPagesInternal( | |
| address, length, accessibility, accessibility_disposition); | |
| } | |
| void DiscardSystemPages(uintptr_t address, size_t length) { | |
| PA_DCHECK(!(length & internal::SystemPageOffsetMask())); | |
| internal::DiscardSystemPagesInternal(address, length); | |
| } | |
| void DiscardSystemPages(void* address, size_t length) { | |
| DiscardSystemPages(reinterpret_cast<uintptr_t>(address), length); | |
| } | |
| bool ReserveAddressSpace(size_t size) { | |
| // To avoid deadlock, call only SystemAllocPages. | |
| internal::ScopedGuard guard(GetReserveLock()); | |
| if (!s_reservation_address) { | |
| uintptr_t mem = internal::SystemAllocPages( | |
| 0, size, PageAccessibilityConfiguration::kInaccessible, | |
| PageTag::kChromium); | |
| if (mem) { | |
| // We guarantee this alignment when reserving address space. | |
| PA_DCHECK(!(mem & internal::PageAllocationGranularityOffsetMask())); | |
| s_reservation_address = mem; | |
| s_reservation_size = size; | |
| return true; | |
| } | |
| } | |
| return false; | |
| } | |
| bool ReleaseReservation() { | |
| // To avoid deadlock, call only FreePages. | |
| internal::ScopedGuard guard(GetReserveLock()); | |
| if (!s_reservation_address) | |
| return false; | |
| FreePages(s_reservation_address, s_reservation_size); | |
| s_reservation_address = 0; | |
| s_reservation_size = 0; | |
| return true; | |
| } | |
| bool HasReservationForTesting() { | |
| internal::ScopedGuard guard(GetReserveLock()); | |
| return s_reservation_address; | |
| } | |
| uint32_t GetAllocPageErrorCode() { | |
| return internal::s_allocPageErrorCode; | |
| } | |
| size_t GetTotalMappedSize() { | |
| return g_total_mapped_address_space; | |
| } | |
| } // namespace partition_alloc |