[PA] Support 16kb pagesize on Linux+ARM64
This makes the system pagesize a run-time property.
ARM64 supports 4kb, 16kb, and 64kb page sizes. Previously, only 4kb
was supported by Chromium. This patch adds 16kb support, as is used
for example by Asahi Linux on M1 Macs. The rare 64kb case is still
not supported due to further changes needed to SlotSpanMetadata.
The implementation follows the changes made to support run-time page
size on macOS. On macOS, the required constants are conveniently
injected before any code runs, while on Linux a function call is
needed, complicating initialization.
The new PageCharacteristics structure holds the page size and shift
as std::atomic<int> which are initialized on first use.
Bug: 1301788
Change-Id: I8ceead40de53ba7a2ec248bd6ef46f2a521dd29c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3545665
Reviewed-by: Benoit Lize <lizeb@chromium.org>
Reviewed-by: Mark Mentovai <mark@chromium.org>
Commit-Queue: Mark Mentovai <mark@chromium.org>
Cr-Commit-Position: refs/heads/main@{#991588}
diff --git a/AUTHORS b/AUTHORS
index 9a05bca..8d638184 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -581,6 +581,7 @@
Joone Hur <joone.hur@intel.com>
Joonghun Park <pjh0718@gmail.com>
Jorge Villatoro <jorge@tomatocannon.com>
+Jorrit Jongma <jorrit@jongma.org>
Joseph Gentle <josephg@gmail.com>
Joseph Lolak <joseph.lolak@samsung.com>
Josh Triplett <josh.triplett@intel.com>
diff --git a/base/allocator/partition_allocator/address_space_randomization.h b/base/allocator/partition_allocator/address_space_randomization.h
index 43033d7..e77757c3 100644
--- a/base/allocator/partition_allocator/address_space_randomization.h
+++ b/base/allocator/partition_allocator/address_space_randomization.h
@@ -121,6 +121,21 @@
return AslrAddress(0x20000000ULL);
}
+ #elif BUILDFLAG(IS_LINUX)
+
+ // Linux on arm64 can use 39, 42, 48, or 52-bit user space, depending on
+ // page size and number of levels of translation pages used. We use
+ // 39-bit as base as all setups should support this, lowered to 38-bit
+ // as ASLROffset() could cause a carry.
+ PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE uintptr_t
+ ASLRMask() {
+ return AslrMask(38);
+ }
+ PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE uintptr_t
+ ASLROffset() {
+ return AslrAddress(0x1000000000ULL);
+ }
+
#else
// ARM64 on Linux has 39-bit user space. Use 38 bits since ASLROffset()
diff --git a/base/allocator/partition_allocator/page_allocator_constants.h b/base/allocator/partition_allocator/page_allocator_constants.h
index 12515b9..0a996cfd 100644
--- a/base/allocator/partition_allocator/page_allocator_constants.h
+++ b/base/allocator/partition_allocator/page_allocator_constants.h
@@ -24,6 +24,31 @@
// elimination.
#define PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR __attribute__((const))
+#elif BUILDFLAG(IS_LINUX) && defined(ARCH_CPU_ARM64)
+// This should work for all POSIX (if needed), but currently all other
+// supported OS/architecture combinations use either hard-coded values
+// (such as x86) or have means to determine these values without needing
+// atomics (such as macOS on arm64).
+
+// Page allocator constants are run-time constant
+#define PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR __attribute__((const))
+
+#include <unistd.h>
+#include <atomic>
+
+namespace partition_alloc::internal {
+
+// Holds the current page size and shift, where size = 1 << shift
+// Use PageAllocationGranularity(), PageAllocationGranularityShift()
+// to initialize and retrieve these values safely.
+struct PageCharacteristics {
+ std::atomic<int> size;
+ std::atomic<int> shift;
+};
+extern PageCharacteristics page_characteristics;
+
+} // namespace partition_alloc::internal
+
#else
// When defined, page size constants are fixed at compile time. When not
@@ -38,6 +63,10 @@
namespace partition_alloc::internal {
+// Forward declaration, implementation below
+PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t
+PageAllocationGranularity();
+
PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t
PageAllocationGranularityShift() {
#if BUILDFLAG(IS_WIN) || defined(ARCH_CPU_PPC64)
@@ -50,6 +79,15 @@
return 14; // 16kB
#elif BUILDFLAG(IS_APPLE) && defined(ARCH_CPU_64_BITS)
return vm_page_shift;
+#elif BUILDFLAG(IS_LINUX) && defined(ARCH_CPU_ARM64)
+ // arm64 supports 4kb (shift = 12), 16kb (shift = 14), and 64kb (shift = 16)
+ // page sizes. Retrieve from or initialize cache.
+ int shift = page_characteristics.shift.load(std::memory_order_relaxed);
+ if (UNLIKELY(shift == 0)) {
+ shift = __builtin_ctz((int)PageAllocationGranularity());
+ page_characteristics.shift.store(shift, std::memory_order_relaxed);
+ }
+ return shift;
#else
return 12; // 4kB
#endif
@@ -59,8 +97,17 @@
PageAllocationGranularity() {
#if BUILDFLAG(IS_APPLE) && defined(ARCH_CPU_64_BITS)
// This is literally equivalent to |1 << PageAllocationGranularityShift()|
- // below, but was separated out for OS_APPLE to avoid << on a non-constexpr.
+ // below, but was separated out for IS_APPLE to avoid << on a non-constexpr.
return vm_page_size;
+#elif BUILDFLAG(IS_LINUX) && defined(ARCH_CPU_ARM64)
+ // arm64 supports 4kb, 16kb, and 64kb page sizes. Retrieve from or
+ // initialize cache.
+ int size = page_characteristics.size.load(std::memory_order_relaxed);
+ if (UNLIKELY(size == 0)) {
+ size = getpagesize();
+ page_characteristics.size.store(size, std::memory_order_relaxed);
+ }
+ return size;
#else
return 1 << PageAllocationGranularityShift();
#endif
@@ -90,9 +137,11 @@
PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t
SystemPageSize() {
-#if BUILDFLAG(IS_APPLE) && defined(ARCH_CPU_64_BITS)
+#if (BUILDFLAG(IS_APPLE) && defined(ARCH_CPU_64_BITS)) || \
+ (BUILDFLAG(IS_LINUX) && defined(ARCH_CPU_ARM64))
// This is literally equivalent to |1 << SystemPageShift()| below, but was
- // separated out for 64-bit OS_APPLE to avoid << on a non-constexpr.
+ // separated out for 64-bit IS_APPLE and arm64 on Linux to avoid << on a
+ // non-constexpr.
return PageAllocationGranularity();
#else
return 1 << SystemPageShift();
diff --git a/base/allocator/partition_allocator/partition_address_space.cc b/base/allocator/partition_allocator/partition_address_space.cc
index 6e488dd..caa69c0 100644
--- a/base/allocator/partition_allocator/partition_address_space.cc
+++ b/base/allocator/partition_allocator/partition_address_space.cc
@@ -184,6 +184,12 @@
setup_.configurable_pool_ = 0;
}
+#if BUILDFLAG(IS_LINUX) && defined(ARCH_CPU_ARM64)
+
+PageCharacteristics page_characteristics;
+
+#endif // BUILDFLAG(IS_LINUX) && defined(ARCH_CPU_ARM64)
+
#endif // defined(PA_HAS_64_BITS_POINTERS)
} // namespace partition_alloc::internal
diff --git a/base/allocator/partition_allocator/partition_alloc_constants.h b/base/allocator/partition_allocator/partition_alloc_constants.h
index d8eb68f..48117ad 100644
--- a/base/allocator/partition_allocator/partition_alloc_constants.h
+++ b/base/allocator/partition_allocator/partition_alloc_constants.h
@@ -79,10 +79,11 @@
PartitionPageShift() {
return 18; // 256 KiB
}
-#elif BUILDFLAG(IS_APPLE) && defined(ARCH_CPU_64_BITS)
+#elif (BUILDFLAG(IS_APPLE) && defined(ARCH_CPU_64_BITS)) || \
+ (BUILDFLAG(IS_LINUX) && defined(ARCH_CPU_ARM64))
PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t
PartitionPageShift() {
- return vm_page_shift + 2;
+ return PageAllocationGranularityShift() + 2;
}
#else
PAGE_ALLOCATOR_CONSTANTS_DECLARE_CONSTEXPR ALWAYS_INLINE size_t
diff --git a/base/allocator/partition_allocator/partition_page.h b/base/allocator/partition_allocator/partition_page.h
index 593d9d5..1a67254c 100644
--- a/base/allocator/partition_allocator/partition_page.h
+++ b/base/allocator/partition_allocator/partition_page.h
@@ -135,6 +135,12 @@
// PartitionPageSize() is 4 times the OS page size.
static constexpr size_t kMaxSlotsPerSlotSpan =
4 * (1 << 14) / kSmallestBucket;
+#elif BUILDFLAG(IS_LINUX) && defined(ARCH_CPU_ARM64)
+ // System page size can be 4, 16, or 64 kiB on Linux on arm64. 64 kiB is
+ // currently (kMaxSlotsPerSlotSpanBits == 13) not supported by the code,
+ // so we use the 16 kiB maximum (64 kiB will crash).
+ static constexpr size_t kMaxSlotsPerSlotSpan =
+ 4 * (1 << 14) / kSmallestBucket;
#else
// A slot span can "span" multiple PartitionPages, but then its slot size is
// larger, so it doesn't have as many slots.
diff --git a/base/allocator/partition_allocator/partition_root.cc b/base/allocator/partition_allocator/partition_root.cc
index 5e53b1c..f5ff97f9 100644
--- a/base/allocator/partition_allocator/partition_root.cc
+++ b/base/allocator/partition_allocator/partition_root.cc
@@ -305,11 +305,12 @@
constexpr size_t kMaxSlotCount =
(PartitionPageSize() * kMaxPartitionPagesPerRegularSlotSpan) /
SystemPageSize();
-#elif BUILDFLAG(IS_APPLE)
+#elif BUILDFLAG(IS_APPLE) || (BUILDFLAG(IS_LINUX) && defined(ARCH_CPU_ARM64))
// It's better for slot_usage to be stack-allocated and fixed-size, which
- // demands that its size be constexpr. On OS_APPLE, PartitionPageSize() is
- // always SystemPageSize() << 2, so regardless of what the run time page size
- // is, kMaxSlotCount can always be simplified to this expression.
+ // demands that its size be constexpr. On IS_APPLE and Linux on arm64,
+ // PartitionPageSize() is always SystemPageSize() << 2, so regardless of
+ // what the run time page size is, kMaxSlotCount can always be simplified
+ // to this expression.
constexpr size_t kMaxSlotCount =
4 * internal::kMaxPartitionPagesPerRegularSlotSpan;
PA_CHECK(kMaxSlotCount == (PartitionPageSize() *
@@ -647,6 +648,14 @@
// apple OSes.
PA_CHECK((internal::SystemPageSize() == (size_t{1} << 12)) ||
(internal::SystemPageSize() == (size_t{1} << 14)));
+#elif BUILDFLAG(IS_LINUX) && defined(ARCH_CPU_ARM64)
+ // Check runtime pagesize. Though the code is currently the same, it is
+ // not merged with the IS_APPLE case above as a 1 << 16 case needs to be
+ // added here in the future, to allow 64 kiB pagesize. That is only
+ // supported on Linux on arm64, not on IS_APPLE, but not yet present here
+ // as the rest of the partition allocator does not currently support it.
+ PA_CHECK((internal::SystemPageSize() == (size_t{1} << 12)) ||
+ (internal::SystemPageSize() == (size_t{1} << 14)));
#endif
::partition_alloc::internal::ScopedGuard guard{lock_};
