blob: 217cbff4d6a7069cf514b9d316e3acf4030452c5 [file] [log] [blame]
// Copyright 2023 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "src/heap/main-allocator.h"
#include "src/base/logging.h"
#include "src/base/optional.h"
#include "src/common/globals.h"
#include "src/execution/vm-state-inl.h"
#include "src/execution/vm-state.h"
#include "src/heap/concurrent-marking.h"
#include "src/heap/free-list-inl.h"
#include "src/heap/gc-tracer-inl.h"
#include "src/heap/heap.h"
#include "src/heap/incremental-marking.h"
#include "src/heap/main-allocator-inl.h"
#include "src/heap/new-spaces.h"
#include "src/heap/page-inl.h"
#include "src/heap/paged-spaces.h"
#include "src/heap/spaces.h"
namespace v8 {
namespace internal {
MainAllocator::MainAllocator(LocalHeap* local_heap, SpaceWithLinearArea* space,
LinearAllocationArea* allocation_info)
: local_heap_(local_heap),
isolate_heap_(local_heap->heap()),
space_(space),
allocation_info_(allocation_info != nullptr ? allocation_info
: &owned_allocation_info_),
allocator_policy_(space->CreateAllocatorPolicy(this)),
supports_extending_lab_(allocator_policy_->SupportsExtendingLAB()) {
CHECK_NOT_NULL(local_heap_);
if (local_heap_->is_main_thread()) {
allocation_counter_.emplace();
linear_area_original_data_.emplace();
}
}
MainAllocator::MainAllocator(Heap* heap, SpaceWithLinearArea* space, InGCTag)
: local_heap_(nullptr),
isolate_heap_(heap),
space_(space),
allocation_info_(&owned_allocation_info_),
allocator_policy_(space->CreateAllocatorPolicy(this)),
supports_extending_lab_(false) {
DCHECK(!allocation_counter_.has_value());
DCHECK(!linear_area_original_data_.has_value());
}
Address MainAllocator::AlignTopForTesting(AllocationAlignment alignment,
int offset) {
DCHECK(top());
int filler_size = Heap::GetFillToAlign(top(), alignment);
if (filler_size + offset) {
space_heap()->CreateFillerObjectAt(top(), filler_size + offset);
allocation_info().IncrementTop(filler_size + offset);
}
return top();
}
AllocationResult MainAllocator::AllocateRawForceAlignmentForTesting(
int size_in_bytes, AllocationAlignment alignment, AllocationOrigin origin) {
size_in_bytes = ALIGN_TO_ALLOCATION_ALIGNMENT(size_in_bytes);
AllocationResult result =
AllocateFastAligned(size_in_bytes, nullptr, alignment, origin);
return V8_UNLIKELY(result.IsFailure())
? AllocateRawSlowAligned(size_in_bytes, alignment, origin)
: result;
}
bool MainAllocator::IsBlackAllocationEnabled() const {
// Use the space heap to check whether black allocation is enabled. For shared
// space this might be different from the LocalHeap's heap.
return identity() != NEW_SPACE && !in_gc() &&
space_heap()->incremental_marking()->black_allocation();
}
void MainAllocator::AddAllocationObserver(AllocationObserver* observer) {
// Adding an allocation observer may decrease the inline allocation limit, so
// we check here that we don't have an existing LAB.
CHECK(!allocation_counter().IsStepInProgress());
DCHECK(!IsLabValid());
allocation_counter().AddAllocationObserver(observer);
}
void MainAllocator::RemoveAllocationObserver(AllocationObserver* observer) {
// AllocationObserver can remove themselves. So we can't CHECK here that no
// allocation step is in progress. It is also okay if there are existing LABs
// because removing an allocation observer can only increase the distance to
// the next step.
allocation_counter().RemoveAllocationObserver(observer);
}
void MainAllocator::PauseAllocationObservers() { DCHECK(!IsLabValid()); }
void MainAllocator::ResumeAllocationObservers() { DCHECK(!IsLabValid()); }
void MainAllocator::AdvanceAllocationObservers() {
if (SupportsAllocationObserver() && allocation_info().top() &&
allocation_info().start() != allocation_info().top()) {
if (isolate_heap()->IsAllocationObserverActive()) {
allocation_counter().AdvanceAllocationObservers(
allocation_info().top() - allocation_info().start());
}
MarkLabStartInitialized();
}
}
void MainAllocator::MarkLabStartInitialized() {
allocation_info().ResetStart();
#if DEBUG
Verify();
#endif
}
// Perform an allocation step when the step is reached. size_in_bytes is the
// actual size needed for the object (required for InvokeAllocationObservers).
// aligned_size_in_bytes is the size of the object including the filler right
// before it to reach the right alignment (required to DCHECK the start of the
// object). allocation_size is the size of the actual allocation which needs to
// be used for the accounting. It can be different from aligned_size_in_bytes in
// PagedSpace::AllocateRawAligned, where we have to overallocate in order to be
// able to align the allocation afterwards.
void MainAllocator::InvokeAllocationObservers(Address soon_object,
size_t size_in_bytes,
size_t aligned_size_in_bytes,
size_t allocation_size) {
DCHECK_LE(size_in_bytes, aligned_size_in_bytes);
DCHECK_LE(aligned_size_in_bytes, allocation_size);
DCHECK(size_in_bytes == aligned_size_in_bytes ||
aligned_size_in_bytes == allocation_size);
if (!SupportsAllocationObserver() ||
!isolate_heap()->IsAllocationObserverActive()) {
return;
}
if (allocation_size >= allocation_counter().NextBytes()) {
// Only the first object in a LAB should reach the next step.
DCHECK_EQ(soon_object, allocation_info().start() + aligned_size_in_bytes -
size_in_bytes);
// Right now the LAB only contains that one object.
DCHECK_EQ(allocation_info().top() + allocation_size - aligned_size_in_bytes,
allocation_info().limit());
// Ensure that there is a valid object
space_heap()->CreateFillerObjectAt(soon_object,
static_cast<int>(size_in_bytes));
#if DEBUG
// Ensure that allocation_info_ isn't modified during one of the
// AllocationObserver::Step methods.
LinearAllocationArea saved_allocation_info = allocation_info();
#endif
// Run AllocationObserver::Step through the AllocationCounter.
allocation_counter().InvokeAllocationObservers(soon_object, size_in_bytes,
allocation_size);
// Ensure that start/top/limit didn't change.
DCHECK_EQ(saved_allocation_info.start(), allocation_info().start());
DCHECK_EQ(saved_allocation_info.top(), allocation_info().top());
DCHECK_EQ(saved_allocation_info.limit(), allocation_info().limit());
}
DCHECK_LT(allocation_info().limit() - allocation_info().start(),
allocation_counter().NextBytes());
}
AllocationResult MainAllocator::AllocateRawSlow(int size_in_bytes,
AllocationAlignment alignment,
AllocationOrigin origin) {
// We are not supposed to allocate in fast c calls.
CHECK_IMPLIES(is_main_thread(),
v8_flags.allow_allocation_in_fast_api_call ||
!isolate_heap()->isolate()->InFastCCall());
AllocationResult result =
USE_ALLOCATION_ALIGNMENT_BOOL && alignment != kTaggedAligned
? AllocateRawSlowAligned(size_in_bytes, alignment, origin)
: AllocateRawSlowUnaligned(size_in_bytes, origin);
return result;
}
AllocationResult MainAllocator::AllocateRawSlowUnaligned(
int size_in_bytes, AllocationOrigin origin) {
DCHECK(!v8_flags.enable_third_party_heap);
if (!EnsureAllocation(size_in_bytes, kTaggedAligned, origin)) {
return AllocationResult::Failure();
}
AllocationResult result = AllocateFastUnaligned(size_in_bytes, origin);
DCHECK(!result.IsFailure());
InvokeAllocationObservers(result.ToAddress(), size_in_bytes, size_in_bytes,
size_in_bytes);
return result;
}
AllocationResult MainAllocator::AllocateRawSlowAligned(
int size_in_bytes, AllocationAlignment alignment, AllocationOrigin origin) {
DCHECK(!v8_flags.enable_third_party_heap);
if (!EnsureAllocation(size_in_bytes, alignment, origin)) {
return AllocationResult::Failure();
}
int max_aligned_size = size_in_bytes + Heap::GetMaximumFillToAlign(alignment);
int aligned_size_in_bytes;
AllocationResult result = AllocateFastAligned(
size_in_bytes, &aligned_size_in_bytes, alignment, origin);
DCHECK_GE(max_aligned_size, aligned_size_in_bytes);
DCHECK(!result.IsFailure());
InvokeAllocationObservers(result.ToAddress(), size_in_bytes,
aligned_size_in_bytes, max_aligned_size);
return result;
}
void MainAllocator::MakeLinearAllocationAreaIterable() {
if (!IsLabValid()) return;
#if DEBUG
Verify();
#endif // DEBUG
Address current_top = top();
Address current_limit = limit();
if (current_top != current_limit) {
space_heap()->CreateFillerObjectAt(
current_top, static_cast<int>(current_limit - current_top));
}
}
void MainAllocator::MarkLinearAllocationAreaBlack() {
DCHECK(IsBlackAllocationEnabled());
Address current_top = top();
Address current_limit = limit();
if (current_top != kNullAddress && current_top != current_limit) {
Page::FromAllocationAreaAddress(current_top)
->CreateBlackArea(current_top, current_limit);
}
}
void MainAllocator::UnmarkLinearAllocationArea() {
Address current_top = top();
Address current_limit = limit();
if (current_top != kNullAddress && current_top != current_limit) {
Page::FromAllocationAreaAddress(current_top)
->DestroyBlackArea(current_top, current_limit);
}
}
void MainAllocator::MoveOriginalTopForward() {
DCHECK(SupportsPendingAllocation());
base::SharedMutexGuard<base::kExclusive> guard(
linear_area_original_data().linear_area_lock());
DCHECK_GE(top(), linear_area_original_data().get_original_top_acquire());
DCHECK_LE(top(), linear_area_original_data().get_original_limit_relaxed());
linear_area_original_data().set_original_top_release(top());
}
void MainAllocator::ResetLab(Address start, Address end, Address extended_end) {
DCHECK_LE(start, end);
DCHECK_LE(end, extended_end);
if (IsLabValid()) {
BasicMemoryChunk::UpdateHighWaterMark(top());
}
allocation_info().Reset(start, end);
if (SupportsPendingAllocation()) {
base::SharedMutexGuard<base::kExclusive> guard(
linear_area_original_data().linear_area_lock());
linear_area_original_data().set_original_limit_relaxed(extended_end);
linear_area_original_data().set_original_top_release(start);
}
}
bool MainAllocator::IsPendingAllocation(Address object_address) {
DCHECK(SupportsPendingAllocation());
base::SharedMutexGuard<base::kShared> guard(
linear_area_original_data().linear_area_lock());
Address top = original_top_acquire();
Address limit = original_limit_relaxed();
DCHECK_LE(top, limit);
return top && top <= object_address && object_address < limit;
}
bool MainAllocator::EnsureAllocation(int size_in_bytes,
AllocationAlignment alignment,
AllocationOrigin origin) {
#ifdef V8_RUNTIME_CALL_STATS
base::Optional<RuntimeCallTimerScope> rcs_scope;
if (is_main_thread()) {
rcs_scope.emplace(isolate_heap()->isolate(),
RuntimeCallCounterId::kGC_Custom_SlowAllocateRaw);
}
#endif // V8_RUNTIME_CALL_STATS
base::Optional<VMState<GC>> vmstate;
if (is_main_thread()) {
vmstate.emplace(isolate_heap()->isolate());
}
return allocator_policy_->EnsureAllocation(size_in_bytes, alignment, origin);
}
void MainAllocator::FreeLinearAllocationArea() {
if (!IsLabValid()) return;
#if DEBUG
Verify();
#endif // DEBUG
base::Optional<CodePageHeaderModificationScope> optional_scope;
if (identity() == CODE_SPACE) {
optional_scope.emplace(
"FreeLinearAllocationArea writes to the page header.");
}
BasicMemoryChunk::UpdateHighWaterMark(top());
allocator_policy_->FreeLinearAllocationArea();
}
void MainAllocator::ExtendLAB(Address limit) {
DCHECK(supports_extending_lab());
DCHECK_LE(limit, original_limit_relaxed());
allocation_info().SetLimit(limit);
}
Address MainAllocator::ComputeLimit(Address start, Address end,
size_t min_size) const {
DCHECK_GE(end - start, min_size);
// Use the full LAB when allocation observers aren't enabled.
if (!SupportsAllocationObserver()) return end;
// LABs with allocation observers are only used outside GC and on the main
// thread.
DCHECK(!isolate_heap()->IsInGC());
DCHECK(is_main_thread());
if (!isolate_heap()->IsInlineAllocationEnabled()) {
// LABs are disabled, so we fit the requested area exactly.
return start + min_size;
}
// When LABs are enabled, pick the largest possible LAB size by default.
size_t step_size = end - start;
if (isolate_heap()->IsAllocationObserverActive()) {
// Ensure there are no unaccounted allocations.
DCHECK_EQ(allocation_info().start(), allocation_info().top());
size_t step = allocation_counter().NextBytes();
DCHECK_NE(step, 0);
// Generated code may allocate inline from the linear allocation area. To
// make sure we can observe these allocations, we use a lower limit.
size_t rounded_step = static_cast<size_t>(
RoundDown(static_cast<int>(step - 1), ObjectAlignment()));
step_size = std::min(step_size, rounded_step);
}
if (v8_flags.stress_marking) {
step_size = std::min(step_size, static_cast<size_t>(64));
}
DCHECK_LE(start + step_size, end);
return start + std::max(step_size, min_size);
}
#if DEBUG
void MainAllocator::Verify() const {
// Ensure validity of LAB: start <= top.
DCHECK_LE(allocation_info().start(), allocation_info().top());
if (top()) {
Page* page = Page::FromAllocationAreaAddress(top());
// Can't compare owner directly because of new space semi spaces.
DCHECK_EQ(page->owner_identity(), identity());
}
if (SupportsPendingAllocation()) {
// Ensure that original_top <= top <= limit <= original_limit.
DCHECK_LE(linear_area_original_data().get_original_top_acquire(),
allocation_info().top());
DCHECK_LE(allocation_info().top(), allocation_info().limit());
DCHECK_LE(allocation_info().limit(),
linear_area_original_data().get_original_limit_relaxed());
} else {
DCHECK_LE(allocation_info().top(), allocation_info().limit());
}
}
#endif // DEBUG
bool MainAllocator::EnsureAllocationForTesting(int size_in_bytes,
AllocationAlignment alignment,
AllocationOrigin origin) {
base::Optional<CodePageHeaderModificationScope> optional_scope;
if (identity() == CODE_SPACE) {
optional_scope.emplace("Slow allocation path writes to the page header.");
}
return EnsureAllocation(size_in_bytes, alignment, origin);
}
int MainAllocator::ObjectAlignment() const {
if (identity() == CODE_SPACE) {
return kCodeAlignment;
} else if (V8_COMPRESS_POINTERS_8GB_BOOL) {
return kObjectAlignment8GbHeap;
} else {
return kTaggedSize;
}
}
AllocationSpace MainAllocator::identity() const { return space_->identity(); }
bool MainAllocator::is_main_thread() const {
return !in_gc() && local_heap()->is_main_thread();
}
bool MainAllocator::in_gc_for_space() const {
return in_gc() && isolate_heap() == space_heap();
}
Heap* MainAllocator::space_heap() const { return space_->heap(); }
AllocatorPolicy::AllocatorPolicy(MainAllocator* allocator)
: allocator_(allocator) {}
Heap* AllocatorPolicy::space_heap() const { return allocator_->space_heap(); }
Heap* AllocatorPolicy::isolate_heap() const {
return allocator_->isolate_heap();
}
bool SemiSpaceNewSpaceAllocatorPolicy::EnsureAllocation(
int size_in_bytes, AllocationAlignment alignment, AllocationOrigin origin) {
base::Optional<base::MutexGuard> guard;
if (allocator_->in_gc()) guard.emplace(space_->mutex());
FreeLinearAllocationAreaUnsynchronized();
base::Optional<std::pair<Address, Address>> allocation_result =
space_->Allocate(size_in_bytes, alignment);
if (!allocation_result) return false;
Address start = allocation_result->first;
Address end = allocation_result->second;
int filler_size = Heap::GetFillToAlign(start, alignment);
int aligned_size_in_bytes = size_in_bytes + filler_size;
DCHECK_LE(start + aligned_size_in_bytes, end);
Address limit;
if (allocator_->in_gc()) {
// During GC we allow multiple LABs in new space and since Allocate() above
// returns the whole remaining page by default, we limit the size of the LAB
// here.
size_t used = std::max(aligned_size_in_bytes, kLabSizeInGC);
limit = std::min(end, start + used);
} else {
limit = allocator_->ComputeLimit(start, end, aligned_size_in_bytes);
}
CHECK_LE(limit, end);
if (limit != end) {
space_->Free(limit, end);
}
allocator_->ResetLab(start, limit, limit);
space_->to_space().AddRangeToActiveSystemPages(allocator_->top(),
allocator_->limit());
return true;
}
void SemiSpaceNewSpaceAllocatorPolicy::FreeLinearAllocationArea() {
if (!allocator_->IsLabValid()) return;
#if DEBUG
allocator_->Verify();
#endif // DEBUG
base::Optional<base::MutexGuard> guard;
if (allocator_->in_gc()) guard.emplace(space_->mutex());
FreeLinearAllocationAreaUnsynchronized();
}
void SemiSpaceNewSpaceAllocatorPolicy::
FreeLinearAllocationAreaUnsynchronized() {
if (!allocator_->IsLabValid()) return;
Address current_top = allocator_->top();
Address current_limit = allocator_->limit();
allocator_->AdvanceAllocationObservers();
allocator_->ResetLab(kNullAddress, kNullAddress, kNullAddress);
space_->Free(current_top, current_limit);
}
PagedNewSpaceAllocatorPolicy::PagedNewSpaceAllocatorPolicy(
PagedNewSpace* space, MainAllocator* allocator)
: AllocatorPolicy(allocator),
space_(space),
paged_space_allocator_policy_(
new PagedSpaceAllocatorPolicy(space->paged_space(), allocator)) {}
bool PagedNewSpaceAllocatorPolicy::EnsureAllocation(
int size_in_bytes, AllocationAlignment alignment, AllocationOrigin origin) {
if (space_->paged_space()->last_lab_page_) {
space_->paged_space()->last_lab_page_->DecreaseAllocatedLabSize(
allocator_->limit() - allocator_->top());
allocator_->ExtendLAB(allocator_->top());
// No need to write a filler to the remaining lab because it will either be
// reallocated if the lab can be extended or freed otherwise.
}
if (!paged_space_allocator_policy_->EnsureAllocation(size_in_bytes, alignment,
origin)) {
if (!TryAllocatePage(size_in_bytes, origin)) {
if (!WaitForSweepingForAllocation(size_in_bytes, origin)) {
return false;
}
}
}
space_->paged_space()->last_lab_page_ =
Page::FromAllocationAreaAddress(allocator_->top());
DCHECK_NOT_NULL(space_->paged_space()->last_lab_page_);
space_->paged_space()->last_lab_page_->IncreaseAllocatedLabSize(
allocator_->limit() - allocator_->top());
if (space_heap()->incremental_marking()->IsMinorMarking()) {
space_heap()->concurrent_marking()->RescheduleJobIfNeeded(
GarbageCollector::MINOR_MARK_SWEEPER);
}
return true;
}
bool PagedNewSpaceAllocatorPolicy::WaitForSweepingForAllocation(
int size_in_bytes, AllocationOrigin origin) {
// This method should be called only when there are no more pages for main
// thread to sweep.
DCHECK(space_heap()->sweeper()->IsSweepingDoneForSpace(NEW_SPACE));
if (!v8_flags.concurrent_sweeping || !space_heap()->sweeping_in_progress())
return false;
Sweeper* sweeper = space_heap()->sweeper();
if (!sweeper->AreMinorSweeperTasksRunning() &&
!sweeper->ShouldRefillFreelistForSpace(NEW_SPACE)) {
#if DEBUG
for (Page* p : *space_) {
DCHECK(p->SweepingDone());
p->ForAllFreeListCategories(
[space = space_->paged_space()](FreeListCategory* category) {
DCHECK_IMPLIES(!category->is_empty(),
category->is_linked(space->free_list()));
});
}
#endif // DEBUG
// All pages are already swept and relinked to the free list
return false;
}
// When getting here we know that any unswept new space page is currently
// being handled by a concurrent sweeping thread. Rather than try to cancel
// tasks and restart them, we wait "per page". This should be faster.
for (Page* p : *space_) {
if (!p->SweepingDone()) sweeper->WaitForPageToBeSwept(p);
}
space_->paged_space()->RefillFreeList();
DCHECK(!sweeper->ShouldRefillFreelistForSpace(NEW_SPACE));
return paged_space_allocator_policy_->TryAllocationFromFreeList(
static_cast<size_t>(size_in_bytes), origin);
}
namespace {
bool IsPagedNewSpaceAtFullCapacity(const PagedNewSpace* space) {
const auto* paged_space = space->paged_space();
if ((paged_space->UsableCapacity() < paged_space->TotalCapacity()) &&
(paged_space->TotalCapacity() - paged_space->UsableCapacity() >=
Page::kPageSize)) {
// Adding another page would exceed the target capacity of the space.
return false;
}
return true;
}
} // namespace
bool PagedNewSpaceAllocatorPolicy::TryAllocatePage(int size_in_bytes,
AllocationOrigin origin) {
if (IsPagedNewSpaceAtFullCapacity(space_) &&
!space_->heap()->ShouldExpandYoungGenerationOnSlowAllocation())
return false;
if (!space_->paged_space()->AllocatePage()) return false;
return paged_space_allocator_policy_->TryAllocationFromFreeList(size_in_bytes,
origin);
}
void PagedNewSpaceAllocatorPolicy::FreeLinearAllocationArea() {
if (!allocator_->IsLabValid()) return;
Page::FromAllocationAreaAddress(allocator_->top())
->DecreaseAllocatedLabSize(allocator_->limit() - allocator_->top());
paged_space_allocator_policy_->FreeLinearAllocationAreaUnsynchronized();
}
bool PagedSpaceAllocatorPolicy::EnsureAllocation(int size_in_bytes,
AllocationAlignment alignment,
AllocationOrigin origin) {
if (!allocator_->in_gc()) {
// Start incremental marking before the actual allocation, this allows the
// allocation function to mark the object black when incremental marking is
// running.
space_heap()->StartIncrementalMarkingIfAllocationLimitIsReached(
allocator_->local_heap(), space_heap()->GCFlagsForIncrementalMarking(),
kGCCallbackScheduleIdleGarbageCollection);
}
if (allocator_->identity() == NEW_SPACE &&
space_heap()->incremental_marking()->IsStopped()) {
DCHECK(allocator_->is_main_thread());
space_heap()->StartMinorMSIncrementalMarkingIfNeeded();
}
// We don't know exactly how much filler we need to align until space is
// allocated, so assume the worst case.
size_in_bytes += Heap::GetMaximumFillToAlign(alignment);
if (allocator_->allocation_info().top() + size_in_bytes <=
allocator_->allocation_info().limit()) {
return true;
}
return RefillLab(size_in_bytes, origin);
}
bool PagedSpaceAllocatorPolicy::RefillLab(int size_in_bytes,
AllocationOrigin origin) {
// Allocation in this space has failed.
DCHECK_GE(size_in_bytes, 0);
if (TryExtendLAB(size_in_bytes)) return true;
if (TryAllocationFromFreeList(size_in_bytes, origin)) return true;
// Sweeping is still in progress.
if (space_heap()->sweeping_in_progress()) {
// First try to refill the free-list, concurrent sweeper threads
// may have freed some objects in the meantime.
if (space_heap()->sweeper()->ShouldRefillFreelistForSpace(
allocator_->identity())) {
space_->RefillFreeList();
// Retry the free list allocation.
if (TryAllocationFromFreeList(static_cast<size_t>(size_in_bytes), origin))
return true;
}
static constexpr int kMaxPagesToSweep = 1;
if (ContributeToSweeping(kMaxPagesToSweep)) {
if (TryAllocationFromFreeList(size_in_bytes, origin)) {
return true;
}
}
}
if (space_->is_compaction_space()) {
DCHECK_NE(NEW_SPACE, allocator_->identity());
// If there is not enough memory in the compaction space left, try to steal
// a page from the corresponding "regular" page space.
PagedSpaceBase* main_space =
space_heap()->paged_space(allocator_->identity());
Page* page = main_space->RemovePageSafe(size_in_bytes);
if (page != nullptr) {
space_->AddPage(page);
if (TryAllocationFromFreeList(static_cast<size_t>(size_in_bytes), origin))
return true;
}
}
if (allocator_->identity() != NEW_SPACE &&
space_heap()->ShouldExpandOldGenerationOnSlowAllocation(
allocator_->local_heap(), origin) &&
space_heap()->CanExpandOldGeneration(space_->AreaSize())) {
if (TryExpandAndAllocate(static_cast<size_t>(size_in_bytes), origin)) {
return true;
}
}
// Try sweeping all pages.
if (ContributeToSweeping()) {
if (TryAllocationFromFreeList(size_in_bytes, origin)) {
return true;
}
}
if (allocator_->identity() != NEW_SPACE && allocator_->in_gc() &&
!space_heap()->force_oom()) {
// Avoid OOM crash in the GC in order to invoke NearHeapLimitCallback after
// GC and give it a chance to increase the heap limit.
if (TryExpandAndAllocate(size_in_bytes, origin)) {
return true;
}
}
return false;
}
bool PagedSpaceAllocatorPolicy::TryExpandAndAllocate(size_t size_in_bytes,
AllocationOrigin origin) {
// Run in a loop because concurrent threads might allocate from the new free
// list entries before this thread gets a chance.
while (space_->TryExpand(allocator_->local_heap(), origin)) {
if (TryAllocationFromFreeList(static_cast<size_t>(size_in_bytes), origin)) {
return true;
}
}
return false;
}
bool PagedSpaceAllocatorPolicy::ContributeToSweeping(uint32_t max_pages) {
if (!space_heap()->sweeping_in_progress_for_space(allocator_->identity()))
return false;
if (space_heap()->sweeper()->IsSweepingDoneForSpace(allocator_->identity()))
return false;
const bool is_main_thread =
allocator_->is_main_thread() ||
(allocator_->in_gc() && isolate_heap()->IsMainThread());
const auto sweeping_scope_kind =
is_main_thread ? ThreadKind::kMain : ThreadKind::kBackground;
const auto sweeping_scope_id = space_heap()->sweeper()->GetTracingScope(
allocator_->identity(), is_main_thread);
TRACE_GC_EPOCH_WITH_FLOW(
isolate_heap()->tracer(), sweeping_scope_id, sweeping_scope_kind,
isolate_heap()->sweeper()->GetTraceIdForFlowEvent(sweeping_scope_id),
TRACE_EVENT_FLAG_FLOW_IN | TRACE_EVENT_FLAG_FLOW_OUT);
// Cleanup invalidated old-to-new refs for compaction space in the
// final atomic pause.
Sweeper::SweepingMode sweeping_mode =
allocator_->in_gc_for_space() ? Sweeper::SweepingMode::kEagerDuringGC
: Sweeper::SweepingMode::kLazyOrConcurrent;
if (!space_heap()->sweeper()->ParallelSweepSpace(allocator_->identity(),
sweeping_mode, max_pages)) {
return false;
}
space_->RefillFreeList();
return true;
}
void PagedSpaceAllocatorPolicy::SetLinearAllocationArea(Address top,
Address limit,
Address end) {
allocator_->ResetLab(top, limit, end);
if (top != kNullAddress && top != limit) {
Page* page = Page::FromAllocationAreaAddress(top);
if (allocator_->IsBlackAllocationEnabled()) {
page->CreateBlackArea(top, limit);
}
}
}
bool PagedSpaceAllocatorPolicy::TryAllocationFromFreeList(
size_t size_in_bytes, AllocationOrigin origin) {
PagedSpace::ConcurrentAllocationMutex guard(space_);
DCHECK(IsAligned(size_in_bytes, kTaggedSize));
DCHECK_LE(allocator_->top(), allocator_->limit());
#ifdef DEBUG
if (allocator_->top() != allocator_->limit()) {
DCHECK_EQ(Page::FromAddress(allocator_->top()),
Page::FromAddress(allocator_->limit() - 1));
}
#endif
// Don't free list allocate if there is linear space available.
DCHECK_LT(static_cast<size_t>(allocator_->limit() - allocator_->top()),
size_in_bytes);
size_t new_node_size = 0;
Tagged<FreeSpace> new_node =
space_->free_list_->Allocate(size_in_bytes, &new_node_size, origin);
if (new_node.is_null()) return false;
DCHECK_GE(new_node_size, size_in_bytes);
// The old-space-step might have finished sweeping and restarted marking.
// Verify that it did not turn the page of the new node into an evacuation
// candidate.
DCHECK(!MarkCompactCollector::IsOnEvacuationCandidate(new_node));
// Mark the old linear allocation area with a free space map so it can be
// skipped when scanning the heap. This also puts it back in the free list
// if it is big enough.
FreeLinearAllocationAreaUnsynchronized();
// Memory in the linear allocation area is counted as allocated. We may free
// a little of this again immediately - see below.
Page* page = Page::FromHeapObject(new_node);
space_->IncreaseAllocatedBytes(new_node_size, page);
DCHECK_EQ(allocator_->allocation_info().start(),
allocator_->allocation_info().top());
Address start = new_node.address();
Address end = new_node.address() + new_node_size;
Address limit = allocator_->ComputeLimit(start, end, size_in_bytes);
DCHECK_LE(limit, end);
DCHECK_LE(size_in_bytes, limit - start);
if (limit != end) {
if (!allocator_->supports_extending_lab()) {
space_->Free(limit, end - limit);
end = limit;
} else {
DCHECK(allocator_->is_main_thread());
space_heap()->CreateFillerObjectAt(limit, static_cast<int>(end - limit));
}
}
SetLinearAllocationArea(start, limit, end);
space_->AddRangeToActiveSystemPages(page, start, limit);
return true;
}
bool PagedSpaceAllocatorPolicy::TryExtendLAB(int size_in_bytes) {
if (!allocator_->supports_extending_lab()) return false;
Address current_top = allocator_->top();
if (current_top == kNullAddress) return false;
Address current_limit = allocator_->limit();
Address max_limit = allocator_->original_limit_relaxed();
if (current_top + size_in_bytes > max_limit) {
return false;
}
allocator_->AdvanceAllocationObservers();
Address new_limit =
allocator_->ComputeLimit(current_top, max_limit, size_in_bytes);
allocator_->ExtendLAB(new_limit);
DCHECK(allocator_->is_main_thread());
space_heap()->CreateFillerObjectAt(new_limit,
static_cast<int>(max_limit - new_limit));
Page* page = Page::FromAddress(current_top);
// No need to create a black allocation area since new space doesn't use
// black allocation.
DCHECK_EQ(NEW_SPACE, allocator_->identity());
space_->AddRangeToActiveSystemPages(page, current_limit, new_limit);
return true;
}
void PagedSpaceAllocatorPolicy::FreeLinearAllocationArea() {
if (!allocator_->IsLabValid()) return;
base::MutexGuard guard(space_->mutex());
FreeLinearAllocationAreaUnsynchronized();
}
void PagedSpaceAllocatorPolicy::FreeLinearAllocationAreaUnsynchronized() {
if (!allocator_->IsLabValid()) return;
#if DEBUG
allocator_->Verify();
#endif // DEBUG
Address current_top = allocator_->top();
Address current_limit = allocator_->limit();
Address current_max_limit = allocator_->supports_extending_lab()
? allocator_->original_limit_relaxed()
: current_limit;
DCHECK_IMPLIES(!allocator_->supports_extending_lab(),
current_max_limit == current_limit);
allocator_->AdvanceAllocationObservers();
if (current_top != current_limit && allocator_->IsBlackAllocationEnabled()) {
Page::FromAddress(current_top)
->DestroyBlackArea(current_top, current_limit);
}
allocator_->ResetLab(kNullAddress, kNullAddress, kNullAddress);
DCHECK_GE(current_limit, current_top);
DCHECK_IMPLIES(current_limit - current_top >= 2 * kTaggedSize,
space_heap()->marking_state()->IsUnmarked(
HeapObject::FromAddress(current_top)));
space_->Free(current_top, current_max_limit - current_top);
}
} // namespace internal
} // namespace v8