blob: 9ac3f64242b6534f47bb9ff266fa357ca69d6491 [file] [log] [blame]
// Copyright 2016 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "src/compiler/memory-optimizer.h"
#include "src/codegen/interface-descriptors.h"
#include "src/compiler/js-graph.h"
#include "src/compiler/linkage.h"
#include "src/compiler/node-matchers.h"
#include "src/compiler/node-properties.h"
#include "src/compiler/node.h"
#include "src/compiler/simplified-operator.h"
#include "src/roots/roots-inl.h"
namespace v8 {
namespace internal {
namespace compiler {
MemoryOptimizer::MemoryOptimizer(JSGraph* jsgraph, Zone* zone,
PoisoningMitigationLevel poisoning_level,
AllocationFolding allocation_folding,
const char* function_debug_name)
: jsgraph_(jsgraph),
empty_state_(AllocationState::Empty(zone)),
pending_(zone),
tokens_(zone),
zone_(zone),
graph_assembler_(jsgraph, nullptr, nullptr, zone),
poisoning_level_(poisoning_level),
allocation_folding_(allocation_folding),
function_debug_name_(function_debug_name) {}
void MemoryOptimizer::Optimize() {
EnqueueUses(graph()->start(), empty_state());
while (!tokens_.empty()) {
Token const token = tokens_.front();
tokens_.pop();
VisitNode(token.node, token.state);
}
DCHECK(pending_.empty());
DCHECK(tokens_.empty());
}
MemoryOptimizer::AllocationGroup::AllocationGroup(Node* node,
AllocationType allocation,
Zone* zone)
: node_ids_(zone), allocation_(allocation), size_(nullptr) {
node_ids_.insert(node->id());
}
MemoryOptimizer::AllocationGroup::AllocationGroup(Node* node,
AllocationType allocation,
Node* size, Zone* zone)
: node_ids_(zone), allocation_(allocation), size_(size) {
node_ids_.insert(node->id());
}
void MemoryOptimizer::AllocationGroup::Add(Node* node) {
node_ids_.insert(node->id());
}
bool MemoryOptimizer::AllocationGroup::Contains(Node* node) const {
// Additions should stay within the same allocated object, so it's safe to
// ignore them.
while (node_ids_.find(node->id()) == node_ids_.end()) {
switch (node->opcode()) {
case IrOpcode::kBitcastTaggedToWord:
case IrOpcode::kBitcastWordToTagged:
case IrOpcode::kInt32Add:
case IrOpcode::kInt64Add:
node = NodeProperties::GetValueInput(node, 0);
break;
default:
return false;
}
}
return true;
}
MemoryOptimizer::AllocationState::AllocationState()
: group_(nullptr), size_(std::numeric_limits<int>::max()), top_(nullptr) {}
MemoryOptimizer::AllocationState::AllocationState(AllocationGroup* group)
: group_(group), size_(std::numeric_limits<int>::max()), top_(nullptr) {}
MemoryOptimizer::AllocationState::AllocationState(AllocationGroup* group,
intptr_t size, Node* top)
: group_(group), size_(size), top_(top) {}
bool MemoryOptimizer::AllocationState::IsYoungGenerationAllocation() const {
return group() && group()->IsYoungGenerationAllocation();
}
namespace {
bool CanAllocate(const Node* node) {
switch (node->opcode()) {
case IrOpcode::kBitcastTaggedToWord:
case IrOpcode::kBitcastWordToTagged:
case IrOpcode::kComment:
case IrOpcode::kDebugAbort:
case IrOpcode::kDebugBreak:
case IrOpcode::kDeoptimizeIf:
case IrOpcode::kDeoptimizeUnless:
case IrOpcode::kEffectPhi:
case IrOpcode::kIfException:
case IrOpcode::kLoad:
case IrOpcode::kLoadElement:
case IrOpcode::kLoadField:
case IrOpcode::kPoisonedLoad:
case IrOpcode::kProtectedLoad:
case IrOpcode::kProtectedStore:
case IrOpcode::kRetain:
// TODO(tebbi): Store nodes might do a bump-pointer allocation.
// We should introduce a special bump-pointer store node to
// differentiate that.
case IrOpcode::kStore:
case IrOpcode::kStoreElement:
case IrOpcode::kStoreField:
case IrOpcode::kTaggedPoisonOnSpeculation:
case IrOpcode::kUnalignedLoad:
case IrOpcode::kUnalignedStore:
case IrOpcode::kUnsafePointerAdd:
case IrOpcode::kUnreachable:
case IrOpcode::kStaticAssert:
case IrOpcode::kWord32AtomicAdd:
case IrOpcode::kWord32AtomicAnd:
case IrOpcode::kWord32AtomicCompareExchange:
case IrOpcode::kWord32AtomicExchange:
case IrOpcode::kWord32AtomicLoad:
case IrOpcode::kWord32AtomicOr:
case IrOpcode::kWord32AtomicPairAdd:
case IrOpcode::kWord32AtomicPairAnd:
case IrOpcode::kWord32AtomicPairCompareExchange:
case IrOpcode::kWord32AtomicPairExchange:
case IrOpcode::kWord32AtomicPairLoad:
case IrOpcode::kWord32AtomicPairOr:
case IrOpcode::kWord32AtomicPairStore:
case IrOpcode::kWord32AtomicPairSub:
case IrOpcode::kWord32AtomicPairXor:
case IrOpcode::kWord32AtomicStore:
case IrOpcode::kWord32AtomicSub:
case IrOpcode::kWord32AtomicXor:
case IrOpcode::kWord32PoisonOnSpeculation:
case IrOpcode::kWord64AtomicAdd:
case IrOpcode::kWord64AtomicAnd:
case IrOpcode::kWord64AtomicCompareExchange:
case IrOpcode::kWord64AtomicExchange:
case IrOpcode::kWord64AtomicLoad:
case IrOpcode::kWord64AtomicOr:
case IrOpcode::kWord64AtomicStore:
case IrOpcode::kWord64AtomicSub:
case IrOpcode::kWord64AtomicXor:
case IrOpcode::kWord64PoisonOnSpeculation:
return false;
case IrOpcode::kCall:
case IrOpcode::kCallWithCallerSavedRegisters:
return !(CallDescriptorOf(node->op())->flags() &
CallDescriptor::kNoAllocate);
default:
break;
}
return true;
}
Node* SearchAllocatingNode(Node* start, Node* limit, Zone* temp_zone) {
ZoneQueue<Node*> queue(temp_zone);
ZoneSet<Node*> visited(temp_zone);
visited.insert(limit);
queue.push(start);
while (!queue.empty()) {
Node* const current = queue.front();
queue.pop();
if (visited.find(current) == visited.end()) {
visited.insert(current);
if (CanAllocate(current)) {
return current;
}
for (int i = 0; i < current->op()->EffectInputCount(); ++i) {
queue.push(NodeProperties::GetEffectInput(current, i));
}
}
}
return nullptr;
}
bool CanLoopAllocate(Node* loop_effect_phi, Zone* temp_zone) {
Node* const control = NodeProperties::GetControlInput(loop_effect_phi);
// Start the effect chain walk from the loop back edges.
for (int i = 1; i < control->InputCount(); ++i) {
if (SearchAllocatingNode(loop_effect_phi->InputAt(i), loop_effect_phi,
temp_zone) != nullptr) {
return true;
}
}
return false;
}
Node* EffectPhiForPhi(Node* phi) {
Node* control = NodeProperties::GetControlInput(phi);
for (Node* use : control->uses()) {
if (use->opcode() == IrOpcode::kEffectPhi) {
return use;
}
}
return nullptr;
}
} // namespace
void MemoryOptimizer::VisitNode(Node* node, AllocationState const* state) {
DCHECK(!node->IsDead());
DCHECK_LT(0, node->op()->EffectInputCount());
switch (node->opcode()) {
case IrOpcode::kAllocate:
// Allocate nodes were purged from the graph in effect-control
// linearization.
UNREACHABLE();
case IrOpcode::kAllocateRaw:
return VisitAllocateRaw(node, state);
case IrOpcode::kCall:
return VisitCall(node, state);
case IrOpcode::kCallWithCallerSavedRegisters:
return VisitCallWithCallerSavedRegisters(node, state);
case IrOpcode::kLoadElement:
return VisitLoadElement(node, state);
case IrOpcode::kLoadField:
return VisitLoadField(node, state);
case IrOpcode::kStoreElement:
return VisitStoreElement(node, state);
case IrOpcode::kStoreField:
return VisitStoreField(node, state);
case IrOpcode::kStore:
return VisitStore(node, state);
default:
if (!CanAllocate(node)) {
// These operations cannot trigger GC.
return VisitOtherEffect(node, state);
}
}
DCHECK_EQ(0, node->op()->EffectOutputCount());
}
#define __ gasm()->
void MemoryOptimizer::VisitAllocateRaw(Node* node,
AllocationState const* state) {
DCHECK_EQ(IrOpcode::kAllocateRaw, node->opcode());
Node* value;
Node* size = node->InputAt(0);
Node* effect = node->InputAt(1);
Node* control = node->InputAt(2);
gasm()->Reset(effect, control);
const AllocateParameters& allocation = AllocateParametersOf(node->op());
AllocationType allocation_type = allocation.allocation_type();
// Propagate tenuring from outer allocations to inner allocations, i.e.
// when we allocate an object in old space and store a newly allocated
// child object into the pretenured object, then the newly allocated
// child object also should get pretenured to old space.
if (allocation_type == AllocationType::kOld) {
for (Edge const edge : node->use_edges()) {
Node* const user = edge.from();
if (user->opcode() == IrOpcode::kStoreField && edge.index() == 0) {
Node* const child = user->InputAt(1);
if (child->opcode() == IrOpcode::kAllocateRaw &&
AllocationTypeOf(child->op()) == AllocationType::kYoung) {
NodeProperties::ChangeOp(child, node->op());
break;
}
}
}
} else {
DCHECK_EQ(AllocationType::kYoung, allocation_type);
for (Edge const edge : node->use_edges()) {
Node* const user = edge.from();
if (user->opcode() == IrOpcode::kStoreField && edge.index() == 1) {
Node* const parent = user->InputAt(0);
if (parent->opcode() == IrOpcode::kAllocateRaw &&
AllocationTypeOf(parent->op()) == AllocationType::kOld) {
allocation_type = AllocationType::kOld;
break;
}
}
}
}
// Determine the top/limit addresses.
Node* top_address = __ ExternalConstant(
allocation_type == AllocationType::kYoung
? ExternalReference::new_space_allocation_top_address(isolate())
: ExternalReference::old_space_allocation_top_address(isolate()));
Node* limit_address = __ ExternalConstant(
allocation_type == AllocationType::kYoung
? ExternalReference::new_space_allocation_limit_address(isolate())
: ExternalReference::old_space_allocation_limit_address(isolate()));
// Check if we can fold this allocation into a previous allocation represented
// by the incoming {state}.
IntPtrMatcher m(size);
if (m.IsInRange(0, kMaxRegularHeapObjectSize)) {
intptr_t const object_size = m.Value();
if (allocation_folding_ == AllocationFolding::kDoAllocationFolding &&
state->size() <= kMaxRegularHeapObjectSize - object_size &&
state->group()->allocation() == allocation_type) {
// We can fold this Allocate {node} into the allocation {group}
// represented by the given {state}. Compute the upper bound for
// the new {state}.
intptr_t const state_size = state->size() + object_size;
// Update the reservation check to the actual maximum upper bound.
AllocationGroup* const group = state->group();
if (machine()->Is64()) {
if (OpParameter<int64_t>(group->size()->op()) < state_size) {
NodeProperties::ChangeOp(group->size(),
common()->Int64Constant(state_size));
}
} else {
if (OpParameter<int32_t>(group->size()->op()) < state_size) {
NodeProperties::ChangeOp(
group->size(),
common()->Int32Constant(static_cast<int32_t>(state_size)));
}
}
// Update the allocation top with the new object allocation.
// TODO(bmeurer): Defer writing back top as much as possible.
Node* top = __ IntAdd(state->top(), size);
__ Store(StoreRepresentation(MachineType::PointerRepresentation(),
kNoWriteBarrier),
top_address, __ IntPtrConstant(0), top);
// Compute the effective inner allocated address.
value = __ BitcastWordToTagged(
__ IntAdd(state->top(), __ IntPtrConstant(kHeapObjectTag)));
// Extend the allocation {group}.
group->Add(value);
state = AllocationState::Open(group, state_size, top, zone());
} else {
auto call_runtime = __ MakeDeferredLabel();
auto done = __ MakeLabel(MachineType::PointerRepresentation());
// Setup a mutable reservation size node; will be patched as we fold
// additional allocations into this new group.
Node* size = __ UniqueIntPtrConstant(object_size);
// Load allocation top and limit.
Node* top =
__ Load(MachineType::Pointer(), top_address, __ IntPtrConstant(0));
Node* limit =
__ Load(MachineType::Pointer(), limit_address, __ IntPtrConstant(0));
// Check if we need to collect garbage before we can start bump pointer
// allocation (always done for folded allocations).
Node* check = __ UintLessThan(__ IntAdd(top, size), limit);
__ GotoIfNot(check, &call_runtime);
__ Goto(&done, top);
__ Bind(&call_runtime);
{
Node* target = allocation_type == AllocationType::kYoung
? __
AllocateInYoungGenerationStubConstant()
: __
AllocateInOldGenerationStubConstant();
if (!allocate_operator_.is_set()) {
auto descriptor = AllocateDescriptor{};
auto call_descriptor = Linkage::GetStubCallDescriptor(
graph()->zone(), descriptor, descriptor.GetStackParameterCount(),
CallDescriptor::kCanUseRoots, Operator::kNoThrow);
allocate_operator_.set(common()->Call(call_descriptor));
}
Node* vfalse = __ BitcastTaggedToWord(
__ Call(allocate_operator_.get(), target, size));
vfalse = __ IntSub(vfalse, __ IntPtrConstant(kHeapObjectTag));
__ Goto(&done, vfalse);
}
__ Bind(&done);
// Compute the new top and write it back.
top = __ IntAdd(done.PhiAt(0), __ IntPtrConstant(object_size));
__ Store(StoreRepresentation(MachineType::PointerRepresentation(),
kNoWriteBarrier),
top_address, __ IntPtrConstant(0), top);
// Compute the initial object address.
value = __ BitcastWordToTagged(
__ IntAdd(done.PhiAt(0), __ IntPtrConstant(kHeapObjectTag)));
// Start a new allocation group.
AllocationGroup* group =
new (zone()) AllocationGroup(value, allocation_type, size, zone());
state = AllocationState::Open(group, object_size, top, zone());
}
} else {
auto call_runtime = __ MakeDeferredLabel();
auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer);
// Load allocation top and limit.
Node* top =
__ Load(MachineType::Pointer(), top_address, __ IntPtrConstant(0));
Node* limit =
__ Load(MachineType::Pointer(), limit_address, __ IntPtrConstant(0));
// Compute the new top.
Node* new_top = __ IntAdd(top, size);
// Check if we can do bump pointer allocation here.
Node* check = __ UintLessThan(new_top, limit);
__ GotoIfNot(check, &call_runtime);
if (allocation.allow_large_objects() == AllowLargeObjects::kTrue) {
__ GotoIfNot(
__ UintLessThan(size, __ IntPtrConstant(kMaxRegularHeapObjectSize)),
&call_runtime);
}
__ Store(StoreRepresentation(MachineType::PointerRepresentation(),
kNoWriteBarrier),
top_address, __ IntPtrConstant(0), new_top);
__ Goto(&done, __ BitcastWordToTagged(
__ IntAdd(top, __ IntPtrConstant(kHeapObjectTag))));
__ Bind(&call_runtime);
Node* target = allocation_type == AllocationType::kYoung
? __
AllocateInYoungGenerationStubConstant()
: __
AllocateInOldGenerationStubConstant();
if (!allocate_operator_.is_set()) {
auto descriptor = AllocateDescriptor{};
auto call_descriptor = Linkage::GetStubCallDescriptor(
graph()->zone(), descriptor, descriptor.GetStackParameterCount(),
CallDescriptor::kCanUseRoots, Operator::kNoThrow);
allocate_operator_.set(common()->Call(call_descriptor));
}
__ Goto(&done, __ Call(allocate_operator_.get(), target, size));
__ Bind(&done);
value = done.PhiAt(0);
// Create an unfoldable allocation group.
AllocationGroup* group =
new (zone()) AllocationGroup(value, allocation_type, zone());
state = AllocationState::Closed(group, zone());
}
effect = __ ExtractCurrentEffect();
control = __ ExtractCurrentControl();
// Replace all effect uses of {node} with the {effect}, enqueue the
// effect uses for further processing, and replace all value uses of
// {node} with the {value}.
for (Edge edge : node->use_edges()) {
if (NodeProperties::IsEffectEdge(edge)) {
EnqueueUse(edge.from(), edge.index(), state);
edge.UpdateTo(effect);
} else if (NodeProperties::IsValueEdge(edge)) {
edge.UpdateTo(value);
} else {
DCHECK(NodeProperties::IsControlEdge(edge));
edge.UpdateTo(control);
}
}
// Kill the {node} to make sure we don't leave dangling dead uses.
node->Kill();
}
#undef __
void MemoryOptimizer::VisitCall(Node* node, AllocationState const* state) {
DCHECK_EQ(IrOpcode::kCall, node->opcode());
// If the call can allocate, we start with a fresh state.
if (!(CallDescriptorOf(node->op())->flags() & CallDescriptor::kNoAllocate)) {
state = empty_state();
}
EnqueueUses(node, state);
}
void MemoryOptimizer::VisitCallWithCallerSavedRegisters(
Node* node, AllocationState const* state) {
DCHECK_EQ(IrOpcode::kCallWithCallerSavedRegisters, node->opcode());
// If the call can allocate, we start with a fresh state.
if (!(CallDescriptorOf(node->op())->flags() & CallDescriptor::kNoAllocate)) {
state = empty_state();
}
EnqueueUses(node, state);
}
void MemoryOptimizer::VisitLoadElement(Node* node,
AllocationState const* state) {
DCHECK_EQ(IrOpcode::kLoadElement, node->opcode());
ElementAccess const& access = ElementAccessOf(node->op());
Node* index = node->InputAt(1);
node->ReplaceInput(1, ComputeIndex(access, index));
MachineType type = access.machine_type;
if (NeedsPoisoning(access.load_sensitivity) &&
type.representation() != MachineRepresentation::kTaggedPointer &&
type.representation() != MachineRepresentation::kCompressedPointer) {
NodeProperties::ChangeOp(node, machine()->PoisonedLoad(type));
} else {
NodeProperties::ChangeOp(node, machine()->Load(type));
}
EnqueueUses(node, state);
}
void MemoryOptimizer::VisitLoadField(Node* node, AllocationState const* state) {
DCHECK_EQ(IrOpcode::kLoadField, node->opcode());
FieldAccess const& access = FieldAccessOf(node->op());
Node* offset = jsgraph()->IntPtrConstant(access.offset - access.tag());
node->InsertInput(graph()->zone(), 1, offset);
MachineType type = access.machine_type;
if (NeedsPoisoning(access.load_sensitivity) &&
type.representation() != MachineRepresentation::kTaggedPointer &&
type.representation() != MachineRepresentation::kCompressedPointer) {
NodeProperties::ChangeOp(node, machine()->PoisonedLoad(type));
} else {
NodeProperties::ChangeOp(node, machine()->Load(type));
}
EnqueueUses(node, state);
}
void MemoryOptimizer::VisitStoreElement(Node* node,
AllocationState const* state) {
DCHECK_EQ(IrOpcode::kStoreElement, node->opcode());
ElementAccess const& access = ElementAccessOf(node->op());
Node* object = node->InputAt(0);
Node* index = node->InputAt(1);
Node* value = node->InputAt(2);
WriteBarrierKind write_barrier_kind = ComputeWriteBarrierKind(
node, object, value, state, access.write_barrier_kind);
node->ReplaceInput(1, ComputeIndex(access, index));
NodeProperties::ChangeOp(
node, machine()->Store(StoreRepresentation(
access.machine_type.representation(), write_barrier_kind)));
EnqueueUses(node, state);
}
void MemoryOptimizer::VisitStoreField(Node* node,
AllocationState const* state) {
DCHECK_EQ(IrOpcode::kStoreField, node->opcode());
FieldAccess const& access = FieldAccessOf(node->op());
Node* object = node->InputAt(0);
Node* value = node->InputAt(1);
WriteBarrierKind write_barrier_kind = ComputeWriteBarrierKind(
node, object, value, state, access.write_barrier_kind);
Node* offset = jsgraph()->IntPtrConstant(access.offset - access.tag());
node->InsertInput(graph()->zone(), 1, offset);
NodeProperties::ChangeOp(
node, machine()->Store(StoreRepresentation(
access.machine_type.representation(), write_barrier_kind)));
EnqueueUses(node, state);
}
void MemoryOptimizer::VisitStore(Node* node, AllocationState const* state) {
DCHECK_EQ(IrOpcode::kStore, node->opcode());
StoreRepresentation representation = StoreRepresentationOf(node->op());
Node* object = node->InputAt(0);
Node* value = node->InputAt(2);
WriteBarrierKind write_barrier_kind = ComputeWriteBarrierKind(
node, object, value, state, representation.write_barrier_kind());
if (write_barrier_kind != representation.write_barrier_kind()) {
NodeProperties::ChangeOp(
node, machine()->Store(StoreRepresentation(
representation.representation(), write_barrier_kind)));
}
EnqueueUses(node, state);
}
void MemoryOptimizer::VisitOtherEffect(Node* node,
AllocationState const* state) {
EnqueueUses(node, state);
}
Node* MemoryOptimizer::ComputeIndex(ElementAccess const& access, Node* index) {
int const element_size_shift =
ElementSizeLog2Of(access.machine_type.representation());
if (element_size_shift) {
index = graph()->NewNode(machine()->WordShl(), index,
jsgraph()->IntPtrConstant(element_size_shift));
}
int const fixed_offset = access.header_size - access.tag();
if (fixed_offset) {
index = graph()->NewNode(machine()->IntAdd(), index,
jsgraph()->IntPtrConstant(fixed_offset));
}
return index;
}
namespace {
bool ValueNeedsWriteBarrier(Node* value, Isolate* isolate) {
while (true) {
switch (value->opcode()) {
case IrOpcode::kBitcastWordToTaggedSigned:
case IrOpcode::kChangeTaggedSignedToCompressedSigned:
case IrOpcode::kChangeTaggedToCompressedSigned:
return false;
case IrOpcode::kChangeTaggedPointerToCompressedPointer:
case IrOpcode::kChangeTaggedToCompressed:
value = NodeProperties::GetValueInput(value, 0);
continue;
case IrOpcode::kHeapConstant: {
RootIndex root_index;
if (isolate->roots_table().IsRootHandle(HeapConstantOf(value->op()),
&root_index) &&
RootsTable::IsImmortalImmovable(root_index)) {
return false;
}
break;
}
default:
break;
}
return true;
}
}
void WriteBarrierAssertFailed(Node* node, Node* object, const char* name,
Zone* temp_zone) {
std::stringstream str;
str << "MemoryOptimizer could not remove write barrier for node #"
<< node->id() << "\n";
str << " Run mksnapshot with --csa-trap-on-node=" << name << ","
<< node->id() << " to break in CSA code.\n";
Node* object_position = object;
if (object_position->opcode() == IrOpcode::kPhi) {
object_position = EffectPhiForPhi(object_position);
}
Node* allocating_node = nullptr;
if (object_position && object_position->op()->EffectOutputCount() > 0) {
allocating_node = SearchAllocatingNode(node, object_position, temp_zone);
}
if (allocating_node) {
str << "\n There is a potentially allocating node in between:\n";
str << " " << *allocating_node << "\n";
str << " Run mksnapshot with --csa-trap-on-node=" << name << ","
<< allocating_node->id() << " to break there.\n";
if (allocating_node->opcode() == IrOpcode::kCall) {
str << " If this is a never-allocating runtime call, you can add an "
"exception to Runtime::MayAllocate.\n";
}
} else {
str << "\n It seems the store happened to something different than a "
"direct "
"allocation:\n";
str << " " << *object << "\n";
str << " Run mksnapshot with --csa-trap-on-node=" << name << ","
<< object->id() << " to break there.\n";
}
FATAL("%s", str.str().c_str());
}
} // namespace
WriteBarrierKind MemoryOptimizer::ComputeWriteBarrierKind(
Node* node, Node* object, Node* value, AllocationState const* state,
WriteBarrierKind write_barrier_kind) {
if (state->IsYoungGenerationAllocation() &&
state->group()->Contains(object)) {
write_barrier_kind = kNoWriteBarrier;
}
if (!ValueNeedsWriteBarrier(value, isolate())) {
write_barrier_kind = kNoWriteBarrier;
}
if (write_barrier_kind == WriteBarrierKind::kAssertNoWriteBarrier) {
WriteBarrierAssertFailed(node, object, function_debug_name_, zone());
}
return write_barrier_kind;
}
MemoryOptimizer::AllocationState const* MemoryOptimizer::MergeStates(
AllocationStates const& states) {
// Check if all states are the same; or at least if all allocation
// states belong to the same allocation group.
AllocationState const* state = states.front();
AllocationGroup* group = state->group();
for (size_t i = 1; i < states.size(); ++i) {
if (states[i] != state) state = nullptr;
if (states[i]->group() != group) group = nullptr;
}
if (state == nullptr) {
if (group != nullptr) {
// We cannot fold any more allocations into this group, but we can still
// eliminate write barriers on stores to this group.
// TODO(bmeurer): We could potentially just create a Phi here to merge
// the various tops; but we need to pay special attention not to create
// an unschedulable graph.
state = AllocationState::Closed(group, zone());
} else {
// The states are from different allocation groups.
state = empty_state();
}
}
return state;
}
void MemoryOptimizer::EnqueueMerge(Node* node, int index,
AllocationState const* state) {
DCHECK_EQ(IrOpcode::kEffectPhi, node->opcode());
int const input_count = node->InputCount() - 1;
DCHECK_LT(0, input_count);
Node* const control = node->InputAt(input_count);
if (control->opcode() == IrOpcode::kLoop) {
if (index == 0) {
if (CanLoopAllocate(node, zone())) {
// If the loop can allocate, we start with an empty state at the
// beginning.
EnqueueUses(node, empty_state());
} else {
// If the loop cannot allocate, we can just propagate the state from
// before the loop.
EnqueueUses(node, state);
}
} else {
// Do not revisit backedges.
}
} else {
DCHECK_EQ(IrOpcode::kMerge, control->opcode());
// Check if we already know about this pending merge.
NodeId const id = node->id();
auto it = pending_.find(id);
if (it == pending_.end()) {
// Insert a new pending merge.
it = pending_.insert(std::make_pair(id, AllocationStates(zone()))).first;
}
// Add the next input state.
it->second.push_back(state);
// Check if states for all inputs are available by now.
if (it->second.size() == static_cast<size_t>(input_count)) {
// All inputs to this effect merge are done, merge the states given all
// input constraints, drop the pending merge and enqueue uses of the
// EffectPhi {node}.
state = MergeStates(it->second);
EnqueueUses(node, state);
pending_.erase(it);
}
}
}
void MemoryOptimizer::EnqueueUses(Node* node, AllocationState const* state) {
for (Edge const edge : node->use_edges()) {
if (NodeProperties::IsEffectEdge(edge)) {
EnqueueUse(edge.from(), edge.index(), state);
}
}
}
void MemoryOptimizer::EnqueueUse(Node* node, int index,
AllocationState const* state) {
if (node->opcode() == IrOpcode::kEffectPhi) {
// An EffectPhi represents a merge of different effect chains, which
// needs special handling depending on whether the merge is part of a
// loop or just a normal control join.
EnqueueMerge(node, index, state);
} else {
Token token = {node, state};
tokens_.push(token);
}
}
Graph* MemoryOptimizer::graph() const { return jsgraph()->graph(); }
Isolate* MemoryOptimizer::isolate() const { return jsgraph()->isolate(); }
CommonOperatorBuilder* MemoryOptimizer::common() const {
return jsgraph()->common();
}
MachineOperatorBuilder* MemoryOptimizer::machine() const {
return jsgraph()->machine();
}
bool MemoryOptimizer::NeedsPoisoning(LoadSensitivity load_sensitivity) const {
// Safe loads do not need poisoning.
if (load_sensitivity == LoadSensitivity::kSafe) return false;
switch (poisoning_level_) {
case PoisoningMitigationLevel::kDontPoison:
return false;
case PoisoningMitigationLevel::kPoisonAll:
return true;
case PoisoningMitigationLevel::kPoisonCriticalOnly:
return load_sensitivity == LoadSensitivity::kCritical;
}
UNREACHABLE();
}
} // namespace compiler
} // namespace internal
} // namespace v8