blob: c58fe5e8fd627621deaac8124cfb3e0da309aaa2 [file] [log] [blame]
// Copyright 2013 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#if V8_TARGET_ARCH_ARM64
#include "src/base/bits.h"
#include "src/base/division-by-constant.h"
#include "src/bootstrapper.h"
#include "src/codegen.h"
#include "src/debug/debug.h"
#include "src/register-configuration.h"
#include "src/runtime/runtime.h"
#include "src/arm64/frames-arm64.h"
#include "src/arm64/macro-assembler-arm64.h"
namespace v8 {
namespace internal {
// Define a fake double underscore to use with the ASM_UNIMPLEMENTED macros.
#define __
MacroAssembler::MacroAssembler(Isolate* arg_isolate, byte* buffer,
unsigned buffer_size,
CodeObjectRequired create_code_object)
: Assembler(arg_isolate, buffer, buffer_size),
generating_stub_(false),
#if DEBUG
allow_macro_instructions_(true),
#endif
has_frame_(false),
use_real_aborts_(true),
sp_(jssp),
tmp_list_(DefaultTmpList()),
fptmp_list_(DefaultFPTmpList()) {
if (create_code_object == CodeObjectRequired::kYes) {
code_object_ =
Handle<Object>::New(isolate()->heap()->undefined_value(), isolate());
}
}
CPURegList MacroAssembler::DefaultTmpList() {
return CPURegList(ip0, ip1);
}
CPURegList MacroAssembler::DefaultFPTmpList() {
return CPURegList(fp_scratch1, fp_scratch2);
}
void MacroAssembler::LogicalMacro(const Register& rd,
const Register& rn,
const Operand& operand,
LogicalOp op) {
UseScratchRegisterScope temps(this);
if (operand.NeedsRelocation(this)) {
Register temp = temps.AcquireX();
Ldr(temp, operand.immediate());
Logical(rd, rn, temp, op);
} else if (operand.IsImmediate()) {
int64_t immediate = operand.ImmediateValue();
unsigned reg_size = rd.SizeInBits();
// If the operation is NOT, invert the operation and immediate.
if ((op & NOT) == NOT) {
op = static_cast<LogicalOp>(op & ~NOT);
immediate = ~immediate;
}
// Ignore the top 32 bits of an immediate if we're moving to a W register.
if (rd.Is32Bits()) {
// Check that the top 32 bits are consistent.
DCHECK(((immediate >> kWRegSizeInBits) == 0) ||
((immediate >> kWRegSizeInBits) == -1));
immediate &= kWRegMask;
}
DCHECK(rd.Is64Bits() || is_uint32(immediate));
// Special cases for all set or all clear immediates.
if (immediate == 0) {
switch (op) {
case AND:
Mov(rd, 0);
return;
case ORR: // Fall through.
case EOR:
Mov(rd, rn);
return;
case ANDS: // Fall through.
case BICS:
break;
default:
UNREACHABLE();
}
} else if ((rd.Is64Bits() && (immediate == -1L)) ||
(rd.Is32Bits() && (immediate == 0xffffffffL))) {
switch (op) {
case AND:
Mov(rd, rn);
return;
case ORR:
Mov(rd, immediate);
return;
case EOR:
Mvn(rd, rn);
return;
case ANDS: // Fall through.
case BICS:
break;
default:
UNREACHABLE();
}
}
unsigned n, imm_s, imm_r;
if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
// Immediate can be encoded in the instruction.
LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
} else {
// Immediate can't be encoded: synthesize using move immediate.
Register temp = temps.AcquireSameSizeAs(rn);
Operand imm_operand = MoveImmediateForShiftedOp(temp, immediate);
if (rd.Is(csp)) {
// If rd is the stack pointer we cannot use it as the destination
// register so we use the temp register as an intermediate again.
Logical(temp, rn, imm_operand, op);
Mov(csp, temp);
AssertStackConsistency();
} else {
Logical(rd, rn, imm_operand, op);
}
}
} else if (operand.IsExtendedRegister()) {
DCHECK(operand.reg().SizeInBits() <= rd.SizeInBits());
// Add/sub extended supports shift <= 4. We want to support exactly the
// same modes here.
DCHECK(operand.shift_amount() <= 4);
DCHECK(operand.reg().Is64Bits() ||
((operand.extend() != UXTX) && (operand.extend() != SXTX)));
Register temp = temps.AcquireSameSizeAs(rn);
EmitExtendShift(temp, operand.reg(), operand.extend(),
operand.shift_amount());
Logical(rd, rn, temp, op);
} else {
// The operand can be encoded in the instruction.
DCHECK(operand.IsShiftedRegister());
Logical(rd, rn, operand, op);
}
}
void MacroAssembler::Mov(const Register& rd, uint64_t imm) {
DCHECK(allow_macro_instructions_);
DCHECK(is_uint32(imm) || is_int32(imm) || rd.Is64Bits());
DCHECK(!rd.IsZero());
// TODO(all) extend to support more immediates.
//
// Immediates on Aarch64 can be produced using an initial value, and zero to
// three move keep operations.
//
// Initial values can be generated with:
// 1. 64-bit move zero (movz).
// 2. 32-bit move inverted (movn).
// 3. 64-bit move inverted.
// 4. 32-bit orr immediate.
// 5. 64-bit orr immediate.
// Move-keep may then be used to modify each of the 16-bit half-words.
//
// The code below supports all five initial value generators, and
// applying move-keep operations to move-zero and move-inverted initial
// values.
// Try to move the immediate in one instruction, and if that fails, switch to
// using multiple instructions.
if (!TryOneInstrMoveImmediate(rd, imm)) {
unsigned reg_size = rd.SizeInBits();
// Generic immediate case. Imm will be represented by
// [imm3, imm2, imm1, imm0], where each imm is 16 bits.
// A move-zero or move-inverted is generated for the first non-zero or
// non-0xffff immX, and a move-keep for subsequent non-zero immX.
uint64_t ignored_halfword = 0;
bool invert_move = false;
// If the number of 0xffff halfwords is greater than the number of 0x0000
// halfwords, it's more efficient to use move-inverted.
if (CountClearHalfWords(~imm, reg_size) >
CountClearHalfWords(imm, reg_size)) {
ignored_halfword = 0xffffL;
invert_move = true;
}
// Mov instructions can't move immediate values into the stack pointer, so
// set up a temporary register, if needed.
UseScratchRegisterScope temps(this);
Register temp = rd.IsSP() ? temps.AcquireSameSizeAs(rd) : rd;
// Iterate through the halfwords. Use movn/movz for the first non-ignored
// halfword, and movk for subsequent halfwords.
DCHECK((reg_size % 16) == 0);
bool first_mov_done = false;
for (int i = 0; i < (rd.SizeInBits() / 16); i++) {
uint64_t imm16 = (imm >> (16 * i)) & 0xffffL;
if (imm16 != ignored_halfword) {
if (!first_mov_done) {
if (invert_move) {
movn(temp, (~imm16) & 0xffffL, 16 * i);
} else {
movz(temp, imm16, 16 * i);
}
first_mov_done = true;
} else {
// Construct a wider constant.
movk(temp, imm16, 16 * i);
}
}
}
DCHECK(first_mov_done);
// Move the temporary if the original destination register was the stack
// pointer.
if (rd.IsSP()) {
mov(rd, temp);
AssertStackConsistency();
}
}
}
void MacroAssembler::Mov(const Register& rd,
const Operand& operand,
DiscardMoveMode discard_mode) {
DCHECK(allow_macro_instructions_);
DCHECK(!rd.IsZero());
// Provide a swap register for instructions that need to write into the
// system stack pointer (and can't do this inherently).
UseScratchRegisterScope temps(this);
Register dst = (rd.IsSP()) ? temps.AcquireSameSizeAs(rd) : rd;
if (operand.NeedsRelocation(this)) {
Ldr(dst, operand.immediate());
} else if (operand.IsImmediate()) {
// Call the macro assembler for generic immediates.
Mov(dst, operand.ImmediateValue());
} else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) {
// Emit a shift instruction if moving a shifted register. This operation
// could also be achieved using an orr instruction (like orn used by Mvn),
// but using a shift instruction makes the disassembly clearer.
EmitShift(dst, operand.reg(), operand.shift(), operand.shift_amount());
} else if (operand.IsExtendedRegister()) {
// Emit an extend instruction if moving an extended register. This handles
// extend with post-shift operations, too.
EmitExtendShift(dst, operand.reg(), operand.extend(),
operand.shift_amount());
} else {
// Otherwise, emit a register move only if the registers are distinct, or
// if they are not X registers.
//
// Note that mov(w0, w0) is not a no-op because it clears the top word of
// x0. A flag is provided (kDiscardForSameWReg) if a move between the same W
// registers is not required to clear the top word of the X register. In
// this case, the instruction is discarded.
//
// If csp is an operand, add #0 is emitted, otherwise, orr #0.
if (!rd.Is(operand.reg()) || (rd.Is32Bits() &&
(discard_mode == kDontDiscardForSameWReg))) {
Assembler::mov(rd, operand.reg());
}
// This case can handle writes into the system stack pointer directly.
dst = rd;
}
// Copy the result to the system stack pointer.
if (!dst.Is(rd)) {
DCHECK(rd.IsSP());
Assembler::mov(rd, dst);
}
}
void MacroAssembler::Mvn(const Register& rd, const Operand& operand) {
DCHECK(allow_macro_instructions_);
if (operand.NeedsRelocation(this)) {
Ldr(rd, operand.immediate());
mvn(rd, rd);
} else if (operand.IsImmediate()) {
// Call the macro assembler for generic immediates.
Mov(rd, ~operand.ImmediateValue());
} else if (operand.IsExtendedRegister()) {
// Emit two instructions for the extend case. This differs from Mov, as
// the extend and invert can't be achieved in one instruction.
EmitExtendShift(rd, operand.reg(), operand.extend(),
operand.shift_amount());
mvn(rd, rd);
} else {
mvn(rd, operand);
}
}
unsigned MacroAssembler::CountClearHalfWords(uint64_t imm, unsigned reg_size) {
DCHECK((reg_size % 8) == 0);
int count = 0;
for (unsigned i = 0; i < (reg_size / 16); i++) {
if ((imm & 0xffff) == 0) {
count++;
}
imm >>= 16;
}
return count;
}
// The movz instruction can generate immediates containing an arbitrary 16-bit
// half-word, with remaining bits clear, eg. 0x00001234, 0x0000123400000000.
bool MacroAssembler::IsImmMovz(uint64_t imm, unsigned reg_size) {
DCHECK((reg_size == kXRegSizeInBits) || (reg_size == kWRegSizeInBits));
return CountClearHalfWords(imm, reg_size) >= ((reg_size / 16) - 1);
}
// The movn instruction can generate immediates containing an arbitrary 16-bit
// half-word, with remaining bits set, eg. 0xffff1234, 0xffff1234ffffffff.
bool MacroAssembler::IsImmMovn(uint64_t imm, unsigned reg_size) {
return IsImmMovz(~imm, reg_size);
}
void MacroAssembler::ConditionalCompareMacro(const Register& rn,
const Operand& operand,
StatusFlags nzcv,
Condition cond,
ConditionalCompareOp op) {
DCHECK((cond != al) && (cond != nv));
if (operand.NeedsRelocation(this)) {
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
Ldr(temp, operand.immediate());
ConditionalCompareMacro(rn, temp, nzcv, cond, op);
} else if ((operand.IsShiftedRegister() && (operand.shift_amount() == 0)) ||
(operand.IsImmediate() &&
IsImmConditionalCompare(operand.ImmediateValue()))) {
// The immediate can be encoded in the instruction, or the operand is an
// unshifted register: call the assembler.
ConditionalCompare(rn, operand, nzcv, cond, op);
} else {
// The operand isn't directly supported by the instruction: perform the
// operation on a temporary register.
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireSameSizeAs(rn);
Mov(temp, operand);
ConditionalCompare(rn, temp, nzcv, cond, op);
}
}
void MacroAssembler::Csel(const Register& rd,
const Register& rn,
const Operand& operand,
Condition cond) {
DCHECK(allow_macro_instructions_);
DCHECK(!rd.IsZero());
DCHECK((cond != al) && (cond != nv));
if (operand.IsImmediate()) {
// Immediate argument. Handle special cases of 0, 1 and -1 using zero
// register.
int64_t imm = operand.ImmediateValue();
Register zr = AppropriateZeroRegFor(rn);
if (imm == 0) {
csel(rd, rn, zr, cond);
} else if (imm == 1) {
csinc(rd, rn, zr, cond);
} else if (imm == -1) {
csinv(rd, rn, zr, cond);
} else {
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireSameSizeAs(rn);
Mov(temp, imm);
csel(rd, rn, temp, cond);
}
} else if (operand.IsShiftedRegister() && (operand.shift_amount() == 0)) {
// Unshifted register argument.
csel(rd, rn, operand.reg(), cond);
} else {
// All other arguments.
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireSameSizeAs(rn);
Mov(temp, operand);
csel(rd, rn, temp, cond);
}
}
bool MacroAssembler::TryOneInstrMoveImmediate(const Register& dst,
int64_t imm) {
unsigned n, imm_s, imm_r;
int reg_size = dst.SizeInBits();
if (IsImmMovz(imm, reg_size) && !dst.IsSP()) {
// Immediate can be represented in a move zero instruction. Movz can't write
// to the stack pointer.
movz(dst, imm);
return true;
} else if (IsImmMovn(imm, reg_size) && !dst.IsSP()) {
// Immediate can be represented in a move not instruction. Movn can't write
// to the stack pointer.
movn(dst, dst.Is64Bits() ? ~imm : (~imm & kWRegMask));
return true;
} else if (IsImmLogical(imm, reg_size, &n, &imm_s, &imm_r)) {
// Immediate can be represented in a logical orr instruction.
LogicalImmediate(dst, AppropriateZeroRegFor(dst), n, imm_s, imm_r, ORR);
return true;
}
return false;
}
Operand MacroAssembler::MoveImmediateForShiftedOp(const Register& dst,
int64_t imm) {
int reg_size = dst.SizeInBits();
// Encode the immediate in a single move instruction, if possible.
if (TryOneInstrMoveImmediate(dst, imm)) {
// The move was successful; nothing to do here.
} else {
// Pre-shift the immediate to the least-significant bits of the register.
int shift_low = CountTrailingZeros(imm, reg_size);
int64_t imm_low = imm >> shift_low;
// Pre-shift the immediate to the most-significant bits of the register. We
// insert set bits in the least-significant bits, as this creates a
// different immediate that may be encodable using movn or orr-immediate.
// If this new immediate is encodable, the set bits will be eliminated by
// the post shift on the following instruction.
int shift_high = CountLeadingZeros(imm, reg_size);
int64_t imm_high = (imm << shift_high) | ((1 << shift_high) - 1);
if (TryOneInstrMoveImmediate(dst, imm_low)) {
// The new immediate has been moved into the destination's low bits:
// return a new leftward-shifting operand.
return Operand(dst, LSL, shift_low);
} else if (TryOneInstrMoveImmediate(dst, imm_high)) {
// The new immediate has been moved into the destination's high bits:
// return a new rightward-shifting operand.
return Operand(dst, LSR, shift_high);
} else {
// Use the generic move operation to set up the immediate.
Mov(dst, imm);
}
}
return Operand(dst);
}
void MacroAssembler::AddSubMacro(const Register& rd,
const Register& rn,
const Operand& operand,
FlagsUpdate S,
AddSubOp op) {
if (operand.IsZero() && rd.Is(rn) && rd.Is64Bits() && rn.Is64Bits() &&
!operand.NeedsRelocation(this) && (S == LeaveFlags)) {
// The instruction would be a nop. Avoid generating useless code.
return;
}
if (operand.NeedsRelocation(this)) {
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
Ldr(temp, operand.immediate());
AddSubMacro(rd, rn, temp, S, op);
} else if ((operand.IsImmediate() &&
!IsImmAddSub(operand.ImmediateValue())) ||
(rn.IsZero() && !operand.IsShiftedRegister()) ||
(operand.IsShiftedRegister() && (operand.shift() == ROR))) {
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireSameSizeAs(rn);
if (operand.IsImmediate()) {
Operand imm_operand =
MoveImmediateForShiftedOp(temp, operand.ImmediateValue());
AddSub(rd, rn, imm_operand, S, op);
} else {
Mov(temp, operand);
AddSub(rd, rn, temp, S, op);
}
} else {
AddSub(rd, rn, operand, S, op);
}
}
void MacroAssembler::AddSubWithCarryMacro(const Register& rd,
const Register& rn,
const Operand& operand,
FlagsUpdate S,
AddSubWithCarryOp op) {
DCHECK(rd.SizeInBits() == rn.SizeInBits());
UseScratchRegisterScope temps(this);
if (operand.NeedsRelocation(this)) {
Register temp = temps.AcquireX();
Ldr(temp, operand.immediate());
AddSubWithCarryMacro(rd, rn, temp, S, op);
} else if (operand.IsImmediate() ||
(operand.IsShiftedRegister() && (operand.shift() == ROR))) {
// Add/sub with carry (immediate or ROR shifted register.)
Register temp = temps.AcquireSameSizeAs(rn);
Mov(temp, operand);
AddSubWithCarry(rd, rn, temp, S, op);
} else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) {
// Add/sub with carry (shifted register).
DCHECK(operand.reg().SizeInBits() == rd.SizeInBits());
DCHECK(operand.shift() != ROR);
DCHECK(is_uintn(operand.shift_amount(),
rd.SizeInBits() == kXRegSizeInBits ? kXRegSizeInBitsLog2
: kWRegSizeInBitsLog2));
Register temp = temps.AcquireSameSizeAs(rn);
EmitShift(temp, operand.reg(), operand.shift(), operand.shift_amount());
AddSubWithCarry(rd, rn, temp, S, op);
} else if (operand.IsExtendedRegister()) {
// Add/sub with carry (extended register).
DCHECK(operand.reg().SizeInBits() <= rd.SizeInBits());
// Add/sub extended supports a shift <= 4. We want to support exactly the
// same modes.
DCHECK(operand.shift_amount() <= 4);
DCHECK(operand.reg().Is64Bits() ||
((operand.extend() != UXTX) && (operand.extend() != SXTX)));
Register temp = temps.AcquireSameSizeAs(rn);
EmitExtendShift(temp, operand.reg(), operand.extend(),
operand.shift_amount());
AddSubWithCarry(rd, rn, temp, S, op);
} else {
// The addressing mode is directly supported by the instruction.
AddSubWithCarry(rd, rn, operand, S, op);
}
}
void MacroAssembler::LoadStoreMacro(const CPURegister& rt,
const MemOperand& addr,
LoadStoreOp op) {
int64_t offset = addr.offset();
LSDataSize size = CalcLSDataSize(op);
// Check if an immediate offset fits in the immediate field of the
// appropriate instruction. If not, emit two instructions to perform
// the operation.
if (addr.IsImmediateOffset() && !IsImmLSScaled(offset, size) &&
!IsImmLSUnscaled(offset)) {
// Immediate offset that can't be encoded using unsigned or unscaled
// addressing modes.
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireSameSizeAs(addr.base());
Mov(temp, addr.offset());
LoadStore(rt, MemOperand(addr.base(), temp), op);
} else if (addr.IsPostIndex() && !IsImmLSUnscaled(offset)) {
// Post-index beyond unscaled addressing range.
LoadStore(rt, MemOperand(addr.base()), op);
add(addr.base(), addr.base(), offset);
} else if (addr.IsPreIndex() && !IsImmLSUnscaled(offset)) {
// Pre-index beyond unscaled addressing range.
add(addr.base(), addr.base(), offset);
LoadStore(rt, MemOperand(addr.base()), op);
} else {
// Encodable in one load/store instruction.
LoadStore(rt, addr, op);
}
}
void MacroAssembler::LoadStorePairMacro(const CPURegister& rt,
const CPURegister& rt2,
const MemOperand& addr,
LoadStorePairOp op) {
// TODO(all): Should we support register offset for load-store-pair?
DCHECK(!addr.IsRegisterOffset());
int64_t offset = addr.offset();
LSDataSize size = CalcLSPairDataSize(op);
// Check if the offset fits in the immediate field of the appropriate
// instruction. If not, emit two instructions to perform the operation.
if (IsImmLSPair(offset, size)) {
// Encodable in one load/store pair instruction.
LoadStorePair(rt, rt2, addr, op);
} else {
Register base = addr.base();
if (addr.IsImmediateOffset()) {
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireSameSizeAs(base);
Add(temp, base, offset);
LoadStorePair(rt, rt2, MemOperand(temp), op);
} else if (addr.IsPostIndex()) {
LoadStorePair(rt, rt2, MemOperand(base), op);
Add(base, base, offset);
} else {
DCHECK(addr.IsPreIndex());
Add(base, base, offset);
LoadStorePair(rt, rt2, MemOperand(base), op);
}
}
}
void MacroAssembler::Load(const Register& rt,
const MemOperand& addr,
Representation r) {
DCHECK(!r.IsDouble());
if (r.IsInteger8()) {
Ldrsb(rt, addr);
} else if (r.IsUInteger8()) {
Ldrb(rt, addr);
} else if (r.IsInteger16()) {
Ldrsh(rt, addr);
} else if (r.IsUInteger16()) {
Ldrh(rt, addr);
} else if (r.IsInteger32()) {
Ldr(rt.W(), addr);
} else {
DCHECK(rt.Is64Bits());
Ldr(rt, addr);
}
}
void MacroAssembler::Store(const Register& rt,
const MemOperand& addr,
Representation r) {
DCHECK(!r.IsDouble());
if (r.IsInteger8() || r.IsUInteger8()) {
Strb(rt, addr);
} else if (r.IsInteger16() || r.IsUInteger16()) {
Strh(rt, addr);
} else if (r.IsInteger32()) {
Str(rt.W(), addr);
} else {
DCHECK(rt.Is64Bits());
if (r.IsHeapObject()) {
AssertNotSmi(rt);
} else if (r.IsSmi()) {
AssertSmi(rt);
}
Str(rt, addr);
}
}
bool MacroAssembler::NeedExtraInstructionsOrRegisterBranch(
Label *label, ImmBranchType b_type) {
bool need_longer_range = false;
// There are two situations in which we care about the offset being out of
// range:
// - The label is bound but too far away.
// - The label is not bound but linked, and the previous branch
// instruction in the chain is too far away.
if (label->is_bound() || label->is_linked()) {
need_longer_range =
!Instruction::IsValidImmPCOffset(b_type, label->pos() - pc_offset());
}
if (!need_longer_range && !label->is_bound()) {
int max_reachable_pc = pc_offset() + Instruction::ImmBranchRange(b_type);
unresolved_branches_.insert(
std::pair<int, FarBranchInfo>(max_reachable_pc,
FarBranchInfo(pc_offset(), label)));
// Also maintain the next pool check.
next_veneer_pool_check_ =
Min(next_veneer_pool_check_,
max_reachable_pc - kVeneerDistanceCheckMargin);
}
return need_longer_range;
}
void MacroAssembler::Adr(const Register& rd, Label* label, AdrHint hint) {
DCHECK(allow_macro_instructions_);
DCHECK(!rd.IsZero());
if (hint == kAdrNear) {
adr(rd, label);
return;
}
DCHECK(hint == kAdrFar);
if (label->is_bound()) {
int label_offset = label->pos() - pc_offset();
if (Instruction::IsValidPCRelOffset(label_offset)) {
adr(rd, label);
} else {
DCHECK(label_offset <= 0);
int min_adr_offset = -(1 << (Instruction::ImmPCRelRangeBitwidth - 1));
adr(rd, min_adr_offset);
Add(rd, rd, label_offset - min_adr_offset);
}
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.AcquireX();
InstructionAccurateScope scope(
this, PatchingAssembler::kAdrFarPatchableNInstrs);
adr(rd, label);
for (int i = 0; i < PatchingAssembler::kAdrFarPatchableNNops; ++i) {
nop(ADR_FAR_NOP);
}
movz(scratch, 0);
}
}
void MacroAssembler::B(Label* label, BranchType type, Register reg, int bit) {
DCHECK((reg.Is(NoReg) || type >= kBranchTypeFirstUsingReg) &&
(bit == -1 || type >= kBranchTypeFirstUsingBit));
if (kBranchTypeFirstCondition <= type && type <= kBranchTypeLastCondition) {
B(static_cast<Condition>(type), label);
} else {
switch (type) {
case always: B(label); break;
case never: break;
case reg_zero: Cbz(reg, label); break;
case reg_not_zero: Cbnz(reg, label); break;
case reg_bit_clear: Tbz(reg, bit, label); break;
case reg_bit_set: Tbnz(reg, bit, label); break;
default:
UNREACHABLE();
}
}
}
void MacroAssembler::B(Label* label, Condition cond) {
DCHECK(allow_macro_instructions_);
DCHECK((cond != al) && (cond != nv));
Label done;
bool need_extra_instructions =
NeedExtraInstructionsOrRegisterBranch(label, CondBranchType);
if (need_extra_instructions) {
b(&done, NegateCondition(cond));
B(label);
} else {
b(label, cond);
}
bind(&done);
}
void MacroAssembler::Tbnz(const Register& rt, unsigned bit_pos, Label* label) {
DCHECK(allow_macro_instructions_);
Label done;
bool need_extra_instructions =
NeedExtraInstructionsOrRegisterBranch(label, TestBranchType);
if (need_extra_instructions) {
tbz(rt, bit_pos, &done);
B(label);
} else {
tbnz(rt, bit_pos, label);
}
bind(&done);
}
void MacroAssembler::Tbz(const Register& rt, unsigned bit_pos, Label* label) {
DCHECK(allow_macro_instructions_);
Label done;
bool need_extra_instructions =
NeedExtraInstructionsOrRegisterBranch(label, TestBranchType);
if (need_extra_instructions) {
tbnz(rt, bit_pos, &done);
B(label);
} else {
tbz(rt, bit_pos, label);
}
bind(&done);
}
void MacroAssembler::Cbnz(const Register& rt, Label* label) {
DCHECK(allow_macro_instructions_);
Label done;
bool need_extra_instructions =
NeedExtraInstructionsOrRegisterBranch(label, CompareBranchType);
if (need_extra_instructions) {
cbz(rt, &done);
B(label);
} else {
cbnz(rt, label);
}
bind(&done);
}
void MacroAssembler::Cbz(const Register& rt, Label* label) {
DCHECK(allow_macro_instructions_);
Label done;
bool need_extra_instructions =
NeedExtraInstructionsOrRegisterBranch(label, CompareBranchType);
if (need_extra_instructions) {
cbnz(rt, &done);
B(label);
} else {
cbz(rt, label);
}
bind(&done);
}
// Pseudo-instructions.
void MacroAssembler::Abs(const Register& rd, const Register& rm,
Label* is_not_representable,
Label* is_representable) {
DCHECK(allow_macro_instructions_);
DCHECK(AreSameSizeAndType(rd, rm));
Cmp(rm, 1);
Cneg(rd, rm, lt);
// If the comparison sets the v flag, the input was the smallest value
// representable by rm, and the mathematical result of abs(rm) is not
// representable using two's complement.
if ((is_not_representable != NULL) && (is_representable != NULL)) {
B(is_not_representable, vs);
B(is_representable);
} else if (is_not_representable != NULL) {
B(is_not_representable, vs);
} else if (is_representable != NULL) {
B(is_representable, vc);
}
}
// Abstracted stack operations.
void MacroAssembler::Push(const CPURegister& src0, const CPURegister& src1,
const CPURegister& src2, const CPURegister& src3) {
DCHECK(AreSameSizeAndType(src0, src1, src2, src3));
int count = 1 + src1.IsValid() + src2.IsValid() + src3.IsValid();
int size = src0.SizeInBytes();
PushPreamble(count, size);
PushHelper(count, size, src0, src1, src2, src3);
}
void MacroAssembler::Push(const CPURegister& src0, const CPURegister& src1,
const CPURegister& src2, const CPURegister& src3,
const CPURegister& src4, const CPURegister& src5,
const CPURegister& src6, const CPURegister& src7) {
DCHECK(AreSameSizeAndType(src0, src1, src2, src3, src4, src5, src6, src7));
int count = 5 + src5.IsValid() + src6.IsValid() + src6.IsValid();
int size = src0.SizeInBytes();
PushPreamble(count, size);
PushHelper(4, size, src0, src1, src2, src3);
PushHelper(count - 4, size, src4, src5, src6, src7);
}
void MacroAssembler::Pop(const CPURegister& dst0, const CPURegister& dst1,
const CPURegister& dst2, const CPURegister& dst3) {
// It is not valid to pop into the same register more than once in one
// instruction, not even into the zero register.
DCHECK(!AreAliased(dst0, dst1, dst2, dst3));
DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3));
DCHECK(dst0.IsValid());
int count = 1 + dst1.IsValid() + dst2.IsValid() + dst3.IsValid();
int size = dst0.SizeInBytes();
PopHelper(count, size, dst0, dst1, dst2, dst3);
PopPostamble(count, size);
}
void MacroAssembler::Pop(const CPURegister& dst0, const CPURegister& dst1,
const CPURegister& dst2, const CPURegister& dst3,
const CPURegister& dst4, const CPURegister& dst5,
const CPURegister& dst6, const CPURegister& dst7) {
// It is not valid to pop into the same register more than once in one
// instruction, not even into the zero register.
DCHECK(!AreAliased(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7));
DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7));
DCHECK(dst0.IsValid());
int count = 5 + dst5.IsValid() + dst6.IsValid() + dst7.IsValid();
int size = dst0.SizeInBytes();
PopHelper(4, size, dst0, dst1, dst2, dst3);
PopHelper(count - 4, size, dst4, dst5, dst6, dst7);
PopPostamble(count, size);
}
void MacroAssembler::Push(const Register& src0, const FPRegister& src1) {
int size = src0.SizeInBytes() + src1.SizeInBytes();
PushPreamble(size);
// Reserve room for src0 and push src1.
str(src1, MemOperand(StackPointer(), -size, PreIndex));
// Fill the gap with src0.
str(src0, MemOperand(StackPointer(), src1.SizeInBytes()));
}
void MacroAssembler::PushPopQueue::PushQueued(
PreambleDirective preamble_directive) {
if (queued_.empty()) return;
if (preamble_directive == WITH_PREAMBLE) {
masm_->PushPreamble(size_);
}
size_t count = queued_.size();
size_t index = 0;
while (index < count) {
// PushHelper can only handle registers with the same size and type, and it
// can handle only four at a time. Batch them up accordingly.
CPURegister batch[4] = {NoReg, NoReg, NoReg, NoReg};
int batch_index = 0;
do {
batch[batch_index++] = queued_[index++];
} while ((batch_index < 4) && (index < count) &&
batch[0].IsSameSizeAndType(queued_[index]));
masm_->PushHelper(batch_index, batch[0].SizeInBytes(),
batch[0], batch[1], batch[2], batch[3]);
}
queued_.clear();
}
void MacroAssembler::PushPopQueue::PopQueued() {
if (queued_.empty()) return;
size_t count = queued_.size();
size_t index = 0;
while (index < count) {
// PopHelper can only handle registers with the same size and type, and it
// can handle only four at a time. Batch them up accordingly.
CPURegister batch[4] = {NoReg, NoReg, NoReg, NoReg};
int batch_index = 0;
do {
batch[batch_index++] = queued_[index++];
} while ((batch_index < 4) && (index < count) &&
batch[0].IsSameSizeAndType(queued_[index]));
masm_->PopHelper(batch_index, batch[0].SizeInBytes(),
batch[0], batch[1], batch[2], batch[3]);
}
masm_->PopPostamble(size_);
queued_.clear();
}
void MacroAssembler::PushCPURegList(CPURegList registers) {
int size = registers.RegisterSizeInBytes();
PushPreamble(registers.Count(), size);
// Push up to four registers at a time because if the current stack pointer is
// csp and reg_size is 32, registers must be pushed in blocks of four in order
// to maintain the 16-byte alignment for csp.
while (!registers.IsEmpty()) {
int count_before = registers.Count();
const CPURegister& src0 = registers.PopHighestIndex();
const CPURegister& src1 = registers.PopHighestIndex();
const CPURegister& src2 = registers.PopHighestIndex();
const CPURegister& src3 = registers.PopHighestIndex();
int count = count_before - registers.Count();
PushHelper(count, size, src0, src1, src2, src3);
}
}
void MacroAssembler::PopCPURegList(CPURegList registers) {
int size = registers.RegisterSizeInBytes();
// Pop up to four registers at a time because if the current stack pointer is
// csp and reg_size is 32, registers must be pushed in blocks of four in
// order to maintain the 16-byte alignment for csp.
while (!registers.IsEmpty()) {
int count_before = registers.Count();
const CPURegister& dst0 = registers.PopLowestIndex();
const CPURegister& dst1 = registers.PopLowestIndex();
const CPURegister& dst2 = registers.PopLowestIndex();
const CPURegister& dst3 = registers.PopLowestIndex();
int count = count_before - registers.Count();
PopHelper(count, size, dst0, dst1, dst2, dst3);
}
PopPostamble(registers.Count(), size);
}
void MacroAssembler::PushMultipleTimes(CPURegister src, int count) {
int size = src.SizeInBytes();
PushPreamble(count, size);
if (FLAG_optimize_for_size && count > 8) {
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
Label loop;
__ Mov(temp, count / 2);
__ Bind(&loop);
PushHelper(2, size, src, src, NoReg, NoReg);
__ Subs(temp, temp, 1);
__ B(ne, &loop);
count %= 2;
}
// Push up to four registers at a time if possible because if the current
// stack pointer is csp and the register size is 32, registers must be pushed
// in blocks of four in order to maintain the 16-byte alignment for csp.
while (count >= 4) {
PushHelper(4, size, src, src, src, src);
count -= 4;
}
if (count >= 2) {
PushHelper(2, size, src, src, NoReg, NoReg);
count -= 2;
}
if (count == 1) {
PushHelper(1, size, src, NoReg, NoReg, NoReg);
count -= 1;
}
DCHECK(count == 0);
}
void MacroAssembler::PushMultipleTimes(CPURegister src, Register count) {
PushPreamble(Operand(count, UXTW, WhichPowerOf2(src.SizeInBytes())));
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireSameSizeAs(count);
if (FLAG_optimize_for_size) {
Label loop, done;
Subs(temp, count, 1);
B(mi, &done);
// Push all registers individually, to save code size.
Bind(&loop);
Subs(temp, temp, 1);
PushHelper(1, src.SizeInBytes(), src, NoReg, NoReg, NoReg);
B(pl, &loop);
Bind(&done);
} else {
Label loop, leftover2, leftover1, done;
Subs(temp, count, 4);
B(mi, &leftover2);
// Push groups of four first.
Bind(&loop);
Subs(temp, temp, 4);
PushHelper(4, src.SizeInBytes(), src, src, src, src);
B(pl, &loop);
// Push groups of two.
Bind(&leftover2);
Tbz(count, 1, &leftover1);
PushHelper(2, src.SizeInBytes(), src, src, NoReg, NoReg);
// Push the last one (if required).
Bind(&leftover1);
Tbz(count, 0, &done);
PushHelper(1, src.SizeInBytes(), src, NoReg, NoReg, NoReg);
Bind(&done);
}
}
void MacroAssembler::PushHelper(int count, int size,
const CPURegister& src0,
const CPURegister& src1,
const CPURegister& src2,
const CPURegister& src3) {
// Ensure that we don't unintentially modify scratch or debug registers.
InstructionAccurateScope scope(this);
DCHECK(AreSameSizeAndType(src0, src1, src2, src3));
DCHECK(size == src0.SizeInBytes());
// When pushing multiple registers, the store order is chosen such that
// Push(a, b) is equivalent to Push(a) followed by Push(b).
switch (count) {
case 1:
DCHECK(src1.IsNone() && src2.IsNone() && src3.IsNone());
str(src0, MemOperand(StackPointer(), -1 * size, PreIndex));
break;
case 2:
DCHECK(src2.IsNone() && src3.IsNone());
stp(src1, src0, MemOperand(StackPointer(), -2 * size, PreIndex));
break;
case 3:
DCHECK(src3.IsNone());
stp(src2, src1, MemOperand(StackPointer(), -3 * size, PreIndex));
str(src0, MemOperand(StackPointer(), 2 * size));
break;
case 4:
// Skip over 4 * size, then fill in the gap. This allows four W registers
// to be pushed using csp, whilst maintaining 16-byte alignment for csp
// at all times.
stp(src3, src2, MemOperand(StackPointer(), -4 * size, PreIndex));
stp(src1, src0, MemOperand(StackPointer(), 2 * size));
break;
default:
UNREACHABLE();
}
}
void MacroAssembler::PopHelper(int count, int size,
const CPURegister& dst0,
const CPURegister& dst1,
const CPURegister& dst2,
const CPURegister& dst3) {
// Ensure that we don't unintentially modify scratch or debug registers.
InstructionAccurateScope scope(this);
DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3));
DCHECK(size == dst0.SizeInBytes());
// When popping multiple registers, the load order is chosen such that
// Pop(a, b) is equivalent to Pop(a) followed by Pop(b).
switch (count) {
case 1:
DCHECK(dst1.IsNone() && dst2.IsNone() && dst3.IsNone());
ldr(dst0, MemOperand(StackPointer(), 1 * size, PostIndex));
break;
case 2:
DCHECK(dst2.IsNone() && dst3.IsNone());
ldp(dst0, dst1, MemOperand(StackPointer(), 2 * size, PostIndex));
break;
case 3:
DCHECK(dst3.IsNone());
ldr(dst2, MemOperand(StackPointer(), 2 * size));
ldp(dst0, dst1, MemOperand(StackPointer(), 3 * size, PostIndex));
break;
case 4:
// Load the higher addresses first, then load the lower addresses and
// skip the whole block in the second instruction. This allows four W
// registers to be popped using csp, whilst maintaining 16-byte alignment
// for csp at all times.
ldp(dst2, dst3, MemOperand(StackPointer(), 2 * size));
ldp(dst0, dst1, MemOperand(StackPointer(), 4 * size, PostIndex));
break;
default:
UNREACHABLE();
}
}
void MacroAssembler::PushPreamble(Operand total_size) {
if (csp.Is(StackPointer())) {
// If the current stack pointer is csp, then it must be aligned to 16 bytes
// on entry and the total size of the specified registers must also be a
// multiple of 16 bytes.
if (total_size.IsImmediate()) {
DCHECK((total_size.ImmediateValue() % 16) == 0);
}
// Don't check access size for non-immediate sizes. It's difficult to do
// well, and it will be caught by hardware (or the simulator) anyway.
} else {
// Even if the current stack pointer is not the system stack pointer (csp),
// the system stack pointer will still be modified in order to comply with
// ABI rules about accessing memory below the system stack pointer.
BumpSystemStackPointer(total_size);
}
}
void MacroAssembler::PopPostamble(Operand total_size) {
if (csp.Is(StackPointer())) {
// If the current stack pointer is csp, then it must be aligned to 16 bytes
// on entry and the total size of the specified registers must also be a
// multiple of 16 bytes.
if (total_size.IsImmediate()) {
DCHECK((total_size.ImmediateValue() % 16) == 0);
}
// Don't check access size for non-immediate sizes. It's difficult to do
// well, and it will be caught by hardware (or the simulator) anyway.
} else if (emit_debug_code()) {
// It is safe to leave csp where it is when unwinding the JavaScript stack,
// but if we keep it matching StackPointer, the simulator can detect memory
// accesses in the now-free part of the stack.
SyncSystemStackPointer();
}
}
void MacroAssembler::Poke(const CPURegister& src, const Operand& offset) {
if (offset.IsImmediate()) {
DCHECK(offset.ImmediateValue() >= 0);
} else if (emit_debug_code()) {
Cmp(xzr, offset);
Check(le, kStackAccessBelowStackPointer);
}
Str(src, MemOperand(StackPointer(), offset));
}
void MacroAssembler::Peek(const CPURegister& dst, const Operand& offset) {
if (offset.IsImmediate()) {
DCHECK(offset.ImmediateValue() >= 0);
} else if (emit_debug_code()) {
Cmp(xzr, offset);
Check(le, kStackAccessBelowStackPointer);
}
Ldr(dst, MemOperand(StackPointer(), offset));
}
void MacroAssembler::PokePair(const CPURegister& src1,
const CPURegister& src2,
int offset) {
DCHECK(AreSameSizeAndType(src1, src2));
DCHECK((offset >= 0) && ((offset % src1.SizeInBytes()) == 0));
Stp(src1, src2, MemOperand(StackPointer(), offset));
}
void MacroAssembler::PeekPair(const CPURegister& dst1,
const CPURegister& dst2,
int offset) {
DCHECK(AreSameSizeAndType(dst1, dst2));
DCHECK((offset >= 0) && ((offset % dst1.SizeInBytes()) == 0));
Ldp(dst1, dst2, MemOperand(StackPointer(), offset));
}
void MacroAssembler::PushCalleeSavedRegisters() {
// Ensure that the macro-assembler doesn't use any scratch registers.
InstructionAccurateScope scope(this);
// This method must not be called unless the current stack pointer is the
// system stack pointer (csp).
DCHECK(csp.Is(StackPointer()));
MemOperand tos(csp, -2 * static_cast<int>(kXRegSize), PreIndex);
stp(d14, d15, tos);
stp(d12, d13, tos);
stp(d10, d11, tos);
stp(d8, d9, tos);
stp(x29, x30, tos);
stp(x27, x28, tos); // x28 = jssp
stp(x25, x26, tos);
stp(x23, x24, tos);
stp(x21, x22, tos);
stp(x19, x20, tos);
}
void MacroAssembler::PopCalleeSavedRegisters() {
// Ensure that the macro-assembler doesn't use any scratch registers.
InstructionAccurateScope scope(this);
// This method must not be called unless the current stack pointer is the
// system stack pointer (csp).
DCHECK(csp.Is(StackPointer()));
MemOperand tos(csp, 2 * kXRegSize, PostIndex);
ldp(x19, x20, tos);
ldp(x21, x22, tos);
ldp(x23, x24, tos);
ldp(x25, x26, tos);
ldp(x27, x28, tos); // x28 = jssp
ldp(x29, x30, tos);
ldp(d8, d9, tos);
ldp(d10, d11, tos);
ldp(d12, d13, tos);
ldp(d14, d15, tos);
}
void MacroAssembler::AssertStackConsistency() {
// Avoid emitting code when !use_real_abort() since non-real aborts cause too
// much code to be generated.
if (emit_debug_code() && use_real_aborts()) {
if (csp.Is(StackPointer())) {
// Always check the alignment of csp if ALWAYS_ALIGN_CSP is true. We
// can't check the alignment of csp without using a scratch register (or
// clobbering the flags), but the processor (or simulator) will abort if
// it is not properly aligned during a load.
ldr(xzr, MemOperand(csp, 0));
}
if (FLAG_enable_slow_asserts && !csp.Is(StackPointer())) {
Label ok;
// Check that csp <= StackPointer(), preserving all registers and NZCV.
sub(StackPointer(), csp, StackPointer());
cbz(StackPointer(), &ok); // Ok if csp == StackPointer().
tbnz(StackPointer(), kXSignBit, &ok); // Ok if csp < StackPointer().
// Avoid generating AssertStackConsistency checks for the Push in Abort.
{ DontEmitDebugCodeScope dont_emit_debug_code_scope(this);
// Restore StackPointer().
sub(StackPointer(), csp, StackPointer());
Abort(kTheCurrentStackPointerIsBelowCsp);
}
bind(&ok);
// Restore StackPointer().
sub(StackPointer(), csp, StackPointer());
}
}
}
void MacroAssembler::AssertCspAligned() {
if (emit_debug_code() && use_real_aborts()) {
// TODO(titzer): use a real assert for alignment check?
UseScratchRegisterScope scope(this);
Register temp = scope.AcquireX();
ldr(temp, MemOperand(csp));
}
}
void MacroAssembler::AssertFPCRState(Register fpcr) {
if (emit_debug_code()) {
Label unexpected_mode, done;
UseScratchRegisterScope temps(this);
if (fpcr.IsNone()) {
fpcr = temps.AcquireX();
Mrs(fpcr, FPCR);
}
// Settings left to their default values:
// - Assert that flush-to-zero is not set.
Tbnz(fpcr, FZ_offset, &unexpected_mode);
// - Assert that the rounding mode is nearest-with-ties-to-even.
STATIC_ASSERT(FPTieEven == 0);
Tst(fpcr, RMode_mask);
B(eq, &done);
Bind(&unexpected_mode);
Abort(kUnexpectedFPCRMode);
Bind(&done);
}
}
void MacroAssembler::CanonicalizeNaN(const FPRegister& dst,
const FPRegister& src) {
AssertFPCRState();
// Subtracting 0.0 preserves all inputs except for signalling NaNs, which
// become quiet NaNs. We use fsub rather than fadd because fsub preserves -0.0
// inputs: -0.0 + 0.0 = 0.0, but -0.0 - 0.0 = -0.0.
Fsub(dst, src, fp_zero);
}
void MacroAssembler::LoadRoot(CPURegister destination,
Heap::RootListIndex index) {
// TODO(jbramley): Most root values are constants, and can be synthesized
// without a load. Refer to the ARM back end for details.
Ldr(destination, MemOperand(root, index << kPointerSizeLog2));
}
void MacroAssembler::StoreRoot(Register source,
Heap::RootListIndex index) {
DCHECK(Heap::RootCanBeWrittenAfterInitialization(index));
Str(source, MemOperand(root, index << kPointerSizeLog2));
}
void MacroAssembler::LoadTrueFalseRoots(Register true_root,
Register false_root) {
STATIC_ASSERT((Heap::kTrueValueRootIndex + 1) == Heap::kFalseValueRootIndex);
Ldp(true_root, false_root,
MemOperand(root, Heap::kTrueValueRootIndex << kPointerSizeLog2));
}
void MacroAssembler::LoadHeapObject(Register result,
Handle<HeapObject> object) {
Mov(result, Operand(object));
}
void MacroAssembler::LoadInstanceDescriptors(Register map,
Register descriptors) {
Ldr(descriptors, FieldMemOperand(map, Map::kDescriptorsOffset));
}
void MacroAssembler::NumberOfOwnDescriptors(Register dst, Register map) {
Ldr(dst, FieldMemOperand(map, Map::kBitField3Offset));
DecodeField<Map::NumberOfOwnDescriptorsBits>(dst);
}
void MacroAssembler::EnumLengthUntagged(Register dst, Register map) {
STATIC_ASSERT(Map::EnumLengthBits::kShift == 0);
Ldrsw(dst, FieldMemOperand(map, Map::kBitField3Offset));
And(dst, dst, Map::EnumLengthBits::kMask);
}
void MacroAssembler::EnumLengthSmi(Register dst, Register map) {
EnumLengthUntagged(dst, map);
SmiTag(dst, dst);
}
void MacroAssembler::LoadAccessor(Register dst, Register holder,
int accessor_index,
AccessorComponent accessor) {
Ldr(dst, FieldMemOperand(holder, HeapObject::kMapOffset));
LoadInstanceDescriptors(dst, dst);
Ldr(dst,
FieldMemOperand(dst, DescriptorArray::GetValueOffset(accessor_index)));
int offset = accessor == ACCESSOR_GETTER ? AccessorPair::kGetterOffset
: AccessorPair::kSetterOffset;
Ldr(dst, FieldMemOperand(dst, offset));
}
void MacroAssembler::CheckEnumCache(Register object, Register scratch0,
Register scratch1, Register scratch2,
Register scratch3, Register scratch4,
Label* call_runtime) {
DCHECK(!AreAliased(object, scratch0, scratch1, scratch2, scratch3, scratch4));
Register empty_fixed_array_value = scratch0;
Register current_object = scratch1;
Register null_value = scratch4;
LoadRoot(empty_fixed_array_value, Heap::kEmptyFixedArrayRootIndex);
Label next, start;
Mov(current_object, object);
// Check if the enum length field is properly initialized, indicating that
// there is an enum cache.
Register map = scratch2;
Register enum_length = scratch3;
Ldr(map, FieldMemOperand(current_object, HeapObject::kMapOffset));
EnumLengthUntagged(enum_length, map);
Cmp(enum_length, kInvalidEnumCacheSentinel);
B(eq, call_runtime);
LoadRoot(null_value, Heap::kNullValueRootIndex);
B(&start);
Bind(&next);
Ldr(map, FieldMemOperand(current_object, HeapObject::kMapOffset));
// For all objects but the receiver, check that the cache is empty.
EnumLengthUntagged(enum_length, map);
Cbnz(enum_length, call_runtime);
Bind(&start);
// Check that there are no elements. Register current_object contains the
// current JS object we've reached through the prototype chain.
Label no_elements;
Ldr(current_object, FieldMemOperand(current_object,
JSObject::kElementsOffset));
Cmp(current_object, empty_fixed_array_value);
B(eq, &no_elements);
// Second chance, the object may be using the empty slow element dictionary.
CompareRoot(current_object, Heap::kEmptySlowElementDictionaryRootIndex);
B(ne, call_runtime);
Bind(&no_elements);
Ldr(current_object, FieldMemOperand(map, Map::kPrototypeOffset));
Cmp(current_object, null_value);
B(ne, &next);
}
void MacroAssembler::TestJSArrayForAllocationMemento(Register receiver,
Register scratch1,
Register scratch2,
Label* no_memento_found) {
Label map_check;
Label top_check;
ExternalReference new_space_allocation_top_adr =
ExternalReference::new_space_allocation_top_address(isolate());
const int kMementoMapOffset = JSArray::kSize - kHeapObjectTag;
const int kMementoLastWordOffset =
kMementoMapOffset + AllocationMemento::kSize - kPointerSize;
// Bail out if the object is not in new space.
JumpIfNotInNewSpace(receiver, no_memento_found);
Add(scratch1, receiver, kMementoLastWordOffset);
// If the object is in new space, we need to check whether it is on the same
// page as the current top.
Mov(scratch2, new_space_allocation_top_adr);
Ldr(scratch2, MemOperand(scratch2));
Eor(scratch2, scratch1, scratch2);
Tst(scratch2, ~Page::kPageAlignmentMask);
B(eq, &top_check);
// The object is on a different page than allocation top. Bail out if the
// object sits on the page boundary as no memento can follow and we cannot
// touch the memory following it.
Eor(scratch2, scratch1, receiver);
Tst(scratch2, ~Page::kPageAlignmentMask);
B(ne, no_memento_found);
// Continue with the actual map check.
jmp(&map_check);
// If top is on the same page as the current object, we need to check whether
// we are below top.
bind(&top_check);
Mov(scratch2, new_space_allocation_top_adr);
Ldr(scratch2, MemOperand(scratch2));
Cmp(scratch1, scratch2);
B(ge, no_memento_found);
// Memento map check.
bind(&map_check);
Ldr(scratch1, MemOperand(receiver, kMementoMapOffset));
Cmp(scratch1, Operand(isolate()->factory()->allocation_memento_map()));
}
void MacroAssembler::InNewSpace(Register object,
Condition cond,
Label* branch) {
DCHECK(cond == eq || cond == ne);
UseScratchRegisterScope temps(this);
CheckPageFlag(object, temps.AcquireSameSizeAs(object),
MemoryChunk::kIsInNewSpaceMask, cond, branch);
}
void MacroAssembler::AssertSmi(Register object, BailoutReason reason) {
if (emit_debug_code()) {
STATIC_ASSERT(kSmiTag == 0);
Tst(object, kSmiTagMask);
Check(eq, reason);
}
}
void MacroAssembler::AssertNotSmi(Register object, BailoutReason reason) {
if (emit_debug_code()) {
STATIC_ASSERT(kSmiTag == 0);
Tst(object, kSmiTagMask);
Check(ne, reason);
}
}
void MacroAssembler::AssertName(Register object) {
if (emit_debug_code()) {
AssertNotSmi(object, kOperandIsASmiAndNotAName);
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
Ldr(temp, FieldMemOperand(object, HeapObject::kMapOffset));
CompareInstanceType(temp, temp, LAST_NAME_TYPE);
Check(ls, kOperandIsNotAName);
}
}
void MacroAssembler::AssertFunction(Register object) {
if (emit_debug_code()) {
AssertNotSmi(object, kOperandIsASmiAndNotAFunction);
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
CompareObjectType(object, temp, temp, JS_FUNCTION_TYPE);
Check(eq, kOperandIsNotAFunction);
}
}
void MacroAssembler::AssertBoundFunction(Register object) {
if (emit_debug_code()) {
AssertNotSmi(object, kOperandIsASmiAndNotABoundFunction);
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
CompareObjectType(object, temp, temp, JS_BOUND_FUNCTION_TYPE);
Check(eq, kOperandIsNotABoundFunction);
}
}
void MacroAssembler::AssertGeneratorObject(Register object) {
if (emit_debug_code()) {
AssertNotSmi(object, kOperandIsASmiAndNotAGeneratorObject);
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
CompareObjectType(object, temp, temp, JS_GENERATOR_OBJECT_TYPE);
Check(eq, kOperandIsNotAGeneratorObject);
}
}
void MacroAssembler::AssertReceiver(Register object) {
if (emit_debug_code()) {
AssertNotSmi(object, kOperandIsASmiAndNotAReceiver);
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
CompareObjectType(object, temp, temp, FIRST_JS_RECEIVER_TYPE);
Check(hs, kOperandIsNotAReceiver);
}
}
void MacroAssembler::AssertUndefinedOrAllocationSite(Register object,
Register scratch) {
if (emit_debug_code()) {
Label done_checking;
AssertNotSmi(object);
JumpIfRoot(object, Heap::kUndefinedValueRootIndex, &done_checking);
Ldr(scratch, FieldMemOperand(object, HeapObject::kMapOffset));
CompareRoot(scratch, Heap::kAllocationSiteMapRootIndex);
Assert(eq, kExpectedUndefinedOrCell);
Bind(&done_checking);
}
}
void MacroAssembler::AssertString(Register object) {
if (emit_debug_code()) {
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
STATIC_ASSERT(kSmiTag == 0);
Tst(object, kSmiTagMask);
Check(ne, kOperandIsASmiAndNotAString);
Ldr(temp, FieldMemOperand(object, HeapObject::kMapOffset));
CompareInstanceType(temp, temp, FIRST_NONSTRING_TYPE);
Check(lo, kOperandIsNotAString);
}
}
void MacroAssembler::AssertPositiveOrZero(Register value) {
if (emit_debug_code()) {
Label done;
int sign_bit = value.Is64Bits() ? kXSignBit : kWSignBit;
Tbz(value, sign_bit, &done);
Abort(kUnexpectedNegativeValue);
Bind(&done);
}
}
void MacroAssembler::AssertNotNumber(Register value) {
if (emit_debug_code()) {
STATIC_ASSERT(kSmiTag == 0);
Tst(value, kSmiTagMask);
Check(ne, kOperandIsANumber);
Label done;
JumpIfNotHeapNumber(value, &done);
Abort(kOperandIsANumber);
Bind(&done);
}
}
void MacroAssembler::AssertNumber(Register value) {
if (emit_debug_code()) {
Label done;
JumpIfSmi(value, &done);
JumpIfHeapNumber(value, &done);
Abort(kOperandIsNotANumber);
Bind(&done);
}
}
void MacroAssembler::CallStub(CodeStub* stub, TypeFeedbackId ast_id) {
DCHECK(AllowThisStubCall(stub)); // Stub calls are not allowed in some stubs.
Call(stub->GetCode(), RelocInfo::CODE_TARGET, ast_id);
}
void MacroAssembler::TailCallStub(CodeStub* stub) {
Jump(stub->GetCode(), RelocInfo::CODE_TARGET);
}
void MacroAssembler::CallRuntime(const Runtime::Function* f,
int num_arguments,
SaveFPRegsMode save_doubles) {
// All arguments must be on the stack before this function is called.
// x0 holds the return value after the call.
// Check that the number of arguments matches what the function expects.
// If f->nargs is -1, the function can accept a variable number of arguments.
CHECK(f->nargs < 0 || f->nargs == num_arguments);
// Place the necessary arguments.
Mov(x0, num_arguments);
Mov(x1, ExternalReference(f, isolate()));
CEntryStub stub(isolate(), 1, save_doubles);
CallStub(&stub);
}
void MacroAssembler::CallExternalReference(const ExternalReference& ext,
int num_arguments) {
Mov(x0, num_arguments);
Mov(x1, ext);
CEntryStub stub(isolate(), 1);
CallStub(&stub);
}
void MacroAssembler::JumpToExternalReference(const ExternalReference& builtin,
bool builtin_exit_frame) {
Mov(x1, builtin);
CEntryStub stub(isolate(), 1, kDontSaveFPRegs, kArgvOnStack,
builtin_exit_frame);
Jump(stub.GetCode(), RelocInfo::CODE_TARGET);
}
void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
const Runtime::Function* function = Runtime::FunctionForId(fid);
DCHECK_EQ(1, function->result_size);
if (function->nargs >= 0) {
// TODO(1236192): Most runtime routines don't need the number of
// arguments passed in because it is constant. At some point we
// should remove this need and make the runtime routine entry code
// smarter.
Mov(x0, function->nargs);
}
JumpToExternalReference(ExternalReference(fid, isolate()));
}
void MacroAssembler::InitializeNewString(Register string,
Register length,
Heap::RootListIndex map_index,
Register scratch1,
Register scratch2) {
DCHECK(!AreAliased(string, length, scratch1, scratch2));
LoadRoot(scratch2, map_index);
SmiTag(scratch1, length);
Str(scratch2, FieldMemOperand(string, HeapObject::kMapOffset));
Mov(scratch2, String::kEmptyHashField);
Str(scratch1, FieldMemOperand(string, String::kLengthOffset));
Str(scratch2, FieldMemOperand(string, String::kHashFieldOffset));
}
int MacroAssembler::ActivationFrameAlignment() {
#if V8_HOST_ARCH_ARM64
// Running on the real platform. Use the alignment as mandated by the local
// environment.
// Note: This will break if we ever start generating snapshots on one ARM
// platform for another ARM platform with a different alignment.
return base::OS::ActivationFrameAlignment();
#else // V8_HOST_ARCH_ARM64
// If we are using the simulator then we should always align to the expected
// alignment. As the simulator is used to generate snapshots we do not know
// if the target platform will need alignment, so this is controlled from a
// flag.
return FLAG_sim_stack_alignment;
#endif // V8_HOST_ARCH_ARM64
}
void MacroAssembler::CallCFunction(ExternalReference function,
int num_of_reg_args) {
CallCFunction(function, num_of_reg_args, 0);
}
void MacroAssembler::CallCFunction(ExternalReference function,
int num_of_reg_args,
int num_of_double_args) {
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
Mov(temp, function);
CallCFunction(temp, num_of_reg_args, num_of_double_args);
}
void MacroAssembler::CallCFunction(Register function,
int num_of_reg_args,
int num_of_double_args) {
DCHECK(has_frame());
// We can pass 8 integer arguments in registers. If we need to pass more than
// that, we'll need to implement support for passing them on the stack.
DCHECK(num_of_reg_args <= 8);
// If we're passing doubles, we're limited to the following prototypes
// (defined by ExternalReference::Type):
// BUILTIN_COMPARE_CALL: int f(double, double)
// BUILTIN_FP_FP_CALL: double f(double, double)
// BUILTIN_FP_CALL: double f(double)
// BUILTIN_FP_INT_CALL: double f(double, int)
if (num_of_double_args > 0) {
DCHECK(num_of_reg_args <= 1);
DCHECK((num_of_double_args + num_of_reg_args) <= 2);
}
// If the stack pointer is not csp, we need to derive an aligned csp from the
// current stack pointer.
const Register old_stack_pointer = StackPointer();
if (!csp.Is(old_stack_pointer)) {
AssertStackConsistency();
int sp_alignment = ActivationFrameAlignment();
// The ABI mandates at least 16-byte alignment.
DCHECK(sp_alignment >= 16);
DCHECK(base::bits::IsPowerOfTwo32(sp_alignment));
// The current stack pointer is a callee saved register, and is preserved
// across the call.
DCHECK(kCalleeSaved.IncludesAliasOf(old_stack_pointer));
// Align and synchronize the system stack pointer with jssp.
Bic(csp, old_stack_pointer, sp_alignment - 1);
SetStackPointer(csp);
}
// Call directly. The function called cannot cause a GC, or allow preemption,
// so the return address in the link register stays correct.
Call(function);
if (!csp.Is(old_stack_pointer)) {
if (emit_debug_code()) {
// Because the stack pointer must be aligned on a 16-byte boundary, the
// aligned csp can be up to 12 bytes below the jssp. This is the case
// where we only pushed one W register on top of an aligned jssp.
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
DCHECK(ActivationFrameAlignment() == 16);
Sub(temp, csp, old_stack_pointer);
// We want temp <= 0 && temp >= -12.
Cmp(temp, 0);
Ccmp(temp, -12, NFlag, le);
Check(ge, kTheStackWasCorruptedByMacroAssemblerCall);
}
SetStackPointer(old_stack_pointer);
}
}
void MacroAssembler::Jump(Register target) {
Br(target);
}
void MacroAssembler::Jump(intptr_t target, RelocInfo::Mode rmode,
Condition cond) {
if (cond == nv) return;
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
Label done;
if (cond != al) B(NegateCondition(cond), &done);
Mov(temp, Operand(target, rmode));
Br(temp);
Bind(&done);
}
void MacroAssembler::Jump(Address target, RelocInfo::Mode rmode,
Condition cond) {
DCHECK(!RelocInfo::IsCodeTarget(rmode));
Jump(reinterpret_cast<intptr_t>(target), rmode, cond);
}
void MacroAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
Condition cond) {
DCHECK(RelocInfo::IsCodeTarget(rmode));
AllowDeferredHandleDereference embedding_raw_address;
Jump(reinterpret_cast<intptr_t>(code.location()), rmode, cond);
}
void MacroAssembler::Call(Register target) {
BlockPoolsScope scope(this);
#ifdef DEBUG
Label start_call;
Bind(&start_call);
#endif
Blr(target);
#ifdef DEBUG
AssertSizeOfCodeGeneratedSince(&start_call, CallSize(target));
#endif
}
void MacroAssembler::Call(Label* target) {
BlockPoolsScope scope(this);
#ifdef DEBUG
Label start_call;
Bind(&start_call);
#endif
Bl(target);
#ifdef DEBUG
AssertSizeOfCodeGeneratedSince(&start_call, CallSize(target));
#endif
}
// MacroAssembler::CallSize is sensitive to changes in this function, as it
// requires to know how many instructions are used to branch to the target.
void MacroAssembler::Call(Address target, RelocInfo::Mode rmode) {
BlockPoolsScope scope(this);
#ifdef DEBUG
Label start_call;
Bind(&start_call);
#endif
// Addresses always have 64 bits, so we shouldn't encounter NONE32.
DCHECK(rmode != RelocInfo::NONE32);
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
if (rmode == RelocInfo::NONE64) {
// Addresses are 48 bits so we never need to load the upper 16 bits.
uint64_t imm = reinterpret_cast<uint64_t>(target);
// If we don't use ARM tagged addresses, the 16 higher bits must be 0.
DCHECK(((imm >> 48) & 0xffff) == 0);
movz(temp, (imm >> 0) & 0xffff, 0);
movk(temp, (imm >> 16) & 0xffff, 16);
movk(temp, (imm >> 32) & 0xffff, 32);
} else {
Ldr(temp, Immediate(reinterpret_cast<intptr_t>(target), rmode));
}
Blr(temp);
#ifdef DEBUG
AssertSizeOfCodeGeneratedSince(&start_call, CallSize(target, rmode));
#endif
}
void MacroAssembler::Call(Handle<Code> code,
RelocInfo::Mode rmode,
TypeFeedbackId ast_id) {
#ifdef DEBUG
Label start_call;
Bind(&start_call);
#endif
if ((rmode == RelocInfo::CODE_TARGET) && (!ast_id.IsNone())) {
SetRecordedAstId(ast_id);
rmode = RelocInfo::CODE_TARGET_WITH_ID;
}
AllowDeferredHandleDereference embedding_raw_address;
Call(reinterpret_cast<Address>(code.location()), rmode);
#ifdef DEBUG
// Check the size of the code generated.
AssertSizeOfCodeGeneratedSince(&start_call, CallSize(code, rmode, ast_id));
#endif
}
int MacroAssembler::CallSize(Register target) {
USE(target);
return kInstructionSize;
}
int MacroAssembler::CallSize(Label* target) {
USE(target);
return kInstructionSize;
}
int MacroAssembler::CallSize(Address target, RelocInfo::Mode rmode) {
USE(target);
// Addresses always have 64 bits, so we shouldn't encounter NONE32.
DCHECK(rmode != RelocInfo::NONE32);
if (rmode == RelocInfo::NONE64) {
return kCallSizeWithoutRelocation;
} else {
return kCallSizeWithRelocation;
}
}
int MacroAssembler::CallSize(Handle<Code> code,
RelocInfo::Mode rmode,
TypeFeedbackId ast_id) {
USE(code);
USE(ast_id);
// Addresses always have 64 bits, so we shouldn't encounter NONE32.
DCHECK(rmode != RelocInfo::NONE32);
if (rmode == RelocInfo::NONE64) {
return kCallSizeWithoutRelocation;
} else {
return kCallSizeWithRelocation;
}
}
void MacroAssembler::JumpIfHeapNumber(Register object, Label* on_heap_number,
SmiCheckType smi_check_type) {
Label on_not_heap_number;
if (smi_check_type == DO_SMI_CHECK) {
JumpIfSmi(object, &on_not_heap_number);
}
AssertNotSmi(object);
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
Ldr(temp, FieldMemOperand(object, HeapObject::kMapOffset));
JumpIfRoot(temp, Heap::kHeapNumberMapRootIndex, on_heap_number);
Bind(&on_not_heap_number);
}
void MacroAssembler::JumpIfNotHeapNumber(Register object,
Label* on_not_heap_number,
SmiCheckType smi_check_type) {
if (smi_check_type == DO_SMI_CHECK) {
JumpIfSmi(object, on_not_heap_number);
}
AssertNotSmi(object);
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
Ldr(temp, FieldMemOperand(object, HeapObject::kMapOffset));
JumpIfNotRoot(temp, Heap::kHeapNumberMapRootIndex, on_not_heap_number);
}
void MacroAssembler::TryRepresentDoubleAsInt(Register as_int,
FPRegister value,
FPRegister scratch_d,
Label* on_successful_conversion,
Label* on_failed_conversion) {
// Convert to an int and back again, then compare with the original value.
Fcvtzs(as_int, value);
Scvtf(scratch_d, as_int);
Fcmp(value, scratch_d);
if (on_successful_conversion) {
B(on_successful_conversion, eq);
}
if (on_failed_conversion) {
B(on_failed_conversion, ne);
}
}
void MacroAssembler::TestForMinusZero(DoubleRegister input) {
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
// Floating point -0.0 is kMinInt as an integer, so subtracting 1 (cmp) will
// cause overflow.
Fmov(temp, input);
Cmp(temp, 1);
}
void MacroAssembler::JumpIfMinusZero(DoubleRegister input,
Label* on_negative_zero) {
TestForMinusZero(input);
B(vs, on_negative_zero);
}
void MacroAssembler::JumpIfMinusZero(Register input,
Label* on_negative_zero) {
DCHECK(input.Is64Bits());
// Floating point value is in an integer register. Detect -0.0 by subtracting
// 1 (cmp), which will cause overflow.
Cmp(input, 1);
B(vs, on_negative_zero);
}
void MacroAssembler::ClampInt32ToUint8(Register output, Register input) {
// Clamp the value to [0..255].
Cmp(input.W(), Operand(input.W(), UXTB));
// If input < input & 0xff, it must be < 0, so saturate to 0.
Csel(output.W(), wzr, input.W(), lt);
// If input <= input & 0xff, it must be <= 255. Otherwise, saturate to 255.
Csel(output.W(), output.W(), 255, le);
}
void MacroAssembler::ClampInt32ToUint8(Register in_out) {
ClampInt32ToUint8(in_out, in_out);
}
void MacroAssembler::ClampDoubleToUint8(Register output,
DoubleRegister input,
DoubleRegister dbl_scratch) {
// This conversion follows the WebIDL "[Clamp]" rules for PIXEL types:
// - Inputs lower than 0 (including -infinity) produce 0.
// - Inputs higher than 255 (including +infinity) produce 255.
// Also, it seems that PIXEL types use round-to-nearest rather than
// round-towards-zero.
// Squash +infinity before the conversion, since Fcvtnu will normally
// convert it to 0.
Fmov(dbl_scratch, 255);
Fmin(dbl_scratch, dbl_scratch, input);
// Convert double to unsigned integer. Values less than zero become zero.
// Values greater than 255 have already been clamped to 255.
Fcvtnu(output, dbl_scratch);
}
void MacroAssembler::InitializeFieldsWithFiller(Register current_address,
Register end_address,
Register filler) {
DCHECK(!current_address.Is(csp));
UseScratchRegisterScope temps(this);
Register distance_in_words = temps.AcquireX();
Label done;
// Calculate the distance. If it's <= zero then there's nothing to do.
Subs(distance_in_words, end_address, current_address);
B(le, &done);
// There's at least one field to fill, so do this unconditionally.
Str(filler, MemOperand(current_address));
// If the distance_in_words consists of odd number of words we advance
// start_address by one word, otherwise the pairs loop will ovwerite the
// field that was stored above.
And(distance_in_words, distance_in_words, kPointerSize);
Add(current_address, current_address, distance_in_words);
// Store filler to memory in pairs.
Label loop, entry;
B(&entry);
Bind(&loop);
Stp(filler, filler, MemOperand(current_address, 2 * kPointerSize, PostIndex));
Bind(&entry);
Cmp(current_address, end_address);
B(lo, &loop);
Bind(&done);
}
void MacroAssembler::JumpIfEitherIsNotSequentialOneByteStrings(
Register first, Register second, Register scratch1, Register scratch2,
Label* failure, SmiCheckType smi_check) {
if (smi_check == DO_SMI_CHECK) {
JumpIfEitherSmi(first, second, failure);
} else if (emit_debug_code()) {
DCHECK(smi_check == DONT_DO_SMI_CHECK);
Label not_smi;
JumpIfEitherSmi(first, second, NULL, &not_smi);
// At least one input is a smi, but the flags indicated a smi check wasn't
// needed.
Abort(kUnexpectedSmi);
Bind(&not_smi);
}
// Test that both first and second are sequential one-byte strings.
Ldr(scratch1, FieldMemOperand(first, HeapObject::kMapOffset));
Ldr(scratch2, FieldMemOperand(second, HeapObject::kMapOffset));
Ldrb(scratch1, FieldMemOperand(scratch1, Map::kInstanceTypeOffset));
Ldrb(scratch2, FieldMemOperand(scratch2, Map::kInstanceTypeOffset));
JumpIfEitherInstanceTypeIsNotSequentialOneByte(scratch1, scratch2, scratch1,
scratch2, failure);
}
void MacroAssembler::JumpIfEitherInstanceTypeIsNotSequentialOneByte(
Register first, Register second, Register scratch1, Register scratch2,
Label* failure) {
DCHECK(!AreAliased(scratch1, second));
DCHECK(!AreAliased(scratch1, scratch2));
const int kFlatOneByteStringMask =
kIsNotStringMask | kStringEncodingMask | kStringRepresentationMask;
const int kFlatOneByteStringTag =
kStringTag | kOneByteStringTag | kSeqStringTag;
And(scratch1, first, kFlatOneByteStringMask);
And(scratch2, second, kFlatOneByteStringMask);
Cmp(scratch1, kFlatOneByteStringTag);
Ccmp(scratch2, kFlatOneByteStringTag, NoFlag, eq);
B(ne, failure);
}
void MacroAssembler::JumpIfInstanceTypeIsNotSequentialOneByte(Register type,
Register scratch,
Label* failure) {
const int kFlatOneByteStringMask =
kIsNotStringMask | kStringEncodingMask | kStringRepresentationMask;
const int kFlatOneByteStringTag =
kStringTag | kOneByteStringTag | kSeqStringTag;
And(scratch, type, kFlatOneByteStringMask);
Cmp(scratch, kFlatOneByteStringTag);
B(ne, failure);
}
void MacroAssembler::JumpIfBothInstanceTypesAreNotSequentialOneByte(
Register first, Register second, Register scratch1, Register scratch2,
Label* failure) {
DCHECK(!AreAliased(first, second, scratch1, scratch2));
const int kFlatOneByteStringMask =
kIsNotStringMask | kStringEncodingMask | kStringRepresentationMask;
const int kFlatOneByteStringTag =
kStringTag | kOneByteStringTag | kSeqStringTag;
And(scratch1, first, kFlatOneByteStringMask);
And(scratch2, second, kFlatOneByteStringMask);
Cmp(scratch1, kFlatOneByteStringTag);
Ccmp(scratch2, kFlatOneByteStringTag, NoFlag, eq);
B(ne, failure);
}
void MacroAssembler::JumpIfNotUniqueNameInstanceType(Register type,
Label* not_unique_name) {
STATIC_ASSERT((kInternalizedTag == 0) && (kStringTag == 0));
// if ((type is string && type is internalized) || type == SYMBOL_TYPE) {
// continue
// } else {
// goto not_unique_name
// }
Tst(type, kIsNotStringMask | kIsNotInternalizedMask);
Ccmp(type, SYMBOL_TYPE, ZFlag, ne);
B(ne, not_unique_name);
}
void MacroAssembler::PrepareForTailCall(const ParameterCount& callee_args_count,
Register caller_args_count_reg,
Register scratch0, Register scratch1) {
#if DEBUG
if (callee_args_count.is_reg()) {
DCHECK(!AreAliased(callee_args_count.reg(), caller_args_count_reg, scratch0,
scratch1));
} else {
DCHECK(!AreAliased(caller_args_count_reg, scratch0, scratch1));
}
#endif
// Calculate the end of destination area where we will put the arguments
// after we drop current frame. We add kPointerSize to count the receiver
// argument which is not included into formal parameters count.
Register dst_reg = scratch0;
__ add(dst_reg, fp, Operand(caller_args_count_reg, LSL, kPointerSizeLog2));
__ add(dst_reg, dst_reg,
Operand(StandardFrameConstants::kCallerSPOffset + kPointerSize));
Register src_reg = caller_args_count_reg;
// Calculate the end of source area. +kPointerSize is for the receiver.
if (callee_args_count.is_reg()) {
add(src_reg, jssp, Operand(callee_args_count.reg(), LSL, kPointerSizeLog2));
add(src_reg, src_reg, Operand(kPointerSize));
} else {
add(src_reg, jssp,
Operand((callee_args_count.immediate() + 1) * kPointerSize));
}
if (FLAG_debug_code) {
__ Cmp(src_reg, dst_reg);
__ Check(lo, kStackAccessBelowStackPointer);
}
// Restore caller's frame pointer and return address now as they will be
// overwritten by the copying loop.
__ Ldr(lr, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
__ Ldr(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
// Now copy callee arguments to the caller frame going backwards to avoid
// callee arguments corruption (source and destination areas could overlap).
// Both src_reg and dst_reg are pointing to the word after the one to copy,
// so they must be pre-decremented in the loop.
Register tmp_reg = scratch1;
Label loop, entry;
__ B(&entry);
__ bind(&loop);
__ Ldr(tmp_reg, MemOperand(src_reg, -kPointerSize, PreIndex));
__ Str(tmp_reg, MemOperand(dst_reg, -kPointerSize, PreIndex));
__ bind(&entry);
__ Cmp(jssp, src_reg);
__ B(ne, &loop);
// Leave current frame.
__ Mov(jssp, dst_reg);
__ SetStackPointer(jssp);
__ AssertStackConsistency();
}
void MacroAssembler::InvokePrologue(const ParameterCount& expected,
const ParameterCount& actual,
Label* done,
InvokeFlag flag,
bool* definitely_mismatches,
const CallWrapper& call_wrapper) {
bool definitely_matches = false;
*definitely_mismatches = false;
Label regular_invoke;
// Check whether the expected and actual arguments count match. If not,
// setup registers according to contract with ArgumentsAdaptorTrampoline:
// x0: actual arguments count.
// x1: function (passed through to callee).
// x2: expected arguments count.
// The code below is made a lot easier because the calling code already sets
// up actual and expected registers according to the contract if values are
// passed in registers.
DCHECK(actual.is_immediate() || actual.reg().is(x0));
DCHECK(expected.is_immediate() || expected.reg().is(x2));
if (expected.is_immediate()) {
DCHECK(actual.is_immediate());
Mov(x0, actual.immediate());
if (expected.immediate() == actual.immediate()) {
definitely_matches = true;
} else {
if (expected.immediate() ==
SharedFunctionInfo::kDontAdaptArgumentsSentinel) {
// Don't worry about adapting arguments for builtins that
// don't want that done. Skip adaption code by making it look
// like we have a match between expected and actual number of
// arguments.
definitely_matches = true;
} else {
*definitely_mismatches = true;
// Set up x2 for the argument adaptor.
Mov(x2, expected.immediate());
}
}
} else { // expected is a register.
Operand actual_op = actual.is_immediate() ? Operand(actual.immediate())
: Operand(actual.reg());
Mov(x0, actual_op);
// If actual == expected perform a regular invocation.
Cmp(expected.reg(), actual_op);
B(eq, &regular_invoke);
}
// If the argument counts may mismatch, generate a call to the argument
// adaptor.
if (!definitely_matches) {
Handle<Code> adaptor =
isolate()->builtins()->ArgumentsAdaptorTrampoline();
if (flag == CALL_FUNCTION) {
call_wrapper.BeforeCall(CallSize(adaptor));
Call(adaptor);
call_wrapper.AfterCall();
if (!*definitely_mismatches) {
// If the arg counts don't match, no extra code is emitted by
// MAsm::InvokeFunctionCode and we can just fall through.
B(done);
}
} else {
Jump(adaptor, RelocInfo::CODE_TARGET);
}
}
Bind(&regular_invoke);
}
void MacroAssembler::FloodFunctionIfStepping(Register fun, Register new_target,
const ParameterCount& expected,
const ParameterCount& actual) {
Label skip_flooding;
ExternalReference last_step_action =
ExternalReference::debug_last_step_action_address(isolate());
STATIC_ASSERT(StepFrame > StepIn);
Mov(x4, Operand(last_step_action));
Ldrsb(x4, MemOperand(x4));
CompareAndBranch(x4, Operand(StepIn), lt, &skip_flooding);
{
FrameScope frame(this,
has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
if (expected.is_reg()) {
SmiTag(expected.reg());
Push(expected.reg());
}
if (actual.is_reg()) {
SmiTag(actual.reg());
Push(actual.reg());
}
if (new_target.is_valid()) {
Push(new_target);
}
Push(fun);
Push(fun);
CallRuntime(Runtime::kDebugPrepareStepInIfStepping);
Pop(fun);
if (new_target.is_valid()) {
Pop(new_target);
}
if (actual.is_reg()) {
Pop(actual.reg());
SmiUntag(actual.reg());
}
if (expected.is_reg()) {
Pop(expected.reg());
SmiUntag(expected.reg());
}
}
bind(&skip_flooding);
}
void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
const ParameterCount& expected,
const ParameterCount& actual,
InvokeFlag flag,
const CallWrapper& call_wrapper) {
// You can't call a function without a valid frame.
DCHECK(flag == JUMP_FUNCTION || has_frame());
DCHECK(function.is(x1));
DCHECK_IMPLIES(new_target.is_valid(), new_target.is(x3));
FloodFunctionIfStepping(function, new_target, expected, actual);
// Clear the new.target register if not given.
if (!new_target.is_valid()) {
LoadRoot(x3, Heap::kUndefinedValueRootIndex);
}
Label done;
bool definitely_mismatches = false;
InvokePrologue(expected, actual, &done, flag, &definitely_mismatches,
call_wrapper);
// If we are certain that actual != expected, then we know InvokePrologue will
// have handled the call through the argument adaptor mechanism.
// The called function expects the call kind in x5.
if (!definitely_mismatches) {
// We call indirectly through the code field in the function to
// allow recompilation to take effect without changing any of the
// call sites.
Register code = x4;
Ldr(code, FieldMemOperand(function, JSFunction::kCodeEntryOffset));
if (flag == CALL_FUNCTION) {
call_wrapper.BeforeCall(CallSize(code));
Call(code);
call_wrapper.AfterCall();
} else {
DCHECK(flag == JUMP_FUNCTION);
Jump(code);
}
}
// Continue here if InvokePrologue does handle the invocation due to
// mismatched parameter counts.
Bind(&done);
}
void MacroAssembler::InvokeFunction(Register function,
Register new_target,
const ParameterCount& actual,
InvokeFlag flag,
const CallWrapper& call_wrapper) {
// You can't call a function without a valid frame.
DCHECK(flag == JUMP_FUNCTION || has_frame());
// Contract with called JS functions requires that function is passed in x1.
// (See FullCodeGenerator::Generate().)
DCHECK(function.is(x1));
Register expected_reg = x2;
Ldr(cp, FieldMemOperand(function, JSFunction::kContextOffset));
// The number of arguments is stored as an int32_t, and -1 is a marker
// (SharedFunctionInfo::kDontAdaptArgumentsSentinel), so we need sign
// extension to correctly handle it.
Ldr(expected_reg, FieldMemOperand(function,
JSFunction::kSharedFunctionInfoOffset));
Ldrsw(expected_reg,
FieldMemOperand(expected_reg,
SharedFunctionInfo::kFormalParameterCountOffset));
ParameterCount expected(expected_reg);
InvokeFunctionCode(function, new_target, expected, actual, flag,
call_wrapper);
}
void MacroAssembler::InvokeFunction(Register function,
const ParameterCount& expected,
const ParameterCount& actual,
InvokeFlag flag,
const CallWrapper& call_wrapper) {
// You can't call a function without a valid frame.
DCHECK(flag == JUMP_FUNCTION || has_frame());
// Contract with called JS functions requires that function is passed in x1.
// (See FullCodeGenerator::Generate().)
DCHECK(function.Is(x1));
// Set up the context.
Ldr(cp, FieldMemOperand(function, JSFunction::kContextOffset));
InvokeFunctionCode(function, no_reg, expected, actual, flag, call_wrapper);
}
void MacroAssembler::InvokeFunction(Handle<JSFunction> function,
const ParameterCount& expected,
const ParameterCount& actual,
InvokeFlag flag,
const CallWrapper& call_wrapper) {
// Contract with called JS functions requires that function is passed in x1.
// (See FullCodeGenerator::Generate().)
__ LoadObject(x1, function);
InvokeFunction(x1, expected, actual, flag, call_wrapper);
}
void MacroAssembler::TryConvertDoubleToInt64(Register result,
DoubleRegister double_input,
Label* done) {
// Try to convert with an FPU convert instruction. It's trivial to compute
// the modulo operation on an integer register so we convert to a 64-bit
// integer.
//
// Fcvtzs will saturate to INT64_MIN (0x800...00) or INT64_MAX (0x7ff...ff)
// when the double is out of range. NaNs and infinities will be converted to 0
// (as ECMA-262 requires).
Fcvtzs(result.X(),