Google Git
Sign in
chromium / v8 / v8 / dc9ed94efdac30bdbe88e81f4cf08783c1dc952f / . / src / codegen / mips64 / macro-assembler-mips64.cc
blob: 61d1dd059a2e387c800ef436b31e73af5e5cba03 [file] [log] [blame]
// Copyright 2012 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <limits.h> // For LONG_MIN, LONG_MAX.
#if V8_TARGET_ARCH_MIPS64
#include "src/base/bits.h"
#include "src/base/division-by-constant.h"
#include "src/codegen/assembler-inl.h"
#include "src/codegen/callable.h"
#include "src/codegen/code-factory.h"
#include "src/codegen/external-reference-table.h"
#include "src/codegen/interface-descriptors-inl.h"
#include "src/codegen/macro-assembler.h"
#include "src/codegen/register-configuration.h"
#include "src/debug/debug.h"
#include "src/deoptimizer/deoptimizer.h"
#include "src/execution/frames-inl.h"
#include "src/heap/memory-chunk.h"
#include "src/init/bootstrapper.h"
#include "src/logging/counters.h"
#include "src/objects/heap-number.h"
#include "src/runtime/runtime.h"
#include "src/snapshot/snapshot.h"
#if V8_ENABLE_WEBASSEMBLY
#include "src/wasm/wasm-code-manager.h"
#endif // V8_ENABLE_WEBASSEMBLY
// Satisfy cpplint check, but don't include platform-specific header. It is
// included recursively via macro-assembler.h.
#if 0
#include "src/codegen/mips64/macro-assembler-mips64.h"
#endif
namespace v8 {
namespace internal {
static inline bool IsZero(const Operand& rt) {
if (rt.is_reg()) {
return rt.rm() == zero_reg;
} else {
return rt.immediate() == 0;
}
}
int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
Register exclusion1,
Register exclusion2,
Register exclusion3) const {
int bytes = 0;
RegList exclusions = {exclusion1, exclusion2, exclusion3};
RegList list = kJSCallerSaved - exclusions;
bytes += list.Count() * kPointerSize;
if (fp_mode == SaveFPRegsMode::kSave) {
bytes += kCallerSavedFPU.Count() * kDoubleSize;
}
return bytes;
}
int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
Register exclusion2, Register exclusion3) {
ASM_CODE_COMMENT(this);
int bytes = 0;
RegList exclusions = {exclusion1, exclusion2, exclusion3};
RegList list = kJSCallerSaved - exclusions;
MultiPush(list);
bytes += list.Count() * kPointerSize;
if (fp_mode == SaveFPRegsMode::kSave) {
MultiPushFPU(kCallerSavedFPU);
bytes += kCallerSavedFPU.Count() * kDoubleSize;
}
return bytes;
}
int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
Register exclusion2, Register exclusion3) {
ASM_CODE_COMMENT(this);
int bytes = 0;
if (fp_mode == SaveFPRegsMode::kSave) {
MultiPopFPU(kCallerSavedFPU);
bytes += kCallerSavedFPU.Count() * kDoubleSize;
}
RegList exclusions = {exclusion1, exclusion2, exclusion3};
RegList list = kJSCallerSaved - exclusions;
MultiPop(list);
bytes += list.Count() * kPointerSize;
return bytes;
}
void TurboAssembler::LoadRoot(Register destination, RootIndex index) {
Ld(destination, MemOperand(s6, RootRegisterOffsetForRootIndex(index)));
}
void TurboAssembler::LoadRoot(Register destination, RootIndex index,
Condition cond, Register src1,
const Operand& src2) {
Branch(2, NegateCondition(cond), src1, src2);
Ld(destination, MemOperand(s6, RootRegisterOffsetForRootIndex(index)));
}
void TurboAssembler::PushCommonFrame(Register marker_reg) {
if (marker_reg.is_valid()) {
Push(ra, fp, marker_reg);
Daddu(fp, sp, Operand(kPointerSize));
} else {
Push(ra, fp);
mov(fp, sp);
}
}
void TurboAssembler::PushStandardFrame(Register function_reg) {
int offset = -StandardFrameConstants::kContextOffset;
if (function_reg.is_valid()) {
Push(ra, fp, cp, function_reg, kJavaScriptCallArgCountRegister);
offset += 2 * kPointerSize;
} else {
Push(ra, fp, cp, kJavaScriptCallArgCountRegister);
offset += kPointerSize;
}
Daddu(fp, sp, Operand(offset));
}
int MacroAssembler::SafepointRegisterStackIndex(int reg_code) {
// The registers are pushed starting with the highest encoding,
// which means that lowest encodings are closest to the stack pointer.
return kSafepointRegisterStackIndexMap[reg_code];
}
// Clobbers object, dst, value, and ra, if (ra_status == kRAHasBeenSaved)
// The register 'object' contains a heap object pointer. The heap object
// tag is shifted away.
void MacroAssembler::RecordWriteField(Register object, int offset,
Register value, Register dst,
RAStatus ra_status,
SaveFPRegsMode save_fp,
RememberedSetAction remembered_set_action,
SmiCheck smi_check) {
ASM_CODE_COMMENT(this);
DCHECK(!AreAliased(value, dst, t8, object));
// First, check if a write barrier is even needed. The tests below
// catch stores of Smis.
Label done;
// Skip barrier if writing a smi.
if (smi_check == SmiCheck::kInline) {
JumpIfSmi(value, &done);
}
// Although the object register is tagged, the offset is relative to the start
// of the object, so offset must be a multiple of kPointerSize.
DCHECK(IsAligned(offset, kPointerSize));
Daddu(dst, object, Operand(offset - kHeapObjectTag));
if (FLAG_debug_code) {
BlockTrampolinePoolScope block_trampoline_pool(this);
Label ok;
And(t8, dst, Operand(kPointerSize - 1));
Branch(&ok, eq, t8, Operand(zero_reg));
stop();
bind(&ok);
}
RecordWrite(object, dst, value, ra_status, save_fp, remembered_set_action,
SmiCheck::kOmit);
bind(&done);
// Clobber clobbered input registers when running with the debug-code flag
// turned on to provoke errors.
if (FLAG_debug_code) {
li(value, Operand(bit_cast<int64_t>(kZapValue + 4)));
li(dst, Operand(bit_cast<int64_t>(kZapValue + 8)));
}
}
void TurboAssembler::MaybeSaveRegisters(RegList registers) {
if (registers.is_empty()) return;
MultiPush(registers);
}
void TurboAssembler::MaybeRestoreRegisters(RegList registers) {
if (registers.is_empty()) return;
MultiPop(registers);
}
void TurboAssembler::CallEphemeronKeyBarrier(Register object,
Register slot_address,
SaveFPRegsMode fp_mode) {
ASM_CODE_COMMENT(this);
DCHECK(!AreAliased(object, slot_address));
RegList registers =
WriteBarrierDescriptor::ComputeSavedRegisters(object, slot_address);
MaybeSaveRegisters(registers);
Register object_parameter = WriteBarrierDescriptor::ObjectRegister();
Register slot_address_parameter =
WriteBarrierDescriptor::SlotAddressRegister();
Push(object);
Push(slot_address);
Pop(slot_address_parameter);
Pop(object_parameter);
Call(isolate()->builtins()->code_handle(
Builtins::GetEphemeronKeyBarrierStub(fp_mode)),
RelocInfo::CODE_TARGET);
MaybeRestoreRegisters(registers);
}
void TurboAssembler::CallRecordWriteStubSaveRegisters(
Register object, Register slot_address,
RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
StubCallMode mode) {
DCHECK(!AreAliased(object, slot_address));
RegList registers =
WriteBarrierDescriptor::ComputeSavedRegisters(object, slot_address);
MaybeSaveRegisters(registers);
Register object_parameter = WriteBarrierDescriptor::ObjectRegister();
Register slot_address_parameter =
WriteBarrierDescriptor::SlotAddressRegister();
Push(object);
Push(slot_address);
Pop(slot_address_parameter);
Pop(object_parameter);
CallRecordWriteStub(object_parameter, slot_address_parameter,
remembered_set_action, fp_mode, mode);
MaybeRestoreRegisters(registers);
}
void TurboAssembler::CallRecordWriteStub(
Register object, Register slot_address,
RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
StubCallMode mode) {
// Use CallRecordWriteStubSaveRegisters if the object and slot registers
// need to be caller saved.
DCHECK_EQ(WriteBarrierDescriptor::ObjectRegister(), object);
DCHECK_EQ(WriteBarrierDescriptor::SlotAddressRegister(), slot_address);
#if V8_ENABLE_WEBASSEMBLY
if (mode == StubCallMode::kCallWasmRuntimeStub) {
auto wasm_target =
wasm::WasmCode::GetRecordWriteStub(remembered_set_action, fp_mode);
Call(wasm_target, RelocInfo::WASM_STUB_CALL);
#else
if (false) {
#endif
} else {
auto builtin = Builtins::GetRecordWriteStub(remembered_set_action, fp_mode);
if (options().inline_offheap_trampolines) {
// Inline the trampoline.
RecordCommentForOffHeapTrampoline(builtin);
li(t9, Operand(BuiltinEntry(builtin), RelocInfo::OFF_HEAP_TARGET));
Call(t9);
RecordComment("]");
} else {
Handle<Code> code_target = isolate()->builtins()->code_handle(builtin);
Call(code_target, RelocInfo::CODE_TARGET);
}
}
}
// Clobbers object, address, value, and ra, if (ra_status == kRAHasBeenSaved)
// The register 'object' contains a heap object pointer. The heap object
// tag is shifted away.
void MacroAssembler::RecordWrite(Register object, Register address,
Register value, RAStatus ra_status,
SaveFPRegsMode fp_mode,
RememberedSetAction remembered_set_action,
SmiCheck smi_check) {
DCHECK(!AreAliased(object, address, value, t8));
DCHECK(!AreAliased(object, address, value, t9));
if (FLAG_debug_code) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(!AreAliased(object, value, scratch));
Ld(scratch, MemOperand(address));
Assert(eq, AbortReason::kWrongAddressOrValuePassedToRecordWrite, scratch,
Operand(value));
}
if ((remembered_set_action == RememberedSetAction::kOmit &&
!FLAG_incremental_marking) ||
FLAG_disable_write_barriers) {
return;
}
// First, check if a write barrier is even needed. The tests below
// catch stores of smis and stores into the young generation.
Label done;
if (smi_check == SmiCheck::kInline) {
DCHECK_EQ(0, kSmiTag);
JumpIfSmi(value, &done);
}
CheckPageFlag(value,
value, // Used as scratch.
MemoryChunk::kPointersToHereAreInterestingMask, eq, &done);
CheckPageFlag(object,
value, // Used as scratch.
MemoryChunk::kPointersFromHereAreInterestingMask, eq, &done);
// Record the actual write.
if (ra_status == kRAHasNotBeenSaved) {
push(ra);
}
Register slot_address = WriteBarrierDescriptor::SlotAddressRegister();
DCHECK(!AreAliased(object, slot_address, value));
mov(slot_address, address);
CallRecordWriteStub(object, slot_address, remembered_set_action, fp_mode);
if (ra_status == kRAHasNotBeenSaved) {
pop(ra);
}
bind(&done);
// Clobber clobbered registers when running with the debug-code flag
// turned on to provoke errors.
if (FLAG_debug_code) {
li(address, Operand(bit_cast<int64_t>(kZapValue + 12)));
li(value, Operand(bit_cast<int64_t>(kZapValue + 16)));
li(slot_address, Operand(bit_cast<int64_t>(kZapValue + 20)));
}
}
// ---------------------------------------------------------------------------
// Instruction macros.
void TurboAssembler::Addu(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
addu(rd, rs, rt.rm());
} else {
if (is_int16(rt.immediate()) && !MustUseReg(rt.rmode())) {
addiu(rd, rs, static_cast<int32_t>(rt.immediate()));
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
addu(rd, rs, scratch);
}
}
}
void TurboAssembler::Daddu(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
daddu(rd, rs, rt.rm());
} else {
if (is_int16(rt.immediate()) && !MustUseReg(rt.rmode())) {
daddiu(rd, rs, static_cast<int32_t>(rt.immediate()));
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
daddu(rd, rs, scratch);
}
}
}
void TurboAssembler::Subu(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
subu(rd, rs, rt.rm());
} else {
DCHECK(is_int32(rt.immediate()));
if (is_int16(-rt.immediate()) && !MustUseReg(rt.rmode())) {
addiu(rd, rs,
static_cast<int32_t>(
-rt.immediate())); // No subiu instr, use addiu(x, y, -imm).
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
if (-rt.immediate() >> 16 == 0 && !MustUseReg(rt.rmode())) {
// Use load -imm and addu when loading -imm generates one instruction.
li(scratch, -rt.immediate());
addu(rd, rs, scratch);
} else {
// li handles the relocation.
li(scratch, rt);
subu(rd, rs, scratch);
}
}
}
}
void TurboAssembler::Dsubu(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
dsubu(rd, rs, rt.rm());
} else if (is_int16(-rt.immediate()) && !MustUseReg(rt.rmode())) {
daddiu(rd, rs,
static_cast<int32_t>(
-rt.immediate())); // No dsubiu instr, use daddiu(x, y, -imm).
} else {
DCHECK(rs != at);
int li_count = InstrCountForLi64Bit(rt.immediate());
int li_neg_count = InstrCountForLi64Bit(-rt.immediate());
if (li_neg_count < li_count && !MustUseReg(rt.rmode())) {
// Use load -imm and daddu when loading -imm generates one instruction.
DCHECK(rt.immediate() != std::numeric_limits<int32_t>::min());
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
li(scratch, Operand(-rt.immediate()));
Daddu(rd, rs, scratch);
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
li(scratch, rt);
dsubu(rd, rs, scratch);
}
}
}
void TurboAssembler::Mul(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
mul(rd, rs, rt.rm());
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
mul(rd, rs, scratch);
}
}
void TurboAssembler::Mulh(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
if (kArchVariant != kMips64r6) {
mult(rs, rt.rm());
mfhi(rd);
} else {
muh(rd, rs, rt.rm());
}
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
if (kArchVariant != kMips64r6) {
mult(rs, scratch);
mfhi(rd);
} else {
muh(rd, rs, scratch);
}
}
}
void TurboAssembler::Mulhu(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
if (kArchVariant != kMips64r6) {
multu(rs, rt.rm());
mfhi(rd);
} else {
muhu(rd, rs, rt.rm());
}
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
if (kArchVariant != kMips64r6) {
multu(rs, scratch);
mfhi(rd);
} else {
muhu(rd, rs, scratch);
}
}
}
void TurboAssembler::Dmul(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
if (kArchVariant == kMips64r6) {
dmul(rd, rs, rt.rm());
} else {
dmult(rs, rt.rm());
mflo(rd);
}
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
if (kArchVariant == kMips64r6) {
dmul(rd, rs, scratch);
} else {
dmult(rs, scratch);
mflo(rd);
}
}
}
void TurboAssembler::Dmulh(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
if (kArchVariant == kMips64r6) {
dmuh(rd, rs, rt.rm());
} else {
dmult(rs, rt.rm());
mfhi(rd);
}
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
if (kArchVariant == kMips64r6) {
dmuh(rd, rs, scratch);
} else {
dmult(rs, scratch);
mfhi(rd);
}
}
}
void TurboAssembler::Mult(Register rs, const Operand& rt) {
if (rt.is_reg()) {
mult(rs, rt.rm());
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
mult(rs, scratch);
}
}
void TurboAssembler::Dmult(Register rs, const Operand& rt) {
if (rt.is_reg()) {
dmult(rs, rt.rm());
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
dmult(rs, scratch);
}
}
void TurboAssembler::Multu(Register rs, const Operand& rt) {
if (rt.is_reg()) {
multu(rs, rt.rm());
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
multu(rs, scratch);
}
}
void TurboAssembler::Dmultu(Register rs, const Operand& rt) {
if (rt.is_reg()) {
dmultu(rs, rt.rm());
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
dmultu(rs, scratch);
}
}
void TurboAssembler::Div(Register rs, const Operand& rt) {
if (rt.is_reg()) {
div(rs, rt.rm());
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
div(rs, scratch);
}
}
void TurboAssembler::Div(Register res, Register rs, const Operand& rt) {
if (rt.is_reg()) {
if (kArchVariant != kMips64r6) {
div(rs, rt.rm());
mflo(res);
} else {
div(res, rs, rt.rm());
}
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
if (kArchVariant != kMips64r6) {
div(rs, scratch);
mflo(res);
} else {
div(res, rs, scratch);
}
}
}
void TurboAssembler::Mod(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
if (kArchVariant != kMips64r6) {
div(rs, rt.rm());
mfhi(rd);
} else {
mod(rd, rs, rt.rm());
}
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
if (kArchVariant != kMips64r6) {
div(rs, scratch);
mfhi(rd);
} else {
mod(rd, rs, scratch);
}
}
}
void TurboAssembler::Modu(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
if (kArchVariant != kMips64r6) {
divu(rs, rt.rm());
mfhi(rd);
} else {
modu(rd, rs, rt.rm());
}
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
if (kArchVariant != kMips64r6) {
divu(rs, scratch);
mfhi(rd);
} else {
modu(rd, rs, scratch);
}
}
}
void TurboAssembler::Ddiv(Register rs, const Operand& rt) {
if (rt.is_reg()) {
ddiv(rs, rt.rm());
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
ddiv(rs, scratch);
}
}
void TurboAssembler::Ddiv(Register rd, Register rs, const Operand& rt) {
if (kArchVariant != kMips64r6) {
if (rt.is_reg()) {
ddiv(rs, rt.rm());
mflo(rd);
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
ddiv(rs, scratch);
mflo(rd);
}
} else {
if (rt.is_reg()) {
ddiv(rd, rs, rt.rm());
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
ddiv(rd, rs, scratch);
}
}
}
void TurboAssembler::Divu(Register rs, const Operand& rt) {
if (rt.is_reg()) {
divu(rs, rt.rm());
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
divu(rs, scratch);
}
}
void TurboAssembler::Divu(Register res, Register rs, const Operand& rt) {
if (rt.is_reg()) {
if (kArchVariant != kMips64r6) {
divu(rs, rt.rm());
mflo(res);
} else {
divu(res, rs, rt.rm());
}
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
if (kArchVariant != kMips64r6) {
divu(rs, scratch);
mflo(res);
} else {
divu(res, rs, scratch);
}
}
}
void TurboAssembler::Ddivu(Register rs, const Operand& rt) {
if (rt.is_reg()) {
ddivu(rs, rt.rm());
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
ddivu(rs, scratch);
}
}
void TurboAssembler::Ddivu(Register res, Register rs, const Operand& rt) {
if (rt.is_reg()) {
if (kArchVariant != kMips64r6) {
ddivu(rs, rt.rm());
mflo(res);
} else {
ddivu(res, rs, rt.rm());
}
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
if (kArchVariant != kMips64r6) {
ddivu(rs, scratch);
mflo(res);
} else {
ddivu(res, rs, scratch);
}
}
}
void TurboAssembler::Dmod(Register rd, Register rs, const Operand& rt) {
if (kArchVariant != kMips64r6) {
if (rt.is_reg()) {
ddiv(rs, rt.rm());
mfhi(rd);
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
ddiv(rs, scratch);
mfhi(rd);
}
} else {
if (rt.is_reg()) {
dmod(rd, rs, rt.rm());
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
dmod(rd, rs, scratch);
}
}
}
void TurboAssembler::Dmodu(Register rd, Register rs, const Operand& rt) {
if (kArchVariant != kMips64r6) {
if (rt.is_reg()) {
ddivu(rs, rt.rm());
mfhi(rd);
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
ddivu(rs, scratch);
mfhi(rd);
}
} else {
if (rt.is_reg()) {
dmodu(rd, rs, rt.rm());
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
dmodu(rd, rs, scratch);
}
}
}
void TurboAssembler::And(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
and_(rd, rs, rt.rm());
} else {
if (is_uint16(rt.immediate()) && !MustUseReg(rt.rmode())) {
andi(rd, rs, static_cast<int32_t>(rt.immediate()));
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
and_(rd, rs, scratch);
}
}
}
void TurboAssembler::Or(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
or_(rd, rs, rt.rm());
} else {
if (is_uint16(rt.immediate()) && !MustUseReg(rt.rmode())) {
ori(rd, rs, static_cast<int32_t>(rt.immediate()));
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
or_(rd, rs, scratch);
}
}
}
void TurboAssembler::Xor(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
xor_(rd, rs, rt.rm());
} else {
if (is_uint16(rt.immediate()) && !MustUseReg(rt.rmode())) {
xori(rd, rs, static_cast<int32_t>(rt.immediate()));
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
xor_(rd, rs, scratch);
}
}
}
void TurboAssembler::Nor(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
nor(rd, rs, rt.rm());
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
DCHECK(rs != scratch);
li(scratch, rt);
nor(rd, rs, scratch);
}
}
void TurboAssembler::Neg(Register rs, const Operand& rt) {
dsubu(rs, zero_reg, rt.rm());
}
void TurboAssembler::Slt(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
slt(rd, rs, rt.rm());
} else {
if (is_int16(rt.immediate()) && !MustUseReg(rt.rmode())) {
slti(rd, rs, static_cast<int32_t>(rt.immediate()));
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
BlockTrampolinePoolScope block_trampoline_pool(this);
Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
DCHECK(rs != scratch);
li(scratch, rt);
slt(rd, rs, scratch);
}
}
}
void TurboAssembler::Sltu(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
sltu(rd, rs, rt.rm());
} else {
const uint64_t int16_min = std::numeric_limits<int16_t>::min();
if (is_uint15(rt.immediate()) && !MustUseReg(rt.rmode())) {
// Imm range is: [0, 32767].
sltiu(rd, rs, static_cast<int32_t>(rt.immediate()));
} else if (is_uint15(rt.immediate() - int16_min) &&
!MustUseReg(rt.rmode())) {
// Imm range is: [max_unsigned-32767,max_unsigned].
sltiu(rd, rs, static_cast<uint16_t>(rt.immediate()));
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
BlockTrampolinePoolScope block_trampoline_pool(this);
Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
DCHECK(rs != scratch);
li(scratch, rt);
sltu(rd, rs, scratch);
}
}
}
void TurboAssembler::Sle(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
slt(rd, rt.rm(), rs);
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
BlockTrampolinePoolScope block_trampoline_pool(this);
DCHECK(rs != scratch);
li(scratch, rt);
slt(rd, scratch, rs);
}
xori(rd, rd, 1);
}
void TurboAssembler::Sleu(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
sltu(rd, rt.rm(), rs);
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
BlockTrampolinePoolScope block_trampoline_pool(this);
DCHECK(rs != scratch);
li(scratch, rt);
sltu(rd, scratch, rs);
}
xori(rd, rd, 1);
}
void TurboAssembler::Sge(Register rd, Register rs, const Operand& rt) {
Slt(rd, rs, rt);
xori(rd, rd, 1);
}
void TurboAssembler::Sgeu(Register rd, Register rs, const Operand& rt) {
Sltu(rd, rs, rt);
xori(rd, rd, 1);
}
void TurboAssembler::Sgt(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
slt(rd, rt.rm(), rs);
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
BlockTrampolinePoolScope block_trampoline_pool(this);
DCHECK(rs != scratch);
li(scratch, rt);
slt(rd, scratch, rs);
}
}
void TurboAssembler::Sgtu(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
sltu(rd, rt.rm(), rs);
} else {
// li handles the relocation.
UseScratchRegisterScope temps(this);
Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
BlockTrampolinePoolScope block_trampoline_pool(this);
DCHECK(rs != scratch);
li(scratch, rt);
sltu(rd, scratch, rs);
}
}
void TurboAssembler::Ror(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
rotrv(rd, rs, rt.rm());
} else {
int64_t ror_value = rt.immediate() % 32;
if (ror_value < 0) {
ror_value += 32;
}
rotr(rd, rs, ror_value);
}
}
void TurboAssembler::Dror(Register rd, Register rs, const Operand& rt) {
if (rt.is_reg()) {
drotrv(rd, rs, rt.rm());
} else {
int64_t dror_value = rt.immediate() % 64;
if (dror_value < 0) dror_value += 64;
if (dror_value <= 31) {
drotr(rd, rs, dror_value);
} else {
drotr32(rd, rs, dror_value - 32);
}
}
}
void MacroAssembler::Pref(int32_t hint, const MemOperand& rs) {
pref(hint, rs);
}
void TurboAssembler::Lsa(Register rd, Register rt, Register rs, uint8_t sa,
Register scratch) {
DCHECK(sa >= 1 && sa <= 31);
if (kArchVariant == kMips64r6 && sa <= 4) {
lsa(rd, rt, rs, sa - 1);
} else {
Register tmp = rd == rt ? scratch : rd;
DCHECK(tmp != rt);
sll(tmp, rs, sa);
Addu(rd, rt, tmp);
}
}
void TurboAssembler::Dlsa(Register rd, Register rt, Register rs, uint8_t sa,
Register scratch) {
DCHECK(sa >= 1 && sa <= 63);
if (kArchVariant == kMips64r6 && sa <= 4) {
dlsa(rd, rt, rs, sa - 1);
} else {
Register tmp = rd == rt ? scratch : rd;
DCHECK(tmp != rt);
if (sa <= 31)
dsll(tmp, rs, sa);
else
dsll32(tmp, rs, sa - 32);
Daddu(rd, rt, tmp);
}
}
void TurboAssembler::Bovc(Register rs, Register rt, Label* L) {
if (is_trampoline_emitted()) {
Label skip;
bnvc(rs, rt, &skip);
BranchLong(L, PROTECT);
bind(&skip);
} else {
bovc(rs, rt, L);
}
}
void TurboAssembler::Bnvc(Register rs, Register rt, Label* L) {
if (is_trampoline_emitted()) {
Label skip;
bovc(rs, rt, &skip);
BranchLong(L, PROTECT);
bind(&skip);
} else {
bnvc(rs, rt, L);
}
}
// ------------Pseudo-instructions-------------
// Change endianness
void TurboAssembler::ByteSwapSigned(Register dest, Register src,
int operand_size) {
DCHECK(operand_size == 2 || operand_size == 4 || operand_size == 8);
DCHECK(kArchVariant == kMips64r6 || kArchVariant == kMips64r2);
if (operand_size == 2) {
wsbh(dest, src);
seh(dest, dest);
} else if (operand_size == 4) {
wsbh(dest, src);
rotr(dest, dest, 16);
} else {
dsbh(dest, src);
dshd(dest, dest);
}
}
void TurboAssembler::ByteSwapUnsigned(Register dest, Register src,
int operand_size) {
DCHECK(operand_size == 2 || operand_size == 4);
if (operand_size == 2) {
wsbh(dest, src);
andi(dest, dest, 0xFFFF);
} else {
wsbh(dest, src);
rotr(dest, dest, 16);
dinsu_(dest, zero_reg, 32, 32);
}
}
void TurboAssembler::Ulw(Register rd, const MemOperand& rs) {
DCHECK(rd != at);
DCHECK(rs.rm() != at);
if (kArchVariant == kMips64r6) {
Lw(rd, rs);
} else {
DCHECK_EQ(kArchVariant, kMips64r2);
DCHECK(kMipsLwrOffset <= 3 && kMipsLwlOffset <= 3);
MemOperand source = rs;
// Adjust offset for two accesses and check if offset + 3 fits into int16_t.
AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 3);
if (rd != source.rm()) {
lwr(rd, MemOperand(source.rm(), source.offset() + kMipsLwrOffset));
lwl(rd, MemOperand(source.rm(), source.offset() + kMipsLwlOffset));
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
lwr(scratch, MemOperand(rs.rm(), rs.offset() + kMipsLwrOffset));
lwl(scratch, MemOperand(rs.rm(), rs.offset() + kMipsLwlOffset));
mov(rd, scratch);
}
}
}
void TurboAssembler::Ulwu(Register rd, const MemOperand& rs) {
if (kArchVariant == kMips64r6) {
Lwu(rd, rs);
} else {
DCHECK_EQ(kArchVariant, kMips64r2);
Ulw(rd, rs);
Dext(rd, rd, 0, 32);
}
}
void TurboAssembler::Usw(Register rd, const MemOperand& rs) {
DCHECK(rd != at);
DCHECK(rs.rm() != at);
DCHECK(rd != rs.rm());
if (kArchVariant == kMips64r6) {
Sw(rd, rs);
} else {
DCHECK_EQ(kArchVariant, kMips64r2);
DCHECK(kMipsSwrOffset <= 3 && kMipsSwlOffset <= 3);
MemOperand source = rs;
// Adjust offset for two accesses and check if offset + 3 fits into int16_t.
AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 3);
swr(rd, MemOperand(source.rm(), source.offset() + kMipsSwrOffset));
swl(rd, MemOperand(source.rm(), source.offset() + kMipsSwlOffset));
}
}
void TurboAssembler::Ulh(Register rd, const MemOperand& rs) {
DCHECK(rd != at);
DCHECK(rs.rm() != at);
if (kArchVariant == kMips64r6) {
Lh(rd, rs);
} else {
DCHECK_EQ(kArchVariant, kMips64r2);
MemOperand source = rs;
// Adjust offset for two accesses and check if offset + 1 fits into int16_t.
AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 1);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
if (source.rm() == scratch) {
#if defined(V8_TARGET_LITTLE_ENDIAN)
Lb(rd, MemOperand(source.rm(), source.offset() + 1));
Lbu(scratch, source);
#elif defined(V8_TARGET_BIG_ENDIAN)
Lb(rd, source);
Lbu(scratch, MemOperand(source.rm(), source.offset() + 1));
#endif
} else {
#if defined(V8_TARGET_LITTLE_ENDIAN)
Lbu(scratch, source);
Lb(rd, MemOperand(source.rm(), source.offset() + 1));
#elif defined(V8_TARGET_BIG_ENDIAN)
Lbu(scratch, MemOperand(source.rm(), source.offset() + 1));
Lb(rd, source);
#endif
}
dsll(rd, rd, 8);
or_(rd, rd, scratch);
}
}
void TurboAssembler::Ulhu(Register rd, const MemOperand& rs) {
DCHECK(rd != at);
DCHECK(rs.rm() != at);
if (kArchVariant == kMips64r6) {
Lhu(rd, rs);
} else {
DCHECK_EQ(kArchVariant, kMips64r2);
MemOperand source = rs;
// Adjust offset for two accesses and check if offset + 1 fits into int16_t.
AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 1);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
if (source.rm() == scratch) {
#if defined(V8_TARGET_LITTLE_ENDIAN)
Lbu(rd, MemOperand(source.rm(), source.offset() + 1));
Lbu(scratch, source);
#elif defined(V8_TARGET_BIG_ENDIAN)
Lbu(rd, source);
Lbu(scratch, MemOperand(source.rm(), source.offset() + 1));
#endif
} else {
#if defined(V8_TARGET_LITTLE_ENDIAN)
Lbu(scratch, source);
Lbu(rd, MemOperand(source.rm(), source.offset() + 1));
#elif defined(V8_TARGET_BIG_ENDIAN)
Lbu(scratch, MemOperand(source.rm(), source.offset() + 1));
Lbu(rd, source);
#endif
}
dsll(rd, rd, 8);
or_(rd, rd, scratch);
}
}
void TurboAssembler::Ush(Register rd, const MemOperand& rs, Register scratch) {
DCHECK(rd != at);
DCHECK(rs.rm() != at);
DCHECK(rs.rm() != scratch);
DCHECK(scratch != at);
if (kArchVariant == kMips64r6) {
Sh(rd, rs);
} else {
DCHECK_EQ(kArchVariant, kMips64r2);
MemOperand source = rs;
// Adjust offset for two accesses and check if offset + 1 fits into int16_t.
AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 1);
if (scratch != rd) {
mov(scratch, rd);
}
#if defined(V8_TARGET_LITTLE_ENDIAN)
Sb(scratch, source);
srl(scratch, scratch, 8);
Sb(scratch, MemOperand(source.rm(), source.offset() + 1));
#elif defined(V8_TARGET_BIG_ENDIAN)
Sb(scratch, MemOperand(source.rm(), source.offset() + 1));
srl(scratch, scratch, 8);
Sb(scratch, source);
#endif
}
}
void TurboAssembler::Uld(Register rd, const MemOperand& rs) {
DCHECK(rd != at);
DCHECK(rs.rm() != at);
if (kArchVariant == kMips64r6) {
Ld(rd, rs);
} else {
DCHECK_EQ(kArchVariant, kMips64r2);
DCHECK(kMipsLdrOffset <= 7 && kMipsLdlOffset <= 7);
MemOperand source = rs;
// Adjust offset for two accesses and check if offset + 7 fits into int16_t.
AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 7);
if (rd != source.rm()) {
ldr(rd, MemOperand(source.rm(), source.offset() + kMipsLdrOffset));
ldl(rd, MemOperand(source.rm(), source.offset() + kMipsLdlOffset));
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
ldr(scratch, MemOperand(rs.rm(), rs.offset() + kMipsLdrOffset));
ldl(scratch, MemOperand(rs.rm(), rs.offset() + kMipsLdlOffset));
mov(rd, scratch);
}
}
}
// Load consequent 32-bit word pair in 64-bit reg. and put first word in low
// bits,
// second word in high bits.
void MacroAssembler::LoadWordPair(Register rd, const MemOperand& rs,
Register scratch) {
Lwu(rd, rs);
Lw(scratch, MemOperand(rs.rm(), rs.offset() + kPointerSize / 2));
dsll32(scratch, scratch, 0);
Daddu(rd, rd, scratch);
}
void TurboAssembler::Usd(Register rd, const MemOperand& rs) {
DCHECK(rd != at);
DCHECK(rs.rm() != at);
if (kArchVariant == kMips64r6) {
Sd(rd, rs);
} else {
DCHECK_EQ(kArchVariant, kMips64r2);
DCHECK(kMipsSdrOffset <= 7 && kMipsSdlOffset <= 7);
MemOperand source = rs;
// Adjust offset for two accesses and check if offset + 7 fits into int16_t.
AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 7);
sdr(rd, MemOperand(source.rm(), source.offset() + kMipsSdrOffset));
sdl(rd, MemOperand(source.rm(), source.offset() + kMipsSdlOffset));
}
}
// Do 64-bit store as two consequent 32-bit stores to unaligned address.
void MacroAssembler::StoreWordPair(Register rd, const MemOperand& rs,
Register scratch) {
Sw(rd, rs);
dsrl32(scratch, rd, 0);
Sw(scratch, MemOperand(rs.rm(), rs.offset() + kPointerSize / 2));
}
void TurboAssembler::Ulwc1(FPURegister fd, const MemOperand& rs,
Register scratch) {
if (kArchVariant == kMips64r6) {
Lwc1(fd, rs);
} else {
DCHECK_EQ(kArchVariant, kMips64r2);
Ulw(scratch, rs);
mtc1(scratch, fd);
}
}
void TurboAssembler::Uswc1(FPURegister fd, const MemOperand& rs,
Register scratch) {
if (kArchVariant == kMips64r6) {
Swc1(fd, rs);
} else {
DCHECK_EQ(kArchVariant, kMips64r2);
mfc1(scratch, fd);
Usw(scratch, rs);
}
}
void TurboAssembler::Uldc1(FPURegister fd, const MemOperand& rs,
Register scratch) {
DCHECK(scratch != at);
if (kArchVariant == kMips64r6) {
Ldc1(fd, rs);
} else {
DCHECK_EQ(kArchVariant, kMips64r2);
Uld(scratch, rs);
dmtc1(scratch, fd);
}
}
void TurboAssembler::Usdc1(FPURegister fd, const MemOperand& rs,
Register scratch) {
DCHECK(scratch != at);
if (kArchVariant == kMips64r6) {
Sdc1(fd, rs);
} else {
DCHECK_EQ(kArchVariant, kMips64r2);
dmfc1(scratch, fd);
Usd(scratch, rs);
}
}
void TurboAssembler::Lb(Register rd, const MemOperand& rs) {
MemOperand source = rs;
AdjustBaseAndOffset(&source);
lb(rd, source);
}
void TurboAssembler::Lbu(Register rd, const MemOperand& rs) {
MemOperand source = rs;
AdjustBaseAndOffset(&source);
lbu(rd, source);
}
void TurboAssembler::Sb(Register rd, const MemOperand& rs) {
MemOperand source = rs;
AdjustBaseAndOffset(&source);
sb(rd, source);
}
void TurboAssembler::Lh(Register rd, const MemOperand& rs) {
MemOperand source = rs;
AdjustBaseAndOffset(&source);
lh(rd, source);
}
void TurboAssembler::Lhu(Register rd, const MemOperand& rs) {
MemOperand source = rs;
AdjustBaseAndOffset(&source);
lhu(rd, source);
}
void TurboAssembler::Sh(Register rd, const MemOperand& rs) {
MemOperand source = rs;
AdjustBaseAndOffset(&source);
sh(rd, source);
}
void TurboAssembler::Lw(Register rd, const MemOperand& rs) {
MemOperand source = rs;
AdjustBaseAndOffset(&source);
lw(rd, source);
}
void TurboAssembler::Lwu(Register rd, const MemOperand& rs) {
MemOperand source = rs;
AdjustBaseAndOffset(&source);
lwu(rd, source);
}
void TurboAssembler::Sw(Register rd, const MemOperand& rs) {
MemOperand source = rs;
AdjustBaseAndOffset(&source);
sw(rd, source);
}
void TurboAssembler::Ld(Register rd, const MemOperand& rs) {
MemOperand source = rs;
AdjustBaseAndOffset(&source);
ld(rd, source);
}
void TurboAssembler::Sd(Register rd, const MemOperand& rs) {
MemOperand source = rs;
AdjustBaseAndOffset(&source);
sd(rd, source);
}
void TurboAssembler::Lwc1(FPURegister fd, const MemOperand& src) {
MemOperand tmp = src;
AdjustBaseAndOffset(&tmp);
lwc1(fd, tmp);
}
void TurboAssembler::Swc1(FPURegister fs, const MemOperand& src) {
MemOperand tmp = src;
AdjustBaseAndOffset(&tmp);
swc1(fs, tmp);
}
void TurboAssembler::Ldc1(FPURegister fd, const MemOperand& src) {
MemOperand tmp = src;
AdjustBaseAndOffset(&tmp);
ldc1(fd, tmp);
}
void TurboAssembler::Sdc1(FPURegister fs, const MemOperand& src) {
MemOperand tmp = src;
AdjustBaseAndOffset(&tmp);
sdc1(fs, tmp);
}
void TurboAssembler::Ll(Register rd, const MemOperand& rs) {
bool is_one_instruction = (kArchVariant == kMips64r6) ? is_int9(rs.offset())
: is_int16(rs.offset());
if (is_one_instruction) {
ll(rd, rs);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
li(scratch, rs.offset());
daddu(scratch, scratch, rs.rm());
ll(rd, MemOperand(scratch, 0));
}
}
void TurboAssembler::Lld(Register rd, const MemOperand& rs) {
bool is_one_instruction = (kArchVariant == kMips64r6) ? is_int9(rs.offset())
: is_int16(rs.offset());
if (is_one_instruction) {
lld(rd, rs);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
li(scratch, rs.offset());
daddu(scratch, scratch, rs.rm());
lld(rd, MemOperand(scratch, 0));
}
}
void TurboAssembler::Sc(Register rd, const MemOperand& rs) {
bool is_one_instruction = (kArchVariant == kMips64r6) ? is_int9(rs.offset())
: is_int16(rs.offset());
if (is_one_instruction) {
sc(rd, rs);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
li(scratch, rs.offset());
daddu(scratch, scratch, rs.rm());
sc(rd, MemOperand(scratch, 0));
}
}
void TurboAssembler::Scd(Register rd, const MemOperand& rs) {
bool is_one_instruction = (kArchVariant == kMips64r6) ? is_int9(rs.offset())
: is_int16(rs.offset());
if (is_one_instruction) {
scd(rd, rs);
} else {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
li(scratch, rs.offset());
daddu(scratch, scratch, rs.rm());
scd(rd, MemOperand(scratch, 0));
}
}
void TurboAssembler::li(Register dst, Handle<HeapObject> value, LiFlags mode) {
// TODO(jgruber,v8:8887): Also consider a root-relative load when generating
// non-isolate-independent code. In many cases it might be cheaper than
// embedding the relocatable value.
if (root_array_available_ && options().isolate_independent_code) {
IndirectLoadConstant(dst, value);
return;
}
li(dst, Operand(value), mode);
}
void TurboAssembler::li(Register dst, ExternalReference value, LiFlags mode) {
// TODO(jgruber,v8:8887): Also consider a root-relative load when generating
// non-isolate-independent code. In many cases it might be cheaper than
// embedding the relocatable value.
if (root_array_available_ && options().isolate_independent_code) {
IndirectLoadExternalReference(dst, value);
return;
}
li(dst, Operand(value), mode);
}
void TurboAssembler::li(Register dst, const StringConstantBase* string,
LiFlags mode) {
li(dst, Operand::EmbeddedStringConstant(string), mode);
}
static inline int InstrCountForLiLower32Bit(int64_t value) {
if (!is_int16(static_cast<int32_t>(value)) && (value & kUpper16MaskOf64) &&
(value & kImm16Mask)) {
return 2;
} else {
return 1;
}
}
void TurboAssembler::LiLower32BitHelper(Register rd, Operand j) {
if (is_int16(static_cast<int32_t>(j.immediate()))) {
daddiu(rd, zero_reg, (j.immediate() & kImm16Mask));
} else if (!(j.immediate() & kUpper16MaskOf64)) {
ori(rd, zero_reg, j.immediate() & kImm16Mask);
} else {
lui(rd, j.immediate() >> kLuiShift & kImm16Mask);
if (j.immediate() & kImm16Mask) {
ori(rd, rd, j.immediate() & kImm16Mask);
}
}
}
static inline int InstrCountForLoadReplicatedConst32(int64_t value) {
uint32_t x = static_cast<uint32_t>(value);
uint32_t y = static_cast<uint32_t>(value >> 32);
if (x == y) {
return (is_uint16(x) || is_int16(x) || (x & kImm16Mask) == 0) ? 2 : 3;
}
return INT_MAX;
}
int TurboAssembler::InstrCountForLi64Bit(int64_t value) {
if (is_int32(value)) {
return InstrCountForLiLower32Bit(value);
} else {
int bit31 = value >> 31 & 0x1;
if ((value & kUpper16MaskOf64) == 0 && is_int16(value >> 32) &&
kArchVariant == kMips64r6) {
return 2;
} else if ((value & (kHigher16MaskOf64 | kUpper16MaskOf64)) == 0 &&
kArchVariant == kMips64r6) {
return 2;
} else if ((value & kImm16Mask) == 0 && is_int16((value >> 32) + bit31) &&
kArchVariant == kMips64r6) {
return 2;
} else if ((value & kImm16Mask) == 0 &&
((value >> 31) & 0x1FFFF) == ((0x20000 - bit31) & 0x1FFFF) &&
kArchVariant == kMips64r6) {
return 2;
} else if (is_int16(static_cast<int32_t>(value)) &&
is_int16((value >> 32) + bit31) && kArchVariant == kMips64r6) {
return 2;
} else if (is_int16(static_cast<int32_t>(value)) &&
((value >> 31) & 0x1FFFF) == ((0x20000 - bit31) & 0x1FFFF) &&
kArchVariant == kMips64r6) {
return 2;
} else if (base::bits::IsPowerOfTwo(value + 1) ||
value == std::numeric_limits<int64_t>::max()) {
return 2;
} else {
int shift_cnt = base::bits::CountTrailingZeros64(value);
int rep32_count = InstrCountForLoadReplicatedConst32(value);
int64_t tmp = value >> shift_cnt;
if (is_uint16(tmp)) {
return 2;
} else if (is_int16(tmp)) {
return 2;
} else if (rep32_count < 3) {
return 2;
} else if (is_int32(tmp)) {
return 3;
} else {
shift_cnt = 16 + base::bits::CountTrailingZeros64(value >> 16);
tmp = value >> shift_cnt;
if (is_uint16(tmp)) {
return 3;
} else if (is_int16(tmp)) {
return 3;
} else if (rep32_count < 4) {
return 3;
} else if (kArchVariant == kMips64r6) {
int64_t imm = value;
int count = InstrCountForLiLower32Bit(imm);
imm = (imm >> 32) + bit31;
if (imm & kImm16Mask) {
count++;
}
imm = (imm >> 16) + (imm >> 15 & 0x1);
if (imm & kImm16Mask) {
count++;
}
return count;
} else {
if (is_int48(value)) {
int64_t k = value >> 16;
int count = InstrCountForLiLower32Bit(k) + 1;
if (value & kImm16Mask) {
count++;
}
return count;
} else {
int64_t k = value >> 32;
int count = InstrCountForLiLower32Bit(k);
if ((value >> 16) & kImm16Mask) {
count += 3;
if (value & kImm16Mask) {
count++;
}
} else {
count++;
if (value & kImm16Mask) {
count++;
}
}
return count;
}
}
}
}
}
UNREACHABLE();
return INT_MAX;
}
// All changes to if...else conditions here must be added to
// InstrCountForLi64Bit as well.
void TurboAssembler::li_optimized(Register rd, Operand j, LiFlags mode) {
DCHECK(!j.is_reg());
DCHECK(!MustUseReg(j.rmode()));
DCHECK(mode == OPTIMIZE_SIZE);
BlockTrampolinePoolScope block_trampoline_pool(this);
// Normal load of an immediate value which does not need Relocation Info.
if (is_int32(j.immediate())) {
LiLower32BitHelper(rd, j);
} else {
int bit31 = j.immediate() >> 31 & 0x1;
if ((j.immediate() & kUpper16MaskOf64) == 0 &&
is_int16(j.immediate() >> 32) && kArchVariant == kMips64r6) {
// 64-bit value which consists of an unsigned 16-bit value in its
// least significant 32-bits, and a signed 16-bit value in its
// most significant 32-bits.
ori(rd, zero_reg, j.immediate() & kImm16Mask);
dahi(rd, j.immediate() >> 32 & kImm16Mask);
} else if ((j.immediate() & (kHigher16MaskOf64 | kUpper16MaskOf64)) == 0 &&
kArchVariant == kMips64r6) {
// 64-bit value which consists of an unsigned 16-bit value in its
// least significant 48-bits, and a signed 16-bit value in its
// most significant 16-bits.
ori(rd, zero_reg, j.immediate() & kImm16Mask);
dati(rd, j.immediate() >> 48 & kImm16Mask);
} else if ((j.immediate() & kImm16Mask) == 0 &&
is_int16((j.immediate() >> 32) + bit31) &&
kArchVariant == kMips64r6) {
// 16 LSBs (Least Significant Bits) all set to zero.
// 48 MSBs (Most Significant Bits) hold a signed 32-bit value.
lui(rd, j.immediate() >> kLuiShift & kImm16Mask);
dahi(rd, ((j.immediate() >> 32) + bit31) & kImm16Mask);
} else if ((j.immediate() & kImm16Mask) == 0 &&
((j.immediate() >> 31) & 0x1FFFF) ==
((0x20000 - bit31) & 0x1FFFF) &&
kArchVariant == kMips64r6) {
// 16 LSBs all set to zero.
// 48 MSBs hold a signed value which can't be represented by signed
// 32-bit number, and the middle 16 bits are all zero, or all one.
lui(rd, j.immediate() >> kLuiShift & kImm16Mask);
dati(rd, ((j.immediate() >> 48) + bit31) & kImm16Mask);
} else if (is_int16(static_cast<int32_t>(j.immediate())) &&
is_int16((j.immediate() >> 32) + bit31) &&
kArchVariant == kMips64r6) {
// 32 LSBs contain a signed 16-bit number.
// 32 MSBs contain a signed 16-bit number.
daddiu(rd, zero_reg, j.immediate() & kImm16Mask);
dahi(rd, ((j.immediate() >> 32) + bit31) & kImm16Mask);
} else if (is_int16(static_cast<int32_t>(j.immediate())) &&
((j.immediate() >> 31) & 0x1FFFF) ==
((0x20000 - bit31) & 0x1FFFF) &&
kArchVariant == kMips64r6) {
// 48 LSBs contain an unsigned 16-bit number.
// 16 MSBs contain a signed 16-bit number.
daddiu(rd, zero_reg, j.immediate() & kImm16Mask);
dati(rd, ((j.immediate() >> 48) + bit31) & kImm16Mask);
} else if (base::bits::IsPowerOfTwo(j.immediate() + 1) ||
j.immediate() == std::numeric_limits<int64_t>::max()) {
// 64-bit values which have their "n" LSBs set to one, and their
// "64-n" MSBs set to zero. "n" must meet the restrictions 0 < n < 64.
int shift_cnt = 64 - base::bits::CountTrailingZeros64(j.immediate() + 1);
daddiu(rd, zero_reg, -1);
if (shift_cnt < 32) {
dsrl(rd, rd, shift_cnt);
} else {
dsrl32(rd, rd, shift_cnt & 31);
}
} else {
int shift_cnt = base::bits::CountTrailingZeros64(j.immediate());
int rep32_count = InstrCountForLoadReplicatedConst32(j.immediate());
int64_t tmp = j.immediate() >> shift_cnt;
if (is_uint16(tmp)) {
// Value can be computed by loading a 16-bit unsigned value, and
// then shifting left.
ori(rd, zero_reg, tmp & kImm16Mask);
if (shift_cnt < 32) {
dsll(rd, rd, shift_cnt);
} else {
dsll32(rd, rd, shift_cnt & 31);
}
} else if (is_int16(tmp)) {
// Value can be computed by loading a 16-bit signed value, and
// then shifting left.
daddiu(rd, zero_reg, static_cast<int32_t>(tmp));
if (shift_cnt < 32) {
dsll(rd, rd, shift_cnt);
} else {
dsll32(rd, rd, shift_cnt & 31);
}
} else if (rep32_count < 3) {
// Value being loaded has 32 LSBs equal to the 32 MSBs, and the
// value loaded into the 32 LSBs can be loaded with a single
// MIPS instruction.
LiLower32BitHelper(rd, j);
Dins(rd, rd, 32, 32);
} else if (is_int32(tmp)) {
// Loads with 3 instructions.
// Value can be computed by loading a 32-bit signed value, and
// then shifting left.
lui(rd, tmp >> kLuiShift & kImm16Mask);
ori(rd, rd, tmp & kImm16Mask);
if (shift_cnt < 32) {
dsll(rd, rd, shift_cnt);
} else {
dsll32(rd, rd, shift_cnt & 31);
}
} else {
shift_cnt = 16 + base::bits::CountTrailingZeros64(j.immediate() >> 16);
tmp = j.immediate() >> shift_cnt;
if (is_uint16(tmp)) {
// Value can be computed by loading a 16-bit unsigned value,
// shifting left, and "or"ing in another 16-bit unsigned value.
ori(rd, zero_reg, tmp & kImm16Mask);
if (shift_cnt < 32) {
dsll(rd, rd, shift_cnt);
} else {
dsll32(rd, rd, shift_cnt & 31);
}
ori(rd, rd, j.immediate() & kImm16Mask);
} else if (is_int16(tmp)) {
// Value can be computed by loading a 16-bit signed value,
// shifting left, and "or"ing in a 16-bit unsigned value.
daddiu(rd, zero_reg, static_cast<int32_t>(tmp));
if (shift_cnt < 32) {
dsll(rd, rd, shift_cnt);
} else {
dsll32(rd, rd, shift_cnt & 31);
}
ori(rd, rd, j.immediate() & kImm16Mask);
} else if (rep32_count < 4) {
// Value being loaded has 32 LSBs equal to the 32 MSBs, and the
// value in the 32 LSBs requires 2 MIPS instructions to load.
LiLower32BitHelper(rd, j);
Dins(rd, rd, 32, 32);
} else if (kArchVariant == kMips64r6) {
// Loads with 3-4 instructions.
// Catch-all case to get any other 64-bit values which aren't
// handled by special cases above.
int64_t imm = j.immediate();
LiLower32BitHelper(rd, j);
imm = (imm >> 32) + bit31;
if (imm & kImm16Mask) {
dahi(rd, imm & kImm16Mask);
}
imm = (imm >> 16) + (imm >> 15 & 0x1);
if (imm & kImm16Mask) {
dati(rd, imm & kImm16Mask);
}
} else {
if (is_int48(j.immediate())) {
Operand k = Operand(j.immediate() >> 16);
LiLower32BitHelper(rd, k);
dsll(rd, rd, 16);
if (j.immediate() & kImm16Mask) {
ori(rd, rd, j.immediate() & kImm16Mask);
}
} else {
Operand k = Operand(j.immediate() >> 32);
LiLower32BitHelper(rd, k);
if ((j.immediate() >> 16) & kImm16Mask) {
dsll(rd, rd, 16);
ori(rd, rd, (j.immediate() >> 16) & kImm16Mask);
dsll(rd, rd, 16);
if (j.immediate() & kImm16Mask) {
ori(rd, rd, j.immediate() & kImm16Mask);
}
} else {
dsll32(rd, rd, 0);
if (j.immediate() & kImm16Mask) {
ori(rd, rd, j.immediate() & kImm16Mask);
}
}
}
}
}
}
}
}
void TurboAssembler::li(Register rd, Operand j, LiFlags mode) {
DCHECK(!j.is_reg());
BlockTrampolinePoolScope block_trampoline_pool(this);
if (!MustUseReg(j.rmode()) && mode == OPTIMIZE_SIZE) {
int li_count = InstrCountForLi64Bit(j.immediate());
int li_neg_count = InstrCountForLi64Bit(-j.immediate());
int li_not_count = InstrCountForLi64Bit(~j.immediate());
// Loading -MIN_INT64 could cause problems, but loading MIN_INT64 takes only
// two instructions so no need to check for this.
if (li_neg_count <= li_not_count && li_neg_count < li_count - 1) {
DCHECK(j.immediate() != std::numeric_limits<int64_t>::min());
li_optimized(rd, Operand(-j.immediate()), mode);
Dsubu(rd, zero_reg, rd);
} else if (li_neg_count > li_not_count && li_not_count < li_count - 1) {
DCHECK(j.immediate() != std::numeric_limits<int64_t>::min());
li_optimized(rd, Operand(~j.immediate()), mode);
nor(rd, rd, rd);
} else {
li_optimized(rd, j, mode);
}
} else if (MustUseReg(j.rmode())) {
int64_t immediate;
if (j.IsHeapObjectRequest()) {
RequestHeapObject(j.heap_object_request());
immediate = 0;
} else {
immediate = j.immediate();
}
RecordRelocInfo(j.rmode(), immediate);
lui(rd, (immediate >> 32) & kImm16Mask);
ori(rd, rd, (immediate >> 16) & kImm16Mask);
dsll(rd, rd, 16);
ori(rd, rd, immediate & kImm16Mask);
} else if (mode == ADDRESS_LOAD) {
// We always need the same number of instructions as we may need to patch
// this code to load another value which may need all 4 instructions.
lui(rd, (j.immediate() >> 32) & kImm16Mask);
ori(rd, rd, (j.immediate() >> 16) & kImm16Mask);
dsll(rd, rd, 16);
ori(rd, rd, j.immediate() & kImm16Mask);
} else { // mode == CONSTANT_SIZE - always emit the same instruction
// sequence.
if (kArchVariant == kMips64r6) {
int64_t imm = j.immediate();
lui(rd, imm >> kLuiShift & kImm16Mask);
ori(rd, rd, (imm & kImm16Mask));
imm = (imm >> 32) + ((imm >> 31) & 0x1);
dahi(rd, imm & kImm16Mask & kImm16Mask);
imm = (imm >> 16) + ((imm >> 15) & 0x1);
dati(rd, imm & kImm16Mask & kImm16Mask);
} else {
lui(rd, (j.immediate() >> 48) & kImm16Mask);
ori(rd, rd, (j.immediate() >> 32) & kImm16Mask);
dsll(rd, rd, 16);
ori(rd, rd, (j.immediate() >> 16) & kImm16Mask);
dsll(rd, rd, 16);
ori(rd, rd, j.immediate() & kImm16Mask);
}
}
}
void TurboAssembler::MultiPush(RegList regs) {
int16_t num_to_push = base::bits::CountPopulation(regs);
int16_t stack_offset = num_to_push * kPointerSize;
Dsubu(sp, sp, Operand(stack_offset));
for (int16_t i = kNumRegisters - 1; i >= 0; i--) {
if ((regs & (1 << i)) != 0) {
stack_offset -= kPointerSize;
Sd(ToRegister(i), MemOperand(sp, stack_offset));
}
}
}
void TurboAssembler::MultiPop(RegList regs) {
int16_t stack_offset = 0;
for (int16_t i = 0; i < kNumRegisters; i++) {
if ((regs & (1 << i)) != 0) {
Ld(ToRegister(i), MemOperand(sp, stack_offset));
stack_offset += kPointerSize;
}
}
daddiu(sp, sp, stack_offset);
}
void TurboAssembler::MultiPushFPU(RegList regs) {
int16_t num_to_push = base::bits::CountPopulation(regs);
int16_t stack_offset = num_to_push * kDoubleSize;
Dsubu(sp, sp, Operand(stack_offset));
for (int16_t i = kNumRegisters - 1; i >= 0; i--) {
if ((regs & (1 << i)) != 0) {
stack_offset -= kDoubleSize;
Sdc1(FPURegister::from_code(i), MemOperand(sp, stack_offset));
}
}
}
void TurboAssembler::MultiPopFPU(RegList regs) {
int16_t stack_offset = 0;
for (int16_t i = 0; i < kNumRegisters; i++) {
if ((regs & (1 << i)) != 0) {
Ldc1(FPURegister::from_code(i), MemOperand(sp, stack_offset));
stack_offset += kDoubleSize;
}
}
daddiu(sp, sp, stack_offset);
}
void TurboAssembler::MultiPushMSA(RegList regs) {
int16_t num_to_push = base::bits::CountPopulation(regs);
int16_t stack_offset = num_to_push * kSimd128Size;
Dsubu(sp, sp, Operand(stack_offset));
for (int16_t i = kNumRegisters - 1; i >= 0; i--) {
if ((regs & (1 << i)) != 0) {
stack_offset -= kSimd128Size;
st_d(MSARegister::from_code(i), MemOperand(sp, stack_offset));
}
}
}
void TurboAssembler::MultiPopMSA(RegList regs) {
int16_t stack_offset = 0;
for (int16_t i = 0; i < kNumRegisters; i++) {
if ((regs & (1 << i)) != 0) {
ld_d(MSARegister::from_code(i), MemOperand(sp, stack_offset));
stack_offset += kSimd128Size;
}
}
daddiu(sp, sp, stack_offset);
}
void TurboAssembler::Ext(Register rt, Register rs, uint16_t pos,
uint16_t size) {
DCHECK_LT(pos, 32);
DCHECK_LT(pos + size, 33);
ext_(rt, rs, pos, size);
}
void TurboAssembler::Dext(Register rt, Register rs, uint16_t pos,
uint16_t size) {
DCHECK(pos < 64 && 0 < size && size <= 64 && 0 < pos + size &&
pos + size <= 64);
if (size > 32) {
dextm_(rt, rs, pos, size);
} else if (pos >= 32) {
dextu_(rt, rs, pos, size);
} else {
dext_(rt, rs, pos, size);
}
}
void TurboAssembler::Ins(Register rt, Register rs, uint16_t pos,
uint16_t size) {
DCHECK_LT(pos, 32);
DCHECK_LE(pos + size, 32);
DCHECK_NE(size, 0);
ins_(rt, rs, pos, size);
}
void TurboAssembler::Dins(Register rt, Register rs, uint16_t pos,
uint16_t size) {
DCHECK(pos < 64 && 0 < size && size <= 64 && 0 < pos + size &&
pos + size <= 64);
if (pos + size <= 32) {
dins_(rt, rs, pos, size);
} else if (pos < 32) {
dinsm_(rt, rs, pos, size);
} else {
dinsu_(rt, rs, pos, size);
}
}
void TurboAssembler::ExtractBits(Register dest, Register source, Register pos,
int size, bool sign_extend) {
dsrav(dest, source, pos);
Dext(dest, dest, 0, size);
if (sign_extend) {
switch (size) {
case 8:
seb(dest, dest);
break;
case 16:
seh(dest, dest);
break;
case 32:
// sign-extend word
sll(dest, dest, 0);
break;
default:
UNREACHABLE();
}
}
}
void TurboAssembler::InsertBits(Register dest, Register source, Register pos,
int size) {
Dror(dest, dest, pos);
Dins(dest, source, 0, size);
{
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
Dsubu(scratch, zero_reg, pos);
Dror(dest, dest, scratch);
}
}
void TurboAssembler::Neg_s(FPURegister fd, FPURegister fs) {
if (kArchVariant == kMips64r6) {
// r6 neg_s changes the sign for NaN-like operands as well.
neg_s(fd, fs);
} else {
DCHECK_EQ(kArchVariant, kMips64r2);
BlockTrampolinePoolScope block_trampoline_pool(this);
Label is_nan, done;
Register scratch1 = t8;
Register scratch2 = t9;
CompareIsNanF32(fs, fs);
BranchTrueShortF(&is_nan);
Branch(USE_DELAY_SLOT, &done);
// For NaN input, neg_s will return the same NaN value,
// while the sign has to be changed separately.
neg_s(fd, fs); // In delay slot.
bind(&is_nan);
mfc1(scratch1, fs);
li(scratch2, kBinary32SignMask);
Xor(scratch1, scratch1, scratch2);
mtc1(scratch1, fd);
bind(&done);
}
}
void TurboAssembler::Neg_d(FPURegister fd, FPURegister fs) {
if (kArchVariant == kMips64r6) {
// r6 neg_d changes the sign for NaN-like operands as well.
neg_d(fd, fs);
} else {
DCHECK_EQ(kArchVariant, kMips64r2);
BlockTrampolinePoolScope block_trampoline_pool(this);
Label is_nan, done;
Register scratch1 = t8;
Register scratch2 = t9;
CompareIsNanF64(fs, fs);
BranchTrueShortF(&is_nan);
Branch(USE_DELAY_SLOT, &done);
// For NaN input, neg_d will return the same NaN value,
// while the sign has to be changed separately.
neg_d(fd, fs); // In delay slot.
bind(&is_nan);
dmfc1(scratch1, fs);
li(scratch2, base::Double::kSignMask);
Xor(scratch1, scratch1, scratch2);
dmtc1(scratch1, fd);
bind(&done);
}
}
void TurboAssembler::Cvt_d_uw(FPURegister fd, FPURegister fs) {
// Move the data from fs to t8.
BlockTrampolinePoolScope block_trampoline_pool(this);
mfc1(t8, fs);
Cvt_d_uw(fd, t8);
}
void TurboAssembler::Cvt_d_uw(FPURegister fd, Register rs) {
BlockTrampolinePoolScope block_trampoline_pool(this);
// Convert rs to a FP value in fd.
DCHECK(rs != t9);
DCHECK(rs != at);
// Zero extend int32 in rs.
Dext(t9, rs, 0, 32);
dmtc1(t9, fd);
cvt_d_l(fd, fd);
}
void TurboAssembler::Cvt_d_ul(FPURegister fd, FPURegister fs) {
BlockTrampolinePoolScope block_trampoline_pool(this);
// Move the data from fs to t8.
dmfc1(t8, fs);
Cvt_d_ul(fd, t8);
}
void TurboAssembler::Cvt_d_ul(FPURegister fd, Register rs) {
BlockTrampolinePoolScope block_trampoline_pool(this);
// Convert rs to a FP value in fd.
DCHECK(rs != t9);
DCHECK(rs != at);
Label msb_clear, conversion_done;
Branch(&msb_clear, ge, rs, Operand(zero_reg));
// Rs >= 2^63
andi(t9, rs, 1);
dsrl(rs, rs, 1);
or_(t9, t9, rs);
dmtc1(t9, fd);
cvt_d_l(fd, fd);
Branch(USE_DELAY_SLOT, &conversion_done);
add_d(fd, fd, fd); // In delay slot.
bind(&msb_clear);
// Rs < 2^63, we can do simple conversion.
dmtc1(rs, fd);
cvt_d_l(fd, fd);
bind(&conversion_done);
}
void TurboAssembler::Cvt_s_uw(FPURegister fd, FPURegister fs) {
BlockTrampolinePoolScope block_trampoline_pool(this);
// Move the data from fs to t8.
mfc1(t8, fs);
Cvt_s_uw(fd, t8);
}
void TurboAssembler::Cvt_s_uw(FPURegister fd, Register rs) {
BlockTrampolinePoolScope block_trampoline_pool(this);
// Convert rs to a FP value in fd.
DCHECK(rs != t9);
DCHECK(rs != at);
// Zero extend int32 in rs.
Dext(t9, rs, 0, 32);
dmtc1(t9, fd);
cvt_s_l(fd, fd);
}
void TurboAssembler::Cvt_s_ul(FPURegister fd, FPURegister fs) {
BlockTrampolinePoolScope block_trampoline_pool(this);
// Move the data from fs to t8.
dmfc1(t8, fs);
Cvt_s_ul(fd, t8);
}
void TurboAssembler::Cvt_s_ul(FPURegister fd, Register rs) {
BlockTrampolinePoolScope block_trampoline_pool(this);
// Convert rs to a FP value in fd.
DCHECK(rs != t9);
DCHECK(rs != at);
Label positive, conversion_done;
Branch(&positive, ge, rs, Operand(zero_reg));
// Rs >= 2^31.
andi(t9, rs, 1);
dsrl(rs, rs, 1);
or_(t9, t9, rs);
dmtc1(t9, fd);
cvt_s_l(fd, fd);
Branch(USE_DELAY_SLOT, &conversion_done);
add_s(fd, fd, fd); // In delay slot.
bind(&positive);
// Rs < 2^31, we can do simple conversion.
dmtc1(rs, fd);
cvt_s_l(fd, fd);
bind(&conversion_done);
}
void MacroAssembler::Round_l_d(FPURegister fd, FPURegister fs) {
round_l_d(fd, fs);
}
void MacroAssembler::Floor_l_d(FPURegister fd, FPURegister fs) {
floor_l_d(fd, fs);
}
void MacroAssembler::Ceil_l_d(FPURegister fd, FPURegister fs) {
ceil_l_d(fd, fs);
}
void MacroAssembler::Trunc_l_d(FPURegister fd, FPURegister fs) {
trunc_l_d(fd, fs);
}
void MacroAssembler::Trunc_l_ud(FPURegister fd, FPURegister fs,
FPURegister scratch) {
BlockTrampolinePoolScope block_trampoline_pool(this);
// Load to GPR.
dmfc1(t8, fs);
// Reset sign bit.
{
UseScratchRegisterScope temps(this);
Register scratch1 = temps.Acquire();
li(scratch1, 0x7FFFFFFFFFFFFFFF);
and_(t8, t8, scratch1);
}
dmtc1(t8, fs);
trunc_l_d(fd, fs);
}
void TurboAssembler::Trunc_uw_d(FPURegister fd, FPURegister fs,
FPURegister scratch) {
BlockTrampolinePoolScope block_trampoline_pool(this);
Trunc_uw_d(t8, fs, scratch);
mtc1(t8, fd);
}
void TurboAssembler::Trunc_uw_s(FPURegister fd, FPURegister fs,
FPURegister scratch) {
BlockTrampolinePoolScope block_trampoline_pool(this);
Trunc_uw_s(t8, fs, scratch);
mtc1(t8, fd);
}
void TurboAssembler::Trunc_ul_d(FPURegister fd, FPURegister fs,
FPURegister scratch, Register result) {
BlockTrampolinePoolScope block_trampoline_pool(this);
Trunc_ul_d(t8, fs, scratch, result);
dmtc1(t8, fd);
}
void TurboAssembler::Trunc_ul_s(FPURegister fd, FPURegister fs,
FPURegister scratch, Register result) {
BlockTrampolinePoolScope block_trampoline_pool(this);
Trunc_ul_s(t8, fs, scratch, result);
dmtc1(t8, fd);
}
void MacroAssembler::Trunc_w_d(FPURegister fd, FPURegister fs) {
trunc_w_d(fd, fs);
}
void MacroAssembler::Round_w_d(FPURegister fd, FPURegister fs) {
round_w_d(fd, fs);
}
void MacroAssembler::Floor_w_d(FPURegister fd, FPURegister fs) {
floor_w_d(fd, fs);
}
void MacroAssembler::Ceil_w_d(FPURegister fd, FPURegister fs) {
ceil_w_d(fd, fs);
}
void TurboAssembler::Trunc_uw_d(Register rd, FPURegister fs,
FPURegister scratch) {
DCHECK(fs != scratch);
DCHECK(rd != at);
{
// Load 2^31 into scratch as its float representation.
UseScratchRegisterScope temps(this);
Register scratch1 = temps.Acquire();
li(scratch1, 0x41E00000);
mtc1(zero_reg, scratch);
mthc1(scratch1, scratch);
}
// Test if scratch > fd.
// If fd < 2^31 we can convert it normally.
Label simple_convert;
CompareF64(OLT, fs, scratch);
BranchTrueShortF(&simple_convert);
// First we subtract 2^31 from fd, then trunc it to rs
// and add 2^31 to rs.
sub_d(scratch, fs, scratch);
trunc_w_d(scratch, scratch);
mfc1(rd, scratch);
Or(rd, rd, 1 << 31);
Label done;
Branch(&done);
// Simple conversion.
bind(&simple_convert);
trunc_w_d(scratch, fs);
mfc1(rd, scratch);
bind(&done);
}
void TurboAssembler::Trunc_uw_s(Register rd, FPURegister fs,
FPURegister scratch) {
DCHECK(fs != scratch);
DCHECK(rd != at);
{
// Load 2^31 into scratch as its float representation.
UseScratchRegisterScope temps(this);
Register scratch1 = temps.Acquire();
li(scratch1, 0x4F000000);
mtc1(scratch1, scratch);
}
// Test if scratch > fs.
// If fs < 2^31 we can convert it normally.
Label simple_convert;
CompareF32(OLT, fs, scratch);
BranchTrueShortF(&simple_convert);
// First we subtract 2^31 from fs, then trunc it to rd
// and add 2^31 to rd.
sub_s(scratch, fs, scratch);
trunc_w_s(scratch, scratch);
mfc1(rd, scratch);
Or(rd, rd, 1 << 31);
Label done;
Branch(&done);
// Simple conversion.
bind(&simple_convert);
trunc_w_s(scratch, fs);
mfc1(rd, scratch);
bind(&done);
}
void TurboAssembler::Trunc_ul_d(Register rd, FPURegister fs,
FPURegister scratch, Register result) {
DCHECK(fs != scratch);
DCHECK(result.is_valid() ? !AreAliased(rd, result, at) : !AreAliased(rd, at));
Label simple_convert, done, fail;
if (result.is_valid()) {
mov(result, zero_reg);
Move(scratch, -1.0);
// If fd =< -1 or unordered, then the conversion fails.
CompareF64(OLE, fs, scratch);
BranchTrueShortF(&fail);
CompareIsNanF64(fs, scratch);
BranchTrueShortF(&fail);
}
// Load 2^63 into scratch as its double representation.
li(at, 0x43E0000000000000);
dmtc1(at, scratch);
// Test if scratch > fs.
// If fs < 2^63 we can convert it normally.
CompareF64(OLT, fs, scratch);
BranchTrueShortF(&simple_convert);
// First we subtract 2^63 from fs, then trunc it to rd
// and add 2^63 to rd.
sub_d(scratch, fs, scratch);
trunc_l_d(scratch, scratch);
dmfc1(rd, scratch);
Or(rd, rd, Operand(1UL << 63));
Branch(&done);
// Simple conversion.
bind(&simple_convert);
trunc_l_d(scratch, fs);
dmfc1(rd, scratch);
bind(&done);
if (result.is_valid()) {
// Conversion is failed if the result is negative.
{
UseScratchRegisterScope temps(this);
Register scratch1 = temps.Acquire();
addiu(scratch1, zero_reg, -1);
dsrl(scratch1, scratch1, 1); // Load 2^62.
dmfc1(result, scratch);
xor_(result, result, scratch1);
}
Slt(result, zero_reg, result);
}
bind(&fail);
}
void TurboAssembler::Trunc_ul_s(Register rd, FPURegister fs,
FPURegister scratch, Register result) {
DCHECK(fs != scratch);
DCHECK(result.is_valid() ? !AreAliased(rd, result, at) : !AreAliased(rd, at));
Label simple_convert, done, fail;
if (result.is_valid()) {
mov(result, zero_reg);
Move(scratch, -1.0f);
// If fd =< -1 or unordered, then the conversion fails.
CompareF32(OLE, fs, scratch);
BranchTrueShortF(&fail);
CompareIsNanF32(fs, scratch);
BranchTrueShortF(&fail);
}
{
// Load 2^63 into scratch as its float representation.
UseScratchRegisterScope temps(this);
Register scratch1 = temps.Acquire();
li(scratch1, 0x5F000000);
mtc1(scratch1, scratch);
}
// Test if scratch > fs.
// If fs < 2^63 we can convert it normally.
CompareF32(OLT, fs, scratch);
BranchTrueShortF(&simple_convert);
// First we subtract 2^63 from fs, then trunc it to rd
// and add 2^63 to rd.
sub_s(scratch, fs, scratch);
trunc_l_s(scratch, scratch);
dmfc1(rd, scratch);
Or(rd, rd, Operand(1UL << 63));
Branch(&done);
// Simple conversion.
bind(&simple_convert);
trunc_l_s(scratch, fs);
dmfc1(rd, scratch);
bind(&done);
if (result.is_valid()) {
// Conversion is failed if the result is negative or unordered.
{
UseScratchRegisterScope temps(this);
Register scratch1 = temps.Acquire();
addiu(scratch1, zero_reg, -1);
dsrl(scratch1, scratch1, 1); // Load 2^62.
dmfc1(result, scratch);
xor_(result, result, scratch1);
}
Slt(result, zero_reg, result);
}
bind(&fail);
}
template <typename RoundFunc>
void TurboAssembler::RoundDouble(FPURegister dst, FPURegister src,
FPURoundingMode mode, RoundFunc round) {
BlockTrampolinePoolScope block_trampoline_pool(this);
Register scratch = t8;
if (kArchVariant == kMips64r6) {
cfc1(scratch, FCSR);
li(at, Operand(mode));
ctc1(at, FCSR);
rint_d(dst, src);
ctc1(scratch, FCSR);
} else {
Label done;
if (!IsDoubleZeroRegSet()) {
Move(kDoubleRegZero, 0.0);
}
mfhc1(scratch, src);
Ext(at, scratch, HeapNumber::kExponentShift, HeapNumber::kExponentBits);
Branch(USE_DELAY_SLOT, &done, hs, at,
Operand(HeapNumber::kExponentBias + HeapNumber::kMantissaBits));
// Canonicalize the result.
sub_d(dst, src, kDoubleRegZero);
round(this, dst, src);
dmfc1(at, dst);
Branch(USE_DELAY_SLOT, &done, ne, at, Operand(zero_reg));
cvt_d_l(dst, dst);
srl(at, scratch, 31);
sll(at, at, 31);
mthc1(at, dst);
bind(&done);
}
}
void TurboAssembler::Floor_d_d(FPURegister dst, FPURegister src) {
RoundDouble(dst, src, mode_floor,
[](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
tasm->floor_l_d(dst, src);
});
}
void TurboAssembler::Ceil_d_d(FPURegister dst, FPURegister src) {
RoundDouble(dst, src, mode_ceil,
[](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
tasm->ceil_l_d(dst, src);
});
}
void TurboAssembler::Trunc_d_d(FPURegister dst, FPURegister src) {
RoundDouble(dst, src, mode_trunc,
[](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
tasm->trunc_l_d(dst, src);
});
}
void TurboAssembler::Round_d_d(FPURegister dst, FPURegister src) {
RoundDouble(dst, src, mode_round,
[](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
tasm->round_l_d(dst, src);
});
}
template <typename RoundFunc>
void TurboAssembler::RoundFloat(FPURegister dst, FPURegister src,
FPURoundingMode mode, RoundFunc round) {
BlockTrampolinePoolScope block_trampoline_pool(this);
Register scratch = t8;
if (kArchVariant == kMips64r6) {
cfc1(scratch, FCSR);
li(at, Operand(mode));
ctc1(at, FCSR);
rint_s(dst, src);
ctc1(scratch, FCSR);
} else {
int32_t kFloat32ExponentBias = 127;
int32_t kFloat32MantissaBits = 23;
int32_t kFloat32ExponentBits = 8;
Label done;
if (!IsDoubleZeroRegSet()) {
Move(kDoubleRegZero, 0.0);
}
mfc1(scratch, src);
Ext(at, scratch, kFloat32MantissaBits, kFloat32ExponentBits);
Branch(USE_DELAY_SLOT, &done, hs, at,
Operand(kFloat32ExponentBias + kFloat32MantissaBits));
// Canonicalize the result.
sub_s(dst, src, kDoubleRegZero);
round(this, dst, src);
mfc1(at, dst);
Branch(USE_DELAY_SLOT, &done, ne, at, Operand(zero_reg));
cvt_s_w(dst, dst);
srl(at, scratch, 31);
sll(at, at, 31);
mtc1(at, dst);
bind(&done);
}
}
void TurboAssembler::Floor_s_s(FPURegister dst, FPURegister src) {
RoundFloat(dst, src, mode_floor,
[](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
tasm->floor_w_s(dst, src);
});
}
void TurboAssembler::Ceil_s_s(FPURegister dst, FPURegister src) {
RoundFloat(dst, src, mode_ceil,
[](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
tasm->ceil_w_s(dst, src);
});
}
void TurboAssembler::Trunc_s_s(FPURegister dst, FPURegister src) {
RoundFloat(dst, src, mode_trunc,
[](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
tasm->trunc_w_s(dst, src);
});
}
void TurboAssembler::Round_s_s(FPURegister dst, FPURegister src) {
RoundFloat(dst, src, mode_round,
[](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
tasm->round_w_s(dst, src);
});
}
void TurboAssembler::LoadLane(MSASize sz, MSARegister dst, uint8_t laneidx,
MemOperand src) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
switch (sz) {
case MSA_B:
Lbu(scratch, src);
insert_b(dst, laneidx, scratch);
break;
case MSA_H:
Lhu(scratch, src);
insert_h(dst, laneidx, scratch);
break;
case MSA_W:
Lwu(scratch, src);
insert_w(dst, laneidx, scratch);
break;
case MSA_D:
Ld(scratch, src);
insert_d(dst, laneidx, scratch);
break;
default:
UNREACHABLE();
}
}
void TurboAssembler::StoreLane(MSASize sz, MSARegister src, uint8_t laneidx,
MemOperand dst) {
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
switch (sz) {
case MSA_B:
copy_u_b(scratch, src, laneidx);
Sb(scratch, dst);
break;
case MSA_H:
copy_u_h(scratch, src, laneidx);
Sh(scratch, dst);
break;
case MSA_W:
if (laneidx == 0) {
FPURegister src_reg = FPURegister::from_code(src.code());
Swc1(src_reg, dst);
} else {
copy_u_w(scratch, src, laneidx);
Sw(scratch, dst);
}
break;
case MSA_D:
if (laneidx == 0) {
FPURegister src_reg = FPURegister::from_code(src.code());
Sdc1(src_reg, dst);
} else {
copy_s_d(scratch, src, laneidx);
Sd(scratch, dst);
}
break;
default:
UNREACHABLE();
}
}
#define EXT_MUL_BINOP(type, ilv_instr, dotp_instr) \
case type: \
xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero); \
ilv_instr(kSimd128ScratchReg, kSimd128RegZero, src1); \
ilv_instr(kSimd128RegZero, kSimd128RegZero, src2); \
dotp_instr(dst, kSimd128ScratchReg, kSimd128RegZero); \
break;
void TurboAssembler::ExtMulLow(MSADataType type, MSARegister dst,
MSARegister src1, MSARegister src2) {
switch (type) {
EXT_MUL_BINOP(MSAS8, ilvr_b, dotp_s_h)
EXT_MUL_BINOP(MSAS16, ilvr_h, dotp_s_w)