| // Copyright 2013 the V8 project authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #ifndef V8_CODEGEN_ARM64_ASSEMBLER_ARM64_INL_H_ |
| #define V8_CODEGEN_ARM64_ASSEMBLER_ARM64_INL_H_ |
| |
| #include <type_traits> |
| |
| #include "src/base/memory.h" |
| #include "src/codegen/arm64/assembler-arm64.h" |
| #include "src/codegen/assembler.h" |
| #include "src/debug/debug.h" |
| #include "src/objects/objects-inl.h" |
| #include "src/objects/smi.h" |
| |
| namespace v8 { |
| namespace internal { |
| |
| bool CpuFeatures::SupportsOptimizer() { return true; } |
| |
| void RelocInfo::apply(intptr_t delta) { |
| // On arm64 only internal references and immediate branches need extra work. |
| if (RelocInfo::IsInternalReference(rmode_)) { |
| // Absolute code pointer inside code object moves with the code object. |
| intptr_t internal_ref = ReadUnalignedValue<intptr_t>(pc_); |
| internal_ref += delta; // Relocate entry. |
| WriteUnalignedValue<intptr_t>(pc_, internal_ref); |
| } else { |
| Instruction* instr = reinterpret_cast<Instruction*>(pc_); |
| if (instr->IsBranchAndLink() || instr->IsUnconditionalBranch()) { |
| Address old_target = |
| reinterpret_cast<Address>(instr->ImmPCOffsetTarget()); |
| Address new_target = old_target - delta; |
| instr->SetBranchImmTarget(reinterpret_cast<Instruction*>(new_target)); |
| } |
| } |
| } |
| |
| inline bool CPURegister::IsSameSizeAndType(const CPURegister& other) const { |
| return (reg_size_ == other.reg_size_) && (reg_type_ == other.reg_type_); |
| } |
| |
| inline bool CPURegister::IsZero() const { |
| DCHECK(is_valid()); |
| return IsRegister() && (code() == kZeroRegCode); |
| } |
| |
| inline bool CPURegister::IsSP() const { |
| DCHECK(is_valid()); |
| return IsRegister() && (code() == kSPRegInternalCode); |
| } |
| |
| inline void CPURegList::Combine(const CPURegList& other) { |
| DCHECK(other.type() == type_); |
| DCHECK(other.RegisterSizeInBits() == size_); |
| list_ |= other.list_; |
| } |
| |
| inline void CPURegList::Remove(const CPURegList& other) { |
| if (other.type() == type_) { |
| list_ &= ~other.list_; |
| } |
| } |
| |
| inline void CPURegList::Combine(const CPURegister& other) { |
| DCHECK(other.type() == type_); |
| DCHECK(other.SizeInBits() == size_); |
| Combine(other.code()); |
| } |
| |
| inline void CPURegList::Remove(const CPURegister& other1, |
| const CPURegister& other2, |
| const CPURegister& other3, |
| const CPURegister& other4) { |
| if (!other1.IsNone() && (other1.type() == type_)) Remove(other1.code()); |
| if (!other2.IsNone() && (other2.type() == type_)) Remove(other2.code()); |
| if (!other3.IsNone() && (other3.type() == type_)) Remove(other3.code()); |
| if (!other4.IsNone() && (other4.type() == type_)) Remove(other4.code()); |
| } |
| |
| inline void CPURegList::Combine(int code) { |
| DCHECK(CPURegister::Create(code, size_, type_).is_valid()); |
| list_ |= (1ULL << code); |
| DCHECK(is_valid()); |
| } |
| |
| inline void CPURegList::Remove(int code) { |
| DCHECK(CPURegister::Create(code, size_, type_).is_valid()); |
| list_ &= ~(1ULL << code); |
| } |
| |
| inline Register Register::XRegFromCode(unsigned code) { |
| if (code == kSPRegInternalCode) { |
| return sp; |
| } else { |
| DCHECK_LT(code, static_cast<unsigned>(kNumberOfRegisters)); |
| return Register::Create(code, kXRegSizeInBits); |
| } |
| } |
| |
| inline Register Register::WRegFromCode(unsigned code) { |
| if (code == kSPRegInternalCode) { |
| return wsp; |
| } else { |
| DCHECK_LT(code, static_cast<unsigned>(kNumberOfRegisters)); |
| return Register::Create(code, kWRegSizeInBits); |
| } |
| } |
| |
| inline VRegister VRegister::BRegFromCode(unsigned code) { |
| DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters)); |
| return VRegister::Create(code, kBRegSizeInBits); |
| } |
| |
| inline VRegister VRegister::HRegFromCode(unsigned code) { |
| DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters)); |
| return VRegister::Create(code, kHRegSizeInBits); |
| } |
| |
| inline VRegister VRegister::SRegFromCode(unsigned code) { |
| DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters)); |
| return VRegister::Create(code, kSRegSizeInBits); |
| } |
| |
| inline VRegister VRegister::DRegFromCode(unsigned code) { |
| DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters)); |
| return VRegister::Create(code, kDRegSizeInBits); |
| } |
| |
| inline VRegister VRegister::QRegFromCode(unsigned code) { |
| DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters)); |
| return VRegister::Create(code, kQRegSizeInBits); |
| } |
| |
| inline VRegister VRegister::VRegFromCode(unsigned code) { |
| DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters)); |
| return VRegister::Create(code, kVRegSizeInBits); |
| } |
| |
| inline Register CPURegister::W() const { |
| DCHECK(IsRegister()); |
| return Register::WRegFromCode(code()); |
| } |
| |
| inline Register CPURegister::Reg() const { |
| DCHECK(IsRegister()); |
| return Register::Create(code(), reg_size_); |
| } |
| |
| inline VRegister CPURegister::VReg() const { |
| DCHECK(IsVRegister()); |
| return VRegister::Create(code(), reg_size_); |
| } |
| |
| inline Register CPURegister::X() const { |
| DCHECK(IsRegister()); |
| return Register::XRegFromCode(code()); |
| } |
| |
| inline VRegister CPURegister::V() const { |
| DCHECK(IsVRegister()); |
| return VRegister::VRegFromCode(code()); |
| } |
| |
| inline VRegister CPURegister::B() const { |
| DCHECK(IsVRegister()); |
| return VRegister::BRegFromCode(code()); |
| } |
| |
| inline VRegister CPURegister::H() const { |
| DCHECK(IsVRegister()); |
| return VRegister::HRegFromCode(code()); |
| } |
| |
| inline VRegister CPURegister::S() const { |
| DCHECK(IsVRegister()); |
| return VRegister::SRegFromCode(code()); |
| } |
| |
| inline VRegister CPURegister::D() const { |
| DCHECK(IsVRegister()); |
| return VRegister::DRegFromCode(code()); |
| } |
| |
| inline VRegister CPURegister::Q() const { |
| DCHECK(IsVRegister()); |
| return VRegister::QRegFromCode(code()); |
| } |
| |
| // Immediate. |
| // Default initializer is for int types |
| template <typename T> |
| struct ImmediateInitializer { |
| static inline RelocInfo::Mode rmode_for(T) { return RelocInfo::NO_INFO; } |
| static inline int64_t immediate_for(T t) { |
| STATIC_ASSERT(sizeof(T) <= 8); |
| STATIC_ASSERT(std::is_integral<T>::value || std::is_enum<T>::value); |
| return t; |
| } |
| }; |
| |
| template <> |
| struct ImmediateInitializer<Smi> { |
| static inline RelocInfo::Mode rmode_for(Smi t) { return RelocInfo::NO_INFO; } |
| static inline int64_t immediate_for(Smi t) { |
| return static_cast<int64_t>(t.ptr()); |
| } |
| }; |
| |
| template <> |
| struct ImmediateInitializer<ExternalReference> { |
| static inline RelocInfo::Mode rmode_for(ExternalReference t) { |
| return RelocInfo::EXTERNAL_REFERENCE; |
| } |
| static inline int64_t immediate_for(ExternalReference t) { |
| return static_cast<int64_t>(t.address()); |
| } |
| }; |
| |
| template <typename T> |
| Immediate::Immediate(Handle<T> handle, RelocInfo::Mode mode) |
| : value_(static_cast<intptr_t>(handle.address())), rmode_(mode) { |
| DCHECK(RelocInfo::IsEmbeddedObjectMode(mode)); |
| } |
| |
| template <typename T> |
| Immediate::Immediate(T t) |
| : value_(ImmediateInitializer<T>::immediate_for(t)), |
| rmode_(ImmediateInitializer<T>::rmode_for(t)) {} |
| |
| template <typename T> |
| Immediate::Immediate(T t, RelocInfo::Mode rmode) |
| : value_(ImmediateInitializer<T>::immediate_for(t)), rmode_(rmode) { |
| STATIC_ASSERT(std::is_integral<T>::value); |
| } |
| |
| template <typename T> |
| Operand::Operand(T t) : immediate_(t), reg_(NoReg) {} |
| |
| template <typename T> |
| Operand::Operand(T t, RelocInfo::Mode rmode) |
| : immediate_(t, rmode), reg_(NoReg) {} |
| |
| Operand::Operand(Register reg, Shift shift, unsigned shift_amount) |
| : immediate_(0), |
| reg_(reg), |
| shift_(shift), |
| extend_(NO_EXTEND), |
| shift_amount_(shift_amount) { |
| DCHECK(reg.Is64Bits() || (shift_amount < kWRegSizeInBits)); |
| DCHECK(reg.Is32Bits() || (shift_amount < kXRegSizeInBits)); |
| DCHECK_IMPLIES(reg.IsSP(), shift_amount == 0); |
| } |
| |
| Operand::Operand(Register reg, Extend extend, unsigned shift_amount) |
| : immediate_(0), |
| reg_(reg), |
| shift_(NO_SHIFT), |
| extend_(extend), |
| shift_amount_(shift_amount) { |
| DCHECK(reg.is_valid()); |
| DCHECK_LE(shift_amount, 4); |
| DCHECK(!reg.IsSP()); |
| |
| // Extend modes SXTX and UXTX require a 64-bit register. |
| DCHECK(reg.Is64Bits() || ((extend != SXTX) && (extend != UXTX))); |
| } |
| |
| bool Operand::IsHeapObjectRequest() const { |
| DCHECK_IMPLIES(heap_object_request_.has_value(), reg_ == NoReg); |
| DCHECK_IMPLIES(heap_object_request_.has_value(), |
| immediate_.rmode() == RelocInfo::FULL_EMBEDDED_OBJECT || |
| immediate_.rmode() == RelocInfo::CODE_TARGET); |
| return heap_object_request_.has_value(); |
| } |
| |
| HeapObjectRequest Operand::heap_object_request() const { |
| DCHECK(IsHeapObjectRequest()); |
| return *heap_object_request_; |
| } |
| |
| bool Operand::IsImmediate() const { |
| return reg_ == NoReg && !IsHeapObjectRequest(); |
| } |
| |
| bool Operand::IsShiftedRegister() const { |
| return reg_.is_valid() && (shift_ != NO_SHIFT); |
| } |
| |
| bool Operand::IsExtendedRegister() const { |
| return reg_.is_valid() && (extend_ != NO_EXTEND); |
| } |
| |
| bool Operand::IsZero() const { |
| if (IsImmediate()) { |
| return ImmediateValue() == 0; |
| } else { |
| return reg().IsZero(); |
| } |
| } |
| |
| Operand Operand::ToExtendedRegister() const { |
| DCHECK(IsShiftedRegister()); |
| DCHECK((shift_ == LSL) && (shift_amount_ <= 4)); |
| return Operand(reg_, reg_.Is64Bits() ? UXTX : UXTW, shift_amount_); |
| } |
| |
| Operand Operand::ToW() const { |
| if (IsShiftedRegister()) { |
| DCHECK(reg_.Is64Bits()); |
| return Operand(reg_.W(), shift(), shift_amount()); |
| } else if (IsExtendedRegister()) { |
| DCHECK(reg_.Is64Bits()); |
| return Operand(reg_.W(), extend(), shift_amount()); |
| } |
| DCHECK(IsImmediate()); |
| return *this; |
| } |
| |
| Immediate Operand::immediate_for_heap_object_request() const { |
| DCHECK((heap_object_request().kind() == HeapObjectRequest::kHeapNumber && |
| immediate_.rmode() == RelocInfo::FULL_EMBEDDED_OBJECT) || |
| (heap_object_request().kind() == HeapObjectRequest::kStringConstant && |
| immediate_.rmode() == RelocInfo::FULL_EMBEDDED_OBJECT)); |
| return immediate_; |
| } |
| |
| Immediate Operand::immediate() const { |
| DCHECK(IsImmediate()); |
| return immediate_; |
| } |
| |
| int64_t Operand::ImmediateValue() const { |
| DCHECK(IsImmediate()); |
| return immediate_.value(); |
| } |
| |
| RelocInfo::Mode Operand::ImmediateRMode() const { |
| DCHECK(IsImmediate() || IsHeapObjectRequest()); |
| return immediate_.rmode(); |
| } |
| |
| Register Operand::reg() const { |
| DCHECK(IsShiftedRegister() || IsExtendedRegister()); |
| return reg_; |
| } |
| |
| Shift Operand::shift() const { |
| DCHECK(IsShiftedRegister()); |
| return shift_; |
| } |
| |
| Extend Operand::extend() const { |
| DCHECK(IsExtendedRegister()); |
| return extend_; |
| } |
| |
| unsigned Operand::shift_amount() const { |
| DCHECK(IsShiftedRegister() || IsExtendedRegister()); |
| return shift_amount_; |
| } |
| |
| MemOperand::MemOperand() |
| : base_(NoReg), |
| regoffset_(NoReg), |
| offset_(0), |
| addrmode_(Offset), |
| shift_(NO_SHIFT), |
| extend_(NO_EXTEND), |
| shift_amount_(0) {} |
| |
| MemOperand::MemOperand(Register base, int64_t offset, AddrMode addrmode) |
| : base_(base), |
| regoffset_(NoReg), |
| offset_(offset), |
| addrmode_(addrmode), |
| shift_(NO_SHIFT), |
| extend_(NO_EXTEND), |
| shift_amount_(0) { |
| DCHECK(base.Is64Bits() && !base.IsZero()); |
| } |
| |
| MemOperand::MemOperand(Register base, Register regoffset, Extend extend, |
| unsigned shift_amount) |
| : base_(base), |
| regoffset_(regoffset), |
| offset_(0), |
| addrmode_(Offset), |
| shift_(NO_SHIFT), |
| extend_(extend), |
| shift_amount_(shift_amount) { |
| DCHECK(base.Is64Bits() && !base.IsZero()); |
| DCHECK(!regoffset.IsSP()); |
| DCHECK((extend == UXTW) || (extend == SXTW) || (extend == SXTX)); |
| |
| // SXTX extend mode requires a 64-bit offset register. |
| DCHECK(regoffset.Is64Bits() || (extend != SXTX)); |
| } |
| |
| MemOperand::MemOperand(Register base, Register regoffset, Shift shift, |
| unsigned shift_amount) |
| : base_(base), |
| regoffset_(regoffset), |
| offset_(0), |
| addrmode_(Offset), |
| shift_(shift), |
| extend_(NO_EXTEND), |
| shift_amount_(shift_amount) { |
| DCHECK(base.Is64Bits() && !base.IsZero()); |
| DCHECK(regoffset.Is64Bits() && !regoffset.IsSP()); |
| DCHECK(shift == LSL); |
| } |
| |
| MemOperand::MemOperand(Register base, const Operand& offset, AddrMode addrmode) |
| : base_(base), regoffset_(NoReg), addrmode_(addrmode) { |
| DCHECK(base.Is64Bits() && !base.IsZero()); |
| |
| if (offset.IsImmediate()) { |
| offset_ = offset.ImmediateValue(); |
| } else if (offset.IsShiftedRegister()) { |
| DCHECK((addrmode == Offset) || (addrmode == PostIndex)); |
| |
| regoffset_ = offset.reg(); |
| shift_ = offset.shift(); |
| shift_amount_ = offset.shift_amount(); |
| |
| extend_ = NO_EXTEND; |
| offset_ = 0; |
| |
| // These assertions match those in the shifted-register constructor. |
| DCHECK(regoffset_.Is64Bits() && !regoffset_.IsSP()); |
| DCHECK(shift_ == LSL); |
| } else { |
| DCHECK(offset.IsExtendedRegister()); |
| DCHECK(addrmode == Offset); |
| |
| regoffset_ = offset.reg(); |
| extend_ = offset.extend(); |
| shift_amount_ = offset.shift_amount(); |
| |
| shift_ = NO_SHIFT; |
| offset_ = 0; |
| |
| // These assertions match those in the extended-register constructor. |
| DCHECK(!regoffset_.IsSP()); |
| DCHECK((extend_ == UXTW) || (extend_ == SXTW) || (extend_ == SXTX)); |
| DCHECK((regoffset_.Is64Bits() || (extend_ != SXTX))); |
| } |
| } |
| |
| bool MemOperand::IsImmediateOffset() const { |
| return (addrmode_ == Offset) && regoffset_ == NoReg; |
| } |
| |
| bool MemOperand::IsRegisterOffset() const { |
| return (addrmode_ == Offset) && regoffset_ != NoReg; |
| } |
| |
| bool MemOperand::IsPreIndex() const { return addrmode_ == PreIndex; } |
| |
| bool MemOperand::IsPostIndex() const { return addrmode_ == PostIndex; } |
| |
| void Assembler::Unreachable() { debug("UNREACHABLE", __LINE__, BREAK); } |
| |
| Address Assembler::target_pointer_address_at(Address pc) { |
| Instruction* instr = reinterpret_cast<Instruction*>(pc); |
| DCHECK(instr->IsLdrLiteralX() || instr->IsLdrLiteralW()); |
| return reinterpret_cast<Address>(instr->ImmPCOffsetTarget()); |
| } |
| |
| // Read/Modify the code target address in the branch/call instruction at pc. |
| Address Assembler::target_address_at(Address pc, Address constant_pool) { |
| Instruction* instr = reinterpret_cast<Instruction*>(pc); |
| if (instr->IsLdrLiteralX()) { |
| return Memory<Address>(target_pointer_address_at(pc)); |
| } else { |
| DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch()); |
| return reinterpret_cast<Address>(instr->ImmPCOffsetTarget()); |
| } |
| } |
| |
| Tagged_t Assembler::target_compressed_address_at(Address pc, |
| Address constant_pool) { |
| Instruction* instr = reinterpret_cast<Instruction*>(pc); |
| CHECK(instr->IsLdrLiteralW()); |
| return Memory<Tagged_t>(target_pointer_address_at(pc)); |
| } |
| |
| Handle<CodeT> Assembler::code_target_object_handle_at(Address pc) { |
| Instruction* instr = reinterpret_cast<Instruction*>(pc); |
| if (instr->IsLdrLiteralX()) { |
| return Handle<CodeT>(reinterpret_cast<Address*>( |
| Assembler::target_address_at(pc, 0 /* unused */))); |
| } else { |
| DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch()); |
| DCHECK_EQ(instr->ImmPCOffset() % kInstrSize, 0); |
| return Handle<CodeT>::cast( |
| GetEmbeddedObject(instr->ImmPCOffset() >> kInstrSizeLog2)); |
| } |
| } |
| |
| AssemblerBase::EmbeddedObjectIndex |
| Assembler::embedded_object_index_referenced_from(Address pc) { |
| Instruction* instr = reinterpret_cast<Instruction*>(pc); |
| if (instr->IsLdrLiteralX()) { |
| STATIC_ASSERT(sizeof(EmbeddedObjectIndex) == sizeof(intptr_t)); |
| return Memory<EmbeddedObjectIndex>(target_pointer_address_at(pc)); |
| } else { |
| DCHECK(instr->IsLdrLiteralW()); |
| return Memory<uint32_t>(target_pointer_address_at(pc)); |
| } |
| } |
| |
| void Assembler::set_embedded_object_index_referenced_from( |
| Address pc, EmbeddedObjectIndex data) { |
| Instruction* instr = reinterpret_cast<Instruction*>(pc); |
| if (instr->IsLdrLiteralX()) { |
| Memory<EmbeddedObjectIndex>(target_pointer_address_at(pc)) = data; |
| } else { |
| DCHECK(instr->IsLdrLiteralW()); |
| DCHECK(is_uint32(data)); |
| WriteUnalignedValue<uint32_t>(target_pointer_address_at(pc), |
| static_cast<uint32_t>(data)); |
| } |
| } |
| |
| Handle<HeapObject> Assembler::target_object_handle_at(Address pc) { |
| return GetEmbeddedObject( |
| Assembler::embedded_object_index_referenced_from(pc)); |
| } |
| |
| Address Assembler::runtime_entry_at(Address pc) { |
| Instruction* instr = reinterpret_cast<Instruction*>(pc); |
| if (instr->IsLdrLiteralX()) { |
| return Assembler::target_address_at(pc, 0 /* unused */); |
| } else { |
| DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch()); |
| return instr->ImmPCOffset() + options().code_range_base; |
| } |
| } |
| |
| int Assembler::deserialization_special_target_size(Address location) { |
| Instruction* instr = reinterpret_cast<Instruction*>(location); |
| if (instr->IsBranchAndLink() || instr->IsUnconditionalBranch()) { |
| return kSpecialTargetSize; |
| } else { |
| DCHECK_EQ(instr->InstructionBits(), 0); |
| return kSystemPointerSize; |
| } |
| } |
| |
| void Assembler::deserialization_set_special_target_at(Address location, |
| Code code, |
| Address target) { |
| Instruction* instr = reinterpret_cast<Instruction*>(location); |
| if (instr->IsBranchAndLink() || instr->IsUnconditionalBranch()) { |
| if (target == 0) { |
| // We are simply wiping the target out for serialization. Set the offset |
| // to zero instead. |
| target = location; |
| } |
| instr->SetBranchImmTarget(reinterpret_cast<Instruction*>(target)); |
| FlushInstructionCache(location, kInstrSize); |
| } else { |
| DCHECK_EQ(instr->InstructionBits(), 0); |
| Memory<Address>(location) = target; |
| // Intuitively, we would think it is necessary to always flush the |
| // instruction cache after patching a target address in the code. However, |
| // in this case, only the constant pool contents change. The instruction |
| // accessing the constant pool remains unchanged, so a flush is not |
| // required. |
| } |
| } |
| |
| void Assembler::deserialization_set_target_internal_reference_at( |
| Address pc, Address target, RelocInfo::Mode mode) { |
| WriteUnalignedValue<Address>(pc, target); |
| } |
| |
| void Assembler::set_target_address_at(Address pc, Address constant_pool, |
| Address target, |
| ICacheFlushMode icache_flush_mode) { |
| Instruction* instr = reinterpret_cast<Instruction*>(pc); |
| if (instr->IsLdrLiteralX()) { |
| Memory<Address>(target_pointer_address_at(pc)) = target; |
| // Intuitively, we would think it is necessary to always flush the |
| // instruction cache after patching a target address in the code. However, |
| // in this case, only the constant pool contents change. The instruction |
| // accessing the constant pool remains unchanged, so a flush is not |
| // required. |
| } else { |
| DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch()); |
| if (target == 0) { |
| // We are simply wiping the target out for serialization. Set the offset |
| // to zero instead. |
| target = pc; |
| } |
| instr->SetBranchImmTarget(reinterpret_cast<Instruction*>(target)); |
| if (icache_flush_mode != SKIP_ICACHE_FLUSH) { |
| FlushInstructionCache(pc, kInstrSize); |
| } |
| } |
| } |
| |
| void Assembler::set_target_compressed_address_at( |
| Address pc, Address constant_pool, Tagged_t target, |
| ICacheFlushMode icache_flush_mode) { |
| Instruction* instr = reinterpret_cast<Instruction*>(pc); |
| CHECK(instr->IsLdrLiteralW()); |
| Memory<Tagged_t>(target_pointer_address_at(pc)) = target; |
| } |
| |
| int RelocInfo::target_address_size() { |
| if (IsCodedSpecially()) { |
| return Assembler::kSpecialTargetSize; |
| } else { |
| Instruction* instr = reinterpret_cast<Instruction*>(pc_); |
| DCHECK(instr->IsLdrLiteralX() || instr->IsLdrLiteralW()); |
| return instr->IsLdrLiteralW() ? kTaggedSize : kSystemPointerSize; |
| } |
| } |
| |
| Address RelocInfo::target_address() { |
| DCHECK(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_) || IsWasmCall(rmode_)); |
| return Assembler::target_address_at(pc_, constant_pool_); |
| } |
| |
| Address RelocInfo::target_address_address() { |
| DCHECK(HasTargetAddressAddress()); |
| Instruction* instr = reinterpret_cast<Instruction*>(pc_); |
| // Read the address of the word containing the target_address in an |
| // instruction stream. |
| // The only architecture-independent user of this function is the serializer. |
| // The serializer uses it to find out how many raw bytes of instruction to |
| // output before the next target. |
| // For an instruction like B/BL, where the target bits are mixed into the |
| // instruction bits, the size of the target will be zero, indicating that the |
| // serializer should not step forward in memory after a target is resolved |
| // and written. |
| // For LDR literal instructions, we can skip up to the constant pool entry |
| // address. We make sure that RelocInfo is ordered by the |
| // target_address_address so that we do not skip over any relocatable |
| // instruction sequences. |
| if (instr->IsLdrLiteralX()) { |
| return constant_pool_entry_address(); |
| } else { |
| DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch()); |
| return pc_; |
| } |
| } |
| |
| Address RelocInfo::constant_pool_entry_address() { |
| DCHECK(IsInConstantPool()); |
| return Assembler::target_pointer_address_at(pc_); |
| } |
| |
| HeapObject RelocInfo::target_object(PtrComprCageBase cage_base) { |
| DCHECK(IsCodeTarget(rmode_) || IsEmbeddedObjectMode(rmode_)); |
| if (IsDataEmbeddedObject(rmode_)) { |
| return HeapObject::cast(Object(ReadUnalignedValue<Address>(pc_))); |
| } else if (IsCompressedEmbeddedObject(rmode_)) { |
| Tagged_t compressed = |
| Assembler::target_compressed_address_at(pc_, constant_pool_); |
| DCHECK(!HAS_SMI_TAG(compressed)); |
| Object obj(DecompressTaggedPointer(cage_base, compressed)); |
| // Embedding of compressed Code objects must not happen when external code |
| // space is enabled, because CodeDataContainers must be used instead. |
| DCHECK_IMPLIES(V8_EXTERNAL_CODE_SPACE_BOOL, |
| !IsCodeSpaceObject(HeapObject::cast(obj))); |
| return HeapObject::cast(obj); |
| } else { |
| return HeapObject::cast( |
| Object(Assembler::target_address_at(pc_, constant_pool_))); |
| } |
| } |
| |
| Handle<HeapObject> RelocInfo::target_object_handle(Assembler* origin) { |
| if (IsDataEmbeddedObject(rmode_)) { |
| return Handle<HeapObject>::cast(ReadUnalignedValue<Handle<Object>>(pc_)); |
| } else if (IsEmbeddedObjectMode(rmode_)) { |
| return origin->target_object_handle_at(pc_); |
| } else { |
| DCHECK(IsCodeTarget(rmode_)); |
| return origin->code_target_object_handle_at(pc_); |
| } |
| } |
| |
| void RelocInfo::set_target_object(Heap* heap, HeapObject target, |
| WriteBarrierMode write_barrier_mode, |
| ICacheFlushMode icache_flush_mode) { |
| DCHECK(IsCodeTarget(rmode_) || IsEmbeddedObjectMode(rmode_)); |
| if (IsDataEmbeddedObject(rmode_)) { |
| WriteUnalignedValue(pc_, target.ptr()); |
| // No need to flush icache since no instructions were changed. |
| } else if (IsCompressedEmbeddedObject(rmode_)) { |
| Assembler::set_target_compressed_address_at( |
| pc_, constant_pool_, CompressTagged(target.ptr()), icache_flush_mode); |
| } else { |
| DCHECK(IsFullEmbeddedObject(rmode_)); |
| Assembler::set_target_address_at(pc_, constant_pool_, target.ptr(), |
| icache_flush_mode); |
| } |
| if (write_barrier_mode == UPDATE_WRITE_BARRIER && !host().is_null() && |
| !FLAG_disable_write_barriers) { |
| WriteBarrierForCode(host(), this, target); |
| } |
| } |
| |
| Address RelocInfo::target_external_reference() { |
| DCHECK(rmode_ == EXTERNAL_REFERENCE); |
| return Assembler::target_address_at(pc_, constant_pool_); |
| } |
| |
| void RelocInfo::set_target_external_reference( |
| Address target, ICacheFlushMode icache_flush_mode) { |
| DCHECK(rmode_ == RelocInfo::EXTERNAL_REFERENCE); |
| Assembler::set_target_address_at(pc_, constant_pool_, target, |
| icache_flush_mode); |
| } |
| |
| Address RelocInfo::target_internal_reference() { |
| DCHECK(rmode_ == INTERNAL_REFERENCE); |
| return ReadUnalignedValue<Address>(pc_); |
| } |
| |
| Address RelocInfo::target_internal_reference_address() { |
| DCHECK(rmode_ == INTERNAL_REFERENCE); |
| return pc_; |
| } |
| |
| Address RelocInfo::target_runtime_entry(Assembler* origin) { |
| DCHECK(IsRuntimeEntry(rmode_)); |
| return origin->runtime_entry_at(pc_); |
| } |
| |
| void RelocInfo::set_target_runtime_entry(Address target, |
| WriteBarrierMode write_barrier_mode, |
| ICacheFlushMode icache_flush_mode) { |
| DCHECK(IsRuntimeEntry(rmode_)); |
| if (target_address() != target) { |
| set_target_address(target, write_barrier_mode, icache_flush_mode); |
| } |
| } |
| |
| Address RelocInfo::target_off_heap_target() { |
| DCHECK(IsOffHeapTarget(rmode_)); |
| return Assembler::target_address_at(pc_, constant_pool_); |
| } |
| |
| void RelocInfo::WipeOut() { |
| DCHECK(IsEmbeddedObjectMode(rmode_) || IsCodeTarget(rmode_) || |
| IsRuntimeEntry(rmode_) || IsExternalReference(rmode_) || |
| IsInternalReference(rmode_) || IsOffHeapTarget(rmode_)); |
| if (IsInternalReference(rmode_)) { |
| WriteUnalignedValue<Address>(pc_, kNullAddress); |
| } else if (IsCompressedEmbeddedObject(rmode_)) { |
| Assembler::set_target_compressed_address_at(pc_, constant_pool_, |
| kNullAddress); |
| } else { |
| Assembler::set_target_address_at(pc_, constant_pool_, kNullAddress); |
| } |
| } |
| |
| LoadStoreOp Assembler::LoadOpFor(const CPURegister& rt) { |
| DCHECK(rt.is_valid()); |
| if (rt.IsRegister()) { |
| return rt.Is64Bits() ? LDR_x : LDR_w; |
| } else { |
| DCHECK(rt.IsVRegister()); |
| switch (rt.SizeInBits()) { |
| case kBRegSizeInBits: |
| return LDR_b; |
| case kHRegSizeInBits: |
| return LDR_h; |
| case kSRegSizeInBits: |
| return LDR_s; |
| case kDRegSizeInBits: |
| return LDR_d; |
| default: |
| DCHECK(rt.IsQ()); |
| return LDR_q; |
| } |
| } |
| } |
| |
| LoadStoreOp Assembler::StoreOpFor(const CPURegister& rt) { |
| DCHECK(rt.is_valid()); |
| if (rt.IsRegister()) { |
| return rt.Is64Bits() ? STR_x : STR_w; |
| } else { |
| DCHECK(rt.IsVRegister()); |
| switch (rt.SizeInBits()) { |
| case kBRegSizeInBits: |
| return STR_b; |
| case kHRegSizeInBits: |
| return STR_h; |
| case kSRegSizeInBits: |
| return STR_s; |
| case kDRegSizeInBits: |
| return STR_d; |
| default: |
| DCHECK(rt.IsQ()); |
| return STR_q; |
| } |
| } |
| } |
| |
| LoadStorePairOp Assembler::LoadPairOpFor(const CPURegister& rt, |
| const CPURegister& rt2) { |
| DCHECK_EQ(STP_w | LoadStorePairLBit, LDP_w); |
| return static_cast<LoadStorePairOp>(StorePairOpFor(rt, rt2) | |
| LoadStorePairLBit); |
| } |
| |
| LoadStorePairOp Assembler::StorePairOpFor(const CPURegister& rt, |
| const CPURegister& rt2) { |
| DCHECK(AreSameSizeAndType(rt, rt2)); |
| USE(rt2); |
| if (rt.IsRegister()) { |
| return rt.Is64Bits() ? STP_x : STP_w; |
| } else { |
| DCHECK(rt.IsVRegister()); |
| switch (rt.SizeInBits()) { |
| case kSRegSizeInBits: |
| return STP_s; |
| case kDRegSizeInBits: |
| return STP_d; |
| default: |
| DCHECK(rt.IsQ()); |
| return STP_q; |
| } |
| } |
| } |
| |
| LoadLiteralOp Assembler::LoadLiteralOpFor(const CPURegister& rt) { |
| if (rt.IsRegister()) { |
| return rt.Is64Bits() ? LDR_x_lit : LDR_w_lit; |
| } else { |
| DCHECK(rt.IsVRegister()); |
| return rt.Is64Bits() ? LDR_d_lit : LDR_s_lit; |
| } |
| } |
| |
| int Assembler::LinkAndGetInstructionOffsetTo(Label* label) { |
| DCHECK_EQ(kStartOfLabelLinkChain, 0); |
| int offset = LinkAndGetByteOffsetTo(label); |
| DCHECK(IsAligned(offset, kInstrSize)); |
| return offset >> kInstrSizeLog2; |
| } |
| |
| Instr Assembler::Flags(FlagsUpdate S) { |
| if (S == SetFlags) { |
| return 1 << FlagsUpdate_offset; |
| } else if (S == LeaveFlags) { |
| return 0 << FlagsUpdate_offset; |
| } |
| UNREACHABLE(); |
| } |
| |
| Instr Assembler::Cond(Condition cond) { return cond << Condition_offset; } |
| |
| Instr Assembler::ImmPCRelAddress(int imm21) { |
| CHECK(is_int21(imm21)); |
| Instr imm = static_cast<Instr>(truncate_to_int21(imm21)); |
| Instr immhi = (imm >> ImmPCRelLo_width) << ImmPCRelHi_offset; |
| Instr immlo = imm << ImmPCRelLo_offset; |
| return (immhi & ImmPCRelHi_mask) | (immlo & ImmPCRelLo_mask); |
| } |
| |
| Instr Assembler::ImmUncondBranch(int imm26) { |
| CHECK(is_int26(imm26)); |
| return truncate_to_int26(imm26) << ImmUncondBranch_offset; |
| } |
| |
| Instr Assembler::ImmCondBranch(int imm19) { |
| CHECK(is_int19(imm19)); |
| return truncate_to_int19(imm19) << ImmCondBranch_offset; |
| } |
| |
| Instr Assembler::ImmCmpBranch(int imm19) { |
| CHECK(is_int19(imm19)); |
| return truncate_to_int19(imm19) << ImmCmpBranch_offset; |
| } |
| |
| Instr Assembler::ImmTestBranch(int imm14) { |
| CHECK(is_int14(imm14)); |
| return truncate_to_int14(imm14) << ImmTestBranch_offset; |
| } |
| |
| Instr Assembler::ImmTestBranchBit(unsigned bit_pos) { |
| DCHECK(is_uint6(bit_pos)); |
| // Subtract five from the shift offset, as we need bit 5 from bit_pos. |
| unsigned b5 = bit_pos << (ImmTestBranchBit5_offset - 5); |
| unsigned b40 = bit_pos << ImmTestBranchBit40_offset; |
| b5 &= ImmTestBranchBit5_mask; |
| b40 &= ImmTestBranchBit40_mask; |
| return b5 | b40; |
| } |
| |
| Instr Assembler::SF(Register rd) { |
| return rd.Is64Bits() ? SixtyFourBits : ThirtyTwoBits; |
| } |
| |
| Instr Assembler::ImmAddSub(int imm) { |
| DCHECK(IsImmAddSub(imm)); |
| if (is_uint12(imm)) { // No shift required. |
| imm <<= ImmAddSub_offset; |
| } else { |
| imm = ((imm >> 12) << ImmAddSub_offset) | (1 << ShiftAddSub_offset); |
| } |
| return imm; |
| } |
| |
| Instr Assembler::ImmS(unsigned imms, unsigned reg_size) { |
| DCHECK(((reg_size == kXRegSizeInBits) && is_uint6(imms)) || |
| ((reg_size == kWRegSizeInBits) && is_uint5(imms))); |
| USE(reg_size); |
| return imms << ImmS_offset; |
| } |
| |
| Instr Assembler::ImmR(unsigned immr, unsigned reg_size) { |
| DCHECK(((reg_size == kXRegSizeInBits) && is_uint6(immr)) || |
| ((reg_size == kWRegSizeInBits) && is_uint5(immr))); |
| USE(reg_size); |
| DCHECK(is_uint6(immr)); |
| return immr << ImmR_offset; |
| } |
| |
| Instr Assembler::ImmSetBits(unsigned imms, unsigned reg_size) { |
| DCHECK((reg_size == kWRegSizeInBits) || (reg_size == kXRegSizeInBits)); |
| DCHECK(is_uint6(imms)); |
| DCHECK((reg_size == kXRegSizeInBits) || is_uint6(imms + 3)); |
| USE(reg_size); |
| return imms << ImmSetBits_offset; |
| } |
| |
| Instr Assembler::ImmRotate(unsigned immr, unsigned reg_size) { |
| DCHECK((reg_size == kWRegSizeInBits) || (reg_size == kXRegSizeInBits)); |
| DCHECK(((reg_size == kXRegSizeInBits) && is_uint6(immr)) || |
| ((reg_size == kWRegSizeInBits) && is_uint5(immr))); |
| USE(reg_size); |
| return immr << ImmRotate_offset; |
| } |
| |
| Instr Assembler::ImmLLiteral(int imm19) { |
| CHECK(is_int19(imm19)); |
| return truncate_to_int19(imm19) << ImmLLiteral_offset; |
| } |
| |
| Instr Assembler::BitN(unsigned bitn, unsigned reg_size) { |
| DCHECK((reg_size == kWRegSizeInBits) || (reg_size == kXRegSizeInBits)); |
| DCHECK((reg_size == kXRegSizeInBits) || (bitn == 0)); |
| USE(reg_size); |
| return bitn << BitN_offset; |
| } |
| |
| Instr Assembler::ShiftDP(Shift shift) { |
| DCHECK(shift == LSL || shift == LSR || shift == ASR || shift == ROR); |
| return shift << ShiftDP_offset; |
| } |
| |
| Instr Assembler::ImmDPShift(unsigned amount) { |
| DCHECK(is_uint6(amount)); |
| return amount << ImmDPShift_offset; |
| } |
| |
| Instr Assembler::ExtendMode(Extend extend) { |
| return extend << ExtendMode_offset; |
| } |
| |
| Instr Assembler::ImmExtendShift(unsigned left_shift) { |
| DCHECK_LE(left_shift, 4); |
| return left_shift << ImmExtendShift_offset; |
| } |
| |
| Instr Assembler::ImmCondCmp(unsigned imm) { |
| DCHECK(is_uint5(imm)); |
| return imm << ImmCondCmp_offset; |
| } |
| |
| Instr Assembler::Nzcv(StatusFlags nzcv) { |
| return ((nzcv >> Flags_offset) & 0xf) << Nzcv_offset; |
| } |
| |
| Instr Assembler::ImmLSUnsigned(int imm12) { |
| DCHECK(is_uint12(imm12)); |
| return imm12 << ImmLSUnsigned_offset; |
| } |
| |
| Instr Assembler::ImmLS(int imm9) { |
| DCHECK(is_int9(imm9)); |
| return truncate_to_int9(imm9) << ImmLS_offset; |
| } |
| |
| Instr Assembler::ImmLSPair(int imm7, unsigned size) { |
| DCHECK_EQ(imm7, |
| static_cast<int>(static_cast<uint32_t>(imm7 >> size) << size)); |
| int scaled_imm7 = imm7 >> size; |
| DCHECK(is_int7(scaled_imm7)); |
| return truncate_to_int7(scaled_imm7) << ImmLSPair_offset; |
| } |
| |
| Instr Assembler::ImmShiftLS(unsigned shift_amount) { |
| DCHECK(is_uint1(shift_amount)); |
| return shift_amount << ImmShiftLS_offset; |
| } |
| |
| Instr Assembler::ImmException(int imm16) { |
| DCHECK(is_uint16(imm16)); |
| return imm16 << ImmException_offset; |
| } |
| |
| Instr Assembler::ImmSystemRegister(int imm15) { |
| DCHECK(is_uint15(imm15)); |
| return imm15 << ImmSystemRegister_offset; |
| } |
| |
| Instr Assembler::ImmHint(int imm7) { |
| DCHECK(is_uint7(imm7)); |
| return imm7 << ImmHint_offset; |
| } |
| |
| Instr Assembler::ImmBarrierDomain(int imm2) { |
| DCHECK(is_uint2(imm2)); |
| return imm2 << ImmBarrierDomain_offset; |
| } |
| |
| Instr Assembler::ImmBarrierType(int imm2) { |
| DCHECK(is_uint2(imm2)); |
| return imm2 << ImmBarrierType_offset; |
| } |
| |
| unsigned Assembler::CalcLSDataSize(LoadStoreOp op) { |
| DCHECK((LSSize_offset + LSSize_width) == (kInstrSize * 8)); |
| unsigned size = static_cast<Instr>(op >> LSSize_offset); |
| if ((op & LSVector_mask) != 0) { |
| // Vector register memory operations encode the access size in the "size" |
| // and "opc" fields. |
| if ((size == 0) && ((op & LSOpc_mask) >> LSOpc_offset) >= 2) { |
| size = kQRegSizeLog2; |
| } |
| } |
| return size; |
| } |
| |
| Instr Assembler::ImmMoveWide(int imm) { |
| DCHECK(is_uint16(imm)); |
| return imm << ImmMoveWide_offset; |
| } |
| |
| Instr Assembler::ShiftMoveWide(int shift) { |
| DCHECK(is_uint2(shift)); |
| return shift << ShiftMoveWide_offset; |
| } |
| |
| Instr Assembler::FPType(VRegister fd) { return fd.Is64Bits() ? FP64 : FP32; } |
| |
| Instr Assembler::FPScale(unsigned scale) { |
| DCHECK(is_uint6(scale)); |
| return scale << FPScale_offset; |
| } |
| |
| const Register& Assembler::AppropriateZeroRegFor(const CPURegister& reg) const { |
| return reg.Is64Bits() ? xzr : wzr; |
| } |
| |
| inline void Assembler::CheckBufferSpace() { |
| DCHECK_LT(pc_, buffer_start_ + buffer_->size()); |
| if (V8_UNLIKELY(buffer_space() < kGap)) { |
| GrowBuffer(); |
| } |
| } |
| |
| V8_INLINE void Assembler::CheckBuffer() { |
| CheckBufferSpace(); |
| if (pc_offset() >= next_veneer_pool_check_) { |
| CheckVeneerPool(false, true); |
| } |
| constpool_.MaybeCheck(); |
| } |
| |
| EnsureSpace::EnsureSpace(Assembler* assembler) : block_pools_scope_(assembler) { |
| assembler->CheckBufferSpace(); |
| } |
| |
| } // namespace internal |
| } // namespace v8 |
| |
| #endif // V8_CODEGEN_ARM64_ASSEMBLER_ARM64_INL_H_ |