Remove code/istream fields from RelocInfo .. and simplify RelocIterator uses. The code/istream fields on RelocInfo were (surprisingly) nullable and were actually unset in many cases. It was never clear when one could rely on them being set, and OTOH when one had to pass valid code/istream objects to the ctor. This CL simply removes them and explicitly passes code/istream to the use-site when needed. As a consequence of these weird nullable fields, RelocIterator constructors also accepted nullable code/istream objects, and were similarly hard to understand. They also attempted to handle calls from within a GC context (i.e. in the presence of forwarding pointers) by sprinkling various unchecked casts around. This CL removes unused ctors, and extensively comments the remaining ones. Finally, as a drive-by, various minor refactors in reloc-info.h and friends. No-Try: true Bug: v8:13789 Change-Id: I2ef3c27faaec84e4bff8f42b32c62bfd631ec6c2 Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/4406533 Commit-Queue: Jakob Linke <jgruber@chromium.org> Reviewed-by: Michael Lippautz <mlippautz@chromium.org> Reviewed-by: Igor Sheludko <ishell@chromium.org> Cr-Commit-Position: refs/heads/main@{#87031}
diff --git a/BUILD.bazel b/BUILD.bazel index 4984bea..7047dfe 100644 --- a/BUILD.bazel +++ b/BUILD.bazel
@@ -1250,6 +1250,7 @@ "src/codegen/reglist-base.h", "src/codegen/reloc-info.cc", "src/codegen/reloc-info.h", + "src/codegen/reloc-info-inl.h", "src/codegen/safepoint-table.cc", "src/codegen/safepoint-table.h", "src/codegen/safepoint-table-base.h",
diff --git a/BUILD.gn b/BUILD.gn index 2bb63dc..07db010 100644 --- a/BUILD.gn +++ b/BUILD.gn
@@ -2920,6 +2920,7 @@ "src/codegen/register.h", "src/codegen/reglist-base.h", "src/codegen/reglist.h", + "src/codegen/reloc-info-inl.h", "src/codegen/reloc-info.h", "src/codegen/safepoint-table-base.h", "src/codegen/safepoint-table.h",
diff --git a/src/api/api.cc b/src/api/api.cc index 3f4ea44..8e8d513 100644 --- a/src/api/api.cc +++ b/src/api/api.cc
@@ -6938,8 +6938,6 @@ VisitPointer(host, host.map_slot()); } void VisitCodePointer(i::Code host, i::CodeObjectSlot slot) final {} - void VisitCodeTarget(i::RelocInfo* rinfo) final {} - void VisitEmbeddedPointer(i::RelocInfo* rinfo) final {} void VisitCustomWeakPointers(i::HeapObject host, i::ObjectSlot start, i::ObjectSlot end) final {}
diff --git a/src/builtins/setup-builtins-internal.cc b/src/builtins/setup-builtins-internal.cc index da00c9c..68a364a 100644 --- a/src/builtins/setup-builtins-internal.cc +++ b/src/builtins/setup-builtins-internal.cc
@@ -8,6 +8,7 @@ #include "src/codegen/interface-descriptors.h" #include "src/codegen/macro-assembler-inl.h" #include "src/codegen/macro-assembler.h" +#include "src/codegen/reloc-info-inl.h" #include "src/compiler/code-assembler.h" #include "src/execution/isolate.h" #include "src/handles/handles-inl.h" @@ -232,6 +233,7 @@ for (Builtin builtin = Builtins::kFirst; builtin <= Builtins::kLast; ++builtin) { Code code = builtins->code(builtin); + InstructionStream istream = code.instruction_stream(); isolate->heap()->UnprotectAndRegisterMemoryChunk( code, UnprotectMemoryOrigin::kMainThread); bool flush_icache = false; @@ -244,7 +246,7 @@ Builtins::IsIsolateIndependent(target_code.builtin_id())); if (!target_code.is_builtin()) continue; Code new_target = builtins->code(target_code.builtin_id()); - rinfo->set_target_address(new_target.instruction_start(), + rinfo->set_target_address(istream, new_target.instruction_start(), UPDATE_WRITE_BARRIER, SKIP_ICACHE_FLUSH); } else { DCHECK(RelocInfo::IsEmbeddedObjectMode(rinfo->rmode())); @@ -253,8 +255,8 @@ Code target = Code::cast(object); if (!target.is_builtin()) continue; Code new_target = builtins->code(target.builtin_id()); - rinfo->set_target_object(isolate->heap(), new_target, - UPDATE_WRITE_BARRIER, SKIP_ICACHE_FLUSH); + rinfo->set_target_object(istream, new_target, UPDATE_WRITE_BARRIER, + SKIP_ICACHE_FLUSH); } flush_icache = true; }
diff --git a/src/codegen/arm/assembler-arm-inl.h b/src/codegen/arm/assembler-arm-inl.h index 0a0880d..5a7e14f 100644 --- a/src/codegen/arm/assembler-arm-inl.h +++ b/src/codegen/arm/assembler-arm-inl.h
@@ -106,15 +106,11 @@ return origin->relative_code_target_object_handle_at(pc_); } -void RelocInfo::set_target_object(Heap* heap, HeapObject target, - WriteBarrierMode write_barrier_mode, +void RelocInfo::set_target_object(HeapObject target, ICacheFlushMode icache_flush_mode) { DCHECK(IsCodeTarget(rmode_) || IsFullEmbeddedObject(rmode_)); Assembler::set_target_address_at(pc_, constant_pool_, target.ptr(), icache_flush_mode); - if (!instruction_stream().is_null() && !v8_flags.disable_write_barriers) { - WriteBarrierForCode(instruction_stream(), this, target, write_barrier_mode); - } } Address RelocInfo::target_external_reference() {
diff --git a/src/codegen/arm/assembler-arm.cc b/src/codegen/arm/assembler-arm.cc index 179c309..009f325 100644 --- a/src/codegen/arm/assembler-arm.cc +++ b/src/codegen/arm/assembler-arm.cc
@@ -5225,28 +5225,20 @@ pc_ += sizeof(uint8_t); } -void Assembler::dd(uint32_t data, RelocInfo::Mode rmode) { +void Assembler::dd(uint32_t data) { // dd is used to write raw data. The constant pool should be emitted or // blocked before using dd. DCHECK(is_const_pool_blocked() || pending_32_bit_constants_.empty()); CheckBuffer(); - if (!RelocInfo::IsNoInfo(rmode)) { - DCHECK(RelocInfo::IsLiteralConstant(rmode)); - RecordRelocInfo(rmode); - } base::WriteUnalignedValue(reinterpret_cast<Address>(pc_), data); pc_ += sizeof(uint32_t); } -void Assembler::dq(uint64_t value, RelocInfo::Mode rmode) { +void Assembler::dq(uint64_t value) { // dq is used to write raw data. The constant pool should be emitted or // blocked before using dq. DCHECK(is_const_pool_blocked() || pending_32_bit_constants_.empty()); CheckBuffer(); - if (!RelocInfo::IsNoInfo(rmode)) { - DCHECK(RelocInfo::IsLiteralConstant(rmode)); - RecordRelocInfo(rmode); - } base::WriteUnalignedValue(reinterpret_cast<Address>(pc_), value); pc_ += sizeof(uint64_t); } @@ -5254,8 +5246,7 @@ void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) { if (!ShouldRecordRelocInfo(rmode)) return; DCHECK_GE(buffer_space(), kMaxRelocSize); // too late to grow buffer here - RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, Code(), - InstructionStream()); + RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data); reloc_info_writer.Write(&rinfo); }
diff --git a/src/codegen/arm/assembler-arm.h b/src/codegen/arm/assembler-arm.h index 9444abe..298ca83 100644 --- a/src/codegen/arm/assembler-arm.h +++ b/src/codegen/arm/assembler-arm.h
@@ -1093,11 +1093,9 @@ // called before any use of db/dd/dq/dp to ensure that constant pools // are not emitted as part of the tables generated. void db(uint8_t data); - void dd(uint32_t data, RelocInfo::Mode rmode = RelocInfo::NO_INFO); - void dq(uint64_t data, RelocInfo::Mode rmode = RelocInfo::NO_INFO); - void dp(uintptr_t data, RelocInfo::Mode rmode = RelocInfo::NO_INFO) { - dd(data, rmode); - } + void dd(uint32_t data); + void dq(uint64_t data); + void dp(uintptr_t data) { dd(data); } // Read/patch instructions Instr instr_at(int pos) {
diff --git a/src/codegen/arm64/assembler-arm64-inl.h b/src/codegen/arm64/assembler-arm64-inl.h index d564d06..bc8e2ed 100644 --- a/src/codegen/arm64/assembler-arm64-inl.h +++ b/src/codegen/arm64/assembler-arm64-inl.h
@@ -682,8 +682,7 @@ } } -void RelocInfo::set_target_object(Heap* heap, HeapObject target, - WriteBarrierMode write_barrier_mode, +void RelocInfo::set_target_object(HeapObject target, ICacheFlushMode icache_flush_mode) { DCHECK(IsCodeTarget(rmode_) || IsEmbeddedObjectMode(rmode_)); if (IsCompressedEmbeddedObject(rmode_)) { @@ -696,9 +695,6 @@ Assembler::set_target_address_at(pc_, constant_pool_, target.ptr(), icache_flush_mode); } - if (!instruction_stream().is_null() && !v8_flags.disable_write_barriers) { - WriteBarrierForCode(instruction_stream(), this, target, write_barrier_mode); - } } Address RelocInfo::target_external_reference() {
diff --git a/src/codegen/arm64/assembler-arm64.cc b/src/codegen/arm64/assembler-arm64.cc index f753e0b..9d606be 100644 --- a/src/codegen/arm64/assembler-arm64.cc +++ b/src/codegen/arm64/assembler-arm64.cc
@@ -4491,19 +4491,17 @@ void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data, ConstantPoolMode constant_pool_mode) { - if ((rmode == RelocInfo::INTERNAL_REFERENCE) || - (rmode == RelocInfo::CONST_POOL) || (rmode == RelocInfo::VENEER_POOL) || - (rmode == RelocInfo::DEOPT_SCRIPT_OFFSET) || - (rmode == RelocInfo::DEOPT_INLINING_ID) || - (rmode == RelocInfo::DEOPT_REASON) || (rmode == RelocInfo::DEOPT_ID) || - (rmode == RelocInfo::LITERAL_CONSTANT) || - (rmode == RelocInfo::DEOPT_NODE_ID)) { + if (rmode == RelocInfo::INTERNAL_REFERENCE || + rmode == RelocInfo::CONST_POOL || rmode == RelocInfo::VENEER_POOL || + rmode == RelocInfo::DEOPT_SCRIPT_OFFSET || + rmode == RelocInfo::DEOPT_INLINING_ID || + rmode == RelocInfo::DEOPT_REASON || rmode == RelocInfo::DEOPT_ID || + rmode == RelocInfo::DEOPT_NODE_ID) { // Adjust code for new modes. DCHECK(RelocInfo::IsDeoptReason(rmode) || RelocInfo::IsDeoptId(rmode) || RelocInfo::IsDeoptNodeId(rmode) || RelocInfo::IsDeoptPosition(rmode) || RelocInfo::IsInternalReference(rmode) || - RelocInfo::IsLiteralConstant(rmode) || RelocInfo::IsConstPool(rmode) || RelocInfo::IsVeneerPool(rmode)); // These modes do not need an entry in the constant pool. } else if (constant_pool_mode == NEEDS_POOL_ENTRY) { @@ -4533,9 +4531,7 @@ DCHECK(constpool_.IsBlocked()); // We do not try to reuse pool constants. - RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, Code(), - InstructionStream()); - + RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data); DCHECK_GE(buffer_space(), kMaxRelocSize); // too late to grow buffer here reloc_info_writer.Write(&rinfo); } @@ -4660,8 +4656,7 @@ void Assembler::RecordVeneerPool(int location_offset, int size) { Assembler::BlockPoolsScope block_pools(this, PoolEmissionCheck::kSkip); RelocInfo rinfo(reinterpret_cast<Address>(buffer_start_) + location_offset, - RelocInfo::VENEER_POOL, static_cast<intptr_t>(size), Code(), - InstructionStream()); + RelocInfo::VENEER_POOL, static_cast<intptr_t>(size)); reloc_info_writer.Write(&rinfo); }
diff --git a/src/codegen/arm64/assembler-arm64.h b/src/codegen/arm64/assembler-arm64.h index 5c10dd8..658a927 100644 --- a/src/codegen/arm64/assembler-arm64.h +++ b/src/codegen/arm64/assembler-arm64.h
@@ -2690,28 +2690,16 @@ // Required by V8. void db(uint8_t data) { dc8(data); } - void dd(uint32_t data, RelocInfo::Mode rmode = RelocInfo::NO_INFO) { + void dd(uint32_t data) { BlockPoolsScope no_pool_scope(this); - if (!RelocInfo::IsNoInfo(rmode)) { - DCHECK(RelocInfo::IsLiteralConstant(rmode)); - RecordRelocInfo(rmode); - } dc32(data); } - void dq(uint64_t data, RelocInfo::Mode rmode = RelocInfo::NO_INFO) { + void dq(uint64_t data) { BlockPoolsScope no_pool_scope(this); - if (!RelocInfo::IsNoInfo(rmode)) { - DCHECK(RelocInfo::IsLiteralConstant(rmode)); - RecordRelocInfo(rmode); - } dc64(data); } - void dp(uintptr_t data, RelocInfo::Mode rmode = RelocInfo::NO_INFO) { + void dp(uintptr_t data) { BlockPoolsScope no_pool_scope(this); - if (!RelocInfo::IsNoInfo(rmode)) { - DCHECK(RelocInfo::IsLiteralConstant(rmode)); - RecordRelocInfo(rmode); - } dc64(data); }
diff --git a/src/codegen/assembler.h b/src/codegen/assembler.h index e1ee491..b06eb86 100644 --- a/src/codegen/assembler.h +++ b/src/codegen/assembler.h
@@ -396,9 +396,6 @@ !v8_flags.debug_code) { return false; } -#ifndef ENABLE_DISASSEMBLER - if (RelocInfo::IsLiteralConstant(rmode)) return false; -#endif return true; }
diff --git a/src/codegen/compiler.cc b/src/codegen/compiler.cc index 81d823d..06a7255 100644 --- a/src/codegen/compiler.cc +++ b/src/codegen/compiler.cc
@@ -1736,8 +1736,12 @@ void VisitCodePointer(Code host, CodeObjectSlot slot) override { UNREACHABLE(); } - void VisitCodeTarget(RelocInfo* rinfo) override { UNREACHABLE(); } - void VisitEmbeddedPointer(RelocInfo* rinfo) override { UNREACHABLE(); } + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) override { + UNREACHABLE(); + } + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo) override { + UNREACHABLE(); + } private: enum ObjectKind {
diff --git a/src/codegen/ia32/assembler-ia32-inl.h b/src/codegen/ia32/assembler-ia32-inl.h index d5040b8..c7c13cc 100644 --- a/src/codegen/ia32/assembler-ia32-inl.h +++ b/src/codegen/ia32/assembler-ia32-inl.h
@@ -90,17 +90,13 @@ return Handle<HeapObject>::cast(ReadUnalignedValue<Handle<Object>>(pc_)); } -void RelocInfo::set_target_object(Heap* heap, HeapObject target, - WriteBarrierMode write_barrier_mode, +void RelocInfo::set_target_object(HeapObject target, ICacheFlushMode icache_flush_mode) { DCHECK(IsCodeTarget(rmode_) || IsFullEmbeddedObject(rmode_)); WriteUnalignedValue(pc_, target.ptr()); if (icache_flush_mode != SKIP_ICACHE_FLUSH) { FlushInstructionCache(pc_, sizeof(Address)); } - if (!instruction_stream().is_null() && !v8_flags.disable_write_barriers) { - WriteBarrierForCode(instruction_stream(), this, target, write_barrier_mode); - } } Address RelocInfo::target_external_reference() {
diff --git a/src/codegen/ia32/assembler-ia32.cc b/src/codegen/ia32/assembler-ia32.cc index bb76c82..0ae5f8b 100644 --- a/src/codegen/ia32/assembler-ia32.cc +++ b/src/codegen/ia32/assembler-ia32.cc
@@ -3384,18 +3384,13 @@ EMIT(data); } -void Assembler::dd(uint32_t data, RelocInfo::Mode rmode) { +void Assembler::dd(uint32_t data) { EnsureSpace ensure_space(this); - if (!RelocInfo::IsNoInfo(rmode)) { - DCHECK(RelocInfo::IsLiteralConstant(rmode)); - RecordRelocInfo(rmode); - } emit(data); } -void Assembler::dq(uint64_t data, RelocInfo::Mode rmode) { +void Assembler::dq(uint64_t data) { EnsureSpace ensure_space(this); - DCHECK(RelocInfo::IsNoInfo(rmode)); emit_q(data); } @@ -3407,8 +3402,7 @@ void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) { if (!ShouldRecordRelocInfo(rmode)) return; - RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, Code(), - InstructionStream()); + RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data); reloc_info_writer.Write(&rinfo); }
diff --git a/src/codegen/ia32/assembler-ia32.h b/src/codegen/ia32/assembler-ia32.h index 54cbbf6..7119ca5 100644 --- a/src/codegen/ia32/assembler-ia32.h +++ b/src/codegen/ia32/assembler-ia32.h
@@ -1650,11 +1650,9 @@ // Writes a single byte or word of data in the code stream. Used for // inline tables, e.g., jump-tables. void db(uint8_t data); - void dd(uint32_t data, RelocInfo::Mode rmode = RelocInfo::NO_INFO); - void dq(uint64_t data, RelocInfo::Mode rmode = RelocInfo::NO_INFO); - void dp(uintptr_t data, RelocInfo::Mode rmode = RelocInfo::NO_INFO) { - dd(data, rmode); - } + void dd(uint32_t data); + void dq(uint64_t data); + void dp(uintptr_t data) { dd(data); } void dd(Label* label); // Check if there is less than kGap bytes available in the buffer.
diff --git a/src/codegen/reloc-info-inl.h b/src/codegen/reloc-info-inl.h new file mode 100644 index 0000000..70fe88f --- /dev/null +++ b/src/codegen/reloc-info-inl.h
@@ -0,0 +1,27 @@ +// Copyright 2023 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef V8_CODEGEN_RELOC_INFO_INL_H_ +#define V8_CODEGEN_RELOC_INFO_INL_H_ + +#include "src/codegen/assembler-inl.h" +#include "src/codegen/reloc-info.h" +#include "src/heap/heap-write-barrier-inl.h" + +namespace v8 { +namespace internal { + +void RelocInfo::set_target_object(InstructionStream host, HeapObject target, + WriteBarrierMode write_barrier_mode, + ICacheFlushMode icache_flush_mode) { + set_target_object(target, icache_flush_mode); + if (!v8_flags.disable_write_barriers) { + WriteBarrierForCode(host, this, target, write_barrier_mode); + } +} + +} // namespace internal +} // namespace v8 + +#endif // V8_CODEGEN_RELOC_INFO_INL_H_
diff --git a/src/codegen/reloc-info.cc b/src/codegen/reloc-info.cc index 66dab8c..f7b48fd 100644 --- a/src/codegen/reloc-info.cc +++ b/src/codegen/reloc-info.cc
@@ -17,63 +17,7 @@ namespace v8 { namespace internal { -const char* const RelocInfo::kFillerCommentString = "DEOPTIMIZATION PADDING"; - -// ----------------------------------------------------------------------------- -// Implementation of RelocInfoWriter and RelocIterator -// -// Relocation information is written backwards in memory, from high addresses -// towards low addresses, byte by byte. Therefore, in the encodings listed -// below, the first byte listed it at the highest address, and successive -// bytes in the record are at progressively lower addresses. -// -// Encoding -// -// The most common modes are given single-byte encodings. Also, it is -// easy to identify the type of reloc info and skip unwanted modes in -// an iteration. -// -// The encoding relies on the fact that there are fewer than 14 -// different relocation modes using standard non-compact encoding. -// -// The first byte of a relocation record has a tag in its low 2 bits: -// Here are the record schemes, depending on the low tag and optional higher -// tags. -// -// Low tag: -// 00: embedded_object: [6-bit pc delta] 00 -// -// 01: code_target: [6-bit pc delta] 01 -// -// 10: wasm_stub_call: [6-bit pc delta] 10 -// -// 11: long_record [6 bit reloc mode] 11 -// followed by pc delta -// followed by optional data depending on type. -// -// If a pc delta exceeds 6 bits, it is split into a remainder that fits into -// 6 bits and a part that does not. The latter is encoded as a long record -// with PC_JUMP as pseudo reloc info mode. The former is encoded as part of -// the following record in the usual way. The long pc jump record has variable -// length: -// pc-jump: [PC_JUMP] 11 -// 1 [7 bits data] -// ... -// 0 [7 bits data] -// (Bits 6..31 of pc delta, encoded with VLQ.) - -const int kTagBits = 2; -const int kTagMask = (1 << kTagBits) - 1; -const int kLongTagBits = 6; - -const int kEmbeddedObjectTag = 0; -const int kCodeTargetTag = 1; -const int kWasmStubCallTag = 2; -const int kDefaultTag = 3; - -const int kSmallPCDeltaBits = kBitsPerByte - kTagBits; -const int kSmallPCDeltaMask = (1 << kSmallPCDeltaBits) - 1; -const int RelocInfo::kMaxSmallPCDelta = kSmallPCDeltaMask; +using namespace detail; uint32_t RelocInfoWriter::WriteLongPCJump(uint32_t pc_delta) { // Return if the pc_delta can fit in kSmallPCDeltaBits bits. @@ -160,19 +104,6 @@ #endif } -inline int RelocIterator::AdvanceGetTag() { return *--pos_ & kTagMask; } - -inline RelocInfo::Mode RelocIterator::GetMode() { - return static_cast<RelocInfo::Mode>((*pos_ >> kTagBits) & - ((1 << kLongTagBits) - 1)); -} - -inline void RelocIterator::ReadShortTaggedPC() { - rinfo_.pc_ += *pos_ >> kTagBits; -} - -inline void RelocIterator::AdvanceReadPC() { rinfo_.pc_ += *--pos_; } - void RelocIterator::AdvanceReadInt() { int x = 0; for (int i = 0; i < kIntSize; i++) { @@ -181,14 +112,6 @@ rinfo_.data_ = x; } -void RelocIterator::AdvanceReadData() { - intptr_t x = 0; - for (int i = 0; i < kIntptrSize; i++) { - x |= static_cast<intptr_t>(*--pos_) << i * kBitsPerByte; - } - rinfo_.data_ = x; -} - void RelocIterator::AdvanceReadLongPCJump() { // Read the 32-kSmallPCDeltaBits most significant bits of the // pc jump as a VLQ encoded integer. @@ -255,65 +178,45 @@ RelocIterator::RelocIterator(Code code, int mode_mask) : RelocIterator( - code, code.unchecked_instruction_stream().unchecked_relocation_info(), + code.instruction_start(), code.constant_pool(), + code.instruction_stream().relocation_info().GetDataEndAddress(), + code.instruction_stream().relocation_info().GetDataStartAddress(), mode_mask) {} -RelocIterator::RelocIterator(Code code, ByteArray relocation_info, - int mode_mask) - : RelocIterator( - code, - InstructionStream::unchecked_cast(code.raw_instruction_stream()), - InstructionStream::unchecked_cast(code.raw_instruction_stream()) - .instruction_start(), - code.constant_pool(), relocation_info.GetDataEndAddress(), - relocation_info.GetDataStartAddress(), mode_mask) {} - RelocIterator::RelocIterator(Code code, InstructionStream instruction_stream, ByteArray relocation_info, int mode_mask) - : RelocIterator(code, instruction_stream, - instruction_stream.instruction_start(), + : RelocIterator(instruction_stream.instruction_start(), code.constant_pool(instruction_stream), relocation_info.GetDataEndAddress(), relocation_info.GetDataStartAddress(), mode_mask) {} -RelocIterator::RelocIterator(const CodeReference code_reference, int mode_mask) - : RelocIterator( - Code(), InstructionStream(), code_reference.instruction_start(), - code_reference.constant_pool(), code_reference.relocation_end(), - code_reference.relocation_start(), mode_mask) {} +RelocIterator::RelocIterator(const CodeReference code_reference) + : RelocIterator(code_reference.instruction_start(), + code_reference.constant_pool(), + code_reference.relocation_end(), + code_reference.relocation_start(), kAllModesMask) {} RelocIterator::RelocIterator(EmbeddedData* embedded_data, Code code, int mode_mask) - : RelocIterator(code, code.instruction_stream(), - embedded_data->InstructionStartOf(code.builtin_id()), - code.constant_pool(), - code.relocation_start() + code.relocation_size(), + : RelocIterator(embedded_data->InstructionStartOf(code.builtin_id()), + code.constant_pool(), code.relocation_end(), code.relocation_start(), mode_mask) {} -RelocIterator::RelocIterator(const CodeDesc& desc, int mode_mask) - : RelocIterator( - Code(), InstructionStream(), reinterpret_cast<Address>(desc.buffer), - 0, desc.buffer + desc.buffer_size, - desc.buffer + desc.buffer_size - desc.reloc_size, mode_mask) {} - RelocIterator::RelocIterator(base::Vector<byte> instructions, base::Vector<const byte> reloc_info, Address const_pool, int mode_mask) - : RelocIterator(Code(), InstructionStream(), - reinterpret_cast<Address>(instructions.begin()), const_pool, + : RelocIterator(reinterpret_cast<Address>(instructions.begin()), const_pool, reloc_info.begin() + reloc_info.size(), reloc_info.begin(), mode_mask) {} -RelocIterator::RelocIterator(Code code, InstructionStream instruction_stream, - Address pc, Address constant_pool, const byte* pos, +RelocIterator::RelocIterator(Address pc, Address constant_pool, const byte* pos, const byte* end, int mode_mask) - : pos_(pos), end_(end), mode_mask_(mode_mask) { + : pos_(pos), + end_(end), + rinfo_(pc, RelocInfo::NO_INFO, 0, constant_pool), + mode_mask_(mode_mask) { // Relocation info is read backwards. DCHECK_GE(pos_, end_); - rinfo_.code_ = code; - rinfo_.pc_ = pc; - rinfo_.instruction_stream_ = instruction_stream; - rinfo_.constant_pool_ = constant_pool; if (mode_mask_ == 0) pos_ = end_; next(); } @@ -359,18 +262,21 @@ } void RelocInfo::set_target_address(Address target, - WriteBarrierMode write_barrier_mode, ICacheFlushMode icache_flush_mode) { DCHECK(IsCodeTargetMode(rmode_) || IsNearBuiltinEntry(rmode_) || IsWasmCall(rmode_)); Assembler::set_target_address_at(pc_, constant_pool_, target, icache_flush_mode); - if (!instruction_stream().is_null() && IsCodeTargetMode(rmode_) && - !v8_flags.disable_write_barriers) { +} + +void RelocInfo::set_target_address(InstructionStream host, Address target, + WriteBarrierMode write_barrier_mode, + ICacheFlushMode icache_flush_mode) { + set_target_address(target, icache_flush_mode); + if (IsCodeTargetMode(rmode_) && !v8_flags.disable_write_barriers) { InstructionStream target_code = InstructionStream::FromTargetAddress(target); - WriteBarrierForCode(instruction_stream(), this, target_code, - write_barrier_mode); + WriteBarrierForCode(host, this, target_code, write_barrier_mode); } } @@ -430,8 +336,6 @@ return "deopt reason"; case DEOPT_ID: return "deopt index"; - case LITERAL_CONSTANT: - return "literal constant"; case DEOPT_NODE_ID: return "deopt node id"; case CONST_POOL: @@ -544,7 +448,6 @@ case DEOPT_INLINING_ID: case DEOPT_REASON: case DEOPT_ID: - case LITERAL_CONSTANT: case DEOPT_NODE_ID: case CONST_POOL: case VENEER_POOL:
diff --git a/src/codegen/reloc-info.h b/src/codegen/reloc-info.h index f54dafb..064854d 100644 --- a/src/codegen/reloc-info.h +++ b/src/codegen/reloc-info.h
@@ -22,6 +22,63 @@ // executed). enum ICacheFlushMode { FLUSH_ICACHE_IF_NEEDED, SKIP_ICACHE_FLUSH }; +namespace detail { +// ----------------------------------------------------------------------------- +// Implementation of RelocInfoWriter and RelocIterator +// +// Relocation information is written backwards in memory, from high addresses +// towards low addresses, byte by byte. Therefore, in the encodings listed +// below, the first byte listed it at the highest address, and successive +// bytes in the record are at progressively lower addresses. +// +// Encoding +// +// The most common modes are given single-byte encodings. Also, it is +// easy to identify the type of reloc info and skip unwanted modes in +// an iteration. +// +// The encoding relies on the fact that there are fewer than 14 +// different relocation modes using standard non-compact encoding. +// +// The first byte of a relocation record has a tag in its low 2 bits: +// Here are the record schemes, depending on the low tag and optional higher +// tags. +// +// Low tag: +// 00: embedded_object: [6-bit pc delta] 00 +// +// 01: code_target: [6-bit pc delta] 01 +// +// 10: wasm_stub_call: [6-bit pc delta] 10 +// +// 11: long_record [6 bit reloc mode] 11 +// followed by pc delta +// followed by optional data depending on type. +// +// If a pc delta exceeds 6 bits, it is split into a remainder that fits into +// 6 bits and a part that does not. The latter is encoded as a long record +// with PC_JUMP as pseudo reloc info mode. The former is encoded as part of +// the following record in the usual way. The long pc jump record has variable +// length: +// pc-jump: [PC_JUMP] 11 +// 1 [7 bits data] +// ... +// 0 [7 bits data] +// (Bits 6..31 of pc delta, encoded with VLQ.) + +constexpr int kTagBits = 2; +constexpr int kTagMask = (1 << kTagBits) - 1; +constexpr int kLongTagBits = 6; + +constexpr int kEmbeddedObjectTag = 0; +constexpr int kCodeTargetTag = 1; +constexpr int kWasmStubCallTag = 2; +constexpr int kDefaultTag = 3; + +constexpr int kSmallPCDeltaBits = kBitsPerByte - kTagBits; +constexpr int kSmallPCDeltaMask = (1 << kSmallPCDeltaBits) - 1; +} // namespace detail + // ----------------------------------------------------------------------------- // Relocation information @@ -34,21 +91,15 @@ class RelocInfo { public: - // This string is used to add padding comments to the reloc info in cases - // where we are not sure to have enough space for patching in during - // lazy deoptimization. This is the case if we have indirect calls for which - // we do not normally record relocation info. - static const char* const kFillerCommentString; - // The minimum size of a comment is equal to two bytes for the extra tagged // pc and kSystemPointerSize for the actual pointer to the comment. - static const int kMinRelocCommentSize = 2 + kSystemPointerSize; + static constexpr int kMinRelocCommentSize = 2 + kSystemPointerSize; // The maximum size for a call instruction including pc-jump. - static const int kMaxCallSize = 6; + static constexpr int kMaxCallSize = 6; // The maximum pc delta that will use the short encoding. - static const int kMaxSmallPCDelta; + static constexpr int kMaxSmallPCDelta = detail::kSmallPCDeltaMask; enum Mode : int8_t { // Please note the order is important (see IsRealRelocMode, IsGCRelocMode, @@ -91,8 +142,6 @@ DEOPT_NODE_ID, // Id of the node that caused deoptimization. This // information is only recorded in debug builds. - LITERAL_CONSTANT, // An constant embedded in the instruction stream. - // This is not an actual reloc mode, but used to encode a long pc jump that // cannot be encoded as part of another record. PC_JUMP, @@ -115,19 +164,16 @@ RelocInfo() = default; - RelocInfo(Address pc, Mode rmode, intptr_t data, Code code, - InstructionStream instruction_stream, + RelocInfo(Address pc, Mode rmode, intptr_t data, Address constant_pool = kNullAddress) - : pc_(pc), - rmode_(rmode), - data_(data), - code_(code), - instruction_stream_(instruction_stream), - constant_pool_(constant_pool) { + : pc_(pc), rmode_(rmode), data_(data), constant_pool_(constant_pool) { DCHECK_IMPLIES(!COMPRESS_POINTERS_BOOL, rmode != COMPRESSED_EMBEDDED_OBJECT); } + // Convenience ctor. + RelocInfo(Address pc, Mode rmode) : RelocInfo(pc, rmode, 0) {} + static constexpr bool IsRealRelocMode(Mode mode) { return mode >= FIRST_REAL_RELOC_MODE && mode <= LAST_REAL_RELOC_MODE; } @@ -170,9 +216,6 @@ return mode == DEOPT_REASON; } static constexpr bool IsDeoptId(Mode mode) { return mode == DEOPT_ID; } - static constexpr bool IsLiteralConstant(Mode mode) { - return mode == LITERAL_CONSTANT; - } static constexpr bool IsDeoptNodeId(Mode mode) { return mode == DEOPT_NODE_ID; } @@ -216,9 +259,6 @@ Address pc() const { return pc_; } Mode rmode() const { return rmode_; } intptr_t data() const { return data_; } - Code code() const { return code_; } - InstructionStream instruction_stream() const { return instruction_stream_; } - Address constant_pool() const { return constant_pool_; } // Apply a relocation by delta bytes. When the code object is moved, PC // relative addresses have to be updated as well as absolute addresses @@ -250,9 +290,12 @@ Address, ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED); void set_target_address( - Address target, + InstructionStream host, Address target, WriteBarrierMode write_barrier_mode = UPDATE_WRITE_BARRIER, ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED); + // Use this overload only when an InstructionStream host is not available. + void set_target_address(Address target, ICacheFlushMode icache_flush_mode = + FLUSH_ICACHE_IF_NEEDED); void set_off_heap_target_address( Address target, @@ -267,9 +310,14 @@ V8_INLINE Handle<HeapObject> target_object_handle(Assembler* origin); V8_INLINE void set_target_object( - Heap* heap, HeapObject target, + InstructionStream host, HeapObject target, WriteBarrierMode write_barrier_mode = UPDATE_WRITE_BARRIER, ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED); + // Use this overload only when an InstructionStream host is not available. + V8_INLINE void set_target_object( + HeapObject target, + ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED); + // Decodes builtin ID encoded as a PC-relative offset. This encoding is used // during code generation of call/jump with NEAR_BUILTIN_ENTRY. V8_INLINE Builtin target_builtin_at(Assembler* origin); @@ -318,18 +366,18 @@ V8_INLINE void WipeOut(); template <typename ObjectVisitor> - void Visit(ObjectVisitor* visitor) { + void Visit(InstructionStream host, ObjectVisitor* visitor) { Mode mode = rmode(); if (IsEmbeddedObjectMode(mode)) { - visitor->VisitEmbeddedPointer(this); + visitor->VisitEmbeddedPointer(host, this); } else if (IsCodeTargetMode(mode)) { - visitor->VisitCodeTarget(this); + visitor->VisitCodeTarget(host, this); } else if (IsExternalReference(mode)) { - visitor->VisitExternalReference(this); + visitor->VisitExternalReference(host, this); } else if (IsInternalReference(mode) || IsInternalReferenceEncoded(mode)) { - visitor->VisitInternalReference(this); + visitor->VisitInternalReference(host, this); } else if (IsBuiltinEntryMode(mode)) { - visitor->VisitOffHeapTarget(this); + visitor->VisitOffHeapTarget(host, this); } } @@ -374,9 +422,8 @@ Address pc_; Mode rmode_; intptr_t data_ = 0; - Code code_; - InstructionStream instruction_stream_; Address constant_pool_ = kNullAddress; + friend class RelocIterator; }; @@ -427,66 +474,68 @@ // } // // A mask can be specified to skip unwanted modes. -class V8_EXPORT_PRIVATE RelocIterator : public Malloced { +class V8_EXPORT_PRIVATE RelocIterator { + static constexpr int kAllModesMask = -1; + public: - // Create a new iterator positioned at the beginning of the reloc info. - // Relocation information with mode k is included in the iteration iff bit k - // of mode_mask is set. - explicit RelocIterator(Code code, int mode_mask = -1); - explicit RelocIterator(Code code, ByteArray relocation_info, int mode_mask); + // Prefer using this ctor when possible: + explicit RelocIterator(Code code, int mode_mask = kAllModesMask); + // For when GC may be in progress and thus pointers on the Code object may be + // stale (or forwarding pointers); or when objects are not fully constructed, + // or we're operating on fake objects for some reason. Then, we pass relevant + // objects explicitly. Note they must all refer to the same underlying + // {Code,IStream} composite object. explicit RelocIterator(Code code, InstructionStream instruction_stream, ByteArray relocation_info, int mode_mask); - explicit RelocIterator(EmbeddedData* embedded_data, Code code, int mode_mask); - explicit RelocIterator(const CodeDesc& desc, int mode_mask = -1); - explicit RelocIterator(const CodeReference code_reference, - int mode_mask = -1); + // For Wasm. explicit RelocIterator(base::Vector<byte> instructions, base::Vector<const byte> reloc_info, - Address const_pool, int mode_mask = -1); - RelocIterator(RelocIterator&&) V8_NOEXCEPT = default; + Address const_pool, int mode_mask = kAllModesMask); + // For the disassembler. + explicit RelocIterator(const CodeReference code_reference); + // For FinalizeEmbeddedCodeTargets when creating embedded builtins. + explicit RelocIterator(EmbeddedData* embedded_data, Code code, int mode_mask); + RelocIterator(RelocIterator&&) V8_NOEXCEPT = default; RelocIterator(const RelocIterator&) = delete; RelocIterator& operator=(const RelocIterator&) = delete; - // Iteration bool done() const { return done_; } void next(); - // Return pointer valid until next next(). + // The returned pointer is valid until the next call to next(). RelocInfo* rinfo() { DCHECK(!done()); return &rinfo_; } private: - RelocIterator(Code code, InstructionStream instruction_stream, Address pc, - Address constant_pool, const byte* pos, const byte* end, - int mode_mask); + RelocIterator(Address pc, Address constant_pool, const byte* pos, + const byte* end, int mode_mask); - // Advance* moves the position before/after reading. - // *Read* reads from current byte(s) into rinfo_. - // *Get* just reads and returns info on current byte. - void Advance(int bytes = 1) { pos_ -= bytes; } - int AdvanceGetTag(); - RelocInfo::Mode GetMode(); - - void AdvanceReadLongPCJump(); - - void ReadShortTaggedPC(); - void ReadShortData(); - - void AdvanceReadPC(); - void AdvanceReadInt(); - void AdvanceReadData(); - - // If the given mode is wanted, set it in rinfo_ and return true. - // Else return false. Used for efficiently skipping unwanted modes. + // Used for efficiently skipping unwanted modes. bool SetMode(RelocInfo::Mode mode) { - return (mode_mask_ & (1 << mode)) ? (rinfo_.rmode_ = mode, true) : false; + if ((mode_mask_ & (1 << mode)) == 0) return false; + rinfo_.rmode_ = mode; + return true; } + RelocInfo::Mode GetMode() const { + return static_cast<RelocInfo::Mode>((*pos_ >> detail::kTagBits) & + ((1 << detail::kLongTagBits) - 1)); + } + + void Advance(int bytes = 1) { pos_ -= bytes; } + int AdvanceGetTag() { return *--pos_ & detail::kTagMask; } + void AdvanceReadLongPCJump(); + void AdvanceReadPC() { rinfo_.pc_ += *--pos_; } + void AdvanceReadInt(); + + void ReadShortTaggedPC() { rinfo_.pc_ += *pos_ >> detail::kTagBits; } + void ReadShortData(); + const byte* pos_; - const byte* end_; + const byte* const end_; RelocInfo rinfo_; bool done_ = false; const int mode_mask_;
diff --git a/src/codegen/x64/assembler-x64-inl.h b/src/codegen/x64/assembler-x64-inl.h index dc51e84..bc514c0 100644 --- a/src/codegen/x64/assembler-x64-inl.h +++ b/src/codegen/x64/assembler-x64-inl.h
@@ -334,8 +334,7 @@ return pc_; } -void RelocInfo::set_target_object(Heap* heap, HeapObject target, - WriteBarrierMode write_barrier_mode, +void RelocInfo::set_target_object(HeapObject target, ICacheFlushMode icache_flush_mode) { DCHECK(IsCodeTarget(rmode_) || IsEmbeddedObjectMode(rmode_)); if (IsCompressedEmbeddedObject(rmode_)) { @@ -349,9 +348,6 @@ if (icache_flush_mode != SKIP_ICACHE_FLUSH) { FlushInstructionCache(pc_, sizeof(Address)); } - if (!instruction_stream().is_null() && !v8_flags.disable_write_barriers) { - WriteBarrierForCode(instruction_stream(), this, target, write_barrier_mode); - } } Builtin RelocInfo::target_builtin_at(Assembler* origin) {
diff --git a/src/codegen/x64/assembler-x64.cc b/src/codegen/x64/assembler-x64.cc index 5b69f28..588c64d 100644 --- a/src/codegen/x64/assembler-x64.cc +++ b/src/codegen/x64/assembler-x64.cc
@@ -4451,21 +4451,13 @@ emit(data); } -void Assembler::dd(uint32_t data, RelocInfo::Mode rmode) { +void Assembler::dd(uint32_t data) { EnsureSpace ensure_space(this); - if (!RelocInfo::IsNoInfo(rmode)) { - DCHECK(RelocInfo::IsLiteralConstant(rmode)); - RecordRelocInfo(rmode); - } emitl(data); } -void Assembler::dq(uint64_t data, RelocInfo::Mode rmode) { +void Assembler::dq(uint64_t data) { EnsureSpace ensure_space(this); - if (!RelocInfo::IsNoInfo(rmode)) { - DCHECK(RelocInfo::IsLiteralConstant(rmode)); - RecordRelocInfo(rmode); - } emitq(data); } @@ -4494,8 +4486,7 @@ void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) { if (!ShouldRecordRelocInfo(rmode)) return; - RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, Code(), - InstructionStream()); + RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data); reloc_info_writer.Write(&rinfo); }
diff --git a/src/codegen/x64/assembler-x64.h b/src/codegen/x64/assembler-x64.h index b8f50fc..75a107e 100644 --- a/src/codegen/x64/assembler-x64.h +++ b/src/codegen/x64/assembler-x64.h
@@ -2157,11 +2157,9 @@ // Writes a single word of data in the code stream. // Used for inline tables, e.g., jump-tables. void db(uint8_t data); - void dd(uint32_t data, RelocInfo::Mode rmode = RelocInfo::NO_INFO); - void dq(uint64_t data, RelocInfo::Mode rmode = RelocInfo::NO_INFO); - void dp(uintptr_t data, RelocInfo::Mode rmode = RelocInfo::NO_INFO) { - dq(data, rmode); - } + void dd(uint32_t data); + void dq(uint64_t data); + void dp(uintptr_t data) { dq(data); } void dq(Label* label); // Patch entries for partial constant pool.
diff --git a/src/common/ptr-compr-inl.h b/src/common/ptr-compr-inl.h index 565b48d..294900d 100644 --- a/src/common/ptr-compr-inl.h +++ b/src/common/ptr-compr-inl.h
@@ -213,6 +213,10 @@ V8HeapCompressionScheme::GetPtrComprCageBaseAddress(address)); } +V8_INLINE PtrComprCageBase GetPtrComprCageBase() { + return PtrComprCageBase(V8HeapCompressionScheme::base()); +} + #else // @@ -262,6 +266,8 @@ return PtrComprCageBase(); } +V8_INLINE PtrComprCageBase GetPtrComprCageBase() { return PtrComprCageBase(); } + #endif // V8_COMPRESS_POINTERS V8_INLINE PtrComprCageBase GetPtrComprCageBase(HeapObject object) {
diff --git a/src/diagnostics/disassembler.cc b/src/diagnostics/disassembler.cc index 6f0b60f..c4dc628 100644 --- a/src/diagnostics/disassembler.cc +++ b/src/diagnostics/disassembler.cc
@@ -313,24 +313,13 @@ pc += 4; } else if (!rit.done() && rit.rinfo()->pc() == reinterpret_cast<Address>(pc) && - (rit.rinfo()->rmode() == RelocInfo::INTERNAL_REFERENCE || - rit.rinfo()->rmode() == RelocInfo::LITERAL_CONSTANT)) { - // raw pointer embedded in code stream, e.g., jump table + rit.rinfo()->rmode() == RelocInfo::INTERNAL_REFERENCE) { + // A raw pointer embedded in code stream. byte* ptr = base::ReadUnalignedValue<byte*>(reinterpret_cast<Address>(pc)); - if (RelocInfo::IsInternalReference(rit.rinfo()->rmode())) { - SNPrintF(decode_buffer, - "%08" V8PRIxPTR " jump table entry %4zu", - reinterpret_cast<intptr_t>(ptr), - static_cast<size_t>(ptr - begin)); - } else { - const char* kType = RelocInfo::IsLiteralConstant(rit.rinfo()->rmode()) - ? " literal constant" - : "embedded data object"; - SNPrintF(decode_buffer, "%08" V8PRIxPTR " %s 0x%08" V8PRIxPTR, - reinterpret_cast<intptr_t>(ptr), kType, - reinterpret_cast<intptr_t>(ptr)); - } + SNPrintF(decode_buffer, "%08" V8PRIxPTR " jump table entry %4zu", + reinterpret_cast<intptr_t>(ptr), + static_cast<size_t>(ptr - begin)); pc += sizeof(ptr); } else { decode_buffer[0] = '\0'; @@ -384,9 +373,7 @@ if (host.is_code()) { code_handle = host.as_code(); - RelocInfo relocinfo(pcs[i], rmodes[i], datas[i], *code_handle, - code_handle->instruction_stream(), constant_pool); - + RelocInfo relocinfo(pcs[i], rmodes[i], datas[i], constant_pool); bool first_reloc_info = (i == 0); PrintRelocInfo(out, isolate, ref_encoder, os, code, &relocinfo, first_reloc_info); @@ -402,7 +389,7 @@ // by IsInConstantPool() below. if (pcs.empty() && !code.is_null() && !decoding_constant_pool) { RelocInfo dummy_rinfo(reinterpret_cast<Address>(prev_pc), - RelocInfo::NO_INFO, 0, Code(), InstructionStream()); + RelocInfo::NO_INFO); if (dummy_rinfo.IsInConstantPool()) { Address constant_pool_entry_address = dummy_rinfo.constant_pool_entry_address();
diff --git a/src/heap/concurrent-marking.cc b/src/heap/concurrent-marking.cc index f38739e..2f0cd75 100644 --- a/src/heap/concurrent-marking.cc +++ b/src/heap/concurrent-marking.cc
@@ -153,11 +153,14 @@ ConcurrentMarkingState* marking_state() { return &marking_state_; } private: - void RecordRelocSlot(RelocInfo* rinfo, HeapObject target) { - if (!MarkCompactCollector::ShouldRecordRelocSlot(rinfo, target)) return; + void RecordRelocSlot(InstructionStream host, RelocInfo* rinfo, + HeapObject target) { + if (!MarkCompactCollector::ShouldRecordRelocSlot(host, rinfo, target)) { + return; + } MarkCompactCollector::RecordRelocSlotInfo info = - MarkCompactCollector::ProcessRelocInfo(rinfo, target); + MarkCompactCollector::ProcessRelocInfo(host, rinfo, target); MemoryChunkData& data = (*memory_chunk_data_)[info.memory_chunk]; if (!data.typed_slots) {
diff --git a/src/heap/evacuation-verifier.cc b/src/heap/evacuation-verifier.cc index 5134b7f..adaa704 100644 --- a/src/heap/evacuation-verifier.cc +++ b/src/heap/evacuation-verifier.cc
@@ -111,12 +111,14 @@ VerifyHeapObjectImpl(code); } } -void FullEvacuationVerifier::VisitCodeTarget(RelocInfo* rinfo) { +void FullEvacuationVerifier::VisitCodeTarget(InstructionStream host, + RelocInfo* rinfo) { InstructionStream target = InstructionStream::FromTargetAddress(rinfo->target_address()); VerifyHeapObjectImpl(target); } -void FullEvacuationVerifier::VisitEmbeddedPointer(RelocInfo* rinfo) { +void FullEvacuationVerifier::VisitEmbeddedPointer(InstructionStream host, + RelocInfo* rinfo) { VerifyHeapObjectImpl(rinfo->target_object(cage_base())); } void FullEvacuationVerifier::VerifyRootPointers(FullObjectSlot start, @@ -155,12 +157,14 @@ VerifyHeapObjectImpl(code); } } -void YoungGenerationEvacuationVerifier::VisitCodeTarget(RelocInfo* rinfo) { +void YoungGenerationEvacuationVerifier::VisitCodeTarget(InstructionStream host, + RelocInfo* rinfo) { InstructionStream target = InstructionStream::FromTargetAddress(rinfo->target_address()); VerifyHeapObjectImpl(target); } -void YoungGenerationEvacuationVerifier::VisitEmbeddedPointer(RelocInfo* rinfo) { +void YoungGenerationEvacuationVerifier::VisitEmbeddedPointer( + InstructionStream host, RelocInfo* rinfo) { VerifyHeapObjectImpl(rinfo->target_object(cage_base())); } void YoungGenerationEvacuationVerifier::VerifyRootPointers(FullObjectSlot start,
diff --git a/src/heap/evacuation-verifier.h b/src/heap/evacuation-verifier.h index 039d793..4ee866b 100644 --- a/src/heap/evacuation-verifier.h +++ b/src/heap/evacuation-verifier.h
@@ -70,8 +70,8 @@ void VerifyPointers(ObjectSlot start, ObjectSlot end) override; void VerifyPointers(MaybeObjectSlot start, MaybeObjectSlot end) override; void VerifyCodePointer(CodeObjectSlot slot) override; - void VisitCodeTarget(RelocInfo* rinfo) override; - void VisitEmbeddedPointer(RelocInfo* rinfo) override; + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) override; + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo) override; void VerifyRootPointers(FullObjectSlot start, FullObjectSlot end) override; }; @@ -91,8 +91,8 @@ void VerifyPointers(ObjectSlot start, ObjectSlot end) override; void VerifyPointers(MaybeObjectSlot start, MaybeObjectSlot end) override; void VerifyCodePointer(CodeObjectSlot slot) override; - void VisitCodeTarget(RelocInfo* rinfo) override; - void VisitEmbeddedPointer(RelocInfo* rinfo) override; + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) override; + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo) override; void VerifyRootPointers(FullObjectSlot start, FullObjectSlot end) override; };
diff --git a/src/heap/heap-verifier.cc b/src/heap/heap-verifier.cc index f4abc97..d9d1cb2 100644 --- a/src/heap/heap-verifier.cc +++ b/src/heap/heap-verifier.cc
@@ -63,8 +63,8 @@ void VisitPointers(HeapObject host, MaybeObjectSlot start, MaybeObjectSlot end) override; void VisitCodePointer(Code host, CodeObjectSlot slot) override; - void VisitCodeTarget(RelocInfo* rinfo) override; - void VisitEmbeddedPointer(RelocInfo* rinfo) override; + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) override; + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo) override; void VisitRootPointers(Root root, const char* description, FullObjectSlot start, FullObjectSlot end) override; @@ -163,13 +163,15 @@ VerifyPointersImpl(start, end); } -void VerifyPointersVisitor::VisitCodeTarget(RelocInfo* rinfo) { +void VerifyPointersVisitor::VisitCodeTarget(InstructionStream host, + RelocInfo* rinfo) { InstructionStream target = InstructionStream::FromTargetAddress(rinfo->target_address()); VerifyHeapObjectImpl(target); } -void VerifyPointersVisitor::VisitEmbeddedPointer(RelocInfo* rinfo) { +void VerifyPointersVisitor::VisitEmbeddedPointer(InstructionStream host, + RelocInfo* rinfo) { VerifyHeapObjectImpl(rinfo->target_object(cage_base())); } @@ -481,11 +483,10 @@ } } - void VisitCodeTarget(RelocInfo* rinfo) override { + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) override { Object target = InstructionStream::FromTargetAddress(rinfo->target_address()); - if (ShouldHaveBeenRecorded(rinfo->instruction_stream(), - MaybeObject::FromObject(target))) { + if (ShouldHaveBeenRecorded(host, MaybeObject::FromObject(target))) { CHECK(InTypedSet(SlotType::kCodeEntry, rinfo->pc()) || (rinfo->IsInConstantPool() && InTypedSet(SlotType::kConstPoolCodeEntry, @@ -493,10 +494,9 @@ } } - void VisitEmbeddedPointer(RelocInfo* rinfo) override { + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo) override { Object target = rinfo->target_object(cage_base()); - if (ShouldHaveBeenRecorded(rinfo->instruction_stream(), - MaybeObject::FromObject(target))) { + if (ShouldHaveBeenRecorded(host, MaybeObject::FromObject(target))) { CHECK(InTypedSet(SlotType::kEmbeddedObjectFull, rinfo->pc()) || InTypedSet(SlotType::kEmbeddedObjectCompressed, rinfo->pc()) || (rinfo->IsInConstantPool() && @@ -613,9 +613,13 @@ #endif } - void VisitCodeTarget(RelocInfo* rinfo) final { UNREACHABLE(); } + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) final { + UNREACHABLE(); + } - void VisitEmbeddedPointer(RelocInfo* rinfo) override { UNREACHABLE(); } + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo) override { + UNREACHABLE(); + } void VisitMapPointer(HeapObject object) override {} // do nothing by default
diff --git a/src/heap/heap-write-barrier-inl.h b/src/heap/heap-write-barrier-inl.h index bbb092a..32c18e9 100644 --- a/src/heap/heap-write-barrier-inl.h +++ b/src/heap/heap-write-barrier-inl.h
@@ -31,8 +31,8 @@ V8_EXPORT_PRIVATE void Heap_CombinedGenerationalAndSharedEphemeronBarrierSlow( EphemeronHashTable table, Address slot, HeapObject value); -V8_EXPORT_PRIVATE void Heap_GenerationalBarrierForCodeSlow(RelocInfo* rinfo, - HeapObject object); +V8_EXPORT_PRIVATE void Heap_GenerationalBarrierForCodeSlow( + InstructionStream host, RelocInfo* rinfo, HeapObject object); V8_EXPORT_PRIVATE void Heap_GenerationalEphemeronKeyBarrierSlow( Heap* heap, HeapObject table, Address slot); @@ -147,7 +147,7 @@ } DCHECK_EQ(mode, UPDATE_WRITE_BARRIER); - GenerationalBarrierForCode(rinfo, value); + GenerationalBarrierForCode(host, rinfo, value); WriteBarrier::Shared(host, rinfo, value); WriteBarrier::Marking(host, rinfo, value); } @@ -215,12 +215,13 @@ } } -inline void GenerationalBarrierForCode(RelocInfo* rinfo, HeapObject object) { +inline void GenerationalBarrierForCode(InstructionStream host, RelocInfo* rinfo, + HeapObject object) { if (V8_ENABLE_THIRD_PARTY_HEAP_BOOL) return; heap_internals::MemoryChunk* object_chunk = heap_internals::MemoryChunk::FromHeapObject(object); if (!object_chunk->InYoungGeneration()) return; - Heap_GenerationalBarrierForCodeSlow(rinfo, object); + Heap_GenerationalBarrierForCodeSlow(host, rinfo, object); } inline WriteBarrierMode GetWriteBarrierModeForObject( @@ -306,7 +307,7 @@ heap_internals::MemoryChunk::FromHeapObject(value); if (!value_chunk->InWritableSharedSpace()) return; - SharedSlow(reloc_info, value); + SharedSlow(host, reloc_info, value); } void WriteBarrier::Marking(JSArrayBuffer host,
diff --git a/src/heap/heap-write-barrier.cc b/src/heap/heap-write-barrier.cc index b730ce2..9c73095 100644 --- a/src/heap/heap-write-barrier.cc +++ b/src/heap/heap-write-barrier.cc
@@ -68,9 +68,10 @@ marking_barrier->Write(host, reloc_info, value); } -void WriteBarrier::SharedSlow(RelocInfo* reloc_info, HeapObject value) { +void WriteBarrier::SharedSlow(InstructionStream host, RelocInfo* reloc_info, + HeapObject value) { MarkCompactCollector::RecordRelocSlotInfo info = - MarkCompactCollector::ProcessRelocInfo(reloc_info, value); + MarkCompactCollector::ProcessRelocInfo(host, reloc_info, value); base::MutexGuard write_scope(info.memory_chunk->mutex()); RememberedSet<OLD_TO_SHARED>::InsertTyped(info.memory_chunk, info.slot_type,
diff --git a/src/heap/heap-write-barrier.h b/src/heap/heap-write-barrier.h index f44af9a..0ea460a 100644 --- a/src/heap/heap-write-barrier.h +++ b/src/heap/heap-write-barrier.h
@@ -44,7 +44,8 @@ Object value, WriteBarrierMode mode); // Generational write barrier. -void GenerationalBarrierForCode(RelocInfo* rinfo, HeapObject object); +void GenerationalBarrierForCode(InstructionStream host, RelocInfo* rinfo, + HeapObject object); inline bool IsReadOnlyHeapObject(HeapObject object); @@ -104,7 +105,7 @@ size_t argc, void** values); - static void SharedSlow(RelocInfo*, HeapObject value); + static void SharedSlow(InstructionStream host, RelocInfo*, HeapObject value); friend class Heap; };
diff --git a/src/heap/heap.cc b/src/heap/heap.cc index 4f0b67f..7920af5 100644 --- a/src/heap/heap.cc +++ b/src/heap/heap.cc
@@ -158,8 +158,9 @@ Heap::CombinedGenerationalAndSharedEphemeronBarrierSlow(table, slot, value); } -void Heap_GenerationalBarrierForCodeSlow(RelocInfo* rinfo, HeapObject object) { - Heap::GenerationalBarrierForCodeSlow(rinfo, object); +void Heap_GenerationalBarrierForCodeSlow(InstructionStream host, + RelocInfo* rinfo, HeapObject object) { + Heap::GenerationalBarrierForCodeSlow(host, rinfo, object); } void Heap::SetConstructStubCreateDeoptPCOffset(int pc_offset) { @@ -6285,12 +6286,12 @@ } } - void VisitCodeTarget(RelocInfo* rinfo) final { + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) final { InstructionStream target = InstructionStream::FromTargetAddress(rinfo->target_address()); MarkHeapObject(target); } - void VisitEmbeddedPointer(RelocInfo* rinfo) final { + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo) final { MarkHeapObject(rinfo->target_object(cage_base())); } @@ -7074,10 +7075,11 @@ } } -void Heap::GenerationalBarrierForCodeSlow(RelocInfo* rinfo, HeapObject object) { +void Heap::GenerationalBarrierForCodeSlow(InstructionStream host, + RelocInfo* rinfo, HeapObject object) { DCHECK(InYoungGeneration(object)); const MarkCompactCollector::RecordRelocSlotInfo info = - MarkCompactCollector::ProcessRelocInfo(rinfo, object); + MarkCompactCollector::ProcessRelocInfo(host, rinfo, object); RememberedSet<OLD_TO_NEW>::InsertTyped(info.memory_chunk, info.slot_type, info.offset);
diff --git a/src/heap/heap.h b/src/heap/heap.h index 7648f48..14f2840 100644 --- a/src/heap/heap.h +++ b/src/heap/heap.h
@@ -507,7 +507,7 @@ V8_EXPORT_PRIVATE static void EphemeronKeyWriteBarrierFromCode( Address raw_object, Address address, Isolate* isolate); V8_EXPORT_PRIVATE static void GenerationalBarrierForCodeSlow( - RelocInfo* rinfo, HeapObject value); + InstructionStream host, RelocInfo* rinfo, HeapObject value); V8_EXPORT_PRIVATE static bool PageFlagsAreConsistent(HeapObject object); // Notifies the heap that is ok to start marking or other activities that
diff --git a/src/heap/mark-compact-inl.h b/src/heap/mark-compact-inl.h index 58c6fa3..1e69f9f 100644 --- a/src/heap/mark-compact-inl.h +++ b/src/heap/mark-compact-inl.h
@@ -111,9 +111,10 @@ } template <typename MarkingState> -void MainMarkingVisitor<MarkingState>::RecordRelocSlot(RelocInfo* rinfo, +void MainMarkingVisitor<MarkingState>::RecordRelocSlot(InstructionStream host, + RelocInfo* rinfo, HeapObject target) { - MarkCompactCollector::RecordRelocSlot(rinfo, target); + MarkCompactCollector::RecordRelocSlot(host, rinfo, target); } Isolate* CollectorBase::isolate() { return heap()->isolate(); }
diff --git a/src/heap/mark-compact.cc b/src/heap/mark-compact.cc index 69973a9..6a2cfe30 100644 --- a/src/heap/mark-compact.cc +++ b/src/heap/mark-compact.cc
@@ -266,16 +266,17 @@ VerifyPointersImpl(start, end); } - void VisitCodeTarget(RelocInfo* rinfo) override { + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) override { InstructionStream target = InstructionStream::FromTargetAddress(rinfo->target_address()); VerifyHeapObjectImpl(target); } - void VisitEmbeddedPointer(RelocInfo* rinfo) override { + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo) override { DCHECK(RelocInfo::IsEmbeddedObjectMode(rinfo->rmode())); HeapObject target_object = rinfo->target_object(cage_base()); - if (!rinfo->code().IsWeakObject(target_object)) { + Code code = Code::unchecked_cast(host.raw_code(kAcquireLoad)); + if (!code.IsWeakObject(target_object)) { VerifyHeapObjectImpl(target_object); } } @@ -1139,14 +1140,14 @@ UNREACHABLE(); } - void VisitCodeTarget(RelocInfo* rinfo) override { + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) override { InstructionStream target = InstructionStream::FromTargetAddress(rinfo->target_address()); - MarkObject(rinfo->instruction_stream(), target); + MarkObject(host, target); } - void VisitEmbeddedPointer(RelocInfo* rinfo) override { - MarkObject(rinfo->instruction_stream(), rinfo->target_object(cage_base())); + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo) override { + MarkObject(host, rinfo->target_object(cage_base())); } private: @@ -1200,7 +1201,7 @@ UNREACHABLE(); } - void VisitCodeTarget(RelocInfo* rinfo) override { + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) override { #if DEBUG InstructionStream target = InstructionStream::FromTargetAddress(rinfo->target_address()); @@ -1208,8 +1209,8 @@ #endif } - void VisitEmbeddedPointer(RelocInfo* rinfo) override { - MarkObject(rinfo->instruction_stream(), rinfo->target_object(cage_base())); + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo) override { + MarkObject(host, rinfo->target_object(cage_base())); } private: @@ -1267,9 +1268,13 @@ } } - void VisitCodeTarget(RelocInfo* rinfo) override { UNREACHABLE(); } + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) override { + UNREACHABLE(); + } - void VisitEmbeddedPointer(RelocInfo* rinfo) override { UNREACHABLE(); } + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo) override { + UNREACHABLE(); + } private: V8_INLINE void CheckForSharedObject(HeapObject host, ObjectSlot slot, @@ -1414,8 +1419,13 @@ void VisitCodePointer(Code host, CodeObjectSlot slot) override { UNREACHABLE(); } - void VisitCodeTarget(RelocInfo* rinfo) override { UNREACHABLE(); } - void VisitEmbeddedPointer(RelocInfo* rinfo) override { UNREACHABLE(); } + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) override { + UNREACHABLE(); + } + void VisitEmbeddedPointer(InstructionStream host, + RelocInfo* rinfo) override { + UNREACHABLE(); + } private: ExternalPointerTable* table_; @@ -1533,7 +1543,8 @@ } } - inline void VisitCodeTarget(RelocInfo* rinfo) override { + inline void VisitCodeTarget(InstructionStream host, + RelocInfo* rinfo) override { DCHECK(RelocInfo::IsCodeTargetMode(rinfo->rmode())); InstructionStream target = InstructionStream::FromTargetAddress(rinfo->target_address()); @@ -1541,20 +1552,23 @@ // the old-to-new remembered set. DCHECK(!Heap::InYoungGeneration(target)); DCHECK(!target.InWritableSharedSpace()); - heap_->mark_compact_collector()->RecordRelocSlot(rinfo, target); + heap_->mark_compact_collector()->RecordRelocSlot(host, rinfo, target); } - inline void VisitEmbeddedPointer(RelocInfo* rinfo) override { + inline void VisitEmbeddedPointer(InstructionStream host, + RelocInfo* rinfo) override { DCHECK(RelocInfo::IsEmbeddedObjectMode(rinfo->rmode())); HeapObject object = rinfo->target_object(cage_base()); - GenerationalBarrierForCode(rinfo, object); - WriteBarrier::Shared(rinfo->instruction_stream(), rinfo, object); - heap_->mark_compact_collector()->RecordRelocSlot(rinfo, object); + GenerationalBarrierForCode(host, rinfo, object); + WriteBarrier::Shared(host, rinfo, object); + heap_->mark_compact_collector()->RecordRelocSlot(host, rinfo, object); } // Entries that are skipped for recording. - inline void VisitExternalReference(RelocInfo* rinfo) final {} - inline void VisitInternalReference(RelocInfo* rinfo) final {} + inline void VisitExternalReference(InstructionStream host, + RelocInfo* rinfo) final {} + inline void VisitInternalReference(InstructionStream host, + RelocInfo* rinfo) final {} inline void VisitExternalPointer(HeapObject host, ExternalPointerSlot slot, ExternalPointerTag tag) final {} @@ -3637,10 +3651,10 @@ } // static -bool MarkCompactCollector::ShouldRecordRelocSlot(RelocInfo* rinfo, +bool MarkCompactCollector::ShouldRecordRelocSlot(InstructionStream host, + RelocInfo* rinfo, HeapObject target) { - MemoryChunk* source_chunk = - MemoryChunk::FromHeapObject(rinfo->instruction_stream()); + MemoryChunk* source_chunk = MemoryChunk::FromHeapObject(host); BasicMemoryChunk* target_chunk = BasicMemoryChunk::FromHeapObject(target); return target_chunk->IsEvacuationCandidate() && !source_chunk->ShouldSkipEvacuationSlotRecording(); @@ -3648,7 +3662,8 @@ // static MarkCompactCollector::RecordRelocSlotInfo -MarkCompactCollector::ProcessRelocInfo(RelocInfo* rinfo, HeapObject target) { +MarkCompactCollector::ProcessRelocInfo(InstructionStream host, RelocInfo* rinfo, + HeapObject target) { RecordRelocSlotInfo result; const RelocInfo::Mode rmode = rinfo->rmode(); Address addr; @@ -3678,8 +3693,7 @@ } } - MemoryChunk* const source_chunk = - MemoryChunk::FromHeapObject(rinfo->instruction_stream()); + MemoryChunk* const source_chunk = MemoryChunk::FromHeapObject(host); const uintptr_t offset = addr - source_chunk->address(); DCHECK_LT(offset, static_cast<uintptr_t>(TypedSlotSet::kMaxOffset)); result.memory_chunk = source_chunk; @@ -3690,10 +3704,11 @@ } // static -void MarkCompactCollector::RecordRelocSlot(RelocInfo* rinfo, +void MarkCompactCollector::RecordRelocSlot(InstructionStream host, + RelocInfo* rinfo, HeapObject target) { - if (!ShouldRecordRelocSlot(rinfo, target)) return; - RecordRelocSlotInfo info = ProcessRelocInfo(rinfo, target); + if (!ShouldRecordRelocSlot(host, rinfo, target)) return; + RecordRelocSlotInfo info = ProcessRelocInfo(host, rinfo, target); // Access to TypeSlots need to be protected, since LocalHeaps might // publish code in the background thread. @@ -3935,12 +3950,12 @@ } } - void VisitCodeTarget(RelocInfo* rinfo) override { + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) override { // This visitor nevers visits code objects. UNREACHABLE(); } - void VisitEmbeddedPointer(RelocInfo* rinfo) override { + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo) override { // This visitor nevers visits code objects. UNREACHABLE(); } @@ -5512,12 +5527,12 @@ UNREACHABLE(); } - void VisitCodeTarget(RelocInfo* rinfo) override { + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) override { InstructionStream target = InstructionStream::FromTargetAddress(rinfo->target_address()); VerifyHeapObjectImpl(target); } - void VisitEmbeddedPointer(RelocInfo* rinfo) override { + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo) override { VerifyHeapObjectImpl(rinfo->target_object(cage_base())); } void VerifyRootPointers(FullObjectSlot start, FullObjectSlot end) override {
diff --git a/src/heap/mark-compact.h b/src/heap/mark-compact.h index 40816ac..42d42ad 100644 --- a/src/heap/mark-compact.h +++ b/src/heap/mark-compact.h
@@ -88,7 +88,8 @@ template <typename TSlot> void RecordSlot(HeapObject object, TSlot slot, HeapObject target); - void RecordRelocSlot(RelocInfo* rinfo, HeapObject target); + void RecordRelocSlot(InstructionStream host, RelocInfo* rinfo, + HeapObject target); MarkingState* marking_state() { return marking_state_; } @@ -277,11 +278,14 @@ uint32_t offset; }; - static bool ShouldRecordRelocSlot(RelocInfo* rinfo, HeapObject target); - static RecordRelocSlotInfo ProcessRelocInfo(RelocInfo* rinfo, + static bool ShouldRecordRelocSlot(InstructionStream host, RelocInfo* rinfo, + HeapObject target); + static RecordRelocSlotInfo ProcessRelocInfo(InstructionStream host, + RelocInfo* rinfo, HeapObject target); - static void RecordRelocSlot(RelocInfo* rinfo, HeapObject target); + static void RecordRelocSlot(InstructionStream host, RelocInfo* rinfo, + HeapObject target); V8_INLINE static void RecordSlot(HeapObject object, ObjectSlot slot, HeapObject target); V8_INLINE static void RecordSlot(HeapObject object, HeapObjectSlot slot,
diff --git a/src/heap/marking-barrier.cc b/src/heap/marking-barrier.cc index f6f10d7..64e4fc4 100644 --- a/src/heap/marking-barrier.cc +++ b/src/heap/marking-barrier.cc
@@ -77,9 +77,9 @@ if (is_main_thread_barrier_) { // An optimization to avoid allocating additional typed slots for the // main thread. - major_collector_->RecordRelocSlot(reloc_info, value); + major_collector_->RecordRelocSlot(host, reloc_info, value); } else { - RecordRelocSlot(reloc_info, value); + RecordRelocSlot(host, reloc_info, value); } } } @@ -143,12 +143,13 @@ } } -void MarkingBarrier::RecordRelocSlot(RelocInfo* rinfo, HeapObject target) { - DCHECK(IsCurrentMarkingBarrier(rinfo->instruction_stream())); - if (!MarkCompactCollector::ShouldRecordRelocSlot(rinfo, target)) return; +void MarkingBarrier::RecordRelocSlot(InstructionStream host, RelocInfo* rinfo, + HeapObject target) { + DCHECK(IsCurrentMarkingBarrier(host)); + if (!MarkCompactCollector::ShouldRecordRelocSlot(host, rinfo, target)) return; MarkCompactCollector::RecordRelocSlotInfo info = - MarkCompactCollector::ProcessRelocInfo(rinfo, target); + MarkCompactCollector::ProcessRelocInfo(host, rinfo, target); auto& typed_slots = typed_slots_map_[info.memory_chunk]; if (!typed_slots) {
diff --git a/src/heap/marking-barrier.h b/src/heap/marking-barrier.h index 382267e..35d59d3 100644 --- a/src/heap/marking-barrier.h +++ b/src/heap/marking-barrier.h
@@ -66,7 +66,8 @@ inline bool WhiteToGreyAndPush(HeapObject value); - void RecordRelocSlot(RelocInfo* rinfo, HeapObject target); + void RecordRelocSlot(InstructionStream host, RelocInfo* rinfo, + HeapObject target); bool IsCurrentMarkingBarrier(HeapObject verification_candidate);
diff --git a/src/heap/marking-visitor-inl.h b/src/heap/marking-visitor-inl.h index 7ade85b..b0f78b5 100644 --- a/src/heap/marking-visitor-inl.h +++ b/src/heap/marking-visitor-inl.h
@@ -117,35 +117,35 @@ template <typename ConcreteVisitor, typename MarkingState> void MarkingVisitorBase<ConcreteVisitor, MarkingState>::VisitEmbeddedPointer( - RelocInfo* rinfo) { + InstructionStream host, RelocInfo* rinfo) { DCHECK(RelocInfo::IsEmbeddedObjectMode(rinfo->rmode())); HeapObject object = rinfo->target_object(ObjectVisitorWithCageBases::cage_base()); if (!ShouldMarkObject(object)) return; if (!concrete_visitor()->marking_state()->IsMarked(object)) { - if (rinfo->code().IsWeakObject(object)) { + Code code = Code::unchecked_cast(host.raw_code(kAcquireLoad)); + if (code.IsWeakObject(object)) { local_weak_objects_->weak_objects_in_code_local.Push( - std::make_pair(object, rinfo->code())); - AddWeakReferenceForReferenceSummarizer(rinfo->instruction_stream(), - object); + std::make_pair(object, code)); + AddWeakReferenceForReferenceSummarizer(host, object); } else { - MarkObject(rinfo->instruction_stream(), object); + MarkObject(host, object); } } - concrete_visitor()->RecordRelocSlot(rinfo, object); + concrete_visitor()->RecordRelocSlot(host, rinfo, object); } template <typename ConcreteVisitor, typename MarkingState> void MarkingVisitorBase<ConcreteVisitor, MarkingState>::VisitCodeTarget( - RelocInfo* rinfo) { + InstructionStream host, RelocInfo* rinfo) { DCHECK(RelocInfo::IsCodeTargetMode(rinfo->rmode())); InstructionStream target = InstructionStream::FromTargetAddress(rinfo->target_address()); if (!ShouldMarkObject(target)) return; - MarkObject(rinfo->instruction_stream(), target); - concrete_visitor()->RecordRelocSlot(rinfo, target); + MarkObject(host, target); + concrete_visitor()->RecordRelocSlot(host, rinfo, target); } template <typename ConcreteVisitor, typename MarkingState>
diff --git a/src/heap/marking-visitor.h b/src/heap/marking-visitor.h index 547985f..403a060 100644 --- a/src/heap/marking-visitor.h +++ b/src/heap/marking-visitor.h
@@ -101,8 +101,10 @@ V8_INLINE void VisitCodePointer(Code host, CodeObjectSlot slot) final { VisitCodePointerImpl(host, slot); } - V8_INLINE void VisitEmbeddedPointer(RelocInfo* rinfo) final; - V8_INLINE void VisitCodeTarget(RelocInfo* rinfo) final; + V8_INLINE void VisitEmbeddedPointer(InstructionStream host, + RelocInfo* rinfo) final; + V8_INLINE void VisitCodeTarget(InstructionStream host, + RelocInfo* rinfo) final; void VisitCustomWeakPointers(HeapObject host, ObjectSlot start, ObjectSlot end) final { // Weak list pointers should be ignored during marking. The lists are
diff --git a/src/heap/object-stats.cc b/src/heap/object-stats.cc index 8677615..06328cf 100644 --- a/src/heap/object-stats.cc +++ b/src/heap/object-stats.cc
@@ -101,12 +101,12 @@ *tagged_fields_count_ += 1; } - void VisitCodeTarget(RelocInfo* rinfo) override { + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) override { // InstructionStream target is most likely encoded as a relative 32-bit // offset and not as a full tagged value, so there's nothing to count. } - void VisitEmbeddedPointer(RelocInfo* rinfo) override { + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo) override { *tagged_fields_count_ += 1; }
diff --git a/src/heap/objects-visiting.h b/src/heap/objects-visiting.h index 89808a0..f678f26 100644 --- a/src/heap/objects-visiting.h +++ b/src/heap/objects-visiting.h
@@ -200,8 +200,12 @@ // Special cases: Unreachable visitors for objects that are never found in the // young generation. void VisitCodePointer(Code, CodeObjectSlot) final { UNREACHABLE(); } - void VisitCodeTarget(RelocInfo*) final { UNREACHABLE(); } - void VisitEmbeddedPointer(RelocInfo*) final { UNREACHABLE(); } + void VisitCodeTarget(InstructionStream host, RelocInfo*) final { + UNREACHABLE(); + } + void VisitEmbeddedPointer(InstructionStream host, RelocInfo*) final { + UNREACHABLE(); + } void VisitMapPointer(HeapObject) override { UNREACHABLE(); } protected:
diff --git a/src/heap/remembered-set-inl.h b/src/heap/remembered-set-inl.h index d30f887..e17fe74 100644 --- a/src/heap/remembered-set-inl.h +++ b/src/heap/remembered-set-inl.h
@@ -20,21 +20,18 @@ Callback callback) { switch (slot_type) { case SlotType::kCodeEntry: { - RelocInfo rinfo(addr, RelocInfo::CODE_TARGET, 0, Code(), - InstructionStream()); + RelocInfo rinfo(addr, RelocInfo::CODE_TARGET); return UpdateCodeTarget(&rinfo, callback); } case SlotType::kConstPoolCodeEntry: { return UpdateCodeEntry(addr, callback); } case SlotType::kEmbeddedObjectCompressed: { - RelocInfo rinfo(addr, RelocInfo::COMPRESSED_EMBEDDED_OBJECT, 0, Code(), - InstructionStream()); + RelocInfo rinfo(addr, RelocInfo::COMPRESSED_EMBEDDED_OBJECT); return UpdateEmbeddedPointer(heap, &rinfo, callback); } case SlotType::kEmbeddedObjectFull: { - RelocInfo rinfo(addr, RelocInfo::FULL_EMBEDDED_OBJECT, 0, Code(), - InstructionStream()); + RelocInfo rinfo(addr, RelocInfo::FULL_EMBEDDED_OBJECT); return UpdateEmbeddedPointer(heap, &rinfo, callback); } case SlotType::kConstPoolEmbeddedObjectCompressed: { @@ -64,21 +61,18 @@ Address addr) { switch (slot_type) { case SlotType::kCodeEntry: { - RelocInfo rinfo(addr, RelocInfo::CODE_TARGET, 0, Code(), - InstructionStream()); + RelocInfo rinfo(addr, RelocInfo::CODE_TARGET); return InstructionStream::FromTargetAddress(rinfo.target_address()); } case SlotType::kConstPoolCodeEntry: { return InstructionStream::FromEntryAddress(addr); } case SlotType::kEmbeddedObjectCompressed: { - RelocInfo rinfo(addr, RelocInfo::COMPRESSED_EMBEDDED_OBJECT, 0, Code(), - InstructionStream()); + RelocInfo rinfo(addr, RelocInfo::COMPRESSED_EMBEDDED_OBJECT); return rinfo.target_object(heap->isolate()); } case SlotType::kEmbeddedObjectFull: { - RelocInfo rinfo(addr, RelocInfo::FULL_EMBEDDED_OBJECT, 0, Code(), - InstructionStream()); + RelocInfo rinfo(addr, RelocInfo::FULL_EMBEDDED_OBJECT); return rinfo.target_object(heap->isolate()); } case SlotType::kConstPoolEmbeddedObjectCompressed: {
diff --git a/src/heap/remembered-set.h b/src/heap/remembered-set.h index c31ef19..5471b0f 100644 --- a/src/heap/remembered-set.h +++ b/src/heap/remembered-set.h
@@ -362,7 +362,7 @@ SlotCallbackResult result = callback(FullMaybeObjectSlot(&new_target)); DCHECK(!HasWeakHeapObjectTag(new_target)); if (new_target != old_target) { - rinfo->set_target_object(heap, HeapObject::cast(new_target)); + rinfo->set_target_object(HeapObject::cast(new_target)); } return result; }
diff --git a/src/heap/scavenger.cc b/src/heap/scavenger.cc index e16d002..87de6ce 100644 --- a/src/heap/scavenger.cc +++ b/src/heap/scavenger.cc
@@ -69,17 +69,17 @@ UNREACHABLE(); } - V8_INLINE void VisitCodeTarget(RelocInfo* rinfo) final { + V8_INLINE void VisitCodeTarget(InstructionStream host, + RelocInfo* rinfo) final { InstructionStream target = InstructionStream::FromTargetAddress(rinfo->target_address()); - HandleSlot(rinfo->instruction_stream(), FullHeapObjectSlot(&target), - target); + HandleSlot(host, FullHeapObjectSlot(&target), target); } - V8_INLINE void VisitEmbeddedPointer(RelocInfo* rinfo) final { - PtrComprCageBase cage_base = GetPtrComprCageBase(rinfo->code()); + V8_INLINE void VisitEmbeddedPointer(InstructionStream host, + RelocInfo* rinfo) final { + PtrComprCageBase cage_base = GetPtrComprCageBase(); HeapObject heap_object = rinfo->target_object(cage_base); - HandleSlot(rinfo->instruction_stream(), FullHeapObjectSlot(&heap_object), - heap_object); + HandleSlot(host, FullHeapObjectSlot(&heap_object), heap_object); } inline void VisitEphemeron(HeapObject obj, int entry, ObjectSlot key,
diff --git a/src/heap/sweeper.cc b/src/heap/sweeper.cc index f99ad60..daec243 100644 --- a/src/heap/sweeper.cc +++ b/src/heap/sweeper.cc
@@ -809,8 +809,8 @@ } // Entries that are skipped for recording. - void VisitExternalReference(RelocInfo* rinfo) final {} - void VisitInternalReference(RelocInfo* rinfo) final {} + void VisitExternalReference(InstructionStream host, RelocInfo* rinfo) final {} + void VisitInternalReference(InstructionStream host, RelocInfo* rinfo) final {} void VisitExternalPointer(HeapObject host, ExternalPointerSlot slot, ExternalPointerTag tag) final {}
diff --git a/src/objects/code-inl.h b/src/objects/code-inl.h index 59ea6c8..1ecb085 100644 --- a/src/objects/code-inl.h +++ b/src/objects/code-inl.h
@@ -428,6 +428,7 @@ Address Code::constant_pool(InstructionStream instruction_stream) const { if (!has_constant_pool()) return kNullAddress; + static_assert(InstructionStream::kOnHeapBodyIsContiguous); return instruction_stream.instruction_start() + instruction_size() + constant_pool_offset(); }
diff --git a/src/objects/code.cc b/src/objects/code.cc index c4f4291..cda766e 100644 --- a/src/objects/code.cc +++ b/src/objects/code.cc
@@ -8,7 +8,7 @@ #include "src/codegen/assembler-inl.h" #include "src/codegen/flush-instruction-cache.h" -#include "src/codegen/reloc-info.h" +#include "src/codegen/reloc-info-inl.h" #include "src/deoptimizer/deoptimizer.h" #include "src/objects/code-inl.h" @@ -30,11 +30,13 @@ } void Code::ClearEmbeddedObjects(Heap* heap) { + DisallowGarbageCollection no_gc; HeapObject undefined = ReadOnlyRoots(heap).undefined_value(); + InstructionStream istream = unchecked_instruction_stream(); int mode_mask = RelocInfo::EmbeddedObjectModeMask(); for (RelocIterator it(*this, mode_mask); !it.done(); it.next()) { DCHECK(RelocInfo::IsEmbeddedObjectMode(it.rinfo()->rmode())); - it.rinfo()->set_target_object(heap, undefined, SKIP_WRITE_BARRIER); + it.rinfo()->set_target_object(istream, undefined, SKIP_WRITE_BARRIER); } set_embedded_objects_cleared(true); } @@ -45,49 +47,80 @@ void Code::CopyFromNoFlush(ByteArray reloc_info, Heap* heap, const CodeDesc& desc) { - // Copy code. + // Copy from compilation artifacts stored in CodeDesc to the target on-heap + // objects. + // + // Note this is quite convoluted for historical reasons. The CodeDesc buffer + // contains instructions, a part of inline metadata, and the relocation info. + // Additionally, the unwinding_info is stored in a separate buffer + // `desc.unwinding_info`. In this method, we copy all these parts into the + // final on-heap representation. + // + // The off-heap representation: + // + // CodeDesc.buffer: + // + // +------------------- + // | instructions + // +------------------- + // | inline metadata + // | .. safepoint table + // | .. handler table + // | .. constant pool + // | .. code comments + // +------------------- + // | reloc info + // +------------------- + // + // CodeDesc.unwinding_info: .. the unwinding info. + // + // This is transformed into the on-heap representation, where + // InstructionStream contains all instructions and inline metadata, and a + // pointer to the relocation info byte array. + + // Copy code and inline metadata. static_assert(InstructionStream::kOnHeapBodyIsContiguous); CopyBytes(reinterpret_cast<byte*>(instruction_start()), desc.buffer, static_cast<size_t>(desc.instr_size)); - // TODO(jgruber,v8:11036): Merge with the above. CopyBytes(reinterpret_cast<byte*>(instruction_start() + desc.instr_size), desc.unwinding_info, static_cast<size_t>(desc.unwinding_info_size)); + DCHECK_EQ(desc.body_size(), desc.instr_size + desc.unwinding_info_size); + DCHECK_EQ(body_size(), instruction_size() + metadata_size()); - // Copy reloc info. + // Copy the relocation info. DCHECK_EQ(reloc_info.length(), desc.reloc_size); - CopyBytes(reloc_info.GetDataStartAddress(), - desc.buffer + desc.buffer_size - desc.reloc_size, + CopyBytes(reloc_info.GetDataStartAddress(), desc.buffer + desc.reloc_offset, static_cast<size_t>(desc.reloc_size)); - // Unbox handles and relocate. - RelocateFromDesc(reloc_info, heap, desc); + RelocateFromDesc(heap, desc); } -void Code::RelocateFromDesc(ByteArray reloc_info, Heap* heap, - const CodeDesc& desc) { - // Unbox handles and relocate. +void Code::RelocateFromDesc(Heap* heap, const CodeDesc& desc) { + DisallowGarbageCollection no_gc; Assembler* origin = desc.origin; + InstructionStream istream = instruction_stream(); const int mode_mask = RelocInfo::PostCodegenRelocationMask(); - for (RelocIterator it(*this, reloc_info, mode_mask); !it.done(); it.next()) { + for (RelocIterator it(*this, mode_mask); !it.done(); it.next()) { RelocInfo::Mode mode = it.rinfo()->rmode(); if (RelocInfo::IsEmbeddedObjectMode(mode)) { Handle<HeapObject> p = it.rinfo()->target_object_handle(origin); - it.rinfo()->set_target_object(heap, *p, UPDATE_WRITE_BARRIER, + it.rinfo()->set_target_object(istream, *p, UPDATE_WRITE_BARRIER, SKIP_ICACHE_FLUSH); } else if (RelocInfo::IsCodeTargetMode(mode)) { // Rewrite code handles to direct pointers to the first instruction in the // code object. Handle<HeapObject> p = it.rinfo()->target_object_handle(origin); DCHECK(p->IsCode(GetPtrComprCageBaseSlow(*p))); - InstructionStream istream = Code::cast(*p).instruction_stream(); - it.rinfo()->set_target_address(istream.instruction_start(), + InstructionStream target_istream = Code::cast(*p).instruction_stream(); + it.rinfo()->set_target_address(istream, + target_istream.instruction_start(), UPDATE_WRITE_BARRIER, SKIP_ICACHE_FLUSH); } else if (RelocInfo::IsNearBuiltinEntry(mode)) { // Rewrite builtin IDs to PC-relative offset to the builtin entry point. Builtin builtin = it.rinfo()->target_builtin_at(origin); Address p = heap->isolate()->builtin_entry_table()[Builtins::ToInt(builtin)]; - it.rinfo()->set_target_address(p, UPDATE_WRITE_BARRIER, + it.rinfo()->set_target_address(istream, p, UPDATE_WRITE_BARRIER, SKIP_ICACHE_FLUSH); DCHECK_EQ(p, it.rinfo()->target_address()); } else if (RelocInfo::IsWasmStubCall(mode)) {
diff --git a/src/objects/code.h b/src/objects/code.h index e6dd503..2af9541 100644 --- a/src/objects/code.h +++ b/src/objects/code.h
@@ -260,7 +260,7 @@ // Migrate code from desc without flushing the instruction cache. void CopyFromNoFlush(ByteArray reloc_info, Heap* heap, const CodeDesc& desc); - void RelocateFromDesc(ByteArray reloc_info, Heap* heap, const CodeDesc& desc); + void RelocateFromDesc(Heap* heap, const CodeDesc& desc); bool IsIsolateIndependent(Isolate* isolate);
diff --git a/src/objects/objects-body-descriptors-inl.h b/src/objects/objects-body-descriptors-inl.h index f015611..9393eb1 100644 --- a/src/objects/objects-body-descriptors-inl.h +++ b/src/objects/objects-body-descriptors-inl.h
@@ -992,7 +992,7 @@ if (istream.TryGetCodeUnchecked(&code, kAcquireLoad)) { RelocIterator it(code, istream, istream.unchecked_relocation_info(), kRelocModeMask); - v->VisitRelocInfo(&it); + v->VisitRelocInfo(istream, &it); } }
diff --git a/src/objects/visitors.cc b/src/objects/visitors.cc index 434255b..6aa9adb 100644 --- a/src/objects/visitors.cc +++ b/src/objects/visitors.cc
@@ -6,6 +6,11 @@ #include "src/codegen/reloc-info.h" +#ifdef DEBUG +#include "src/objects/instruction-stream-inl.h" +#include "src/objects/smi.h" +#endif // DEBUG + namespace v8 { namespace internal { @@ -22,9 +27,12 @@ UNREACHABLE(); } -void ObjectVisitor::VisitRelocInfo(RelocIterator* it) { +void ObjectVisitor::VisitRelocInfo(InstructionStream host, RelocIterator* it) { + // RelocInfo iteration is only valid for fully-initialized InstructionStream + // objects. Callers must ensure this. + DCHECK_NE(host.raw_code(kAcquireLoad), Smi::zero()); for (; !it->done(); it->next()) { - it->rinfo()->Visit(this); + it->rinfo()->Visit(host, this); } }
diff --git a/src/objects/visitors.h b/src/objects/visitors.h index ab16adf..140ab11 100644 --- a/src/objects/visitors.h +++ b/src/objects/visitors.h
@@ -9,6 +9,7 @@ #include "src/objects/code.h" #include "src/objects/compressed-slots.h" #include "src/objects/foreign.h" +#include "src/objects/instruction-stream.h" #include "src/objects/slots.h" namespace v8 { @@ -164,23 +165,21 @@ VisitPointer(host, value); } - virtual void VisitCodeTarget(RelocInfo* rinfo) = 0; + // Visits the relocation info using the given iterator. + void VisitRelocInfo(InstructionStream host, RelocIterator* it); - virtual void VisitEmbeddedPointer(RelocInfo* rinfo) = 0; - - virtual void VisitExternalReference(RelocInfo* rinfo) {} + virtual void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) {} + virtual void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo) {} + virtual void VisitExternalReference(InstructionStream host, + RelocInfo* rinfo) {} + virtual void VisitInternalReference(InstructionStream host, + RelocInfo* rinfo) {} + // TODO(ishell): rename to VisitBuiltinEntry. + virtual void VisitOffHeapTarget(InstructionStream host, RelocInfo* rinfo) {} virtual void VisitExternalPointer(HeapObject host, ExternalPointerSlot slot, ExternalPointerTag tag) {} - virtual void VisitInternalReference(RelocInfo* rinfo) {} - - // TODO(ishell): rename to VisitBuiltinEntry. - virtual void VisitOffHeapTarget(RelocInfo* rinfo) {} - - // Visits the relocation info using the given iterator. - void VisitRelocInfo(RelocIterator* it); - virtual void VisitMapPointer(HeapObject host) { UNREACHABLE(); } };
diff --git a/src/profiler/heap-snapshot-generator.cc b/src/profiler/heap-snapshot-generator.cc index ae413e5..6bf1abc 100644 --- a/src/profiler/heap-snapshot-generator.cc +++ b/src/profiler/heap-snapshot-generator.cc
@@ -1087,15 +1087,16 @@ VisitSlotImpl(code_cage_base(), slot); } - void VisitCodeTarget(RelocInfo* rinfo) override { + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo) override { InstructionStream target = InstructionStream::FromTargetAddress(rinfo->target_address()); VisitHeapObjectImpl(target, -1); } - void VisitEmbeddedPointer(RelocInfo* rinfo) override { + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo) override { HeapObject object = rinfo->target_object(cage_base()); - if (rinfo->code().IsWeakObject(object)) { + Code code = Code::unchecked_cast(host.raw_code(kAcquireLoad)); + if (code.IsWeakObject(object)) { generator_->SetWeakReference(parent_, next_index_++, object, {}); } else { VisitHeapObjectImpl(object, -1);
diff --git a/src/snapshot/deserializer.cc b/src/snapshot/deserializer.cc index d438087..2959688 100644 --- a/src/snapshot/deserializer.cc +++ b/src/snapshot/deserializer.cc
@@ -6,6 +6,7 @@ #include "src/base/logging.h" #include "src/codegen/assembler-inl.h" +#include "src/codegen/reloc-info-inl.h" #include "src/common/assert-scope.h" #include "src/common/globals.h" #include "src/execution/isolate.h" @@ -731,11 +732,11 @@ DCHECK_EQ(current_object_, objects_->size()); } - void VisitCodeTarget(RelocInfo* rinfo); - void VisitEmbeddedPointer(RelocInfo* rinfo); - void VisitExternalReference(RelocInfo* rinfo); - void VisitInternalReference(RelocInfo* rinfo); - void VisitOffHeapTarget(RelocInfo* rinfo); + void VisitCodeTarget(InstructionStream host, RelocInfo* rinfo); + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* rinfo); + void VisitExternalReference(InstructionStream host, RelocInfo* rinfo); + void VisitInternalReference(InstructionStream host, RelocInfo* rinfo); + void VisitOffHeapTarget(InstructionStream host, RelocInfo* rinfo); private: Isolate* isolate() { return deserializer_->isolate(); } @@ -746,19 +747,22 @@ int current_object_; }; -void DeserializerRelocInfoVisitor::VisitCodeTarget(RelocInfo* rinfo) { +void DeserializerRelocInfoVisitor::VisitCodeTarget(InstructionStream host, + RelocInfo* rinfo) { HeapObject object = *objects_->at(current_object_++); rinfo->set_target_address( - InstructionStream::cast(object).instruction_start()); + host, InstructionStream::cast(object).instruction_start()); } -void DeserializerRelocInfoVisitor::VisitEmbeddedPointer(RelocInfo* rinfo) { +void DeserializerRelocInfoVisitor::VisitEmbeddedPointer(InstructionStream host, + RelocInfo* rinfo) { HeapObject object = *objects_->at(current_object_++); // Embedded object reference must be a strong one. - rinfo->set_target_object(isolate()->heap(), object); + rinfo->set_target_object(host, object); } -void DeserializerRelocInfoVisitor::VisitExternalReference(RelocInfo* rinfo) { +void DeserializerRelocInfoVisitor::VisitExternalReference( + InstructionStream host, RelocInfo* rinfo) { byte data = source().Get(); CHECK_EQ(data, Deserializer<Isolate>::kExternalReference); @@ -766,14 +770,15 @@ if (rinfo->IsCodedSpecially()) { Address location_of_branch_data = rinfo->pc(); - Assembler::deserialization_set_special_target_at(location_of_branch_data, - rinfo->code(), address); + Assembler::deserialization_set_special_target_at( + location_of_branch_data, host.code(kAcquireLoad), address); } else { WriteUnalignedValue(rinfo->target_address_address(), address); } } -void DeserializerRelocInfoVisitor::VisitInternalReference(RelocInfo* rinfo) { +void DeserializerRelocInfoVisitor::VisitInternalReference( + InstructionStream host, RelocInfo* rinfo) { byte data = source().Get(); CHECK_EQ(data, Deserializer<Isolate>::kInternalReference); @@ -781,13 +786,14 @@ int target_offset = source().GetInt(); static_assert(InstructionStream::kOnHeapBodyIsContiguous); DCHECK_LT(static_cast<unsigned>(target_offset), - static_cast<unsigned>(rinfo->code().instruction_size())); - Address target = rinfo->code().instruction_start() + target_offset; + static_cast<unsigned>(host.code(kAcquireLoad).instruction_size())); + Address target = host.instruction_start() + target_offset; Assembler::deserialization_set_target_internal_reference_at( rinfo->pc(), target, rinfo->rmode()); } -void DeserializerRelocInfoVisitor::VisitOffHeapTarget(RelocInfo* rinfo) { +void DeserializerRelocInfoVisitor::VisitOffHeapTarget(InstructionStream host, + RelocInfo* rinfo) { // Currently we don't serialize code that contains near builtin entries. DCHECK_NE(rinfo->rmode(), RelocInfo::NEAR_BUILTIN_ENTRY); @@ -804,8 +810,8 @@ // TODO(ishell): implement RelocInfo::set_target_off_heap_target() if (RelocInfo::OffHeapTargetIsCodedSpecially()) { Address location_of_branch_data = rinfo->pc(); - Assembler::deserialization_set_special_target_at(location_of_branch_data, - rinfo->code(), address); + Assembler::deserialization_set_special_target_at( + location_of_branch_data, host.code(kAcquireLoad), address); } else { WriteUnalignedValue(rinfo->target_address_address(), address); } @@ -1225,7 +1231,7 @@ for (RelocIterator it(code, istream, istream.relocation_info(), InstructionStream::BodyDescriptor::kRelocModeMask); !it.done(); it.next()) { - it.rinfo()->Visit(&visitor); + it.rinfo()->Visit(istream, &visitor); } }
diff --git a/src/snapshot/serializer.cc b/src/snapshot/serializer.cc index 8586412..dcb2d38a 100644 --- a/src/snapshot/serializer.cc +++ b/src/snapshot/serializer.cc
@@ -1067,12 +1067,12 @@ explicit RelocInfoObjectPreSerializer(Serializer* serializer) : serializer_(serializer) {} - void VisitEmbeddedPointer(RelocInfo* target) { + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* target) { HeapObject object = target->target_object(isolate()); serializer_->SerializeObject(handle(object, isolate()), SlotType::kAnySlot); num_serialized_objects_++; } - void VisitCodeTarget(RelocInfo* target) { + void VisitCodeTarget(InstructionStream host, RelocInfo* target) { #ifdef V8_TARGET_ARCH_ARM DCHECK(!RelocInfo::IsRelativeCodeTarget(target->rmode())); #endif @@ -1082,9 +1082,9 @@ num_serialized_objects_++; } - void VisitExternalReference(RelocInfo* rinfo) {} - void VisitInternalReference(RelocInfo* rinfo) {} - void VisitOffHeapTarget(RelocInfo* target) {} + void VisitExternalReference(InstructionStream host, RelocInfo* rinfo) {} + void VisitInternalReference(InstructionStream host, RelocInfo* rinfo) {} + void VisitOffHeapTarget(InstructionStream host, RelocInfo* target) {} int num_serialized_objects() const { return num_serialized_objects_; } @@ -1095,7 +1095,8 @@ int num_serialized_objects_ = 0; }; -void Serializer::ObjectSerializer::VisitEmbeddedPointer(RelocInfo* rinfo) { +void Serializer::ObjectSerializer::VisitEmbeddedPointer(InstructionStream host, + RelocInfo* rinfo) { // Target object should be pre-serialized by RelocInfoObjectPreSerializer, so // just track the pointer's existence as kTaggedSize in // bytes_processed_so_far_. @@ -1104,7 +1105,8 @@ bytes_processed_so_far_ += kTaggedSize; } -void Serializer::ObjectSerializer::VisitExternalReference(RelocInfo* rinfo) { +void Serializer::ObjectSerializer::VisitExternalReference( + InstructionStream host, RelocInfo* rinfo) { Address target = rinfo->target_external_reference(); DCHECK_NE(target, kNullAddress); // InstructionStream does not reference null. @@ -1115,12 +1117,13 @@ kExternalPointerNullTag); } -void Serializer::ObjectSerializer::VisitInternalReference(RelocInfo* rinfo) { - Address entry = rinfo->code().instruction_start(); +void Serializer::ObjectSerializer::VisitInternalReference( + InstructionStream host, RelocInfo* rinfo) { + Address entry = host.instruction_start(); DCHECK_GE(rinfo->target_internal_reference(), entry); uintptr_t target_offset = rinfo->target_internal_reference() - entry; static_assert(InstructionStream::kOnHeapBodyIsContiguous); - DCHECK_LT(target_offset, rinfo->code().instruction_size()); + DCHECK_LT(target_offset, host.code(kAcquireLoad).instruction_size()); sink_->Put(kInternalReference, "InternalRef"); sink_->PutInt(target_offset, "internal ref value"); } @@ -1164,7 +1167,8 @@ } } -void Serializer::ObjectSerializer::VisitOffHeapTarget(RelocInfo* rinfo) { +void Serializer::ObjectSerializer::VisitOffHeapTarget(InstructionStream host, + RelocInfo* rinfo) { static_assert(EmbeddedData::kTableSize == Builtins::kBuiltinCount); // Currently we don't serialize code that contains near builtin entries. @@ -1181,7 +1185,8 @@ sink_->PutInt(static_cast<int>(builtin), "builtin index"); } -void Serializer::ObjectSerializer::VisitCodeTarget(RelocInfo* rinfo) { +void Serializer::ObjectSerializer::VisitCodeTarget(InstructionStream host, + RelocInfo* rinfo) { // Target object should be pre-serialized by RelocInfoObjectPreSerializer, so // just track the pointer's existence as kTaggedSize in // bytes_processed_so_far_. @@ -1356,7 +1361,7 @@ for (RelocIterator it(*code, *on_heap_istream, relocation_info, InstructionStream::BodyDescriptor::kRelocModeMask); !it.done(); it.next()) { - it.rinfo()->Visit(&pre_serializer); + it.rinfo()->Visit(*on_heap_istream, &pre_serializer); } // Mark that the pre-serialization finished with a kSynchronize bytecode. sink_->Put(kSynchronize, "PreSerializationFinished"); @@ -1367,7 +1372,7 @@ for (RelocIterator it(*code, *on_heap_istream, relocation_info, InstructionStream::BodyDescriptor::kRelocModeMask); !it.done(); it.next()) { - it.rinfo()->Visit(this); + it.rinfo()->Visit(*on_heap_istream, this); } // We record a kTaggedSize for every object encountered during the
diff --git a/src/snapshot/serializer.h b/src/snapshot/serializer.h index 1b5340f..67cdaba 100644 --- a/src/snapshot/serializer.h +++ b/src/snapshot/serializer.h
@@ -470,11 +470,13 @@ void VisitPointers(HeapObject host, MaybeObjectSlot start, MaybeObjectSlot end) override; void VisitCodePointer(Code host, CodeObjectSlot slot) override; - void VisitEmbeddedPointer(RelocInfo* target) override; - void VisitExternalReference(RelocInfo* rinfo) override; - void VisitInternalReference(RelocInfo* rinfo) override; - void VisitCodeTarget(RelocInfo* target) override; - void VisitOffHeapTarget(RelocInfo* target) override; + void VisitEmbeddedPointer(InstructionStream host, RelocInfo* target) override; + void VisitExternalReference(InstructionStream host, + RelocInfo* rinfo) override; + void VisitInternalReference(InstructionStream host, + RelocInfo* rinfo) override; + void VisitCodeTarget(InstructionStream host, RelocInfo* target) override; + void VisitOffHeapTarget(InstructionStream host, RelocInfo* target) override; void VisitExternalPointer(HeapObject host, ExternalPointerSlot slot, ExternalPointerTag tag) override;
diff --git a/src/wasm/wasm-serialization.cc b/src/wasm/wasm-serialization.cc index 7661ae8..083e931 100644 --- a/src/wasm/wasm-serialization.cc +++ b/src/wasm/wasm-serialization.cc
@@ -161,7 +161,7 @@ } else if (rinfo->rmode() == RelocInfo::WASM_STUB_CALL) { rinfo->set_wasm_stub_call_address(addr, SKIP_ICACHE_FLUSH); } else { - rinfo->set_target_address(addr, SKIP_WRITE_BARRIER, SKIP_ICACHE_FLUSH); + rinfo->set_target_address(addr, SKIP_ICACHE_FLUSH); } #endif }
diff --git a/test/cctest/heap/test-concurrent-allocation.cc b/test/cctest/heap/test-concurrent-allocation.cc index 54fb289..d3b49ee 100644 --- a/test/cctest/heap/test-concurrent-allocation.cc +++ b/test/cctest/heap/test-concurrent-allocation.cc
@@ -12,6 +12,7 @@ #include "src/codegen/assembler.h" #include "src/codegen/macro-assembler-inl.h" #include "src/codegen/macro-assembler.h" +#include "src/codegen/reloc-info-inl.h" #include "src/common/globals.h" #include "src/handles/handles-inl.h" #include "src/handles/handles.h" @@ -485,10 +486,12 @@ UnparkedScope unparked_scope(&local_heap); // Modification of InstructionStream object requires write access. RwxMemoryWriteScopeForTesting rwx_write_scope; + DisallowGarbageCollection no_gc; + InstructionStream istream = code_.instruction_stream(); int mode_mask = RelocInfo::EmbeddedObjectModeMask(); for (RelocIterator it(code_, mode_mask); !it.done(); it.next()) { DCHECK(RelocInfo::IsEmbeddedObjectMode(it.rinfo()->rmode())); - it.rinfo()->set_target_object(heap_, value_); + it.rinfo()->set_target_object(istream, value_); } }