Source/JavaScriptCore/assembler/LinkBuffer.h - external/github.com/WebKit/webkit - Git at Google

 /*
  * Copyright (C) 2009-2021 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */

 #pragma once

 #if ENABLE(ASSEMBLER)

 #define DUMP_LINK_STATISTICS 0
 #define DUMP_CODE 0

 #define GLOBAL_THUNK_ID reinterpret_cast<void*>(static_cast<intptr_t>(-1))
 #define REGEXP_CODE_ID reinterpret_cast<void*>(static_cast<intptr_t>(-2))
 #define CSS_CODE_ID reinterpret_cast<void*>(static_cast<intptr_t>(-3))

 #include "JITCompilationEffort.h"
 #include "MacroAssembler.h"
 #include "MacroAssemblerCodeRef.h"
 #include <wtf/DataLog.h>
 #include <wtf/FastMalloc.h>
 #include <wtf/Noncopyable.h>

 namespace JSC {

 // LinkBuffer:
 //
 // This class assists in linking code generated by the macro assembler, once code generation
 // has been completed, and the code has been copied to is final location in memory.  At this
 // time pointers to labels within the code may be resolved, and relative offsets to external
 // addresses may be fixed.
 //
 // Specifically:
 //   * Jump objects may be linked to external targets,
 //   * The address of Jump objects may taken, such that it can later be relinked.
 //   * The return address of a Call may be acquired.
 //   * The address of a Label pointing into the code may be resolved.
 //   * The value referenced by a DataLabel may be set.
 //
 class LinkBuffer {
     WTF_MAKE_NONCOPYABLE(LinkBuffer); WTF_MAKE_FAST_ALLOCATED;

     template<PtrTag tag> using CodePtr = MacroAssemblerCodePtr<tag>;
     template<PtrTag tag> using CodeRef = MacroAssemblerCodeRef<tag>;
     typedef MacroAssembler::Label Label;
     typedef MacroAssembler::Jump Jump;
     typedef MacroAssembler::PatchableJump PatchableJump;
     typedef MacroAssembler::JumpList JumpList;
     typedef MacroAssembler::Call Call;
     typedef MacroAssembler::DataLabelCompact DataLabelCompact;
     typedef MacroAssembler::DataLabel32 DataLabel32;
     typedef MacroAssembler::DataLabelPtr DataLabelPtr;
     typedef MacroAssembler::ConvertibleLoadLabel ConvertibleLoadLabel;
 #if ENABLE(BRANCH_COMPACTION)
     typedef MacroAssembler::LinkRecord LinkRecord;
     typedef MacroAssembler::JumpLinkType JumpLinkType;
 #endif

 public:

 #define FOR_EACH_LINKBUFFER_PROFILE(v) \
     v(BaselineJIT) \
     v(DFG) \
     v(FTL) \
     v(DFGOSREntry) \
     v(DFGOSRExit) \
     v(FTLOSRExit) \
     v(InlineCache) \
     v(JumpIsland) \
     v(Thunk) \
     v(LLIntThunk) \
     v(DFGThunk) \
     v(FTLThunk) \
     v(BoundFunctionThunk) \
     v(SpecializedThunk) \
     v(VirtualThunk) \
     v(WasmThunk) \
     v(ExtraCTIThunk) \
     v(Wasm) \
     v(YarrJIT) \
     v(CSSJIT) \
     v(Uncategorized) \
     v(Total) \

 #define DECLARE_LINKBUFFER_PROFILE(name) name,
     enum class Profile {
         FOR_EACH_LINKBUFFER_PROFILE(DECLARE_LINKBUFFER_PROFILE)
     };
 #undef DECLARE_LINKBUFFER_PROFILE

 #define COUNT_LINKBUFFER_PROFILE(name) + 1
     static constexpr unsigned numberOfProfiles = FOR_EACH_LINKBUFFER_PROFILE(COUNT_LINKBUFFER_PROFILE);
 #undef COUNT_LINKBUFFER_PROFILE
     static constexpr unsigned numberOfProfilesExcludingTotal = numberOfProfiles - 1;

     LinkBuffer(MacroAssembler& macroAssembler, void* ownerUID, Profile profile = Profile::Uncategorized, JITCompilationEffort effort = JITCompilationMustSucceed)
         : m_size(0)
         , m_didAllocate(false)
 #ifndef NDEBUG
         , m_completed(false)
 #endif
         , m_profile(profile)
     {
         UNUSED_PARAM(ownerUID);
         linkCode(macroAssembler, effort);
     }

     template<PtrTag tag>
     LinkBuffer(MacroAssembler& macroAssembler, MacroAssemblerCodePtr<tag> code, size_t size, Profile profile = Profile::Uncategorized, JITCompilationEffort effort = JITCompilationMustSucceed, bool shouldPerformBranchCompaction = true)
         : m_size(size)
         , m_didAllocate(false)
 #ifndef NDEBUG
         , m_completed(false)
 #endif
         , m_profile(profile)
         , m_code(code.template retagged<LinkBufferPtrTag>())
     {
 #if ENABLE(BRANCH_COMPACTION)
         m_shouldPerformBranchCompaction = shouldPerformBranchCompaction;
 #else
         UNUSED_PARAM(shouldPerformBranchCompaction);
 #endif
         linkCode(macroAssembler, effort);
     }

     ~LinkBuffer()
     {
     }

     void runMainThreadFinalizationTasks();

     bool didFailToAllocate() const
     {
         return !m_didAllocate;
     }

     bool isValid() const
     {
         return !didFailToAllocate();
     }

     void setIsJumpIsland()
     {
 #if ASSERT_ENABLED
         m_isJumpIsland = true;
 #endif
     }

     // These methods are used to link or set values at code generation time.

     template<PtrTag tag, typename Func, typename = std::enable_if_t<std::is_function<typename std::remove_pointer<Func>::type>::value>>
     void link(Call call, Func funcName)
     {
         FunctionPtr<tag> function(funcName);
         link(call, function);
     }

     template<PtrTag tag>
     void link(Call call, FunctionPtr<tag> function)
     {
         ASSERT(call.isFlagSet(Call::Linkable));
         call.m_label = applyOffset(call.m_label);
         MacroAssembler::linkCall(code(), call, function);
     }

     template<PtrTag tag>
     void link(Call call, CodeLocationLabel<tag> label)
     {
         link(call, FunctionPtr<tag>(label));
     }

     template<PtrTag tag>
     void link(Jump jump, CodeLocationLabel<tag> label)
     {
         jump.m_label = applyOffset(jump.m_label);
         MacroAssembler::linkJump(code(), jump, label);
     }

     template<PtrTag tag>
     void link(const JumpList& list, CodeLocationLabel<tag> label)
     {
         for (const Jump& jump : list.jumps())
             link(jump, label);
     }

     void patch(DataLabelPtr label, void* value)
     {
         AssemblerLabel target = applyOffset(label.m_label);
         MacroAssembler::linkPointer(code(), target, value);
     }

     template<PtrTag tag>
     void patch(DataLabelPtr label, CodeLocationLabel<tag> value)
     {
         AssemblerLabel target = applyOffset(label.m_label);
         MacroAssembler::linkPointer(code(), target, value);
     }

     // These methods are used to obtain handles to allow the code to be relinked / repatched later.

     template<PtrTag tag>
     CodeLocationLabel<tag> entrypoint()
     {
         return CodeLocationLabel<tag>(tagCodePtr<tag>(code()));
     }

     template<PtrTag tag>
     CodeLocationCall<tag> locationOf(Call call)
     {
         ASSERT(call.isFlagSet(Call::Linkable));
         ASSERT(!call.isFlagSet(Call::Near));
         return CodeLocationCall<tag>(getLinkerAddress<tag>(applyOffset(call.m_label)));
     }

     template<PtrTag tag>
     CodeLocationNearCall<tag> locationOfNearCall(Call call)
     {
         ASSERT(call.isFlagSet(Call::Linkable));
         ASSERT(call.isFlagSet(Call::Near));
         return CodeLocationNearCall<tag>(getLinkerAddress<tag>(applyOffset(call.m_label)),
             call.isFlagSet(Call::Tail) ? NearCallMode::Tail : NearCallMode::Regular);
     }

     template<PtrTag tag>
     CodeLocationLabel<tag> locationOf(PatchableJump jump)
     {
         return CodeLocationLabel<tag>(getLinkerAddress<tag>(applyOffset(jump.m_jump.m_label)));
     }

     template<PtrTag tag>
     CodeLocationLabel<tag> locationOf(Label label)
     {
         return CodeLocationLabel<tag>(getLinkerAddress<tag>(applyOffset(label.m_label)));
     }

     template<PtrTag tag>
     CodeLocationDataLabelPtr<tag> locationOf(DataLabelPtr label)
     {
         return CodeLocationDataLabelPtr<tag>(getLinkerAddress<tag>(applyOffset(label.m_label)));
     }

     template<PtrTag tag>
     CodeLocationDataLabel32<tag> locationOf(DataLabel32 label)
     {
         return CodeLocationDataLabel32<tag>(getLinkerAddress<tag>(applyOffset(label.m_label)));
     }

     template<PtrTag tag>
     CodeLocationDataLabelCompact<tag> locationOf(DataLabelCompact label)
     {
         return CodeLocationDataLabelCompact<tag>(getLinkerAddress<tag>(applyOffset(label.m_label)));
     }

     template<PtrTag tag>
     CodeLocationConvertibleLoad<tag> locationOf(ConvertibleLoadLabel label)
     {
         return CodeLocationConvertibleLoad<tag>(getLinkerAddress<tag>(applyOffset(label.m_label)));
     }

     // This method obtains the return address of the call, given as an offset from
     // the start of the code.
     unsigned returnAddressOffset(Call call)
     {
         call.m_label = applyOffset(call.m_label);
         return MacroAssembler::getLinkerCallReturnOffset(call);
     }

     uint32_t offsetOf(Label label)
     {
         return applyOffset(label.m_label).offset();
     }

     unsigned offsetOf(PatchableJump jump)
     {
         return applyOffset(jump.m_jump.m_label).offset();
     }

     // Upon completion of all patching 'FINALIZE_CODE()' should be called once to
     // complete generation of the code. Alternatively, call
     // finalizeCodeWithoutDisassembly() directly if you have your own way of
     // displaying disassembly.

     template<PtrTag tag>
     CodeRef<tag> finalizeCodeWithoutDisassembly()
     {
         return finalizeCodeWithoutDisassemblyImpl().template retagged<tag>();
     }

     template<PtrTag tag, typename... Args>
     CodeRef<tag> finalizeCodeWithDisassembly(bool dumpDisassembly, const char* format, Args... args)
     {
         ALLOW_NONLITERAL_FORMAT_BEGIN
         IGNORE_WARNINGS_BEGIN("format-security")
         return finalizeCodeWithDisassemblyImpl(dumpDisassembly, format, args...).template retagged<tag>();
         IGNORE_WARNINGS_END
         ALLOW_NONLITERAL_FORMAT_END
     }

     template<PtrTag tag>
     CodePtr<tag> trampolineAt(Label label)
     {
         return CodePtr<tag>(MacroAssembler::AssemblerType_T::getRelocatedAddress(code(), applyOffset(label.m_label)));
     }

     void* debugAddress()
     {
         return m_code.dataLocation();
     }

     size_t size() const { return m_size; }

     bool wasAlreadyDisassembled() const { return m_alreadyDisassembled; }
     void didAlreadyDisassemble() { m_alreadyDisassembled = true; }

     JS_EXPORT_PRIVATE static void clearProfileStatistics();
     JS_EXPORT_PRIVATE static void dumpProfileStatistics(std::optional<PrintStream*> = std::nullopt);

     template<typename Functor>
     void addMainThreadFinalizationTask(const Functor& functor)
     {
         m_mainThreadFinalizationTasks.append(createSharedTask<void()>(functor));
     }

 private:
     JS_EXPORT_PRIVATE CodeRef<LinkBufferPtrTag> finalizeCodeWithoutDisassemblyImpl();
     JS_EXPORT_PRIVATE CodeRef<LinkBufferPtrTag> finalizeCodeWithDisassemblyImpl(bool dumpDisassembly, const char* format, ...) WTF_ATTRIBUTE_PRINTF(3, 4);

 #if ENABLE(BRANCH_COMPACTION)
     int executableOffsetFor(int location)
     {
         // Returning 0 in this case works because at location <
         // sizeof(int32_t), no compaction could have happened before this
         // point as the assembler could not have placed a branch instruction
         // within this space that required compaction.
         if (location < static_cast<int>(sizeof(int32_t)))
             return 0;
         return bitwise_cast<int32_t*>(m_assemblerStorage.buffer())[location / sizeof(int32_t) - 1];
     }
 #endif

     template <typename T> T applyOffset(T src)
     {
 #if ENABLE(BRANCH_COMPACTION)
         src = src.labelAtOffset(-executableOffsetFor(src.offset()));
 #endif
         return src;
     }

     // Keep this private! - the underlying code should only be obtained externally via finalizeCode().
     void* code()
     {
         return m_code.dataLocation();
     }

     void allocate(MacroAssembler&, JITCompilationEffort);

     template<PtrTag tag, typename T>
     void* getLinkerAddress(T src)
     {
         void *code = this->code();
         void* address = MacroAssembler::getLinkerAddress<tag>(code, src);
         RELEASE_ASSERT(code <= untagCodePtr<tag>(address) && untagCodePtr<tag>(address) <= static_cast<char*>(code) + size());
         return address;
     }

     JS_EXPORT_PRIVATE void linkCode(MacroAssembler&, JITCompilationEffort);
 #if ENABLE(BRANCH_COMPACTION)
     template <typename InstructionType>
     void copyCompactAndLinkCode(MacroAssembler&, JITCompilationEffort);
 #endif

     void performFinalization();

 #if DUMP_LINK_STATISTICS
     static void dumpLinkStatistics(void* code, size_t initialSize, size_t finalSize);
 #endif

 #if DUMP_CODE
     static void dumpCode(void* code, size_t);
 #endif

     RefPtr<ExecutableMemoryHandle> m_executableMemory;
     size_t m_size;
 #if ENABLE(BRANCH_COMPACTION)
     AssemblerData m_assemblerStorage;
 #if CPU(ARM64E)
     AssemblerData m_assemblerHashesStorage;
 #endif
     bool m_shouldPerformBranchCompaction { true };
 #endif
     bool m_didAllocate;
 #ifndef NDEBUG
     bool m_completed;
 #endif
 #if ASSERT_ENABLED
     bool m_isJumpIsland { false };
 #endif
     bool m_alreadyDisassembled { false };
     Profile m_profile { Profile::Uncategorized };
     MacroAssemblerCodePtr<LinkBufferPtrTag> m_code;
     Vector<RefPtr<SharedTask<void(LinkBuffer&)>>> m_linkTasks;
     Vector<RefPtr<SharedTask<void(LinkBuffer&)>>> m_lateLinkTasks;
     Vector<RefPtr<SharedTask<void()>>> m_mainThreadFinalizationTasks;

     static size_t s_profileCummulativeLinkedSizes[numberOfProfiles];
     static size_t s_profileCummulativeLinkedCounts[numberOfProfiles];
 };

 #if OS(LINUX)
 #define FINALIZE_CODE_IF(condition, linkBufferReference, resultPtrTag, ...) \
     (UNLIKELY((condition) || JSC::Options::logJIT()) \
         ? (linkBufferReference).finalizeCodeWithDisassembly<resultPtrTag>((condition), __VA_ARGS__) \
         : (UNLIKELY(JSC::Options::logJITCodeForPerf()) \
             ? (linkBufferReference).finalizeCodeWithDisassembly<resultPtrTag>(false, __VA_ARGS__) \
             : (linkBufferReference).finalizeCodeWithoutDisassembly<resultPtrTag>()))
 #else
 #define FINALIZE_CODE_IF(condition, linkBufferReference, resultPtrTag, ...) \
     (UNLIKELY((condition) || JSC::Options::logJIT()) \
         ? (linkBufferReference).finalizeCodeWithDisassembly<resultPtrTag>((condition), __VA_ARGS__) \
         : (linkBufferReference).finalizeCodeWithoutDisassembly<resultPtrTag>())
 #endif

 #define FINALIZE_CODE_FOR(codeBlock, linkBufferReference, resultPtrTag, ...)  \
     FINALIZE_CODE_IF((shouldDumpDisassemblyFor(codeBlock) || Options::asyncDisassembly()), linkBufferReference, resultPtrTag, __VA_ARGS__)

 // Use this to finalize code, like so:
 //
 // CodeRef code = FINALIZE_CODE(linkBuffer, tag, "my super thingy number %d", number);
 //
 // Which, in disassembly mode, will print:
 //
 // Generated JIT code for my super thingy number 42:
 //     Code at [0x123456, 0x234567]:
 //         0x123456: mov $0, 0
 //         0x12345a: ret
 //
 // ... and so on.
 //
 // Note that the format string and print arguments are only evaluated when dumpDisassembly
 // is true, so you can hide expensive disassembly-only computations inside there.

 #define FINALIZE_CODE(linkBufferReference, resultPtrTag, ...)  \
     FINALIZE_CODE_IF((JSC::Options::asyncDisassembly() || JSC::Options::dumpDisassembly()), linkBufferReference, resultPtrTag, __VA_ARGS__)

 #define FINALIZE_DFG_CODE(linkBufferReference, resultPtrTag, ...)  \
     FINALIZE_CODE_IF((JSC::Options::asyncDisassembly() || JSC::Options::dumpDisassembly() || Options::dumpDFGDisassembly()), linkBufferReference, resultPtrTag, __VA_ARGS__)

 #define FINALIZE_REGEXP_CODE(linkBufferReference, resultPtrTag, dataLogFArgumentsForHeading)  \
     FINALIZE_CODE_IF(JSC::Options::asyncDisassembly() || JSC::Options::dumpDisassembly() || Options::dumpRegExpDisassembly(), linkBufferReference, resultPtrTag, dataLogFArgumentsForHeading)

 #define FINALIZE_WASM_CODE(linkBufferReference, resultPtrTag, ...)  \
     FINALIZE_CODE_IF((JSC::Options::asyncDisassembly() || JSC::Options::dumpDisassembly() || Options::dumpWasmDisassembly()), linkBufferReference, resultPtrTag, __VA_ARGS__)

 #define FINALIZE_WASM_CODE_FOR_MODE(mode, linkBufferReference, resultPtrTag, ...)  \
     FINALIZE_CODE_IF(shouldDumpDisassemblyFor(mode), linkBufferReference, resultPtrTag, __VA_ARGS__)


 } // namespace JSC

 #endif // ENABLE(ASSEMBLER)
	/*
	* Copyright (C) 2009-2021 Apple Inc. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
	* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
	* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
	* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
	* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
	* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
	* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
	* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	#pragma once

	#if ENABLE(ASSEMBLER)

	#define DUMP_LINK_STATISTICS 0
	#define DUMP_CODE 0

	#define GLOBAL_THUNK_ID reinterpret_cast<void*>(static_cast<intptr_t>(-1))
	#define REGEXP_CODE_ID reinterpret_cast<void*>(static_cast<intptr_t>(-2))
	#define CSS_CODE_ID reinterpret_cast<void*>(static_cast<intptr_t>(-3))

	#include "JITCompilationEffort.h"
	#include "MacroAssembler.h"
	#include "MacroAssemblerCodeRef.h"
	#include <wtf/DataLog.h>
	#include <wtf/FastMalloc.h>
	#include <wtf/Noncopyable.h>

	namespace JSC {

	// LinkBuffer:
	//
	// This class assists in linking code generated by the macro assembler, once code generation
	// has been completed, and the code has been copied to is final location in memory. At this
	// time pointers to labels within the code may be resolved, and relative offsets to external
	// addresses may be fixed.
	//
	// Specifically:
	// * Jump objects may be linked to external targets,
	// * The address of Jump objects may taken, such that it can later be relinked.
	// * The return address of a Call may be acquired.
	// * The address of a Label pointing into the code may be resolved.
	// * The value referenced by a DataLabel may be set.
	//
	class LinkBuffer {
	WTF_MAKE_NONCOPYABLE(LinkBuffer); WTF_MAKE_FAST_ALLOCATED;

	template<PtrTag tag> using CodePtr = MacroAssemblerCodePtr<tag>;
	template<PtrTag tag> using CodeRef = MacroAssemblerCodeRef<tag>;
	typedef MacroAssembler::Label Label;
	typedef MacroAssembler::Jump Jump;
	typedef MacroAssembler::PatchableJump PatchableJump;
	typedef MacroAssembler::JumpList JumpList;
	typedef MacroAssembler::Call Call;
	typedef MacroAssembler::DataLabelCompact DataLabelCompact;
	typedef MacroAssembler::DataLabel32 DataLabel32;
	typedef MacroAssembler::DataLabelPtr DataLabelPtr;
	typedef MacroAssembler::ConvertibleLoadLabel ConvertibleLoadLabel;
	#if ENABLE(BRANCH_COMPACTION)
	typedef MacroAssembler::LinkRecord LinkRecord;
	typedef MacroAssembler::JumpLinkType JumpLinkType;
	#endif

	public:

	#define FOR_EACH_LINKBUFFER_PROFILE(v) \
	v(BaselineJIT) \
	v(DFG) \
	v(FTL) \
	v(DFGOSREntry) \
	v(DFGOSRExit) \
	v(FTLOSRExit) \
	v(InlineCache) \
	v(JumpIsland) \
	v(Thunk) \
	v(LLIntThunk) \
	v(DFGThunk) \
	v(FTLThunk) \
	v(BoundFunctionThunk) \
	v(SpecializedThunk) \
	v(VirtualThunk) \
	v(WasmThunk) \
	v(ExtraCTIThunk) \
	v(Wasm) \
	v(YarrJIT) \
	v(CSSJIT) \
	v(Uncategorized) \
	v(Total) \

	#define DECLARE_LINKBUFFER_PROFILE(name) name,
	enum class Profile {
	FOR_EACH_LINKBUFFER_PROFILE(DECLARE_LINKBUFFER_PROFILE)
	};
	#undef DECLARE_LINKBUFFER_PROFILE

	#define COUNT_LINKBUFFER_PROFILE(name) + 1
	static constexpr unsigned numberOfProfiles = FOR_EACH_LINKBUFFER_PROFILE(COUNT_LINKBUFFER_PROFILE);
	#undef COUNT_LINKBUFFER_PROFILE
	static constexpr unsigned numberOfProfilesExcludingTotal = numberOfProfiles - 1;

	LinkBuffer(MacroAssembler& macroAssembler, void* ownerUID, Profile profile = Profile::Uncategorized, JITCompilationEffort effort = JITCompilationMustSucceed)
	: m_size(0)
	, m_didAllocate(false)
	#ifndef NDEBUG
	, m_completed(false)
	#endif
	, m_profile(profile)
	{
	UNUSED_PARAM(ownerUID);
	linkCode(macroAssembler, effort);
	}

	template<PtrTag tag>
	LinkBuffer(MacroAssembler& macroAssembler, MacroAssemblerCodePtr<tag> code, size_t size, Profile profile = Profile::Uncategorized, JITCompilationEffort effort = JITCompilationMustSucceed, bool shouldPerformBranchCompaction = true)
	: m_size(size)
	, m_didAllocate(false)
	#ifndef NDEBUG
	, m_completed(false)
	#endif
	, m_profile(profile)
	, m_code(code.template retagged<LinkBufferPtrTag>())
	{
	#if ENABLE(BRANCH_COMPACTION)
	m_shouldPerformBranchCompaction = shouldPerformBranchCompaction;
	#else
	UNUSED_PARAM(shouldPerformBranchCompaction);
	#endif
	linkCode(macroAssembler, effort);
	}

	~LinkBuffer()
	{
	}

	void runMainThreadFinalizationTasks();

	bool didFailToAllocate() const
	{
	return !m_didAllocate;
	}

	bool isValid() const
	{
	return !didFailToAllocate();
	}

	void setIsJumpIsland()
	{
	#if ASSERT_ENABLED
	m_isJumpIsland = true;
	#endif
	}

	// These methods are used to link or set values at code generation time.

	template<PtrTag tag, typename Func, typename = std::enable_if_t<std::is_function<typename std::remove_pointer<Func>::type>::value>>
	void link(Call call, Func funcName)
	{
	FunctionPtr<tag> function(funcName);
	link(call, function);
	}

	template<PtrTag tag>
	void link(Call call, FunctionPtr<tag> function)
	{
	ASSERT(call.isFlagSet(Call::Linkable));
	call.m_label = applyOffset(call.m_label);
	MacroAssembler::linkCall(code(), call, function);
	}

	template<PtrTag tag>
	void link(Call call, CodeLocationLabel<tag> label)
	{
	link(call, FunctionPtr<tag>(label));
	}

	template<PtrTag tag>
	void link(Jump jump, CodeLocationLabel<tag> label)
	{
	jump.m_label = applyOffset(jump.m_label);
	MacroAssembler::linkJump(code(), jump, label);
	}

	template<PtrTag tag>
	void link(const JumpList& list, CodeLocationLabel<tag> label)
	{
	for (const Jump& jump : list.jumps())
	link(jump, label);
	}

	void patch(DataLabelPtr label, void* value)
	{
	AssemblerLabel target = applyOffset(label.m_label);
	MacroAssembler::linkPointer(code(), target, value);
	}

	template<PtrTag tag>
	void patch(DataLabelPtr label, CodeLocationLabel<tag> value)
	{
	AssemblerLabel target = applyOffset(label.m_label);
	MacroAssembler::linkPointer(code(), target, value);
	}

	// These methods are used to obtain handles to allow the code to be relinked / repatched later.

	template<PtrTag tag>
	CodeLocationLabel<tag> entrypoint()
	{
	return CodeLocationLabel<tag>(tagCodePtr<tag>(code()));
	}

	template<PtrTag tag>
	CodeLocationCall<tag> locationOf(Call call)
	{
	ASSERT(call.isFlagSet(Call::Linkable));
	ASSERT(!call.isFlagSet(Call::Near));
	return CodeLocationCall<tag>(getLinkerAddress<tag>(applyOffset(call.m_label)));
	}

	template<PtrTag tag>
	CodeLocationNearCall<tag> locationOfNearCall(Call call)
	{
	ASSERT(call.isFlagSet(Call::Linkable));
	ASSERT(call.isFlagSet(Call::Near));
	return CodeLocationNearCall<tag>(getLinkerAddress<tag>(applyOffset(call.m_label)),
	call.isFlagSet(Call::Tail) ? NearCallMode::Tail : NearCallMode::Regular);
	}

	template<PtrTag tag>
	CodeLocationLabel<tag> locationOf(PatchableJump jump)
	{
	return CodeLocationLabel<tag>(getLinkerAddress<tag>(applyOffset(jump.m_jump.m_label)));
	}

	template<PtrTag tag>
	CodeLocationLabel<tag> locationOf(Label label)
	{
	return CodeLocationLabel<tag>(getLinkerAddress<tag>(applyOffset(label.m_label)));
	}

	template<PtrTag tag>
	CodeLocationDataLabelPtr<tag> locationOf(DataLabelPtr label)
	{
	return CodeLocationDataLabelPtr<tag>(getLinkerAddress<tag>(applyOffset(label.m_label)));
	}

	template<PtrTag tag>
	CodeLocationDataLabel32<tag> locationOf(DataLabel32 label)
	{
	return CodeLocationDataLabel32<tag>(getLinkerAddress<tag>(applyOffset(label.m_label)));
	}

	template<PtrTag tag>
	CodeLocationDataLabelCompact<tag> locationOf(DataLabelCompact label)
	{
	return CodeLocationDataLabelCompact<tag>(getLinkerAddress<tag>(applyOffset(label.m_label)));
	}

	template<PtrTag tag>
	CodeLocationConvertibleLoad<tag> locationOf(ConvertibleLoadLabel label)
	{
	return CodeLocationConvertibleLoad<tag>(getLinkerAddress<tag>(applyOffset(label.m_label)));
	}

	// This method obtains the return address of the call, given as an offset from
	// the start of the code.
	unsigned returnAddressOffset(Call call)
	{
	call.m_label = applyOffset(call.m_label);
	return MacroAssembler::getLinkerCallReturnOffset(call);
	}

	uint32_t offsetOf(Label label)
	{
	return applyOffset(label.m_label).offset();
	}

	unsigned offsetOf(PatchableJump jump)
	{
	return applyOffset(jump.m_jump.m_label).offset();
	}

	// Upon completion of all patching 'FINALIZE_CODE()' should be called once to
	// complete generation of the code. Alternatively, call
	// finalizeCodeWithoutDisassembly() directly if you have your own way of
	// displaying disassembly.

	template<PtrTag tag>
	CodeRef<tag> finalizeCodeWithoutDisassembly()
	{
	return finalizeCodeWithoutDisassemblyImpl().template retagged<tag>();
	}

	template<PtrTag tag, typename... Args>
	CodeRef<tag> finalizeCodeWithDisassembly(bool dumpDisassembly, const char* format, Args... args)
	{
	ALLOW_NONLITERAL_FORMAT_BEGIN
	IGNORE_WARNINGS_BEGIN("format-security")
	return finalizeCodeWithDisassemblyImpl(dumpDisassembly, format, args...).template retagged<tag>();
	IGNORE_WARNINGS_END
	ALLOW_NONLITERAL_FORMAT_END
	}

	template<PtrTag tag>
	CodePtr<tag> trampolineAt(Label label)
	{
	return CodePtr<tag>(MacroAssembler::AssemblerType_T::getRelocatedAddress(code(), applyOffset(label.m_label)));
	}

	void* debugAddress()
	{
	return m_code.dataLocation();
	}

	size_t size() const { return m_size; }

	bool wasAlreadyDisassembled() const { return m_alreadyDisassembled; }
	void didAlreadyDisassemble() { m_alreadyDisassembled = true; }

	JS_EXPORT_PRIVATE static void clearProfileStatistics();
	JS_EXPORT_PRIVATE static void dumpProfileStatistics(std::optional<PrintStream*> = std::nullopt);

	template<typename Functor>
	void addMainThreadFinalizationTask(const Functor& functor)
	{
	m_mainThreadFinalizationTasks.append(createSharedTask<void()>(functor));
	}

	private:
	JS_EXPORT_PRIVATE CodeRef<LinkBufferPtrTag> finalizeCodeWithoutDisassemblyImpl();
	JS_EXPORT_PRIVATE CodeRef<LinkBufferPtrTag> finalizeCodeWithDisassemblyImpl(bool dumpDisassembly, const char* format, ...) WTF_ATTRIBUTE_PRINTF(3, 4);

	#if ENABLE(BRANCH_COMPACTION)
	int executableOffsetFor(int location)
	{
	// Returning 0 in this case works because at location <
	// sizeof(int32_t), no compaction could have happened before this
	// point as the assembler could not have placed a branch instruction
	// within this space that required compaction.
	if (location < static_cast<int>(sizeof(int32_t)))
	return 0;
	return bitwise_cast<int32_t*>(m_assemblerStorage.buffer())[location / sizeof(int32_t) - 1];
	}
	#endif

	template <typename T> T applyOffset(T src)
	{
	#if ENABLE(BRANCH_COMPACTION)
	src = src.labelAtOffset(-executableOffsetFor(src.offset()));
	#endif
	return src;
	}

	// Keep this private! - the underlying code should only be obtained externally via finalizeCode().
	void* code()
	{
	return m_code.dataLocation();
	}

	void allocate(MacroAssembler&, JITCompilationEffort);

	template<PtrTag tag, typename T>
	void* getLinkerAddress(T src)
	{
	void *code = this->code();
	void* address = MacroAssembler::getLinkerAddress<tag>(code, src);
	RELEASE_ASSERT(code <= untagCodePtr<tag>(address) && untagCodePtr<tag>(address) <= static_cast<char*>(code) + size());
	return address;
	}

	JS_EXPORT_PRIVATE void linkCode(MacroAssembler&, JITCompilationEffort);
	#if ENABLE(BRANCH_COMPACTION)
	template <typename InstructionType>
	void copyCompactAndLinkCode(MacroAssembler&, JITCompilationEffort);
	#endif

	void performFinalization();

	#if DUMP_LINK_STATISTICS
	static void dumpLinkStatistics(void* code, size_t initialSize, size_t finalSize);
	#endif

	#if DUMP_CODE
	static void dumpCode(void* code, size_t);
	#endif

	RefPtr<ExecutableMemoryHandle> m_executableMemory;
	size_t m_size;
	#if ENABLE(BRANCH_COMPACTION)
	AssemblerData m_assemblerStorage;
	#if CPU(ARM64E)
	AssemblerData m_assemblerHashesStorage;
	#endif
	bool m_shouldPerformBranchCompaction { true };
	#endif
	bool m_didAllocate;
	#ifndef NDEBUG
	bool m_completed;
	#endif
	#if ASSERT_ENABLED
	bool m_isJumpIsland { false };
	#endif
	bool m_alreadyDisassembled { false };
	Profile m_profile { Profile::Uncategorized };
	MacroAssemblerCodePtr<LinkBufferPtrTag> m_code;
	Vector<RefPtr<SharedTask<void(LinkBuffer&)>>> m_linkTasks;
	Vector<RefPtr<SharedTask<void(LinkBuffer&)>>> m_lateLinkTasks;
	Vector<RefPtr<SharedTask<void()>>> m_mainThreadFinalizationTasks;

	static size_t s_profileCummulativeLinkedSizes[numberOfProfiles];
	static size_t s_profileCummulativeLinkedCounts[numberOfProfiles];
	};

	#if OS(LINUX)
	#define FINALIZE_CODE_IF(condition, linkBufferReference, resultPtrTag, ...) \
	(UNLIKELY((condition) \|\| JSC::Options::logJIT()) \
	? (linkBufferReference).finalizeCodeWithDisassembly<resultPtrTag>((condition), __VA_ARGS__) \
	: (UNLIKELY(JSC::Options::logJITCodeForPerf()) \
	? (linkBufferReference).finalizeCodeWithDisassembly<resultPtrTag>(false, __VA_ARGS__) \
	: (linkBufferReference).finalizeCodeWithoutDisassembly<resultPtrTag>()))
	#else
	#define FINALIZE_CODE_IF(condition, linkBufferReference, resultPtrTag, ...) \
	(UNLIKELY((condition) \|\| JSC::Options::logJIT()) \
	? (linkBufferReference).finalizeCodeWithDisassembly<resultPtrTag>((condition), __VA_ARGS__) \
	: (linkBufferReference).finalizeCodeWithoutDisassembly<resultPtrTag>())
	#endif

	#define FINALIZE_CODE_FOR(codeBlock, linkBufferReference, resultPtrTag, ...) \
	FINALIZE_CODE_IF((shouldDumpDisassemblyFor(codeBlock) \|\| Options::asyncDisassembly()), linkBufferReference, resultPtrTag, __VA_ARGS__)

	// Use this to finalize code, like so:
	//
	// CodeRef code = FINALIZE_CODE(linkBuffer, tag, "my super thingy number %d", number);
	//
	// Which, in disassembly mode, will print:
	//
	// Generated JIT code for my super thingy number 42:
	// Code at [0x123456, 0x234567]:
	// 0x123456: mov $0, 0
	// 0x12345a: ret
	//
	// ... and so on.
	//
	// Note that the format string and print arguments are only evaluated when dumpDisassembly
	// is true, so you can hide expensive disassembly-only computations inside there.

	#define FINALIZE_CODE(linkBufferReference, resultPtrTag, ...) \
	FINALIZE_CODE_IF((JSC::Options::asyncDisassembly() \|\| JSC::Options::dumpDisassembly()), linkBufferReference, resultPtrTag, __VA_ARGS__)

	#define FINALIZE_DFG_CODE(linkBufferReference, resultPtrTag, ...) \
	FINALIZE_CODE_IF((JSC::Options::asyncDisassembly() \|\| JSC::Options::dumpDisassembly() \|\| Options::dumpDFGDisassembly()), linkBufferReference, resultPtrTag, __VA_ARGS__)

	#define FINALIZE_REGEXP_CODE(linkBufferReference, resultPtrTag, dataLogFArgumentsForHeading) \
	FINALIZE_CODE_IF(JSC::Options::asyncDisassembly() \|\| JSC::Options::dumpDisassembly() \|\| Options::dumpRegExpDisassembly(), linkBufferReference, resultPtrTag, dataLogFArgumentsForHeading)

	#define FINALIZE_WASM_CODE(linkBufferReference, resultPtrTag, ...) \
	FINALIZE_CODE_IF((JSC::Options::asyncDisassembly() \|\| JSC::Options::dumpDisassembly() \|\| Options::dumpWasmDisassembly()), linkBufferReference, resultPtrTag, __VA_ARGS__)

	#define FINALIZE_WASM_CODE_FOR_MODE(mode, linkBufferReference, resultPtrTag, ...) \
	FINALIZE_CODE_IF(shouldDumpDisassemblyFor(mode), linkBufferReference, resultPtrTag, __VA_ARGS__)



	} // namespace JSC

	#endif // ENABLE(ASSEMBLER)