| /* |
| * Copyright (C) 2015-2020 Apple Inc. All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions |
| * are met: |
| * 1. Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * 2. Redistributions in binary form must reproduce the above copyright |
| * notice, this list of conditions and the following disclaimer in the |
| * documentation and/or other materials provided with the distribution. |
| * |
| * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY |
| * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR |
| * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY |
| * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| #include "config.h" |
| #include "CallFrameShuffler.h" |
| |
| #if ENABLE(JIT) |
| |
| #include "CachedRecovery.h" |
| #include "CCallHelpers.h" |
| #include "CodeBlock.h" |
| |
| namespace JSC { |
| |
| CallFrameShuffler::CallFrameShuffler(CCallHelpers& jit, const CallFrameShuffleData& data) |
| : m_jit(jit) |
| , m_oldFrame(data.numLocals + CallerFrameAndPC::sizeInRegisters, nullptr) |
| , m_newFrame(data.args.size() + CallFrame::headerSizeInRegisters, nullptr) |
| , m_alignedOldFrameSize(CallFrame::headerSizeInRegisters + roundArgumentCountToAlignFrame(data.numParameters)) |
| , m_alignedNewFrameSize(CallFrame::headerSizeInRegisters |
| + roundArgumentCountToAlignFrame(data.args.size())) |
| , m_frameDelta(m_alignedNewFrameSize - m_alignedOldFrameSize) |
| , m_lockedRegisters(RegisterSet::allRegisters()) |
| , m_numPassedArgs(data.numPassedArgs) |
| , m_numParameters(data.numParameters) |
| { |
| // We are allowed all the usual registers... |
| for (unsigned i = GPRInfo::numberOfRegisters; i--; ) |
| m_lockedRegisters.clear(GPRInfo::toRegister(i)); |
| for (unsigned i = FPRInfo::numberOfRegisters; i--; ) |
| m_lockedRegisters.clear(FPRInfo::toRegister(i)); |
| |
| // ... as well as the callee saved registers |
| m_lockedRegisters.exclude(RegisterSet::vmCalleeSaveRegisters()); |
| |
| ASSERT(!data.callee.isInJSStack() || data.callee.virtualRegister().isLocal()); |
| addNew(CallFrameSlot::callee, data.callee); |
| |
| for (size_t i = 0; i < data.args.size(); ++i) { |
| ASSERT(!data.args[i].isInJSStack() || data.args[i].virtualRegister().isLocal()); |
| addNew(virtualRegisterForArgumentIncludingThis(i), data.args[i]); |
| } |
| |
| for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) { |
| if (!data.registers[reg].isSet()) |
| continue; |
| |
| if (reg.isGPR()) { |
| #if USE(JSVALUE64) |
| addNew(JSValueRegs(reg.gpr()), data.registers[reg]); |
| #elif USE(JSVALUE32_64) |
| addNew(reg.gpr(), data.registers[reg]); |
| #endif |
| } else |
| addNew(reg.fpr(), data.registers[reg]); |
| } |
| |
| #if USE(JSVALUE64) |
| m_numberTagRegister = data.numberTagRegister; |
| if (m_numberTagRegister != InvalidGPRReg) |
| lockGPR(m_numberTagRegister); |
| #endif |
| } |
| |
| void CallFrameShuffler::dump(PrintStream& out) const |
| { |
| static const char* delimiter = " +-------------------------------+ "; |
| static const char* dangerDelimiter = " X-------------------------------X "; |
| static const char* dangerBoundsDelimiter = " XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX "; |
| static const char* emptySpace = " "; |
| out.print(" "); |
| out.print(" Old frame "); |
| out.print(" New frame "); |
| out.print("\n"); |
| int totalSize = m_alignedOldFrameSize + std::max(numLocals(), m_alignedNewFrameSize) + 3; |
| for (int i = 0; i < totalSize; ++i) { |
| VirtualRegister old { m_alignedOldFrameSize - i - 1 }; |
| VirtualRegister newReg { old + m_frameDelta }; |
| |
| if (!isValidOld(old) && old != firstOld() - 1 |
| && !isValidNew(newReg) && newReg != firstNew() - 1) |
| continue; |
| |
| out.print(" "); |
| if (dangerFrontier() >= firstNew() |
| && (newReg == dangerFrontier() || newReg == firstNew() - 1)) |
| out.print(dangerBoundsDelimiter); |
| else if (isValidOld(old)) |
| out.print(isValidNew(newReg) && isDangerNew(newReg) ? dangerDelimiter : delimiter); |
| else if (old == firstOld() - 1) |
| out.print(delimiter); |
| else |
| out.print(emptySpace); |
| if (dangerFrontier() >= firstNew() |
| && (newReg == dangerFrontier() || newReg == firstNew() - 1)) |
| out.print(dangerBoundsDelimiter); |
| else if (isValidNew(newReg) || newReg == firstNew() - 1) |
| out.print(isDangerNew(newReg) ? dangerDelimiter : delimiter); |
| else |
| out.print(emptySpace); |
| out.print("\n"); |
| if (old == firstOld()) |
| out.print(" sp --> "); |
| else if (!old.offset()) |
| out.print(" fp --> "); |
| else |
| out.print(" "); |
| if (isValidOld(old)) { |
| if (getOld(old)) { |
| auto str = toCString(old); |
| if (isValidNew(newReg) && isDangerNew(newReg)) |
| out.printf(" X %18s X ", str.data()); |
| else |
| out.printf(" | %18s | ", str.data()); |
| } else if (isValidNew(newReg) && isDangerNew(newReg)) |
| out.printf(" X%30s X ", ""); |
| else |
| out.printf(" |%30s | ", ""); |
| } else |
| out.print(emptySpace); |
| if (isValidNew(newReg)) { |
| const char d = isDangerNew(newReg) ? 'X' : '|'; |
| auto str = toCString(newReg); |
| if (getNew(newReg)) { |
| if (getNew(newReg)->recovery().isConstant()) |
| out.printf(" %c%8s <- constant %c ", d, str.data(), d); |
| else { |
| auto recoveryStr = toCString(getNew(newReg)->recovery()); |
| out.printf(" %c%8s <- %18s %c ", d, str.data(), |
| recoveryStr.data(), d); |
| } |
| } else if (newReg == VirtualRegister { CallFrameSlot::argumentCountIncludingThis }) |
| out.printf(" %c%8s <- %18zu %c ", d, str.data(), argCount(), d); |
| else |
| out.printf(" %c%30s %c ", d, "", d); |
| } else |
| out.print(emptySpace); |
| if (newReg == firstNew() - m_newFrameOffset && !isSlowPath()) |
| out.print(" <-- new sp before jump (current ", m_newFrameBase, ") "); |
| if (newReg == firstNew()) |
| out.print(" <-- new fp after prologue"); |
| out.print("\n"); |
| } |
| out.print(" "); |
| out.print(" Live registers "); |
| out.print(" Wanted registers "); |
| out.print("\n"); |
| for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) { |
| CachedRecovery* oldCachedRecovery { m_registers[reg] }; |
| CachedRecovery* newCachedRecovery { m_newRegisters[reg] }; |
| if (!oldCachedRecovery && !newCachedRecovery) |
| continue; |
| out.print(" "); |
| if (oldCachedRecovery) { |
| auto str = toCString(reg); |
| out.printf(" %8s ", str.data()); |
| } else |
| out.print(emptySpace); |
| #if USE(JSVALUE32_64) |
| if (newCachedRecovery) { |
| JSValueRegs wantedJSValueRegs { newCachedRecovery->wantedJSValueRegs() }; |
| if (reg.isFPR()) |
| out.print(reg, " <- ", newCachedRecovery->recovery()); |
| else { |
| if (reg.gpr() == wantedJSValueRegs.tagGPR()) |
| out.print(reg.gpr(), " <- tag(", newCachedRecovery->recovery(), ")"); |
| else |
| out.print(reg.gpr(), " <- payload(", newCachedRecovery->recovery(), ")"); |
| } |
| } |
| #else |
| if (newCachedRecovery) |
| out.print(" ", reg, " <- ", newCachedRecovery->recovery()); |
| #endif |
| out.print("\n"); |
| } |
| out.print(" Locked registers: "); |
| bool firstLocked { true }; |
| for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) { |
| if (m_lockedRegisters.get(reg)) { |
| out.print(firstLocked ? "" : ", ", reg); |
| firstLocked = false; |
| } |
| } |
| out.print("\n"); |
| |
| if (isSlowPath()) |
| out.print(" Using fp-relative addressing for slow path call\n"); |
| else |
| out.print(" Using sp-relative addressing for jump (using ", m_newFrameBase, " as new sp)\n"); |
| if (m_oldFrameOffset) |
| out.print(" Old frame offset is ", m_oldFrameOffset, "\n"); |
| if (m_newFrameOffset) |
| out.print(" New frame offset is ", m_newFrameOffset, "\n"); |
| #if USE(JSVALUE64) |
| if (m_numberTagRegister != InvalidGPRReg) |
| out.print(" NumberTag is currently in ", m_numberTagRegister, "\n"); |
| #endif |
| } |
| |
| CachedRecovery* CallFrameShuffler::getCachedRecovery(ValueRecovery recovery) |
| { |
| ASSERT(!recovery.isConstant()); |
| if (recovery.isInGPR()) |
| return m_registers[recovery.gpr()]; |
| if (recovery.isInFPR()) |
| return m_registers[recovery.fpr()]; |
| #if USE(JSVALUE32_64) |
| if (recovery.technique() == InPair) { |
| ASSERT(m_registers[recovery.tagGPR()] == m_registers[recovery.payloadGPR()]); |
| return m_registers[recovery.payloadGPR()]; |
| } |
| #endif |
| ASSERT(recovery.isInJSStack()); |
| return getOld(recovery.virtualRegister()); |
| } |
| |
| CachedRecovery* CallFrameShuffler::setCachedRecovery(ValueRecovery recovery, CachedRecovery* cachedRecovery) |
| { |
| ASSERT(!recovery.isConstant()); |
| if (recovery.isInGPR()) |
| return m_registers[recovery.gpr()] = cachedRecovery; |
| if (recovery.isInFPR()) |
| return m_registers[recovery.fpr()] = cachedRecovery; |
| #if USE(JSVALUE32_64) |
| if (recovery.technique() == InPair) { |
| m_registers[recovery.tagGPR()] = cachedRecovery; |
| return m_registers[recovery.payloadGPR()] = cachedRecovery; |
| } |
| #endif |
| ASSERT(recovery.isInJSStack()); |
| setOld(recovery.virtualRegister(), cachedRecovery); |
| return cachedRecovery; |
| } |
| |
| void CallFrameShuffler::spill(CachedRecovery& cachedRecovery) |
| { |
| ASSERT(!isSlowPath()); |
| ASSERT(cachedRecovery.recovery().isInRegisters()); |
| |
| VirtualRegister spillSlot { 0 }; |
| for (VirtualRegister slot = firstOld(); slot <= lastOld(); slot += 1) { |
| if (slot >= newAsOld(firstNew())) |
| break; |
| |
| if (getOld(slot)) |
| continue; |
| |
| spillSlot = slot; |
| break; |
| } |
| // We must have enough slots to be able to fit the whole callee's |
| // frame for the slow path - unless we are in the FTL. In that |
| // case, we are allowed to extend the frame *once*, since we are |
| // guaranteed to have enough available space for that. |
| if (spillSlot >= newAsOld(firstNew()) || !spillSlot.isLocal()) { |
| RELEASE_ASSERT(!m_didExtendFrame); |
| extendFrameIfNeeded(); |
| spill(cachedRecovery); |
| return; |
| } |
| |
| if (verbose) |
| dataLog(" * Spilling ", cachedRecovery.recovery(), " into ", spillSlot, "\n"); |
| auto format = emitStore(cachedRecovery, addressForOld(spillSlot)); |
| ASSERT(format != DataFormatNone); |
| updateRecovery(cachedRecovery, ValueRecovery::displacedInJSStack(spillSlot, format)); |
| } |
| |
| void CallFrameShuffler::emitDeltaCheck() |
| { |
| if (!ASSERT_ENABLED) |
| return; |
| |
| GPRReg scratchGPR { getFreeGPR() }; |
| if (scratchGPR != InvalidGPRReg) { |
| if (verbose) |
| dataLog(" Using ", scratchGPR, " for the fp-sp delta check\n"); |
| m_jit.move(MacroAssembler::stackPointerRegister, scratchGPR); |
| m_jit.subPtr(GPRInfo::callFrameRegister, scratchGPR); |
| MacroAssembler::Jump ok = m_jit.branch32( |
| MacroAssembler::Equal, scratchGPR, |
| MacroAssembler::TrustedImm32(-numLocals() * sizeof(Register))); |
| m_jit.abortWithReason(JITUnexpectedCallFrameSize); |
| ok.link(&m_jit); |
| } else if (verbose) |
| dataLog(" Skipping the fp-sp delta check since there is too much pressure"); |
| } |
| |
| void CallFrameShuffler::extendFrameIfNeeded() |
| { |
| ASSERT(!m_didExtendFrame); |
| |
| VirtualRegister firstRead { firstOld() }; |
| for (; firstRead <= virtualRegisterForLocal(0); firstRead += 1) { |
| if (getOld(firstRead)) |
| break; |
| } |
| size_t availableSize = static_cast<size_t>(firstRead.offset() - firstOld().offset()); |
| size_t wantedSize = m_newFrame.size() + m_newFrameOffset; |
| |
| if (availableSize < wantedSize) { |
| size_t delta = WTF::roundUpToMultipleOf(stackAlignmentRegisters(), wantedSize - availableSize); |
| m_oldFrame.grow(m_oldFrame.size() + delta); |
| for (size_t i = 0; i < delta; ++i) |
| m_oldFrame[m_oldFrame.size() - i - 1] = nullptr; |
| m_jit.subPtr(MacroAssembler::TrustedImm32(delta * sizeof(Register)), MacroAssembler::stackPointerRegister); |
| |
| if (isSlowPath()) |
| m_frameDelta = numLocals() + CallerFrameAndPC::sizeInRegisters; |
| else |
| m_oldFrameOffset = numLocals(); |
| |
| if (verbose) |
| dataLogF(" Not enough space - extending the old frame %zu slot\n", delta); |
| } |
| |
| m_didExtendFrame = true; |
| } |
| |
| void CallFrameShuffler::prepareForSlowPath() |
| { |
| ASSERT(isUndecided()); |
| emitDeltaCheck(); |
| |
| m_frameDelta = numLocals() + CallerFrameAndPC::sizeInRegisters; |
| m_newFrameBase = MacroAssembler::stackPointerRegister; |
| m_newFrameOffset = -CallerFrameAndPC::sizeInRegisters; |
| |
| if (verbose) |
| dataLog("\n\nPreparing frame for slow path call:\n"); |
| |
| // When coming from the FTL, we need to extend the frame. In other |
| // cases, we may end up extending the frame if we previously |
| // spilled things (e.g. in polymorphic cache). |
| extendFrameIfNeeded(); |
| |
| if (verbose) |
| dataLog(*this); |
| |
| prepareAny(); |
| |
| if (verbose) |
| dataLog("Ready for slow path call!\n"); |
| } |
| |
| void CallFrameShuffler::prepareForTailCall() |
| { |
| ASSERT(isUndecided()); |
| emitDeltaCheck(); |
| |
| // We'll use sp-based indexing so that we can load the |
| // caller's frame pointer into the fpr immediately |
| m_oldFrameBase = MacroAssembler::stackPointerRegister; |
| m_oldFrameOffset = numLocals(); |
| m_newFrameBase = acquireGPR(); |
| #if CPU(ARM_THUMB2) || CPU(MIPS) |
| // We load the frame pointer and link register |
| // manually. We could ask the algorithm to load them for us, |
| // and it would allow us to use the link register as an extra |
| // temporary - but it'd mean that the frame pointer can also |
| // be used as an extra temporary, so we keep the link register |
| // locked instead. |
| |
| // sp will point to head1 since the callee's prologue pushes |
| // the call frame and link register. |
| m_newFrameOffset = -1; |
| #elif CPU(ARM64) |
| // We load the frame pointer and link register manually. We |
| // could ask the algorithm to load the link register for us |
| // (which would allow for its use as an extra temporary), but |
| // since its not in GPRInfo, we can't do it. |
| |
| // sp will point to head2 since the callee's prologue pushes the |
| // call frame and link register |
| m_newFrameOffset = -2; |
| #elif CPU(X86_64) |
| // We load the frame pointer manually, but we ask the |
| // algorithm to move the return PC for us (it'd probably |
| // require a write in the danger zone) |
| addNew(VirtualRegister { 1 }, |
| ValueRecovery::displacedInJSStack(VirtualRegister(1), DataFormatJS)); |
| |
| // sp will point to head1 since the callee's prologue pushes |
| // the call frame register |
| m_newFrameOffset = -1; |
| #else |
| UNREACHABLE_FOR_PLATFORM(); |
| #endif |
| |
| if (verbose) |
| dataLog(" Emitting code for computing the new frame base\n"); |
| |
| // We compute the new frame base by first computing the top of the |
| // old frame (taking into account an argument count higher than |
| // the number of parameters), then substracting to it the aligned |
| // new frame size (adjusted). |
| m_jit.load32(MacroAssembler::Address(GPRInfo::callFrameRegister, CallFrameSlot::argumentCountIncludingThis * static_cast<int>(sizeof(Register)) + PayloadOffset), m_newFrameBase); |
| MacroAssembler::Jump argumentCountOK = |
| m_jit.branch32(MacroAssembler::BelowOrEqual, m_newFrameBase, |
| MacroAssembler::TrustedImm32(m_numParameters)); |
| m_jit.add32(MacroAssembler::TrustedImm32(stackAlignmentRegisters() - 1 + CallFrame::headerSizeInRegisters), m_newFrameBase); |
| m_jit.and32(MacroAssembler::TrustedImm32(-stackAlignmentRegisters()), m_newFrameBase); |
| m_jit.mul32(MacroAssembler::TrustedImm32(sizeof(Register)), m_newFrameBase, m_newFrameBase); |
| MacroAssembler::Jump done = m_jit.jump(); |
| argumentCountOK.link(&m_jit); |
| m_jit.move( |
| MacroAssembler::TrustedImm32(m_alignedOldFrameSize * sizeof(Register)), |
| m_newFrameBase); |
| done.link(&m_jit); |
| |
| m_jit.addPtr(GPRInfo::callFrameRegister, m_newFrameBase); |
| m_jit.subPtr( |
| MacroAssembler::TrustedImm32( |
| (m_alignedNewFrameSize + m_newFrameOffset) * sizeof(Register)), |
| m_newFrameBase); |
| |
| // We load the link register manually for architectures that have one |
| #if CPU(ARM_THUMB2) || CPU(ARM64) |
| m_jit.loadPtr(MacroAssembler::Address(MacroAssembler::framePointerRegister, CallFrame::returnPCOffset()), |
| MacroAssembler::linkRegister); |
| #if CPU(ARM64E) |
| m_jit.addPtr(MacroAssembler::TrustedImm32(sizeof(CallerFrameAndPC)), MacroAssembler::framePointerRegister); |
| m_jit.untagPtr(MacroAssembler::framePointerRegister, MacroAssembler::linkRegister); |
| m_jit.subPtr(MacroAssembler::TrustedImm32(sizeof(CallerFrameAndPC)), MacroAssembler::framePointerRegister); |
| m_jit.validateUntaggedPtr(MacroAssembler::linkRegister); |
| #endif |
| |
| #elif CPU(MIPS) |
| m_jit.loadPtr(MacroAssembler::Address(MacroAssembler::framePointerRegister, sizeof(void*)), |
| MacroAssembler::returnAddressRegister); |
| #endif |
| |
| // We want the frame pointer to always point to a valid frame, and |
| // we are going to trash the current one. Let's make it point to |
| // our caller's frame, since that's what we want to end up with. |
| m_jit.loadPtr(MacroAssembler::Address(MacroAssembler::framePointerRegister), |
| MacroAssembler::framePointerRegister); |
| |
| if (verbose) |
| dataLog("Preparing frame for tail call:\n", *this); |
| |
| prepareAny(); |
| |
| if (verbose) |
| dataLog("Ready for tail call!\n"); |
| } |
| |
| bool CallFrameShuffler::tryWrites(CachedRecovery& cachedRecovery) |
| { |
| ASSERT(m_newFrameBase != InvalidGPRReg); |
| |
| // If the value is already set up correctly, we don't have |
| // anything to do. |
| if (isSlowPath() && cachedRecovery.recovery().isInJSStack() |
| && cachedRecovery.targets().size() == 1 |
| && newAsOld(cachedRecovery.targets()[0]) == cachedRecovery.recovery().virtualRegister()) { |
| cachedRecovery.clearTargets(); |
| if (!cachedRecovery.wantedJSValueRegs() && cachedRecovery.wantedFPR() == InvalidFPRReg) |
| clearCachedRecovery(cachedRecovery.recovery()); |
| return true; |
| } |
| |
| if (!canLoadAndBox(cachedRecovery)) |
| return false; |
| |
| emitLoad(cachedRecovery); |
| emitBox(cachedRecovery); |
| ASSERT(cachedRecovery.recovery().isInRegisters() |
| || cachedRecovery.recovery().isConstant()); |
| |
| if (verbose) |
| dataLog(" * Storing ", cachedRecovery.recovery()); |
| for (size_t i = 0; i < cachedRecovery.targets().size(); ++i) { |
| VirtualRegister target { cachedRecovery.targets()[i] }; |
| ASSERT(!isDangerNew(target)); |
| if (verbose) |
| dataLog(!i ? " into " : ", and ", "NEW ", target); |
| emitStore(cachedRecovery, addressForNew(target)); |
| setNew(target, nullptr); |
| } |
| if (verbose) |
| dataLog("\n"); |
| cachedRecovery.clearTargets(); |
| if (!cachedRecovery.wantedJSValueRegs() && cachedRecovery.wantedFPR() == InvalidFPRReg) |
| clearCachedRecovery(cachedRecovery.recovery()); |
| |
| return true; |
| } |
| |
| bool CallFrameShuffler::performSafeWrites() |
| { |
| VirtualRegister firstSafe; |
| VirtualRegister end { lastNew() + 1 }; |
| Vector<VirtualRegister> failures; |
| |
| // For all cachedRecoveries that writes to the safe zone, if it |
| // doesn't also write to the danger zone, we try to perform |
| // the writes. This may free up danger slots, so we iterate |
| // again until it doesn't happen anymore. |
| // |
| // Note that even though we have a while block, we look at |
| // each slot of the new call frame at most once since in each |
| // iteration beyond the first, we only load up the portion of |
| // the new call frame that was dangerous and became safe due |
| // to the previous iteration. |
| do { |
| firstSafe = dangerFrontier() + 1; |
| if (verbose) |
| dataLog(" Trying safe writes (between NEW ", firstSafe, " and NEW ", end - 1, ")\n"); |
| bool didProgress = false; |
| for (VirtualRegister reg = firstSafe; reg < end; reg += 1) { |
| CachedRecovery* cachedRecovery = getNew(reg); |
| if (!cachedRecovery) { |
| if (verbose) |
| dataLog(" + ", reg, " is OK.\n"); |
| continue; |
| } |
| if (!hasOnlySafeWrites(*cachedRecovery)) { |
| if (verbose) { |
| dataLog(" - ", cachedRecovery->recovery(), " writes to NEW ", reg, |
| " but also has dangerous writes.\n"); |
| } |
| continue; |
| } |
| if (cachedRecovery->wantedJSValueRegs()) { |
| if (verbose) { |
| dataLog(" - ", cachedRecovery->recovery(), " writes to NEW ", reg, |
| " but is also needed in registers.\n"); |
| } |
| continue; |
| } |
| if (cachedRecovery->wantedFPR() != InvalidFPRReg) { |
| if (verbose) { |
| dataLog(" - ", cachedRecovery->recovery(), " writes to NEW ", reg, |
| " but is also needed in an FPR.\n"); |
| } |
| continue; |
| } |
| if (!tryWrites(*cachedRecovery)) { |
| if (verbose) |
| dataLog(" - Unable to write to NEW ", reg, " from ", cachedRecovery->recovery(), "\n"); |
| failures.append(reg); |
| } |
| didProgress = true; |
| } |
| end = firstSafe; |
| |
| // If we have cachedRecoveries that failed to write, it is |
| // because they are on the stack and we didn't have enough |
| // registers available at the time to load them into. If |
| // we have a free register, we should try again because it |
| // could free up some danger slots. |
| if (didProgress && hasFreeRegister()) { |
| Vector<VirtualRegister> stillFailing; |
| for (VirtualRegister failed : failures) { |
| CachedRecovery* cachedRecovery = getNew(failed); |
| // It could have been handled later if it had |
| // several targets |
| if (!cachedRecovery) |
| continue; |
| |
| ASSERT(hasOnlySafeWrites(*cachedRecovery) |
| && !cachedRecovery->wantedJSValueRegs() |
| && cachedRecovery->wantedFPR() == InvalidFPRReg); |
| if (!tryWrites(*cachedRecovery)) |
| stillFailing.append(failed); |
| } |
| failures = WTFMove(stillFailing); |
| } |
| if (verbose && firstSafe != dangerFrontier() + 1) |
| dataLog(" We freed up danger slots!\n"); |
| } while (firstSafe != dangerFrontier() + 1); |
| |
| return failures.isEmpty(); |
| } |
| |
| void CallFrameShuffler::prepareAny() |
| { |
| ASSERT(!isUndecided()); |
| |
| updateDangerFrontier(); |
| |
| // First, we try to store any value that goes above the danger |
| // frontier. This will never use more registers since we are only |
| // loading+storing if we ensure that any register used for the load |
| // will be freed up after the stores (i.e., all stores are above |
| // the danger frontier, and there is no wanted register). |
| performSafeWrites(); |
| |
| // At this point, we couldn't have more available registers than |
| // we have withouth spilling: all values currently in registers |
| // either require a write to the danger zone, or have a wanted |
| // register, which means that in any case they will have to go |
| // through registers again. |
| |
| // We now slowly free up the danger zone by first loading the old |
| // value on the danger frontier, spilling as many registers as |
| // needed to do so and ensuring that the corresponding slot in the |
| // new frame is now ready to be written. Then, we store the old |
| // value to its target location if possible (we could have failed |
| // to load it previously due to high pressure). Finally, we write |
| // to any of the newly safe slots that we can, which could free up |
| // registers (hence why we do it eagerly). |
| for (VirtualRegister reg = dangerFrontier(); reg >= firstNew(); reg -= 1) { |
| if (reg == dangerFrontier()) { |
| if (verbose) |
| dataLog(" Next slot (NEW ", reg, ") is the danger frontier\n"); |
| CachedRecovery* cachedRecovery { getOld(newAsOld(dangerFrontier())) }; |
| ASSERT(cachedRecovery); |
| ensureLoad(*cachedRecovery); |
| emitLoad(*cachedRecovery); |
| ensureBox(*cachedRecovery); |
| emitBox(*cachedRecovery); |
| if (hasOnlySafeWrites(*cachedRecovery)) |
| tryWrites(*cachedRecovery); |
| } else if (verbose) |
| dataLog(" Next slot is NEW ", reg, "\n"); |
| |
| ASSERT(!isDangerNew(reg)); |
| CachedRecovery* cachedRecovery = getNew(reg); |
| // This could be one of the header slots we don't care about. |
| if (!cachedRecovery) { |
| if (verbose) |
| dataLog(" + ", reg, " is OK\n"); |
| continue; |
| } |
| |
| if (canLoadAndBox(*cachedRecovery) && hasOnlySafeWrites(*cachedRecovery) |
| && !cachedRecovery->wantedJSValueRegs() |
| && cachedRecovery->wantedFPR() == InvalidFPRReg) { |
| emitLoad(*cachedRecovery); |
| emitBox(*cachedRecovery); |
| bool writesOK = tryWrites(*cachedRecovery); |
| ASSERT_UNUSED(writesOK, writesOK); |
| } else if (verbose) |
| dataLog(" - ", cachedRecovery->recovery(), " can't be handled just yet.\n"); |
| } |
| ASSERT(dangerFrontier() < firstNew()); |
| |
| // Now, the danger zone is empty, but we still have a couple of |
| // things to do: |
| // |
| // 1) There could be remaining safe writes that failed earlier due |
| // to high register pressure and had nothing to do with the |
| // danger zone whatsoever. |
| // |
| // 2) Some wanted registers could have to be loaded (this could |
| // happen either when making a call to a new function with a |
| // lower number of arguments - since above here, we only load |
| // wanted registers when they are at the danger frontier -, or |
| // if a wanted register got spilled). |
| // |
| // 3) Some wanted registers could have been loaded in the wrong |
| // registers |
| // |
| // 4) We have to take care of some bookkeeping - namely, storing |
| // the argument count and updating the stack pointer. |
| |
| // At this point, we must have enough registers available for |
| // handling 1). None of the loads can fail because we have been |
| // eagerly freeing up registers in all the previous phases - so |
| // the only values that are in registers at this point must have |
| // wanted registers. |
| if (verbose) |
| dataLog(" Danger zone is clear, performing remaining writes.\n"); |
| for (VirtualRegister reg = firstNew(); reg <= lastNew(); reg += 1) { |
| CachedRecovery* cachedRecovery { getNew(reg) }; |
| if (!cachedRecovery) |
| continue; |
| |
| emitLoad(*cachedRecovery); |
| emitBox(*cachedRecovery); |
| bool writesOK = tryWrites(*cachedRecovery); |
| ASSERT_UNUSED(writesOK, writesOK); |
| } |
| |
| #if USE(JSVALUE64) |
| if (m_numberTagRegister != InvalidGPRReg && m_newRegisters[m_numberTagRegister]) |
| releaseGPR(m_numberTagRegister); |
| #endif |
| |
| // Handle 2) by loading all registers. We don't have to do any |
| // writes, since they have been taken care of above. |
| if (verbose) |
| dataLog(" Loading wanted registers into registers\n"); |
| for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) { |
| CachedRecovery* cachedRecovery { m_newRegisters[reg] }; |
| if (!cachedRecovery) |
| continue; |
| |
| emitLoad(*cachedRecovery); |
| emitBox(*cachedRecovery); |
| ASSERT(cachedRecovery->targets().isEmpty()); |
| } |
| |
| #if USE(JSVALUE64) |
| if (m_numberTagRegister != InvalidGPRReg) |
| releaseGPR(m_numberTagRegister); |
| #endif |
| |
| // At this point, we have read everything we cared about from the |
| // stack, and written everything we had to to the stack. |
| if (verbose) |
| dataLog(" Callee frame is fully set up\n"); |
| if (ASSERT_ENABLED) { |
| for (VirtualRegister reg = firstNew(); reg <= lastNew(); reg += 1) |
| ASSERT_UNUSED(reg, !getNew(reg)); |
| |
| for (CachedRecovery* cachedRecovery : m_cachedRecoveries) { |
| ASSERT_UNUSED(cachedRecovery, cachedRecovery->targets().isEmpty()); |
| ASSERT(!cachedRecovery->recovery().isInJSStack()); |
| } |
| } |
| |
| // We need to handle 4) first because it implies releasing |
| // m_newFrameBase, which could be a wanted register. |
| if (verbose) |
| dataLog(" * Storing the argument count into ", VirtualRegister { CallFrameSlot::argumentCountIncludingThis }, "\n"); |
| m_jit.store32(MacroAssembler::TrustedImm32(0), |
| addressForNew(VirtualRegister { CallFrameSlot::argumentCountIncludingThis }).withOffset(TagOffset)); |
| RELEASE_ASSERT(m_numPassedArgs != UINT_MAX); |
| m_jit.store32(MacroAssembler::TrustedImm32(m_numPassedArgs), |
| addressForNew(VirtualRegister { CallFrameSlot::argumentCountIncludingThis }).withOffset(PayloadOffset)); |
| |
| if (!isSlowPath()) { |
| ASSERT(m_newFrameBase != MacroAssembler::stackPointerRegister); |
| if (verbose) |
| dataLog(" Releasing the new frame base pointer\n"); |
| m_jit.move(m_newFrameBase, MacroAssembler::stackPointerRegister); |
| releaseGPR(m_newFrameBase); |
| } |
| |
| // Finally we handle 3) |
| if (verbose) |
| dataLog(" Ensuring wanted registers are in the right register\n"); |
| for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) { |
| CachedRecovery* cachedRecovery { m_newRegisters[reg] }; |
| if (!cachedRecovery) |
| continue; |
| |
| emitDisplace(*cachedRecovery); |
| } |
| } |
| |
| } // namespace JSC |
| |
| #endif // ENABLE(JIT) |
