| /* |
| * Copyright (C) 2015-2019 Apple Inc. All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions |
| * are met: |
| * 1. Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * 2. Redistributions in binary form must reproduce the above copyright |
| * notice, this list of conditions and the following disclaimer in the |
| * documentation and/or other materials provided with the distribution. |
| * |
| * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY |
| * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR |
| * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY |
| * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| #include "config.h" |
| #include "CallFrameShuffler.h" |
| |
| #if ENABLE(JIT) && USE(JSVALUE64) |
| |
| #include "CCallHelpers.h" |
| #include "DataFormat.h" |
| #include "JSCJSValueInlines.h" |
| |
| namespace JSC { |
| |
| DataFormat CallFrameShuffler::emitStore( |
| CachedRecovery& cachedRecovery, MacroAssembler::Address address) |
| { |
| ASSERT(!cachedRecovery.recovery().isInJSStack()); |
| |
| switch (cachedRecovery.recovery().technique()) { |
| case InGPR: |
| m_jit.storePtr(cachedRecovery.recovery().gpr(), address); |
| return DataFormatJS; |
| case UnboxedInt32InGPR: |
| m_jit.store32(cachedRecovery.recovery().gpr(), address.withOffset(PayloadOffset)); |
| return DataFormatInt32; |
| case UnboxedInt52InGPR: |
| m_jit.rshift64(MacroAssembler::TrustedImm32(JSValue::int52ShiftAmount), |
| cachedRecovery.recovery().gpr()); |
| FALLTHROUGH; |
| case UnboxedStrictInt52InGPR: |
| m_jit.storePtr(cachedRecovery.recovery().gpr(), address); |
| return DataFormatStrictInt52; |
| case UnboxedBooleanInGPR: |
| m_jit.storePtr(cachedRecovery.recovery().gpr(), address); |
| return DataFormatBoolean; |
| case UnboxedCellInGPR: |
| m_jit.storePtr(cachedRecovery.recovery().gpr(), address); |
| return DataFormatCell; |
| case UnboxedDoubleInFPR: |
| m_jit.storeDouble(cachedRecovery.recovery().fpr(), address); |
| return DataFormatDouble; |
| case InFPR: |
| m_jit.storeDouble(cachedRecovery.recovery().fpr(), address); |
| return DataFormatJS; |
| case Constant: |
| m_jit.storeTrustedValue(cachedRecovery.recovery().constant(), address); |
| return DataFormatJS; |
| default: |
| RELEASE_ASSERT_NOT_REACHED(); |
| } |
| } |
| |
| void CallFrameShuffler::emitBox(CachedRecovery& cachedRecovery) |
| { |
| ASSERT(canBox(cachedRecovery)); |
| if (cachedRecovery.recovery().isConstant()) |
| return; |
| |
| if (cachedRecovery.recovery().isInGPR()) { |
| switch (cachedRecovery.recovery().dataFormat()) { |
| case DataFormatInt32: |
| if (verbose) |
| dataLog(" * Boxing ", cachedRecovery.recovery()); |
| m_jit.zeroExtend32ToWord( |
| cachedRecovery.recovery().gpr(), |
| cachedRecovery.recovery().gpr()); |
| m_lockedRegisters.add(cachedRecovery.recovery().gpr(), IgnoreVectors); |
| if (tryAcquireNumberTagRegister()) |
| m_jit.or64(m_numberTagRegister, cachedRecovery.recovery().gpr()); |
| else { |
| // We have to do this the hard way |
| m_jit.or64(MacroAssembler::TrustedImm64(JSValue::NumberTag), |
| cachedRecovery.recovery().gpr()); |
| } |
| m_lockedRegisters.remove(cachedRecovery.recovery().gpr()); |
| cachedRecovery.setRecovery( |
| ValueRecovery::inGPR(cachedRecovery.recovery().gpr(), DataFormatJS)); |
| if (verbose) |
| dataLog(" into ", cachedRecovery.recovery(), "\n"); |
| return; |
| case DataFormatInt52: |
| if (verbose) |
| dataLog(" * Boxing ", cachedRecovery.recovery()); |
| m_jit.rshift64(MacroAssembler::TrustedImm32(JSValue::int52ShiftAmount), |
| cachedRecovery.recovery().gpr()); |
| cachedRecovery.setRecovery( |
| ValueRecovery::inGPR(cachedRecovery.recovery().gpr(), DataFormatStrictInt52)); |
| if (verbose) |
| dataLog(" into ", cachedRecovery.recovery(), "\n"); |
| FALLTHROUGH; |
| case DataFormatStrictInt52: { |
| if (verbose) |
| dataLog(" * Boxing ", cachedRecovery.recovery()); |
| FPRReg resultFPR = getFreeFPR(); |
| ASSERT(resultFPR != InvalidFPRReg); |
| m_jit.convertInt64ToDouble(cachedRecovery.recovery().gpr(), resultFPR); |
| updateRecovery(cachedRecovery, ValueRecovery::inFPR(resultFPR, DataFormatDouble)); |
| if (verbose) |
| dataLog(" into ", cachedRecovery.recovery(), "\n"); |
| break; |
| } |
| case DataFormatBoolean: |
| if (verbose) |
| dataLog(" * Boxing ", cachedRecovery.recovery()); |
| m_jit.add32(MacroAssembler::TrustedImm32(JSValue::ValueFalse), |
| cachedRecovery.recovery().gpr()); |
| cachedRecovery.setRecovery( |
| ValueRecovery::inGPR(cachedRecovery.recovery().gpr(), DataFormatJS)); |
| if (verbose) |
| dataLog(" into ", cachedRecovery.recovery(), "\n"); |
| return; |
| default: |
| return; |
| } |
| } |
| |
| if (cachedRecovery.recovery().isInFPR()) { |
| if (cachedRecovery.recovery().dataFormat() == DataFormatDouble) { |
| if (verbose) |
| dataLog(" * Boxing ", cachedRecovery.recovery()); |
| GPRReg resultGPR = cachedRecovery.wantedJSValueRegs().gpr(); |
| if (resultGPR == InvalidGPRReg || m_registers[resultGPR]) |
| resultGPR = getFreeGPR(); |
| ASSERT(resultGPR != InvalidGPRReg); |
| ASSERT(Reg::fromIndex(resultGPR).isGPR()); |
| m_jit.purifyNaN(cachedRecovery.recovery().fpr()); |
| m_jit.moveDoubleTo64(cachedRecovery.recovery().fpr(), resultGPR); |
| m_lockedRegisters.add(resultGPR, IgnoreVectors); |
| if (tryAcquireNumberTagRegister()) |
| m_jit.sub64(m_numberTagRegister, resultGPR); |
| else |
| m_jit.sub64(MacroAssembler::TrustedImm64(JSValue::NumberTag), resultGPR); |
| m_lockedRegisters.remove(resultGPR); |
| updateRecovery(cachedRecovery, ValueRecovery::inGPR(resultGPR, DataFormatJS)); |
| if (verbose) |
| dataLog(" into ", cachedRecovery.recovery(), "\n"); |
| return; |
| } |
| ASSERT(cachedRecovery.recovery().dataFormat() == DataFormatJS); |
| return; |
| } |
| |
| RELEASE_ASSERT_NOT_REACHED(); |
| } |
| |
| void CallFrameShuffler::emitLoad(CachedRecovery& cachedRecovery) |
| { |
| if (!cachedRecovery.recovery().isInJSStack()) |
| return; |
| |
| if (verbose) |
| dataLog(" * Loading ", cachedRecovery.recovery(), " into "); |
| |
| VirtualRegister reg = cachedRecovery.recovery().virtualRegister(); |
| MacroAssembler::Address address { addressForOld(reg) }; |
| bool tryFPR { true }; |
| GPRReg resultGPR { cachedRecovery.wantedJSValueRegs().gpr() }; |
| |
| // If we want a GPR and it's available, that's better than loading |
| // into an FPR. |
| if (resultGPR != InvalidGPRReg && !m_registers[resultGPR] |
| && !m_lockedRegisters.contains(resultGPR, IgnoreVectors) && cachedRecovery.loadsIntoGPR()) |
| tryFPR = false; |
| |
| // Otherwise, we prefer loading into FPRs if possible |
| if (tryFPR && cachedRecovery.loadsIntoFPR()) { |
| FPRReg resultFPR { cachedRecovery.wantedFPR() }; |
| if (resultFPR == InvalidFPRReg || m_registers[resultFPR] || m_lockedRegisters.contains(resultFPR, IgnoreVectors)) |
| resultFPR = getFreeFPR(); |
| if (resultFPR != InvalidFPRReg) { |
| m_jit.loadDouble(address, resultFPR); |
| DataFormat dataFormat = DataFormatJS; |
| // We could be transforming a DataFormatCell into a |
| // DataFormatJS here - but that's OK. |
| if (cachedRecovery.recovery().dataFormat() == DataFormatDouble) |
| dataFormat = DataFormatDouble; |
| updateRecovery(cachedRecovery, |
| ValueRecovery::inFPR(resultFPR, dataFormat)); |
| if (verbose) |
| dataLog(cachedRecovery.recovery(), "\n"); |
| if (reg == newAsOld(dangerFrontier())) |
| updateDangerFrontier(); |
| return; |
| } |
| } |
| |
| ASSERT(cachedRecovery.loadsIntoGPR()); |
| if (resultGPR == InvalidGPRReg || m_registers[resultGPR] || m_lockedRegisters.contains(resultGPR, IgnoreVectors)) |
| resultGPR = getFreeGPR(); |
| ASSERT(resultGPR != InvalidGPRReg); |
| m_jit.loadPtr(address, resultGPR); |
| updateRecovery(cachedRecovery, |
| ValueRecovery::inGPR(resultGPR, cachedRecovery.recovery().dataFormat())); |
| if (verbose) |
| dataLog(cachedRecovery.recovery(), "\n"); |
| if (reg == newAsOld(dangerFrontier())) |
| updateDangerFrontier(); |
| } |
| |
| bool CallFrameShuffler::canLoad(CachedRecovery& cachedRecovery) |
| { |
| if (!cachedRecovery.recovery().isInJSStack()) |
| return true; |
| |
| ASSERT(cachedRecovery.loadsIntoFPR() || cachedRecovery.loadsIntoGPR()); |
| |
| if (cachedRecovery.loadsIntoFPR() && getFreeFPR() != InvalidFPRReg) |
| return true; |
| |
| if (cachedRecovery.loadsIntoGPR() && getFreeGPR() != InvalidGPRReg) |
| return true; |
| |
| return false; |
| } |
| |
| void CallFrameShuffler::emitDisplace(CachedRecovery& cachedRecovery) |
| { |
| Reg wantedReg; |
| if (!(wantedReg = Reg { cachedRecovery.wantedJSValueRegs().gpr() })) |
| wantedReg = Reg { cachedRecovery.wantedFPR() }; |
| ASSERT(wantedReg); |
| ASSERT(!m_lockedRegisters.contains(wantedReg, IgnoreVectors)); |
| |
| if (CachedRecovery* current = m_registers[wantedReg]) { |
| if (current == &cachedRecovery) { |
| if (verbose) |
| dataLog(" + ", wantedReg, " is OK\n"); |
| return; |
| } |
| // We could do a more complex thing by finding cycles |
| // etc. in that case. |
| // However, ending up in this situation will be super |
| // rare, and should actually be outright impossible for |
| // non-FTL tiers, since: |
| // (a) All doubles have been converted into JSValues with |
| // ValueRep nodes, so FPRs are initially free |
| // |
| // (b) The only recoveries with wanted registers are the |
| // callee (which always starts out in a register) and |
| // the callee-save registers |
| // |
| // (c) The callee-save registers are the first things we |
| // load (after the return PC), and they are loaded as JSValues |
| // |
| // (d) We prefer loading JSValues into FPRs if their |
| // wanted GPR is not available |
| // |
| // (e) If we end up spilling some registers with a |
| // target, we won't load them again before the very |
| // end of the algorithm |
| // |
| // Combined, this means that we will never load a recovery |
| // with a wanted GPR into any GPR other than its wanted |
| // GPR. The callee could however have been initially in |
| // one of the callee-save registers - but since the wanted |
| // GPR for the callee is always regT0, it will be the |
| // first one to be displaced, and we won't see it when |
| // handling any of the callee-save registers. |
| // |
| // Thus, the only way we could ever reach this path is in |
| // the FTL, when there is so much pressure that we |
| // absolutely need to load the callee-save registers into |
| // different GPRs initially but not enough pressure to |
| // then have to spill all of them. And even in that case, |
| // depending on the order in which B3 saves the |
| // callee-saves, we will probably still be safe. Anyway, |
| // the couple extra move instructions compared to an |
| // efficient cycle-based algorithm are not going to hurt |
| // us. |
| if (wantedReg.isFPR()) { |
| FPRReg tempFPR = getFreeFPR(); |
| if (verbose) |
| dataLog(" * Moving ", wantedReg, " into ", tempFPR, "\n"); |
| m_jit.moveDouble(wantedReg.fpr(), tempFPR); |
| updateRecovery(*current, |
| ValueRecovery::inFPR(tempFPR, current->recovery().dataFormat())); |
| } else { |
| GPRReg tempGPR = getFreeGPR(); |
| if (verbose) |
| dataLog(" * Moving ", wantedReg.gpr(), " into ", tempGPR, "\n"); |
| m_jit.move(wantedReg.gpr(), tempGPR); |
| updateRecovery(*current, |
| ValueRecovery::inGPR(tempGPR, current->recovery().dataFormat())); |
| } |
| } |
| ASSERT(!m_registers[wantedReg]); |
| |
| if (cachedRecovery.recovery().isConstant()) { |
| // We only care about callee saves for wanted FPRs, and those are never constants |
| ASSERT(wantedReg.isGPR()); |
| if (verbose) |
| dataLog(" * Loading ", cachedRecovery.recovery().constant(), " into ", wantedReg, "\n"); |
| m_jit.moveTrustedValue(cachedRecovery.recovery().constant(), JSValueRegs { wantedReg.gpr() }); |
| updateRecovery( |
| cachedRecovery, |
| ValueRecovery::inRegister(wantedReg, DataFormatJS)); |
| } else if (cachedRecovery.recovery().isInGPR()) { |
| if (verbose) |
| dataLog(" * Moving ", cachedRecovery.recovery(), " into ", wantedReg, "\n"); |
| if (wantedReg.isGPR()) |
| m_jit.move(cachedRecovery.recovery().gpr(), wantedReg.gpr()); |
| else |
| m_jit.move64ToDouble(cachedRecovery.recovery().gpr(), wantedReg.fpr()); |
| RELEASE_ASSERT(cachedRecovery.recovery().dataFormat() == DataFormatJS); |
| updateRecovery(cachedRecovery, |
| ValueRecovery::inRegister(wantedReg, DataFormatJS)); |
| } else { |
| ASSERT(cachedRecovery.recovery().isInFPR()); |
| if (cachedRecovery.recovery().dataFormat() == DataFormatDouble) { |
| // We only care about callee saves for wanted FPRs, and those are always DataFormatJS |
| ASSERT(wantedReg.isGPR()); |
| // This will automatically pick the wanted GPR |
| emitBox(cachedRecovery); |
| } else { |
| if (verbose) |
| dataLog(" * Moving ", cachedRecovery.recovery().fpr(), " into ", wantedReg, "\n"); |
| if (wantedReg.isGPR()) |
| m_jit.moveDoubleTo64(cachedRecovery.recovery().fpr(), wantedReg.gpr()); |
| else |
| m_jit.moveDouble(cachedRecovery.recovery().fpr(), wantedReg.fpr()); |
| RELEASE_ASSERT(cachedRecovery.recovery().dataFormat() == DataFormatJS); |
| updateRecovery(cachedRecovery, |
| ValueRecovery::inRegister(wantedReg, DataFormatJS)); |
| } |
| } |
| |
| ASSERT(m_registers[wantedReg] == &cachedRecovery); |
| } |
| |
| bool CallFrameShuffler::tryAcquireNumberTagRegister() |
| { |
| if (m_numberTagRegister != InvalidGPRReg) |
| return true; |
| |
| m_numberTagRegister = getFreeGPR(); |
| |
| if (m_numberTagRegister == InvalidGPRReg) |
| return false; |
| |
| m_lockedRegisters.add(m_numberTagRegister, IgnoreVectors); |
| m_jit.move(MacroAssembler::TrustedImm64(JSValue::NumberTag), m_numberTagRegister); |
| return true; |
| } |
| |
| } // namespace JSC |
| |
| #endif // ENABLE(JIT) && USE(JSVALUE64) |
