| //===----------- PPCVSXSwapRemoval.cpp - Remove VSX LE Swaps -------------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===---------------------------------------------------------------------===// |
| // |
| // This pass analyzes vector computations and removes unnecessary |
| // doubleword swaps (xxswapd instructions). This pass is performed |
| // only for little-endian VSX code generation. |
| // |
| // For this specific case, loads and stores of v4i32, v4f32, v2i64, |
| // and v2f64 vectors are inefficient. These are implemented using |
| // the lxvd2x and stxvd2x instructions, which invert the order of |
| // doublewords in a vector register. Thus code generation inserts |
| // an xxswapd after each such load, and prior to each such store. |
| // |
| // The extra xxswapd instructions reduce performance. The purpose |
| // of this pass is to reduce the number of xxswapd instructions |
| // required for correctness. |
| // |
| // The primary insight is that much code that operates on vectors |
| // does not care about the relative order of elements in a register, |
| // so long as the correct memory order is preserved. If we have a |
| // computation where all input values are provided by lxvd2x/xxswapd, |
| // all outputs are stored using xxswapd/lxvd2x, and all intermediate |
| // computations are lane-insensitive (independent of element order), |
| // then all the xxswapd instructions associated with the loads and |
| // stores may be removed without changing observable semantics. |
| // |
| // This pass uses standard equivalence class infrastructure to create |
| // maximal webs of computations fitting the above description. Each |
| // such web is then optimized by removing its unnecessary xxswapd |
| // instructions. |
| // |
| // There are some lane-sensitive operations for which we can still |
| // permit the optimization, provided we modify those operations |
| // accordingly. Such operations are identified as using "special |
| // handling" within this module. |
| // |
| //===---------------------------------------------------------------------===// |
| |
| #include "PPC.h" |
| #include "PPCInstrBuilder.h" |
| #include "PPCInstrInfo.h" |
| #include "PPCTargetMachine.h" |
| #include "llvm/ADT/DenseMap.h" |
| #include "llvm/ADT/EquivalenceClasses.h" |
| #include "llvm/CodeGen/MachineFunctionPass.h" |
| #include "llvm/CodeGen/MachineInstrBuilder.h" |
| #include "llvm/CodeGen/MachineRegisterInfo.h" |
| #include "llvm/Config/llvm-config.h" |
| #include "llvm/Support/Debug.h" |
| #include "llvm/Support/Format.h" |
| #include "llvm/Support/raw_ostream.h" |
| |
| using namespace llvm; |
| |
| #define DEBUG_TYPE "ppc-vsx-swaps" |
| |
| namespace { |
| |
| // A PPCVSXSwapEntry is created for each machine instruction that |
| // is relevant to a vector computation. |
| struct PPCVSXSwapEntry { |
| // Pointer to the instruction. |
| MachineInstr *VSEMI; |
| |
| // Unique ID (position in the swap vector). |
| int VSEId; |
| |
| // Attributes of this node. |
| unsigned int IsLoad : 1; |
| unsigned int IsStore : 1; |
| unsigned int IsSwap : 1; |
| unsigned int MentionsPhysVR : 1; |
| unsigned int IsSwappable : 1; |
| unsigned int MentionsPartialVR : 1; |
| unsigned int SpecialHandling : 3; |
| unsigned int WebRejected : 1; |
| unsigned int WillRemove : 1; |
| }; |
| |
| enum SHValues { |
| SH_NONE = 0, |
| SH_EXTRACT, |
| SH_INSERT, |
| SH_NOSWAP_LD, |
| SH_NOSWAP_ST, |
| SH_SPLAT, |
| SH_XXPERMDI, |
| SH_COPYWIDEN |
| }; |
| |
| struct PPCVSXSwapRemoval : public MachineFunctionPass { |
| |
| static char ID; |
| const PPCInstrInfo *TII; |
| MachineFunction *MF; |
| MachineRegisterInfo *MRI; |
| |
| // Swap entries are allocated in a vector for better performance. |
| std::vector<PPCVSXSwapEntry> SwapVector; |
| |
| // A mapping is maintained between machine instructions and |
| // their swap entries. The key is the address of the MI. |
| DenseMap<MachineInstr*, int> SwapMap; |
| |
| // Equivalence classes are used to gather webs of related computation. |
| // Swap entries are represented by their VSEId fields. |
| EquivalenceClasses<int> *EC; |
| |
| PPCVSXSwapRemoval() : MachineFunctionPass(ID) { |
| initializePPCVSXSwapRemovalPass(*PassRegistry::getPassRegistry()); |
| } |
| |
| private: |
| // Initialize data structures. |
| void initialize(MachineFunction &MFParm); |
| |
| // Walk the machine instructions to gather vector usage information. |
| // Return true iff vector mentions are present. |
| bool gatherVectorInstructions(); |
| |
| // Add an entry to the swap vector and swap map. |
| int addSwapEntry(MachineInstr *MI, PPCVSXSwapEntry &SwapEntry); |
| |
| // Hunt backwards through COPY and SUBREG_TO_REG chains for a |
| // source register. VecIdx indicates the swap vector entry to |
| // mark as mentioning a physical register if the search leads |
| // to one. |
| unsigned lookThruCopyLike(unsigned SrcReg, unsigned VecIdx); |
| |
| // Generate equivalence classes for related computations (webs). |
| void formWebs(); |
| |
| // Analyze webs and determine those that cannot be optimized. |
| void recordUnoptimizableWebs(); |
| |
| // Record which swap instructions can be safely removed. |
| void markSwapsForRemoval(); |
| |
| // Remove swaps and update other instructions requiring special |
| // handling. Return true iff any changes are made. |
| bool removeSwaps(); |
| |
| // Insert a swap instruction from SrcReg to DstReg at the given |
| // InsertPoint. |
| void insertSwap(MachineInstr *MI, MachineBasicBlock::iterator InsertPoint, |
| unsigned DstReg, unsigned SrcReg); |
| |
| // Update instructions requiring special handling. |
| void handleSpecialSwappables(int EntryIdx); |
| |
| // Dump a description of the entries in the swap vector. |
| void dumpSwapVector(); |
| |
| // Return true iff the given register is in the given class. |
| bool isRegInClass(unsigned Reg, const TargetRegisterClass *RC) { |
| if (TargetRegisterInfo::isVirtualRegister(Reg)) |
| return RC->hasSubClassEq(MRI->getRegClass(Reg)); |
| return RC->contains(Reg); |
| } |
| |
| // Return true iff the given register is a full vector register. |
| bool isVecReg(unsigned Reg) { |
| return (isRegInClass(Reg, &PPC::VSRCRegClass) || |
| isRegInClass(Reg, &PPC::VRRCRegClass)); |
| } |
| |
| // Return true iff the given register is a partial vector register. |
| bool isScalarVecReg(unsigned Reg) { |
| return (isRegInClass(Reg, &PPC::VSFRCRegClass) || |
| isRegInClass(Reg, &PPC::VSSRCRegClass)); |
| } |
| |
| // Return true iff the given register mentions all or part of a |
| // vector register. Also sets Partial to true if the mention |
| // is for just the floating-point register overlap of the register. |
| bool isAnyVecReg(unsigned Reg, bool &Partial) { |
| if (isScalarVecReg(Reg)) |
| Partial = true; |
| return isScalarVecReg(Reg) || isVecReg(Reg); |
| } |
| |
| public: |
| // Main entry point for this pass. |
| bool runOnMachineFunction(MachineFunction &MF) override { |
| if (skipFunction(MF.getFunction())) |
| return false; |
| |
| // If we don't have VSX on the subtarget, don't do anything. |
| // Also, on Power 9 the load and store ops preserve element order and so |
| // the swaps are not required. |
| const PPCSubtarget &STI = MF.getSubtarget<PPCSubtarget>(); |
| if (!STI.hasVSX() || !STI.needsSwapsForVSXMemOps()) |
| return false; |
| |
| bool Changed = false; |
| initialize(MF); |
| |
| if (gatherVectorInstructions()) { |
| formWebs(); |
| recordUnoptimizableWebs(); |
| markSwapsForRemoval(); |
| Changed = removeSwaps(); |
| } |
| |
| // FIXME: See the allocation of EC in initialize(). |
| delete EC; |
| return Changed; |
| } |
| }; |
| |
| // Initialize data structures for this pass. In particular, clear the |
| // swap vector and allocate the equivalence class mapping before |
| // processing each function. |
| void PPCVSXSwapRemoval::initialize(MachineFunction &MFParm) { |
| MF = &MFParm; |
| MRI = &MF->getRegInfo(); |
| TII = MF->getSubtarget<PPCSubtarget>().getInstrInfo(); |
| |
| // An initial vector size of 256 appears to work well in practice. |
| // Small/medium functions with vector content tend not to incur a |
| // reallocation at this size. Three of the vector tests in |
| // projects/test-suite reallocate, which seems like a reasonable rate. |
| const int InitialVectorSize(256); |
| SwapVector.clear(); |
| SwapVector.reserve(InitialVectorSize); |
| |
| // FIXME: Currently we allocate EC each time because we don't have |
| // access to the set representation on which to call clear(). Should |
| // consider adding a clear() method to the EquivalenceClasses class. |
| EC = new EquivalenceClasses<int>; |
| } |
| |
| // Create an entry in the swap vector for each instruction that mentions |
| // a full vector register, recording various characteristics of the |
| // instructions there. |
| bool PPCVSXSwapRemoval::gatherVectorInstructions() { |
| bool RelevantFunction = false; |
| |
| for (MachineBasicBlock &MBB : *MF) { |
| for (MachineInstr &MI : MBB) { |
| |
| if (MI.isDebugInstr()) |
| continue; |
| |
| bool RelevantInstr = false; |
| bool Partial = false; |
| |
| for (const MachineOperand &MO : MI.operands()) { |
| if (!MO.isReg()) |
| continue; |
| unsigned Reg = MO.getReg(); |
| if (isAnyVecReg(Reg, Partial)) { |
| RelevantInstr = true; |
| break; |
| } |
| } |
| |
| if (!RelevantInstr) |
| continue; |
| |
| RelevantFunction = true; |
| |
| // Create a SwapEntry initialized to zeros, then fill in the |
| // instruction and ID fields before pushing it to the back |
| // of the swap vector. |
| PPCVSXSwapEntry SwapEntry{}; |
| int VecIdx = addSwapEntry(&MI, SwapEntry); |
| |
| switch(MI.getOpcode()) { |
| default: |
| // Unless noted otherwise, an instruction is considered |
| // safe for the optimization. There are a large number of |
| // such true-SIMD instructions (all vector math, logical, |
| // select, compare, etc.). However, if the instruction |
| // mentions a partial vector register and does not have |
| // special handling defined, it is not swappable. |
| if (Partial) |
| SwapVector[VecIdx].MentionsPartialVR = 1; |
| else |
| SwapVector[VecIdx].IsSwappable = 1; |
| break; |
| case PPC::XXPERMDI: { |
| // This is a swap if it is of the form XXPERMDI t, s, s, 2. |
| // Unfortunately, MachineCSE ignores COPY and SUBREG_TO_REG, so we |
| // can also see XXPERMDI t, SUBREG_TO_REG(s), SUBREG_TO_REG(s), 2, |
| // for example. We have to look through chains of COPY and |
| // SUBREG_TO_REG to find the real source value for comparison. |
| // If the real source value is a physical register, then mark the |
| // XXPERMDI as mentioning a physical register. |
| int immed = MI.getOperand(3).getImm(); |
| if (immed == 2) { |
| unsigned trueReg1 = lookThruCopyLike(MI.getOperand(1).getReg(), |
| VecIdx); |
| unsigned trueReg2 = lookThruCopyLike(MI.getOperand(2).getReg(), |
| VecIdx); |
| if (trueReg1 == trueReg2) |
| SwapVector[VecIdx].IsSwap = 1; |
| else { |
| // We can still handle these if the two registers are not |
| // identical, by adjusting the form of the XXPERMDI. |
| SwapVector[VecIdx].IsSwappable = 1; |
| SwapVector[VecIdx].SpecialHandling = SHValues::SH_XXPERMDI; |
| } |
| // This is a doubleword splat if it is of the form |
| // XXPERMDI t, s, s, 0 or XXPERMDI t, s, s, 3. As above we |
| // must look through chains of copy-likes to find the source |
| // register. We turn off the marking for mention of a physical |
| // register, because splatting it is safe; the optimization |
| // will not swap the value in the physical register. Whether |
| // or not the two input registers are identical, we can handle |
| // these by adjusting the form of the XXPERMDI. |
| } else if (immed == 0 || immed == 3) { |
| |
| SwapVector[VecIdx].IsSwappable = 1; |
| SwapVector[VecIdx].SpecialHandling = SHValues::SH_XXPERMDI; |
| |
| unsigned trueReg1 = lookThruCopyLike(MI.getOperand(1).getReg(), |
| VecIdx); |
| unsigned trueReg2 = lookThruCopyLike(MI.getOperand(2).getReg(), |
| VecIdx); |
| if (trueReg1 == trueReg2) |
| SwapVector[VecIdx].MentionsPhysVR = 0; |
| |
| } else { |
| // We can still handle these by adjusting the form of the XXPERMDI. |
| SwapVector[VecIdx].IsSwappable = 1; |
| SwapVector[VecIdx].SpecialHandling = SHValues::SH_XXPERMDI; |
| } |
| break; |
| } |
| case PPC::LVX: |
| // Non-permuting loads are currently unsafe. We can use special |
| // handling for this in the future. By not marking these as |
| // IsSwap, we ensure computations containing them will be rejected |
| // for now. |
| SwapVector[VecIdx].IsLoad = 1; |
| break; |
| case PPC::LXVD2X: |
| case PPC::LXVW4X: |
| // Permuting loads are marked as both load and swap, and are |
| // safe for optimization. |
| SwapVector[VecIdx].IsLoad = 1; |
| SwapVector[VecIdx].IsSwap = 1; |
| break; |
| case PPC::LXSDX: |
| case PPC::LXSSPX: |
| case PPC::XFLOADf64: |
| case PPC::XFLOADf32: |
| // A load of a floating-point value into the high-order half of |
| // a vector register is safe, provided that we introduce a swap |
| // following the load, which will be done by the SUBREG_TO_REG |
| // support. So just mark these as safe. |
| SwapVector[VecIdx].IsLoad = 1; |
| SwapVector[VecIdx].IsSwappable = 1; |
| break; |
| case PPC::STVX: |
| // Non-permuting stores are currently unsafe. We can use special |
| // handling for this in the future. By not marking these as |
| // IsSwap, we ensure computations containing them will be rejected |
| // for now. |
| SwapVector[VecIdx].IsStore = 1; |
| break; |
| case PPC::STXVD2X: |
| case PPC::STXVW4X: |
| // Permuting stores are marked as both store and swap, and are |
| // safe for optimization. |
| SwapVector[VecIdx].IsStore = 1; |
| SwapVector[VecIdx].IsSwap = 1; |
| break; |
| case PPC::COPY: |
| // These are fine provided they are moving between full vector |
| // register classes. |
| if (isVecReg(MI.getOperand(0).getReg()) && |
| isVecReg(MI.getOperand(1).getReg())) |
| SwapVector[VecIdx].IsSwappable = 1; |
| // If we have a copy from one scalar floating-point register |
| // to another, we can accept this even if it is a physical |
| // register. The only way this gets involved is if it feeds |
| // a SUBREG_TO_REG, which is handled by introducing a swap. |
| else if (isScalarVecReg(MI.getOperand(0).getReg()) && |
| isScalarVecReg(MI.getOperand(1).getReg())) |
| SwapVector[VecIdx].IsSwappable = 1; |
| break; |
| case PPC::SUBREG_TO_REG: { |
| // These are fine provided they are moving between full vector |
| // register classes. If they are moving from a scalar |
| // floating-point class to a vector class, we can handle those |
| // as well, provided we introduce a swap. It is generally the |
| // case that we will introduce fewer swaps than we remove, but |
| // (FIXME) a cost model could be used. However, introduced |
| // swaps could potentially be CSEd, so this is not trivial. |
| if (isVecReg(MI.getOperand(0).getReg()) && |
| isVecReg(MI.getOperand(2).getReg())) |
| SwapVector[VecIdx].IsSwappable = 1; |
| else if (isVecReg(MI.getOperand(0).getReg()) && |
| isScalarVecReg(MI.getOperand(2).getReg())) { |
| SwapVector[VecIdx].IsSwappable = 1; |
| SwapVector[VecIdx].SpecialHandling = SHValues::SH_COPYWIDEN; |
| } |
| break; |
| } |
| case PPC::VSPLTB: |
| case PPC::VSPLTH: |
| case PPC::VSPLTW: |
| case PPC::XXSPLTW: |
| // Splats are lane-sensitive, but we can use special handling |
| // to adjust the source lane for the splat. |
| SwapVector[VecIdx].IsSwappable = 1; |
| SwapVector[VecIdx].SpecialHandling = SHValues::SH_SPLAT; |
| break; |
| // The presence of the following lane-sensitive operations in a |
| // web will kill the optimization, at least for now. For these |
| // we do nothing, causing the optimization to fail. |
| // FIXME: Some of these could be permitted with special handling, |
| // and will be phased in as time permits. |
| // FIXME: There is no simple and maintainable way to express a set |
| // of opcodes having a common attribute in TableGen. Should this |
| // change, this is a prime candidate to use such a mechanism. |
| case PPC::INLINEASM: |
| case PPC::INLINEASM_BR: |
| case PPC::EXTRACT_SUBREG: |
| case PPC::INSERT_SUBREG: |
| case PPC::COPY_TO_REGCLASS: |
| case PPC::LVEBX: |
| case PPC::LVEHX: |
| case PPC::LVEWX: |
| case PPC::LVSL: |
| case PPC::LVSR: |
| case PPC::LVXL: |
| case PPC::STVEBX: |
| case PPC::STVEHX: |
| case PPC::STVEWX: |
| case PPC::STVXL: |
| // We can handle STXSDX and STXSSPX similarly to LXSDX and LXSSPX, |
| // by adding special handling for narrowing copies as well as |
| // widening ones. However, I've experimented with this, and in |
| // practice we currently do not appear to use STXSDX fed by |
| // a narrowing copy from a full vector register. Since I can't |
| // generate any useful test cases, I've left this alone for now. |
| case PPC::STXSDX: |
| case PPC::STXSSPX: |
| case PPC::VCIPHER: |
| case PPC::VCIPHERLAST: |
| case PPC::VMRGHB: |
| case PPC::VMRGHH: |
| case PPC::VMRGHW: |
| case PPC::VMRGLB: |
| case PPC::VMRGLH: |
| case PPC::VMRGLW: |
| case PPC::VMULESB: |
| case PPC::VMULESH: |
| case PPC::VMULESW: |
| case PPC::VMULEUB: |
| case PPC::VMULEUH: |
| case PPC::VMULEUW: |
| case PPC::VMULOSB: |
| case PPC::VMULOSH: |
| case PPC::VMULOSW: |
| case PPC::VMULOUB: |
| case PPC::VMULOUH: |
| case PPC::VMULOUW: |
| case PPC::VNCIPHER: |
| case PPC::VNCIPHERLAST: |
| case PPC::VPERM: |
| case PPC::VPERMXOR: |
| case PPC::VPKPX: |
| case PPC::VPKSHSS: |
| case PPC::VPKSHUS: |
| case PPC::VPKSDSS: |
| case PPC::VPKSDUS: |
| case PPC::VPKSWSS: |
| case PPC::VPKSWUS: |
| case PPC::VPKUDUM: |
| case PPC::VPKUDUS: |
| case PPC::VPKUHUM: |
| case PPC::VPKUHUS: |
| case PPC::VPKUWUM: |
| case PPC::VPKUWUS: |
| case PPC::VPMSUMB: |
| case PPC::VPMSUMD: |
| case PPC::VPMSUMH: |
| case PPC::VPMSUMW: |
| case PPC::VRLB: |
| case PPC::VRLD: |
| case PPC::VRLH: |
| case PPC::VRLW: |
| case PPC::VSBOX: |
| case PPC::VSHASIGMAD: |
| case PPC::VSHASIGMAW: |
| case PPC::VSL: |
| case PPC::VSLDOI: |
| case PPC::VSLO: |
| case PPC::VSR: |
| case PPC::VSRO: |
| case PPC::VSUM2SWS: |
| case PPC::VSUM4SBS: |
| case PPC::VSUM4SHS: |
| case PPC::VSUM4UBS: |
| case PPC::VSUMSWS: |
| case PPC::VUPKHPX: |
| case PPC::VUPKHSB: |
| case PPC::VUPKHSH: |
| case PPC::VUPKHSW: |
| case PPC::VUPKLPX: |
| case PPC::VUPKLSB: |
| case PPC::VUPKLSH: |
| case PPC::VUPKLSW: |
| case PPC::XXMRGHW: |
| case PPC::XXMRGLW: |
| // XXSLDWI could be replaced by a general permute with one of three |
| // permute control vectors (for shift values 1, 2, 3). However, |
| // VPERM has a more restrictive register class. |
| case PPC::XXSLDWI: |
| case PPC::XSCVDPSPN: |
| case PPC::XSCVSPDPN: |
| break; |
| } |
| } |
| } |
| |
| if (RelevantFunction) { |
| LLVM_DEBUG(dbgs() << "Swap vector when first built\n\n"); |
| LLVM_DEBUG(dumpSwapVector()); |
| } |
| |
| return RelevantFunction; |
| } |
| |
| // Add an entry to the swap vector and swap map, and make a |
| // singleton equivalence class for the entry. |
| int PPCVSXSwapRemoval::addSwapEntry(MachineInstr *MI, |
| PPCVSXSwapEntry& SwapEntry) { |
| SwapEntry.VSEMI = MI; |
| SwapEntry.VSEId = SwapVector.size(); |
| SwapVector.push_back(SwapEntry); |
| EC->insert(SwapEntry.VSEId); |
| SwapMap[MI] = SwapEntry.VSEId; |
| return SwapEntry.VSEId; |
| } |
| |
| // This is used to find the "true" source register for an |
| // XXPERMDI instruction, since MachineCSE does not handle the |
| // "copy-like" operations (Copy and SubregToReg). Returns |
| // the original SrcReg unless it is the target of a copy-like |
| // operation, in which case we chain backwards through all |
| // such operations to the ultimate source register. If a |
| // physical register is encountered, we stop the search and |
| // flag the swap entry indicated by VecIdx (the original |
| // XXPERMDI) as mentioning a physical register. |
| unsigned PPCVSXSwapRemoval::lookThruCopyLike(unsigned SrcReg, |
| unsigned VecIdx) { |
| MachineInstr *MI = MRI->getVRegDef(SrcReg); |
| if (!MI->isCopyLike()) |
| return SrcReg; |
| |
| unsigned CopySrcReg; |
| if (MI->isCopy()) |
| CopySrcReg = MI->getOperand(1).getReg(); |
| else { |
| assert(MI->isSubregToReg() && "bad opcode for lookThruCopyLike"); |
| CopySrcReg = MI->getOperand(2).getReg(); |
| } |
| |
| if (!TargetRegisterInfo::isVirtualRegister(CopySrcReg)) { |
| if (!isScalarVecReg(CopySrcReg)) |
| SwapVector[VecIdx].MentionsPhysVR = 1; |
| return CopySrcReg; |
| } |
| |
| return lookThruCopyLike(CopySrcReg, VecIdx); |
| } |
| |
| // Generate equivalence classes for related computations (webs) by |
| // def-use relationships of virtual registers. Mention of a physical |
| // register terminates the generation of equivalence classes as this |
| // indicates a use of a parameter, definition of a return value, use |
| // of a value returned from a call, or definition of a parameter to a |
| // call. Computations with physical register mentions are flagged |
| // as such so their containing webs will not be optimized. |
| void PPCVSXSwapRemoval::formWebs() { |
| |
| LLVM_DEBUG(dbgs() << "\n*** Forming webs for swap removal ***\n\n"); |
| |
| for (unsigned EntryIdx = 0; EntryIdx < SwapVector.size(); ++EntryIdx) { |
| |
| MachineInstr *MI = SwapVector[EntryIdx].VSEMI; |
| |
| LLVM_DEBUG(dbgs() << "\n" << SwapVector[EntryIdx].VSEId << " "); |
| LLVM_DEBUG(MI->dump()); |
| |
| // It's sufficient to walk vector uses and join them to their unique |
| // definitions. In addition, check full vector register operands |
| // for physical regs. We exclude partial-vector register operands |
| // because we can handle them if copied to a full vector. |
| for (const MachineOperand &MO : MI->operands()) { |
| if (!MO.isReg()) |
| continue; |
| |
| unsigned Reg = MO.getReg(); |
| if (!isVecReg(Reg) && !isScalarVecReg(Reg)) |
| continue; |
| |
| if (!TargetRegisterInfo::isVirtualRegister(Reg)) { |
| if (!(MI->isCopy() && isScalarVecReg(Reg))) |
| SwapVector[EntryIdx].MentionsPhysVR = 1; |
| continue; |
| } |
| |
| if (!MO.isUse()) |
| continue; |
| |
| MachineInstr* DefMI = MRI->getVRegDef(Reg); |
| assert(SwapMap.find(DefMI) != SwapMap.end() && |
| "Inconsistency: def of vector reg not found in swap map!"); |
| int DefIdx = SwapMap[DefMI]; |
| (void)EC->unionSets(SwapVector[DefIdx].VSEId, |
| SwapVector[EntryIdx].VSEId); |
| |
| LLVM_DEBUG(dbgs() << format("Unioning %d with %d\n", |
| SwapVector[DefIdx].VSEId, |
| SwapVector[EntryIdx].VSEId)); |
| LLVM_DEBUG(dbgs() << " Def: "); |
| LLVM_DEBUG(DefMI->dump()); |
| } |
| } |
| } |
| |
| // Walk the swap vector entries looking for conditions that prevent their |
| // containing computations from being optimized. When such conditions are |
| // found, mark the representative of the computation's equivalence class |
| // as rejected. |
| void PPCVSXSwapRemoval::recordUnoptimizableWebs() { |
| |
| LLVM_DEBUG(dbgs() << "\n*** Rejecting webs for swap removal ***\n\n"); |
| |
| for (unsigned EntryIdx = 0; EntryIdx < SwapVector.size(); ++EntryIdx) { |
| int Repr = EC->getLeaderValue(SwapVector[EntryIdx].VSEId); |
| |
| // If representative is already rejected, don't waste further time. |
| if (SwapVector[Repr].WebRejected) |
| continue; |
| |
| // Reject webs containing mentions of physical or partial registers, or |
| // containing operations that we don't know how to handle in a lane- |
| // permuted region. |
| if (SwapVector[EntryIdx].MentionsPhysVR || |
| SwapVector[EntryIdx].MentionsPartialVR || |
| !(SwapVector[EntryIdx].IsSwappable || SwapVector[EntryIdx].IsSwap)) { |
| |
| SwapVector[Repr].WebRejected = 1; |
| |
| LLVM_DEBUG( |
| dbgs() << format("Web %d rejected for physreg, partial reg, or not " |
| "swap[pable]\n", |
| Repr)); |
| LLVM_DEBUG(dbgs() << " in " << EntryIdx << ": "); |
| LLVM_DEBUG(SwapVector[EntryIdx].VSEMI->dump()); |
| LLVM_DEBUG(dbgs() << "\n"); |
| } |
| |
| // Reject webs than contain swapping loads that feed something other |
| // than a swap instruction. |
| else if (SwapVector[EntryIdx].IsLoad && SwapVector[EntryIdx].IsSwap) { |
| MachineInstr *MI = SwapVector[EntryIdx].VSEMI; |
| unsigned DefReg = MI->getOperand(0).getReg(); |
| |
| // We skip debug instructions in the analysis. (Note that debug |
| // location information is still maintained by this optimization |
| // because it remains on the LXVD2X and STXVD2X instructions after |
| // the XXPERMDIs are removed.) |
| for (MachineInstr &UseMI : MRI->use_nodbg_instructions(DefReg)) { |
| int UseIdx = SwapMap[&UseMI]; |
| |
| if (!SwapVector[UseIdx].IsSwap || SwapVector[UseIdx].IsLoad || |
| SwapVector[UseIdx].IsStore) { |
| |
| SwapVector[Repr].WebRejected = 1; |
| |
| LLVM_DEBUG(dbgs() << format( |
| "Web %d rejected for load not feeding swap\n", Repr)); |
| LLVM_DEBUG(dbgs() << " def " << EntryIdx << ": "); |
| LLVM_DEBUG(MI->dump()); |
| LLVM_DEBUG(dbgs() << " use " << UseIdx << ": "); |
| LLVM_DEBUG(UseMI.dump()); |
| LLVM_DEBUG(dbgs() << "\n"); |
| } |
| } |
| |
| // Reject webs that contain swapping stores that are fed by something |
| // other than a swap instruction. |
| } else if (SwapVector[EntryIdx].IsStore && SwapVector[EntryIdx].IsSwap) { |
| MachineInstr *MI = SwapVector[EntryIdx].VSEMI; |
| unsigned UseReg = MI->getOperand(0).getReg(); |
| MachineInstr *DefMI = MRI->getVRegDef(UseReg); |
| unsigned DefReg = DefMI->getOperand(0).getReg(); |
| int DefIdx = SwapMap[DefMI]; |
| |
| if (!SwapVector[DefIdx].IsSwap || SwapVector[DefIdx].IsLoad || |
| SwapVector[DefIdx].IsStore) { |
| |
| SwapVector[Repr].WebRejected = 1; |
| |
| LLVM_DEBUG(dbgs() << format( |
| "Web %d rejected for store not fed by swap\n", Repr)); |
| LLVM_DEBUG(dbgs() << " def " << DefIdx << ": "); |
| LLVM_DEBUG(DefMI->dump()); |
| LLVM_DEBUG(dbgs() << " use " << EntryIdx << ": "); |
| LLVM_DEBUG(MI->dump()); |
| LLVM_DEBUG(dbgs() << "\n"); |
| } |
| |
| // Ensure all uses of the register defined by DefMI feed store |
| // instructions |
| for (MachineInstr &UseMI : MRI->use_nodbg_instructions(DefReg)) { |
| int UseIdx = SwapMap[&UseMI]; |
| |
| if (SwapVector[UseIdx].VSEMI->getOpcode() != MI->getOpcode()) { |
| SwapVector[Repr].WebRejected = 1; |
| |
| LLVM_DEBUG( |
| dbgs() << format( |
| "Web %d rejected for swap not feeding only stores\n", Repr)); |
| LLVM_DEBUG(dbgs() << " def " |
| << " : "); |
| LLVM_DEBUG(DefMI->dump()); |
| LLVM_DEBUG(dbgs() << " use " << UseIdx << ": "); |
| LLVM_DEBUG(SwapVector[UseIdx].VSEMI->dump()); |
| LLVM_DEBUG(dbgs() << "\n"); |
| } |
| } |
| } |
| } |
| |
| LLVM_DEBUG(dbgs() << "Swap vector after web analysis:\n\n"); |
| LLVM_DEBUG(dumpSwapVector()); |
| } |
| |
| // Walk the swap vector entries looking for swaps fed by permuting loads |
| // and swaps that feed permuting stores. If the containing computation |
| // has not been marked rejected, mark each such swap for removal. |
| // (Removal is delayed in case optimization has disturbed the pattern, |
| // such that multiple loads feed the same swap, etc.) |
| void PPCVSXSwapRemoval::markSwapsForRemoval() { |
| |
| LLVM_DEBUG(dbgs() << "\n*** Marking swaps for removal ***\n\n"); |
| |
| for (unsigned EntryIdx = 0; EntryIdx < SwapVector.size(); ++EntryIdx) { |
| |
| if (SwapVector[EntryIdx].IsLoad && SwapVector[EntryIdx].IsSwap) { |
| int Repr = EC->getLeaderValue(SwapVector[EntryIdx].VSEId); |
| |
| if (!SwapVector[Repr].WebRejected) { |
| MachineInstr *MI = SwapVector[EntryIdx].VSEMI; |
| unsigned DefReg = MI->getOperand(0).getReg(); |
| |
| for (MachineInstr &UseMI : MRI->use_nodbg_instructions(DefReg)) { |
| int UseIdx = SwapMap[&UseMI]; |
| SwapVector[UseIdx].WillRemove = 1; |
| |
| LLVM_DEBUG(dbgs() << "Marking swap fed by load for removal: "); |
| LLVM_DEBUG(UseMI.dump()); |
| } |
| } |
| |
| } else if (SwapVector[EntryIdx].IsStore && SwapVector[EntryIdx].IsSwap) { |
| int Repr = EC->getLeaderValue(SwapVector[EntryIdx].VSEId); |
| |
| if (!SwapVector[Repr].WebRejected) { |
| MachineInstr *MI = SwapVector[EntryIdx].VSEMI; |
| unsigned UseReg = MI->getOperand(0).getReg(); |
| MachineInstr *DefMI = MRI->getVRegDef(UseReg); |
| int DefIdx = SwapMap[DefMI]; |
| SwapVector[DefIdx].WillRemove = 1; |
| |
| LLVM_DEBUG(dbgs() << "Marking swap feeding store for removal: "); |
| LLVM_DEBUG(DefMI->dump()); |
| } |
| |
| } else if (SwapVector[EntryIdx].IsSwappable && |
| SwapVector[EntryIdx].SpecialHandling != 0) { |
| int Repr = EC->getLeaderValue(SwapVector[EntryIdx].VSEId); |
| |
| if (!SwapVector[Repr].WebRejected) |
| handleSpecialSwappables(EntryIdx); |
| } |
| } |
| } |
| |
| // Create an xxswapd instruction and insert it prior to the given point. |
| // MI is used to determine basic block and debug loc information. |
| // FIXME: When inserting a swap, we should check whether SrcReg is |
| // defined by another swap: SrcReg = XXPERMDI Reg, Reg, 2; If so, |
| // then instead we should generate a copy from Reg to DstReg. |
| void PPCVSXSwapRemoval::insertSwap(MachineInstr *MI, |
| MachineBasicBlock::iterator InsertPoint, |
| unsigned DstReg, unsigned SrcReg) { |
| BuildMI(*MI->getParent(), InsertPoint, MI->getDebugLoc(), |
| TII->get(PPC::XXPERMDI), DstReg) |
| .addReg(SrcReg) |
| .addReg(SrcReg) |
| .addImm(2); |
| } |
| |
| // The identified swap entry requires special handling to allow its |
| // containing computation to be optimized. Perform that handling |
| // here. |
| // FIXME: Additional opportunities will be phased in with subsequent |
| // patches. |
| void PPCVSXSwapRemoval::handleSpecialSwappables(int EntryIdx) { |
| switch (SwapVector[EntryIdx].SpecialHandling) { |
| |
| default: |
| llvm_unreachable("Unexpected special handling type"); |
| |
| // For splats based on an index into a vector, add N/2 modulo N |
| // to the index, where N is the number of vector elements. |
| case SHValues::SH_SPLAT: { |
| MachineInstr *MI = SwapVector[EntryIdx].VSEMI; |
| unsigned NElts; |
| |
| LLVM_DEBUG(dbgs() << "Changing splat: "); |
| LLVM_DEBUG(MI->dump()); |
| |
| switch (MI->getOpcode()) { |
| default: |
| llvm_unreachable("Unexpected splat opcode"); |
| case PPC::VSPLTB: NElts = 16; break; |
| case PPC::VSPLTH: NElts = 8; break; |
| case PPC::VSPLTW: |
| case PPC::XXSPLTW: NElts = 4; break; |
| } |
| |
| unsigned EltNo; |
| if (MI->getOpcode() == PPC::XXSPLTW) |
| EltNo = MI->getOperand(2).getImm(); |
| else |
| EltNo = MI->getOperand(1).getImm(); |
| |
| EltNo = (EltNo + NElts / 2) % NElts; |
| if (MI->getOpcode() == PPC::XXSPLTW) |
| MI->getOperand(2).setImm(EltNo); |
| else |
| MI->getOperand(1).setImm(EltNo); |
| |
| LLVM_DEBUG(dbgs() << " Into: "); |
| LLVM_DEBUG(MI->dump()); |
| break; |
| } |
| |
| // For an XXPERMDI that isn't handled otherwise, we need to |
| // reverse the order of the operands. If the selector operand |
| // has a value of 0 or 3, we need to change it to 3 or 0, |
| // respectively. Otherwise we should leave it alone. (This |
| // is equivalent to reversing the two bits of the selector |
| // operand and complementing the result.) |
| case SHValues::SH_XXPERMDI: { |
| MachineInstr *MI = SwapVector[EntryIdx].VSEMI; |
| |
| LLVM_DEBUG(dbgs() << "Changing XXPERMDI: "); |
| LLVM_DEBUG(MI->dump()); |
| |
| unsigned Selector = MI->getOperand(3).getImm(); |
| if (Selector == 0 || Selector == 3) |
| Selector = 3 - Selector; |
| MI->getOperand(3).setImm(Selector); |
| |
| unsigned Reg1 = MI->getOperand(1).getReg(); |
| unsigned Reg2 = MI->getOperand(2).getReg(); |
| MI->getOperand(1).setReg(Reg2); |
| MI->getOperand(2).setReg(Reg1); |
| |
| // We also need to swap kill flag associated with the register. |
| bool IsKill1 = MI->getOperand(1).isKill(); |
| bool IsKill2 = MI->getOperand(2).isKill(); |
| MI->getOperand(1).setIsKill(IsKill2); |
| MI->getOperand(2).setIsKill(IsKill1); |
| |
| LLVM_DEBUG(dbgs() << " Into: "); |
| LLVM_DEBUG(MI->dump()); |
| break; |
| } |
| |
| // For a copy from a scalar floating-point register to a vector |
| // register, removing swaps will leave the copied value in the |
| // wrong lane. Insert a swap following the copy to fix this. |
| case SHValues::SH_COPYWIDEN: { |
| MachineInstr *MI = SwapVector[EntryIdx].VSEMI; |
| |
| LLVM_DEBUG(dbgs() << "Changing SUBREG_TO_REG: "); |
| LLVM_DEBUG(MI->dump()); |
| |
| unsigned DstReg = MI->getOperand(0).getReg(); |
| const TargetRegisterClass *DstRC = MRI->getRegClass(DstReg); |
| unsigned NewVReg = MRI->createVirtualRegister(DstRC); |
| |
| MI->getOperand(0).setReg(NewVReg); |
| LLVM_DEBUG(dbgs() << " Into: "); |
| LLVM_DEBUG(MI->dump()); |
| |
| auto InsertPoint = ++MachineBasicBlock::iterator(MI); |
| |
| // Note that an XXPERMDI requires a VSRC, so if the SUBREG_TO_REG |
| // is copying to a VRRC, we need to be careful to avoid a register |
| // assignment problem. In this case we must copy from VRRC to VSRC |
| // prior to the swap, and from VSRC to VRRC following the swap. |
| // Coalescing will usually remove all this mess. |
| if (DstRC == &PPC::VRRCRegClass) { |
| unsigned VSRCTmp1 = MRI->createVirtualRegister(&PPC::VSRCRegClass); |
| unsigned VSRCTmp2 = MRI->createVirtualRegister(&PPC::VSRCRegClass); |
| |
| BuildMI(*MI->getParent(), InsertPoint, MI->getDebugLoc(), |
| TII->get(PPC::COPY), VSRCTmp1) |
| .addReg(NewVReg); |
| LLVM_DEBUG(std::prev(InsertPoint)->dump()); |
| |
| insertSwap(MI, InsertPoint, VSRCTmp2, VSRCTmp1); |
| LLVM_DEBUG(std::prev(InsertPoint)->dump()); |
| |
| BuildMI(*MI->getParent(), InsertPoint, MI->getDebugLoc(), |
| TII->get(PPC::COPY), DstReg) |
| .addReg(VSRCTmp2); |
| LLVM_DEBUG(std::prev(InsertPoint)->dump()); |
| |
| } else { |
| insertSwap(MI, InsertPoint, DstReg, NewVReg); |
| LLVM_DEBUG(std::prev(InsertPoint)->dump()); |
| } |
| break; |
| } |
| } |
| } |
| |
| // Walk the swap vector and replace each entry marked for removal with |
| // a copy operation. |
| bool PPCVSXSwapRemoval::removeSwaps() { |
| |
| LLVM_DEBUG(dbgs() << "\n*** Removing swaps ***\n\n"); |
| |
| bool Changed = false; |
| |
| for (unsigned EntryIdx = 0; EntryIdx < SwapVector.size(); ++EntryIdx) { |
| if (SwapVector[EntryIdx].WillRemove) { |
| Changed = true; |
| MachineInstr *MI = SwapVector[EntryIdx].VSEMI; |
| MachineBasicBlock *MBB = MI->getParent(); |
| BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(TargetOpcode::COPY), |
| MI->getOperand(0).getReg()) |
| .add(MI->getOperand(1)); |
| |
| LLVM_DEBUG(dbgs() << format("Replaced %d with copy: ", |
| SwapVector[EntryIdx].VSEId)); |
| LLVM_DEBUG(MI->dump()); |
| |
| MI->eraseFromParent(); |
| } |
| } |
| |
| return Changed; |
| } |
| |
| #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) |
| // For debug purposes, dump the contents of the swap vector. |
| LLVM_DUMP_METHOD void PPCVSXSwapRemoval::dumpSwapVector() { |
| |
| for (unsigned EntryIdx = 0; EntryIdx < SwapVector.size(); ++EntryIdx) { |
| |
| MachineInstr *MI = SwapVector[EntryIdx].VSEMI; |
| int ID = SwapVector[EntryIdx].VSEId; |
| |
| dbgs() << format("%6d", ID); |
| dbgs() << format("%6d", EC->getLeaderValue(ID)); |
| dbgs() << format(" %bb.%3d", MI->getParent()->getNumber()); |
| dbgs() << format(" %14s ", TII->getName(MI->getOpcode()).str().c_str()); |
| |
| if (SwapVector[EntryIdx].IsLoad) |
| dbgs() << "load "; |
| if (SwapVector[EntryIdx].IsStore) |
| dbgs() << "store "; |
| if (SwapVector[EntryIdx].IsSwap) |
| dbgs() << "swap "; |
| if (SwapVector[EntryIdx].MentionsPhysVR) |
| dbgs() << "physreg "; |
| if (SwapVector[EntryIdx].MentionsPartialVR) |
| dbgs() << "partialreg "; |
| |
| if (SwapVector[EntryIdx].IsSwappable) { |
| dbgs() << "swappable "; |
| switch(SwapVector[EntryIdx].SpecialHandling) { |
| default: |
| dbgs() << "special:**unknown**"; |
| break; |
| case SH_NONE: |
| break; |
| case SH_EXTRACT: |
| dbgs() << "special:extract "; |
| break; |
| case SH_INSERT: |
| dbgs() << "special:insert "; |
| break; |
| case SH_NOSWAP_LD: |
| dbgs() << "special:load "; |
| break; |
| case SH_NOSWAP_ST: |
| dbgs() << "special:store "; |
| break; |
| case SH_SPLAT: |
| dbgs() << "special:splat "; |
| break; |
| case SH_XXPERMDI: |
| dbgs() << "special:xxpermdi "; |
| break; |
| case SH_COPYWIDEN: |
| dbgs() << "special:copywiden "; |
| break; |
| } |
| } |
| |
| if (SwapVector[EntryIdx].WebRejected) |
| dbgs() << "rejected "; |
| if (SwapVector[EntryIdx].WillRemove) |
| dbgs() << "remove "; |
| |
| dbgs() << "\n"; |
| |
| // For no-asserts builds. |
| (void)MI; |
| (void)ID; |
| } |
| |
| dbgs() << "\n"; |
| } |
| #endif |
| |
| } // end default namespace |
| |
| INITIALIZE_PASS_BEGIN(PPCVSXSwapRemoval, DEBUG_TYPE, |
| "PowerPC VSX Swap Removal", false, false) |
| INITIALIZE_PASS_END(PPCVSXSwapRemoval, DEBUG_TYPE, |
| "PowerPC VSX Swap Removal", false, false) |
| |
| char PPCVSXSwapRemoval::ID = 0; |
| FunctionPass* |
| llvm::createPPCVSXSwapRemovalPass() { return new PPCVSXSwapRemoval(); } |
