| //===-- FLATInstructions.td - FLAT Instruction Definitions ----------------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| |
| let WantsRoot = true in { |
| def FlatOffset : ComplexPattern<iPTR, 2, "SelectFlatOffset", [], [], -10>; |
| def GlobalOffset : ComplexPattern<iPTR, 2, "SelectGlobalOffset", [], [], -10>; |
| def ScratchOffset : ComplexPattern<iPTR, 2, "SelectScratchOffset", [], [], -10>; |
| |
| def GlobalSAddr : ComplexPattern<iPTR, 3, "SelectGlobalSAddr", [], [], -10>; |
| def ScratchSAddr : ComplexPattern<iPTR, 2, "SelectScratchSAddr", [], [], -10>; |
| def ScratchSVAddr : ComplexPattern<iPTR, 3, "SelectScratchSVAddr", [], [], -10>; |
| } |
| |
| class True16D16Table <string hiOp, string loOp> { |
| Instruction T16Op = !cast<Instruction>(NAME); |
| Instruction HiOp = !cast<Instruction>(hiOp); |
| Instruction LoOp = !cast<Instruction>(loOp); |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // FLAT classes |
| //===----------------------------------------------------------------------===// |
| |
| class FLAT_Pseudo<string opName, dag outs, dag ins, |
| string asmOps, list<dag> pattern=[]> : |
| InstSI<outs, ins, "", pattern>, |
| SIMCInstr<NAME, SIEncodingFamily.NONE> { |
| |
| let isPseudo = 1; |
| let isCodeGenOnly = 1; |
| |
| let FLAT = 1; |
| |
| let UseNamedOperandTable = 1; |
| let hasSideEffects = 0; |
| let SchedRW = [WriteVMEM]; |
| |
| string Mnemonic = opName; |
| string AsmOperands = asmOps; |
| |
| bits<1> is_flat_global = 0; |
| bits<1> is_flat_scratch = 0; |
| |
| bits<1> has_vdst = 1; |
| |
| // We need to distinguish having saddr and enabling saddr because |
| // saddr is only valid for scratch and global instructions. Pre-gfx9 |
| // these bits were reserved, so we also don't necessarily want to |
| // set these bits to the disabled value for the original flat |
| // segment instructions. |
| bits<1> has_saddr = 0; |
| bits<1> enabled_saddr = 0; |
| bits<7> saddr_value = 0; |
| bits<1> has_vaddr = 1; |
| |
| bits<1> has_data = 1; |
| bits<1> has_glc = 1; |
| bits<1> glcValue = 0; |
| bits<1> has_dlc = 1; |
| bits<1> dlcValue = 0; |
| bits<1> has_sccb = 1; |
| bits<1> sccbValue = 0; |
| bits<1> has_sve = 0; // Scratch VGPR Enable |
| bits<1> lds = 0; |
| bits<1> sve = 0; |
| bits<1> has_offset = 1; |
| |
| let SubtargetPredicate = !if(is_flat_global, HasFlatGlobalInsts, |
| !if(is_flat_scratch, HasFlatScratchInsts, HasFlatAddressSpace)); |
| |
| // TODO: M0 if it could possibly access LDS (before gfx9? only)? |
| let Uses = !if(is_flat_global, [EXEC], [EXEC, FLAT_SCR]); |
| |
| // Internally, FLAT instruction are executed as both an LDS and a |
| // Buffer instruction; so, they increment both VM_CNT and LGKM_CNT |
| // and are not considered done until both have been decremented. |
| let VM_CNT = 1; |
| let LGKM_CNT = !not(!or(is_flat_global, is_flat_scratch)); |
| |
| let FlatGlobal = is_flat_global; |
| |
| let FlatScratch = is_flat_scratch; |
| } |
| |
| class FLAT_Real <bits<7> op, FLAT_Pseudo ps, string opName = ps.Mnemonic> : |
| InstSI <ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands, []>, |
| Enc64 { |
| |
| let isPseudo = 0; |
| let isCodeGenOnly = 0; |
| |
| let FLAT = 1; |
| |
| // copy relevant pseudo op flags |
| let SubtargetPredicate = ps.SubtargetPredicate; |
| let AsmMatchConverter = ps.AsmMatchConverter; |
| let OtherPredicates = ps.OtherPredicates; |
| let TSFlags = ps.TSFlags; |
| let UseNamedOperandTable = ps.UseNamedOperandTable; |
| let SchedRW = ps.SchedRW; |
| let mayLoad = ps.mayLoad; |
| let mayStore = ps.mayStore; |
| let IsAtomicRet = ps.IsAtomicRet; |
| let IsAtomicNoRet = ps.IsAtomicNoRet; |
| let VM_CNT = ps.VM_CNT; |
| let LGKM_CNT = ps.LGKM_CNT; |
| let VALU = ps.VALU; |
| let Uses = ps.Uses; |
| let Defs = ps.Defs; |
| let isConvergent = ps.isConvergent; |
| |
| // encoding fields |
| bits<8> vaddr; |
| bits<10> vdata; |
| bits<7> saddr; |
| bits<10> vdst; |
| |
| bits<5> cpol; |
| |
| // Only valid on gfx9 |
| bits<1> lds = ps.lds; // LDS DMA for global and scratch |
| |
| // Segment, 00=flat, 01=scratch, 10=global, 11=reserved |
| bits<2> seg = {ps.is_flat_global, ps.is_flat_scratch}; |
| |
| // Signed offset. Highest bit ignored for flat and treated as 12-bit |
| // unsigned for flat accesses. |
| bits<13> offset; |
| // GFX90A+ only: instruction uses AccVGPR for data |
| bits<1> acc = !if(ps.has_vdst, vdst{9}, !if(ps.has_data, vdata{9}, 0)); |
| |
| // We don't use tfe right now, and it was removed in gfx9. |
| bits<1> tfe = 0; |
| |
| // Only valid on GFX9+ |
| let Inst{12-0} = offset; |
| let Inst{13} = !if(ps.has_sve, ps.sve, lds); |
| let Inst{15-14} = seg; |
| |
| let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glcValue); |
| let Inst{17} = cpol{CPolBit.SLC}; |
| let Inst{24-18} = op; |
| let Inst{31-26} = 0x37; // Encoding. |
| let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); |
| let Inst{47-40} = !if(ps.has_data, vdata{7-0}, ?); |
| let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7f), 0); |
| |
| // 54-48 is reserved. |
| let Inst{55} = acc; // nv on GFX9+, TFE before. AccVGPR for data on GFX90A. |
| let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, ?); |
| } |
| |
| class VFLAT_Real <bits<8> op, FLAT_Pseudo ps, string opName = ps.Mnemonic> : |
| InstSI <ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands, []>, |
| Enc96 { |
| |
| let FLAT = 1; |
| |
| // copy relevant pseudo op flags |
| let SubtargetPredicate = ps.SubtargetPredicate; |
| let WaveSizePredicate = ps.WaveSizePredicate; |
| let AsmMatchConverter = ps.AsmMatchConverter; |
| let OtherPredicates = ps.OtherPredicates; |
| let TSFlags = ps.TSFlags; |
| let UseNamedOperandTable = ps.UseNamedOperandTable; |
| let SchedRW = ps.SchedRW; |
| let mayLoad = ps.mayLoad; |
| let mayStore = ps.mayStore; |
| let IsAtomicRet = ps.IsAtomicRet; |
| let IsAtomicNoRet = ps.IsAtomicNoRet; |
| let VM_CNT = ps.VM_CNT; |
| let LGKM_CNT = ps.LGKM_CNT; |
| let VALU = ps.VALU; |
| let Uses = ps.Uses; |
| let Defs = ps.Defs; |
| let isConvergent = ps.isConvergent; |
| |
| bits<7> saddr; |
| bits<8> vdst; |
| bits<6> cpol; |
| bits<8> vdata; // vsrc |
| bits<8> vaddr; |
| bits<24> offset; |
| |
| let Inst{6-0} = !if(ps.enabled_saddr, saddr, SGPR_NULL_gfx11plus.Index); |
| let Inst{21-14} = op; |
| let Inst{31-26} = 0x3b; |
| let Inst{39-32} = !if(ps.has_vdst, vdst, ?); |
| let Inst{49} = ps.sve; |
| let Inst{54-53} = cpol{2-1}; // th{2-1} |
| let Inst{52} = !if(ps.IsAtomicRet, 1, cpol{0}); // th{0} |
| let Inst{51-50} = cpol{4-3}; // scope |
| let Inst{62-55} = !if(ps.has_data, vdata{7-0}, ?); |
| let Inst{71-64} = !if(ps.has_vaddr, vaddr, ?); |
| let Inst{95-72} = !if(ps.has_offset, offset, ?); |
| } |
| |
| class GlobalSaddrTable <bit is_saddr, string Name = ""> { |
| bit IsSaddr = is_saddr; |
| string SaddrOp = Name; |
| } |
| |
| // TODO: Is exec allowed for saddr? The disabled value 0x7f is the |
| // same encoding value as exec_hi, so it isn't possible to use that if |
| // saddr is 32-bit (which isn't handled here yet). |
| class FLAT_Load_Pseudo <string opName, RegisterClass regClass, |
| bit HasTiedOutput = 0, |
| bit HasSaddr = 0, bit EnableSaddr = 0, |
| RegisterOperand vdata_op = getLdStRegisterOperand<regClass>.ret> : FLAT_Pseudo< |
| opName, |
| (outs vdata_op:$vdst), |
| !con( |
| !if(EnableSaddr, |
| (ins SReg_64_XEXEC_XNULL:$saddr, VGPR_32:$vaddr), |
| (ins VReg_64:$vaddr)), |
| (ins flat_offset:$offset), |
| // FIXME: Operands with default values do not work with following non-optional operands. |
| !if(HasTiedOutput, (ins CPol:$cpol, vdata_op:$vdst_in), |
| (ins CPol_0:$cpol))), |
| " $vdst, $vaddr"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$cpol"> { |
| let has_data = 0; |
| let mayLoad = 1; |
| let has_saddr = HasSaddr; |
| let enabled_saddr = EnableSaddr; |
| |
| let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); |
| let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); |
| } |
| |
| multiclass FLAT_Load_Pseudo_t16<string opName> { |
| def "" : FLAT_Load_Pseudo<opName, VGPR_32, 1>; |
| let True16Predicate = UseRealTrue16Insts in |
| def _t16 : FLAT_Load_Pseudo<opName#"_t16", VGPR_16>, True16D16Table<NAME#"_HI", NAME>; |
| } |
| |
| class FLAT_Store_Pseudo <string opName, RegisterClass vdataClass, |
| bit HasSaddr = 0, bit EnableSaddr = 0> : FLAT_Pseudo< |
| opName, |
| (outs), |
| !con( |
| !if(EnableSaddr, |
| (ins VGPR_32:$vaddr, getLdStRegisterOperand<vdataClass>.ret:$vdata, SReg_64_XEXEC_XNULL:$saddr), |
| (ins VReg_64:$vaddr, getLdStRegisterOperand<vdataClass>.ret:$vdata)), |
| (ins flat_offset:$offset, CPol_0:$cpol)), |
| " $vaddr, $vdata"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$cpol"> { |
| let mayLoad = 0; |
| let mayStore = 1; |
| let has_vdst = 0; |
| let has_saddr = HasSaddr; |
| let enabled_saddr = EnableSaddr; |
| } |
| |
| multiclass FLAT_Store_Pseudo_t16<string opName> { |
| def "" : FLAT_Store_Pseudo<opName, VGPR_32>; |
| let OtherPredicates = [HasTrue16BitInsts] in |
| def _t16 : FLAT_Store_Pseudo<opName#"_t16", VGPR_16>, True16D16Table<NAME#"_D16_HI", NAME>; |
| } |
| |
| multiclass FLAT_Global_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedInput = 0> { |
| let is_flat_global = 1 in { |
| def "" : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1>, |
| GlobalSaddrTable<0, opName>; |
| def _SADDR : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1, 1>, |
| GlobalSaddrTable<1, opName>; |
| } |
| } |
| |
| multiclass FLAT_Global_Load_Pseudo_t16<string opName> { |
| defm "" : FLAT_Global_Load_Pseudo<opName, VGPR_32, 1>; |
| |
| defvar Name16 = opName#"_t16"; |
| let OtherPredicates = [HasTrue16BitInsts], |
| SubtargetPredicate = HasFlatGlobalInsts, is_flat_global = 1 in { |
| def _t16 : FLAT_Load_Pseudo<Name16, VGPR_16, 0, 1>, |
| GlobalSaddrTable<0, Name16>, |
| True16D16Table<NAME#"_HI", NAME>; |
| def _SADDR_t16 : FLAT_Load_Pseudo<Name16, VGPR_16, 0, 1, 1>, |
| GlobalSaddrTable<1, Name16>, |
| True16D16Table<NAME#"_HI_SADDR", NAME#"_SADDR">; |
| } |
| } |
| |
| class FLAT_Global_Load_AddTid_Pseudo <string opName, RegisterClass regClass, |
| bit HasTiedOutput = 0, bit EnableSaddr = 0> : FLAT_Pseudo< |
| opName, |
| (outs regClass:$vdst), |
| !con(!if(EnableSaddr, (ins SReg_64:$saddr), (ins)), |
| (ins flat_offset:$offset, CPol_0:$cpol), |
| !if(HasTiedOutput, (ins regClass:$vdst_in), (ins))), |
| " $vdst, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { |
| let is_flat_global = 1; |
| let has_data = 0; |
| let mayLoad = 1; |
| let has_vaddr = 0; |
| let has_saddr = 1; |
| let enabled_saddr = EnableSaddr; |
| |
| let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); |
| let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); |
| } |
| |
| multiclass FLAT_Global_Load_AddTid_Pseudo<string opName, RegisterClass regClass, |
| bit HasTiedOutput = 0> { |
| def "" : FLAT_Global_Load_AddTid_Pseudo<opName, regClass, HasTiedOutput>, |
| GlobalSaddrTable<0, opName>; |
| def _SADDR : FLAT_Global_Load_AddTid_Pseudo<opName, regClass, HasTiedOutput, 1>, |
| GlobalSaddrTable<1, opName>; |
| } |
| |
| multiclass FLAT_Global_Store_Pseudo<string opName, RegisterClass regClass> { |
| let is_flat_global = 1 in { |
| def "" : FLAT_Store_Pseudo<opName, regClass, 1>, |
| GlobalSaddrTable<0, opName>; |
| def _SADDR : FLAT_Store_Pseudo<opName, regClass, 1, 1>, |
| GlobalSaddrTable<1, opName>; |
| } |
| } |
| |
| multiclass FLAT_Global_Store_Pseudo_t16<string opName> { |
| defm "" : FLAT_Global_Store_Pseudo<opName, VGPR_32>; |
| |
| defvar Name16 = opName#"_t16"; |
| let OtherPredicates = [HasTrue16BitInsts], |
| SubtargetPredicate = HasFlatGlobalInsts, is_flat_global = 1 in { |
| def _t16 : FLAT_Store_Pseudo<Name16, VGPR_16, 1>, |
| GlobalSaddrTable<0, Name16>, |
| True16D16Table<NAME#"_D16_HI", NAME>; |
| def _SADDR_t16 : FLAT_Store_Pseudo<Name16, VGPR_16, 1, 1>, |
| GlobalSaddrTable<1, Name16>, |
| True16D16Table<NAME#"_D16_HI_SADDR", NAME#"_SADDR">; |
| } |
| } |
| |
| class FLAT_Global_Load_LDS_Pseudo <string opName, bit EnableSaddr = 0> : FLAT_Pseudo< |
| opName, |
| (outs ), |
| !con( |
| !if(EnableSaddr, (ins SReg_64:$saddr, VGPR_32:$vaddr), (ins VReg_64:$vaddr)), |
| (ins flat_offset:$offset, CPol_0:$cpol)), |
| " $vaddr"#!if(EnableSaddr, ", $saddr", ", off")#"$offset$cpol"> { |
| let LGKM_CNT = 1; |
| let is_flat_global = 1; |
| let lds = 1; |
| let has_data = 0; |
| let has_vdst = 0; |
| let mayLoad = 1; |
| let mayStore = 1; |
| let has_saddr = 1; |
| let enabled_saddr = EnableSaddr; |
| let VALU = 1; |
| let Uses = [M0, EXEC]; |
| let SchedRW = [WriteVMEM, WriteLDS]; |
| } |
| |
| multiclass FLAT_Global_Load_LDS_Pseudo<string opName> { |
| def "" : FLAT_Global_Load_LDS_Pseudo<opName>, |
| GlobalSaddrTable<0, opName>; |
| def _SADDR : FLAT_Global_Load_LDS_Pseudo<opName, 1>, |
| GlobalSaddrTable<1, opName>; |
| } |
| |
| class FLAT_Global_Store_AddTid_Pseudo <string opName, RegisterClass vdataClass, |
| bit EnableSaddr = 0> : FLAT_Pseudo< |
| opName, |
| (outs), |
| !con(!if(EnableSaddr, (ins vdataClass:$vdata, SReg_64:$saddr), (ins vdataClass:$vdata)), |
| (ins flat_offset:$offset, CPol:$cpol)), |
| " $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { |
| let is_flat_global = 1; |
| let mayLoad = 0; |
| let mayStore = 1; |
| let has_vdst = 0; |
| let has_vaddr = 0; |
| let has_saddr = 1; |
| let enabled_saddr = EnableSaddr; |
| } |
| |
| multiclass FLAT_Global_Store_AddTid_Pseudo<string opName, RegisterClass regClass> { |
| def "" : FLAT_Global_Store_AddTid_Pseudo<opName, regClass>, |
| GlobalSaddrTable<0, opName>; |
| def _SADDR : FLAT_Global_Store_AddTid_Pseudo<opName, regClass, 1>, |
| GlobalSaddrTable<1, opName>; |
| } |
| |
| class FLAT_Global_Invalidate_Writeback<string opName, SDPatternOperator node = null_frag> : |
| FLAT_Pseudo<opName, (outs), (ins CPol:$cpol), "$cpol", [(node)]> { |
| |
| let AsmMatchConverter = ""; |
| |
| let hasSideEffects = 1; |
| let mayLoad = 0; |
| let mayStore = 0; |
| let is_flat_global = 1; |
| |
| let has_offset = 0; |
| let has_saddr = 0; |
| let enabled_saddr = 0; |
| let saddr_value = 0; |
| let has_vdst = 0; |
| let has_data = 0; |
| let has_vaddr = 0; |
| let has_glc = 0; |
| let has_dlc = 0; |
| let glcValue = 0; |
| let dlcValue = 0; |
| let has_sccb = 0; |
| let sccbValue = 0; |
| let has_sve = 0; |
| let lds = 0; |
| let sve = 0; |
| } |
| |
| class FlatScratchInst <string sv_op, string mode> { |
| string SVOp = sv_op; |
| string Mode = mode; |
| } |
| |
| class FLAT_Scratch_Load_Pseudo <string opName, RegisterClass regClass, |
| bit HasTiedOutput = 0, |
| bit EnableSaddr = 0, |
| bit EnableSVE = 0, |
| bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr))> |
| : FLAT_Pseudo< |
| opName, |
| (outs getLdStRegisterOperand<regClass>.ret:$vdst), |
| !con( |
| !if(EnableSVE, |
| (ins VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset), |
| !if(EnableSaddr, |
| (ins SReg_32_XEXEC_HI:$saddr, flat_offset:$offset), |
| !if(EnableVaddr, |
| (ins VGPR_32:$vaddr, flat_offset:$offset), |
| (ins flat_offset:$offset)))), |
| !if(HasTiedOutput, (ins CPol:$cpol, getLdStRegisterOperand<regClass>.ret:$vdst_in), |
| (ins CPol_0:$cpol))), |
| " $vdst, "#!if(EnableVaddr, "$vaddr, ", "off, ")#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { |
| let is_flat_scratch = 1; |
| let has_data = 0; |
| let mayLoad = 1; |
| let has_saddr = 1; |
| let enabled_saddr = EnableSaddr; |
| let has_vaddr = EnableVaddr; |
| let has_sve = EnableSVE; |
| let sve = EnableVaddr; |
| |
| let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", ""); |
| let DisableEncoding = !if(HasTiedOutput, "$vdst_in", ""); |
| } |
| |
| class FLAT_Scratch_Store_Pseudo <string opName, RegisterClass vdataClass, bit EnableSaddr = 0, |
| bit EnableSVE = 0, |
| bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr)), |
| RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> : FLAT_Pseudo< |
| opName, |
| (outs), |
| !if(EnableSVE, |
| (ins vdata_op:$vdata, VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol_0:$cpol), |
| !if(EnableSaddr, |
| (ins vdata_op:$vdata, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol_0:$cpol), |
| !if(EnableVaddr, |
| (ins vdata_op:$vdata, VGPR_32:$vaddr, flat_offset:$offset, CPol_0:$cpol), |
| (ins vdata_op:$vdata, flat_offset:$offset, CPol_0:$cpol)))), |
| " "#!if(EnableVaddr, "$vaddr", "off")#", $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { |
| let is_flat_scratch = 1; |
| let mayLoad = 0; |
| let mayStore = 1; |
| let has_vdst = 0; |
| let has_saddr = 1; |
| let enabled_saddr = EnableSaddr; |
| let has_vaddr = EnableVaddr; |
| let has_sve = EnableSVE; |
| let sve = EnableVaddr; |
| } |
| |
| multiclass FLAT_Scratch_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedOutput = 0> { |
| def "" : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput>, |
| FlatScratchInst<opName, "SV">; |
| def _SADDR : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 1>, |
| FlatScratchInst<opName, "SS">; |
| |
| let SubtargetPredicate = HasFlatScratchSVSMode in |
| def _SVS : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 1, 1>, |
| FlatScratchInst<opName, "SVS">; |
| |
| let SubtargetPredicate = HasFlatScratchSTMode in |
| def _ST : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 0, 0, 0>, |
| FlatScratchInst<opName, "ST">; |
| } |
| |
| multiclass FLAT_Scratch_Load_Pseudo_t16<string opName> { |
| defm "" : FLAT_Scratch_Load_Pseudo<opName, VGPR_32, 1>; |
| |
| defvar Name16 = opName#"_t16"; |
| let OtherPredicates = [HasTrue16BitInsts], is_flat_scratch = 1 in { |
| def _t16 : FLAT_Scratch_Load_Pseudo<Name16, VGPR_16, 0>, |
| FlatScratchInst<Name16, "SV">, |
| True16D16Table<NAME#"_HI", NAME>; |
| def _SADDR_t16 : FLAT_Scratch_Load_Pseudo<Name16, VGPR_16, 0, 1>, |
| FlatScratchInst<Name16, "SS">, |
| True16D16Table<NAME#"_HI_SADDR", NAME#"_SADDR">; |
| let SubtargetPredicate = HasFlatScratchSVSMode in |
| def _SVS_t16 : FLAT_Scratch_Load_Pseudo<Name16, VGPR_16, 0, 1, 1>, |
| FlatScratchInst<Name16, "SVS">, |
| True16D16Table<NAME#"_HI_SVS", NAME#"_SVS">; |
| |
| let SubtargetPredicate = HasFlatScratchSTMode in |
| def _ST_t16 : FLAT_Scratch_Load_Pseudo<Name16, VGPR_16, 0, 0, 0, 0>, |
| FlatScratchInst<Name16, "ST">, |
| True16D16Table<NAME#"_HI_ST", NAME#"_ST">; |
| } |
| } |
| |
| multiclass FLAT_Scratch_Store_Pseudo<string opName, RegisterClass regClass> { |
| def "" : FLAT_Scratch_Store_Pseudo<opName, regClass>, |
| FlatScratchInst<opName, "SV">; |
| def _SADDR : FLAT_Scratch_Store_Pseudo<opName, regClass, 1>, |
| FlatScratchInst<opName, "SS">; |
| |
| let SubtargetPredicate = HasFlatScratchSVSMode in |
| def _SVS : FLAT_Scratch_Store_Pseudo<opName, regClass, 1, 1>, |
| FlatScratchInst<opName, "SVS">; |
| |
| let SubtargetPredicate = HasFlatScratchSTMode in |
| def _ST : FLAT_Scratch_Store_Pseudo<opName, regClass, 0, 0, 0>, |
| FlatScratchInst<opName, "ST">; |
| } |
| |
| multiclass FLAT_Scratch_Store_Pseudo_t16<string opName> { |
| defm "" : FLAT_Scratch_Store_Pseudo<opName, VGPR_32>; |
| |
| defvar Name16 = opName#"_t16"; |
| let OtherPredicates = [HasTrue16BitInsts], is_flat_scratch = 1 in { |
| def _t16 : FLAT_Scratch_Store_Pseudo<Name16, VGPR_16>, |
| FlatScratchInst<Name16, "SV">, |
| True16D16Table<NAME#"_D16_HI", NAME>; |
| def _SADDR_t16 : FLAT_Scratch_Store_Pseudo<Name16, VGPR_16, 1>, |
| FlatScratchInst<Name16, "SS">, |
| True16D16Table<NAME#"_D16_HI_SADDR", NAME#"_SADDR">; |
| |
| let SubtargetPredicate = HasFlatScratchSVSMode in |
| def _SVS_t16 : FLAT_Scratch_Store_Pseudo<Name16, VGPR_16, 1, 1>, |
| FlatScratchInst<Name16, "SVS">, |
| True16D16Table<NAME#"_D16_HI_SVS", NAME#"_SVS">; |
| |
| let SubtargetPredicate = HasFlatScratchSTMode in |
| def _ST_t16 : FLAT_Scratch_Store_Pseudo<Name16, VGPR_16, 0, 0, 0>, |
| FlatScratchInst<Name16, "ST">, |
| True16D16Table<NAME#"_D16_HI_ST", NAME#"_ST">; |
| } |
| } |
| |
| |
| class FLAT_Scratch_Load_LDS_Pseudo <string opName, bit EnableSaddr = 0, |
| bit EnableSVE = 0, |
| bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr))> : FLAT_Pseudo< |
| opName, |
| (outs ), |
| !if(EnableSVE, |
| (ins VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol:$cpol), |
| !if(EnableSaddr, |
| (ins SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol:$cpol), |
| !if(EnableVaddr, |
| (ins VGPR_32:$vaddr, flat_offset:$offset, CPol:$cpol), |
| (ins flat_offset:$offset, CPol:$cpol)))), |
| " "#!if(EnableVaddr, "$vaddr, ", "off, ")#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> { |
| |
| let LGKM_CNT = 1; |
| let is_flat_scratch = 1; |
| let lds = 1; |
| let has_data = 0; |
| let has_vdst = 0; |
| let mayLoad = 1; |
| let mayStore = 1; |
| let has_saddr = 1; |
| let enabled_saddr = EnableSaddr; |
| let has_vaddr = EnableVaddr; |
| let has_sve = EnableSVE; |
| let sve = EnableVaddr; |
| let VALU = 1; |
| let Uses = [M0, EXEC]; |
| let SchedRW = [WriteVMEM, WriteLDS]; |
| } |
| |
| multiclass FLAT_Scratch_Load_LDS_Pseudo<string opName> { |
| def "" : FLAT_Scratch_Load_LDS_Pseudo<opName>, |
| FlatScratchInst<opName, "SV">; |
| def _SADDR : FLAT_Scratch_Load_LDS_Pseudo<opName, 1>, |
| FlatScratchInst<opName, "SS">; |
| def _SVS : FLAT_Scratch_Load_LDS_Pseudo<opName, 1, 1>, |
| FlatScratchInst<opName, "SVS">; |
| def _ST : FLAT_Scratch_Load_LDS_Pseudo<opName, 0, 0, 0>, |
| FlatScratchInst<opName, "ST">; |
| } |
| |
| class FLAT_AtomicNoRet_Pseudo<string opName, dag outs, dag ins, |
| string asm, list<dag> pattern = []> : |
| FLAT_Pseudo<opName, outs, ins, asm, pattern> { |
| let mayLoad = 1; |
| let mayStore = 1; |
| let has_glc = 0; |
| let glcValue = 0; |
| let has_vdst = 0; |
| let has_sccb = 1; |
| let sccbValue = 0; |
| let IsAtomicNoRet = 1; |
| } |
| |
| class FLAT_AtomicRet_Pseudo<string opName, dag outs, dag ins, |
| string asm, list<dag> pattern = []> |
| : FLAT_AtomicNoRet_Pseudo<opName, outs, ins, asm, pattern> { |
| let has_vdst = 1; |
| let glcValue = 1; |
| let sccbValue = 0; |
| let IsAtomicNoRet = 0; |
| let IsAtomicRet = 1; |
| } |
| |
| multiclass FLAT_Atomic_Pseudo_NO_RTN< |
| string opName, |
| RegisterClass vdst_rc, |
| ValueType vt, |
| ValueType data_vt = vt, |
| RegisterClass data_rc = vdst_rc, |
| RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret> { |
| def "" : FLAT_AtomicNoRet_Pseudo <opName, |
| (outs), |
| (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_0:$cpol), |
| " $vaddr, $vdata$offset$cpol">, |
| GlobalSaddrTable<0, opName> { |
| let FPAtomic = data_vt.isFP; |
| let AddedComplexity = -1; // Prefer global atomics if available |
| } |
| } |
| |
| multiclass FLAT_Atomic_Pseudo_RTN< |
| string opName, |
| RegisterClass vdst_rc, |
| ValueType vt, |
| ValueType data_vt = vt, |
| RegisterClass data_rc = vdst_rc, |
| RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret> { |
| def _RTN : FLAT_AtomicRet_Pseudo <opName, |
| (outs getLdStRegisterOperand<vdst_rc>.ret:$vdst), |
| (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_GLC1:$cpol), |
| " $vdst, $vaddr, $vdata$offset$cpol">, |
| GlobalSaddrTable<0, opName#"_rtn"> { |
| let FPAtomic = data_vt.isFP; |
| let AddedComplexity = -1; // Prefer global atomics if available |
| } |
| } |
| |
| multiclass FLAT_Atomic_Pseudo< |
| string opName, |
| RegisterClass vdst_rc, |
| ValueType vt, |
| ValueType data_vt = vt, |
| RegisterClass data_rc = vdst_rc, |
| RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret> { |
| defm "" : FLAT_Atomic_Pseudo_NO_RTN<opName, vdst_rc, vt, data_vt, data_rc, data_op>; |
| defm "" : FLAT_Atomic_Pseudo_RTN<opName, vdst_rc, vt, data_vt, data_rc, data_op>; |
| } |
| |
| multiclass FLAT_Global_Atomic_Pseudo_NO_RTN< |
| string opName, |
| RegisterClass vdst_rc, |
| ValueType vt, |
| ValueType data_vt = vt, |
| RegisterClass data_rc = vdst_rc, |
| RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret> { |
| |
| let is_flat_global = 1 in { |
| def "" : FLAT_AtomicNoRet_Pseudo <opName, |
| (outs), |
| (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_0:$cpol), |
| " $vaddr, $vdata, off$offset$cpol">, |
| GlobalSaddrTable<0, opName> { |
| let has_saddr = 1; |
| let FPAtomic = data_vt.isFP; |
| } |
| |
| def _SADDR : FLAT_AtomicNoRet_Pseudo <opName, |
| (outs), |
| (ins VGPR_32:$vaddr, data_op:$vdata, SReg_64_XEXEC_XNULL:$saddr, flat_offset:$offset, CPol_0:$cpol), |
| " $vaddr, $vdata, $saddr$offset$cpol">, |
| GlobalSaddrTable<1, opName> { |
| let has_saddr = 1; |
| let enabled_saddr = 1; |
| let FPAtomic = data_vt.isFP; |
| } |
| } |
| } |
| |
| multiclass FLAT_Global_Atomic_Pseudo_RTN< |
| string opName, |
| RegisterClass vdst_rc, |
| ValueType vt, |
| ValueType data_vt = vt, |
| RegisterClass data_rc = vdst_rc, |
| RegisterOperand data_op = getLdStRegisterOperand<data_rc>.ret, |
| RegisterOperand vdst_op = getLdStRegisterOperand<vdst_rc>.ret> { |
| |
| let is_flat_global = 1 in { |
| def _RTN : FLAT_AtomicRet_Pseudo <opName, |
| (outs vdst_op:$vdst), |
| (ins VReg_64:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_GLC1:$cpol), |
| " $vdst, $vaddr, $vdata, off$offset$cpol">, |
| GlobalSaddrTable<0, opName#"_rtn"> { |
| let has_saddr = 1; |
| let FPAtomic = data_vt.isFP; |
| } |
| |
| def _SADDR_RTN : FLAT_AtomicRet_Pseudo <opName, |
| (outs vdst_op:$vdst), |
| (ins VGPR_32:$vaddr, data_op:$vdata, SReg_64_XEXEC_XNULL:$saddr, flat_offset:$offset, CPol_GLC1:$cpol), |
| " $vdst, $vaddr, $vdata, $saddr$offset$cpol">, |
| GlobalSaddrTable<1, opName#"_rtn"> { |
| let has_saddr = 1; |
| let enabled_saddr = 1; |
| let FPAtomic = data_vt.isFP; |
| } |
| } |
| } |
| |
| multiclass FLAT_Global_Atomic_Pseudo< |
| string opName, |
| RegisterClass vdst_rc, |
| ValueType vt, |
| ValueType data_vt = vt, |
| RegisterClass data_rc = vdst_rc> { |
| defm "" : FLAT_Global_Atomic_Pseudo_NO_RTN<opName, vdst_rc, vt, data_vt, data_rc>; |
| defm "" : FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_rc, vt, data_vt, data_rc>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // Flat Instructions |
| //===----------------------------------------------------------------------===// |
| |
| def FLAT_LOAD_UBYTE : FLAT_Load_Pseudo <"flat_load_ubyte", VGPR_32>; |
| def FLAT_LOAD_SBYTE : FLAT_Load_Pseudo <"flat_load_sbyte", VGPR_32>; |
| def FLAT_LOAD_USHORT : FLAT_Load_Pseudo <"flat_load_ushort", VGPR_32>; |
| def FLAT_LOAD_SSHORT : FLAT_Load_Pseudo <"flat_load_sshort", VGPR_32>; |
| def FLAT_LOAD_DWORD : FLAT_Load_Pseudo <"flat_load_dword", VGPR_32>; |
| def FLAT_LOAD_DWORDX2 : FLAT_Load_Pseudo <"flat_load_dwordx2", VReg_64>; |
| def FLAT_LOAD_DWORDX4 : FLAT_Load_Pseudo <"flat_load_dwordx4", VReg_128>; |
| def FLAT_LOAD_DWORDX3 : FLAT_Load_Pseudo <"flat_load_dwordx3", VReg_96>; |
| |
| def FLAT_STORE_DWORD : FLAT_Store_Pseudo <"flat_store_dword", VGPR_32>; |
| def FLAT_STORE_DWORDX2 : FLAT_Store_Pseudo <"flat_store_dwordx2", VReg_64>; |
| def FLAT_STORE_DWORDX4 : FLAT_Store_Pseudo <"flat_store_dwordx4", VReg_128>; |
| def FLAT_STORE_DWORDX3 : FLAT_Store_Pseudo <"flat_store_dwordx3", VReg_96>; |
| |
| let SubtargetPredicate = HasD16LoadStore in { |
| let TiedSourceNotRead = 1 in { |
| def FLAT_LOAD_UBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_ubyte_d16_hi", VGPR_32, 1>; |
| defm FLAT_LOAD_UBYTE_D16 : FLAT_Load_Pseudo_t16 <"flat_load_ubyte_d16">; |
| def FLAT_LOAD_SBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_sbyte_d16_hi", VGPR_32, 1>; |
| defm FLAT_LOAD_SBYTE_D16 : FLAT_Load_Pseudo_t16 <"flat_load_sbyte_d16">; |
| def FLAT_LOAD_SHORT_D16_HI : FLAT_Load_Pseudo <"flat_load_short_d16_hi", VGPR_32, 1>; |
| defm FLAT_LOAD_SHORT_D16 : FLAT_Load_Pseudo_t16 <"flat_load_short_d16">; |
| } |
| |
| def FLAT_STORE_BYTE_D16_HI : FLAT_Store_Pseudo <"flat_store_byte_d16_hi", VGPR_32>; |
| def FLAT_STORE_SHORT_D16_HI : FLAT_Store_Pseudo <"flat_store_short_d16_hi", VGPR_32>; |
| } |
| |
| defm FLAT_STORE_BYTE : FLAT_Store_Pseudo_t16 <"flat_store_byte">; |
| defm FLAT_STORE_SHORT : FLAT_Store_Pseudo_t16 <"flat_store_short">; |
| |
| defm FLAT_ATOMIC_CMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap", |
| VGPR_32, i32, v2i32, VReg_64>; |
| |
| defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap_x2", |
| VReg_64, i64, v2i64, VReg_128>; |
| |
| defm FLAT_ATOMIC_SWAP : FLAT_Atomic_Pseudo <"flat_atomic_swap", |
| VGPR_32, i32>; |
| |
| defm FLAT_ATOMIC_SWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_swap_x2", |
| VReg_64, i64>; |
| |
| defm FLAT_ATOMIC_ADD : FLAT_Atomic_Pseudo <"flat_atomic_add", |
| VGPR_32, i32>; |
| |
| defm FLAT_ATOMIC_SUB : FLAT_Atomic_Pseudo <"flat_atomic_sub", |
| VGPR_32, i32>; |
| |
| defm FLAT_ATOMIC_SMIN : FLAT_Atomic_Pseudo <"flat_atomic_smin", |
| VGPR_32, i32>; |
| |
| defm FLAT_ATOMIC_UMIN : FLAT_Atomic_Pseudo <"flat_atomic_umin", |
| VGPR_32, i32>; |
| |
| defm FLAT_ATOMIC_SMAX : FLAT_Atomic_Pseudo <"flat_atomic_smax", |
| VGPR_32, i32>; |
| |
| defm FLAT_ATOMIC_UMAX : FLAT_Atomic_Pseudo <"flat_atomic_umax", |
| VGPR_32, i32>; |
| |
| defm FLAT_ATOMIC_AND : FLAT_Atomic_Pseudo <"flat_atomic_and", |
| VGPR_32, i32>; |
| |
| defm FLAT_ATOMIC_OR : FLAT_Atomic_Pseudo <"flat_atomic_or", |
| VGPR_32, i32>; |
| |
| defm FLAT_ATOMIC_XOR : FLAT_Atomic_Pseudo <"flat_atomic_xor", |
| VGPR_32, i32>; |
| |
| defm FLAT_ATOMIC_INC : FLAT_Atomic_Pseudo <"flat_atomic_inc", |
| VGPR_32, i32>; |
| |
| defm FLAT_ATOMIC_DEC : FLAT_Atomic_Pseudo <"flat_atomic_dec", |
| VGPR_32, i32>; |
| |
| defm FLAT_ATOMIC_ADD_X2 : FLAT_Atomic_Pseudo <"flat_atomic_add_x2", |
| VReg_64, i64>; |
| |
| defm FLAT_ATOMIC_SUB_X2 : FLAT_Atomic_Pseudo <"flat_atomic_sub_x2", |
| VReg_64, i64>; |
| |
| defm FLAT_ATOMIC_SMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smin_x2", |
| VReg_64, i64>; |
| |
| defm FLAT_ATOMIC_UMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umin_x2", |
| VReg_64, i64>; |
| |
| defm FLAT_ATOMIC_SMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smax_x2", |
| VReg_64, i64>; |
| |
| defm FLAT_ATOMIC_UMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umax_x2", |
| VReg_64, i64>; |
| |
| defm FLAT_ATOMIC_AND_X2 : FLAT_Atomic_Pseudo <"flat_atomic_and_x2", |
| VReg_64, i64>; |
| |
| defm FLAT_ATOMIC_OR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_or_x2", |
| VReg_64, i64>; |
| |
| defm FLAT_ATOMIC_XOR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_xor_x2", |
| VReg_64, i64>; |
| |
| defm FLAT_ATOMIC_INC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_inc_x2", |
| VReg_64, i64>; |
| |
| defm FLAT_ATOMIC_DEC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_dec_x2", |
| VReg_64, i64>; |
| |
| // GFX7-, GFX10-only flat instructions. |
| let SubtargetPredicate = isGFX7GFX10 in { |
| defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap_x2", |
| VReg_64, f64, v2f64, VReg_128>; |
| } // End SubtargetPredicate = isGFX7GFX10 |
| |
| |
| // The names may be flat_atomic_fmin_x2 on some subtargets, but we |
| // choose this as the canonical name. |
| let SubtargetPredicate = HasAtomicFMinFMaxF64FlatInsts in { |
| defm FLAT_ATOMIC_MIN_F64 : FLAT_Atomic_Pseudo <"flat_atomic_min_f64", |
| VReg_64, f64>; |
| |
| defm FLAT_ATOMIC_MAX_F64 : FLAT_Atomic_Pseudo <"flat_atomic_max_f64", |
| VReg_64, f64>; |
| } |
| |
| let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts in { |
| defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_min_f64", VReg_64, f64>; |
| defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_max_f64", VReg_64, f64>; |
| } |
| |
| let SubtargetPredicate = HasFlatBufferGlobalAtomicFaddF64Inst in { |
| defm FLAT_ATOMIC_ADD_F64 : FLAT_Atomic_Pseudo<"flat_atomic_add_f64", VReg_64, f64>; |
| defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_add_f64", VReg_64, f64>; |
| } // End SubtargetPredicate = HasFlatBufferGlobalAtomicFaddF64Inst |
| |
| let SubtargetPredicate = HasAtomicFlatPkAdd16Insts in { |
| defm FLAT_ATOMIC_PK_ADD_F16 : FLAT_Atomic_Pseudo<"flat_atomic_pk_add_f16", VGPR_32, v2f16>; |
| let FPAtomic = 1 in |
| defm FLAT_ATOMIC_PK_ADD_BF16 : FLAT_Atomic_Pseudo<"flat_atomic_pk_add_bf16", VGPR_32, v2i16>; |
| } // End SubtargetPredicate = HasAtomicFlatPkAdd16Insts |
| |
| let SubtargetPredicate = HasAtomicGlobalPkAddBF16Inst, FPAtomic = 1 in |
| defm GLOBAL_ATOMIC_PK_ADD_BF16 : FLAT_Global_Atomic_Pseudo<"global_atomic_pk_add_bf16", VGPR_32, v2i16>; |
| |
| // GFX7-, GFX10-, GFX11-only flat instructions. |
| let SubtargetPredicate = isGFX7GFX10GFX11 in { |
| |
| defm FLAT_ATOMIC_FCMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap", |
| VGPR_32, f32, v2f32, VReg_64>; |
| |
| defm FLAT_ATOMIC_FMIN : FLAT_Atomic_Pseudo <"flat_atomic_fmin", |
| VGPR_32, f32>; |
| |
| defm FLAT_ATOMIC_FMAX : FLAT_Atomic_Pseudo <"flat_atomic_fmax", |
| VGPR_32, f32>; |
| |
| } // End SubtargetPredicate = isGFX7GFX10GFX11 |
| |
| // GFX942-, GFX11-only flat instructions. |
| let SubtargetPredicate = HasFlatAtomicFaddF32Inst in { |
| defm FLAT_ATOMIC_ADD_F32 : FLAT_Atomic_Pseudo<"flat_atomic_add_f32", VGPR_32, f32>; |
| } // End SubtargetPredicate = HasFlatAtomicFaddF32Inst |
| |
| let SubtargetPredicate = isGFX12Plus in { |
| defm FLAT_ATOMIC_CSUB_U32 : FLAT_Atomic_Pseudo <"flat_atomic_csub_u32", VGPR_32, i32>; |
| defm FLAT_ATOMIC_COND_SUB_U32 : FLAT_Atomic_Pseudo <"flat_atomic_cond_sub_u32", VGPR_32, i32>; |
| } // End SubtargetPredicate = isGFX12Plus |
| |
| defm GLOBAL_LOAD_UBYTE : FLAT_Global_Load_Pseudo <"global_load_ubyte", VGPR_32>; |
| defm GLOBAL_LOAD_SBYTE : FLAT_Global_Load_Pseudo <"global_load_sbyte", VGPR_32>; |
| defm GLOBAL_LOAD_USHORT : FLAT_Global_Load_Pseudo <"global_load_ushort", VGPR_32>; |
| defm GLOBAL_LOAD_SSHORT : FLAT_Global_Load_Pseudo <"global_load_sshort", VGPR_32>; |
| defm GLOBAL_LOAD_DWORD : FLAT_Global_Load_Pseudo <"global_load_dword", VGPR_32>; |
| defm GLOBAL_LOAD_DWORDX2 : FLAT_Global_Load_Pseudo <"global_load_dwordx2", VReg_64>; |
| defm GLOBAL_LOAD_DWORDX3 : FLAT_Global_Load_Pseudo <"global_load_dwordx3", VReg_96>; |
| defm GLOBAL_LOAD_DWORDX4 : FLAT_Global_Load_Pseudo <"global_load_dwordx4", VReg_128>; |
| |
| let TiedSourceNotRead = 1 in { |
| defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16_hi", VGPR_32, 1>; |
| defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Global_Load_Pseudo <"global_load_short_d16_hi", VGPR_32, 1>; |
| defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16_hi", VGPR_32, 1>; |
| defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Global_Load_Pseudo_t16 <"global_load_sbyte_d16">; |
| defm GLOBAL_LOAD_SHORT_D16 : FLAT_Global_Load_Pseudo_t16 <"global_load_short_d16">; |
| defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Global_Load_Pseudo_t16 <"global_load_ubyte_d16">; |
| } |
| |
| defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Global_Store_Pseudo <"global_store_byte_d16_hi", VGPR_32>; |
| defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Global_Store_Pseudo <"global_store_short_d16_hi", VGPR_32>; |
| |
| let OtherPredicates = [HasGFX10_BEncoding] in |
| defm GLOBAL_LOAD_DWORD_ADDTID : FLAT_Global_Load_AddTid_Pseudo <"global_load_dword_addtid", VGPR_32>; |
| |
| defm GLOBAL_STORE_BYTE : FLAT_Global_Store_Pseudo_t16 <"global_store_byte">; |
| defm GLOBAL_STORE_SHORT : FLAT_Global_Store_Pseudo_t16 <"global_store_short">; |
| defm GLOBAL_STORE_DWORD : FLAT_Global_Store_Pseudo <"global_store_dword", VGPR_32>; |
| defm GLOBAL_STORE_DWORDX2 : FLAT_Global_Store_Pseudo <"global_store_dwordx2", VReg_64>; |
| defm GLOBAL_STORE_DWORDX3 : FLAT_Global_Store_Pseudo <"global_store_dwordx3", VReg_96>; |
| defm GLOBAL_STORE_DWORDX4 : FLAT_Global_Store_Pseudo <"global_store_dwordx4", VReg_128>; |
| let OtherPredicates = [HasGFX10_BEncoding] in |
| defm GLOBAL_STORE_DWORD_ADDTID : FLAT_Global_Store_AddTid_Pseudo <"global_store_dword_addtid", VGPR_32>; |
| |
| defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap", |
| VGPR_32, i32, v2i32, VReg_64>; |
| |
| defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap_x2", |
| VReg_64, i64, v2i64, VReg_128>; |
| |
| defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_swap", |
| VGPR_32, i32>; |
| |
| defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_swap_x2", |
| VReg_64, i64>; |
| |
| defm GLOBAL_ATOMIC_ADD : FLAT_Global_Atomic_Pseudo <"global_atomic_add", |
| VGPR_32, i32>; |
| |
| defm GLOBAL_ATOMIC_SUB : FLAT_Global_Atomic_Pseudo <"global_atomic_sub", |
| VGPR_32, i32>; |
| |
| defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_smin", |
| VGPR_32, i32>; |
| |
| defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_umin", |
| VGPR_32, i32>; |
| |
| defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_smax", |
| VGPR_32, i32>; |
| |
| defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_umax", |
| VGPR_32, i32>; |
| |
| defm GLOBAL_ATOMIC_AND : FLAT_Global_Atomic_Pseudo <"global_atomic_and", |
| VGPR_32, i32>; |
| |
| defm GLOBAL_ATOMIC_OR : FLAT_Global_Atomic_Pseudo <"global_atomic_or", |
| VGPR_32, i32>; |
| |
| defm GLOBAL_ATOMIC_XOR : FLAT_Global_Atomic_Pseudo <"global_atomic_xor", |
| VGPR_32, i32>; |
| |
| defm GLOBAL_ATOMIC_INC : FLAT_Global_Atomic_Pseudo <"global_atomic_inc", |
| VGPR_32, i32>; |
| |
| defm GLOBAL_ATOMIC_DEC : FLAT_Global_Atomic_Pseudo <"global_atomic_dec", |
| VGPR_32, i32>; |
| |
| defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_add_x2", |
| VReg_64, i64>; |
| |
| defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_sub_x2", |
| VReg_64, i64>; |
| |
| defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smin_x2", |
| VReg_64, i64>; |
| |
| defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umin_x2", |
| VReg_64, i64>; |
| |
| defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smax_x2", |
| VReg_64, i64>; |
| |
| defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umax_x2", |
| VReg_64, i64>; |
| |
| defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_and_x2", |
| VReg_64, i64>; |
| |
| defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_or_x2", |
| VReg_64, i64>; |
| |
| defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_xor_x2", |
| VReg_64, i64>; |
| |
| defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_inc_x2", |
| VReg_64, i64>; |
| |
| defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_dec_x2", |
| VReg_64, i64>; |
| |
| let SubtargetPredicate = HasGFX10_BEncoding in { |
| defm GLOBAL_ATOMIC_CSUB : FLAT_Global_Atomic_Pseudo <"global_atomic_csub", |
| VGPR_32, i32>; |
| } |
| |
| defm GLOBAL_LOAD_LDS_UBYTE : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_ubyte">; |
| defm GLOBAL_LOAD_LDS_SBYTE : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_sbyte">; |
| defm GLOBAL_LOAD_LDS_USHORT : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_ushort">; |
| defm GLOBAL_LOAD_LDS_SSHORT : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_sshort">; |
| defm GLOBAL_LOAD_LDS_DWORD : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_dword">; |
| |
| let SubtargetPredicate = HasGFX950Insts in { |
| defm GLOBAL_LOAD_LDS_DWORDX3 : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_dwordx3">; |
| defm GLOBAL_LOAD_LDS_DWORDX4 : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_dwordx4">; |
| } |
| |
| let SubtargetPredicate = isGFX12Plus in { |
| defm GLOBAL_ATOMIC_COND_SUB_U32 : FLAT_Global_Atomic_Pseudo <"global_atomic_cond_sub_u32", VGPR_32, i32>; |
| defm GLOBAL_ATOMIC_ORDERED_ADD_B64 : FLAT_Global_Atomic_Pseudo <"global_atomic_ordered_add_b64", VReg_64, i64>; |
| |
| def GLOBAL_INV : FLAT_Global_Invalidate_Writeback<"global_inv">; |
| def GLOBAL_WB : FLAT_Global_Invalidate_Writeback<"global_wb">; |
| def GLOBAL_WBINV : FLAT_Global_Invalidate_Writeback<"global_wbinv">; |
| } // End SubtargetPredicate = isGFX12Plus |
| |
| defm SCRATCH_LOAD_UBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte", VGPR_32>; |
| defm SCRATCH_LOAD_SBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte", VGPR_32>; |
| defm SCRATCH_LOAD_USHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_ushort", VGPR_32>; |
| defm SCRATCH_LOAD_SSHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_sshort", VGPR_32>; |
| defm SCRATCH_LOAD_DWORD : FLAT_Scratch_Load_Pseudo <"scratch_load_dword", VGPR_32>; |
| defm SCRATCH_LOAD_DWORDX2 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx2", VReg_64>; |
| defm SCRATCH_LOAD_DWORDX3 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx3", VReg_96>; |
| defm SCRATCH_LOAD_DWORDX4 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx4", VReg_128>; |
| |
| let TiedSourceNotRead = 1 in { |
| defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16_hi", VGPR_32, 1>; |
| defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16_hi", VGPR_32, 1>; |
| defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16_hi", VGPR_32, 1>; |
| defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Scratch_Load_Pseudo_t16 <"scratch_load_ubyte_d16">; |
| defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Scratch_Load_Pseudo_t16 <"scratch_load_sbyte_d16">; |
| defm SCRATCH_LOAD_SHORT_D16 : FLAT_Scratch_Load_Pseudo_t16 <"scratch_load_short_d16">; |
| } |
| |
| defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_byte_d16_hi", VGPR_32>; |
| defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_short_d16_hi", VGPR_32>; |
| |
| defm SCRATCH_STORE_BYTE : FLAT_Scratch_Store_Pseudo_t16 <"scratch_store_byte">; |
| defm SCRATCH_STORE_SHORT : FLAT_Scratch_Store_Pseudo_t16 <"scratch_store_short">; |
| defm SCRATCH_STORE_DWORD : FLAT_Scratch_Store_Pseudo <"scratch_store_dword", VGPR_32>; |
| defm SCRATCH_STORE_DWORDX2 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx2", VReg_64>; |
| defm SCRATCH_STORE_DWORDX3 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx3", VReg_96>; |
| defm SCRATCH_STORE_DWORDX4 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx4", VReg_128>; |
| |
| defm SCRATCH_LOAD_LDS_UBYTE : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_ubyte">; |
| defm SCRATCH_LOAD_LDS_SBYTE : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_sbyte">; |
| defm SCRATCH_LOAD_LDS_USHORT : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_ushort">; |
| defm SCRATCH_LOAD_LDS_SSHORT : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_sshort">; |
| defm SCRATCH_LOAD_LDS_DWORD : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_dword">; |
| |
| let SubtargetPredicate = isGFX12Plus in { |
| let Uses = [EXEC, M0] in { |
| defm GLOBAL_LOAD_BLOCK : FLAT_Global_Load_Pseudo <"global_load_block", VReg_1024>; |
| defm GLOBAL_STORE_BLOCK : FLAT_Global_Store_Pseudo <"global_store_block", VReg_1024>; |
| } |
| let Uses = [EXEC, FLAT_SCR, M0] in { |
| defm SCRATCH_LOAD_BLOCK : FLAT_Scratch_Load_Pseudo <"scratch_load_block", VReg_1024>; |
| defm SCRATCH_STORE_BLOCK : FLAT_Scratch_Store_Pseudo <"scratch_store_block", VReg_1024>; |
| } |
| |
| let WaveSizePredicate = isWave32 in { |
| let Mnemonic = "global_load_tr_b128" in |
| defm GLOBAL_LOAD_TR_B128_w32 : FLAT_Global_Load_Pseudo <"global_load_tr_b128_w32", VReg_128>; |
| let Mnemonic = "global_load_tr_b64" in |
| defm GLOBAL_LOAD_TR_B64_w32 : FLAT_Global_Load_Pseudo <"global_load_tr_b64_w32", VReg_64>; |
| } |
| let WaveSizePredicate = isWave64 in { |
| let Mnemonic = "global_load_tr_b128" in |
| defm GLOBAL_LOAD_TR_B128_w64 : FLAT_Global_Load_Pseudo <"global_load_tr_b128_w64", VReg_64>; |
| let Mnemonic = "global_load_tr_b64" in |
| defm GLOBAL_LOAD_TR_B64_w64 : FLAT_Global_Load_Pseudo <"global_load_tr_b64_w64", VGPR_32>; |
| } |
| } // End SubtargetPredicate = isGFX12Plus |
| |
| let SubtargetPredicate = isGFX10Plus in { |
| defm GLOBAL_ATOMIC_FCMPSWAP : |
| FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap", VGPR_32, f32, v2f32, VReg_64>; |
| defm GLOBAL_ATOMIC_FMIN : |
| FLAT_Global_Atomic_Pseudo<"global_atomic_fmin", VGPR_32, f32>; |
| defm GLOBAL_ATOMIC_FMAX : |
| FLAT_Global_Atomic_Pseudo<"global_atomic_fmax", VGPR_32, f32>; |
| defm GLOBAL_ATOMIC_FCMPSWAP_X2 : |
| FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap_x2", VReg_64, f64, v2f64, VReg_128>; |
| } // End SubtargetPredicate = isGFX10Plus |
| |
| let OtherPredicates = [HasAtomicFaddNoRtnInsts] in |
| defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_NO_RTN < |
| "global_atomic_add_f32", VGPR_32, f32 |
| >; |
| let OtherPredicates = [HasAtomicBufferGlobalPkAddF16NoRtnInsts] in |
| defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_NO_RTN < |
| "global_atomic_pk_add_f16", VGPR_32, v2f16 |
| >; |
| let OtherPredicates = [HasAtomicFaddRtnInsts] in |
| defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_RTN < |
| "global_atomic_add_f32", VGPR_32, f32 |
| >; |
| let OtherPredicates = [HasAtomicBufferGlobalPkAddF16Insts] in |
| defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_RTN < |
| "global_atomic_pk_add_f16", VGPR_32, v2f16 |
| >; |
| |
| //===----------------------------------------------------------------------===// |
| // Flat Patterns |
| //===----------------------------------------------------------------------===// |
| |
| // Patterns for global loads with no offset. |
| class FlatLoadPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (vt (node (FlatOffset i64:$vaddr, i32:$offset))), |
| (inst $vaddr, $offset) |
| >; |
| |
| class FlatLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (node (FlatOffset (i64 VReg_64:$vaddr), i32:$offset), vt:$in), |
| (inst $vaddr, $offset, 0, $in) |
| >; |
| |
| class FlatLoadPat_D16_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (vt (node (FlatOffset (i64 VReg_64:$vaddr), i32:$offset))), |
| (inst $vaddr, $offset, (i32 0)) |
| >; |
| |
| class FlatSignedLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (node (GlobalOffset (i64 VReg_64:$vaddr), i32:$offset), vt:$in), |
| (inst $vaddr, $offset, 0, $in) |
| >; |
| |
| class FlatSignedLoadPat_D16_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (vt (node (GlobalOffset (i64 VReg_64:$vaddr), i32:$offset))), |
| (inst $vaddr, $offset, (i32 0)) |
| >; |
| |
| class GlobalLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset), vt:$in)), |
| (inst $saddr, $voffset, $offset, 0, $in) |
| >; |
| |
| class GlobalLoadSaddrPat_D16_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset))), |
| (inst $saddr, $voffset, $offset, (i32 0)) |
| >; |
| |
| class FlatLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (vt (node (GlobalOffset (i64 VReg_64:$vaddr), i32:$offset))), |
| (inst $vaddr, $offset) |
| >; |
| |
| class GlobalLoadSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset))), |
| (inst $saddr, $voffset, $offset, 0) |
| >; |
| |
| class GlobalStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, |
| ValueType vt> : GCNPat < |
| (node vt:$data, (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset)), |
| (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) |
| >; |
| |
| class GlobalAtomicSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, |
| ValueType vt, ValueType data_vt = vt> : GCNPat < |
| (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset), data_vt:$data)), |
| (inst $voffset, getVregSrcForVT<data_vt>.ret:$data, $saddr, $offset) |
| >; |
| |
| class GlobalAtomicNoRtnSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, |
| ValueType vt> : GCNPat < |
| (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset), vt:$data), |
| (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset) |
| >; |
| |
| class FlatStorePat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (node vt:$data, (FlatOffset i64:$vaddr, i32:$offset)), |
| (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) |
| >; |
| |
| class FlatStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (node vt:$data, (GlobalOffset i64:$vaddr, i32:$offset)), |
| (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset) |
| >; |
| |
| class FlatStoreSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, |
| ValueType vt, ValueType data_vt = vt> : GCNPat < |
| // atomic store follows atomic binop convention so the address comes |
| // first. |
| (node (GlobalOffset i64:$vaddr, i32:$offset), data_vt:$data), |
| (inst $vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset) |
| >; |
| |
| multiclass FlatAtomicNoRtnPatBase <string inst, string node, ValueType vt, |
| ValueType data_vt = vt> { |
| |
| defvar noRtnNode = !cast<PatFrags>(node); |
| |
| let AddedComplexity = 1 in |
| def : GCNPat <(vt (noRtnNode (FlatOffset i64:$vaddr, i32:$offset), data_vt:$data)), |
| (!cast<FLAT_Pseudo>(inst) VReg_64:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset)>; |
| } |
| |
| multiclass FlatAtomicNoRtnPatWithAddrSpace<string inst, string node, string addrSpaceSuffix, |
| ValueType vt> : |
| FlatAtomicNoRtnPatBase<inst, node # "_noret_" # addrSpaceSuffix, vt, vt>; |
| |
| multiclass FlatAtomicNoRtnPat <string inst, string node, ValueType vt, |
| ValueType data_vt = vt, bit isIntr = 0> : |
| FlatAtomicNoRtnPatBase<inst, node # "_noret" # !if(isIntr, "", "_"#vt), vt, data_vt>; |
| |
| |
| multiclass FlatAtomicRtnPatBase <string inst, string node, ValueType vt, |
| ValueType data_vt = vt> { |
| |
| defvar rtnNode = !cast<SDPatternOperator>(node); |
| |
| def : GCNPat <(vt (rtnNode (FlatOffset i64:$vaddr, i32:$offset), data_vt:$data)), |
| (!cast<FLAT_Pseudo>(inst#"_RTN") VReg_64:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset)>; |
| } |
| |
| multiclass FlatAtomicRtnPatWithAddrSpace<string inst, string intr, string addrSpaceSuffix, |
| ValueType vt> : |
| FlatAtomicRtnPatBase<inst, intr # "_" # addrSpaceSuffix, vt, vt>; |
| |
| multiclass FlatAtomicRtnPat <string inst, string node, ValueType vt, |
| ValueType data_vt = vt, bit isIntr = 0> : |
| FlatAtomicRtnPatBase<inst, node # !if(isIntr, "", "_"#vt), vt, data_vt>; |
| |
| |
| multiclass FlatAtomicPat <string inst, string node, ValueType vt, |
| ValueType data_vt = vt, bit isIntr = 0> : |
| FlatAtomicRtnPat<inst, node, vt, data_vt, isIntr>, |
| FlatAtomicNoRtnPat<inst, node, vt, data_vt, isIntr>; |
| |
| multiclass FlatAtomicIntrNoRtnPat <string inst, string node, ValueType vt, |
| ValueType data_vt = vt> { |
| defm : FlatAtomicNoRtnPat<inst, node, vt, data_vt, /* isIntr */ 1>; |
| } |
| |
| multiclass FlatAtomicIntrRtnPat <string inst, string node, ValueType vt, |
| ValueType data_vt = vt> { |
| defm : FlatAtomicRtnPat<inst, node, vt, data_vt, /* isIntr */ 1>; |
| } |
| |
| multiclass FlatAtomicIntrPat <string inst, string node, ValueType vt, |
| ValueType data_vt = vt> : |
| FlatAtomicRtnPat<inst, node, vt, data_vt, /* isIntr */ 1>, |
| FlatAtomicNoRtnPat<inst, node, vt, data_vt, /* isIntr */ 1>; |
| |
| class FlatSignedAtomicPatBase <FLAT_Pseudo inst, SDPatternOperator node, |
| ValueType vt, ValueType data_vt = vt> : GCNPat < |
| (vt (node (GlobalOffset i64:$vaddr, i32:$offset), data_vt:$data)), |
| (inst VReg_64:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset) |
| >; |
| |
| multiclass FlatSignedAtomicPat <string inst, string node, ValueType vt, |
| ValueType data_vt = vt, int complexity = 0, |
| bit isIntr = 0> { |
| defvar rtnNode = !cast<SDPatternOperator>(node # !if(isIntr, "", "_" # vt)); |
| defvar noRtnNode = !cast<PatFrags>(node # "_noret" # !if(isIntr, "", "_" # vt)); |
| |
| let AddedComplexity = complexity in |
| def : FlatSignedAtomicPatBase<!cast<FLAT_Pseudo>(inst#"_RTN"), rtnNode, vt, data_vt>; |
| |
| let AddedComplexity = !add(complexity, 1) in |
| def : FlatSignedAtomicPatBase<!cast<FLAT_Pseudo>(inst), noRtnNode, vt, data_vt>; |
| } |
| |
| class ScratchLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (vt (node (ScratchOffset (i32 VGPR_32:$vaddr), i32:$offset))), |
| (inst $vaddr, $offset) |
| >; |
| |
| class ScratchLoadSignedPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (node (ScratchOffset (i32 VGPR_32:$vaddr), i32:$offset), vt:$in), |
| (inst $vaddr, $offset, 0, $in) |
| >; |
| |
| class ScratchLoadSignedPat_D16_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (vt (node (ScratchOffset (i32 VGPR_32:$vaddr), i32:$offset))), |
| (inst $vaddr, $offset, 0) |
| >; |
| |
| class ScratchStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (node vt:$data, (ScratchOffset (i32 VGPR_32:$vaddr), i32:$offset)), |
| (inst getVregSrcForVT<vt>.ret:$data, $vaddr, $offset) |
| >; |
| |
| class ScratchLoadSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i32:$offset))), |
| (inst $saddr, $offset) |
| >; |
| |
| class ScratchLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i32:$offset), vt:$in)), |
| (inst $saddr, $offset, 0, $in) |
| >; |
| |
| class ScratchLoadSaddrPat_D16_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i32:$offset))), |
| (inst $saddr, $offset, 0) |
| >; |
| |
| class ScratchStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, |
| ValueType vt> : GCNPat < |
| (node vt:$data, (ScratchSAddr (i32 SGPR_32:$saddr), i32:$offset)), |
| (inst getVregSrcForVT<vt>.ret:$data, $saddr, $offset) |
| >; |
| |
| class ScratchLoadSVaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (vt (node (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i32:$offset))), |
| (inst $vaddr, $saddr, $offset, 0) |
| >; |
| |
| class ScratchStoreSVaddrPat <FLAT_Pseudo inst, SDPatternOperator node, |
| ValueType vt> : GCNPat < |
| (node vt:$data, (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i32:$offset)), |
| (inst getVregSrcForVT<vt>.ret:$data, $vaddr, $saddr, $offset) |
| >; |
| |
| class ScratchLoadSVaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (vt (node (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i32:$offset), vt:$in)), |
| (inst $vaddr, $saddr, $offset, 0, $in) |
| >; |
| |
| class ScratchLoadSVaddrPat_D16_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat < |
| (vt (node (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i32:$offset))), |
| (inst $vaddr, $saddr, $offset, 0) |
| >; |
| |
| multiclass GlobalFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { |
| def : FlatLoadSignedPat <inst, node, vt> { |
| let AddedComplexity = 10; |
| } |
| |
| def : GlobalLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { |
| let AddedComplexity = 11; |
| } |
| } |
| |
| multiclass GlobalFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { |
| def : FlatSignedLoadPat_D16 <inst, node, vt> { |
| let AddedComplexity = 10; |
| } |
| |
| def : GlobalLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { |
| let AddedComplexity = 11; |
| } |
| } |
| |
| multiclass GlobalFLATLoadPats_D16_t16<string inst, SDPatternOperator node, ValueType vt> { |
| def : FlatSignedLoadPat_D16_t16<!cast<FLAT_Pseudo>(inst#"_t16"), node, vt> { |
| let AddedComplexity = 10; |
| } |
| |
| def : GlobalLoadSaddrPat_D16_t16<!cast<FLAT_Pseudo>(inst#"_SADDR_t16"), node, vt> { |
| let AddedComplexity = 11; |
| } |
| } |
| |
| multiclass GlobalFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node, |
| ValueType vt> { |
| def : FlatStoreSignedPat <inst, node, vt> { |
| let AddedComplexity = 10; |
| } |
| |
| def : GlobalStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { |
| let AddedComplexity = 11; |
| } |
| } |
| |
| multiclass GlobalFLATStorePats_D16_t16<string inst, SDPatternOperator node, ValueType vt> { |
| def : FlatStoreSignedPat<!cast<FLAT_Pseudo>(inst#"_t16"), node, vt> { |
| let AddedComplexity = 10; |
| } |
| |
| def : GlobalStoreSaddrPat<!cast<FLAT_Pseudo>(inst#"_SADDR_t16"), node, vt> { |
| let AddedComplexity = 11; |
| } |
| } |
| |
| multiclass GlobalFLATAtomicPatsNoRtnBase<string inst, string node, ValueType vt, |
| ValueType data_vt = vt> { |
| let AddedComplexity = 11 in |
| def : FlatSignedAtomicPatBase<!cast<FLAT_Pseudo>(inst), !cast<SDPatternOperator>(node), vt, data_vt>; |
| |
| let AddedComplexity = 13 in |
| def : GlobalAtomicSaddrPat<!cast<FLAT_Pseudo>(inst#"_SADDR"), !cast<SDPatternOperator>(node), vt, data_vt>; |
| } |
| |
| multiclass GlobalFLATAtomicPatsRtnBase<string inst, string node, ValueType vt, |
| ValueType data_vt = vt, bit isPatFrags = 0> { |
| defvar rtnNode = !if(isPatFrags, !cast<PatFrags>(node), !cast<SDPatternOperator>(node)); |
| |
| let AddedComplexity = 10 in |
| def : FlatSignedAtomicPatBase<!cast<FLAT_Pseudo>(inst#"_RTN"), rtnNode, vt, data_vt>; |
| |
| let AddedComplexity = 12 in |
| def : GlobalAtomicSaddrPat<!cast<FLAT_Pseudo>(inst#"_SADDR_RTN"), rtnNode, vt, data_vt>; |
| } |
| |
| multiclass GlobalFLATAtomicPatsNoRtn<string inst, string node, ValueType vt, |
| ValueType data_vt = vt, bit isIntr = 0> : |
| GlobalFLATAtomicPatsNoRtnBase<inst, node # "_noret" # !if(isIntr, "", "_" # vt), vt, data_vt>; |
| |
| multiclass GlobalFLATAtomicPatsRtn<string inst, string node, ValueType vt, |
| ValueType data_vt = vt, bit isIntr = 0> : |
| GlobalFLATAtomicPatsRtnBase<inst, node # !if(isIntr, "", "_" # vt), vt, data_vt>; |
| |
| multiclass GlobalFLATAtomicPats<string inst, string node, ValueType vt, |
| ValueType data_vt = vt, bit isIntr = 0> : |
| GlobalFLATAtomicPatsNoRtn<inst, node, vt, data_vt, isIntr>, |
| GlobalFLATAtomicPatsRtn<inst, node, vt, data_vt, isIntr>; |
| |
| multiclass GlobalFLATAtomicPatsNoRtnWithAddrSpace<string inst, string intr, string addrSpaceSuffix, |
| ValueType vt, ValueType data_vt = vt> : |
| GlobalFLATAtomicPatsNoRtnBase<inst, intr # "_noret_" # addrSpaceSuffix, vt, data_vt>; |
| |
| multiclass GlobalFLATAtomicPatsRtnWithAddrSpace<string inst, string intr, string addrSpaceSuffix, |
| ValueType vt, ValueType data_vt = vt> : |
| GlobalFLATAtomicPatsRtnBase<inst, intr # "_" # addrSpaceSuffix, vt, data_vt, /*isPatFrags*/ 1>; |
| |
| multiclass GlobalFLATAtomicPatsWithAddrSpace<string inst, string intr, string addrSpaceSuffix, |
| ValueType vt, ValueType data_vt = vt> : |
| GlobalFLATAtomicPatsNoRtnWithAddrSpace<inst, intr, addrSpaceSuffix, vt, data_vt>, |
| GlobalFLATAtomicPatsRtnWithAddrSpace<inst, intr, addrSpaceSuffix, vt, data_vt>; |
| |
| multiclass GlobalFLATAtomicIntrPats<string inst, string node, ValueType vt, |
| ValueType data_vt = vt> { |
| defm : GlobalFLATAtomicPats<inst, node, vt, data_vt, /* isIntr */ 1>; |
| } |
| |
| multiclass ScratchFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { |
| def : ScratchLoadSignedPat <inst, node, vt> { |
| let AddedComplexity = 25; |
| } |
| |
| def : ScratchLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { |
| let AddedComplexity = 26; |
| } |
| |
| def : ScratchLoadSVaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> { |
| let SubtargetPredicate = HasFlatScratchSVSMode; |
| let AddedComplexity = 27; |
| } |
| } |
| |
| multiclass ScratchFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node, |
| ValueType vt> { |
| def : ScratchStoreSignedPat <inst, node, vt> { |
| let AddedComplexity = 25; |
| } |
| |
| def : ScratchStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { |
| let AddedComplexity = 26; |
| } |
| |
| def : ScratchStoreSVaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> { |
| let SubtargetPredicate = HasFlatScratchSVSMode; |
| let AddedComplexity = 27; |
| } |
| } |
| |
| multiclass ScratchFLATStorePats_t16<string inst, SDPatternOperator node, |
| ValueType vt> { |
| def : ScratchStoreSignedPat <!cast<FLAT_Pseudo>(inst#"_t16"), node, vt> { |
| let AddedComplexity = 25; |
| } |
| |
| def : ScratchStoreSaddrPat<!cast<FLAT_Pseudo>(inst#"_SADDR_t16"), node, vt> { |
| let AddedComplexity = 26; |
| } |
| |
| def : ScratchStoreSVaddrPat<!cast<FLAT_Pseudo>(inst#"_SVS_t16"), node, vt> { |
| let SubtargetPredicate = HasFlatScratchSVSMode; |
| let AddedComplexity = 27; |
| } |
| } |
| |
| multiclass ScratchFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> { |
| def : ScratchLoadSignedPat_D16 <inst, node, vt> { |
| let AddedComplexity = 25; |
| } |
| |
| def : ScratchLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> { |
| let AddedComplexity = 26; |
| } |
| |
| def : ScratchLoadSVaddrPat_D16 <!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> { |
| let SubtargetPredicate = HasFlatScratchSVSMode; |
| let AddedComplexity = 27; |
| } |
| } |
| |
| multiclass ScratchFLATLoadPats_D16_t16<string inst, SDPatternOperator node, ValueType vt> { |
| def : ScratchLoadSignedPat_D16_t16 <!cast<FLAT_Pseudo>(inst#"_t16"), node, vt> { |
| let AddedComplexity = 25; |
| } |
| |
| def : ScratchLoadSaddrPat_D16_t16<!cast<FLAT_Pseudo>(inst#"_SADDR_t16"), node, vt> { |
| let AddedComplexity = 26; |
| } |
| |
| def : ScratchLoadSVaddrPat_D16_t16 <!cast<FLAT_Pseudo>(inst#"_SVS_t16"), node, vt> { |
| let SubtargetPredicate = HasFlatScratchSVSMode; |
| let AddedComplexity = 27; |
| } |
| } |
| |
| let OtherPredicates = [HasFlatAddressSpace] in { |
| |
| def : FlatLoadPat <FLAT_LOAD_UBYTE, atomic_load_8_flat, i32>; |
| def : FlatLoadPat <FLAT_LOAD_UBYTE, atomic_load_8_flat, i16>; |
| def : FlatLoadPat <FLAT_LOAD_UBYTE, atomic_load_zext_8_flat, i32>; |
| def : FlatLoadPat <FLAT_LOAD_UBYTE, atomic_load_zext_8_flat, i16>; |
| def : FlatLoadPat <FLAT_LOAD_USHORT, atomic_load_16_flat, i32>; |
| def : FlatLoadPat <FLAT_LOAD_USHORT, atomic_load_16_flat, i16>; |
| def : FlatLoadPat <FLAT_LOAD_USHORT, atomic_load_zext_16_flat, i32>; |
| def : FlatLoadPat <FLAT_LOAD_USHORT, atomic_load_zext_16_flat, i16>; |
| def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i32>; |
| def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i32>; |
| def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i32>; |
| def : FlatLoadPat <FLAT_LOAD_SBYTE, atomic_load_sext_8_flat, i32>; |
| def : FlatLoadPat <FLAT_LOAD_SBYTE, atomic_load_sext_8_flat, i16>; |
| def : FlatLoadPat <FLAT_LOAD_USHORT, extloadi16_flat, i32>; |
| def : FlatLoadPat <FLAT_LOAD_USHORT, zextloadi16_flat, i32>; |
| def : FlatLoadPat <FLAT_LOAD_SSHORT, sextloadi16_flat, i32>; |
| def : FlatLoadPat <FLAT_LOAD_SSHORT, atomic_load_sext_16_flat, i32>; |
| def : FlatLoadPat <FLAT_LOAD_DWORDX3, load_flat, v3i32>; |
| |
| foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in |
| let True16Predicate = p in { |
| def : FlatLoadPat <FLAT_LOAD_UBYTE, extloadi8_flat, i16>; |
| def : FlatLoadPat <FLAT_LOAD_UBYTE, zextloadi8_flat, i16>; |
| def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>; |
| def : FlatLoadPat <FLAT_LOAD_USHORT, load_flat, i16>; |
| def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i16>; |
| def : FlatStorePat <FLAT_STORE_SHORT, store_flat, i16>; |
| } |
| |
| let OtherPredicates = [D16PreservesUnusedBits, HasFlatAddressSpace], True16Predicate = UseRealTrue16Insts in { |
| def : FlatLoadPat_D16_t16<FLAT_LOAD_UBYTE_D16_t16, extloadi8_flat, i16>; |
| def : FlatLoadPat_D16_t16<FLAT_LOAD_UBYTE_D16_t16, zextloadi8_flat, i16>; |
| def : FlatLoadPat_D16_t16<FLAT_LOAD_SBYTE_D16_t16, sextloadi8_flat, i16>; |
| def : FlatLoadPat_D16_t16<FLAT_LOAD_SHORT_D16_t16, load_flat, i16>; |
| def : FlatStorePat <FLAT_STORE_BYTE_t16, truncstorei8_flat, i16>; |
| def : FlatStorePat <FLAT_STORE_SHORT_t16, store_flat, i16>; |
| } // End let OtherPredicates = [D16PreservesUnusedBits, HasFlatAddressSpace], True16Predicate = UseRealTrue16Insts |
| |
| def : FlatLoadPat <FLAT_LOAD_DWORD, atomic_load_32_flat, i32>; |
| def : FlatLoadPat <FLAT_LOAD_DWORDX2, atomic_load_64_flat, i64>; |
| |
| def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i32>; |
| def : FlatStorePat <FLAT_STORE_SHORT, truncstorei16_flat, i32>; |
| |
| foreach vt = Reg32Types.types in { |
| def : FlatLoadPat <FLAT_LOAD_DWORD, load_flat, vt>; |
| def : FlatStorePat <FLAT_STORE_DWORD, store_flat, vt>; |
| } |
| |
| foreach vt = VReg_64.RegTypes in { |
| def : FlatStorePat <FLAT_STORE_DWORDX2, store_flat, vt>; |
| def : FlatLoadPat <FLAT_LOAD_DWORDX2, load_flat, vt>; |
| } |
| |
| def : FlatStorePat <FLAT_STORE_DWORDX3, store_flat, v3i32>; |
| |
| foreach vt = VReg_128.RegTypes in { |
| def : FlatLoadPat <FLAT_LOAD_DWORDX4, load_flat, vt>; |
| def : FlatStorePat <FLAT_STORE_DWORDX4, store_flat, vt>; |
| } |
| |
| def : FlatStorePat <FLAT_STORE_DWORD, atomic_store_32_flat, i32>; |
| def : FlatStorePat <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>; |
| def : FlatStorePat <FLAT_STORE_BYTE, atomic_store_8_flat, i32>; |
| def : FlatStorePat <FLAT_STORE_BYTE, atomic_store_8_flat, i16>; |
| def : FlatStorePat <FLAT_STORE_SHORT, atomic_store_16_flat, i32>; |
| def : FlatStorePat <FLAT_STORE_SHORT, atomic_store_16_flat, i16>; |
| |
| foreach as = [ "flat", "global" ] in { |
| defm : FlatAtomicPat <"FLAT_ATOMIC_ADD", "atomic_load_add_"#as, i32>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_SUB", "atomic_load_sub_"#as, i32>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_INC", "atomic_load_uinc_wrap_"#as, i32>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_DEC", "atomic_load_udec_wrap_"#as, i32>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_AND", "atomic_load_and_"#as, i32>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_SMAX", "atomic_load_max_"#as, i32>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_UMAX", "atomic_load_umax_"#as, i32>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_SMIN", "atomic_load_min_"#as, i32>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_UMIN", "atomic_load_umin_"#as, i32>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_OR", "atomic_load_or_"#as, i32>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_SWAP", "atomic_swap_"#as, i32>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_CMPSWAP", "AMDGPUatomic_cmp_swap_"#as, i32, v2i32>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_XOR", "atomic_load_xor_"#as, i32>; |
| |
| defm : FlatAtomicPat <"FLAT_ATOMIC_ADD_X2", "atomic_load_add_"#as, i64>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_SUB_X2", "atomic_load_sub_"#as, i64>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_INC_X2", "atomic_load_uinc_wrap_"#as, i64>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_DEC_X2", "atomic_load_udec_wrap_"#as, i64>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_AND_X2", "atomic_load_and_"#as, i64>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_SMAX_X2", "atomic_load_max_"#as, i64>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_UMAX_X2", "atomic_load_umax_"#as, i64>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_SMIN_X2", "atomic_load_min_"#as, i64>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_UMIN_X2", "atomic_load_umin_"#as, i64>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_OR_X2", "atomic_load_or_"#as, i64>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_SWAP_X2", "atomic_swap_"#as, i64>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_CMPSWAP_X2", "AMDGPUatomic_cmp_swap_"#as, i64, v2i64>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_XOR_X2", "atomic_load_xor_"#as, i64>; |
| |
| let SubtargetPredicate = HasAtomicFMinFMaxF32FlatInsts in { |
| defm : FlatAtomicPat <"FLAT_ATOMIC_FMIN", "atomic_load_fmin_"#as, f32>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_FMAX", "atomic_load_fmax_"#as, f32>; |
| } |
| |
| let SubtargetPredicate = HasAtomicFMinFMaxF64FlatInsts in { |
| defm : FlatAtomicPat <"FLAT_ATOMIC_MIN_F64", "atomic_load_fmin_"#as, f64>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_MAX_F64", "atomic_load_fmax_"#as, f64>; |
| } |
| |
| } // end foreach as |
| |
| let SubtargetPredicate = isGFX12Plus in { |
| defm : FlatAtomicRtnPatWithAddrSpace<"FLAT_ATOMIC_COND_SUB_U32", "int_amdgcn_atomic_cond_sub_u32", "flat_addrspace", i32 >; |
| |
| let OtherPredicates = [HasAtomicCSubNoRtnInsts] in |
| defm : FlatAtomicNoRtnPatWithAddrSpace<"FLAT_ATOMIC_COND_SUB_U32", "int_amdgcn_atomic_cond_sub_u32", "flat_addrspace", i32>; |
| } |
| |
| let OtherPredicates = [HasD16LoadStore] in { |
| def : FlatStorePat <FLAT_STORE_SHORT_D16_HI, truncstorei16_hi16_flat, i32>; |
| def : FlatStorePat <FLAT_STORE_BYTE_D16_HI, truncstorei8_hi16_flat, i32>; |
| } |
| |
| let OtherPredicates = [D16PreservesUnusedBits] in { |
| // TODO: Handle atomic loads |
| def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2i16>; |
| def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2f16>; |
| def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2i16>; |
| def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2f16>; |
| def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2i16>; |
| def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2f16>; |
| |
| def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2i16>; |
| def : FlatLoadPat_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2f16>; |
| def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2i16>; |
| def : FlatLoadPat_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2f16>; |
| def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2i16>; |
| def : FlatLoadPat_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2f16>; |
| } |
| |
| } // End OtherPredicates = [HasFlatAddressSpace] |
| |
| let OtherPredicates = [HasFlatGlobalInsts] in { |
| |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_8_global, i32>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_8_global, i16>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_zext_8_global, i32>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_zext_8_global, i16>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_16_global, i32>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_16_global, i16>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i32>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i16>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, atomic_load_sext_8_global, i32>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, atomic_load_sext_8_global, i16>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i32>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i32>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i32>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, extloadi16_global, i32>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, zextloadi16_global, i32>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_SSHORT, sextloadi16_global, i32>; |
| |
| foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in |
| let OtherPredicates = [HasFlatGlobalInsts], True16Predicate = p in { |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i16>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i16>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_SSHORT, atomic_load_sext_16_global, i32>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i32>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, load_global, i16>; |
| } |
| |
| let OtherPredicates = [HasFlatGlobalInsts, D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts in { |
| defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_UBYTE_D16", extloadi8_global, i16>; |
| defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_UBYTE_D16", zextloadi8_global, i16>; |
| defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SBYTE_D16", sextloadi8_global, i16>; |
| defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SHORT_D16", load_global, i16>; |
| defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_BYTE", truncstorei8_global, i16>; |
| defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_SHORT", store_global, i16>; |
| } // end OtherPredicates = [HasFlatGlobalInsts, D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts |
| |
| foreach vt = Reg32Types.types in { |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, load_global, vt>; |
| defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, store_global, vt>; |
| } |
| |
| foreach vt = VReg_64.RegTypes in { |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, load_global, vt>; |
| defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, store_global, vt>; |
| } |
| |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX3, load_global, v3i32>; |
| |
| foreach vt = VReg_128.RegTypes in { |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, load_global, vt>; |
| defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>; |
| } |
| |
| // There is no distinction for atomic load lowering during selection; |
| // the memory legalizer will set the cache bits and insert the |
| // appropriate waits. |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_32_global, i32>; |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_64_global, i64>; |
| |
| defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>; |
| defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, truncstorei16_global, i32>; |
| defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX3, store_global, v3i32>; |
| |
| foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in |
| let OtherPredicates = [HasFlatGlobalInsts], True16Predicate = p in { |
| defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i16>; |
| defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, store_global, i16>; |
| } |
| |
| let OtherPredicates = [HasD16LoadStore] in { |
| defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT_D16_HI, truncstorei16_hi16_global, i32>; |
| defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE_D16_HI, truncstorei8_hi16_global, i32>; |
| } |
| |
| let OtherPredicates = [D16PreservesUnusedBits] in { |
| // TODO: Handle atomic loads |
| defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2i16>; |
| defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2f16>; |
| defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2i16>; |
| defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2f16>; |
| defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2i16>; |
| defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2f16>; |
| |
| defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2i16>; |
| defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2f16>; |
| defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2i16>; |
| defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2f16>; |
| defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2i16>; |
| defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>; |
| } |
| |
| defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i32>; |
| defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i16>; |
| defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i32>; |
| defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i16>; |
| defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>; |
| defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>; |
| |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", "atomic_load_add_global", i32>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", "atomic_load_sub_global", i32>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC", "atomic_load_uinc_wrap_global", i32>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC", "atomic_load_udec_wrap_global", i32>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND", "atomic_load_and_global", i32>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX", "atomic_load_max_global", i32>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX", "atomic_load_umax_global", i32>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN", "atomic_load_min_global", i32>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN", "atomic_load_umin_global", i32>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR", "atomic_load_or_global", i32>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP", "atomic_swap_global", i32>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP", "AMDGPUatomic_cmp_swap_global", i32, v2i32>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR", "atomic_load_xor_global", i32>; |
| defm : GlobalFLATAtomicPatsRtn <"GLOBAL_ATOMIC_CSUB", "int_amdgcn_global_atomic_csub", i32, i32, /* isIntr */ 1>; |
| |
| let OtherPredicates = [HasAtomicCSubNoRtnInsts] in |
| defm : GlobalFLATAtomicPatsNoRtn <"GLOBAL_ATOMIC_CSUB", "int_amdgcn_global_atomic_csub", i32, i32, /* isIntr */ 1>; |
| |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_X2", "atomic_load_add_global", i64>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB_X2", "atomic_load_sub_global", i64>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC_X2", "atomic_load_uinc_wrap_global", i64>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC_X2", "atomic_load_udec_wrap_global", i64>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND_X2", "atomic_load_and_global", i64>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX_X2", "atomic_load_max_global", i64>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX_X2", "atomic_load_umax_global", i64>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN_X2", "atomic_load_min_global", i64>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN_X2", "atomic_load_umin_global", i64>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR_X2", "atomic_load_or_global", i64>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP_X2", "atomic_swap_global", i64>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP_X2", "AMDGPUatomic_cmp_swap_global", i64, v2i64>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR_X2", "atomic_load_xor_global", i64>; |
| |
| let SubtargetPredicate = isGFX12Plus in { |
| defm : GlobalFLATAtomicPatsRtnWithAddrSpace <"GLOBAL_ATOMIC_COND_SUB_U32", "int_amdgcn_atomic_cond_sub_u32", "global_addrspace", i32>; |
| |
| let OtherPredicates = [HasAtomicCSubNoRtnInsts] in |
| defm : GlobalFLATAtomicPatsNoRtnWithAddrSpace <"GLOBAL_ATOMIC_COND_SUB_U32", "int_amdgcn_atomic_cond_sub_u32", "global_addrspace", i32>; |
| } |
| |
| let OtherPredicates = [isGFX12Plus] in { |
| defm : GlobalFLATAtomicPatsRtn <"GLOBAL_ATOMIC_ORDERED_ADD_B64", "int_amdgcn_global_atomic_ordered_add_b64", i64, i64, /* isIntr */ 1>; |
| |
| let WaveSizePredicate = isWave32 in { |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B64_w32, int_amdgcn_global_load_tr_b64, v2i32>; |
| foreach vt = [v8i16, v8f16, v8bf16] in |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w32, int_amdgcn_global_load_tr_b128, vt>; |
| } |
| let WaveSizePredicate = isWave64 in { |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B64_w64, int_amdgcn_global_load_tr_b64, i32>; |
| foreach vt = [v4i16, v4f16, v4bf16] in |
| defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w64, int_amdgcn_global_load_tr_b128, vt>; |
| } |
| } |
| |
| let SubtargetPredicate = HasAtomicFMinFMaxF32GlobalInsts, OtherPredicates = [HasFlatGlobalInsts] in { |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMIN", "atomic_load_fmin_global", f32>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMAX", "atomic_load_fmax_global", f32>; |
| } |
| |
| let SubtargetPredicate = HasAtomicFMinFMaxF32FlatInsts in { |
| defm : FlatAtomicPat <"FLAT_ATOMIC_FMIN", "atomic_load_fmin_flat", f32>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_FMAX", "atomic_load_fmax_flat", f32>; |
| } |
| |
| let OtherPredicates = [isGFX12Only] in { |
| // FIXME: Remove these intrinsics |
| defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_FMIN", "int_amdgcn_global_atomic_fmin_num", f32>; |
| defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_FMAX", "int_amdgcn_global_atomic_fmax_num", f32>; |
| defm : FlatAtomicIntrPat <"FLAT_ATOMIC_FMIN", "int_amdgcn_flat_atomic_fmin_num", f32>; |
| defm : FlatAtomicIntrPat <"FLAT_ATOMIC_FMAX", "int_amdgcn_flat_atomic_fmax_num", f32>; |
| } |
| |
| let OtherPredicates = [HasAtomicFaddNoRtnInsts] in { |
| defm : GlobalFLATAtomicPatsNoRtn <"GLOBAL_ATOMIC_ADD_F32", "atomic_load_fadd_global", f32>; |
| } |
| |
| let OtherPredicates = [HasAtomicBufferGlobalPkAddF16NoRtnInsts] in { |
| defm : GlobalFLATAtomicPatsNoRtn <"GLOBAL_ATOMIC_PK_ADD_F16", "atomic_load_fadd_global", v2f16>; |
| } |
| |
| let OtherPredicates = [HasAtomicFaddRtnInsts] in { |
| defm : GlobalFLATAtomicPatsRtn <"GLOBAL_ATOMIC_ADD_F32", "atomic_load_fadd_global", f32>; |
| } |
| |
| let OtherPredicates = [HasAtomicBufferGlobalPkAddF16Insts] in { |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_PK_ADD_F16", "atomic_load_fadd_global", v2f16>; |
| } |
| |
| let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts, OtherPredicates = [HasFlatGlobalInsts] in { |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_MIN_F64", "atomic_load_fmin_global", f64>; |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_MAX_F64", "atomic_load_fmax_global", f64>; |
| } |
| |
| let OtherPredicates = [HasFlatBufferGlobalAtomicFaddF64Inst] in { |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_F64", "atomic_load_fadd_global", f64>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_ADD_F64", "atomic_load_fadd_flat", f64>; |
| } |
| |
| let OtherPredicates = [HasFlatAtomicFaddF32Inst] in { |
| defm : FlatAtomicPat <"FLAT_ATOMIC_ADD_F32", "atomic_load_fadd_flat", f32>; |
| } |
| |
| let OtherPredicates = [HasAtomicFlatPkAdd16Insts] in { |
| defm : FlatAtomicPat <"FLAT_ATOMIC_PK_ADD_F16", "atomic_load_fadd_flat", v2f16>; |
| defm : FlatAtomicPat <"FLAT_ATOMIC_PK_ADD_BF16", "atomic_load_fadd_flat", v2bf16>; |
| } |
| |
| let OtherPredicates = [HasAtomicGlobalPkAddBF16Inst] in |
| defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_PK_ADD_BF16", "atomic_load_fadd_global", v2bf16>; |
| } // End OtherPredicates = [HasFlatGlobalInsts], AddedComplexity = 10 |
| |
| let OtherPredicates = [HasFlatScratchInsts, EnableFlatScratch] in { |
| |
| defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, extloadi8_private, i32>; |
| defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, zextloadi8_private, i32>; |
| defm : ScratchFLATLoadPats <SCRATCH_LOAD_SBYTE, sextloadi8_private, i32>; |
| defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, extloadi16_private, i32>; |
| defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, zextloadi16_private, i32>; |
| defm : ScratchFLATLoadPats <SCRATCH_LOAD_SSHORT, sextloadi16_private, i32>; |
| |
| foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in |
| let True16Predicate = p in { |
| defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, extloadi8_private, i16>; |
| defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, zextloadi8_private, i16>; |
| defm : ScratchFLATLoadPats <SCRATCH_LOAD_SBYTE, sextloadi8_private, i16>; |
| defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, load_private, i16>; |
| defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT, store_private, i16>; |
| defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE, truncstorei8_private, i16>; |
| } |
| |
| let True16Predicate = UseRealTrue16Insts in { |
| defm : ScratchFLATLoadPats_D16_t16<"SCRATCH_LOAD_UBYTE_D16", extloadi8_private, i16>; |
| defm : ScratchFLATLoadPats_D16_t16<"SCRATCH_LOAD_UBYTE_D16", zextloadi8_private, i16>; |
| defm : ScratchFLATLoadPats_D16_t16<"SCRATCH_LOAD_SBYTE_D16", sextloadi8_private, i16>; |
| defm : ScratchFLATLoadPats_D16_t16<"SCRATCH_LOAD_SHORT_D16", load_private, i16>; |
| defm : ScratchFLATStorePats_t16 <"SCRATCH_STORE_SHORT", store_private, i16>; |
| defm : ScratchFLATStorePats_t16 <"SCRATCH_STORE_BYTE", truncstorei8_private, i16>; |
| } // End True16Predicate = UseRealTrue16Insts |
| |
| foreach vt = Reg32Types.types in { |
| defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORD, load_private, vt>; |
| defm : ScratchFLATStorePats <SCRATCH_STORE_DWORD, store_private, vt>; |
| } |
| |
| foreach vt = VReg_64.RegTypes in { |
| defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX2, load_private, vt>; |
| defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX2, store_private, vt>; |
| } |
| |
| defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX3, load_private, v3i32>; |
| |
| foreach vt = VReg_128.RegTypes in { |
| defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX4, load_private, vt>; |
| defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX4, store_private, vt>; |
| } |
| |
| defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE, truncstorei8_private, i32>; |
| defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT, truncstorei16_private, i32>; |
| defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX3, store_private, v3i32>; |
| |
| let OtherPredicates = [HasD16LoadStore, HasFlatScratchInsts, EnableFlatScratch] in { |
| defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT_D16_HI, truncstorei16_hi16_private, i32>; |
| defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE_D16_HI, truncstorei8_hi16_private, i32>; |
| } |
| |
| let OtherPredicates = [D16PreservesUnusedBits, HasFlatScratchInsts, EnableFlatScratch] in { |
| defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_private, v2i16>; |
| defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_private, v2f16>; |
| defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_private, v2i16>; |
| defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_private, v2f16>; |
| defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16_HI, load_d16_hi_private, v2i16>; |
| defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16_HI, load_d16_hi_private, v2f16>; |
| |
| defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16, az_extloadi8_d16_lo_private, v2i16>; |
| defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16, az_extloadi8_d16_lo_private, v2f16>; |
| defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16, sextloadi8_d16_lo_private, v2i16>; |
| defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16, sextloadi8_d16_lo_private, v2f16>; |
| defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16, load_d16_lo_private, v2i16>; |
| defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16, load_d16_lo_private, v2f16>; |
| } |
| |
| } // End OtherPredicates = [HasFlatScratchInsts,EnableFlatScratch] |
| |
| //===----------------------------------------------------------------------===// |
| // Target |
| //===----------------------------------------------------------------------===// |
| |
| //===----------------------------------------------------------------------===// |
| // CI |
| //===----------------------------------------------------------------------===// |
| |
| class FLAT_Real_ci <bits<7> op, FLAT_Pseudo ps, string asmName = ps.Mnemonic> : |
| FLAT_Real <op, ps, asmName>, |
| SIMCInstr <ps.PseudoInstr, SIEncodingFamily.SI> { |
| let AssemblerPredicate = isGFX7Only; |
| let DecoderNamespace="GFX7"; |
| } |
| |
| def FLAT_LOAD_UBYTE_ci : FLAT_Real_ci <0x8, FLAT_LOAD_UBYTE>; |
| def FLAT_LOAD_SBYTE_ci : FLAT_Real_ci <0x9, FLAT_LOAD_SBYTE>; |
| def FLAT_LOAD_USHORT_ci : FLAT_Real_ci <0xa, FLAT_LOAD_USHORT>; |
| def FLAT_LOAD_SSHORT_ci : FLAT_Real_ci <0xb, FLAT_LOAD_SSHORT>; |
| def FLAT_LOAD_DWORD_ci : FLAT_Real_ci <0xc, FLAT_LOAD_DWORD>; |
| def FLAT_LOAD_DWORDX2_ci : FLAT_Real_ci <0xd, FLAT_LOAD_DWORDX2>; |
| def FLAT_LOAD_DWORDX4_ci : FLAT_Real_ci <0xe, FLAT_LOAD_DWORDX4>; |
| def FLAT_LOAD_DWORDX3_ci : FLAT_Real_ci <0xf, FLAT_LOAD_DWORDX3>; |
| |
| def FLAT_STORE_BYTE_ci : FLAT_Real_ci <0x18, FLAT_STORE_BYTE>; |
| def FLAT_STORE_SHORT_ci : FLAT_Real_ci <0x1a, FLAT_STORE_SHORT>; |
| def FLAT_STORE_DWORD_ci : FLAT_Real_ci <0x1c, FLAT_STORE_DWORD>; |
| def FLAT_STORE_DWORDX2_ci : FLAT_Real_ci <0x1d, FLAT_STORE_DWORDX2>; |
| def FLAT_STORE_DWORDX4_ci : FLAT_Real_ci <0x1e, FLAT_STORE_DWORDX4>; |
| def FLAT_STORE_DWORDX3_ci : FLAT_Real_ci <0x1f, FLAT_STORE_DWORDX3>; |
| |
| multiclass FLAT_Real_Atomics_ci <bits<7> op, string opName = NAME, |
| string asmName = !cast<FLAT_Pseudo>(opName).Mnemonic> { |
| defvar ps = !cast<FLAT_Pseudo>(opName); |
| defvar ps_rtn = !cast<FLAT_Pseudo>(opName#"_RTN"); |
| |
| def _ci : FLAT_Real_ci<op, ps, asmName>; |
| def _RTN_ci : FLAT_Real_ci<op, ps_rtn, asmName>; |
| } |
| |
| defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_ci <0x30>; |
| defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_ci <0x31>; |
| defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_ci <0x32>; |
| defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_ci <0x33>; |
| defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_ci <0x35>; |
| defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_ci <0x36>; |
| defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_ci <0x37>; |
| defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_ci <0x38>; |
| defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_ci <0x39>; |
| defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_ci <0x3a>; |
| defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_ci <0x3b>; |
| defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_ci <0x3c>; |
| defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_ci <0x3d>; |
| defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_ci <0x50>; |
| defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_ci <0x51>; |
| defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_ci <0x52>; |
| defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_ci <0x53>; |
| defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_ci <0x55>; |
| defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_ci <0x56>; |
| defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_ci <0x57>; |
| defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_ci <0x58>; |
| defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_ci <0x59>; |
| defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_ci <0x5a>; |
| defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_ci <0x5b>; |
| defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_ci <0x5c>; |
| defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_ci <0x5d>; |
| |
| // CI Only flat instructions |
| defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_ci <0x3e>; |
| defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_ci <0x3f>; |
| defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_ci <0x40>; |
| defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_ci <0x5e>; |
| defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_ci <0x5f, "FLAT_ATOMIC_MIN_F64", "flat_atomic_fmin_x2">; |
| defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_ci <0x60, "FLAT_ATOMIC_MAX_F64", "flat_atomic_fmax_x2">; |
| |
| |
| //===----------------------------------------------------------------------===// |
| // VI |
| //===----------------------------------------------------------------------===// |
| |
| class FLAT_Real_vi <bits<7> op, FLAT_Pseudo ps, bit has_sccb = ps.has_sccb> : |
| FLAT_Real <op, ps>, |
| SIMCInstr <ps.PseudoInstr, SIEncodingFamily.VI> { |
| let AssemblerPredicate = isGFX8GFX9; |
| let DecoderNamespace = "GFX8"; |
| |
| let Inst{25} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccbValue); |
| let AsmString = ps.Mnemonic # |
| !subst("$sccb", !if(has_sccb, "$sccb",""), ps.AsmOperands); |
| } |
| |
| multiclass FLAT_Real_AllAddr_vi<bits<7> op, |
| bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> { |
| def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME), has_sccb>; |
| def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR"), has_sccb>; |
| } |
| |
| class FLAT_Real_gfx940 <bits<7> op, FLAT_Pseudo ps> : |
| FLAT_Real <op, ps>, |
| SIMCInstr <ps.PseudoInstr, SIEncodingFamily.GFX940> { |
| let AssemblerPredicate = isGFX940Plus; |
| let DecoderNamespace = "GFX9"; |
| let Inst{13} = ps.sve; |
| let Inst{25} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccbValue); |
| } |
| |
| multiclass FLAT_Real_AllAddr_SVE_vi<bits<7> op> { |
| def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME)> { |
| let AssemblerPredicate = isGFX8GFX9NotGFX940; |
| let OtherPredicates = [isGFX8GFX9NotGFX940]; |
| } |
| def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")> { |
| let DecoderNamespace = "GFX9"; |
| } |
| let AssemblerPredicate = isGFX940Plus in { |
| def _VE_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME)>; |
| def _SVS_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_SVS")>; |
| def _ST_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_ST")>; |
| } |
| } |
| |
| multiclass FLAT_Real_AllAddr_LDS<bits<7> op, bits<7> pre_gfx940_op, |
| string pre_gfx940_name = !subst("_lds", "", !cast<FLAT_Pseudo>(NAME).Mnemonic), |
| bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> { |
| |
| let OtherPredicates = [isGFX8GFX9NotGFX940] in { |
| def _vi : FLAT_Real_vi<pre_gfx940_op, !cast<FLAT_Pseudo>(NAME), has_sccb> { |
| let AsmString = pre_gfx940_name # !cast<FLAT_Pseudo>(NAME).AsmOperands # " lds"; |
| } |
| def _SADDR_vi : FLAT_Real_vi<pre_gfx940_op, !cast<FLAT_Pseudo>(NAME#"_SADDR"), has_sccb> { |
| let AsmString = pre_gfx940_name # !cast<FLAT_Pseudo>(NAME#"_SADDR").AsmOperands # " lds"; |
| } |
| } |
| |
| let AssemblerPredicate = isGFX940Plus in { |
| def _gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME)>; |
| def _SADDR_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>; |
| } |
| } |
| |
| multiclass FLAT_Real_AllAddr_SVE_LDS<bits<7> op, bits<7> pre_gfx940_op> { |
| defm "" : FLAT_Real_AllAddr_LDS<op, pre_gfx940_op>; |
| def _SVS_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_SVS")>; |
| def _ST_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_ST")>; |
| } |
| |
| def FLAT_LOAD_UBYTE_vi : FLAT_Real_vi <0x10, FLAT_LOAD_UBYTE>; |
| def FLAT_LOAD_SBYTE_vi : FLAT_Real_vi <0x11, FLAT_LOAD_SBYTE>; |
| def FLAT_LOAD_USHORT_vi : FLAT_Real_vi <0x12, FLAT_LOAD_USHORT>; |
| def FLAT_LOAD_SSHORT_vi : FLAT_Real_vi <0x13, FLAT_LOAD_SSHORT>; |
| def FLAT_LOAD_DWORD_vi : FLAT_Real_vi <0x14, FLAT_LOAD_DWORD>; |
| def FLAT_LOAD_DWORDX2_vi : FLAT_Real_vi <0x15, FLAT_LOAD_DWORDX2>; |
| def FLAT_LOAD_DWORDX4_vi : FLAT_Real_vi <0x17, FLAT_LOAD_DWORDX4>; |
| def FLAT_LOAD_DWORDX3_vi : FLAT_Real_vi <0x16, FLAT_LOAD_DWORDX3>; |
| |
| def FLAT_STORE_BYTE_vi : FLAT_Real_vi <0x18, FLAT_STORE_BYTE>; |
| def FLAT_STORE_BYTE_D16_HI_vi : FLAT_Real_vi <0x19, FLAT_STORE_BYTE_D16_HI>; |
| def FLAT_STORE_SHORT_vi : FLAT_Real_vi <0x1a, FLAT_STORE_SHORT>; |
| def FLAT_STORE_SHORT_D16_HI_vi : FLAT_Real_vi <0x1b, FLAT_STORE_SHORT_D16_HI>; |
| def FLAT_STORE_DWORD_vi : FLAT_Real_vi <0x1c, FLAT_STORE_DWORD>; |
| def FLAT_STORE_DWORDX2_vi : FLAT_Real_vi <0x1d, FLAT_STORE_DWORDX2>; |
| def FLAT_STORE_DWORDX4_vi : FLAT_Real_vi <0x1f, FLAT_STORE_DWORDX4>; |
| def FLAT_STORE_DWORDX3_vi : FLAT_Real_vi <0x1e, FLAT_STORE_DWORDX3>; |
| |
| def FLAT_LOAD_UBYTE_D16_vi : FLAT_Real_vi <0x20, FLAT_LOAD_UBYTE_D16>; |
| def FLAT_LOAD_UBYTE_D16_HI_vi : FLAT_Real_vi <0x21, FLAT_LOAD_UBYTE_D16_HI>; |
| def FLAT_LOAD_SBYTE_D16_vi : FLAT_Real_vi <0x22, FLAT_LOAD_SBYTE_D16>; |
| def FLAT_LOAD_SBYTE_D16_HI_vi : FLAT_Real_vi <0x23, FLAT_LOAD_SBYTE_D16_HI>; |
| def FLAT_LOAD_SHORT_D16_vi : FLAT_Real_vi <0x24, FLAT_LOAD_SHORT_D16>; |
| def FLAT_LOAD_SHORT_D16_HI_vi : FLAT_Real_vi <0x25, FLAT_LOAD_SHORT_D16_HI>; |
| |
| multiclass FLAT_Real_Atomics_vi <bits<7> op, |
| bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> { |
| defvar ps = !cast<FLAT_Pseudo>(NAME); |
| def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr), has_sccb>; |
| def _RTN_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN"), has_sccb>; |
| } |
| |
| multiclass FLAT_Global_Real_Atomics_vi<bits<7> op, |
| bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> : |
| FLAT_Real_AllAddr_vi<op, has_sccb> { |
| def _RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_RTN"), has_sccb>; |
| def _SADDR_RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN"), has_sccb>; |
| } |
| |
| |
| defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_vi <0x40>; |
| defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_vi <0x41>; |
| defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_vi <0x42>; |
| defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_vi <0x43>; |
| defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_vi <0x44>; |
| defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_vi <0x45>; |
| defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_vi <0x46>; |
| defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_vi <0x47>; |
| defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_vi <0x48>; |
| defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_vi <0x49>; |
| defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_vi <0x4a>; |
| defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_vi <0x4b>; |
| defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_vi <0x4c>; |
| defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_vi <0x60>; |
| defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_vi <0x61>; |
| defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_vi <0x62>; |
| defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_vi <0x63>; |
| defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_vi <0x64>; |
| defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_vi <0x65>; |
| defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_vi <0x66>; |
| defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_vi <0x67>; |
| defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_vi <0x68>; |
| defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_vi <0x69>; |
| defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_vi <0x6a>; |
| defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_vi <0x6b>; |
| defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_vi <0x6c>; |
| |
| defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>; |
| defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>; |
| defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>; |
| defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>; |
| defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>; |
| defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>; |
| defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>; |
| defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>; |
| |
| defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>; |
| defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>; |
| defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>; |
| defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>; |
| defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>; |
| defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>; |
| |
| defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>; |
| defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>; |
| defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>; |
| defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>; |
| defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>; |
| defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>; |
| defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>; |
| defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>; |
| |
| defm GLOBAL_LOAD_LDS_UBYTE : FLAT_Real_AllAddr_LDS <0x026, 0x10>; |
| defm GLOBAL_LOAD_LDS_SBYTE : FLAT_Real_AllAddr_LDS <0x027, 0x11>; |
| defm GLOBAL_LOAD_LDS_USHORT : FLAT_Real_AllAddr_LDS <0x028, 0x12>; |
| defm GLOBAL_LOAD_LDS_SSHORT : FLAT_Real_AllAddr_LDS <0x029, 0x13>; |
| defm GLOBAL_LOAD_LDS_DWORD : FLAT_Real_AllAddr_LDS <0x02a, 0x14>; |
| |
| defm GLOBAL_LOAD_LDS_DWORDX3 : FLAT_Real_AllAddr_LDS <0x07e, 0x07e>; |
| defm GLOBAL_LOAD_LDS_DWORDX4 : FLAT_Real_AllAddr_LDS <0x07d, 0x07d>; |
| |
| |
| defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Real_Atomics_vi <0x40>; |
| defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Real_Atomics_vi <0x41>; |
| defm GLOBAL_ATOMIC_ADD : FLAT_Global_Real_Atomics_vi <0x42>; |
| defm GLOBAL_ATOMIC_SUB : FLAT_Global_Real_Atomics_vi <0x43>; |
| defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Real_Atomics_vi <0x44>; |
| defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Real_Atomics_vi <0x45>; |
| defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Real_Atomics_vi <0x46>; |
| defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Real_Atomics_vi <0x47>; |
| defm GLOBAL_ATOMIC_AND : FLAT_Global_Real_Atomics_vi <0x48>; |
| defm GLOBAL_ATOMIC_OR : FLAT_Global_Real_Atomics_vi <0x49>; |
| defm GLOBAL_ATOMIC_XOR : FLAT_Global_Real_Atomics_vi <0x4a>; |
| defm GLOBAL_ATOMIC_INC : FLAT_Global_Real_Atomics_vi <0x4b>; |
| defm GLOBAL_ATOMIC_DEC : FLAT_Global_Real_Atomics_vi <0x4c>; |
| defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Real_Atomics_vi <0x60>; |
| defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Real_Atomics_vi <0x61>; |
| defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Real_Atomics_vi <0x62>; |
| defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Real_Atomics_vi <0x63>; |
| defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Real_Atomics_vi <0x64>; |
| defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Real_Atomics_vi <0x65>; |
| defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Real_Atomics_vi <0x66>; |
| defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Real_Atomics_vi <0x67>; |
| defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Real_Atomics_vi <0x68>; |
| defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Real_Atomics_vi <0x69>; |
| defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Real_Atomics_vi <0x6a>; |
| defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Real_Atomics_vi <0x6b>; |
| defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Real_Atomics_vi <0x6c>; |
| |
| defm SCRATCH_LOAD_LDS_UBYTE : FLAT_Real_AllAddr_SVE_LDS <0x026, 0x10>; |
| defm SCRATCH_LOAD_LDS_SBYTE : FLAT_Real_AllAddr_SVE_LDS <0x027, 0x11>; |
| defm SCRATCH_LOAD_LDS_USHORT : FLAT_Real_AllAddr_SVE_LDS <0x028, 0x12>; |
| defm SCRATCH_LOAD_LDS_SSHORT : FLAT_Real_AllAddr_SVE_LDS <0x029, 0x13>; |
| defm SCRATCH_LOAD_LDS_DWORD : FLAT_Real_AllAddr_SVE_LDS <0x02a, 0x14>; |
| |
| defm SCRATCH_LOAD_UBYTE : FLAT_Real_AllAddr_SVE_vi <0x10>; |
| defm SCRATCH_LOAD_SBYTE : FLAT_Real_AllAddr_SVE_vi <0x11>; |
| defm SCRATCH_LOAD_USHORT : FLAT_Real_AllAddr_SVE_vi <0x12>; |
| defm SCRATCH_LOAD_SSHORT : FLAT_Real_AllAddr_SVE_vi <0x13>; |
| defm SCRATCH_LOAD_DWORD : FLAT_Real_AllAddr_SVE_vi <0x14>; |
| defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_AllAddr_SVE_vi <0x15>; |
| defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_AllAddr_SVE_vi <0x16>; |
| defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_AllAddr_SVE_vi <0x17>; |
| defm SCRATCH_STORE_BYTE : FLAT_Real_AllAddr_SVE_vi <0x18>; |
| defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x19>; |
| defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_SVE_vi <0x20>; |
| defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x21>; |
| defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_SVE_vi <0x22>; |
| defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x23>; |
| defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_AllAddr_SVE_vi <0x24>; |
| defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x25>; |
| defm SCRATCH_STORE_SHORT : FLAT_Real_AllAddr_SVE_vi <0x1a>; |
| defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x1b>; |
| defm SCRATCH_STORE_DWORD : FLAT_Real_AllAddr_SVE_vi <0x1c>; |
| defm SCRATCH_STORE_DWORDX2 : FLAT_Real_AllAddr_SVE_vi <0x1d>; |
| defm SCRATCH_STORE_DWORDX3 : FLAT_Real_AllAddr_SVE_vi <0x1e>; |
| defm SCRATCH_STORE_DWORDX4 : FLAT_Real_AllAddr_SVE_vi <0x1f>; |
| |
| let AssemblerPredicate = isGFX8GFX9NotGFX940 in { |
| // These instructions are encoded differently on gfx90* and gfx94*. |
| defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Real_Atomics_vi <0x04d, 0>; |
| defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Real_Atomics_vi <0x04e, 0>; |
| } |
| |
| defm FLAT_ATOMIC_ADD_F64 : FLAT_Real_Atomics_vi<0x4f, 0>; |
| defm FLAT_ATOMIC_MIN_F64 : FLAT_Real_Atomics_vi<0x50, 0>; |
| defm FLAT_ATOMIC_MAX_F64 : FLAT_Real_Atomics_vi<0x51, 0>; |
| defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Real_Atomics_vi<0x4f, 0>; |
| defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Real_Atomics_vi<0x50, 0>; |
| defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Real_Atomics_vi<0x51, 0>; |
| |
| multiclass FLAT_Real_AllAddr_gfx940<bits<7> op> { |
| def _gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME)>; |
| def _SADDR_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>; |
| } |
| |
| multiclass FLAT_Real_Atomics_gfx940 <bits<7> op> { |
| defvar ps = !cast<FLAT_Pseudo>(NAME); |
| def _gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>; |
| def _RTN_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>; |
| } |
| |
| multiclass FLAT_Global_Real_Atomics_gfx940<bits<7> op> : |
| FLAT_Real_AllAddr_gfx940<op> { |
| def _RTN_gfx940 : FLAT_Real_gfx940 <op, !cast<FLAT_Pseudo>(NAME#"_RTN")>; |
| def _SADDR_RTN_gfx940 : FLAT_Real_gfx940 <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>; |
| } |
| |
| let AssemblerPredicate = isGFX940Plus in { |
| // These instructions are encoded differently on gfx90* and gfx940. |
| defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Real_Atomics_gfx940 <0x04d>; |
| defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Real_Atomics_gfx940 <0x04e>; |
| |
| defm FLAT_ATOMIC_ADD_F64 : FLAT_Real_Atomics_gfx940<0x4f>; |
| defm FLAT_ATOMIC_MIN_F64 : FLAT_Real_Atomics_gfx940<0x50>; |
| defm FLAT_ATOMIC_MAX_F64 : FLAT_Real_Atomics_gfx940<0x51>; |
| defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Real_Atomics_gfx940<0x4f>; |
| defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Real_Atomics_gfx940<0x50>; |
| defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Real_Atomics_gfx940<0x51>; |
| defm FLAT_ATOMIC_ADD_F32 : FLAT_Real_Atomics_vi<0x4d>; |
| defm FLAT_ATOMIC_PK_ADD_F16 : FLAT_Real_Atomics_vi<0x4e>; |
| defm FLAT_ATOMIC_PK_ADD_BF16 : FLAT_Real_Atomics_vi<0x52>; |
| defm GLOBAL_ATOMIC_PK_ADD_BF16 : FLAT_Global_Real_Atomics_vi<0x52>; |
| } // End AssemblerPredicate = isGFX940Plus |
| |
| //===----------------------------------------------------------------------===// |
| // GFX10. |
| //===----------------------------------------------------------------------===// |
| |
| class FLAT_Real_gfx10<bits<7> op, FLAT_Pseudo ps, string opName = ps.Mnemonic> : |
| FLAT_Real<op, ps, opName>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX10> { |
| let AssemblerPredicate = isGFX10Only; |
| let DecoderNamespace = "GFX10"; |
| |
| let Inst{11-0} = offset{11-0}; |
| let Inst{12} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlcValue); |
| let Inst{54-48} = !cond(ps.enabled_saddr : saddr, |
| !and(ps.is_flat_scratch, !not(ps.has_vaddr)) : EXEC_HI.Index{6-0}, // ST mode |
| true : SGPR_NULL_gfxpre11.Index{6-0}); |
| let Inst{55} = 0; |
| } |
| |
| multiclass FLAT_Real_Base_gfx10<bits<7> op, string psName = NAME, |
| string asmName = !cast<FLAT_Pseudo>(psName).Mnemonic> { |
| def _gfx10 : |
| FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(psName), asmName>; |
| } |
| |
| multiclass FLAT_Real_RTN_gfx10<bits<7> op, string psName = NAME, |
| string asmName = !cast<FLAT_Pseudo>(psName).Mnemonic> { |
| def _RTN_gfx10 : |
| FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(psName#"_RTN"), asmName>; |
| } |
| |
| multiclass FLAT_Real_SADDR_gfx10<bits<7> op, string psName = NAME, |
| string asmName = !cast<FLAT_Pseudo>(psName#"_SADDR").Mnemonic> { |
| def _SADDR_gfx10 : |
| FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(psName#"_SADDR"), asmName>; |
| } |
| |
| multiclass FLAT_Real_SADDR_RTN_gfx10<bits<7> op, string psName = NAME, |
| string asmName = !cast<FLAT_Pseudo>(psName#"_SADDR_RTN").Mnemonic> { |
| def _SADDR_RTN_gfx10 : |
| FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(psName#"_SADDR_RTN"), asmName>; |
| } |
| |
| multiclass FLAT_Real_ST_gfx10<bits<7> op> { |
| def _ST_gfx10 : |
| FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_ST")>; |
| } |
| |
| multiclass FLAT_Real_AllAddr_gfx10<bits<7> op, string OpName = NAME, |
| string asmName = !cast<FLAT_Pseudo>(OpName).Mnemonic> : |
| FLAT_Real_Base_gfx10<op, OpName, asmName>, |
| FLAT_Real_SADDR_gfx10<op, OpName, asmName>; |
| |
| multiclass FLAT_Real_Atomics_gfx10<bits<7> op, string OpName = NAME, |
| string asmName = !cast<FLAT_Pseudo>(OpName).Mnemonic> : |
| FLAT_Real_Base_gfx10<op, OpName, asmName>, |
| FLAT_Real_RTN_gfx10<op, OpName, asmName>; |
| |
| multiclass FLAT_Real_GlblAtomics_gfx10<bits<7> op, string OpName = NAME, |
| string asmName = !cast<FLAT_Pseudo>(OpName).Mnemonic> : |
| FLAT_Real_AllAddr_gfx10<op, OpName, asmName>, |
| FLAT_Real_RTN_gfx10<op, OpName, asmName>, |
| FLAT_Real_SADDR_RTN_gfx10<op, OpName, asmName>; |
| |
| multiclass FLAT_Real_GlblAtomics_RTN_gfx10<bits<7> op, string OpName = NAME> : |
| FLAT_Real_RTN_gfx10<op, OpName>, |
| FLAT_Real_SADDR_RTN_gfx10<op, OpName>; |
| |
| multiclass FLAT_Real_ScratchAllAddr_gfx10<bits<7> op> : |
| FLAT_Real_Base_gfx10<op>, |
| FLAT_Real_SADDR_gfx10<op>, |
| FLAT_Real_ST_gfx10<op>; |
| |
| multiclass FLAT_Real_AllAddr_LDS_gfx10<bits<7> op, |
| string opname = !subst("_lds", "", !cast<FLAT_Pseudo>(NAME).Mnemonic)> { |
| let AsmString = opname # !cast<FLAT_Pseudo>(NAME).AsmOperands # " lds" in |
| defm "" : FLAT_Real_Base_gfx10<op>; |
| |
| let AsmString = opname # !cast<FLAT_Pseudo>(NAME#"_SADDR").AsmOperands # " lds" in |
| defm "" : FLAT_Real_SADDR_gfx10<op>; |
| } |
| |
| multiclass FLAT_Real_ScratchAllAddr_LDS_gfx10<bits<7> op, |
| string opname = !subst("_lds", "", !cast<FLAT_Pseudo>(NAME).Mnemonic)> { |
| defm "" : FLAT_Real_AllAddr_LDS_gfx10<op>; |
| |
| let AsmString = opname # !cast<FLAT_Pseudo>(NAME#"_ST").AsmOperands # " lds" in |
| defm "" : FLAT_Real_ST_gfx10<op>; |
| } |
| |
| // ENC_FLAT. |
| defm FLAT_LOAD_UBYTE : FLAT_Real_Base_gfx10<0x008>; |
| defm FLAT_LOAD_SBYTE : FLAT_Real_Base_gfx10<0x009>; |
| defm FLAT_LOAD_USHORT : FLAT_Real_Base_gfx10<0x00a>; |
| defm FLAT_LOAD_SSHORT : FLAT_Real_Base_gfx10<0x00b>; |
| defm FLAT_LOAD_DWORD : FLAT_Real_Base_gfx10<0x00c>; |
| defm FLAT_LOAD_DWORDX2 : FLAT_Real_Base_gfx10<0x00d>; |
| defm FLAT_LOAD_DWORDX4 : FLAT_Real_Base_gfx10<0x00e>; |
| defm FLAT_LOAD_DWORDX3 : FLAT_Real_Base_gfx10<0x00f>; |
| defm FLAT_STORE_BYTE : FLAT_Real_Base_gfx10<0x018>; |
| defm FLAT_STORE_BYTE_D16_HI : FLAT_Real_Base_gfx10<0x019>; |
| defm FLAT_STORE_SHORT : FLAT_Real_Base_gfx10<0x01a>; |
| defm FLAT_STORE_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x01b>; |
| defm FLAT_STORE_DWORD : FLAT_Real_Base_gfx10<0x01c>; |
| defm FLAT_STORE_DWORDX2 : FLAT_Real_Base_gfx10<0x01d>; |
| defm FLAT_STORE_DWORDX4 : FLAT_Real_Base_gfx10<0x01e>; |
| defm FLAT_STORE_DWORDX3 : FLAT_Real_Base_gfx10<0x01f>; |
| defm FLAT_LOAD_UBYTE_D16 : FLAT_Real_Base_gfx10<0x020>; |
| defm FLAT_LOAD_UBYTE_D16_HI : FLAT_Real_Base_gfx10<0x021>; |
| defm FLAT_LOAD_SBYTE_D16 : FLAT_Real_Base_gfx10<0x022>; |
| defm FLAT_LOAD_SBYTE_D16_HI : FLAT_Real_Base_gfx10<0x023>; |
| defm FLAT_LOAD_SHORT_D16 : FLAT_Real_Base_gfx10<0x024>; |
| defm FLAT_LOAD_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x025>; |
| defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_gfx10<0x030>; |
| defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_gfx10<0x031>; |
| defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_gfx10<0x032>; |
| defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_gfx10<0x033>; |
| defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_gfx10<0x035>; |
| defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_gfx10<0x036>; |
| defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_gfx10<0x037>; |
| defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_gfx10<0x038>; |
| defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_gfx10<0x039>; |
| defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_gfx10<0x03a>; |
| defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_gfx10<0x03b>; |
| defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_gfx10<0x03c>; |
| defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_gfx10<0x03d>; |
| defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_gfx10<0x03e>; |
| defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_gfx10<0x03f>; |
| defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_gfx10<0x040>; |
| defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_gfx10<0x050>; |
| defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x051>; |
| defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_gfx10<0x052>; |
| defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_gfx10<0x053>; |
| defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_gfx10<0x055>; |
| defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_gfx10<0x056>; |
| defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_gfx10<0x057>; |
| defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_gfx10<0x058>; |
| defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_gfx10<0x059>; |
| defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_gfx10<0x05a>; |
| defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_gfx10<0x05b>; |
| defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_gfx10<0x05c>; |
| defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_gfx10<0x05d>; |
| defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x05e>; |
| defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_gfx10<0x05f, "FLAT_ATOMIC_MIN_F64", "flat_atomic_fmin_x2">; |
| defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_gfx10<0x060, "FLAT_ATOMIC_MAX_F64", "flat_atomic_fmax_x2">; |
| |
| |
| // ENC_FLAT_GLBL. |
| defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_gfx10<0x008>; |
| defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_gfx10<0x009>; |
| defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_gfx10<0x00a>; |
| defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_gfx10<0x00b>; |
| defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_gfx10<0x00c>; |
| defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x00d>; |
| defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x00e>; |
| defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x00f>; |
| defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_gfx10<0x018>; |
| defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x019>; |
| defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_gfx10<0x01a>; |
| defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x01b>; |
| defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_gfx10<0x01c>; |
| defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x01d>; |
| defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x01e>; |
| defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x01f>; |
| defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x020>; |
| defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x021>; |
| defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x022>; |
| defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x023>; |
| defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_gfx10<0x024>; |
| defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x025>; |
| defm GLOBAL_ATOMIC_SWAP : FLAT_Real_GlblAtomics_gfx10<0x030>; |
| defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x031>; |
| defm GLOBAL_ATOMIC_ADD : FLAT_Real_GlblAtomics_gfx10<0x032>; |
| defm GLOBAL_ATOMIC_SUB : FLAT_Real_GlblAtomics_gfx10<0x033>; |
| defm GLOBAL_ATOMIC_CSUB : FLAT_Real_GlblAtomics_gfx10<0x034>; |
| defm GLOBAL_ATOMIC_SMIN : FLAT_Real_GlblAtomics_gfx10<0x035>; |
| defm GLOBAL_ATOMIC_UMIN : FLAT_Real_GlblAtomics_gfx10<0x036>; |
| defm GLOBAL_ATOMIC_SMAX : FLAT_Real_GlblAtomics_gfx10<0x037>; |
| defm GLOBAL_ATOMIC_UMAX : FLAT_Real_GlblAtomics_gfx10<0x038>; |
| defm GLOBAL_ATOMIC_AND : FLAT_Real_GlblAtomics_gfx10<0x039>; |
| defm GLOBAL_ATOMIC_OR : FLAT_Real_GlblAtomics_gfx10<0x03a>; |
| defm GLOBAL_ATOMIC_XOR : FLAT_Real_GlblAtomics_gfx10<0x03b>; |
| defm GLOBAL_ATOMIC_INC : FLAT_Real_GlblAtomics_gfx10<0x03c>; |
| defm GLOBAL_ATOMIC_DEC : FLAT_Real_GlblAtomics_gfx10<0x03d>; |
| defm GLOBAL_ATOMIC_FCMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x03e>; |
| defm GLOBAL_ATOMIC_FMIN : FLAT_Real_GlblAtomics_gfx10<0x03f>; |
| defm GLOBAL_ATOMIC_FMAX : FLAT_Real_GlblAtomics_gfx10<0x040>; |
| defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x050>; |
| defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x051>; |
| defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Real_GlblAtomics_gfx10<0x052>; |
| defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Real_GlblAtomics_gfx10<0x053>; |
| defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x055>; |
| defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x056>; |
| defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x057>; |
| defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x058>; |
| defm GLOBAL_ATOMIC_AND_X2 : FLAT_Real_GlblAtomics_gfx10<0x059>; |
| defm GLOBAL_ATOMIC_OR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05a>; |
| defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05b>; |
| defm GLOBAL_ATOMIC_INC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05c>; |
| defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05d>; |
| defm GLOBAL_ATOMIC_FCMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x05e>; |
| defm GLOBAL_ATOMIC_FMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x05f, "GLOBAL_ATOMIC_MIN_F64", "global_atomic_fmin_x2">; |
| defm GLOBAL_ATOMIC_FMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x060, "GLOBAL_ATOMIC_MAX_F64", "global_atomic_fmax_x2">; |
| defm GLOBAL_LOAD_DWORD_ADDTID : FLAT_Real_AllAddr_gfx10<0x016>; |
| defm GLOBAL_STORE_DWORD_ADDTID : FLAT_Real_AllAddr_gfx10<0x017>; |
| |
| defm GLOBAL_LOAD_LDS_UBYTE : FLAT_Real_AllAddr_LDS_gfx10 <0x008>; |
| defm GLOBAL_LOAD_LDS_SBYTE : FLAT_Real_AllAddr_LDS_gfx10 <0x009>; |
| defm GLOBAL_LOAD_LDS_USHORT : FLAT_Real_AllAddr_LDS_gfx10 <0x00a>; |
| defm GLOBAL_LOAD_LDS_SSHORT : FLAT_Real_AllAddr_LDS_gfx10 <0x00b>; |
| defm GLOBAL_LOAD_LDS_DWORD : FLAT_Real_AllAddr_LDS_gfx10 <0x00c>; |
| |
| // ENC_FLAT_SCRATCH. |
| defm SCRATCH_LOAD_UBYTE : FLAT_Real_ScratchAllAddr_gfx10<0x008>; |
| defm SCRATCH_LOAD_SBYTE : FLAT_Real_ScratchAllAddr_gfx10<0x009>; |
| defm SCRATCH_LOAD_USHORT : FLAT_Real_ScratchAllAddr_gfx10<0x00a>; |
| defm SCRATCH_LOAD_SSHORT : FLAT_Real_ScratchAllAddr_gfx10<0x00b>; |
| defm SCRATCH_LOAD_DWORD : FLAT_Real_ScratchAllAddr_gfx10<0x00c>; |
| defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_ScratchAllAddr_gfx10<0x00d>; |
| defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_ScratchAllAddr_gfx10<0x00e>; |
| defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_ScratchAllAddr_gfx10<0x00f>; |
| defm SCRATCH_STORE_BYTE : FLAT_Real_ScratchAllAddr_gfx10<0x018>; |
| defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x019>; |
| defm SCRATCH_STORE_SHORT : FLAT_Real_ScratchAllAddr_gfx10<0x01a>; |
| defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x01b>; |
| defm SCRATCH_STORE_DWORD : FLAT_Real_ScratchAllAddr_gfx10<0x01c>; |
| defm SCRATCH_STORE_DWORDX2 : FLAT_Real_ScratchAllAddr_gfx10<0x01d>; |
| defm SCRATCH_STORE_DWORDX4 : FLAT_Real_ScratchAllAddr_gfx10<0x01e>; |
| defm SCRATCH_STORE_DWORDX3 : FLAT_Real_ScratchAllAddr_gfx10<0x01f>; |
| defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x020>; |
| defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x021>; |
| defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x022>; |
| defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x023>; |
| defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x024>; |
| defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x025>; |
| |
| defm SCRATCH_LOAD_LDS_UBYTE : FLAT_Real_ScratchAllAddr_LDS_gfx10 <0x008>; |
| defm SCRATCH_LOAD_LDS_SBYTE : FLAT_Real_ScratchAllAddr_LDS_gfx10 <0x009>; |
| defm SCRATCH_LOAD_LDS_USHORT : FLAT_Real_ScratchAllAddr_LDS_gfx10 <0x00a>; |
| defm SCRATCH_LOAD_LDS_SSHORT : FLAT_Real_ScratchAllAddr_LDS_gfx10 <0x00b>; |
| defm SCRATCH_LOAD_LDS_DWORD : FLAT_Real_ScratchAllAddr_LDS_gfx10 <0x00c>; |
| |
| //===----------------------------------------------------------------------===// |
| // GFX11 |
| //===----------------------------------------------------------------------===// |
| |
| class get_FLAT_ps<string name> { |
| string Mnemonic = !cast<FLAT_Pseudo>(name).Mnemonic; |
| } |
| |
| multiclass FLAT_Real_gfx11 <bits<7> op, |
| string name = get_FLAT_ps<NAME>.Mnemonic> { |
| defvar ps = !cast<FLAT_Pseudo>(NAME); |
| def _gfx11 : FLAT_Real <op, ps, name>, |
| SIMCInstr <ps.PseudoInstr, SIEncodingFamily.GFX11> { |
| let AssemblerPredicate = isGFX11Only; |
| let DecoderNamespace = "GFX11"; |
| |
| let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlcValue); |
| let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glcValue); |
| let Inst{15} = cpol{CPolBit.SLC}; |
| let Inst{17-16} = seg; |
| let Inst{54-48} = !if(ps.enabled_saddr, saddr, SGPR_NULL_gfx11plus.Index); |
| let Inst{55} = ps.sve; |
| } |
| } |
| |
| multiclass FLAT_Aliases_gfx11<string name> { |
| defvar ps = get_FLAT_ps<NAME>; |
| if !ne(ps.Mnemonic, name) then |
| def : AMDGPUMnemonicAlias<ps.Mnemonic, name> { |
| let AssemblerPredicate = isGFX11Only; |
| } |
| } |
| |
| multiclass FLAT_Real_Base_gfx11<bits<7> op, |
| string name = get_FLAT_ps<NAME>.Mnemonic> : |
| FLAT_Aliases_gfx11<name>, |
| FLAT_Real_gfx11<op, name>; |
| |
| multiclass FLAT_Real_Atomics_gfx11<bits<7> op, |
| string name = get_FLAT_ps<NAME>.Mnemonic> : |
| FLAT_Real_Base_gfx11<op, name> { |
| defm _RTN : FLAT_Real_gfx11<op, name>; |
| } |
| |
| multiclass GLOBAL_Real_AllAddr_gfx11<bits<7> op, |
| string name = get_FLAT_ps<NAME>.Mnemonic> : |
| FLAT_Real_Base_gfx11<op, name> { |
| defm _SADDR : FLAT_Real_gfx11<op, name>; |
| } |
| |
| multiclass GLOBAL_Real_Atomics_gfx11<bits<7> op, |
| string name = get_FLAT_ps<NAME>.Mnemonic> : |
| GLOBAL_Real_AllAddr_gfx11<op, name> { |
| defm _RTN : FLAT_Real_gfx11<op, name>; |
| defm _SADDR_RTN : FLAT_Real_gfx11<op, name>; |
| } |
| |
| multiclass SCRATCH_Real_AllAddr_gfx11<bits<7> op, |
| string name = get_FLAT_ps<NAME>.Mnemonic> : |
| FLAT_Real_Base_gfx11<op, name> { |
| defm _SADDR : FLAT_Real_gfx11<op, name>; |
| defm _ST : FLAT_Real_gfx11<op, name>; |
| defm _SVS : FLAT_Real_gfx11<op, name>; |
| } |
| |
| // ENC_FLAT. |
| defm FLAT_LOAD_UBYTE : FLAT_Real_Base_gfx11<0x010, "flat_load_u8">; |
| defm FLAT_LOAD_SBYTE : FLAT_Real_Base_gfx11<0x011, "flat_load_i8">; |
| defm FLAT_LOAD_USHORT : FLAT_Real_Base_gfx11<0x012, "flat_load_u16">; |
| defm FLAT_LOAD_SSHORT : FLAT_Real_Base_gfx11<0x013, "flat_load_i16">; |
| defm FLAT_LOAD_DWORD : FLAT_Real_Base_gfx11<0x014, "flat_load_b32">; |
| defm FLAT_LOAD_DWORDX2 : FLAT_Real_Base_gfx11<0x015, "flat_load_b64">; |
| defm FLAT_LOAD_DWORDX3 : FLAT_Real_Base_gfx11<0x016, "flat_load_b96">; |
| defm FLAT_LOAD_DWORDX4 : FLAT_Real_Base_gfx11<0x017, "flat_load_b128">; |
| defm FLAT_STORE_BYTE : FLAT_Real_Base_gfx11<0x018, "flat_store_b8">; |
| defm FLAT_STORE_SHORT : FLAT_Real_Base_gfx11<0x019, "flat_store_b16">; |
| defm FLAT_STORE_DWORD : FLAT_Real_Base_gfx11<0x01a, "flat_store_b32">; |
| defm FLAT_STORE_DWORDX2 : FLAT_Real_Base_gfx11<0x01b, "flat_store_b64">; |
| defm FLAT_STORE_DWORDX3 : FLAT_Real_Base_gfx11<0x01c, "flat_store_b96">; |
| defm FLAT_STORE_DWORDX4 : FLAT_Real_Base_gfx11<0x01d, "flat_store_b128">; |
| defm FLAT_LOAD_UBYTE_D16 : FLAT_Real_Base_gfx11<0x01e, "flat_load_d16_u8">; |
| defm FLAT_LOAD_SBYTE_D16 : FLAT_Real_Base_gfx11<0x01f, "flat_load_d16_i8">; |
| defm FLAT_LOAD_SHORT_D16 : FLAT_Real_Base_gfx11<0x020, "flat_load_d16_b16">; |
| defm FLAT_LOAD_UBYTE_D16_HI : FLAT_Real_Base_gfx11<0x021, "flat_load_d16_hi_u8">; |
| defm FLAT_LOAD_SBYTE_D16_HI : FLAT_Real_Base_gfx11<0x022, "flat_load_d16_hi_i8">; |
| defm FLAT_LOAD_SHORT_D16_HI : FLAT_Real_Base_gfx11<0x023, "flat_load_d16_hi_b16">; |
| defm FLAT_STORE_BYTE_D16_HI : FLAT_Real_Base_gfx11<0x024, "flat_store_d16_hi_b8">; |
| defm FLAT_STORE_SHORT_D16_HI : FLAT_Real_Base_gfx11<0x025, "flat_store_d16_hi_b16">; |
| defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_gfx11<0x033, "flat_atomic_swap_b32">; |
| defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_gfx11<0x034, "flat_atomic_cmpswap_b32">; |
| defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_gfx11<0x035, "flat_atomic_add_u32">; |
| defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_gfx11<0x036, "flat_atomic_sub_u32">; |
| defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_gfx11<0x038, "flat_atomic_min_i32">; |
| defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_gfx11<0x039, "flat_atomic_min_u32">; |
| defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_gfx11<0x03a, "flat_atomic_max_i32">; |
| defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_gfx11<0x03b, "flat_atomic_max_u32">; |
| defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_gfx11<0x03c, "flat_atomic_and_b32">; |
| defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_gfx11<0x03d, "flat_atomic_or_b32">; |
| defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_gfx11<0x03e, "flat_atomic_xor_b32">; |
| defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_gfx11<0x03f, "flat_atomic_inc_u32">; |
| defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_gfx11<0x040, "flat_atomic_dec_u32">; |
| defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_gfx11<0x041, "flat_atomic_swap_b64">; |
| defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_gfx11<0x042, "flat_atomic_cmpswap_b64">; |
| defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_gfx11<0x043, "flat_atomic_add_u64">; |
| defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_gfx11<0x044, "flat_atomic_sub_u64">; |
| defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_gfx11<0x045, "flat_atomic_min_i64">; |
| defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_gfx11<0x046, "flat_atomic_min_u64">; |
| defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_gfx11<0x047, "flat_atomic_max_i64">; |
| defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_gfx11<0x048, "flat_atomic_max_u64">; |
| defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_gfx11<0x049, "flat_atomic_and_b64">; |
| defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_gfx11<0x04a, "flat_atomic_or_b64">; |
| defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_gfx11<0x04b, "flat_atomic_xor_b64">; |
| defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_gfx11<0x04c, "flat_atomic_inc_u64">; |
| defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_gfx11<0x04d, "flat_atomic_dec_u64">; |
| defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_gfx11<0x050, "flat_atomic_cmpswap_f32">; |
| defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_gfx11<0x051, "flat_atomic_min_f32">; |
| defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_gfx11<0x052, "flat_atomic_max_f32">; |
| defm FLAT_ATOMIC_ADD_F32 : FLAT_Real_Atomics_gfx11<0x056>; |
| |
| // ENC_FLAT_GLBL. |
| defm GLOBAL_LOAD_UBYTE : GLOBAL_Real_AllAddr_gfx11<0x010, "global_load_u8">; |
| defm GLOBAL_LOAD_SBYTE : GLOBAL_Real_AllAddr_gfx11<0x011, "global_load_i8">; |
| defm GLOBAL_LOAD_USHORT : GLOBAL_Real_AllAddr_gfx11<0x012, "global_load_u16">; |
| defm GLOBAL_LOAD_SSHORT : GLOBAL_Real_AllAddr_gfx11<0x013, "global_load_i16">; |
| defm GLOBAL_LOAD_DWORD : GLOBAL_Real_AllAddr_gfx11<0x014, "global_load_b32">; |
| defm GLOBAL_LOAD_DWORDX2 : GLOBAL_Real_AllAddr_gfx11<0x015, "global_load_b64">; |
| defm GLOBAL_LOAD_DWORDX3 : GLOBAL_Real_AllAddr_gfx11<0x016, "global_load_b96">; |
| defm GLOBAL_LOAD_DWORDX4 : GLOBAL_Real_AllAddr_gfx11<0x017, "global_load_b128">; |
| defm GLOBAL_STORE_BYTE : GLOBAL_Real_AllAddr_gfx11<0x018, "global_store_b8">; |
| defm GLOBAL_STORE_SHORT : GLOBAL_Real_AllAddr_gfx11<0x019, "global_store_b16">; |
| defm GLOBAL_STORE_DWORD : GLOBAL_Real_AllAddr_gfx11<0x01a, "global_store_b32">; |
| defm GLOBAL_STORE_DWORDX2 : GLOBAL_Real_AllAddr_gfx11<0x01b, "global_store_b64">; |
| defm GLOBAL_STORE_DWORDX3 : GLOBAL_Real_AllAddr_gfx11<0x01c, "global_store_b96">; |
| defm GLOBAL_STORE_DWORDX4 : GLOBAL_Real_AllAddr_gfx11<0x01d, "global_store_b128">; |
| defm GLOBAL_LOAD_UBYTE_D16 : GLOBAL_Real_AllAddr_gfx11<0x01e, "global_load_d16_u8">; |
| defm GLOBAL_LOAD_SBYTE_D16 : GLOBAL_Real_AllAddr_gfx11<0x01f, "global_load_d16_i8">; |
| defm GLOBAL_LOAD_SHORT_D16 : GLOBAL_Real_AllAddr_gfx11<0x020, "global_load_d16_b16">; |
| defm GLOBAL_LOAD_UBYTE_D16_HI : GLOBAL_Real_AllAddr_gfx11<0x021, "global_load_d16_hi_u8">; |
| defm GLOBAL_LOAD_SBYTE_D16_HI : GLOBAL_Real_AllAddr_gfx11<0x022, "global_load_d16_hi_i8">; |
| defm GLOBAL_LOAD_SHORT_D16_HI : GLOBAL_Real_AllAddr_gfx11<0x023, "global_load_d16_hi_b16">; |
| defm GLOBAL_STORE_BYTE_D16_HI : GLOBAL_Real_AllAddr_gfx11<0x024, "global_store_d16_hi_b8">; |
| defm GLOBAL_STORE_SHORT_D16_HI : GLOBAL_Real_AllAddr_gfx11<0x025, "global_store_d16_hi_b16">; |
| defm GLOBAL_LOAD_DWORD_ADDTID : GLOBAL_Real_AllAddr_gfx11<0x028, "global_load_addtid_b32">; |
| defm GLOBAL_STORE_DWORD_ADDTID : GLOBAL_Real_AllAddr_gfx11<0x029, "global_store_addtid_b32">; |
| defm GLOBAL_ATOMIC_SWAP : GLOBAL_Real_Atomics_gfx11<0x033, "global_atomic_swap_b32">; |
| defm GLOBAL_ATOMIC_CMPSWAP : GLOBAL_Real_Atomics_gfx11<0x034, "global_atomic_cmpswap_b32">; |
| defm GLOBAL_ATOMIC_ADD : GLOBAL_Real_Atomics_gfx11<0x035, "global_atomic_add_u32">; |
| defm GLOBAL_ATOMIC_SUB : GLOBAL_Real_Atomics_gfx11<0x036, "global_atomic_sub_u32">; |
| defm GLOBAL_ATOMIC_CSUB : GLOBAL_Real_Atomics_gfx11<0x037, "global_atomic_csub_u32">; |
| defm GLOBAL_ATOMIC_SMIN : GLOBAL_Real_Atomics_gfx11<0x038, "global_atomic_min_i32">; |
| defm GLOBAL_ATOMIC_UMIN : GLOBAL_Real_Atomics_gfx11<0x039, "global_atomic_min_u32">; |
| defm GLOBAL_ATOMIC_SMAX : GLOBAL_Real_Atomics_gfx11<0x03a, "global_atomic_max_i32">; |
| defm GLOBAL_ATOMIC_UMAX : GLOBAL_Real_Atomics_gfx11<0x03b, "global_atomic_max_u32">; |
| defm GLOBAL_ATOMIC_AND : GLOBAL_Real_Atomics_gfx11<0x03c, "global_atomic_and_b32">; |
| defm GLOBAL_ATOMIC_OR : GLOBAL_Real_Atomics_gfx11<0x03d, "global_atomic_or_b32">; |
| defm GLOBAL_ATOMIC_XOR : GLOBAL_Real_Atomics_gfx11<0x03e, "global_atomic_xor_b32">; |
| defm GLOBAL_ATOMIC_INC : GLOBAL_Real_Atomics_gfx11<0x03f, "global_atomic_inc_u32">; |
| defm GLOBAL_ATOMIC_DEC : GLOBAL_Real_Atomics_gfx11<0x040, "global_atomic_dec_u32">; |
| defm GLOBAL_ATOMIC_SWAP_X2 : GLOBAL_Real_Atomics_gfx11<0x041, "global_atomic_swap_b64">; |
| defm GLOBAL_ATOMIC_CMPSWAP_X2 : GLOBAL_Real_Atomics_gfx11<0x042, "global_atomic_cmpswap_b64">; |
| defm GLOBAL_ATOMIC_ADD_X2 : GLOBAL_Real_Atomics_gfx11<0x043, "global_atomic_add_u64">; |
| defm GLOBAL_ATOMIC_SUB_X2 : GLOBAL_Real_Atomics_gfx11<0x044, "global_atomic_sub_u64">; |
| defm GLOBAL_ATOMIC_SMIN_X2 : GLOBAL_Real_Atomics_gfx11<0x045, "global_atomic_min_i64">; |
| defm GLOBAL_ATOMIC_UMIN_X2 : GLOBAL_Real_Atomics_gfx11<0x046, "global_atomic_min_u64">; |
| defm GLOBAL_ATOMIC_SMAX_X2 : GLOBAL_Real_Atomics_gfx11<0x047, "global_atomic_max_i64">; |
| defm GLOBAL_ATOMIC_UMAX_X2 : GLOBAL_Real_Atomics_gfx11<0x048, "global_atomic_max_u64">; |
| defm GLOBAL_ATOMIC_AND_X2 : GLOBAL_Real_Atomics_gfx11<0x049, "global_atomic_and_b64">; |
| defm GLOBAL_ATOMIC_OR_X2 : GLOBAL_Real_Atomics_gfx11<0x04a, "global_atomic_or_b64">; |
| defm GLOBAL_ATOMIC_XOR_X2 : GLOBAL_Real_Atomics_gfx11<0x04b, "global_atomic_xor_b64">; |
| defm GLOBAL_ATOMIC_INC_X2 : GLOBAL_Real_Atomics_gfx11<0x04c, "global_atomic_inc_u64">; |
| defm GLOBAL_ATOMIC_DEC_X2 : GLOBAL_Real_Atomics_gfx11<0x04d, "global_atomic_dec_u64">; |
| defm GLOBAL_ATOMIC_FCMPSWAP : GLOBAL_Real_Atomics_gfx11<0x050, "global_atomic_cmpswap_f32">; |
| defm GLOBAL_ATOMIC_FMIN : GLOBAL_Real_Atomics_gfx11<0x051, "global_atomic_min_f32">; |
| defm GLOBAL_ATOMIC_FMAX : GLOBAL_Real_Atomics_gfx11<0x052, "global_atomic_max_f32">; |
| defm GLOBAL_ATOMIC_ADD_F32 : GLOBAL_Real_Atomics_gfx11<0x056>; |
| |
| // ENC_FLAT_SCRATCH. |
| defm SCRATCH_LOAD_UBYTE : SCRATCH_Real_AllAddr_gfx11<0x10, "scratch_load_u8">; |
| defm SCRATCH_LOAD_SBYTE : SCRATCH_Real_AllAddr_gfx11<0x11, "scratch_load_i8">; |
| defm SCRATCH_LOAD_USHORT : SCRATCH_Real_AllAddr_gfx11<0x12, "scratch_load_u16">; |
| defm SCRATCH_LOAD_SSHORT : SCRATCH_Real_AllAddr_gfx11<0x13, "scratch_load_i16">; |
| defm SCRATCH_LOAD_DWORD : SCRATCH_Real_AllAddr_gfx11<0x14, "scratch_load_b32">; |
| defm SCRATCH_LOAD_DWORDX2 : SCRATCH_Real_AllAddr_gfx11<0x15, "scratch_load_b64">; |
| defm SCRATCH_LOAD_DWORDX3 : SCRATCH_Real_AllAddr_gfx11<0x16, "scratch_load_b96">; |
| defm SCRATCH_LOAD_DWORDX4 : SCRATCH_Real_AllAddr_gfx11<0x17, "scratch_load_b128">; |
| defm SCRATCH_STORE_BYTE : SCRATCH_Real_AllAddr_gfx11<0x18, "scratch_store_b8">; |
| defm SCRATCH_STORE_SHORT : SCRATCH_Real_AllAddr_gfx11<0x19, "scratch_store_b16">; |
| defm SCRATCH_STORE_DWORD : SCRATCH_Real_AllAddr_gfx11<0x1a, "scratch_store_b32">; |
| defm SCRATCH_STORE_DWORDX2 : SCRATCH_Real_AllAddr_gfx11<0x1b, "scratch_store_b64">; |
| defm SCRATCH_STORE_DWORDX3 : SCRATCH_Real_AllAddr_gfx11<0x1c, "scratch_store_b96">; |
| defm SCRATCH_STORE_DWORDX4 : SCRATCH_Real_AllAddr_gfx11<0x1d, "scratch_store_b128">; |
| defm SCRATCH_LOAD_UBYTE_D16 : SCRATCH_Real_AllAddr_gfx11<0x1e, "scratch_load_d16_u8">; |
| defm SCRATCH_LOAD_SBYTE_D16 : SCRATCH_Real_AllAddr_gfx11<0x1f, "scratch_load_d16_i8">; |
| defm SCRATCH_LOAD_SHORT_D16 : SCRATCH_Real_AllAddr_gfx11<0x20, "scratch_load_d16_b16">; |
| defm SCRATCH_LOAD_UBYTE_D16_HI : SCRATCH_Real_AllAddr_gfx11<0x21, "scratch_load_d16_hi_u8">; |
| defm SCRATCH_LOAD_SBYTE_D16_HI : SCRATCH_Real_AllAddr_gfx11<0x22, "scratch_load_d16_hi_i8">; |
| defm SCRATCH_LOAD_SHORT_D16_HI : SCRATCH_Real_AllAddr_gfx11<0x23, "scratch_load_d16_hi_b16">; |
| defm SCRATCH_STORE_BYTE_D16_HI : SCRATCH_Real_AllAddr_gfx11<0x24, "scratch_store_d16_hi_b8">; |
| defm SCRATCH_STORE_SHORT_D16_HI : SCRATCH_Real_AllAddr_gfx11<0x25, "scratch_store_d16_hi_b16">; |
| |
| //===----------------------------------------------------------------------===// |
| // GFX12 |
| //===----------------------------------------------------------------------===// |
| |
| multiclass VFLAT_Real_gfx12 <bits<8> op, string name = get_FLAT_ps<NAME>.Mnemonic> { |
| defvar ps = !cast<FLAT_Pseudo>(NAME); |
| def _gfx12 : VFLAT_Real <op, ps, name>, |
| SIMCInstr <ps.PseudoInstr, SIEncodingFamily.GFX12> { |
| let AssemblerPredicate = isGFX12Only; |
| let DecoderNamespace = "GFX12"; |
| |
| let Inst{25-24} = {ps.is_flat_global, ps.is_flat_scratch}; |
| } |
| } |
| |
| multiclass VFLAT_Aliases_gfx12<string name, string alias = name> { |
| defvar ps = get_FLAT_ps<NAME>; |
| let AssemblerPredicate = isGFX12Only in { |
| if !ne(ps.Mnemonic, name) then |
| def : AMDGPUMnemonicAlias<ps.Mnemonic, name>; |
| if !ne(alias, name) then |
| def : AMDGPUMnemonicAlias<alias, name>; |
| } |
| } |
| |
| multiclass VFLAT_Real_Base_gfx12<bits<8> op, |
| string name = get_FLAT_ps<NAME>.Mnemonic, |
| string alias = name> : |
| VFLAT_Aliases_gfx12<name, alias>, |
| VFLAT_Real_gfx12<op, name>; |
| |
| multiclass VFLAT_Real_Atomics_gfx12<bits<8> op, |
| string name = get_FLAT_ps<NAME>.Mnemonic, |
| string alias = name> : |
| VFLAT_Real_Base_gfx12<op, name, alias> { |
| defm _RTN : VFLAT_Real_gfx12<op, name>; |
| } |
| |
| multiclass VGLOBAL_Real_AllAddr_gfx12<bits<8> op, |
| string name = get_FLAT_ps<NAME>.Mnemonic, |
| string alias = name> : |
| VFLAT_Real_Base_gfx12<op, name, alias> { |
| defm _SADDR : VFLAT_Real_gfx12<op, name>; |
| } |
| |
| multiclass VGLOBAL_Real_AllAddr_gfx12_w64<bits<8> op, |
| string name = get_FLAT_ps<NAME>.Mnemonic> : |
| VFLAT_Aliases_gfx12<name> { |
| let DecoderNamespace = "GFX12W64" in { |
| defm "" : VFLAT_Real_gfx12<op, name>; |
| defm _SADDR : VFLAT_Real_gfx12<op, name>; |
| } |
| } |
| |
| multiclass VGLOBAL_Real_Atomics_gfx12<bits<8> op, |
| string name = get_FLAT_ps<NAME>.Mnemonic, |
| string alias = name> : |
| VGLOBAL_Real_AllAddr_gfx12<op, name, alias> { |
| defm _RTN : VFLAT_Real_gfx12<op, name>; |
| defm _SADDR_RTN : VFLAT_Real_gfx12<op, name>; |
| } |
| |
| multiclass VSCRATCH_Real_AllAddr_gfx12<bits<8> op, |
| string name = get_FLAT_ps<NAME>.Mnemonic> : |
| VFLAT_Real_Base_gfx12<op, name> { |
| defm _SADDR : VFLAT_Real_gfx12<op, name>; |
| defm _ST : VFLAT_Real_gfx12<op, name>; |
| defm _SVS : VFLAT_Real_gfx12<op, name>; |
| } |
| |
| // ENC_VFLAT. |
| defm FLAT_LOAD_UBYTE : VFLAT_Real_Base_gfx12<0x010, "flat_load_u8">; |
| defm FLAT_LOAD_SBYTE : VFLAT_Real_Base_gfx12<0x011, "flat_load_i8">; |
| defm FLAT_LOAD_USHORT : VFLAT_Real_Base_gfx12<0x012, "flat_load_u16">; |
| defm FLAT_LOAD_SSHORT : VFLAT_Real_Base_gfx12<0x013, "flat_load_i16">; |
| defm FLAT_LOAD_DWORD : VFLAT_Real_Base_gfx12<0x014, "flat_load_b32">; |
| defm FLAT_LOAD_DWORDX2 : VFLAT_Real_Base_gfx12<0x015, "flat_load_b64">; |
| defm FLAT_LOAD_DWORDX3 : VFLAT_Real_Base_gfx12<0x016, "flat_load_b96">; |
| defm FLAT_LOAD_DWORDX4 : VFLAT_Real_Base_gfx12<0x017, "flat_load_b128">; |
| defm FLAT_STORE_BYTE : VFLAT_Real_Base_gfx12<0x018, "flat_store_b8">; |
| defm FLAT_STORE_SHORT : VFLAT_Real_Base_gfx12<0x019, "flat_store_b16">; |
| defm FLAT_STORE_DWORD : VFLAT_Real_Base_gfx12<0x01a, "flat_store_b32">; |
| defm FLAT_STORE_DWORDX2 : VFLAT_Real_Base_gfx12<0x01b, "flat_store_b64">; |
| defm FLAT_STORE_DWORDX3 : VFLAT_Real_Base_gfx12<0x01c, "flat_store_b96">; |
| defm FLAT_STORE_DWORDX4 : VFLAT_Real_Base_gfx12<0x01d, "flat_store_b128">; |
| defm FLAT_LOAD_UBYTE_D16 : VFLAT_Real_Base_gfx12<0x01e, "flat_load_d16_u8">; |
| defm FLAT_LOAD_SBYTE_D16 : VFLAT_Real_Base_gfx12<0x01f, "flat_load_d16_i8">; |
| defm FLAT_LOAD_SHORT_D16 : VFLAT_Real_Base_gfx12<0x020, "flat_load_d16_b16">; |
| defm FLAT_LOAD_UBYTE_D16_HI : VFLAT_Real_Base_gfx12<0x021, "flat_load_d16_hi_u8">; |
| defm FLAT_LOAD_SBYTE_D16_HI : VFLAT_Real_Base_gfx12<0x022, "flat_load_d16_hi_i8">; |
| defm FLAT_LOAD_SHORT_D16_HI : VFLAT_Real_Base_gfx12<0x023, "flat_load_d16_hi_b16">; |
| defm FLAT_STORE_BYTE_D16_HI : VFLAT_Real_Base_gfx12<0x024, "flat_store_d16_hi_b8">; |
| defm FLAT_STORE_SHORT_D16_HI : VFLAT_Real_Base_gfx12<0x025, "flat_store_d16_hi_b16">; |
| defm FLAT_ATOMIC_SWAP : VFLAT_Real_Atomics_gfx12<0x033, "flat_atomic_swap_b32">; |
| defm FLAT_ATOMIC_CMPSWAP : VFLAT_Real_Atomics_gfx12<0x034, "flat_atomic_cmpswap_b32">; |
| defm FLAT_ATOMIC_ADD : VFLAT_Real_Atomics_gfx12<0x035, "flat_atomic_add_u32">; |
| defm FLAT_ATOMIC_SUB : VFLAT_Real_Atomics_gfx12<0x036, "flat_atomic_sub_u32">; |
| defm FLAT_ATOMIC_CSUB_U32 : VFLAT_Real_Atomics_gfx12<0x037, "flat_atomic_sub_clamp_u32">; |
| defm FLAT_ATOMIC_SMIN : VFLAT_Real_Atomics_gfx12<0x038, "flat_atomic_min_i32">; |
| defm FLAT_ATOMIC_UMIN : VFLAT_Real_Atomics_gfx12<0x039, "flat_atomic_min_u32">; |
| defm FLAT_ATOMIC_SMAX : VFLAT_Real_Atomics_gfx12<0x03a, "flat_atomic_max_i32">; |
| defm FLAT_ATOMIC_UMAX : VFLAT_Real_Atomics_gfx12<0x03b, "flat_atomic_max_u32">; |
| defm FLAT_ATOMIC_AND : VFLAT_Real_Atomics_gfx12<0x03c, "flat_atomic_and_b32">; |
| defm FLAT_ATOMIC_OR : VFLAT_Real_Atomics_gfx12<0x03d, "flat_atomic_or_b32">; |
| defm FLAT_ATOMIC_XOR : VFLAT_Real_Atomics_gfx12<0x03e, "flat_atomic_xor_b32">; |
| defm FLAT_ATOMIC_INC : VFLAT_Real_Atomics_gfx12<0x03f, "flat_atomic_inc_u32">; |
| defm FLAT_ATOMIC_DEC : VFLAT_Real_Atomics_gfx12<0x040, "flat_atomic_dec_u32">; |
| defm FLAT_ATOMIC_SWAP_X2 : VFLAT_Real_Atomics_gfx12<0x041, "flat_atomic_swap_b64">; |
| defm FLAT_ATOMIC_CMPSWAP_X2 : VFLAT_Real_Atomics_gfx12<0x042, "flat_atomic_cmpswap_b64">; |
| defm FLAT_ATOMIC_ADD_X2 : VFLAT_Real_Atomics_gfx12<0x043, "flat_atomic_add_u64">; |
| defm FLAT_ATOMIC_SUB_X2 : VFLAT_Real_Atomics_gfx12<0x044, "flat_atomic_sub_u64">; |
| defm FLAT_ATOMIC_SMIN_X2 : VFLAT_Real_Atomics_gfx12<0x045, "flat_atomic_min_i64">; |
| defm FLAT_ATOMIC_UMIN_X2 : VFLAT_Real_Atomics_gfx12<0x046, "flat_atomic_min_u64">; |
| defm FLAT_ATOMIC_SMAX_X2 : VFLAT_Real_Atomics_gfx12<0x047, "flat_atomic_max_i64">; |
| defm FLAT_ATOMIC_UMAX_X2 : VFLAT_Real_Atomics_gfx12<0x048, "flat_atomic_max_u64">; |
| defm FLAT_ATOMIC_AND_X2 : VFLAT_Real_Atomics_gfx12<0x049, "flat_atomic_and_b64">; |
| defm FLAT_ATOMIC_OR_X2 : VFLAT_Real_Atomics_gfx12<0x04a, "flat_atomic_or_b64">; |
| defm FLAT_ATOMIC_XOR_X2 : VFLAT_Real_Atomics_gfx12<0x04b, "flat_atomic_xor_b64">; |
| defm FLAT_ATOMIC_INC_X2 : VFLAT_Real_Atomics_gfx12<0x04c, "flat_atomic_inc_u64">; |
| defm FLAT_ATOMIC_DEC_X2 : VFLAT_Real_Atomics_gfx12<0x04d, "flat_atomic_dec_u64">; |
| defm FLAT_ATOMIC_COND_SUB_U32 : VFLAT_Real_Atomics_gfx12<0x050>; |
| defm FLAT_ATOMIC_FMIN : VFLAT_Real_Atomics_gfx12<0x051, "flat_atomic_min_num_f32", "flat_atomic_min_f32">; |
| defm FLAT_ATOMIC_FMAX : VFLAT_Real_Atomics_gfx12<0x052, "flat_atomic_max_num_f32", "flat_atomic_max_f32">; |
| defm FLAT_ATOMIC_ADD_F32 : VFLAT_Real_Atomics_gfx12<0x056>; |
| defm FLAT_ATOMIC_PK_ADD_F16 : VFLAT_Real_Atomics_gfx12<0x059>; |
| defm FLAT_ATOMIC_PK_ADD_BF16 : VFLAT_Real_Atomics_gfx12<0x05a>; |
| |
| // ENC_VGLOBAL. |
| defm GLOBAL_LOAD_UBYTE : VGLOBAL_Real_AllAddr_gfx12<0x010, "global_load_u8">; |
| defm GLOBAL_LOAD_SBYTE : VGLOBAL_Real_AllAddr_gfx12<0x011, "global_load_i8">; |
| defm GLOBAL_LOAD_USHORT : VGLOBAL_Real_AllAddr_gfx12<0x012, "global_load_u16">; |
| defm GLOBAL_LOAD_SSHORT : VGLOBAL_Real_AllAddr_gfx12<0x013, "global_load_i16">; |
| defm GLOBAL_LOAD_DWORD : VGLOBAL_Real_AllAddr_gfx12<0x014, "global_load_b32">; |
| defm GLOBAL_LOAD_DWORDX2 : VGLOBAL_Real_AllAddr_gfx12<0x015, "global_load_b64">; |
| defm GLOBAL_LOAD_DWORDX3 : VGLOBAL_Real_AllAddr_gfx12<0x016, "global_load_b96">; |
| defm GLOBAL_LOAD_DWORDX4 : VGLOBAL_Real_AllAddr_gfx12<0x017, "global_load_b128">; |
| defm GLOBAL_STORE_BYTE : VGLOBAL_Real_AllAddr_gfx12<0x018, "global_store_b8">; |
| defm GLOBAL_STORE_SHORT : VGLOBAL_Real_AllAddr_gfx12<0x019, "global_store_b16">; |
| defm GLOBAL_STORE_DWORD : VGLOBAL_Real_AllAddr_gfx12<0x01a, "global_store_b32">; |
| defm GLOBAL_STORE_DWORDX2 : VGLOBAL_Real_AllAddr_gfx12<0x01b, "global_store_b64">; |
| defm GLOBAL_STORE_DWORDX3 : VGLOBAL_Real_AllAddr_gfx12<0x01c, "global_store_b96">; |
| defm GLOBAL_STORE_DWORDX4 : VGLOBAL_Real_AllAddr_gfx12<0x01d, "global_store_b128">; |
| defm GLOBAL_LOAD_UBYTE_D16 : VGLOBAL_Real_AllAddr_gfx12<0x01e, "global_load_d16_u8">; |
| defm GLOBAL_LOAD_SBYTE_D16 : VGLOBAL_Real_AllAddr_gfx12<0x01f, "global_load_d16_i8">; |
| defm GLOBAL_LOAD_SHORT_D16 : VGLOBAL_Real_AllAddr_gfx12<0x020, "global_load_d16_b16">; |
| defm GLOBAL_LOAD_UBYTE_D16_HI : VGLOBAL_Real_AllAddr_gfx12<0x021, "global_load_d16_hi_u8">; |
| defm GLOBAL_LOAD_SBYTE_D16_HI : VGLOBAL_Real_AllAddr_gfx12<0x022, "global_load_d16_hi_i8">; |
| defm GLOBAL_LOAD_SHORT_D16_HI : VGLOBAL_Real_AllAddr_gfx12<0x023, "global_load_d16_hi_b16">; |
| defm GLOBAL_STORE_BYTE_D16_HI : VGLOBAL_Real_AllAddr_gfx12<0x024, "global_store_d16_hi_b8">; |
| defm GLOBAL_STORE_SHORT_D16_HI : VGLOBAL_Real_AllAddr_gfx12<0x025, "global_store_d16_hi_b16">; |
| defm GLOBAL_LOAD_DWORD_ADDTID : VGLOBAL_Real_AllAddr_gfx12<0x028, "global_load_addtid_b32">; |
| defm GLOBAL_STORE_DWORD_ADDTID : VGLOBAL_Real_AllAddr_gfx12<0x029, "global_store_addtid_b32">; |
| defm GLOBAL_LOAD_BLOCK : VGLOBAL_Real_AllAddr_gfx12<0x053>; |
| defm GLOBAL_STORE_BLOCK : VGLOBAL_Real_AllAddr_gfx12<0x054>; |
| |
| defm GLOBAL_ATOMIC_SWAP : VGLOBAL_Real_Atomics_gfx12<0x033, "global_atomic_swap_b32">; |
| defm GLOBAL_ATOMIC_CMPSWAP : VGLOBAL_Real_Atomics_gfx12<0x034, "global_atomic_cmpswap_b32">; |
| defm GLOBAL_ATOMIC_ADD : VGLOBAL_Real_Atomics_gfx12<0x035, "global_atomic_add_u32">; |
| defm GLOBAL_ATOMIC_SUB : VGLOBAL_Real_Atomics_gfx12<0x036, "global_atomic_sub_u32">; |
| defm GLOBAL_ATOMIC_CSUB : VGLOBAL_Real_Atomics_gfx12<0x037, "global_atomic_sub_clamp_u32", "global_atomic_csub_u32">; |
| defm GLOBAL_ATOMIC_SMIN : VGLOBAL_Real_Atomics_gfx12<0x038, "global_atomic_min_i32">; |
| defm GLOBAL_ATOMIC_UMIN : VGLOBAL_Real_Atomics_gfx12<0x039, "global_atomic_min_u32">; |
| defm GLOBAL_ATOMIC_SMAX : VGLOBAL_Real_Atomics_gfx12<0x03a, "global_atomic_max_i32">; |
| defm GLOBAL_ATOMIC_UMAX : VGLOBAL_Real_Atomics_gfx12<0x03b, "global_atomic_max_u32">; |
| defm GLOBAL_ATOMIC_AND : VGLOBAL_Real_Atomics_gfx12<0x03c, "global_atomic_and_b32">; |
| defm GLOBAL_ATOMIC_OR : VGLOBAL_Real_Atomics_gfx12<0x03d, "global_atomic_or_b32">; |
| defm GLOBAL_ATOMIC_XOR : VGLOBAL_Real_Atomics_gfx12<0x03e, "global_atomic_xor_b32">; |
| defm GLOBAL_ATOMIC_INC : VGLOBAL_Real_Atomics_gfx12<0x03f, "global_atomic_inc_u32">; |
| defm GLOBAL_ATOMIC_DEC : VGLOBAL_Real_Atomics_gfx12<0x040, "global_atomic_dec_u32">; |
| defm GLOBAL_ATOMIC_SWAP_X2 : VGLOBAL_Real_Atomics_gfx12<0x041, "global_atomic_swap_b64">; |
| defm GLOBAL_ATOMIC_CMPSWAP_X2 : VGLOBAL_Real_Atomics_gfx12<0x042, "global_atomic_cmpswap_b64">; |
| defm GLOBAL_ATOMIC_ADD_X2 : VGLOBAL_Real_Atomics_gfx12<0x043, "global_atomic_add_u64">; |
| defm GLOBAL_ATOMIC_SUB_X2 : VGLOBAL_Real_Atomics_gfx12<0x044, "global_atomic_sub_u64">; |
| defm GLOBAL_ATOMIC_SMIN_X2 : VGLOBAL_Real_Atomics_gfx12<0x045, "global_atomic_min_i64">; |
| defm GLOBAL_ATOMIC_UMIN_X2 : VGLOBAL_Real_Atomics_gfx12<0x046, "global_atomic_min_u64">; |
| defm GLOBAL_ATOMIC_SMAX_X2 : VGLOBAL_Real_Atomics_gfx12<0x047, "global_atomic_max_i64">; |
| defm GLOBAL_ATOMIC_UMAX_X2 : VGLOBAL_Real_Atomics_gfx12<0x048, "global_atomic_max_u64">; |
| defm GLOBAL_ATOMIC_AND_X2 : VGLOBAL_Real_Atomics_gfx12<0x049, "global_atomic_and_b64">; |
| defm GLOBAL_ATOMIC_OR_X2 : VGLOBAL_Real_Atomics_gfx12<0x04a, "global_atomic_or_b64">; |
| defm GLOBAL_ATOMIC_XOR_X2 : VGLOBAL_Real_Atomics_gfx12<0x04b, "global_atomic_xor_b64">; |
| defm GLOBAL_ATOMIC_INC_X2 : VGLOBAL_Real_Atomics_gfx12<0x04c, "global_atomic_inc_u64">; |
| defm GLOBAL_ATOMIC_DEC_X2 : VGLOBAL_Real_Atomics_gfx12<0x04d, "global_atomic_dec_u64">; |
| defm GLOBAL_ATOMIC_COND_SUB_U32 : VGLOBAL_Real_Atomics_gfx12<0x050>; |
| defm GLOBAL_ATOMIC_FMIN : VGLOBAL_Real_Atomics_gfx12<0x051, "global_atomic_min_num_f32", "global_atomic_min_f32">; |
| defm GLOBAL_ATOMIC_FMAX : VGLOBAL_Real_Atomics_gfx12<0x052, "global_atomic_max_num_f32", "global_atomic_max_f32">; |
| defm GLOBAL_ATOMIC_ADD_F32 : VGLOBAL_Real_Atomics_gfx12<0x056>; |
| |
| defm GLOBAL_LOAD_TR_B128_w32 : VGLOBAL_Real_AllAddr_gfx12<0x057>; |
| defm GLOBAL_LOAD_TR_B64_w32 : VGLOBAL_Real_AllAddr_gfx12<0x058>; |
| |
| defm GLOBAL_LOAD_TR_B128_w64 : VGLOBAL_Real_AllAddr_gfx12_w64<0x057>; |
| defm GLOBAL_LOAD_TR_B64_w64 : VGLOBAL_Real_AllAddr_gfx12_w64<0x058>; |
| |
| defm GLOBAL_ATOMIC_ORDERED_ADD_B64 : VGLOBAL_Real_Atomics_gfx12<0x073>; |
| defm GLOBAL_ATOMIC_PK_ADD_F16 : VGLOBAL_Real_Atomics_gfx12<0x059>; |
| defm GLOBAL_ATOMIC_PK_ADD_BF16 : VGLOBAL_Real_Atomics_gfx12<0x05a>; |
| |
| defm GLOBAL_INV : VFLAT_Real_Base_gfx12<0x02b>; |
| defm GLOBAL_WB : VFLAT_Real_Base_gfx12<0x02c>; |
| defm GLOBAL_WBINV : VFLAT_Real_Base_gfx12<0x04f>; |
| |
| // ENC_VSCRATCH. |
| defm SCRATCH_LOAD_UBYTE : VSCRATCH_Real_AllAddr_gfx12<0x10, "scratch_load_u8">; |
| defm SCRATCH_LOAD_SBYTE : VSCRATCH_Real_AllAddr_gfx12<0x11, "scratch_load_i8">; |
| defm SCRATCH_LOAD_USHORT : VSCRATCH_Real_AllAddr_gfx12<0x12, "scratch_load_u16">; |
| defm SCRATCH_LOAD_SSHORT : VSCRATCH_Real_AllAddr_gfx12<0x13, "scratch_load_i16">; |
| defm SCRATCH_LOAD_DWORD : VSCRATCH_Real_AllAddr_gfx12<0x14, "scratch_load_b32">; |
| defm SCRATCH_LOAD_DWORDX2 : VSCRATCH_Real_AllAddr_gfx12<0x15, "scratch_load_b64">; |
| defm SCRATCH_LOAD_DWORDX3 : VSCRATCH_Real_AllAddr_gfx12<0x16, "scratch_load_b96">; |
| defm SCRATCH_LOAD_DWORDX4 : VSCRATCH_Real_AllAddr_gfx12<0x17, "scratch_load_b128">; |
| defm SCRATCH_STORE_BYTE : VSCRATCH_Real_AllAddr_gfx12<0x18, "scratch_store_b8">; |
| defm SCRATCH_STORE_SHORT : VSCRATCH_Real_AllAddr_gfx12<0x19, "scratch_store_b16">; |
| defm SCRATCH_STORE_DWORD : VSCRATCH_Real_AllAddr_gfx12<0x1a, "scratch_store_b32">; |
| defm SCRATCH_STORE_DWORDX2 : VSCRATCH_Real_AllAddr_gfx12<0x1b, "scratch_store_b64">; |
| defm SCRATCH_STORE_DWORDX3 : VSCRATCH_Real_AllAddr_gfx12<0x1c, "scratch_store_b96">; |
| defm SCRATCH_STORE_DWORDX4 : VSCRATCH_Real_AllAddr_gfx12<0x1d, "scratch_store_b128">; |
| defm SCRATCH_LOAD_UBYTE_D16 : VSCRATCH_Real_AllAddr_gfx12<0x1e, "scratch_load_d16_u8">; |
| defm SCRATCH_LOAD_SBYTE_D16 : VSCRATCH_Real_AllAddr_gfx12<0x1f, "scratch_load_d16_i8">; |
| defm SCRATCH_LOAD_SHORT_D16 : VSCRATCH_Real_AllAddr_gfx12<0x20, "scratch_load_d16_b16">; |
| defm SCRATCH_LOAD_UBYTE_D16_HI : VSCRATCH_Real_AllAddr_gfx12<0x21, "scratch_load_d16_hi_u8">; |
| defm SCRATCH_LOAD_SBYTE_D16_HI : VSCRATCH_Real_AllAddr_gfx12<0x22, "scratch_load_d16_hi_i8">; |
| defm SCRATCH_LOAD_SHORT_D16_HI : VSCRATCH_Real_AllAddr_gfx12<0x23, "scratch_load_d16_hi_b16">; |
| defm SCRATCH_STORE_BYTE_D16_HI : VSCRATCH_Real_AllAddr_gfx12<0x24, "scratch_store_d16_hi_b8">; |
| defm SCRATCH_STORE_SHORT_D16_HI : VSCRATCH_Real_AllAddr_gfx12<0x25, "scratch_store_d16_hi_b16">; |
| |
| defm SCRATCH_LOAD_BLOCK : VSCRATCH_Real_AllAddr_gfx12<0x53>; |
| defm SCRATCH_STORE_BLOCK : VSCRATCH_Real_AllAddr_gfx12<0x54>; |
| |
| def True16D16Table : GenericTable { |
| let FilterClass = "True16D16Table"; |
| let CppTypeName = "True16D16Info"; |
| let Fields = ["T16Op", "HiOp", "LoOp"]; |
| let PrimaryKey = ["T16Op"]; |
| let PrimaryKeyName = "getT16D16Helper"; |
| } |
