1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines an instruction selector for the ARM target. 11// 12//===----------------------------------------------------------------------===// 13 14#include "ARM.h" 15#include "ARMBaseInstrInfo.h" 16#include "ARMTargetMachine.h" 17#include "MCTargetDesc/ARMAddressingModes.h" 18#include "llvm/CodeGen/MachineFrameInfo.h" 19#include "llvm/CodeGen/MachineFunction.h" 20#include "llvm/CodeGen/MachineInstrBuilder.h" 21#include "llvm/CodeGen/MachineRegisterInfo.h" 22#include "llvm/CodeGen/SelectionDAG.h" 23#include "llvm/CodeGen/SelectionDAGISel.h" 24#include "llvm/IR/CallingConv.h" 25#include "llvm/IR/Constants.h" 26#include "llvm/IR/DerivedTypes.h" 27#include "llvm/IR/Function.h" 28#include "llvm/IR/Intrinsics.h" 29#include "llvm/IR/LLVMContext.h" 30#include "llvm/Support/CommandLine.h" 31#include "llvm/Support/Compiler.h" 32#include "llvm/Support/Debug.h" 33#include "llvm/Support/ErrorHandling.h" 34#include "llvm/Target/TargetLowering.h" 35#include "llvm/Target/TargetOptions.h" 36 37using namespace llvm; 38 39#define DEBUG_TYPE "arm-isel" 40 41static cl::opt<bool> 42DisableShifterOp("disable-shifter-op", cl::Hidden, 43 cl::desc("Disable isel of shifter-op"), 44 cl::init(false)); 45 46static cl::opt<bool> 47CheckVMLxHazard("check-vmlx-hazard", cl::Hidden, 48 cl::desc("Check fp vmla / vmls hazard at isel time"), 49 cl::init(true)); 50 51//===--------------------------------------------------------------------===// 52/// ARMDAGToDAGISel - ARM specific code to select ARM machine 53/// instructions for SelectionDAG operations. 54/// 55namespace { 56 57enum AddrMode2Type { 58 AM2_BASE, // Simple AM2 (+-imm12) 59 AM2_SHOP // Shifter-op AM2 60}; 61 62class ARMDAGToDAGISel : public SelectionDAGISel { 63 /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can 64 /// make the right decision when generating code for different targets. 65 const ARMSubtarget *Subtarget; 66 67public: 68 explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, CodeGenOpt::Level OptLevel) 69 : SelectionDAGISel(tm, OptLevel) {} 70 71 bool runOnMachineFunction(MachineFunction &MF) override { 72 // Reset the subtarget each time through. 73 Subtarget = &MF.getSubtarget<ARMSubtarget>(); 74 SelectionDAGISel::runOnMachineFunction(MF); 75 return true; 76 } 77 78 const char *getPassName() const override { 79 return "ARM Instruction Selection"; 80 } 81 82 void PreprocessISelDAG() override; 83 84 /// getI32Imm - Return a target constant of type i32 with the specified 85 /// value. 86 inline SDValue getI32Imm(unsigned Imm) { 87 return CurDAG->getTargetConstant(Imm, MVT::i32); 88 } 89 90 SDNode *Select(SDNode *N) override; 91 92 93 bool hasNoVMLxHazardUse(SDNode *N) const; 94 bool isShifterOpProfitable(const SDValue &Shift, 95 ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt); 96 bool SelectRegShifterOperand(SDValue N, SDValue &A, 97 SDValue &B, SDValue &C, 98 bool CheckProfitability = true); 99 bool SelectImmShifterOperand(SDValue N, SDValue &A, 100 SDValue &B, bool CheckProfitability = true); 101 bool SelectShiftRegShifterOperand(SDValue N, SDValue &A, 102 SDValue &B, SDValue &C) { 103 // Don't apply the profitability check 104 return SelectRegShifterOperand(N, A, B, C, false); 105 } 106 bool SelectShiftImmShifterOperand(SDValue N, SDValue &A, 107 SDValue &B) { 108 // Don't apply the profitability check 109 return SelectImmShifterOperand(N, A, B, false); 110 } 111 112 bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm); 113 bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc); 114 115 AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base, 116 SDValue &Offset, SDValue &Opc); 117 bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset, 118 SDValue &Opc) { 119 return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE; 120 } 121 122 bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset, 123 SDValue &Opc) { 124 return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP; 125 } 126 127 bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset, 128 SDValue &Opc) { 129 SelectAddrMode2Worker(N, Base, Offset, Opc); 130// return SelectAddrMode2ShOp(N, Base, Offset, Opc); 131 // This always matches one way or another. 132 return true; 133 } 134 135 bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) { 136 const ConstantSDNode *CN = cast<ConstantSDNode>(N); 137 Pred = CurDAG->getTargetConstant(CN->getZExtValue(), MVT::i32); 138 Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32); 139 return true; 140 } 141 142 bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N, 143 SDValue &Offset, SDValue &Opc); 144 bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N, 145 SDValue &Offset, SDValue &Opc); 146 bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N, 147 SDValue &Offset, SDValue &Opc); 148 bool SelectAddrOffsetNone(SDValue N, SDValue &Base); 149 bool SelectAddrMode3(SDValue N, SDValue &Base, 150 SDValue &Offset, SDValue &Opc); 151 bool SelectAddrMode3Offset(SDNode *Op, SDValue N, 152 SDValue &Offset, SDValue &Opc); 153 bool SelectAddrMode5(SDValue N, SDValue &Base, 154 SDValue &Offset); 155 bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align); 156 bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset); 157 158 bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label); 159 160 // Thumb Addressing Modes: 161 bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset); 162 bool SelectThumbAddrModeRI(SDValue N, SDValue &Base, SDValue &Offset, 163 unsigned Scale); 164 bool SelectThumbAddrModeRI5S1(SDValue N, SDValue &Base, SDValue &Offset); 165 bool SelectThumbAddrModeRI5S2(SDValue N, SDValue &Base, SDValue &Offset); 166 bool SelectThumbAddrModeRI5S4(SDValue N, SDValue &Base, SDValue &Offset); 167 bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base, 168 SDValue &OffImm); 169 bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base, 170 SDValue &OffImm); 171 bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base, 172 SDValue &OffImm); 173 bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base, 174 SDValue &OffImm); 175 bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm); 176 177 // Thumb 2 Addressing Modes: 178 bool SelectT2ShifterOperandReg(SDValue N, 179 SDValue &BaseReg, SDValue &Opc); 180 bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm); 181 bool SelectT2AddrModeImm8(SDValue N, SDValue &Base, 182 SDValue &OffImm); 183 bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N, 184 SDValue &OffImm); 185 bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base, 186 SDValue &OffReg, SDValue &ShImm); 187 bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm); 188 189 inline bool is_so_imm(unsigned Imm) const { 190 return ARM_AM::getSOImmVal(Imm) != -1; 191 } 192 193 inline bool is_so_imm_not(unsigned Imm) const { 194 return ARM_AM::getSOImmVal(~Imm) != -1; 195 } 196 197 inline bool is_t2_so_imm(unsigned Imm) const { 198 return ARM_AM::getT2SOImmVal(Imm) != -1; 199 } 200 201 inline bool is_t2_so_imm_not(unsigned Imm) const { 202 return ARM_AM::getT2SOImmVal(~Imm) != -1; 203 } 204 205 // Include the pieces autogenerated from the target description. 206#include "ARMGenDAGISel.inc" 207 208private: 209 /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for 210 /// ARM. 211 SDNode *SelectARMIndexedLoad(SDNode *N); 212 SDNode *SelectT2IndexedLoad(SDNode *N); 213 214 /// SelectVLD - Select NEON load intrinsics. NumVecs should be 215 /// 1, 2, 3 or 4. The opcode arrays specify the instructions used for 216 /// loads of D registers and even subregs and odd subregs of Q registers. 217 /// For NumVecs <= 2, QOpcodes1 is not used. 218 SDNode *SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs, 219 const uint16_t *DOpcodes, 220 const uint16_t *QOpcodes0, const uint16_t *QOpcodes1); 221 222 /// SelectVST - Select NEON store intrinsics. NumVecs should 223 /// be 1, 2, 3 or 4. The opcode arrays specify the instructions used for 224 /// stores of D registers and even subregs and odd subregs of Q registers. 225 /// For NumVecs <= 2, QOpcodes1 is not used. 226 SDNode *SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, 227 const uint16_t *DOpcodes, 228 const uint16_t *QOpcodes0, const uint16_t *QOpcodes1); 229 230 /// SelectVLDSTLane - Select NEON load/store lane intrinsics. NumVecs should 231 /// be 2, 3 or 4. The opcode arrays specify the instructions used for 232 /// load/store of D registers and Q registers. 233 SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, 234 bool isUpdating, unsigned NumVecs, 235 const uint16_t *DOpcodes, const uint16_t *QOpcodes); 236 237 /// SelectVLDDup - Select NEON load-duplicate intrinsics. NumVecs 238 /// should be 2, 3 or 4. The opcode array specifies the instructions used 239 /// for loading D registers. (Q registers are not supported.) 240 SDNode *SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs, 241 const uint16_t *Opcodes); 242 243 /// SelectVTBL - Select NEON VTBL and VTBX intrinsics. NumVecs should be 2, 244 /// 3 or 4. These are custom-selected so that a REG_SEQUENCE can be 245 /// generated to force the table registers to be consecutive. 246 SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc); 247 248 /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM. 249 SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned); 250 251 // Select special operations if node forms integer ABS pattern 252 SDNode *SelectABSOp(SDNode *N); 253 254 SDNode *SelectInlineAsm(SDNode *N); 255 256 SDNode *SelectConcatVector(SDNode *N); 257 258 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 259 /// inline asm expressions. 260 bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, 261 std::vector<SDValue> &OutOps) override; 262 263 // Form pairs of consecutive R, S, D, or Q registers. 264 SDNode *createGPRPairNode(EVT VT, SDValue V0, SDValue V1); 265 SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1); 266 SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1); 267 SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1); 268 269 // Form sequences of 4 consecutive S, D, or Q registers. 270 SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); 271 SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); 272 SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); 273 274 // Get the alignment operand for a NEON VLD or VST instruction. 275 SDValue GetVLDSTAlign(SDValue Align, unsigned NumVecs, bool is64BitVector); 276}; 277} 278 279/// isInt32Immediate - This method tests to see if the node is a 32-bit constant 280/// operand. If so Imm will receive the 32-bit value. 281static bool isInt32Immediate(SDNode *N, unsigned &Imm) { 282 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) { 283 Imm = cast<ConstantSDNode>(N)->getZExtValue(); 284 return true; 285 } 286 return false; 287} 288 289// isInt32Immediate - This method tests to see if a constant operand. 290// If so Imm will receive the 32 bit value. 291static bool isInt32Immediate(SDValue N, unsigned &Imm) { 292 return isInt32Immediate(N.getNode(), Imm); 293} 294 295// isOpcWithIntImmediate - This method tests to see if the node is a specific 296// opcode and that it has a immediate integer right operand. 297// If so Imm will receive the 32 bit value. 298static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) { 299 return N->getOpcode() == Opc && 300 isInt32Immediate(N->getOperand(1).getNode(), Imm); 301} 302 303/// \brief Check whether a particular node is a constant value representable as 304/// (N * Scale) where (N in [\p RangeMin, \p RangeMax). 305/// 306/// \param ScaledConstant [out] - On success, the pre-scaled constant value. 307static bool isScaledConstantInRange(SDValue Node, int Scale, 308 int RangeMin, int RangeMax, 309 int &ScaledConstant) { 310 assert(Scale > 0 && "Invalid scale!"); 311 312 // Check that this is a constant. 313 const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node); 314 if (!C) 315 return false; 316 317 ScaledConstant = (int) C->getZExtValue(); 318 if ((ScaledConstant % Scale) != 0) 319 return false; 320 321 ScaledConstant /= Scale; 322 return ScaledConstant >= RangeMin && ScaledConstant < RangeMax; 323} 324 325void ARMDAGToDAGISel::PreprocessISelDAG() { 326 if (!Subtarget->hasV6T2Ops()) 327 return; 328 329 bool isThumb2 = Subtarget->isThumb(); 330 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 331 E = CurDAG->allnodes_end(); I != E; ) { 332 SDNode *N = I++; // Preincrement iterator to avoid invalidation issues. 333 334 if (N->getOpcode() != ISD::ADD) 335 continue; 336 337 // Look for (add X1, (and (srl X2, c1), c2)) where c2 is constant with 338 // leading zeros, followed by consecutive set bits, followed by 1 or 2 339 // trailing zeros, e.g. 1020. 340 // Transform the expression to 341 // (add X1, (shl (and (srl X2, c1), (c2>>tz)), tz)) where tz is the number 342 // of trailing zeros of c2. The left shift would be folded as an shifter 343 // operand of 'add' and the 'and' and 'srl' would become a bits extraction 344 // node (UBFX). 345 346 SDValue N0 = N->getOperand(0); 347 SDValue N1 = N->getOperand(1); 348 unsigned And_imm = 0; 349 if (!isOpcWithIntImmediate(N1.getNode(), ISD::AND, And_imm)) { 350 if (isOpcWithIntImmediate(N0.getNode(), ISD::AND, And_imm)) 351 std::swap(N0, N1); 352 } 353 if (!And_imm) 354 continue; 355 356 // Check if the AND mask is an immediate of the form: 000.....1111111100 357 unsigned TZ = countTrailingZeros(And_imm); 358 if (TZ != 1 && TZ != 2) 359 // Be conservative here. Shifter operands aren't always free. e.g. On 360 // Swift, left shifter operand of 1 / 2 for free but others are not. 361 // e.g. 362 // ubfx r3, r1, #16, #8 363 // ldr.w r3, [r0, r3, lsl #2] 364 // vs. 365 // mov.w r9, #1020 366 // and.w r2, r9, r1, lsr #14 367 // ldr r2, [r0, r2] 368 continue; 369 And_imm >>= TZ; 370 if (And_imm & (And_imm + 1)) 371 continue; 372 373 // Look for (and (srl X, c1), c2). 374 SDValue Srl = N1.getOperand(0); 375 unsigned Srl_imm = 0; 376 if (!isOpcWithIntImmediate(Srl.getNode(), ISD::SRL, Srl_imm) || 377 (Srl_imm <= 2)) 378 continue; 379 380 // Make sure first operand is not a shifter operand which would prevent 381 // folding of the left shift. 382 SDValue CPTmp0; 383 SDValue CPTmp1; 384 SDValue CPTmp2; 385 if (isThumb2) { 386 if (SelectT2ShifterOperandReg(N0, CPTmp0, CPTmp1)) 387 continue; 388 } else { 389 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) || 390 SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2)) 391 continue; 392 } 393 394 // Now make the transformation. 395 Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32, 396 Srl.getOperand(0), 397 CurDAG->getConstant(Srl_imm+TZ, MVT::i32)); 398 N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32, 399 Srl, CurDAG->getConstant(And_imm, MVT::i32)); 400 N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32, 401 N1, CurDAG->getConstant(TZ, MVT::i32)); 402 CurDAG->UpdateNodeOperands(N, N0, N1); 403 } 404} 405 406/// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS 407/// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at 408/// least on current ARM implementations) which should be avoidded. 409bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const { 410 if (OptLevel == CodeGenOpt::None) 411 return true; 412 413 if (!CheckVMLxHazard) 414 return true; 415 416 if (!Subtarget->isCortexA7() && !Subtarget->isCortexA8() && 417 !Subtarget->isCortexA9() && !Subtarget->isSwift()) 418 return true; 419 420 if (!N->hasOneUse()) 421 return false; 422 423 SDNode *Use = *N->use_begin(); 424 if (Use->getOpcode() == ISD::CopyToReg) 425 return true; 426 if (Use->isMachineOpcode()) { 427 const ARMBaseInstrInfo *TII = static_cast<const ARMBaseInstrInfo *>( 428 CurDAG->getSubtarget().getInstrInfo()); 429 430 const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode()); 431 if (MCID.mayStore()) 432 return true; 433 unsigned Opcode = MCID.getOpcode(); 434 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD) 435 return true; 436 // vmlx feeding into another vmlx. We actually want to unfold 437 // the use later in the MLxExpansion pass. e.g. 438 // vmla 439 // vmla (stall 8 cycles) 440 // 441 // vmul (5 cycles) 442 // vadd (5 cycles) 443 // vmla 444 // This adds up to about 18 - 19 cycles. 445 // 446 // vmla 447 // vmul (stall 4 cycles) 448 // vadd adds up to about 14 cycles. 449 return TII->isFpMLxInstruction(Opcode); 450 } 451 452 return false; 453} 454 455bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift, 456 ARM_AM::ShiftOpc ShOpcVal, 457 unsigned ShAmt) { 458 if (!Subtarget->isLikeA9() && !Subtarget->isSwift()) 459 return true; 460 if (Shift.hasOneUse()) 461 return true; 462 // R << 2 is free. 463 return ShOpcVal == ARM_AM::lsl && 464 (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1)); 465} 466 467bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N, 468 SDValue &BaseReg, 469 SDValue &Opc, 470 bool CheckProfitability) { 471 if (DisableShifterOp) 472 return false; 473 474 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 475 476 // Don't match base register only case. That is matched to a separate 477 // lower complexity pattern with explicit register operand. 478 if (ShOpcVal == ARM_AM::no_shift) return false; 479 480 BaseReg = N.getOperand(0); 481 unsigned ShImmVal = 0; 482 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 483 if (!RHS) return false; 484 ShImmVal = RHS->getZExtValue() & 31; 485 Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal), 486 MVT::i32); 487 return true; 488} 489 490bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N, 491 SDValue &BaseReg, 492 SDValue &ShReg, 493 SDValue &Opc, 494 bool CheckProfitability) { 495 if (DisableShifterOp) 496 return false; 497 498 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 499 500 // Don't match base register only case. That is matched to a separate 501 // lower complexity pattern with explicit register operand. 502 if (ShOpcVal == ARM_AM::no_shift) return false; 503 504 BaseReg = N.getOperand(0); 505 unsigned ShImmVal = 0; 506 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 507 if (RHS) return false; 508 509 ShReg = N.getOperand(1); 510 if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal)) 511 return false; 512 Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal), 513 MVT::i32); 514 return true; 515} 516 517 518bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N, 519 SDValue &Base, 520 SDValue &OffImm) { 521 // Match simple R + imm12 operands. 522 523 // Base only. 524 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 525 !CurDAG->isBaseWithConstantOffset(N)) { 526 if (N.getOpcode() == ISD::FrameIndex) { 527 // Match frame index. 528 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 529 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); 530 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 531 return true; 532 } 533 534 if (N.getOpcode() == ARMISD::Wrapper && 535 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) { 536 Base = N.getOperand(0); 537 } else 538 Base = N; 539 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 540 return true; 541 } 542 543 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 544 int RHSC = (int)RHS->getSExtValue(); 545 if (N.getOpcode() == ISD::SUB) 546 RHSC = -RHSC; 547 548 if (RHSC > -0x1000 && RHSC < 0x1000) { // 12 bits 549 Base = N.getOperand(0); 550 if (Base.getOpcode() == ISD::FrameIndex) { 551 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 552 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); 553 } 554 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 555 return true; 556 } 557 } 558 559 // Base only. 560 Base = N; 561 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 562 return true; 563} 564 565 566 567bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, 568 SDValue &Opc) { 569 if (N.getOpcode() == ISD::MUL && 570 ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) { 571 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 572 // X * [3,5,9] -> X + X * [2,4,8] etc. 573 int RHSC = (int)RHS->getZExtValue(); 574 if (RHSC & 1) { 575 RHSC = RHSC & ~1; 576 ARM_AM::AddrOpc AddSub = ARM_AM::add; 577 if (RHSC < 0) { 578 AddSub = ARM_AM::sub; 579 RHSC = - RHSC; 580 } 581 if (isPowerOf2_32(RHSC)) { 582 unsigned ShAmt = Log2_32(RHSC); 583 Base = Offset = N.getOperand(0); 584 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, 585 ARM_AM::lsl), 586 MVT::i32); 587 return true; 588 } 589 } 590 } 591 } 592 593 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 594 // ISD::OR that is equivalent to an ISD::ADD. 595 !CurDAG->isBaseWithConstantOffset(N)) 596 return false; 597 598 // Leave simple R +/- imm12 operands for LDRi12 599 if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) { 600 int RHSC; 601 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, 602 -0x1000+1, 0x1000, RHSC)) // 12 bits. 603 return false; 604 } 605 606 // Otherwise this is R +/- [possibly shifted] R. 607 ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add; 608 ARM_AM::ShiftOpc ShOpcVal = 609 ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode()); 610 unsigned ShAmt = 0; 611 612 Base = N.getOperand(0); 613 Offset = N.getOperand(1); 614 615 if (ShOpcVal != ARM_AM::no_shift) { 616 // Check to see if the RHS of the shift is a constant, if not, we can't fold 617 // it. 618 if (ConstantSDNode *Sh = 619 dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) { 620 ShAmt = Sh->getZExtValue(); 621 if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt)) 622 Offset = N.getOperand(1).getOperand(0); 623 else { 624 ShAmt = 0; 625 ShOpcVal = ARM_AM::no_shift; 626 } 627 } else { 628 ShOpcVal = ARM_AM::no_shift; 629 } 630 } 631 632 // Try matching (R shl C) + (R). 633 if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift && 634 !(Subtarget->isLikeA9() || Subtarget->isSwift() || 635 N.getOperand(0).hasOneUse())) { 636 ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode()); 637 if (ShOpcVal != ARM_AM::no_shift) { 638 // Check to see if the RHS of the shift is a constant, if not, we can't 639 // fold it. 640 if (ConstantSDNode *Sh = 641 dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) { 642 ShAmt = Sh->getZExtValue(); 643 if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) { 644 Offset = N.getOperand(0).getOperand(0); 645 Base = N.getOperand(1); 646 } else { 647 ShAmt = 0; 648 ShOpcVal = ARM_AM::no_shift; 649 } 650 } else { 651 ShOpcVal = ARM_AM::no_shift; 652 } 653 } 654 } 655 656 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), 657 MVT::i32); 658 return true; 659} 660 661 662//----- 663 664AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N, 665 SDValue &Base, 666 SDValue &Offset, 667 SDValue &Opc) { 668 if (N.getOpcode() == ISD::MUL && 669 (!(Subtarget->isLikeA9() || Subtarget->isSwift()) || N.hasOneUse())) { 670 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 671 // X * [3,5,9] -> X + X * [2,4,8] etc. 672 int RHSC = (int)RHS->getZExtValue(); 673 if (RHSC & 1) { 674 RHSC = RHSC & ~1; 675 ARM_AM::AddrOpc AddSub = ARM_AM::add; 676 if (RHSC < 0) { 677 AddSub = ARM_AM::sub; 678 RHSC = - RHSC; 679 } 680 if (isPowerOf2_32(RHSC)) { 681 unsigned ShAmt = Log2_32(RHSC); 682 Base = Offset = N.getOperand(0); 683 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, 684 ARM_AM::lsl), 685 MVT::i32); 686 return AM2_SHOP; 687 } 688 } 689 } 690 } 691 692 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 693 // ISD::OR that is equivalent to an ADD. 694 !CurDAG->isBaseWithConstantOffset(N)) { 695 Base = N; 696 if (N.getOpcode() == ISD::FrameIndex) { 697 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 698 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); 699 } else if (N.getOpcode() == ARMISD::Wrapper && 700 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) { 701 Base = N.getOperand(0); 702 } 703 Offset = CurDAG->getRegister(0, MVT::i32); 704 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0, 705 ARM_AM::no_shift), 706 MVT::i32); 707 return AM2_BASE; 708 } 709 710 // Match simple R +/- imm12 operands. 711 if (N.getOpcode() != ISD::SUB) { 712 int RHSC; 713 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, 714 -0x1000+1, 0x1000, RHSC)) { // 12 bits. 715 Base = N.getOperand(0); 716 if (Base.getOpcode() == ISD::FrameIndex) { 717 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 718 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); 719 } 720 Offset = CurDAG->getRegister(0, MVT::i32); 721 722 ARM_AM::AddrOpc AddSub = ARM_AM::add; 723 if (RHSC < 0) { 724 AddSub = ARM_AM::sub; 725 RHSC = - RHSC; 726 } 727 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC, 728 ARM_AM::no_shift), 729 MVT::i32); 730 return AM2_BASE; 731 } 732 } 733 734 if ((Subtarget->isLikeA9() || Subtarget->isSwift()) && !N.hasOneUse()) { 735 // Compute R +/- (R << N) and reuse it. 736 Base = N; 737 Offset = CurDAG->getRegister(0, MVT::i32); 738 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0, 739 ARM_AM::no_shift), 740 MVT::i32); 741 return AM2_BASE; 742 } 743 744 // Otherwise this is R +/- [possibly shifted] R. 745 ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub; 746 ARM_AM::ShiftOpc ShOpcVal = 747 ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode()); 748 unsigned ShAmt = 0; 749 750 Base = N.getOperand(0); 751 Offset = N.getOperand(1); 752 753 if (ShOpcVal != ARM_AM::no_shift) { 754 // Check to see if the RHS of the shift is a constant, if not, we can't fold 755 // it. 756 if (ConstantSDNode *Sh = 757 dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) { 758 ShAmt = Sh->getZExtValue(); 759 if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt)) 760 Offset = N.getOperand(1).getOperand(0); 761 else { 762 ShAmt = 0; 763 ShOpcVal = ARM_AM::no_shift; 764 } 765 } else { 766 ShOpcVal = ARM_AM::no_shift; 767 } 768 } 769 770 // Try matching (R shl C) + (R). 771 if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift && 772 !(Subtarget->isLikeA9() || Subtarget->isSwift() || 773 N.getOperand(0).hasOneUse())) { 774 ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode()); 775 if (ShOpcVal != ARM_AM::no_shift) { 776 // Check to see if the RHS of the shift is a constant, if not, we can't 777 // fold it. 778 if (ConstantSDNode *Sh = 779 dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) { 780 ShAmt = Sh->getZExtValue(); 781 if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) { 782 Offset = N.getOperand(0).getOperand(0); 783 Base = N.getOperand(1); 784 } else { 785 ShAmt = 0; 786 ShOpcVal = ARM_AM::no_shift; 787 } 788 } else { 789 ShOpcVal = ARM_AM::no_shift; 790 } 791 } 792 } 793 794 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), 795 MVT::i32); 796 return AM2_SHOP; 797} 798 799bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N, 800 SDValue &Offset, SDValue &Opc) { 801 unsigned Opcode = Op->getOpcode(); 802 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 803 ? cast<LoadSDNode>(Op)->getAddressingMode() 804 : cast<StoreSDNode>(Op)->getAddressingMode(); 805 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 806 ? ARM_AM::add : ARM_AM::sub; 807 int Val; 808 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) 809 return false; 810 811 Offset = N; 812 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 813 unsigned ShAmt = 0; 814 if (ShOpcVal != ARM_AM::no_shift) { 815 // Check to see if the RHS of the shift is a constant, if not, we can't fold 816 // it. 817 if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 818 ShAmt = Sh->getZExtValue(); 819 if (isShifterOpProfitable(N, ShOpcVal, ShAmt)) 820 Offset = N.getOperand(0); 821 else { 822 ShAmt = 0; 823 ShOpcVal = ARM_AM::no_shift; 824 } 825 } else { 826 ShOpcVal = ARM_AM::no_shift; 827 } 828 } 829 830 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), 831 MVT::i32); 832 return true; 833} 834 835bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N, 836 SDValue &Offset, SDValue &Opc) { 837 unsigned Opcode = Op->getOpcode(); 838 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 839 ? cast<LoadSDNode>(Op)->getAddressingMode() 840 : cast<StoreSDNode>(Op)->getAddressingMode(); 841 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 842 ? ARM_AM::add : ARM_AM::sub; 843 int Val; 844 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits. 845 if (AddSub == ARM_AM::sub) Val *= -1; 846 Offset = CurDAG->getRegister(0, MVT::i32); 847 Opc = CurDAG->getTargetConstant(Val, MVT::i32); 848 return true; 849 } 850 851 return false; 852} 853 854 855bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N, 856 SDValue &Offset, SDValue &Opc) { 857 unsigned Opcode = Op->getOpcode(); 858 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 859 ? cast<LoadSDNode>(Op)->getAddressingMode() 860 : cast<StoreSDNode>(Op)->getAddressingMode(); 861 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 862 ? ARM_AM::add : ARM_AM::sub; 863 int Val; 864 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits. 865 Offset = CurDAG->getRegister(0, MVT::i32); 866 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val, 867 ARM_AM::no_shift), 868 MVT::i32); 869 return true; 870 } 871 872 return false; 873} 874 875bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) { 876 Base = N; 877 return true; 878} 879 880bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N, 881 SDValue &Base, SDValue &Offset, 882 SDValue &Opc) { 883 if (N.getOpcode() == ISD::SUB) { 884 // X - C is canonicalize to X + -C, no need to handle it here. 885 Base = N.getOperand(0); 886 Offset = N.getOperand(1); 887 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32); 888 return true; 889 } 890 891 if (!CurDAG->isBaseWithConstantOffset(N)) { 892 Base = N; 893 if (N.getOpcode() == ISD::FrameIndex) { 894 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 895 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); 896 } 897 Offset = CurDAG->getRegister(0, MVT::i32); 898 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32); 899 return true; 900 } 901 902 // If the RHS is +/- imm8, fold into addr mode. 903 int RHSC; 904 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, 905 -256 + 1, 256, RHSC)) { // 8 bits. 906 Base = N.getOperand(0); 907 if (Base.getOpcode() == ISD::FrameIndex) { 908 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 909 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); 910 } 911 Offset = CurDAG->getRegister(0, MVT::i32); 912 913 ARM_AM::AddrOpc AddSub = ARM_AM::add; 914 if (RHSC < 0) { 915 AddSub = ARM_AM::sub; 916 RHSC = -RHSC; 917 } 918 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32); 919 return true; 920 } 921 922 Base = N.getOperand(0); 923 Offset = N.getOperand(1); 924 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32); 925 return true; 926} 927 928bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N, 929 SDValue &Offset, SDValue &Opc) { 930 unsigned Opcode = Op->getOpcode(); 931 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 932 ? cast<LoadSDNode>(Op)->getAddressingMode() 933 : cast<StoreSDNode>(Op)->getAddressingMode(); 934 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 935 ? ARM_AM::add : ARM_AM::sub; 936 int Val; 937 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits. 938 Offset = CurDAG->getRegister(0, MVT::i32); 939 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32); 940 return true; 941 } 942 943 Offset = N; 944 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32); 945 return true; 946} 947 948bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N, 949 SDValue &Base, SDValue &Offset) { 950 if (!CurDAG->isBaseWithConstantOffset(N)) { 951 Base = N; 952 if (N.getOpcode() == ISD::FrameIndex) { 953 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 954 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); 955 } else if (N.getOpcode() == ARMISD::Wrapper && 956 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) { 957 Base = N.getOperand(0); 958 } 959 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0), 960 MVT::i32); 961 return true; 962 } 963 964 // If the RHS is +/- imm8, fold into addr mode. 965 int RHSC; 966 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 967 -256 + 1, 256, RHSC)) { 968 Base = N.getOperand(0); 969 if (Base.getOpcode() == ISD::FrameIndex) { 970 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 971 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); 972 } 973 974 ARM_AM::AddrOpc AddSub = ARM_AM::add; 975 if (RHSC < 0) { 976 AddSub = ARM_AM::sub; 977 RHSC = -RHSC; 978 } 979 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC), 980 MVT::i32); 981 return true; 982 } 983 984 Base = N; 985 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0), 986 MVT::i32); 987 return true; 988} 989 990bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr, 991 SDValue &Align) { 992 Addr = N; 993 994 unsigned Alignment = 0; 995 996 MemSDNode *MemN = cast<MemSDNode>(Parent); 997 998 if (isa<LSBaseSDNode>(MemN) || 999 ((MemN->getOpcode() == ARMISD::VST1_UPD || 1000 MemN->getOpcode() == ARMISD::VLD1_UPD) && 1001 MemN->getConstantOperandVal(MemN->getNumOperands() - 1) == 1)) { 1002 // This case occurs only for VLD1-lane/dup and VST1-lane instructions. 1003 // The maximum alignment is equal to the memory size being referenced. 1004 unsigned MMOAlign = MemN->getAlignment(); 1005 unsigned MemSize = MemN->getMemoryVT().getSizeInBits() / 8; 1006 if (MMOAlign >= MemSize && MemSize > 1) 1007 Alignment = MemSize; 1008 } else { 1009 // All other uses of addrmode6 are for intrinsics. For now just record 1010 // the raw alignment value; it will be refined later based on the legal 1011 // alignment operands for the intrinsic. 1012 Alignment = MemN->getAlignment(); 1013 } 1014 1015 Align = CurDAG->getTargetConstant(Alignment, MVT::i32); 1016 return true; 1017} 1018 1019bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N, 1020 SDValue &Offset) { 1021 LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op); 1022 ISD::MemIndexedMode AM = LdSt->getAddressingMode(); 1023 if (AM != ISD::POST_INC) 1024 return false; 1025 Offset = N; 1026 if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) { 1027 if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits()) 1028 Offset = CurDAG->getRegister(0, MVT::i32); 1029 } 1030 return true; 1031} 1032 1033bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N, 1034 SDValue &Offset, SDValue &Label) { 1035 if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) { 1036 Offset = N.getOperand(0); 1037 SDValue N1 = N.getOperand(1); 1038 Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(), 1039 MVT::i32); 1040 return true; 1041 } 1042 1043 return false; 1044} 1045 1046 1047//===----------------------------------------------------------------------===// 1048// Thumb Addressing Modes 1049//===----------------------------------------------------------------------===// 1050 1051bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N, 1052 SDValue &Base, SDValue &Offset){ 1053 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) { 1054 ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N); 1055 if (!NC || !NC->isNullValue()) 1056 return false; 1057 1058 Base = Offset = N; 1059 return true; 1060 } 1061 1062 Base = N.getOperand(0); 1063 Offset = N.getOperand(1); 1064 return true; 1065} 1066 1067bool 1068ARMDAGToDAGISel::SelectThumbAddrModeRI(SDValue N, SDValue &Base, 1069 SDValue &Offset, unsigned Scale) { 1070 if (Scale == 4) { 1071 SDValue TmpBase, TmpOffImm; 1072 if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm)) 1073 return false; // We want to select tLDRspi / tSTRspi instead. 1074 1075 if (N.getOpcode() == ARMISD::Wrapper && 1076 N.getOperand(0).getOpcode() == ISD::TargetConstantPool) 1077 return false; // We want to select tLDRpci instead. 1078 } 1079 1080 if (!CurDAG->isBaseWithConstantOffset(N)) 1081 return false; 1082 1083 // Thumb does not have [sp, r] address mode. 1084 RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0)); 1085 RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1)); 1086 if ((LHSR && LHSR->getReg() == ARM::SP) || 1087 (RHSR && RHSR->getReg() == ARM::SP)) 1088 return false; 1089 1090 // FIXME: Why do we explicitly check for a match here and then return false? 1091 // Presumably to allow something else to match, but shouldn't this be 1092 // documented? 1093 int RHSC; 1094 if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) 1095 return false; 1096 1097 Base = N.getOperand(0); 1098 Offset = N.getOperand(1); 1099 return true; 1100} 1101 1102bool 1103ARMDAGToDAGISel::SelectThumbAddrModeRI5S1(SDValue N, 1104 SDValue &Base, 1105 SDValue &Offset) { 1106 return SelectThumbAddrModeRI(N, Base, Offset, 1); 1107} 1108 1109bool 1110ARMDAGToDAGISel::SelectThumbAddrModeRI5S2(SDValue N, 1111 SDValue &Base, 1112 SDValue &Offset) { 1113 return SelectThumbAddrModeRI(N, Base, Offset, 2); 1114} 1115 1116bool 1117ARMDAGToDAGISel::SelectThumbAddrModeRI5S4(SDValue N, 1118 SDValue &Base, 1119 SDValue &Offset) { 1120 return SelectThumbAddrModeRI(N, Base, Offset, 4); 1121} 1122 1123bool 1124ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, 1125 SDValue &Base, SDValue &OffImm) { 1126 if (Scale == 4) { 1127 SDValue TmpBase, TmpOffImm; 1128 if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm)) 1129 return false; // We want to select tLDRspi / tSTRspi instead. 1130 1131 if (N.getOpcode() == ARMISD::Wrapper && 1132 N.getOperand(0).getOpcode() == ISD::TargetConstantPool) 1133 return false; // We want to select tLDRpci instead. 1134 } 1135 1136 if (!CurDAG->isBaseWithConstantOffset(N)) { 1137 if (N.getOpcode() == ARMISD::Wrapper && 1138 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) { 1139 Base = N.getOperand(0); 1140 } else { 1141 Base = N; 1142 } 1143 1144 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1145 return true; 1146 } 1147 1148 RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0)); 1149 RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1)); 1150 if ((LHSR && LHSR->getReg() == ARM::SP) || 1151 (RHSR && RHSR->getReg() == ARM::SP)) { 1152 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(N.getOperand(0)); 1153 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1154 unsigned LHSC = LHS ? LHS->getZExtValue() : 0; 1155 unsigned RHSC = RHS ? RHS->getZExtValue() : 0; 1156 1157 // Thumb does not have [sp, #imm5] address mode for non-zero imm5. 1158 if (LHSC != 0 || RHSC != 0) return false; 1159 1160 Base = N; 1161 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1162 return true; 1163 } 1164 1165 // If the RHS is + imm5 * scale, fold into addr mode. 1166 int RHSC; 1167 if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) { 1168 Base = N.getOperand(0); 1169 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 1170 return true; 1171 } 1172 1173 Base = N.getOperand(0); 1174 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1175 return true; 1176} 1177 1178bool 1179ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base, 1180 SDValue &OffImm) { 1181 return SelectThumbAddrModeImm5S(N, 4, Base, OffImm); 1182} 1183 1184bool 1185ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base, 1186 SDValue &OffImm) { 1187 return SelectThumbAddrModeImm5S(N, 2, Base, OffImm); 1188} 1189 1190bool 1191ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base, 1192 SDValue &OffImm) { 1193 return SelectThumbAddrModeImm5S(N, 1, Base, OffImm); 1194} 1195 1196bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N, 1197 SDValue &Base, SDValue &OffImm) { 1198 if (N.getOpcode() == ISD::FrameIndex) { 1199 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 1200 // Only multiples of 4 are allowed for the offset, so the frame object 1201 // alignment must be at least 4. 1202 MachineFrameInfo *MFI = MF->getFrameInfo(); 1203 if (MFI->getObjectAlignment(FI) < 4) 1204 MFI->setObjectAlignment(FI, 4); 1205 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); 1206 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1207 return true; 1208 } 1209 1210 if (!CurDAG->isBaseWithConstantOffset(N)) 1211 return false; 1212 1213 RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0)); 1214 if (N.getOperand(0).getOpcode() == ISD::FrameIndex || 1215 (LHSR && LHSR->getReg() == ARM::SP)) { 1216 // If the RHS is + imm8 * scale, fold into addr mode. 1217 int RHSC; 1218 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) { 1219 Base = N.getOperand(0); 1220 if (Base.getOpcode() == ISD::FrameIndex) { 1221 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1222 // For LHS+RHS to result in an offset that's a multiple of 4 the object 1223 // indexed by the LHS must be 4-byte aligned. 1224 MachineFrameInfo *MFI = MF->getFrameInfo(); 1225 if (MFI->getObjectAlignment(FI) < 4) 1226 MFI->setObjectAlignment(FI, 4); 1227 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); 1228 } 1229 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 1230 return true; 1231 } 1232 } 1233 1234 return false; 1235} 1236 1237 1238//===----------------------------------------------------------------------===// 1239// Thumb 2 Addressing Modes 1240//===----------------------------------------------------------------------===// 1241 1242 1243bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg, 1244 SDValue &Opc) { 1245 if (DisableShifterOp) 1246 return false; 1247 1248 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 1249 1250 // Don't match base register only case. That is matched to a separate 1251 // lower complexity pattern with explicit register operand. 1252 if (ShOpcVal == ARM_AM::no_shift) return false; 1253 1254 BaseReg = N.getOperand(0); 1255 unsigned ShImmVal = 0; 1256 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1257 ShImmVal = RHS->getZExtValue() & 31; 1258 Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal)); 1259 return true; 1260 } 1261 1262 return false; 1263} 1264 1265bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N, 1266 SDValue &Base, SDValue &OffImm) { 1267 // Match simple R + imm12 operands. 1268 1269 // Base only. 1270 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 1271 !CurDAG->isBaseWithConstantOffset(N)) { 1272 if (N.getOpcode() == ISD::FrameIndex) { 1273 // Match frame index. 1274 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 1275 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); 1276 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1277 return true; 1278 } 1279 1280 if (N.getOpcode() == ARMISD::Wrapper && 1281 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) { 1282 Base = N.getOperand(0); 1283 if (Base.getOpcode() == ISD::TargetConstantPool) 1284 return false; // We want to select t2LDRpci instead. 1285 } else 1286 Base = N; 1287 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1288 return true; 1289 } 1290 1291 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1292 if (SelectT2AddrModeImm8(N, Base, OffImm)) 1293 // Let t2LDRi8 handle (R - imm8). 1294 return false; 1295 1296 int RHSC = (int)RHS->getZExtValue(); 1297 if (N.getOpcode() == ISD::SUB) 1298 RHSC = -RHSC; 1299 1300 if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned) 1301 Base = N.getOperand(0); 1302 if (Base.getOpcode() == ISD::FrameIndex) { 1303 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1304 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); 1305 } 1306 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 1307 return true; 1308 } 1309 } 1310 1311 // Base only. 1312 Base = N; 1313 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1314 return true; 1315} 1316 1317bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N, 1318 SDValue &Base, SDValue &OffImm) { 1319 // Match simple R - imm8 operands. 1320 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 1321 !CurDAG->isBaseWithConstantOffset(N)) 1322 return false; 1323 1324 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1325 int RHSC = (int)RHS->getSExtValue(); 1326 if (N.getOpcode() == ISD::SUB) 1327 RHSC = -RHSC; 1328 1329 if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative) 1330 Base = N.getOperand(0); 1331 if (Base.getOpcode() == ISD::FrameIndex) { 1332 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1333 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); 1334 } 1335 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 1336 return true; 1337 } 1338 } 1339 1340 return false; 1341} 1342 1343bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N, 1344 SDValue &OffImm){ 1345 unsigned Opcode = Op->getOpcode(); 1346 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 1347 ? cast<LoadSDNode>(Op)->getAddressingMode() 1348 : cast<StoreSDNode>(Op)->getAddressingMode(); 1349 int RHSC; 1350 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits. 1351 OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC)) 1352 ? CurDAG->getTargetConstant(RHSC, MVT::i32) 1353 : CurDAG->getTargetConstant(-RHSC, MVT::i32); 1354 return true; 1355 } 1356 1357 return false; 1358} 1359 1360bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N, 1361 SDValue &Base, 1362 SDValue &OffReg, SDValue &ShImm) { 1363 // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12. 1364 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) 1365 return false; 1366 1367 // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8. 1368 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1369 int RHSC = (int)RHS->getZExtValue(); 1370 if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned) 1371 return false; 1372 else if (RHSC < 0 && RHSC >= -255) // 8 bits 1373 return false; 1374 } 1375 1376 // Look for (R + R) or (R + (R << [1,2,3])). 1377 unsigned ShAmt = 0; 1378 Base = N.getOperand(0); 1379 OffReg = N.getOperand(1); 1380 1381 // Swap if it is ((R << c) + R). 1382 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode()); 1383 if (ShOpcVal != ARM_AM::lsl) { 1384 ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode()); 1385 if (ShOpcVal == ARM_AM::lsl) 1386 std::swap(Base, OffReg); 1387 } 1388 1389 if (ShOpcVal == ARM_AM::lsl) { 1390 // Check to see if the RHS of the shift is a constant, if not, we can't fold 1391 // it. 1392 if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) { 1393 ShAmt = Sh->getZExtValue(); 1394 if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt)) 1395 OffReg = OffReg.getOperand(0); 1396 else { 1397 ShAmt = 0; 1398 } 1399 } 1400 } 1401 1402 ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32); 1403 1404 return true; 1405} 1406 1407bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base, 1408 SDValue &OffImm) { 1409 // This *must* succeed since it's used for the irreplaceable ldrex and strex 1410 // instructions. 1411 Base = N; 1412 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1413 1414 if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N)) 1415 return true; 1416 1417 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1418 if (!RHS) 1419 return true; 1420 1421 uint32_t RHSC = (int)RHS->getZExtValue(); 1422 if (RHSC > 1020 || RHSC % 4 != 0) 1423 return true; 1424 1425 Base = N.getOperand(0); 1426 if (Base.getOpcode() == ISD::FrameIndex) { 1427 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1428 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); 1429 } 1430 1431 OffImm = CurDAG->getTargetConstant(RHSC / 4, MVT::i32); 1432 return true; 1433} 1434 1435//===--------------------------------------------------------------------===// 1436 1437/// getAL - Returns a ARMCC::AL immediate node. 1438static inline SDValue getAL(SelectionDAG *CurDAG) { 1439 return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32); 1440} 1441 1442SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) { 1443 LoadSDNode *LD = cast<LoadSDNode>(N); 1444 ISD::MemIndexedMode AM = LD->getAddressingMode(); 1445 if (AM == ISD::UNINDEXED) 1446 return nullptr; 1447 1448 EVT LoadedVT = LD->getMemoryVT(); 1449 SDValue Offset, AMOpc; 1450 bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC); 1451 unsigned Opcode = 0; 1452 bool Match = false; 1453 if (LoadedVT == MVT::i32 && isPre && 1454 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) { 1455 Opcode = ARM::LDR_PRE_IMM; 1456 Match = true; 1457 } else if (LoadedVT == MVT::i32 && !isPre && 1458 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) { 1459 Opcode = ARM::LDR_POST_IMM; 1460 Match = true; 1461 } else if (LoadedVT == MVT::i32 && 1462 SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) { 1463 Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG; 1464 Match = true; 1465 1466 } else if (LoadedVT == MVT::i16 && 1467 SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { 1468 Match = true; 1469 Opcode = (LD->getExtensionType() == ISD::SEXTLOAD) 1470 ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST) 1471 : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST); 1472 } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) { 1473 if (LD->getExtensionType() == ISD::SEXTLOAD) { 1474 if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { 1475 Match = true; 1476 Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST; 1477 } 1478 } else { 1479 if (isPre && 1480 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) { 1481 Match = true; 1482 Opcode = ARM::LDRB_PRE_IMM; 1483 } else if (!isPre && 1484 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) { 1485 Match = true; 1486 Opcode = ARM::LDRB_POST_IMM; 1487 } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) { 1488 Match = true; 1489 Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG; 1490 } 1491 } 1492 } 1493 1494 if (Match) { 1495 if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) { 1496 SDValue Chain = LD->getChain(); 1497 SDValue Base = LD->getBasePtr(); 1498 SDValue Ops[]= { Base, AMOpc, getAL(CurDAG), 1499 CurDAG->getRegister(0, MVT::i32), Chain }; 1500 return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, 1501 MVT::i32, MVT::Other, Ops); 1502 } else { 1503 SDValue Chain = LD->getChain(); 1504 SDValue Base = LD->getBasePtr(); 1505 SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG), 1506 CurDAG->getRegister(0, MVT::i32), Chain }; 1507 return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, 1508 MVT::i32, MVT::Other, Ops); 1509 } 1510 } 1511 1512 return nullptr; 1513} 1514 1515SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) { 1516 LoadSDNode *LD = cast<LoadSDNode>(N); 1517 ISD::MemIndexedMode AM = LD->getAddressingMode(); 1518 if (AM == ISD::UNINDEXED) 1519 return nullptr; 1520 1521 EVT LoadedVT = LD->getMemoryVT(); 1522 bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD; 1523 SDValue Offset; 1524 bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC); 1525 unsigned Opcode = 0; 1526 bool Match = false; 1527 if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) { 1528 switch (LoadedVT.getSimpleVT().SimpleTy) { 1529 case MVT::i32: 1530 Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST; 1531 break; 1532 case MVT::i16: 1533 if (isSExtLd) 1534 Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST; 1535 else 1536 Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST; 1537 break; 1538 case MVT::i8: 1539 case MVT::i1: 1540 if (isSExtLd) 1541 Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST; 1542 else 1543 Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST; 1544 break; 1545 default: 1546 return nullptr; 1547 } 1548 Match = true; 1549 } 1550 1551 if (Match) { 1552 SDValue Chain = LD->getChain(); 1553 SDValue Base = LD->getBasePtr(); 1554 SDValue Ops[]= { Base, Offset, getAL(CurDAG), 1555 CurDAG->getRegister(0, MVT::i32), Chain }; 1556 return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32, 1557 MVT::Other, Ops); 1558 } 1559 1560 return nullptr; 1561} 1562 1563/// \brief Form a GPRPair pseudo register from a pair of GPR regs. 1564SDNode *ARMDAGToDAGISel::createGPRPairNode(EVT VT, SDValue V0, SDValue V1) { 1565 SDLoc dl(V0.getNode()); 1566 SDValue RegClass = 1567 CurDAG->getTargetConstant(ARM::GPRPairRegClassID, MVT::i32); 1568 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, MVT::i32); 1569 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, MVT::i32); 1570 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1571 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1572} 1573 1574/// \brief Form a D register from a pair of S registers. 1575SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) { 1576 SDLoc dl(V0.getNode()); 1577 SDValue RegClass = 1578 CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, MVT::i32); 1579 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32); 1580 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32); 1581 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1582 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1583} 1584 1585/// \brief Form a quad register from a pair of D registers. 1586SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) { 1587 SDLoc dl(V0.getNode()); 1588 SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, MVT::i32); 1589 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32); 1590 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32); 1591 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1592 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1593} 1594 1595/// \brief Form 4 consecutive D registers from a pair of Q registers. 1596SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) { 1597 SDLoc dl(V0.getNode()); 1598 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32); 1599 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32); 1600 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32); 1601 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1602 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1603} 1604 1605/// \brief Form 4 consecutive S registers. 1606SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, 1607 SDValue V2, SDValue V3) { 1608 SDLoc dl(V0.getNode()); 1609 SDValue RegClass = 1610 CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, MVT::i32); 1611 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32); 1612 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32); 1613 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32); 1614 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32); 1615 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, 1616 V2, SubReg2, V3, SubReg3 }; 1617 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1618} 1619 1620/// \brief Form 4 consecutive D registers. 1621SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, 1622 SDValue V2, SDValue V3) { 1623 SDLoc dl(V0.getNode()); 1624 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32); 1625 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32); 1626 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32); 1627 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32); 1628 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32); 1629 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, 1630 V2, SubReg2, V3, SubReg3 }; 1631 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1632} 1633 1634/// \brief Form 4 consecutive Q registers. 1635SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, 1636 SDValue V2, SDValue V3) { 1637 SDLoc dl(V0.getNode()); 1638 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, MVT::i32); 1639 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32); 1640 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32); 1641 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32); 1642 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32); 1643 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, 1644 V2, SubReg2, V3, SubReg3 }; 1645 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1646} 1647 1648/// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand 1649/// of a NEON VLD or VST instruction. The supported values depend on the 1650/// number of registers being loaded. 1651SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, unsigned NumVecs, 1652 bool is64BitVector) { 1653 unsigned NumRegs = NumVecs; 1654 if (!is64BitVector && NumVecs < 3) 1655 NumRegs *= 2; 1656 1657 unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); 1658 if (Alignment >= 32 && NumRegs == 4) 1659 Alignment = 32; 1660 else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4)) 1661 Alignment = 16; 1662 else if (Alignment >= 8) 1663 Alignment = 8; 1664 else 1665 Alignment = 0; 1666 1667 return CurDAG->getTargetConstant(Alignment, MVT::i32); 1668} 1669 1670static bool isVLDfixed(unsigned Opc) 1671{ 1672 switch (Opc) { 1673 default: return false; 1674 case ARM::VLD1d8wb_fixed : return true; 1675 case ARM::VLD1d16wb_fixed : return true; 1676 case ARM::VLD1d64Qwb_fixed : return true; 1677 case ARM::VLD1d32wb_fixed : return true; 1678 case ARM::VLD1d64wb_fixed : return true; 1679 case ARM::VLD1d64TPseudoWB_fixed : return true; 1680 case ARM::VLD1d64QPseudoWB_fixed : return true; 1681 case ARM::VLD1q8wb_fixed : return true; 1682 case ARM::VLD1q16wb_fixed : return true; 1683 case ARM::VLD1q32wb_fixed : return true; 1684 case ARM::VLD1q64wb_fixed : return true; 1685 case ARM::VLD2d8wb_fixed : return true; 1686 case ARM::VLD2d16wb_fixed : return true; 1687 case ARM::VLD2d32wb_fixed : return true; 1688 case ARM::VLD2q8PseudoWB_fixed : return true; 1689 case ARM::VLD2q16PseudoWB_fixed : return true; 1690 case ARM::VLD2q32PseudoWB_fixed : return true; 1691 case ARM::VLD2DUPd8wb_fixed : return true; 1692 case ARM::VLD2DUPd16wb_fixed : return true; 1693 case ARM::VLD2DUPd32wb_fixed : return true; 1694 } 1695} 1696 1697static bool isVSTfixed(unsigned Opc) 1698{ 1699 switch (Opc) { 1700 default: return false; 1701 case ARM::VST1d8wb_fixed : return true; 1702 case ARM::VST1d16wb_fixed : return true; 1703 case ARM::VST1d32wb_fixed : return true; 1704 case ARM::VST1d64wb_fixed : return true; 1705 case ARM::VST1q8wb_fixed : return true; 1706 case ARM::VST1q16wb_fixed : return true; 1707 case ARM::VST1q32wb_fixed : return true; 1708 case ARM::VST1q64wb_fixed : return true; 1709 case ARM::VST1d64TPseudoWB_fixed : return true; 1710 case ARM::VST1d64QPseudoWB_fixed : return true; 1711 case ARM::VST2d8wb_fixed : return true; 1712 case ARM::VST2d16wb_fixed : return true; 1713 case ARM::VST2d32wb_fixed : return true; 1714 case ARM::VST2q8PseudoWB_fixed : return true; 1715 case ARM::VST2q16PseudoWB_fixed : return true; 1716 case ARM::VST2q32PseudoWB_fixed : return true; 1717 } 1718} 1719 1720// Get the register stride update opcode of a VLD/VST instruction that 1721// is otherwise equivalent to the given fixed stride updating instruction. 1722static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) { 1723 assert((isVLDfixed(Opc) || isVSTfixed(Opc)) 1724 && "Incorrect fixed stride updating instruction."); 1725 switch (Opc) { 1726 default: break; 1727 case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register; 1728 case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register; 1729 case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register; 1730 case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register; 1731 case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register; 1732 case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register; 1733 case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register; 1734 case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register; 1735 case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register; 1736 case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register; 1737 case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register; 1738 case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register; 1739 1740 case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register; 1741 case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register; 1742 case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register; 1743 case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register; 1744 case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register; 1745 case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register; 1746 case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register; 1747 case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register; 1748 case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register; 1749 case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register; 1750 1751 case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register; 1752 case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register; 1753 case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register; 1754 case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register; 1755 case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register; 1756 case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register; 1757 1758 case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register; 1759 case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register; 1760 case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register; 1761 case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register; 1762 case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register; 1763 case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register; 1764 1765 case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register; 1766 case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register; 1767 case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register; 1768 } 1769 return Opc; // If not one we handle, return it unchanged. 1770} 1771 1772SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs, 1773 const uint16_t *DOpcodes, 1774 const uint16_t *QOpcodes0, 1775 const uint16_t *QOpcodes1) { 1776 assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range"); 1777 SDLoc dl(N); 1778 1779 SDValue MemAddr, Align; 1780 unsigned AddrOpIdx = isUpdating ? 1 : 2; 1781 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) 1782 return nullptr; 1783 1784 SDValue Chain = N->getOperand(0); 1785 EVT VT = N->getValueType(0); 1786 bool is64BitVector = VT.is64BitVector(); 1787 Align = GetVLDSTAlign(Align, NumVecs, is64BitVector); 1788 1789 unsigned OpcodeIndex; 1790 switch (VT.getSimpleVT().SimpleTy) { 1791 default: llvm_unreachable("unhandled vld type"); 1792 // Double-register operations: 1793 case MVT::v8i8: OpcodeIndex = 0; break; 1794 case MVT::v4i16: OpcodeIndex = 1; break; 1795 case MVT::v2f32: 1796 case MVT::v2i32: OpcodeIndex = 2; break; 1797 case MVT::v1i64: OpcodeIndex = 3; break; 1798 // Quad-register operations: 1799 case MVT::v16i8: OpcodeIndex = 0; break; 1800 case MVT::v8i16: OpcodeIndex = 1; break; 1801 case MVT::v4f32: 1802 case MVT::v4i32: OpcodeIndex = 2; break; 1803 case MVT::v2f64: 1804 case MVT::v2i64: OpcodeIndex = 3; 1805 assert(NumVecs == 1 && "v2i64 type only supported for VLD1"); 1806 break; 1807 } 1808 1809 EVT ResTy; 1810 if (NumVecs == 1) 1811 ResTy = VT; 1812 else { 1813 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; 1814 if (!is64BitVector) 1815 ResTyElts *= 2; 1816 ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts); 1817 } 1818 std::vector<EVT> ResTys; 1819 ResTys.push_back(ResTy); 1820 if (isUpdating) 1821 ResTys.push_back(MVT::i32); 1822 ResTys.push_back(MVT::Other); 1823 1824 SDValue Pred = getAL(CurDAG); 1825 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 1826 SDNode *VLd; 1827 SmallVector<SDValue, 7> Ops; 1828 1829 // Double registers and VLD1/VLD2 quad registers are directly supported. 1830 if (is64BitVector || NumVecs <= 2) { 1831 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : 1832 QOpcodes0[OpcodeIndex]); 1833 Ops.push_back(MemAddr); 1834 Ops.push_back(Align); 1835 if (isUpdating) { 1836 SDValue Inc = N->getOperand(AddrOpIdx + 1); 1837 // FIXME: VLD1/VLD2 fixed increment doesn't need Reg0. Remove the reg0 1838 // case entirely when the rest are updated to that form, too. 1839 if ((NumVecs <= 2) && !isa<ConstantSDNode>(Inc.getNode())) 1840 Opc = getVLDSTRegisterUpdateOpcode(Opc); 1841 // FIXME: We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so 1842 // check for that explicitly too. Horribly hacky, but temporary. 1843 if ((NumVecs > 2 && !isVLDfixed(Opc)) || 1844 !isa<ConstantSDNode>(Inc.getNode())) 1845 Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc); 1846 } 1847 Ops.push_back(Pred); 1848 Ops.push_back(Reg0); 1849 Ops.push_back(Chain); 1850 VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 1851 1852 } else { 1853 // Otherwise, quad registers are loaded with two separate instructions, 1854 // where one loads the even registers and the other loads the odd registers. 1855 EVT AddrTy = MemAddr.getValueType(); 1856 1857 // Load the even subregs. This is always an updating load, so that it 1858 // provides the address to the second load for the odd subregs. 1859 SDValue ImplDef = 1860 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0); 1861 const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain }; 1862 SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, 1863 ResTy, AddrTy, MVT::Other, OpsA); 1864 Chain = SDValue(VLdA, 2); 1865 1866 // Load the odd subregs. 1867 Ops.push_back(SDValue(VLdA, 1)); 1868 Ops.push_back(Align); 1869 if (isUpdating) { 1870 SDValue Inc = N->getOperand(AddrOpIdx + 1); 1871 assert(isa<ConstantSDNode>(Inc.getNode()) && 1872 "only constant post-increment update allowed for VLD3/4"); 1873 (void)Inc; 1874 Ops.push_back(Reg0); 1875 } 1876 Ops.push_back(SDValue(VLdA, 0)); 1877 Ops.push_back(Pred); 1878 Ops.push_back(Reg0); 1879 Ops.push_back(Chain); 1880 VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops); 1881 } 1882 1883 // Transfer memoperands. 1884 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1885 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 1886 cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1); 1887 1888 if (NumVecs == 1) 1889 return VLd; 1890 1891 // Extract out the subregisters. 1892 SDValue SuperReg = SDValue(VLd, 0); 1893 assert(ARM::dsub_7 == ARM::dsub_0+7 && 1894 ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering"); 1895 unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0); 1896 for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 1897 ReplaceUses(SDValue(N, Vec), 1898 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg)); 1899 ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1)); 1900 if (isUpdating) 1901 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2)); 1902 return nullptr; 1903} 1904 1905SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, 1906 const uint16_t *DOpcodes, 1907 const uint16_t *QOpcodes0, 1908 const uint16_t *QOpcodes1) { 1909 assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range"); 1910 SDLoc dl(N); 1911 1912 SDValue MemAddr, Align; 1913 unsigned AddrOpIdx = isUpdating ? 1 : 2; 1914 unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1) 1915 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) 1916 return nullptr; 1917 1918 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1919 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 1920 1921 SDValue Chain = N->getOperand(0); 1922 EVT VT = N->getOperand(Vec0Idx).getValueType(); 1923 bool is64BitVector = VT.is64BitVector(); 1924 Align = GetVLDSTAlign(Align, NumVecs, is64BitVector); 1925 1926 unsigned OpcodeIndex; 1927 switch (VT.getSimpleVT().SimpleTy) { 1928 default: llvm_unreachable("unhandled vst type"); 1929 // Double-register operations: 1930 case MVT::v8i8: OpcodeIndex = 0; break; 1931 case MVT::v4i16: OpcodeIndex = 1; break; 1932 case MVT::v2f32: 1933 case MVT::v2i32: OpcodeIndex = 2; break; 1934 case MVT::v1i64: OpcodeIndex = 3; break; 1935 // Quad-register operations: 1936 case MVT::v16i8: OpcodeIndex = 0; break; 1937 case MVT::v8i16: OpcodeIndex = 1; break; 1938 case MVT::v4f32: 1939 case MVT::v4i32: OpcodeIndex = 2; break; 1940 case MVT::v2f64: 1941 case MVT::v2i64: OpcodeIndex = 3; 1942 assert(NumVecs == 1 && "v2i64 type only supported for VST1"); 1943 break; 1944 } 1945 1946 std::vector<EVT> ResTys; 1947 if (isUpdating) 1948 ResTys.push_back(MVT::i32); 1949 ResTys.push_back(MVT::Other); 1950 1951 SDValue Pred = getAL(CurDAG); 1952 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 1953 SmallVector<SDValue, 7> Ops; 1954 1955 // Double registers and VST1/VST2 quad registers are directly supported. 1956 if (is64BitVector || NumVecs <= 2) { 1957 SDValue SrcReg; 1958 if (NumVecs == 1) { 1959 SrcReg = N->getOperand(Vec0Idx); 1960 } else if (is64BitVector) { 1961 // Form a REG_SEQUENCE to force register allocation. 1962 SDValue V0 = N->getOperand(Vec0Idx + 0); 1963 SDValue V1 = N->getOperand(Vec0Idx + 1); 1964 if (NumVecs == 2) 1965 SrcReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0); 1966 else { 1967 SDValue V2 = N->getOperand(Vec0Idx + 2); 1968 // If it's a vst3, form a quad D-register and leave the last part as 1969 // an undef. 1970 SDValue V3 = (NumVecs == 3) 1971 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0) 1972 : N->getOperand(Vec0Idx + 3); 1973 SrcReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0); 1974 } 1975 } else { 1976 // Form a QQ register. 1977 SDValue Q0 = N->getOperand(Vec0Idx); 1978 SDValue Q1 = N->getOperand(Vec0Idx + 1); 1979 SrcReg = SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0); 1980 } 1981 1982 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : 1983 QOpcodes0[OpcodeIndex]); 1984 Ops.push_back(MemAddr); 1985 Ops.push_back(Align); 1986 if (isUpdating) { 1987 SDValue Inc = N->getOperand(AddrOpIdx + 1); 1988 // FIXME: VST1/VST2 fixed increment doesn't need Reg0. Remove the reg0 1989 // case entirely when the rest are updated to that form, too. 1990 if (NumVecs <= 2 && !isa<ConstantSDNode>(Inc.getNode())) 1991 Opc = getVLDSTRegisterUpdateOpcode(Opc); 1992 // FIXME: We use a VST1 for v1i64 even if the pseudo says vld2/3/4, so 1993 // check for that explicitly too. Horribly hacky, but temporary. 1994 if (!isa<ConstantSDNode>(Inc.getNode())) 1995 Ops.push_back(Inc); 1996 else if (NumVecs > 2 && !isVSTfixed(Opc)) 1997 Ops.push_back(Reg0); 1998 } 1999 Ops.push_back(SrcReg); 2000 Ops.push_back(Pred); 2001 Ops.push_back(Reg0); 2002 Ops.push_back(Chain); 2003 SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 2004 2005 // Transfer memoperands. 2006 cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1); 2007 2008 return VSt; 2009 } 2010 2011 // Otherwise, quad registers are stored with two separate instructions, 2012 // where one stores the even registers and the other stores the odd registers. 2013 2014 // Form the QQQQ REG_SEQUENCE. 2015 SDValue V0 = N->getOperand(Vec0Idx + 0); 2016 SDValue V1 = N->getOperand(Vec0Idx + 1); 2017 SDValue V2 = N->getOperand(Vec0Idx + 2); 2018 SDValue V3 = (NumVecs == 3) 2019 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) 2020 : N->getOperand(Vec0Idx + 3); 2021 SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0); 2022 2023 // Store the even D registers. This is always an updating store, so that it 2024 // provides the address to the second store for the odd subregs. 2025 const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain }; 2026 SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, 2027 MemAddr.getValueType(), 2028 MVT::Other, OpsA); 2029 cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1); 2030 Chain = SDValue(VStA, 1); 2031 2032 // Store the odd D registers. 2033 Ops.push_back(SDValue(VStA, 0)); 2034 Ops.push_back(Align); 2035 if (isUpdating) { 2036 SDValue Inc = N->getOperand(AddrOpIdx + 1); 2037 assert(isa<ConstantSDNode>(Inc.getNode()) && 2038 "only constant post-increment update allowed for VST3/4"); 2039 (void)Inc; 2040 Ops.push_back(Reg0); 2041 } 2042 Ops.push_back(RegSeq); 2043 Ops.push_back(Pred); 2044 Ops.push_back(Reg0); 2045 Ops.push_back(Chain); 2046 SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, 2047 Ops); 2048 cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1); 2049 return VStB; 2050} 2051 2052SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, 2053 bool isUpdating, unsigned NumVecs, 2054 const uint16_t *DOpcodes, 2055 const uint16_t *QOpcodes) { 2056 assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range"); 2057 SDLoc dl(N); 2058 2059 SDValue MemAddr, Align; 2060 unsigned AddrOpIdx = isUpdating ? 1 : 2; 2061 unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1) 2062 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) 2063 return nullptr; 2064 2065 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 2066 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 2067 2068 SDValue Chain = N->getOperand(0); 2069 unsigned Lane = 2070 cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue(); 2071 EVT VT = N->getOperand(Vec0Idx).getValueType(); 2072 bool is64BitVector = VT.is64BitVector(); 2073 2074 unsigned Alignment = 0; 2075 if (NumVecs != 3) { 2076 Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); 2077 unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8; 2078 if (Alignment > NumBytes) 2079 Alignment = NumBytes; 2080 if (Alignment < 8 && Alignment < NumBytes) 2081 Alignment = 0; 2082 // Alignment must be a power of two; make sure of that. 2083 Alignment = (Alignment & -Alignment); 2084 if (Alignment == 1) 2085 Alignment = 0; 2086 } 2087 Align = CurDAG->getTargetConstant(Alignment, MVT::i32); 2088 2089 unsigned OpcodeIndex; 2090 switch (VT.getSimpleVT().SimpleTy) { 2091 default: llvm_unreachable("unhandled vld/vst lane type"); 2092 // Double-register operations: 2093 case MVT::v8i8: OpcodeIndex = 0; break; 2094 case MVT::v4i16: OpcodeIndex = 1; break; 2095 case MVT::v2f32: 2096 case MVT::v2i32: OpcodeIndex = 2; break; 2097 // Quad-register operations: 2098 case MVT::v8i16: OpcodeIndex = 0; break; 2099 case MVT::v4f32: 2100 case MVT::v4i32: OpcodeIndex = 1; break; 2101 } 2102 2103 std::vector<EVT> ResTys; 2104 if (IsLoad) { 2105 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; 2106 if (!is64BitVector) 2107 ResTyElts *= 2; 2108 ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), 2109 MVT::i64, ResTyElts)); 2110 } 2111 if (isUpdating) 2112 ResTys.push_back(MVT::i32); 2113 ResTys.push_back(MVT::Other); 2114 2115 SDValue Pred = getAL(CurDAG); 2116 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2117 2118 SmallVector<SDValue, 8> Ops; 2119 Ops.push_back(MemAddr); 2120 Ops.push_back(Align); 2121 if (isUpdating) { 2122 SDValue Inc = N->getOperand(AddrOpIdx + 1); 2123 Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc); 2124 } 2125 2126 SDValue SuperReg; 2127 SDValue V0 = N->getOperand(Vec0Idx + 0); 2128 SDValue V1 = N->getOperand(Vec0Idx + 1); 2129 if (NumVecs == 2) { 2130 if (is64BitVector) 2131 SuperReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0); 2132 else 2133 SuperReg = SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0); 2134 } else { 2135 SDValue V2 = N->getOperand(Vec0Idx + 2); 2136 SDValue V3 = (NumVecs == 3) 2137 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) 2138 : N->getOperand(Vec0Idx + 3); 2139 if (is64BitVector) 2140 SuperReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0); 2141 else 2142 SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0); 2143 } 2144 Ops.push_back(SuperReg); 2145 Ops.push_back(getI32Imm(Lane)); 2146 Ops.push_back(Pred); 2147 Ops.push_back(Reg0); 2148 Ops.push_back(Chain); 2149 2150 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : 2151 QOpcodes[OpcodeIndex]); 2152 SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 2153 cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1); 2154 if (!IsLoad) 2155 return VLdLn; 2156 2157 // Extract the subregisters. 2158 SuperReg = SDValue(VLdLn, 0); 2159 assert(ARM::dsub_7 == ARM::dsub_0+7 && 2160 ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering"); 2161 unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0; 2162 for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 2163 ReplaceUses(SDValue(N, Vec), 2164 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg)); 2165 ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1)); 2166 if (isUpdating) 2167 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2)); 2168 return nullptr; 2169} 2170 2171SDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating, 2172 unsigned NumVecs, 2173 const uint16_t *Opcodes) { 2174 assert(NumVecs >=2 && NumVecs <= 4 && "VLDDup NumVecs out-of-range"); 2175 SDLoc dl(N); 2176 2177 SDValue MemAddr, Align; 2178 if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align)) 2179 return nullptr; 2180 2181 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 2182 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 2183 2184 SDValue Chain = N->getOperand(0); 2185 EVT VT = N->getValueType(0); 2186 2187 unsigned Alignment = 0; 2188 if (NumVecs != 3) { 2189 Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); 2190 unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8; 2191 if (Alignment > NumBytes) 2192 Alignment = NumBytes; 2193 if (Alignment < 8 && Alignment < NumBytes) 2194 Alignment = 0; 2195 // Alignment must be a power of two; make sure of that. 2196 Alignment = (Alignment & -Alignment); 2197 if (Alignment == 1) 2198 Alignment = 0; 2199 } 2200 Align = CurDAG->getTargetConstant(Alignment, MVT::i32); 2201 2202 unsigned OpcodeIndex; 2203 switch (VT.getSimpleVT().SimpleTy) { 2204 default: llvm_unreachable("unhandled vld-dup type"); 2205 case MVT::v8i8: OpcodeIndex = 0; break; 2206 case MVT::v4i16: OpcodeIndex = 1; break; 2207 case MVT::v2f32: 2208 case MVT::v2i32: OpcodeIndex = 2; break; 2209 } 2210 2211 SDValue Pred = getAL(CurDAG); 2212 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2213 SDValue SuperReg; 2214 unsigned Opc = Opcodes[OpcodeIndex]; 2215 SmallVector<SDValue, 6> Ops; 2216 Ops.push_back(MemAddr); 2217 Ops.push_back(Align); 2218 if (isUpdating) { 2219 // fixed-stride update instructions don't have an explicit writeback 2220 // operand. It's implicit in the opcode itself. 2221 SDValue Inc = N->getOperand(2); 2222 if (!isa<ConstantSDNode>(Inc.getNode())) 2223 Ops.push_back(Inc); 2224 // FIXME: VLD3 and VLD4 haven't been updated to that form yet. 2225 else if (NumVecs > 2) 2226 Ops.push_back(Reg0); 2227 } 2228 Ops.push_back(Pred); 2229 Ops.push_back(Reg0); 2230 Ops.push_back(Chain); 2231 2232 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; 2233 std::vector<EVT> ResTys; 2234 ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts)); 2235 if (isUpdating) 2236 ResTys.push_back(MVT::i32); 2237 ResTys.push_back(MVT::Other); 2238 SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 2239 cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1); 2240 SuperReg = SDValue(VLdDup, 0); 2241 2242 // Extract the subregisters. 2243 assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering"); 2244 unsigned SubIdx = ARM::dsub_0; 2245 for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 2246 ReplaceUses(SDValue(N, Vec), 2247 CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg)); 2248 ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1)); 2249 if (isUpdating) 2250 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2)); 2251 return nullptr; 2252} 2253 2254SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, 2255 unsigned Opc) { 2256 assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range"); 2257 SDLoc dl(N); 2258 EVT VT = N->getValueType(0); 2259 unsigned FirstTblReg = IsExt ? 2 : 1; 2260 2261 // Form a REG_SEQUENCE to force register allocation. 2262 SDValue RegSeq; 2263 SDValue V0 = N->getOperand(FirstTblReg + 0); 2264 SDValue V1 = N->getOperand(FirstTblReg + 1); 2265 if (NumVecs == 2) 2266 RegSeq = SDValue(createDRegPairNode(MVT::v16i8, V0, V1), 0); 2267 else { 2268 SDValue V2 = N->getOperand(FirstTblReg + 2); 2269 // If it's a vtbl3, form a quad D-register and leave the last part as 2270 // an undef. 2271 SDValue V3 = (NumVecs == 3) 2272 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) 2273 : N->getOperand(FirstTblReg + 3); 2274 RegSeq = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0); 2275 } 2276 2277 SmallVector<SDValue, 6> Ops; 2278 if (IsExt) 2279 Ops.push_back(N->getOperand(1)); 2280 Ops.push_back(RegSeq); 2281 Ops.push_back(N->getOperand(FirstTblReg + NumVecs)); 2282 Ops.push_back(getAL(CurDAG)); // predicate 2283 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register 2284 return CurDAG->getMachineNode(Opc, dl, VT, Ops); 2285} 2286 2287SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N, 2288 bool isSigned) { 2289 if (!Subtarget->hasV6T2Ops()) 2290 return nullptr; 2291 2292 unsigned Opc = isSigned 2293 ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX) 2294 : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX); 2295 2296 // For unsigned extracts, check for a shift right and mask 2297 unsigned And_imm = 0; 2298 if (N->getOpcode() == ISD::AND) { 2299 if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) { 2300 2301 // The immediate is a mask of the low bits iff imm & (imm+1) == 0 2302 if (And_imm & (And_imm + 1)) 2303 return nullptr; 2304 2305 unsigned Srl_imm = 0; 2306 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, 2307 Srl_imm)) { 2308 assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!"); 2309 2310 // Note: The width operand is encoded as width-1. 2311 unsigned Width = countTrailingOnes(And_imm) - 1; 2312 unsigned LSB = Srl_imm; 2313 2314 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2315 2316 if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) { 2317 // It's cheaper to use a right shift to extract the top bits. 2318 if (Subtarget->isThumb()) { 2319 Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri; 2320 SDValue Ops[] = { N->getOperand(0).getOperand(0), 2321 CurDAG->getTargetConstant(LSB, MVT::i32), 2322 getAL(CurDAG), Reg0, Reg0 }; 2323 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); 2324 } 2325 2326 // ARM models shift instructions as MOVsi with shifter operand. 2327 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(ISD::SRL); 2328 SDValue ShOpc = 2329 CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB), 2330 MVT::i32); 2331 SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc, 2332 getAL(CurDAG), Reg0, Reg0 }; 2333 return CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops); 2334 } 2335 2336 SDValue Ops[] = { N->getOperand(0).getOperand(0), 2337 CurDAG->getTargetConstant(LSB, MVT::i32), 2338 CurDAG->getTargetConstant(Width, MVT::i32), 2339 getAL(CurDAG), Reg0 }; 2340 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); 2341 } 2342 } 2343 return nullptr; 2344 } 2345 2346 // Otherwise, we're looking for a shift of a shift 2347 unsigned Shl_imm = 0; 2348 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) { 2349 assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!"); 2350 unsigned Srl_imm = 0; 2351 if (isInt32Immediate(N->getOperand(1), Srl_imm)) { 2352 assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!"); 2353 // Note: The width operand is encoded as width-1. 2354 unsigned Width = 32 - Srl_imm - 1; 2355 int LSB = Srl_imm - Shl_imm; 2356 if (LSB < 0) 2357 return nullptr; 2358 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2359 SDValue Ops[] = { N->getOperand(0).getOperand(0), 2360 CurDAG->getTargetConstant(LSB, MVT::i32), 2361 CurDAG->getTargetConstant(Width, MVT::i32), 2362 getAL(CurDAG), Reg0 }; 2363 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); 2364 } 2365 } 2366 2367 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) { 2368 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits(); 2369 unsigned LSB = 0; 2370 if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, LSB) && 2371 !isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRA, LSB)) 2372 return nullptr; 2373 2374 if (LSB + Width > 32) 2375 return nullptr; 2376 2377 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2378 SDValue Ops[] = { N->getOperand(0).getOperand(0), 2379 CurDAG->getTargetConstant(LSB, MVT::i32), 2380 CurDAG->getTargetConstant(Width - 1, MVT::i32), 2381 getAL(CurDAG), Reg0 }; 2382 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); 2383 } 2384 2385 return nullptr; 2386} 2387 2388/// Target-specific DAG combining for ISD::XOR. 2389/// Target-independent combining lowers SELECT_CC nodes of the form 2390/// select_cc setg[ge] X, 0, X, -X 2391/// select_cc setgt X, -1, X, -X 2392/// select_cc setl[te] X, 0, -X, X 2393/// select_cc setlt X, 1, -X, X 2394/// which represent Integer ABS into: 2395/// Y = sra (X, size(X)-1); xor (add (X, Y), Y) 2396/// ARM instruction selection detects the latter and matches it to 2397/// ARM::ABS or ARM::t2ABS machine node. 2398SDNode *ARMDAGToDAGISel::SelectABSOp(SDNode *N){ 2399 SDValue XORSrc0 = N->getOperand(0); 2400 SDValue XORSrc1 = N->getOperand(1); 2401 EVT VT = N->getValueType(0); 2402 2403 if (Subtarget->isThumb1Only()) 2404 return nullptr; 2405 2406 if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA) 2407 return nullptr; 2408 2409 SDValue ADDSrc0 = XORSrc0.getOperand(0); 2410 SDValue ADDSrc1 = XORSrc0.getOperand(1); 2411 SDValue SRASrc0 = XORSrc1.getOperand(0); 2412 SDValue SRASrc1 = XORSrc1.getOperand(1); 2413 ConstantSDNode *SRAConstant = dyn_cast<ConstantSDNode>(SRASrc1); 2414 EVT XType = SRASrc0.getValueType(); 2415 unsigned Size = XType.getSizeInBits() - 1; 2416 2417 if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 && 2418 XType.isInteger() && SRAConstant != nullptr && 2419 Size == SRAConstant->getZExtValue()) { 2420 unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS; 2421 return CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0); 2422 } 2423 2424 return nullptr; 2425} 2426 2427SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) { 2428 // The only time a CONCAT_VECTORS operation can have legal types is when 2429 // two 64-bit vectors are concatenated to a 128-bit vector. 2430 EVT VT = N->getValueType(0); 2431 if (!VT.is128BitVector() || N->getNumOperands() != 2) 2432 llvm_unreachable("unexpected CONCAT_VECTORS"); 2433 return createDRegPairNode(VT, N->getOperand(0), N->getOperand(1)); 2434} 2435 2436SDNode *ARMDAGToDAGISel::Select(SDNode *N) { 2437 SDLoc dl(N); 2438 2439 if (N->isMachineOpcode()) { 2440 N->setNodeId(-1); 2441 return nullptr; // Already selected. 2442 } 2443 2444 switch (N->getOpcode()) { 2445 default: break; 2446 case ISD::INLINEASM: { 2447 SDNode *ResNode = SelectInlineAsm(N); 2448 if (ResNode) 2449 return ResNode; 2450 break; 2451 } 2452 case ISD::XOR: { 2453 // Select special operations if XOR node forms integer ABS pattern 2454 SDNode *ResNode = SelectABSOp(N); 2455 if (ResNode) 2456 return ResNode; 2457 // Other cases are autogenerated. 2458 break; 2459 } 2460 case ISD::Constant: { 2461 unsigned Val = cast<ConstantSDNode>(N)->getZExtValue(); 2462 bool UseCP = true; 2463 if (Subtarget->useMovt(*MF)) 2464 // Thumb2-aware targets have the MOVT instruction, so all immediates can 2465 // be done with MOV + MOVT, at worst. 2466 UseCP = false; 2467 else { 2468 if (Subtarget->isThumb()) { 2469 UseCP = (Val > 255 && // MOV 2470 ~Val > 255 && // MOV + MVN 2471 !ARM_AM::isThumbImmShiftedVal(Val) && // MOV + LSL 2472 !(Subtarget->hasV6T2Ops() && Val <= 0xffff)); // MOVW 2473 } else 2474 UseCP = (ARM_AM::getSOImmVal(Val) == -1 && // MOV 2475 ARM_AM::getSOImmVal(~Val) == -1 && // MVN 2476 !ARM_AM::isSOImmTwoPartVal(Val) && // two instrs. 2477 !(Subtarget->hasV6T2Ops() && Val <= 0xffff)); // MOVW 2478 } 2479 2480 if (UseCP) { 2481 SDValue CPIdx = CurDAG->getTargetConstantPool( 2482 ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), Val), 2483 TLI->getPointerTy()); 2484 2485 SDNode *ResNode; 2486 if (Subtarget->isThumb()) { 2487 SDValue Pred = getAL(CurDAG); 2488 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2489 SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() }; 2490 ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other, 2491 Ops); 2492 } else { 2493 SDValue Ops[] = { 2494 CPIdx, 2495 CurDAG->getTargetConstant(0, MVT::i32), 2496 getAL(CurDAG), 2497 CurDAG->getRegister(0, MVT::i32), 2498 CurDAG->getEntryNode() 2499 }; 2500 ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other, 2501 Ops); 2502 } 2503 ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0)); 2504 return nullptr; 2505 } 2506 2507 // Other cases are autogenerated. 2508 break; 2509 } 2510 case ISD::FrameIndex: { 2511 // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm. 2512 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 2513 SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy()); 2514 if (Subtarget->isThumb1Only()) { 2515 // Set the alignment of the frame object to 4, to avoid having to generate 2516 // more than one ADD 2517 MachineFrameInfo *MFI = MF->getFrameInfo(); 2518 if (MFI->getObjectAlignment(FI) < 4) 2519 MFI->setObjectAlignment(FI, 4); 2520 return CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI, 2521 CurDAG->getTargetConstant(0, MVT::i32)); 2522 } else { 2523 unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ? 2524 ARM::t2ADDri : ARM::ADDri); 2525 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32), 2526 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), 2527 CurDAG->getRegister(0, MVT::i32) }; 2528 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); 2529 } 2530 } 2531 case ISD::SRL: 2532 if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false)) 2533 return I; 2534 break; 2535 case ISD::SIGN_EXTEND_INREG: 2536 case ISD::SRA: 2537 if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true)) 2538 return I; 2539 break; 2540 case ISD::MUL: 2541 if (Subtarget->isThumb1Only()) 2542 break; 2543 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 2544 unsigned RHSV = C->getZExtValue(); 2545 if (!RHSV) break; 2546 if (isPowerOf2_32(RHSV-1)) { // 2^n+1? 2547 unsigned ShImm = Log2_32(RHSV-1); 2548 if (ShImm >= 32) 2549 break; 2550 SDValue V = N->getOperand(0); 2551 ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm); 2552 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32); 2553 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2554 if (Subtarget->isThumb()) { 2555 SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; 2556 return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops); 2557 } else { 2558 SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; 2559 return CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops); 2560 } 2561 } 2562 if (isPowerOf2_32(RHSV+1)) { // 2^n-1? 2563 unsigned ShImm = Log2_32(RHSV+1); 2564 if (ShImm >= 32) 2565 break; 2566 SDValue V = N->getOperand(0); 2567 ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm); 2568 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32); 2569 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2570 if (Subtarget->isThumb()) { 2571 SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; 2572 return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops); 2573 } else { 2574 SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; 2575 return CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops); 2576 } 2577 } 2578 } 2579 break; 2580 case ISD::AND: { 2581 // Check for unsigned bitfield extract 2582 if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false)) 2583 return I; 2584 2585 // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits 2586 // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits 2587 // are entirely contributed by c2 and lower 16-bits are entirely contributed 2588 // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)). 2589 // Select it to: "movt x, ((c1 & 0xffff) >> 16) 2590 EVT VT = N->getValueType(0); 2591 if (VT != MVT::i32) 2592 break; 2593 unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2()) 2594 ? ARM::t2MOVTi16 2595 : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0); 2596 if (!Opc) 2597 break; 2598 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); 2599 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 2600 if (!N1C) 2601 break; 2602 if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) { 2603 SDValue N2 = N0.getOperand(1); 2604 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2); 2605 if (!N2C) 2606 break; 2607 unsigned N1CVal = N1C->getZExtValue(); 2608 unsigned N2CVal = N2C->getZExtValue(); 2609 if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) && 2610 (N1CVal & 0xffffU) == 0xffffU && 2611 (N2CVal & 0xffffU) == 0x0U) { 2612 SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16, 2613 MVT::i32); 2614 SDValue Ops[] = { N0.getOperand(0), Imm16, 2615 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; 2616 return CurDAG->getMachineNode(Opc, dl, VT, Ops); 2617 } 2618 } 2619 break; 2620 } 2621 case ARMISD::VMOVRRD: 2622 return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32, 2623 N->getOperand(0), getAL(CurDAG), 2624 CurDAG->getRegister(0, MVT::i32)); 2625 case ISD::UMUL_LOHI: { 2626 if (Subtarget->isThumb1Only()) 2627 break; 2628 if (Subtarget->isThumb()) { 2629 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2630 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; 2631 return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops); 2632 } else { 2633 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2634 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), 2635 CurDAG->getRegister(0, MVT::i32) }; 2636 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2637 ARM::UMULL : ARM::UMULLv5, 2638 dl, MVT::i32, MVT::i32, Ops); 2639 } 2640 } 2641 case ISD::SMUL_LOHI: { 2642 if (Subtarget->isThumb1Only()) 2643 break; 2644 if (Subtarget->isThumb()) { 2645 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2646 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; 2647 return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops); 2648 } else { 2649 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2650 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), 2651 CurDAG->getRegister(0, MVT::i32) }; 2652 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2653 ARM::SMULL : ARM::SMULLv5, 2654 dl, MVT::i32, MVT::i32, Ops); 2655 } 2656 } 2657 case ARMISD::UMLAL:{ 2658 if (Subtarget->isThumb()) { 2659 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 2660 N->getOperand(3), getAL(CurDAG), 2661 CurDAG->getRegister(0, MVT::i32)}; 2662 return CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops); 2663 }else{ 2664 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 2665 N->getOperand(3), getAL(CurDAG), 2666 CurDAG->getRegister(0, MVT::i32), 2667 CurDAG->getRegister(0, MVT::i32) }; 2668 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2669 ARM::UMLAL : ARM::UMLALv5, 2670 dl, MVT::i32, MVT::i32, Ops); 2671 } 2672 } 2673 case ARMISD::SMLAL:{ 2674 if (Subtarget->isThumb()) { 2675 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 2676 N->getOperand(3), getAL(CurDAG), 2677 CurDAG->getRegister(0, MVT::i32)}; 2678 return CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops); 2679 }else{ 2680 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 2681 N->getOperand(3), getAL(CurDAG), 2682 CurDAG->getRegister(0, MVT::i32), 2683 CurDAG->getRegister(0, MVT::i32) }; 2684 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2685 ARM::SMLAL : ARM::SMLALv5, 2686 dl, MVT::i32, MVT::i32, Ops); 2687 } 2688 } 2689 case ISD::LOAD: { 2690 SDNode *ResNode = nullptr; 2691 if (Subtarget->isThumb() && Subtarget->hasThumb2()) 2692 ResNode = SelectT2IndexedLoad(N); 2693 else 2694 ResNode = SelectARMIndexedLoad(N); 2695 if (ResNode) 2696 return ResNode; 2697 // Other cases are autogenerated. 2698 break; 2699 } 2700 case ARMISD::BRCOND: { 2701 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) 2702 // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc) 2703 // Pattern complexity = 6 cost = 1 size = 0 2704 2705 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) 2706 // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc) 2707 // Pattern complexity = 6 cost = 1 size = 0 2708 2709 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) 2710 // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc) 2711 // Pattern complexity = 6 cost = 1 size = 0 2712 2713 unsigned Opc = Subtarget->isThumb() ? 2714 ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc; 2715 SDValue Chain = N->getOperand(0); 2716 SDValue N1 = N->getOperand(1); 2717 SDValue N2 = N->getOperand(2); 2718 SDValue N3 = N->getOperand(3); 2719 SDValue InFlag = N->getOperand(4); 2720 assert(N1.getOpcode() == ISD::BasicBlock); 2721 assert(N2.getOpcode() == ISD::Constant); 2722 assert(N3.getOpcode() == ISD::Register); 2723 2724 SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned) 2725 cast<ConstantSDNode>(N2)->getZExtValue()), 2726 MVT::i32); 2727 SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag }; 2728 SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, 2729 MVT::Glue, Ops); 2730 Chain = SDValue(ResNode, 0); 2731 if (N->getNumValues() == 2) { 2732 InFlag = SDValue(ResNode, 1); 2733 ReplaceUses(SDValue(N, 1), InFlag); 2734 } 2735 ReplaceUses(SDValue(N, 0), 2736 SDValue(Chain.getNode(), Chain.getResNo())); 2737 return nullptr; 2738 } 2739 case ARMISD::VZIP: { 2740 unsigned Opc = 0; 2741 EVT VT = N->getValueType(0); 2742 switch (VT.getSimpleVT().SimpleTy) { 2743 default: return nullptr; 2744 case MVT::v8i8: Opc = ARM::VZIPd8; break; 2745 case MVT::v4i16: Opc = ARM::VZIPd16; break; 2746 case MVT::v2f32: 2747 // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm. 2748 case MVT::v2i32: Opc = ARM::VTRNd32; break; 2749 case MVT::v16i8: Opc = ARM::VZIPq8; break; 2750 case MVT::v8i16: Opc = ARM::VZIPq16; break; 2751 case MVT::v4f32: 2752 case MVT::v4i32: Opc = ARM::VZIPq32; break; 2753 } 2754 SDValue Pred = getAL(CurDAG); 2755 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2756 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; 2757 return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops); 2758 } 2759 case ARMISD::VUZP: { 2760 unsigned Opc = 0; 2761 EVT VT = N->getValueType(0); 2762 switch (VT.getSimpleVT().SimpleTy) { 2763 default: return nullptr; 2764 case MVT::v8i8: Opc = ARM::VUZPd8; break; 2765 case MVT::v4i16: Opc = ARM::VUZPd16; break; 2766 case MVT::v2f32: 2767 // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm. 2768 case MVT::v2i32: Opc = ARM::VTRNd32; break; 2769 case MVT::v16i8: Opc = ARM::VUZPq8; break; 2770 case MVT::v8i16: Opc = ARM::VUZPq16; break; 2771 case MVT::v4f32: 2772 case MVT::v4i32: Opc = ARM::VUZPq32; break; 2773 } 2774 SDValue Pred = getAL(CurDAG); 2775 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2776 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; 2777 return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops); 2778 } 2779 case ARMISD::VTRN: { 2780 unsigned Opc = 0; 2781 EVT VT = N->getValueType(0); 2782 switch (VT.getSimpleVT().SimpleTy) { 2783 default: return nullptr; 2784 case MVT::v8i8: Opc = ARM::VTRNd8; break; 2785 case MVT::v4i16: Opc = ARM::VTRNd16; break; 2786 case MVT::v2f32: 2787 case MVT::v2i32: Opc = ARM::VTRNd32; break; 2788 case MVT::v16i8: Opc = ARM::VTRNq8; break; 2789 case MVT::v8i16: Opc = ARM::VTRNq16; break; 2790 case MVT::v4f32: 2791 case MVT::v4i32: Opc = ARM::VTRNq32; break; 2792 } 2793 SDValue Pred = getAL(CurDAG); 2794 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2795 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; 2796 return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops); 2797 } 2798 case ARMISD::BUILD_VECTOR: { 2799 EVT VecVT = N->getValueType(0); 2800 EVT EltVT = VecVT.getVectorElementType(); 2801 unsigned NumElts = VecVT.getVectorNumElements(); 2802 if (EltVT == MVT::f64) { 2803 assert(NumElts == 2 && "unexpected type for BUILD_VECTOR"); 2804 return createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)); 2805 } 2806 assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR"); 2807 if (NumElts == 2) 2808 return createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)); 2809 assert(NumElts == 4 && "unexpected type for BUILD_VECTOR"); 2810 return createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1), 2811 N->getOperand(2), N->getOperand(3)); 2812 } 2813 2814 case ARMISD::VLD2DUP: { 2815 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16, 2816 ARM::VLD2DUPd32 }; 2817 return SelectVLDDup(N, false, 2, Opcodes); 2818 } 2819 2820 case ARMISD::VLD3DUP: { 2821 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo, 2822 ARM::VLD3DUPd16Pseudo, 2823 ARM::VLD3DUPd32Pseudo }; 2824 return SelectVLDDup(N, false, 3, Opcodes); 2825 } 2826 2827 case ARMISD::VLD4DUP: { 2828 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo, 2829 ARM::VLD4DUPd16Pseudo, 2830 ARM::VLD4DUPd32Pseudo }; 2831 return SelectVLDDup(N, false, 4, Opcodes); 2832 } 2833 2834 case ARMISD::VLD2DUP_UPD: { 2835 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed, 2836 ARM::VLD2DUPd16wb_fixed, 2837 ARM::VLD2DUPd32wb_fixed }; 2838 return SelectVLDDup(N, true, 2, Opcodes); 2839 } 2840 2841 case ARMISD::VLD3DUP_UPD: { 2842 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD, 2843 ARM::VLD3DUPd16Pseudo_UPD, 2844 ARM::VLD3DUPd32Pseudo_UPD }; 2845 return SelectVLDDup(N, true, 3, Opcodes); 2846 } 2847 2848 case ARMISD::VLD4DUP_UPD: { 2849 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD, 2850 ARM::VLD4DUPd16Pseudo_UPD, 2851 ARM::VLD4DUPd32Pseudo_UPD }; 2852 return SelectVLDDup(N, true, 4, Opcodes); 2853 } 2854 2855 case ARMISD::VLD1_UPD: { 2856 static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed, 2857 ARM::VLD1d16wb_fixed, 2858 ARM::VLD1d32wb_fixed, 2859 ARM::VLD1d64wb_fixed }; 2860 static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed, 2861 ARM::VLD1q16wb_fixed, 2862 ARM::VLD1q32wb_fixed, 2863 ARM::VLD1q64wb_fixed }; 2864 return SelectVLD(N, true, 1, DOpcodes, QOpcodes, nullptr); 2865 } 2866 2867 case ARMISD::VLD2_UPD: { 2868 static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed, 2869 ARM::VLD2d16wb_fixed, 2870 ARM::VLD2d32wb_fixed, 2871 ARM::VLD1q64wb_fixed}; 2872 static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed, 2873 ARM::VLD2q16PseudoWB_fixed, 2874 ARM::VLD2q32PseudoWB_fixed }; 2875 return SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr); 2876 } 2877 2878 case ARMISD::VLD3_UPD: { 2879 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD, 2880 ARM::VLD3d16Pseudo_UPD, 2881 ARM::VLD3d32Pseudo_UPD, 2882 ARM::VLD1d64TPseudoWB_fixed}; 2883 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD, 2884 ARM::VLD3q16Pseudo_UPD, 2885 ARM::VLD3q32Pseudo_UPD }; 2886 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD, 2887 ARM::VLD3q16oddPseudo_UPD, 2888 ARM::VLD3q32oddPseudo_UPD }; 2889 return SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); 2890 } 2891 2892 case ARMISD::VLD4_UPD: { 2893 static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD, 2894 ARM::VLD4d16Pseudo_UPD, 2895 ARM::VLD4d32Pseudo_UPD, 2896 ARM::VLD1d64QPseudoWB_fixed}; 2897 static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD, 2898 ARM::VLD4q16Pseudo_UPD, 2899 ARM::VLD4q32Pseudo_UPD }; 2900 static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD, 2901 ARM::VLD4q16oddPseudo_UPD, 2902 ARM::VLD4q32oddPseudo_UPD }; 2903 return SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); 2904 } 2905 2906 case ARMISD::VLD2LN_UPD: { 2907 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD, 2908 ARM::VLD2LNd16Pseudo_UPD, 2909 ARM::VLD2LNd32Pseudo_UPD }; 2910 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD, 2911 ARM::VLD2LNq32Pseudo_UPD }; 2912 return SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes); 2913 } 2914 2915 case ARMISD::VLD3LN_UPD: { 2916 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD, 2917 ARM::VLD3LNd16Pseudo_UPD, 2918 ARM::VLD3LNd32Pseudo_UPD }; 2919 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD, 2920 ARM::VLD3LNq32Pseudo_UPD }; 2921 return SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes); 2922 } 2923 2924 case ARMISD::VLD4LN_UPD: { 2925 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD, 2926 ARM::VLD4LNd16Pseudo_UPD, 2927 ARM::VLD4LNd32Pseudo_UPD }; 2928 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD, 2929 ARM::VLD4LNq32Pseudo_UPD }; 2930 return SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes); 2931 } 2932 2933 case ARMISD::VST1_UPD: { 2934 static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed, 2935 ARM::VST1d16wb_fixed, 2936 ARM::VST1d32wb_fixed, 2937 ARM::VST1d64wb_fixed }; 2938 static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed, 2939 ARM::VST1q16wb_fixed, 2940 ARM::VST1q32wb_fixed, 2941 ARM::VST1q64wb_fixed }; 2942 return SelectVST(N, true, 1, DOpcodes, QOpcodes, nullptr); 2943 } 2944 2945 case ARMISD::VST2_UPD: { 2946 static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed, 2947 ARM::VST2d16wb_fixed, 2948 ARM::VST2d32wb_fixed, 2949 ARM::VST1q64wb_fixed}; 2950 static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed, 2951 ARM::VST2q16PseudoWB_fixed, 2952 ARM::VST2q32PseudoWB_fixed }; 2953 return SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr); 2954 } 2955 2956 case ARMISD::VST3_UPD: { 2957 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD, 2958 ARM::VST3d16Pseudo_UPD, 2959 ARM::VST3d32Pseudo_UPD, 2960 ARM::VST1d64TPseudoWB_fixed}; 2961 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD, 2962 ARM::VST3q16Pseudo_UPD, 2963 ARM::VST3q32Pseudo_UPD }; 2964 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD, 2965 ARM::VST3q16oddPseudo_UPD, 2966 ARM::VST3q32oddPseudo_UPD }; 2967 return SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); 2968 } 2969 2970 case ARMISD::VST4_UPD: { 2971 static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD, 2972 ARM::VST4d16Pseudo_UPD, 2973 ARM::VST4d32Pseudo_UPD, 2974 ARM::VST1d64QPseudoWB_fixed}; 2975 static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD, 2976 ARM::VST4q16Pseudo_UPD, 2977 ARM::VST4q32Pseudo_UPD }; 2978 static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD, 2979 ARM::VST4q16oddPseudo_UPD, 2980 ARM::VST4q32oddPseudo_UPD }; 2981 return SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); 2982 } 2983 2984 case ARMISD::VST2LN_UPD: { 2985 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD, 2986 ARM::VST2LNd16Pseudo_UPD, 2987 ARM::VST2LNd32Pseudo_UPD }; 2988 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD, 2989 ARM::VST2LNq32Pseudo_UPD }; 2990 return SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes); 2991 } 2992 2993 case ARMISD::VST3LN_UPD: { 2994 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD, 2995 ARM::VST3LNd16Pseudo_UPD, 2996 ARM::VST3LNd32Pseudo_UPD }; 2997 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD, 2998 ARM::VST3LNq32Pseudo_UPD }; 2999 return SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes); 3000 } 3001 3002 case ARMISD::VST4LN_UPD: { 3003 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD, 3004 ARM::VST4LNd16Pseudo_UPD, 3005 ARM::VST4LNd32Pseudo_UPD }; 3006 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD, 3007 ARM::VST4LNq32Pseudo_UPD }; 3008 return SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes); 3009 } 3010 3011 case ISD::INTRINSIC_VOID: 3012 case ISD::INTRINSIC_W_CHAIN: { 3013 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 3014 switch (IntNo) { 3015 default: 3016 break; 3017 3018 case Intrinsic::arm_ldaexd: 3019 case Intrinsic::arm_ldrexd: { 3020 SDLoc dl(N); 3021 SDValue Chain = N->getOperand(0); 3022 SDValue MemAddr = N->getOperand(2); 3023 bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2(); 3024 3025 bool IsAcquire = IntNo == Intrinsic::arm_ldaexd; 3026 unsigned NewOpc = isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD) 3027 : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD); 3028 3029 // arm_ldrexd returns a i64 value in {i32, i32} 3030 std::vector<EVT> ResTys; 3031 if (isThumb) { 3032 ResTys.push_back(MVT::i32); 3033 ResTys.push_back(MVT::i32); 3034 } else 3035 ResTys.push_back(MVT::Untyped); 3036 ResTys.push_back(MVT::Other); 3037 3038 // Place arguments in the right order. 3039 SmallVector<SDValue, 7> Ops; 3040 Ops.push_back(MemAddr); 3041 Ops.push_back(getAL(CurDAG)); 3042 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); 3043 Ops.push_back(Chain); 3044 SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops); 3045 // Transfer memoperands. 3046 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 3047 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 3048 cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1); 3049 3050 // Remap uses. 3051 SDValue OutChain = isThumb ? SDValue(Ld, 2) : SDValue(Ld, 1); 3052 if (!SDValue(N, 0).use_empty()) { 3053 SDValue Result; 3054 if (isThumb) 3055 Result = SDValue(Ld, 0); 3056 else { 3057 SDValue SubRegIdx = CurDAG->getTargetConstant(ARM::gsub_0, MVT::i32); 3058 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 3059 dl, MVT::i32, SDValue(Ld, 0), SubRegIdx); 3060 Result = SDValue(ResNode,0); 3061 } 3062 ReplaceUses(SDValue(N, 0), Result); 3063 } 3064 if (!SDValue(N, 1).use_empty()) { 3065 SDValue Result; 3066 if (isThumb) 3067 Result = SDValue(Ld, 1); 3068 else { 3069 SDValue SubRegIdx = CurDAG->getTargetConstant(ARM::gsub_1, MVT::i32); 3070 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 3071 dl, MVT::i32, SDValue(Ld, 0), SubRegIdx); 3072 Result = SDValue(ResNode,0); 3073 } 3074 ReplaceUses(SDValue(N, 1), Result); 3075 } 3076 ReplaceUses(SDValue(N, 2), OutChain); 3077 return nullptr; 3078 } 3079 case Intrinsic::arm_stlexd: 3080 case Intrinsic::arm_strexd: { 3081 SDLoc dl(N); 3082 SDValue Chain = N->getOperand(0); 3083 SDValue Val0 = N->getOperand(2); 3084 SDValue Val1 = N->getOperand(3); 3085 SDValue MemAddr = N->getOperand(4); 3086 3087 // Store exclusive double return a i32 value which is the return status 3088 // of the issued store. 3089 const EVT ResTys[] = {MVT::i32, MVT::Other}; 3090 3091 bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2(); 3092 // Place arguments in the right order. 3093 SmallVector<SDValue, 7> Ops; 3094 if (isThumb) { 3095 Ops.push_back(Val0); 3096 Ops.push_back(Val1); 3097 } else 3098 // arm_strexd uses GPRPair. 3099 Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, Val0, Val1), 0)); 3100 Ops.push_back(MemAddr); 3101 Ops.push_back(getAL(CurDAG)); 3102 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); 3103 Ops.push_back(Chain); 3104 3105 bool IsRelease = IntNo == Intrinsic::arm_stlexd; 3106 unsigned NewOpc = isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD) 3107 : (IsRelease ? ARM::STLEXD : ARM::STREXD); 3108 3109 SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops); 3110 // Transfer memoperands. 3111 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 3112 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 3113 cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1); 3114 3115 return St; 3116 } 3117 3118 case Intrinsic::arm_neon_vld1: { 3119 static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16, 3120 ARM::VLD1d32, ARM::VLD1d64 }; 3121 static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16, 3122 ARM::VLD1q32, ARM::VLD1q64}; 3123 return SelectVLD(N, false, 1, DOpcodes, QOpcodes, nullptr); 3124 } 3125 3126 case Intrinsic::arm_neon_vld2: { 3127 static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16, 3128 ARM::VLD2d32, ARM::VLD1q64 }; 3129 static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo, 3130 ARM::VLD2q32Pseudo }; 3131 return SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr); 3132 } 3133 3134 case Intrinsic::arm_neon_vld3: { 3135 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo, 3136 ARM::VLD3d16Pseudo, 3137 ARM::VLD3d32Pseudo, 3138 ARM::VLD1d64TPseudo }; 3139 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD, 3140 ARM::VLD3q16Pseudo_UPD, 3141 ARM::VLD3q32Pseudo_UPD }; 3142 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo, 3143 ARM::VLD3q16oddPseudo, 3144 ARM::VLD3q32oddPseudo }; 3145 return SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); 3146 } 3147 3148 case Intrinsic::arm_neon_vld4: { 3149 static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo, 3150 ARM::VLD4d16Pseudo, 3151 ARM::VLD4d32Pseudo, 3152 ARM::VLD1d64QPseudo }; 3153 static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD, 3154 ARM::VLD4q16Pseudo_UPD, 3155 ARM::VLD4q32Pseudo_UPD }; 3156 static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo, 3157 ARM::VLD4q16oddPseudo, 3158 ARM::VLD4q32oddPseudo }; 3159 return SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); 3160 } 3161 3162 case Intrinsic::arm_neon_vld2lane: { 3163 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo, 3164 ARM::VLD2LNd16Pseudo, 3165 ARM::VLD2LNd32Pseudo }; 3166 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo, 3167 ARM::VLD2LNq32Pseudo }; 3168 return SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes); 3169 } 3170 3171 case Intrinsic::arm_neon_vld3lane: { 3172 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo, 3173 ARM::VLD3LNd16Pseudo, 3174 ARM::VLD3LNd32Pseudo }; 3175 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo, 3176 ARM::VLD3LNq32Pseudo }; 3177 return SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes); 3178 } 3179 3180 case Intrinsic::arm_neon_vld4lane: { 3181 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo, 3182 ARM::VLD4LNd16Pseudo, 3183 ARM::VLD4LNd32Pseudo }; 3184 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo, 3185 ARM::VLD4LNq32Pseudo }; 3186 return SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes); 3187 } 3188 3189 case Intrinsic::arm_neon_vst1: { 3190 static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16, 3191 ARM::VST1d32, ARM::VST1d64 }; 3192 static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16, 3193 ARM::VST1q32, ARM::VST1q64 }; 3194 return SelectVST(N, false, 1, DOpcodes, QOpcodes, nullptr); 3195 } 3196 3197 case Intrinsic::arm_neon_vst2: { 3198 static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16, 3199 ARM::VST2d32, ARM::VST1q64 }; 3200 static uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo, 3201 ARM::VST2q32Pseudo }; 3202 return SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr); 3203 } 3204 3205 case Intrinsic::arm_neon_vst3: { 3206 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo, 3207 ARM::VST3d16Pseudo, 3208 ARM::VST3d32Pseudo, 3209 ARM::VST1d64TPseudo }; 3210 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD, 3211 ARM::VST3q16Pseudo_UPD, 3212 ARM::VST3q32Pseudo_UPD }; 3213 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo, 3214 ARM::VST3q16oddPseudo, 3215 ARM::VST3q32oddPseudo }; 3216 return SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); 3217 } 3218 3219 case Intrinsic::arm_neon_vst4: { 3220 static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo, 3221 ARM::VST4d16Pseudo, 3222 ARM::VST4d32Pseudo, 3223 ARM::VST1d64QPseudo }; 3224 static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD, 3225 ARM::VST4q16Pseudo_UPD, 3226 ARM::VST4q32Pseudo_UPD }; 3227 static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo, 3228 ARM::VST4q16oddPseudo, 3229 ARM::VST4q32oddPseudo }; 3230 return SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); 3231 } 3232 3233 case Intrinsic::arm_neon_vst2lane: { 3234 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo, 3235 ARM::VST2LNd16Pseudo, 3236 ARM::VST2LNd32Pseudo }; 3237 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo, 3238 ARM::VST2LNq32Pseudo }; 3239 return SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes); 3240 } 3241 3242 case Intrinsic::arm_neon_vst3lane: { 3243 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo, 3244 ARM::VST3LNd16Pseudo, 3245 ARM::VST3LNd32Pseudo }; 3246 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo, 3247 ARM::VST3LNq32Pseudo }; 3248 return SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes); 3249 } 3250 3251 case Intrinsic::arm_neon_vst4lane: { 3252 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo, 3253 ARM::VST4LNd16Pseudo, 3254 ARM::VST4LNd32Pseudo }; 3255 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo, 3256 ARM::VST4LNq32Pseudo }; 3257 return SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes); 3258 } 3259 } 3260 break; 3261 } 3262 3263 case ISD::INTRINSIC_WO_CHAIN: { 3264 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 3265 switch (IntNo) { 3266 default: 3267 break; 3268 3269 case Intrinsic::arm_neon_vtbl2: 3270 return SelectVTBL(N, false, 2, ARM::VTBL2); 3271 case Intrinsic::arm_neon_vtbl3: 3272 return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo); 3273 case Intrinsic::arm_neon_vtbl4: 3274 return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo); 3275 3276 case Intrinsic::arm_neon_vtbx2: 3277 return SelectVTBL(N, true, 2, ARM::VTBX2); 3278 case Intrinsic::arm_neon_vtbx3: 3279 return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo); 3280 case Intrinsic::arm_neon_vtbx4: 3281 return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo); 3282 } 3283 break; 3284 } 3285 3286 case ARMISD::VTBL1: { 3287 SDLoc dl(N); 3288 EVT VT = N->getValueType(0); 3289 SmallVector<SDValue, 6> Ops; 3290 3291 Ops.push_back(N->getOperand(0)); 3292 Ops.push_back(N->getOperand(1)); 3293 Ops.push_back(getAL(CurDAG)); // Predicate 3294 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register 3295 return CurDAG->getMachineNode(ARM::VTBL1, dl, VT, Ops); 3296 } 3297 case ARMISD::VTBL2: { 3298 SDLoc dl(N); 3299 EVT VT = N->getValueType(0); 3300 3301 // Form a REG_SEQUENCE to force register allocation. 3302 SDValue V0 = N->getOperand(0); 3303 SDValue V1 = N->getOperand(1); 3304 SDValue RegSeq = SDValue(createDRegPairNode(MVT::v16i8, V0, V1), 0); 3305 3306 SmallVector<SDValue, 6> Ops; 3307 Ops.push_back(RegSeq); 3308 Ops.push_back(N->getOperand(2)); 3309 Ops.push_back(getAL(CurDAG)); // Predicate 3310 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register 3311 return CurDAG->getMachineNode(ARM::VTBL2, dl, VT, Ops); 3312 } 3313 3314 case ISD::CONCAT_VECTORS: 3315 return SelectConcatVector(N); 3316 } 3317 3318 return SelectCode(N); 3319} 3320 3321SDNode *ARMDAGToDAGISel::SelectInlineAsm(SDNode *N){ 3322 std::vector<SDValue> AsmNodeOperands; 3323 unsigned Flag, Kind; 3324 bool Changed = false; 3325 unsigned NumOps = N->getNumOperands(); 3326 3327 // Normally, i64 data is bounded to two arbitrary GRPs for "%r" constraint. 3328 // However, some instrstions (e.g. ldrexd/strexd in ARM mode) require 3329 // (even/even+1) GPRs and use %n and %Hn to refer to the individual regs 3330 // respectively. Since there is no constraint to explicitly specify a 3331 // reg pair, we use GPRPair reg class for "%r" for 64-bit data. For Thumb, 3332 // the 64-bit data may be referred by H, Q, R modifiers, so we still pack 3333 // them into a GPRPair. 3334 3335 SDLoc dl(N); 3336 SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps-1) 3337 : SDValue(nullptr,0); 3338 3339 SmallVector<bool, 8> OpChanged; 3340 // Glue node will be appended late. 3341 for(unsigned i = 0, e = N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) { 3342 SDValue op = N->getOperand(i); 3343 AsmNodeOperands.push_back(op); 3344 3345 if (i < InlineAsm::Op_FirstOperand) 3346 continue; 3347 3348 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) { 3349 Flag = C->getZExtValue(); 3350 Kind = InlineAsm::getKind(Flag); 3351 } 3352 else 3353 continue; 3354 3355 // Immediate operands to inline asm in the SelectionDAG are modeled with 3356 // two operands. The first is a constant of value InlineAsm::Kind_Imm, and 3357 // the second is a constant with the value of the immediate. If we get here 3358 // and we have a Kind_Imm, skip the next operand, and continue. 3359 if (Kind == InlineAsm::Kind_Imm) { 3360 SDValue op = N->getOperand(++i); 3361 AsmNodeOperands.push_back(op); 3362 continue; 3363 } 3364 3365 unsigned NumRegs = InlineAsm::getNumOperandRegisters(Flag); 3366 if (NumRegs) 3367 OpChanged.push_back(false); 3368 3369 unsigned DefIdx = 0; 3370 bool IsTiedToChangedOp = false; 3371 // If it's a use that is tied with a previous def, it has no 3372 // reg class constraint. 3373 if (Changed && InlineAsm::isUseOperandTiedToDef(Flag, DefIdx)) 3374 IsTiedToChangedOp = OpChanged[DefIdx]; 3375 3376 if (Kind != InlineAsm::Kind_RegUse && Kind != InlineAsm::Kind_RegDef 3377 && Kind != InlineAsm::Kind_RegDefEarlyClobber) 3378 continue; 3379 3380 unsigned RC; 3381 bool HasRC = InlineAsm::hasRegClassConstraint(Flag, RC); 3382 if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID)) 3383 || NumRegs != 2) 3384 continue; 3385 3386 assert((i+2 < NumOps) && "Invalid number of operands in inline asm"); 3387 SDValue V0 = N->getOperand(i+1); 3388 SDValue V1 = N->getOperand(i+2); 3389 unsigned Reg0 = cast<RegisterSDNode>(V0)->getReg(); 3390 unsigned Reg1 = cast<RegisterSDNode>(V1)->getReg(); 3391 SDValue PairedReg; 3392 MachineRegisterInfo &MRI = MF->getRegInfo(); 3393 3394 if (Kind == InlineAsm::Kind_RegDef || 3395 Kind == InlineAsm::Kind_RegDefEarlyClobber) { 3396 // Replace the two GPRs with 1 GPRPair and copy values from GPRPair to 3397 // the original GPRs. 3398 3399 unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass); 3400 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped); 3401 SDValue Chain = SDValue(N,0); 3402 3403 SDNode *GU = N->getGluedUser(); 3404 SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped, 3405 Chain.getValue(1)); 3406 3407 // Extract values from a GPRPair reg and copy to the original GPR reg. 3408 SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32, 3409 RegCopy); 3410 SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32, 3411 RegCopy); 3412 SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0, 3413 RegCopy.getValue(1)); 3414 SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1)); 3415 3416 // Update the original glue user. 3417 std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1); 3418 Ops.push_back(T1.getValue(1)); 3419 CurDAG->UpdateNodeOperands(GU, Ops); 3420 } 3421 else { 3422 // For Kind == InlineAsm::Kind_RegUse, we first copy two GPRs into a 3423 // GPRPair and then pass the GPRPair to the inline asm. 3424 SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain]; 3425 3426 // As REG_SEQ doesn't take RegisterSDNode, we copy them first. 3427 SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32, 3428 Chain.getValue(1)); 3429 SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32, 3430 T0.getValue(1)); 3431 SDValue Pair = SDValue(createGPRPairNode(MVT::Untyped, T0, T1), 0); 3432 3433 // Copy REG_SEQ into a GPRPair-typed VR and replace the original two 3434 // i32 VRs of inline asm with it. 3435 unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass); 3436 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped); 3437 Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1)); 3438 3439 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain; 3440 Glue = Chain.getValue(1); 3441 } 3442 3443 Changed = true; 3444 3445 if(PairedReg.getNode()) { 3446 OpChanged[OpChanged.size() -1 ] = true; 3447 Flag = InlineAsm::getFlagWord(Kind, 1 /* RegNum*/); 3448 if (IsTiedToChangedOp) 3449 Flag = InlineAsm::getFlagWordForMatchingOp(Flag, DefIdx); 3450 else 3451 Flag = InlineAsm::getFlagWordForRegClass(Flag, ARM::GPRPairRegClassID); 3452 // Replace the current flag. 3453 AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant( 3454 Flag, MVT::i32); 3455 // Add the new register node and skip the original two GPRs. 3456 AsmNodeOperands.push_back(PairedReg); 3457 // Skip the next two GPRs. 3458 i += 2; 3459 } 3460 } 3461 3462 if (Glue.getNode()) 3463 AsmNodeOperands.push_back(Glue); 3464 if (!Changed) 3465 return nullptr; 3466 3467 SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N), 3468 CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands); 3469 New->setNodeId(-1); 3470 return New.getNode(); 3471} 3472 3473 3474bool ARMDAGToDAGISel:: 3475SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, 3476 std::vector<SDValue> &OutOps) { 3477 assert(ConstraintID == InlineAsm::Constraint_m && 3478 "unexpected asm memory constraint"); 3479 // Require the address to be in a register. That is safe for all ARM 3480 // variants and it is hard to do anything much smarter without knowing 3481 // how the operand is used. 3482 OutOps.push_back(Op); 3483 return false; 3484} 3485 3486/// createARMISelDag - This pass converts a legalized DAG into a 3487/// ARM-specific DAG, ready for instruction scheduling. 3488/// 3489FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM, 3490 CodeGenOpt::Level OptLevel) { 3491 return new ARMDAGToDAGISel(TM, OptLevel); 3492} 3493