ARMBaseInstrInfo.cpp revision 0553e1efcd3f8ccd8b45302e033924d9f85a5d2f
1//===-- ARMBaseInstrInfo.cpp - ARM Instruction Information ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file contains the Base ARM implementation of the TargetInstrInfo class. 11// 12//===----------------------------------------------------------------------===// 13 14#include "ARMBaseInstrInfo.h" 15#include "ARM.h" 16#include "ARMBaseRegisterInfo.h" 17#include "ARMConstantPoolValue.h" 18#include "ARMHazardRecognizer.h" 19#include "ARMMachineFunctionInfo.h" 20#include "MCTargetDesc/ARMAddressingModes.h" 21#include "llvm/ADT/STLExtras.h" 22#include "llvm/CodeGen/LiveVariables.h" 23#include "llvm/CodeGen/MachineConstantPool.h" 24#include "llvm/CodeGen/MachineFrameInfo.h" 25#include "llvm/CodeGen/MachineInstrBuilder.h" 26#include "llvm/CodeGen/MachineJumpTableInfo.h" 27#include "llvm/CodeGen/MachineMemOperand.h" 28#include "llvm/CodeGen/MachineRegisterInfo.h" 29#include "llvm/CodeGen/SelectionDAGNodes.h" 30#include "llvm/IR/Constants.h" 31#include "llvm/IR/Function.h" 32#include "llvm/IR/GlobalValue.h" 33#include "llvm/MC/MCAsmInfo.h" 34#include "llvm/Support/BranchProbability.h" 35#include "llvm/Support/CommandLine.h" 36#include "llvm/Support/Debug.h" 37#include "llvm/Support/ErrorHandling.h" 38 39#define GET_INSTRINFO_CTOR 40#include "ARMGenInstrInfo.inc" 41 42using namespace llvm; 43 44static cl::opt<bool> 45EnableARM3Addr("enable-arm-3-addr-conv", cl::Hidden, 46 cl::desc("Enable ARM 2-addr to 3-addr conv")); 47 48static cl::opt<bool> 49WidenVMOVS("widen-vmovs", cl::Hidden, cl::init(true), 50 cl::desc("Widen ARM vmovs to vmovd when possible")); 51 52static cl::opt<unsigned> 53SwiftPartialUpdateClearance("swift-partial-update-clearance", 54 cl::Hidden, cl::init(12), 55 cl::desc("Clearance before partial register updates")); 56 57/// ARM_MLxEntry - Record information about MLA / MLS instructions. 58struct ARM_MLxEntry { 59 uint16_t MLxOpc; // MLA / MLS opcode 60 uint16_t MulOpc; // Expanded multiplication opcode 61 uint16_t AddSubOpc; // Expanded add / sub opcode 62 bool NegAcc; // True if the acc is negated before the add / sub. 63 bool HasLane; // True if instruction has an extra "lane" operand. 64}; 65 66static const ARM_MLxEntry ARM_MLxTable[] = { 67 // MLxOpc, MulOpc, AddSubOpc, NegAcc, HasLane 68 // fp scalar ops 69 { ARM::VMLAS, ARM::VMULS, ARM::VADDS, false, false }, 70 { ARM::VMLSS, ARM::VMULS, ARM::VSUBS, false, false }, 71 { ARM::VMLAD, ARM::VMULD, ARM::VADDD, false, false }, 72 { ARM::VMLSD, ARM::VMULD, ARM::VSUBD, false, false }, 73 { ARM::VNMLAS, ARM::VNMULS, ARM::VSUBS, true, false }, 74 { ARM::VNMLSS, ARM::VMULS, ARM::VSUBS, true, false }, 75 { ARM::VNMLAD, ARM::VNMULD, ARM::VSUBD, true, false }, 76 { ARM::VNMLSD, ARM::VMULD, ARM::VSUBD, true, false }, 77 78 // fp SIMD ops 79 { ARM::VMLAfd, ARM::VMULfd, ARM::VADDfd, false, false }, 80 { ARM::VMLSfd, ARM::VMULfd, ARM::VSUBfd, false, false }, 81 { ARM::VMLAfq, ARM::VMULfq, ARM::VADDfq, false, false }, 82 { ARM::VMLSfq, ARM::VMULfq, ARM::VSUBfq, false, false }, 83 { ARM::VMLAslfd, ARM::VMULslfd, ARM::VADDfd, false, true }, 84 { ARM::VMLSslfd, ARM::VMULslfd, ARM::VSUBfd, false, true }, 85 { ARM::VMLAslfq, ARM::VMULslfq, ARM::VADDfq, false, true }, 86 { ARM::VMLSslfq, ARM::VMULslfq, ARM::VSUBfq, false, true }, 87}; 88 89ARMBaseInstrInfo::ARMBaseInstrInfo(const ARMSubtarget& STI) 90 : ARMGenInstrInfo(ARM::ADJCALLSTACKDOWN, ARM::ADJCALLSTACKUP), 91 Subtarget(STI) { 92 for (unsigned i = 0, e = array_lengthof(ARM_MLxTable); i != e; ++i) { 93 if (!MLxEntryMap.insert(std::make_pair(ARM_MLxTable[i].MLxOpc, i)).second) 94 assert(false && "Duplicated entries?"); 95 MLxHazardOpcodes.insert(ARM_MLxTable[i].AddSubOpc); 96 MLxHazardOpcodes.insert(ARM_MLxTable[i].MulOpc); 97 } 98} 99 100// Use a ScoreboardHazardRecognizer for prepass ARM scheduling. TargetInstrImpl 101// currently defaults to no prepass hazard recognizer. 102ScheduleHazardRecognizer *ARMBaseInstrInfo:: 103CreateTargetHazardRecognizer(const TargetMachine *TM, 104 const ScheduleDAG *DAG) const { 105 if (usePreRAHazardRecognizer()) { 106 const InstrItineraryData *II = TM->getInstrItineraryData(); 107 return new ScoreboardHazardRecognizer(II, DAG, "pre-RA-sched"); 108 } 109 return TargetInstrInfo::CreateTargetHazardRecognizer(TM, DAG); 110} 111 112ScheduleHazardRecognizer *ARMBaseInstrInfo:: 113CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, 114 const ScheduleDAG *DAG) const { 115 if (Subtarget.isThumb2() || Subtarget.hasVFP2()) 116 return (ScheduleHazardRecognizer *) 117 new ARMHazardRecognizer(II, *this, getRegisterInfo(), Subtarget, DAG); 118 return TargetInstrInfo::CreateTargetPostRAHazardRecognizer(II, DAG); 119} 120 121MachineInstr * 122ARMBaseInstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI, 123 MachineBasicBlock::iterator &MBBI, 124 LiveVariables *LV) const { 125 // FIXME: Thumb2 support. 126 127 if (!EnableARM3Addr) 128 return NULL; 129 130 MachineInstr *MI = MBBI; 131 MachineFunction &MF = *MI->getParent()->getParent(); 132 uint64_t TSFlags = MI->getDesc().TSFlags; 133 bool isPre = false; 134 switch ((TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift) { 135 default: return NULL; 136 case ARMII::IndexModePre: 137 isPre = true; 138 break; 139 case ARMII::IndexModePost: 140 break; 141 } 142 143 // Try splitting an indexed load/store to an un-indexed one plus an add/sub 144 // operation. 145 unsigned MemOpc = getUnindexedOpcode(MI->getOpcode()); 146 if (MemOpc == 0) 147 return NULL; 148 149 MachineInstr *UpdateMI = NULL; 150 MachineInstr *MemMI = NULL; 151 unsigned AddrMode = (TSFlags & ARMII::AddrModeMask); 152 const MCInstrDesc &MCID = MI->getDesc(); 153 unsigned NumOps = MCID.getNumOperands(); 154 bool isLoad = !MI->mayStore(); 155 const MachineOperand &WB = isLoad ? MI->getOperand(1) : MI->getOperand(0); 156 const MachineOperand &Base = MI->getOperand(2); 157 const MachineOperand &Offset = MI->getOperand(NumOps-3); 158 unsigned WBReg = WB.getReg(); 159 unsigned BaseReg = Base.getReg(); 160 unsigned OffReg = Offset.getReg(); 161 unsigned OffImm = MI->getOperand(NumOps-2).getImm(); 162 ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI->getOperand(NumOps-1).getImm(); 163 switch (AddrMode) { 164 default: llvm_unreachable("Unknown indexed op!"); 165 case ARMII::AddrMode2: { 166 bool isSub = ARM_AM::getAM2Op(OffImm) == ARM_AM::sub; 167 unsigned Amt = ARM_AM::getAM2Offset(OffImm); 168 if (OffReg == 0) { 169 if (ARM_AM::getSOImmVal(Amt) == -1) 170 // Can't encode it in a so_imm operand. This transformation will 171 // add more than 1 instruction. Abandon! 172 return NULL; 173 UpdateMI = BuildMI(MF, MI->getDebugLoc(), 174 get(isSub ? ARM::SUBri : ARM::ADDri), WBReg) 175 .addReg(BaseReg).addImm(Amt) 176 .addImm(Pred).addReg(0).addReg(0); 177 } else if (Amt != 0) { 178 ARM_AM::ShiftOpc ShOpc = ARM_AM::getAM2ShiftOpc(OffImm); 179 unsigned SOOpc = ARM_AM::getSORegOpc(ShOpc, Amt); 180 UpdateMI = BuildMI(MF, MI->getDebugLoc(), 181 get(isSub ? ARM::SUBrsi : ARM::ADDrsi), WBReg) 182 .addReg(BaseReg).addReg(OffReg).addReg(0).addImm(SOOpc) 183 .addImm(Pred).addReg(0).addReg(0); 184 } else 185 UpdateMI = BuildMI(MF, MI->getDebugLoc(), 186 get(isSub ? ARM::SUBrr : ARM::ADDrr), WBReg) 187 .addReg(BaseReg).addReg(OffReg) 188 .addImm(Pred).addReg(0).addReg(0); 189 break; 190 } 191 case ARMII::AddrMode3 : { 192 bool isSub = ARM_AM::getAM3Op(OffImm) == ARM_AM::sub; 193 unsigned Amt = ARM_AM::getAM3Offset(OffImm); 194 if (OffReg == 0) 195 // Immediate is 8-bits. It's guaranteed to fit in a so_imm operand. 196 UpdateMI = BuildMI(MF, MI->getDebugLoc(), 197 get(isSub ? ARM::SUBri : ARM::ADDri), WBReg) 198 .addReg(BaseReg).addImm(Amt) 199 .addImm(Pred).addReg(0).addReg(0); 200 else 201 UpdateMI = BuildMI(MF, MI->getDebugLoc(), 202 get(isSub ? ARM::SUBrr : ARM::ADDrr), WBReg) 203 .addReg(BaseReg).addReg(OffReg) 204 .addImm(Pred).addReg(0).addReg(0); 205 break; 206 } 207 } 208 209 std::vector<MachineInstr*> NewMIs; 210 if (isPre) { 211 if (isLoad) 212 MemMI = BuildMI(MF, MI->getDebugLoc(), 213 get(MemOpc), MI->getOperand(0).getReg()) 214 .addReg(WBReg).addImm(0).addImm(Pred); 215 else 216 MemMI = BuildMI(MF, MI->getDebugLoc(), 217 get(MemOpc)).addReg(MI->getOperand(1).getReg()) 218 .addReg(WBReg).addReg(0).addImm(0).addImm(Pred); 219 NewMIs.push_back(MemMI); 220 NewMIs.push_back(UpdateMI); 221 } else { 222 if (isLoad) 223 MemMI = BuildMI(MF, MI->getDebugLoc(), 224 get(MemOpc), MI->getOperand(0).getReg()) 225 .addReg(BaseReg).addImm(0).addImm(Pred); 226 else 227 MemMI = BuildMI(MF, MI->getDebugLoc(), 228 get(MemOpc)).addReg(MI->getOperand(1).getReg()) 229 .addReg(BaseReg).addReg(0).addImm(0).addImm(Pred); 230 if (WB.isDead()) 231 UpdateMI->getOperand(0).setIsDead(); 232 NewMIs.push_back(UpdateMI); 233 NewMIs.push_back(MemMI); 234 } 235 236 // Transfer LiveVariables states, kill / dead info. 237 if (LV) { 238 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { 239 MachineOperand &MO = MI->getOperand(i); 240 if (MO.isReg() && TargetRegisterInfo::isVirtualRegister(MO.getReg())) { 241 unsigned Reg = MO.getReg(); 242 243 LiveVariables::VarInfo &VI = LV->getVarInfo(Reg); 244 if (MO.isDef()) { 245 MachineInstr *NewMI = (Reg == WBReg) ? UpdateMI : MemMI; 246 if (MO.isDead()) 247 LV->addVirtualRegisterDead(Reg, NewMI); 248 } 249 if (MO.isUse() && MO.isKill()) { 250 for (unsigned j = 0; j < 2; ++j) { 251 // Look at the two new MI's in reverse order. 252 MachineInstr *NewMI = NewMIs[j]; 253 if (!NewMI->readsRegister(Reg)) 254 continue; 255 LV->addVirtualRegisterKilled(Reg, NewMI); 256 if (VI.removeKill(MI)) 257 VI.Kills.push_back(NewMI); 258 break; 259 } 260 } 261 } 262 } 263 } 264 265 MFI->insert(MBBI, NewMIs[1]); 266 MFI->insert(MBBI, NewMIs[0]); 267 return NewMIs[0]; 268} 269 270// Branch analysis. 271bool 272ARMBaseInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,MachineBasicBlock *&TBB, 273 MachineBasicBlock *&FBB, 274 SmallVectorImpl<MachineOperand> &Cond, 275 bool AllowModify) const { 276 // If the block has no terminators, it just falls into the block after it. 277 MachineBasicBlock::iterator I = MBB.end(); 278 if (I == MBB.begin()) 279 return false; 280 --I; 281 while (I->isDebugValue()) { 282 if (I == MBB.begin()) 283 return false; 284 --I; 285 } 286 287 // Get the last instruction in the block. 288 MachineInstr *LastInst = I; 289 unsigned LastOpc = LastInst->getOpcode(); 290 291 // Check if it's an indirect branch first, this should return 'unanalyzable' 292 // even if it's predicated. 293 if (isIndirectBranchOpcode(LastOpc)) 294 return true; 295 296 if (!isUnpredicatedTerminator(I)) 297 return false; 298 299 // If there is only one terminator instruction, process it. 300 if (I == MBB.begin() || !isUnpredicatedTerminator(--I)) { 301 if (isUncondBranchOpcode(LastOpc)) { 302 TBB = LastInst->getOperand(0).getMBB(); 303 return false; 304 } 305 if (isCondBranchOpcode(LastOpc)) { 306 // Block ends with fall-through condbranch. 307 TBB = LastInst->getOperand(0).getMBB(); 308 Cond.push_back(LastInst->getOperand(1)); 309 Cond.push_back(LastInst->getOperand(2)); 310 return false; 311 } 312 return true; // Can't handle indirect branch. 313 } 314 315 // Get the instruction before it if it is a terminator. 316 MachineInstr *SecondLastInst = I; 317 unsigned SecondLastOpc = SecondLastInst->getOpcode(); 318 319 // If AllowModify is true and the block ends with two or more unconditional 320 // branches, delete all but the first unconditional branch. 321 if (AllowModify && isUncondBranchOpcode(LastOpc)) { 322 while (isUncondBranchOpcode(SecondLastOpc)) { 323 LastInst->eraseFromParent(); 324 LastInst = SecondLastInst; 325 LastOpc = LastInst->getOpcode(); 326 if (I == MBB.begin() || !isUnpredicatedTerminator(--I)) { 327 // Return now the only terminator is an unconditional branch. 328 TBB = LastInst->getOperand(0).getMBB(); 329 return false; 330 } else { 331 SecondLastInst = I; 332 SecondLastOpc = SecondLastInst->getOpcode(); 333 } 334 } 335 } 336 337 // If there are three terminators, we don't know what sort of block this is. 338 if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(--I)) 339 return true; 340 341 // If the block ends with a B and a Bcc, handle it. 342 if (isCondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) { 343 TBB = SecondLastInst->getOperand(0).getMBB(); 344 Cond.push_back(SecondLastInst->getOperand(1)); 345 Cond.push_back(SecondLastInst->getOperand(2)); 346 FBB = LastInst->getOperand(0).getMBB(); 347 return false; 348 } 349 350 // If the block ends with two unconditional branches, handle it. The second 351 // one is not executed, so remove it. 352 if (isUncondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) { 353 TBB = SecondLastInst->getOperand(0).getMBB(); 354 I = LastInst; 355 if (AllowModify) 356 I->eraseFromParent(); 357 return false; 358 } 359 360 // ...likewise if it ends with a branch table followed by an unconditional 361 // branch. The branch folder can create these, and we must get rid of them for 362 // correctness of Thumb constant islands. 363 if ((isJumpTableBranchOpcode(SecondLastOpc) || 364 isIndirectBranchOpcode(SecondLastOpc)) && 365 isUncondBranchOpcode(LastOpc)) { 366 I = LastInst; 367 if (AllowModify) 368 I->eraseFromParent(); 369 return true; 370 } 371 372 // Otherwise, can't handle this. 373 return true; 374} 375 376 377unsigned ARMBaseInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const { 378 MachineBasicBlock::iterator I = MBB.end(); 379 if (I == MBB.begin()) return 0; 380 --I; 381 while (I->isDebugValue()) { 382 if (I == MBB.begin()) 383 return 0; 384 --I; 385 } 386 if (!isUncondBranchOpcode(I->getOpcode()) && 387 !isCondBranchOpcode(I->getOpcode())) 388 return 0; 389 390 // Remove the branch. 391 I->eraseFromParent(); 392 393 I = MBB.end(); 394 395 if (I == MBB.begin()) return 1; 396 --I; 397 if (!isCondBranchOpcode(I->getOpcode())) 398 return 1; 399 400 // Remove the branch. 401 I->eraseFromParent(); 402 return 2; 403} 404 405unsigned 406ARMBaseInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, 407 MachineBasicBlock *FBB, 408 const SmallVectorImpl<MachineOperand> &Cond, 409 DebugLoc DL) const { 410 ARMFunctionInfo *AFI = MBB.getParent()->getInfo<ARMFunctionInfo>(); 411 int BOpc = !AFI->isThumbFunction() 412 ? ARM::B : (AFI->isThumb2Function() ? ARM::t2B : ARM::tB); 413 int BccOpc = !AFI->isThumbFunction() 414 ? ARM::Bcc : (AFI->isThumb2Function() ? ARM::t2Bcc : ARM::tBcc); 415 bool isThumb = AFI->isThumbFunction() || AFI->isThumb2Function(); 416 417 // Shouldn't be a fall through. 418 assert(TBB && "InsertBranch must not be told to insert a fallthrough"); 419 assert((Cond.size() == 2 || Cond.size() == 0) && 420 "ARM branch conditions have two components!"); 421 422 if (FBB == 0) { 423 if (Cond.empty()) { // Unconditional branch? 424 if (isThumb) 425 BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB).addImm(ARMCC::AL).addReg(0); 426 else 427 BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB); 428 } else 429 BuildMI(&MBB, DL, get(BccOpc)).addMBB(TBB) 430 .addImm(Cond[0].getImm()).addReg(Cond[1].getReg()); 431 return 1; 432 } 433 434 // Two-way conditional branch. 435 BuildMI(&MBB, DL, get(BccOpc)).addMBB(TBB) 436 .addImm(Cond[0].getImm()).addReg(Cond[1].getReg()); 437 if (isThumb) 438 BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB).addImm(ARMCC::AL).addReg(0); 439 else 440 BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB); 441 return 2; 442} 443 444bool ARMBaseInstrInfo:: 445ReverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { 446 ARMCC::CondCodes CC = (ARMCC::CondCodes)(int)Cond[0].getImm(); 447 Cond[0].setImm(ARMCC::getOppositeCondition(CC)); 448 return false; 449} 450 451bool ARMBaseInstrInfo::isPredicated(const MachineInstr *MI) const { 452 if (MI->isBundle()) { 453 MachineBasicBlock::const_instr_iterator I = MI; 454 MachineBasicBlock::const_instr_iterator E = MI->getParent()->instr_end(); 455 while (++I != E && I->isInsideBundle()) { 456 int PIdx = I->findFirstPredOperandIdx(); 457 if (PIdx != -1 && I->getOperand(PIdx).getImm() != ARMCC::AL) 458 return true; 459 } 460 return false; 461 } 462 463 int PIdx = MI->findFirstPredOperandIdx(); 464 return PIdx != -1 && MI->getOperand(PIdx).getImm() != ARMCC::AL; 465} 466 467bool ARMBaseInstrInfo:: 468PredicateInstruction(MachineInstr *MI, 469 const SmallVectorImpl<MachineOperand> &Pred) const { 470 unsigned Opc = MI->getOpcode(); 471 if (isUncondBranchOpcode(Opc)) { 472 MI->setDesc(get(getMatchingCondBranchOpcode(Opc))); 473 MachineInstrBuilder(*MI->getParent()->getParent(), MI) 474 .addImm(Pred[0].getImm()) 475 .addReg(Pred[1].getReg()); 476 return true; 477 } 478 479 int PIdx = MI->findFirstPredOperandIdx(); 480 if (PIdx != -1) { 481 MachineOperand &PMO = MI->getOperand(PIdx); 482 PMO.setImm(Pred[0].getImm()); 483 MI->getOperand(PIdx+1).setReg(Pred[1].getReg()); 484 return true; 485 } 486 return false; 487} 488 489bool ARMBaseInstrInfo:: 490SubsumesPredicate(const SmallVectorImpl<MachineOperand> &Pred1, 491 const SmallVectorImpl<MachineOperand> &Pred2) const { 492 if (Pred1.size() > 2 || Pred2.size() > 2) 493 return false; 494 495 ARMCC::CondCodes CC1 = (ARMCC::CondCodes)Pred1[0].getImm(); 496 ARMCC::CondCodes CC2 = (ARMCC::CondCodes)Pred2[0].getImm(); 497 if (CC1 == CC2) 498 return true; 499 500 switch (CC1) { 501 default: 502 return false; 503 case ARMCC::AL: 504 return true; 505 case ARMCC::HS: 506 return CC2 == ARMCC::HI; 507 case ARMCC::LS: 508 return CC2 == ARMCC::LO || CC2 == ARMCC::EQ; 509 case ARMCC::GE: 510 return CC2 == ARMCC::GT; 511 case ARMCC::LE: 512 return CC2 == ARMCC::LT; 513 } 514} 515 516bool ARMBaseInstrInfo::DefinesPredicate(MachineInstr *MI, 517 std::vector<MachineOperand> &Pred) const { 518 bool Found = false; 519 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { 520 const MachineOperand &MO = MI->getOperand(i); 521 if ((MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR)) || 522 (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR)) { 523 Pred.push_back(MO); 524 Found = true; 525 } 526 } 527 528 return Found; 529} 530 531/// isPredicable - Return true if the specified instruction can be predicated. 532/// By default, this returns true for every instruction with a 533/// PredicateOperand. 534bool ARMBaseInstrInfo::isPredicable(MachineInstr *MI) const { 535 if (!MI->isPredicable()) 536 return false; 537 538 if ((MI->getDesc().TSFlags & ARMII::DomainMask) == ARMII::DomainNEON) { 539 ARMFunctionInfo *AFI = 540 MI->getParent()->getParent()->getInfo<ARMFunctionInfo>(); 541 return AFI->isThumb2Function(); 542 } 543 return true; 544} 545 546/// FIXME: Works around a gcc miscompilation with -fstrict-aliasing. 547LLVM_ATTRIBUTE_NOINLINE 548static unsigned getNumJTEntries(const std::vector<MachineJumpTableEntry> &JT, 549 unsigned JTI); 550static unsigned getNumJTEntries(const std::vector<MachineJumpTableEntry> &JT, 551 unsigned JTI) { 552 assert(JTI < JT.size()); 553 return JT[JTI].MBBs.size(); 554} 555 556/// GetInstSize - Return the size of the specified MachineInstr. 557/// 558unsigned ARMBaseInstrInfo::GetInstSizeInBytes(const MachineInstr *MI) const { 559 const MachineBasicBlock &MBB = *MI->getParent(); 560 const MachineFunction *MF = MBB.getParent(); 561 const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo(); 562 563 const MCInstrDesc &MCID = MI->getDesc(); 564 if (MCID.getSize()) 565 return MCID.getSize(); 566 567 // If this machine instr is an inline asm, measure it. 568 if (MI->getOpcode() == ARM::INLINEASM) 569 return getInlineAsmLength(MI->getOperand(0).getSymbolName(), *MAI); 570 if (MI->isLabel()) 571 return 0; 572 unsigned Opc = MI->getOpcode(); 573 switch (Opc) { 574 case TargetOpcode::IMPLICIT_DEF: 575 case TargetOpcode::KILL: 576 case TargetOpcode::PROLOG_LABEL: 577 case TargetOpcode::EH_LABEL: 578 case TargetOpcode::DBG_VALUE: 579 return 0; 580 case TargetOpcode::BUNDLE: 581 return getInstBundleLength(MI); 582 case ARM::MOVi16_ga_pcrel: 583 case ARM::MOVTi16_ga_pcrel: 584 case ARM::t2MOVi16_ga_pcrel: 585 case ARM::t2MOVTi16_ga_pcrel: 586 return 4; 587 case ARM::MOVi32imm: 588 case ARM::t2MOVi32imm: 589 return 8; 590 case ARM::CONSTPOOL_ENTRY: 591 // If this machine instr is a constant pool entry, its size is recorded as 592 // operand #2. 593 return MI->getOperand(2).getImm(); 594 case ARM::Int_eh_sjlj_longjmp: 595 return 16; 596 case ARM::tInt_eh_sjlj_longjmp: 597 return 10; 598 case ARM::Int_eh_sjlj_setjmp: 599 case ARM::Int_eh_sjlj_setjmp_nofp: 600 return 20; 601 case ARM::tInt_eh_sjlj_setjmp: 602 case ARM::t2Int_eh_sjlj_setjmp: 603 case ARM::t2Int_eh_sjlj_setjmp_nofp: 604 return 12; 605 case ARM::BR_JTr: 606 case ARM::BR_JTm: 607 case ARM::BR_JTadd: 608 case ARM::tBR_JTr: 609 case ARM::t2BR_JT: 610 case ARM::t2TBB_JT: 611 case ARM::t2TBH_JT: { 612 // These are jumptable branches, i.e. a branch followed by an inlined 613 // jumptable. The size is 4 + 4 * number of entries. For TBB, each 614 // entry is one byte; TBH two byte each. 615 unsigned EntrySize = (Opc == ARM::t2TBB_JT) 616 ? 1 : ((Opc == ARM::t2TBH_JT) ? 2 : 4); 617 unsigned NumOps = MCID.getNumOperands(); 618 MachineOperand JTOP = 619 MI->getOperand(NumOps - (MI->isPredicable() ? 3 : 2)); 620 unsigned JTI = JTOP.getIndex(); 621 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo(); 622 assert(MJTI != 0); 623 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 624 assert(JTI < JT.size()); 625 // Thumb instructions are 2 byte aligned, but JT entries are 4 byte 626 // 4 aligned. The assembler / linker may add 2 byte padding just before 627 // the JT entries. The size does not include this padding; the 628 // constant islands pass does separate bookkeeping for it. 629 // FIXME: If we know the size of the function is less than (1 << 16) *2 630 // bytes, we can use 16-bit entries instead. Then there won't be an 631 // alignment issue. 632 unsigned InstSize = (Opc == ARM::tBR_JTr || Opc == ARM::t2BR_JT) ? 2 : 4; 633 unsigned NumEntries = getNumJTEntries(JT, JTI); 634 if (Opc == ARM::t2TBB_JT && (NumEntries & 1)) 635 // Make sure the instruction that follows TBB is 2-byte aligned. 636 // FIXME: Constant island pass should insert an "ALIGN" instruction 637 // instead. 638 ++NumEntries; 639 return NumEntries * EntrySize + InstSize; 640 } 641 default: 642 // Otherwise, pseudo-instruction sizes are zero. 643 return 0; 644 } 645} 646 647unsigned ARMBaseInstrInfo::getInstBundleLength(const MachineInstr *MI) const { 648 unsigned Size = 0; 649 MachineBasicBlock::const_instr_iterator I = MI; 650 MachineBasicBlock::const_instr_iterator E = MI->getParent()->instr_end(); 651 while (++I != E && I->isInsideBundle()) { 652 assert(!I->isBundle() && "No nested bundle!"); 653 Size += GetInstSizeInBytes(&*I); 654 } 655 return Size; 656} 657 658void ARMBaseInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 659 MachineBasicBlock::iterator I, DebugLoc DL, 660 unsigned DestReg, unsigned SrcReg, 661 bool KillSrc) const { 662 bool GPRDest = ARM::GPRRegClass.contains(DestReg); 663 bool GPRSrc = ARM::GPRRegClass.contains(SrcReg); 664 665 if (GPRDest && GPRSrc) { 666 AddDefaultCC(AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::MOVr), DestReg) 667 .addReg(SrcReg, getKillRegState(KillSrc)))); 668 return; 669 } 670 671 bool SPRDest = ARM::SPRRegClass.contains(DestReg); 672 bool SPRSrc = ARM::SPRRegClass.contains(SrcReg); 673 674 unsigned Opc = 0; 675 if (SPRDest && SPRSrc) 676 Opc = ARM::VMOVS; 677 else if (GPRDest && SPRSrc) 678 Opc = ARM::VMOVRS; 679 else if (SPRDest && GPRSrc) 680 Opc = ARM::VMOVSR; 681 else if (ARM::DPRRegClass.contains(DestReg, SrcReg)) 682 Opc = ARM::VMOVD; 683 else if (ARM::QPRRegClass.contains(DestReg, SrcReg)) 684 Opc = ARM::VORRq; 685 686 if (Opc) { 687 MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opc), DestReg); 688 MIB.addReg(SrcReg, getKillRegState(KillSrc)); 689 if (Opc == ARM::VORRq) 690 MIB.addReg(SrcReg, getKillRegState(KillSrc)); 691 AddDefaultPred(MIB); 692 return; 693 } 694 695 // Handle register classes that require multiple instructions. 696 unsigned BeginIdx = 0; 697 unsigned SubRegs = 0; 698 int Spacing = 1; 699 700 // Use VORRq when possible. 701 if (ARM::QQPRRegClass.contains(DestReg, SrcReg)) 702 Opc = ARM::VORRq, BeginIdx = ARM::qsub_0, SubRegs = 2; 703 else if (ARM::QQQQPRRegClass.contains(DestReg, SrcReg)) 704 Opc = ARM::VORRq, BeginIdx = ARM::qsub_0, SubRegs = 4; 705 // Fall back to VMOVD. 706 else if (ARM::DPairRegClass.contains(DestReg, SrcReg)) 707 Opc = ARM::VMOVD, BeginIdx = ARM::dsub_0, SubRegs = 2; 708 else if (ARM::DTripleRegClass.contains(DestReg, SrcReg)) 709 Opc = ARM::VMOVD, BeginIdx = ARM::dsub_0, SubRegs = 3; 710 else if (ARM::DQuadRegClass.contains(DestReg, SrcReg)) 711 Opc = ARM::VMOVD, BeginIdx = ARM::dsub_0, SubRegs = 4; 712 else if (ARM::GPRPairRegClass.contains(DestReg, SrcReg)) 713 Opc = ARM::MOVr, BeginIdx = ARM::gsub_0, SubRegs = 2; 714 715 else if (ARM::DPairSpcRegClass.contains(DestReg, SrcReg)) 716 Opc = ARM::VMOVD, BeginIdx = ARM::dsub_0, SubRegs = 2, Spacing = 2; 717 else if (ARM::DTripleSpcRegClass.contains(DestReg, SrcReg)) 718 Opc = ARM::VMOVD, BeginIdx = ARM::dsub_0, SubRegs = 3, Spacing = 2; 719 else if (ARM::DQuadSpcRegClass.contains(DestReg, SrcReg)) 720 Opc = ARM::VMOVD, BeginIdx = ARM::dsub_0, SubRegs = 4, Spacing = 2; 721 722 assert(Opc && "Impossible reg-to-reg copy"); 723 724 const TargetRegisterInfo *TRI = &getRegisterInfo(); 725 MachineInstrBuilder Mov; 726 727 // Copy register tuples backward when the first Dest reg overlaps with SrcReg. 728 if (TRI->regsOverlap(SrcReg, TRI->getSubReg(DestReg, BeginIdx))) { 729 BeginIdx = BeginIdx + ((SubRegs-1)*Spacing); 730 Spacing = -Spacing; 731 } 732#ifndef NDEBUG 733 SmallSet<unsigned, 4> DstRegs; 734#endif 735 for (unsigned i = 0; i != SubRegs; ++i) { 736 unsigned Dst = TRI->getSubReg(DestReg, BeginIdx + i*Spacing); 737 unsigned Src = TRI->getSubReg(SrcReg, BeginIdx + i*Spacing); 738 assert(Dst && Src && "Bad sub-register"); 739#ifndef NDEBUG 740 assert(!DstRegs.count(Src) && "destructive vector copy"); 741 DstRegs.insert(Dst); 742#endif 743 Mov = BuildMI(MBB, I, I->getDebugLoc(), get(Opc), Dst) 744 .addReg(Src); 745 // VORR takes two source operands. 746 if (Opc == ARM::VORRq) 747 Mov.addReg(Src); 748 Mov = AddDefaultPred(Mov); 749 } 750 // Add implicit super-register defs and kills to the last instruction. 751 Mov->addRegisterDefined(DestReg, TRI); 752 if (KillSrc) 753 Mov->addRegisterKilled(SrcReg, TRI); 754} 755 756const MachineInstrBuilder & 757ARMBaseInstrInfo::AddDReg(MachineInstrBuilder &MIB, unsigned Reg, 758 unsigned SubIdx, unsigned State, 759 const TargetRegisterInfo *TRI) const { 760 if (!SubIdx) 761 return MIB.addReg(Reg, State); 762 763 if (TargetRegisterInfo::isPhysicalRegister(Reg)) 764 return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State); 765 return MIB.addReg(Reg, State, SubIdx); 766} 767 768void ARMBaseInstrInfo:: 769storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, 770 unsigned SrcReg, bool isKill, int FI, 771 const TargetRegisterClass *RC, 772 const TargetRegisterInfo *TRI) const { 773 DebugLoc DL; 774 if (I != MBB.end()) DL = I->getDebugLoc(); 775 MachineFunction &MF = *MBB.getParent(); 776 MachineFrameInfo &MFI = *MF.getFrameInfo(); 777 unsigned Align = MFI.getObjectAlignment(FI); 778 779 MachineMemOperand *MMO = 780 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(FI), 781 MachineMemOperand::MOStore, 782 MFI.getObjectSize(FI), 783 Align); 784 785 switch (RC->getSize()) { 786 case 4: 787 if (ARM::GPRRegClass.hasSubClassEq(RC)) { 788 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::STRi12)) 789 .addReg(SrcReg, getKillRegState(isKill)) 790 .addFrameIndex(FI).addImm(0).addMemOperand(MMO)); 791 } else if (ARM::SPRRegClass.hasSubClassEq(RC)) { 792 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTRS)) 793 .addReg(SrcReg, getKillRegState(isKill)) 794 .addFrameIndex(FI).addImm(0).addMemOperand(MMO)); 795 } else 796 llvm_unreachable("Unknown reg class!"); 797 break; 798 case 8: 799 if (ARM::DPRRegClass.hasSubClassEq(RC)) { 800 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTRD)) 801 .addReg(SrcReg, getKillRegState(isKill)) 802 .addFrameIndex(FI).addImm(0).addMemOperand(MMO)); 803 } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) { 804 if (Subtarget.hasV5TEOps()) { 805 MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::STRD)); 806 AddDReg(MIB, SrcReg, ARM::gsub_0, getKillRegState(isKill), TRI); 807 AddDReg(MIB, SrcReg, ARM::gsub_1, 0, TRI); 808 MIB.addFrameIndex(FI).addReg(0).addImm(0).addMemOperand(MMO); 809 810 AddDefaultPred(MIB); 811 } else { 812 // Fallback to STM instruction, which has existed since the dawn of 813 // time. 814 MachineInstrBuilder MIB = 815 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::STMIA)) 816 .addFrameIndex(FI).addMemOperand(MMO)); 817 AddDReg(MIB, SrcReg, ARM::gsub_0, getKillRegState(isKill), TRI); 818 AddDReg(MIB, SrcReg, ARM::gsub_1, 0, TRI); 819 } 820 } else 821 llvm_unreachable("Unknown reg class!"); 822 break; 823 case 16: 824 if (ARM::DPairRegClass.hasSubClassEq(RC)) { 825 // Use aligned spills if the stack can be realigned. 826 if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) { 827 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VST1q64)) 828 .addFrameIndex(FI).addImm(16) 829 .addReg(SrcReg, getKillRegState(isKill)) 830 .addMemOperand(MMO)); 831 } else { 832 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTMQIA)) 833 .addReg(SrcReg, getKillRegState(isKill)) 834 .addFrameIndex(FI) 835 .addMemOperand(MMO)); 836 } 837 } else 838 llvm_unreachable("Unknown reg class!"); 839 break; 840 case 24: 841 if (ARM::DTripleRegClass.hasSubClassEq(RC)) { 842 // Use aligned spills if the stack can be realigned. 843 if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) { 844 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VST1d64TPseudo)) 845 .addFrameIndex(FI).addImm(16) 846 .addReg(SrcReg, getKillRegState(isKill)) 847 .addMemOperand(MMO)); 848 } else { 849 MachineInstrBuilder MIB = 850 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTMDIA)) 851 .addFrameIndex(FI)) 852 .addMemOperand(MMO); 853 MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI); 854 MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI); 855 AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI); 856 } 857 } else 858 llvm_unreachable("Unknown reg class!"); 859 break; 860 case 32: 861 if (ARM::QQPRRegClass.hasSubClassEq(RC) || ARM::DQuadRegClass.hasSubClassEq(RC)) { 862 if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) { 863 // FIXME: It's possible to only store part of the QQ register if the 864 // spilled def has a sub-register index. 865 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VST1d64QPseudo)) 866 .addFrameIndex(FI).addImm(16) 867 .addReg(SrcReg, getKillRegState(isKill)) 868 .addMemOperand(MMO)); 869 } else { 870 MachineInstrBuilder MIB = 871 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTMDIA)) 872 .addFrameIndex(FI)) 873 .addMemOperand(MMO); 874 MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI); 875 MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI); 876 MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI); 877 AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI); 878 } 879 } else 880 llvm_unreachable("Unknown reg class!"); 881 break; 882 case 64: 883 if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) { 884 MachineInstrBuilder MIB = 885 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTMDIA)) 886 .addFrameIndex(FI)) 887 .addMemOperand(MMO); 888 MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI); 889 MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI); 890 MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI); 891 MIB = AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI); 892 MIB = AddDReg(MIB, SrcReg, ARM::dsub_4, 0, TRI); 893 MIB = AddDReg(MIB, SrcReg, ARM::dsub_5, 0, TRI); 894 MIB = AddDReg(MIB, SrcReg, ARM::dsub_6, 0, TRI); 895 AddDReg(MIB, SrcReg, ARM::dsub_7, 0, TRI); 896 } else 897 llvm_unreachable("Unknown reg class!"); 898 break; 899 default: 900 llvm_unreachable("Unknown reg class!"); 901 } 902} 903 904unsigned 905ARMBaseInstrInfo::isStoreToStackSlot(const MachineInstr *MI, 906 int &FrameIndex) const { 907 switch (MI->getOpcode()) { 908 default: break; 909 case ARM::STRrs: 910 case ARM::t2STRs: // FIXME: don't use t2STRs to access frame. 911 if (MI->getOperand(1).isFI() && 912 MI->getOperand(2).isReg() && 913 MI->getOperand(3).isImm() && 914 MI->getOperand(2).getReg() == 0 && 915 MI->getOperand(3).getImm() == 0) { 916 FrameIndex = MI->getOperand(1).getIndex(); 917 return MI->getOperand(0).getReg(); 918 } 919 break; 920 case ARM::STRi12: 921 case ARM::t2STRi12: 922 case ARM::tSTRspi: 923 case ARM::VSTRD: 924 case ARM::VSTRS: 925 if (MI->getOperand(1).isFI() && 926 MI->getOperand(2).isImm() && 927 MI->getOperand(2).getImm() == 0) { 928 FrameIndex = MI->getOperand(1).getIndex(); 929 return MI->getOperand(0).getReg(); 930 } 931 break; 932 case ARM::VST1q64: 933 case ARM::VST1d64TPseudo: 934 case ARM::VST1d64QPseudo: 935 if (MI->getOperand(0).isFI() && 936 MI->getOperand(2).getSubReg() == 0) { 937 FrameIndex = MI->getOperand(0).getIndex(); 938 return MI->getOperand(2).getReg(); 939 } 940 break; 941 case ARM::VSTMQIA: 942 if (MI->getOperand(1).isFI() && 943 MI->getOperand(0).getSubReg() == 0) { 944 FrameIndex = MI->getOperand(1).getIndex(); 945 return MI->getOperand(0).getReg(); 946 } 947 break; 948 } 949 950 return 0; 951} 952 953unsigned ARMBaseInstrInfo::isStoreToStackSlotPostFE(const MachineInstr *MI, 954 int &FrameIndex) const { 955 const MachineMemOperand *Dummy; 956 return MI->mayStore() && hasStoreToStackSlot(MI, Dummy, FrameIndex); 957} 958 959void ARMBaseInstrInfo:: 960loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, 961 unsigned DestReg, int FI, 962 const TargetRegisterClass *RC, 963 const TargetRegisterInfo *TRI) const { 964 DebugLoc DL; 965 if (I != MBB.end()) DL = I->getDebugLoc(); 966 MachineFunction &MF = *MBB.getParent(); 967 MachineFrameInfo &MFI = *MF.getFrameInfo(); 968 unsigned Align = MFI.getObjectAlignment(FI); 969 MachineMemOperand *MMO = 970 MF.getMachineMemOperand( 971 MachinePointerInfo::getFixedStack(FI), 972 MachineMemOperand::MOLoad, 973 MFI.getObjectSize(FI), 974 Align); 975 976 switch (RC->getSize()) { 977 case 4: 978 if (ARM::GPRRegClass.hasSubClassEq(RC)) { 979 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::LDRi12), DestReg) 980 .addFrameIndex(FI).addImm(0).addMemOperand(MMO)); 981 982 } else if (ARM::SPRRegClass.hasSubClassEq(RC)) { 983 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDRS), DestReg) 984 .addFrameIndex(FI).addImm(0).addMemOperand(MMO)); 985 } else 986 llvm_unreachable("Unknown reg class!"); 987 break; 988 case 8: 989 if (ARM::DPRRegClass.hasSubClassEq(RC)) { 990 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDRD), DestReg) 991 .addFrameIndex(FI).addImm(0).addMemOperand(MMO)); 992 } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) { 993 MachineInstrBuilder MIB; 994 995 if (Subtarget.hasV5TEOps()) { 996 MIB = BuildMI(MBB, I, DL, get(ARM::LDRD)); 997 AddDReg(MIB, DestReg, ARM::gsub_0, RegState::DefineNoRead, TRI); 998 AddDReg(MIB, DestReg, ARM::gsub_1, RegState::DefineNoRead, TRI); 999 MIB.addFrameIndex(FI).addReg(0).addImm(0).addMemOperand(MMO); 1000 1001 AddDefaultPred(MIB); 1002 } else { 1003 // Fallback to LDM instruction, which has existed since the dawn of 1004 // time. 1005 MIB = AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::LDMIA)) 1006 .addFrameIndex(FI).addMemOperand(MMO)); 1007 MIB = AddDReg(MIB, DestReg, ARM::gsub_0, RegState::DefineNoRead, TRI); 1008 MIB = AddDReg(MIB, DestReg, ARM::gsub_1, RegState::DefineNoRead, TRI); 1009 } 1010 1011 if (TargetRegisterInfo::isPhysicalRegister(DestReg)) 1012 MIB.addReg(DestReg, RegState::ImplicitDefine); 1013 } else 1014 llvm_unreachable("Unknown reg class!"); 1015 break; 1016 case 16: 1017 if (ARM::DPairRegClass.hasSubClassEq(RC)) { 1018 if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) { 1019 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLD1q64), DestReg) 1020 .addFrameIndex(FI).addImm(16) 1021 .addMemOperand(MMO)); 1022 } else { 1023 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDMQIA), DestReg) 1024 .addFrameIndex(FI) 1025 .addMemOperand(MMO)); 1026 } 1027 } else 1028 llvm_unreachable("Unknown reg class!"); 1029 break; 1030 case 24: 1031 if (ARM::DTripleRegClass.hasSubClassEq(RC)) { 1032 if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) { 1033 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLD1d64TPseudo), DestReg) 1034 .addFrameIndex(FI).addImm(16) 1035 .addMemOperand(MMO)); 1036 } else { 1037 MachineInstrBuilder MIB = 1038 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDMDIA)) 1039 .addFrameIndex(FI) 1040 .addMemOperand(MMO)); 1041 MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI); 1042 MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI); 1043 MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI); 1044 if (TargetRegisterInfo::isPhysicalRegister(DestReg)) 1045 MIB.addReg(DestReg, RegState::ImplicitDefine); 1046 } 1047 } else 1048 llvm_unreachable("Unknown reg class!"); 1049 break; 1050 case 32: 1051 if (ARM::QQPRRegClass.hasSubClassEq(RC) || ARM::DQuadRegClass.hasSubClassEq(RC)) { 1052 if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) { 1053 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLD1d64QPseudo), DestReg) 1054 .addFrameIndex(FI).addImm(16) 1055 .addMemOperand(MMO)); 1056 } else { 1057 MachineInstrBuilder MIB = 1058 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDMDIA)) 1059 .addFrameIndex(FI)) 1060 .addMemOperand(MMO); 1061 MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI); 1062 MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI); 1063 MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI); 1064 MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::DefineNoRead, TRI); 1065 if (TargetRegisterInfo::isPhysicalRegister(DestReg)) 1066 MIB.addReg(DestReg, RegState::ImplicitDefine); 1067 } 1068 } else 1069 llvm_unreachable("Unknown reg class!"); 1070 break; 1071 case 64: 1072 if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) { 1073 MachineInstrBuilder MIB = 1074 AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDMDIA)) 1075 .addFrameIndex(FI)) 1076 .addMemOperand(MMO); 1077 MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI); 1078 MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI); 1079 MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI); 1080 MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::DefineNoRead, TRI); 1081 MIB = AddDReg(MIB, DestReg, ARM::dsub_4, RegState::DefineNoRead, TRI); 1082 MIB = AddDReg(MIB, DestReg, ARM::dsub_5, RegState::DefineNoRead, TRI); 1083 MIB = AddDReg(MIB, DestReg, ARM::dsub_6, RegState::DefineNoRead, TRI); 1084 MIB = AddDReg(MIB, DestReg, ARM::dsub_7, RegState::DefineNoRead, TRI); 1085 if (TargetRegisterInfo::isPhysicalRegister(DestReg)) 1086 MIB.addReg(DestReg, RegState::ImplicitDefine); 1087 } else 1088 llvm_unreachable("Unknown reg class!"); 1089 break; 1090 default: 1091 llvm_unreachable("Unknown regclass!"); 1092 } 1093} 1094 1095unsigned 1096ARMBaseInstrInfo::isLoadFromStackSlot(const MachineInstr *MI, 1097 int &FrameIndex) const { 1098 switch (MI->getOpcode()) { 1099 default: break; 1100 case ARM::LDRrs: 1101 case ARM::t2LDRs: // FIXME: don't use t2LDRs to access frame. 1102 if (MI->getOperand(1).isFI() && 1103 MI->getOperand(2).isReg() && 1104 MI->getOperand(3).isImm() && 1105 MI->getOperand(2).getReg() == 0 && 1106 MI->getOperand(3).getImm() == 0) { 1107 FrameIndex = MI->getOperand(1).getIndex(); 1108 return MI->getOperand(0).getReg(); 1109 } 1110 break; 1111 case ARM::LDRi12: 1112 case ARM::t2LDRi12: 1113 case ARM::tLDRspi: 1114 case ARM::VLDRD: 1115 case ARM::VLDRS: 1116 if (MI->getOperand(1).isFI() && 1117 MI->getOperand(2).isImm() && 1118 MI->getOperand(2).getImm() == 0) { 1119 FrameIndex = MI->getOperand(1).getIndex(); 1120 return MI->getOperand(0).getReg(); 1121 } 1122 break; 1123 case ARM::VLD1q64: 1124 case ARM::VLD1d64TPseudo: 1125 case ARM::VLD1d64QPseudo: 1126 if (MI->getOperand(1).isFI() && 1127 MI->getOperand(0).getSubReg() == 0) { 1128 FrameIndex = MI->getOperand(1).getIndex(); 1129 return MI->getOperand(0).getReg(); 1130 } 1131 break; 1132 case ARM::VLDMQIA: 1133 if (MI->getOperand(1).isFI() && 1134 MI->getOperand(0).getSubReg() == 0) { 1135 FrameIndex = MI->getOperand(1).getIndex(); 1136 return MI->getOperand(0).getReg(); 1137 } 1138 break; 1139 } 1140 1141 return 0; 1142} 1143 1144unsigned ARMBaseInstrInfo::isLoadFromStackSlotPostFE(const MachineInstr *MI, 1145 int &FrameIndex) const { 1146 const MachineMemOperand *Dummy; 1147 return MI->mayLoad() && hasLoadFromStackSlot(MI, Dummy, FrameIndex); 1148} 1149 1150bool ARMBaseInstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const{ 1151 // This hook gets to expand COPY instructions before they become 1152 // copyPhysReg() calls. Look for VMOVS instructions that can legally be 1153 // widened to VMOVD. We prefer the VMOVD when possible because it may be 1154 // changed into a VORR that can go down the NEON pipeline. 1155 if (!WidenVMOVS || !MI->isCopy() || Subtarget.isCortexA15()) 1156 return false; 1157 1158 // Look for a copy between even S-registers. That is where we keep floats 1159 // when using NEON v2f32 instructions for f32 arithmetic. 1160 unsigned DstRegS = MI->getOperand(0).getReg(); 1161 unsigned SrcRegS = MI->getOperand(1).getReg(); 1162 if (!ARM::SPRRegClass.contains(DstRegS, SrcRegS)) 1163 return false; 1164 1165 const TargetRegisterInfo *TRI = &getRegisterInfo(); 1166 unsigned DstRegD = TRI->getMatchingSuperReg(DstRegS, ARM::ssub_0, 1167 &ARM::DPRRegClass); 1168 unsigned SrcRegD = TRI->getMatchingSuperReg(SrcRegS, ARM::ssub_0, 1169 &ARM::DPRRegClass); 1170 if (!DstRegD || !SrcRegD) 1171 return false; 1172 1173 // We want to widen this into a DstRegD = VMOVD SrcRegD copy. This is only 1174 // legal if the COPY already defines the full DstRegD, and it isn't a 1175 // sub-register insertion. 1176 if (!MI->definesRegister(DstRegD, TRI) || MI->readsRegister(DstRegD, TRI)) 1177 return false; 1178 1179 // A dead copy shouldn't show up here, but reject it just in case. 1180 if (MI->getOperand(0).isDead()) 1181 return false; 1182 1183 // All clear, widen the COPY. 1184 DEBUG(dbgs() << "widening: " << *MI); 1185 MachineInstrBuilder MIB(*MI->getParent()->getParent(), MI); 1186 1187 // Get rid of the old <imp-def> of DstRegD. Leave it if it defines a Q-reg 1188 // or some other super-register. 1189 int ImpDefIdx = MI->findRegisterDefOperandIdx(DstRegD); 1190 if (ImpDefIdx != -1) 1191 MI->RemoveOperand(ImpDefIdx); 1192 1193 // Change the opcode and operands. 1194 MI->setDesc(get(ARM::VMOVD)); 1195 MI->getOperand(0).setReg(DstRegD); 1196 MI->getOperand(1).setReg(SrcRegD); 1197 AddDefaultPred(MIB); 1198 1199 // We are now reading SrcRegD instead of SrcRegS. This may upset the 1200 // register scavenger and machine verifier, so we need to indicate that we 1201 // are reading an undefined value from SrcRegD, but a proper value from 1202 // SrcRegS. 1203 MI->getOperand(1).setIsUndef(); 1204 MIB.addReg(SrcRegS, RegState::Implicit); 1205 1206 // SrcRegD may actually contain an unrelated value in the ssub_1 1207 // sub-register. Don't kill it. Only kill the ssub_0 sub-register. 1208 if (MI->getOperand(1).isKill()) { 1209 MI->getOperand(1).setIsKill(false); 1210 MI->addRegisterKilled(SrcRegS, TRI, true); 1211 } 1212 1213 DEBUG(dbgs() << "replaced by: " << *MI); 1214 return true; 1215} 1216 1217MachineInstr* 1218ARMBaseInstrInfo::emitFrameIndexDebugValue(MachineFunction &MF, 1219 int FrameIx, uint64_t Offset, 1220 const MDNode *MDPtr, 1221 DebugLoc DL) const { 1222 MachineInstrBuilder MIB = BuildMI(MF, DL, get(ARM::DBG_VALUE)) 1223 .addFrameIndex(FrameIx).addImm(0).addImm(Offset).addMetadata(MDPtr); 1224 return &*MIB; 1225} 1226 1227/// Create a copy of a const pool value. Update CPI to the new index and return 1228/// the label UID. 1229static unsigned duplicateCPV(MachineFunction &MF, unsigned &CPI) { 1230 MachineConstantPool *MCP = MF.getConstantPool(); 1231 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 1232 1233 const MachineConstantPoolEntry &MCPE = MCP->getConstants()[CPI]; 1234 assert(MCPE.isMachineConstantPoolEntry() && 1235 "Expecting a machine constantpool entry!"); 1236 ARMConstantPoolValue *ACPV = 1237 static_cast<ARMConstantPoolValue*>(MCPE.Val.MachineCPVal); 1238 1239 unsigned PCLabelId = AFI->createPICLabelUId(); 1240 ARMConstantPoolValue *NewCPV = 0; 1241 // FIXME: The below assumes PIC relocation model and that the function 1242 // is Thumb mode (t1 or t2). PCAdjustment would be 8 for ARM mode PIC, and 1243 // zero for non-PIC in ARM or Thumb. The callers are all of thumb LDR 1244 // instructions, so that's probably OK, but is PIC always correct when 1245 // we get here? 1246 if (ACPV->isGlobalValue()) 1247 NewCPV = ARMConstantPoolConstant:: 1248 Create(cast<ARMConstantPoolConstant>(ACPV)->getGV(), PCLabelId, 1249 ARMCP::CPValue, 4); 1250 else if (ACPV->isExtSymbol()) 1251 NewCPV = ARMConstantPoolSymbol:: 1252 Create(MF.getFunction()->getContext(), 1253 cast<ARMConstantPoolSymbol>(ACPV)->getSymbol(), PCLabelId, 4); 1254 else if (ACPV->isBlockAddress()) 1255 NewCPV = ARMConstantPoolConstant:: 1256 Create(cast<ARMConstantPoolConstant>(ACPV)->getBlockAddress(), PCLabelId, 1257 ARMCP::CPBlockAddress, 4); 1258 else if (ACPV->isLSDA()) 1259 NewCPV = ARMConstantPoolConstant::Create(MF.getFunction(), PCLabelId, 1260 ARMCP::CPLSDA, 4); 1261 else if (ACPV->isMachineBasicBlock()) 1262 NewCPV = ARMConstantPoolMBB:: 1263 Create(MF.getFunction()->getContext(), 1264 cast<ARMConstantPoolMBB>(ACPV)->getMBB(), PCLabelId, 4); 1265 else 1266 llvm_unreachable("Unexpected ARM constantpool value type!!"); 1267 CPI = MCP->getConstantPoolIndex(NewCPV, MCPE.getAlignment()); 1268 return PCLabelId; 1269} 1270 1271void ARMBaseInstrInfo:: 1272reMaterialize(MachineBasicBlock &MBB, 1273 MachineBasicBlock::iterator I, 1274 unsigned DestReg, unsigned SubIdx, 1275 const MachineInstr *Orig, 1276 const TargetRegisterInfo &TRI) const { 1277 unsigned Opcode = Orig->getOpcode(); 1278 switch (Opcode) { 1279 default: { 1280 MachineInstr *MI = MBB.getParent()->CloneMachineInstr(Orig); 1281 MI->substituteRegister(Orig->getOperand(0).getReg(), DestReg, SubIdx, TRI); 1282 MBB.insert(I, MI); 1283 break; 1284 } 1285 case ARM::tLDRpci_pic: 1286 case ARM::t2LDRpci_pic: { 1287 MachineFunction &MF = *MBB.getParent(); 1288 unsigned CPI = Orig->getOperand(1).getIndex(); 1289 unsigned PCLabelId = duplicateCPV(MF, CPI); 1290 MachineInstrBuilder MIB = BuildMI(MBB, I, Orig->getDebugLoc(), get(Opcode), 1291 DestReg) 1292 .addConstantPoolIndex(CPI).addImm(PCLabelId); 1293 MIB->setMemRefs(Orig->memoperands_begin(), Orig->memoperands_end()); 1294 break; 1295 } 1296 } 1297} 1298 1299MachineInstr * 1300ARMBaseInstrInfo::duplicate(MachineInstr *Orig, MachineFunction &MF) const { 1301 MachineInstr *MI = TargetInstrInfo::duplicate(Orig, MF); 1302 switch(Orig->getOpcode()) { 1303 case ARM::tLDRpci_pic: 1304 case ARM::t2LDRpci_pic: { 1305 unsigned CPI = Orig->getOperand(1).getIndex(); 1306 unsigned PCLabelId = duplicateCPV(MF, CPI); 1307 Orig->getOperand(1).setIndex(CPI); 1308 Orig->getOperand(2).setImm(PCLabelId); 1309 break; 1310 } 1311 } 1312 return MI; 1313} 1314 1315bool ARMBaseInstrInfo::produceSameValue(const MachineInstr *MI0, 1316 const MachineInstr *MI1, 1317 const MachineRegisterInfo *MRI) const { 1318 int Opcode = MI0->getOpcode(); 1319 if (Opcode == ARM::t2LDRpci || 1320 Opcode == ARM::t2LDRpci_pic || 1321 Opcode == ARM::tLDRpci || 1322 Opcode == ARM::tLDRpci_pic || 1323 Opcode == ARM::MOV_ga_dyn || 1324 Opcode == ARM::MOV_ga_pcrel || 1325 Opcode == ARM::MOV_ga_pcrel_ldr || 1326 Opcode == ARM::t2MOV_ga_dyn || 1327 Opcode == ARM::t2MOV_ga_pcrel) { 1328 if (MI1->getOpcode() != Opcode) 1329 return false; 1330 if (MI0->getNumOperands() != MI1->getNumOperands()) 1331 return false; 1332 1333 const MachineOperand &MO0 = MI0->getOperand(1); 1334 const MachineOperand &MO1 = MI1->getOperand(1); 1335 if (MO0.getOffset() != MO1.getOffset()) 1336 return false; 1337 1338 if (Opcode == ARM::MOV_ga_dyn || 1339 Opcode == ARM::MOV_ga_pcrel || 1340 Opcode == ARM::MOV_ga_pcrel_ldr || 1341 Opcode == ARM::t2MOV_ga_dyn || 1342 Opcode == ARM::t2MOV_ga_pcrel) 1343 // Ignore the PC labels. 1344 return MO0.getGlobal() == MO1.getGlobal(); 1345 1346 const MachineFunction *MF = MI0->getParent()->getParent(); 1347 const MachineConstantPool *MCP = MF->getConstantPool(); 1348 int CPI0 = MO0.getIndex(); 1349 int CPI1 = MO1.getIndex(); 1350 const MachineConstantPoolEntry &MCPE0 = MCP->getConstants()[CPI0]; 1351 const MachineConstantPoolEntry &MCPE1 = MCP->getConstants()[CPI1]; 1352 bool isARMCP0 = MCPE0.isMachineConstantPoolEntry(); 1353 bool isARMCP1 = MCPE1.isMachineConstantPoolEntry(); 1354 if (isARMCP0 && isARMCP1) { 1355 ARMConstantPoolValue *ACPV0 = 1356 static_cast<ARMConstantPoolValue*>(MCPE0.Val.MachineCPVal); 1357 ARMConstantPoolValue *ACPV1 = 1358 static_cast<ARMConstantPoolValue*>(MCPE1.Val.MachineCPVal); 1359 return ACPV0->hasSameValue(ACPV1); 1360 } else if (!isARMCP0 && !isARMCP1) { 1361 return MCPE0.Val.ConstVal == MCPE1.Val.ConstVal; 1362 } 1363 return false; 1364 } else if (Opcode == ARM::PICLDR) { 1365 if (MI1->getOpcode() != Opcode) 1366 return false; 1367 if (MI0->getNumOperands() != MI1->getNumOperands()) 1368 return false; 1369 1370 unsigned Addr0 = MI0->getOperand(1).getReg(); 1371 unsigned Addr1 = MI1->getOperand(1).getReg(); 1372 if (Addr0 != Addr1) { 1373 if (!MRI || 1374 !TargetRegisterInfo::isVirtualRegister(Addr0) || 1375 !TargetRegisterInfo::isVirtualRegister(Addr1)) 1376 return false; 1377 1378 // This assumes SSA form. 1379 MachineInstr *Def0 = MRI->getVRegDef(Addr0); 1380 MachineInstr *Def1 = MRI->getVRegDef(Addr1); 1381 // Check if the loaded value, e.g. a constantpool of a global address, are 1382 // the same. 1383 if (!produceSameValue(Def0, Def1, MRI)) 1384 return false; 1385 } 1386 1387 for (unsigned i = 3, e = MI0->getNumOperands(); i != e; ++i) { 1388 // %vreg12<def> = PICLDR %vreg11, 0, pred:14, pred:%noreg 1389 const MachineOperand &MO0 = MI0->getOperand(i); 1390 const MachineOperand &MO1 = MI1->getOperand(i); 1391 if (!MO0.isIdenticalTo(MO1)) 1392 return false; 1393 } 1394 return true; 1395 } 1396 1397 return MI0->isIdenticalTo(MI1, MachineInstr::IgnoreVRegDefs); 1398} 1399 1400/// areLoadsFromSameBasePtr - This is used by the pre-regalloc scheduler to 1401/// determine if two loads are loading from the same base address. It should 1402/// only return true if the base pointers are the same and the only differences 1403/// between the two addresses is the offset. It also returns the offsets by 1404/// reference. 1405/// 1406/// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched 1407/// is permanently disabled. 1408bool ARMBaseInstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, 1409 int64_t &Offset1, 1410 int64_t &Offset2) const { 1411 // Don't worry about Thumb: just ARM and Thumb2. 1412 if (Subtarget.isThumb1Only()) return false; 1413 1414 if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode()) 1415 return false; 1416 1417 switch (Load1->getMachineOpcode()) { 1418 default: 1419 return false; 1420 case ARM::LDRi12: 1421 case ARM::LDRBi12: 1422 case ARM::LDRD: 1423 case ARM::LDRH: 1424 case ARM::LDRSB: 1425 case ARM::LDRSH: 1426 case ARM::VLDRD: 1427 case ARM::VLDRS: 1428 case ARM::t2LDRi8: 1429 case ARM::t2LDRDi8: 1430 case ARM::t2LDRSHi8: 1431 case ARM::t2LDRi12: 1432 case ARM::t2LDRSHi12: 1433 break; 1434 } 1435 1436 switch (Load2->getMachineOpcode()) { 1437 default: 1438 return false; 1439 case ARM::LDRi12: 1440 case ARM::LDRBi12: 1441 case ARM::LDRD: 1442 case ARM::LDRH: 1443 case ARM::LDRSB: 1444 case ARM::LDRSH: 1445 case ARM::VLDRD: 1446 case ARM::VLDRS: 1447 case ARM::t2LDRi8: 1448 case ARM::t2LDRSHi8: 1449 case ARM::t2LDRi12: 1450 case ARM::t2LDRSHi12: 1451 break; 1452 } 1453 1454 // Check if base addresses and chain operands match. 1455 if (Load1->getOperand(0) != Load2->getOperand(0) || 1456 Load1->getOperand(4) != Load2->getOperand(4)) 1457 return false; 1458 1459 // Index should be Reg0. 1460 if (Load1->getOperand(3) != Load2->getOperand(3)) 1461 return false; 1462 1463 // Determine the offsets. 1464 if (isa<ConstantSDNode>(Load1->getOperand(1)) && 1465 isa<ConstantSDNode>(Load2->getOperand(1))) { 1466 Offset1 = cast<ConstantSDNode>(Load1->getOperand(1))->getSExtValue(); 1467 Offset2 = cast<ConstantSDNode>(Load2->getOperand(1))->getSExtValue(); 1468 return true; 1469 } 1470 1471 return false; 1472} 1473 1474/// shouldScheduleLoadsNear - This is a used by the pre-regalloc scheduler to 1475/// determine (in conjunction with areLoadsFromSameBasePtr) if two loads should 1476/// be scheduled togther. On some targets if two loads are loading from 1477/// addresses in the same cache line, it's better if they are scheduled 1478/// together. This function takes two integers that represent the load offsets 1479/// from the common base address. It returns true if it decides it's desirable 1480/// to schedule the two loads together. "NumLoads" is the number of loads that 1481/// have already been scheduled after Load1. 1482/// 1483/// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched 1484/// is permanently disabled. 1485bool ARMBaseInstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, 1486 int64_t Offset1, int64_t Offset2, 1487 unsigned NumLoads) const { 1488 // Don't worry about Thumb: just ARM and Thumb2. 1489 if (Subtarget.isThumb1Only()) return false; 1490 1491 assert(Offset2 > Offset1); 1492 1493 if ((Offset2 - Offset1) / 8 > 64) 1494 return false; 1495 1496 if (Load1->getMachineOpcode() != Load2->getMachineOpcode()) 1497 return false; // FIXME: overly conservative? 1498 1499 // Four loads in a row should be sufficient. 1500 if (NumLoads >= 3) 1501 return false; 1502 1503 return true; 1504} 1505 1506bool ARMBaseInstrInfo::isSchedulingBoundary(const MachineInstr *MI, 1507 const MachineBasicBlock *MBB, 1508 const MachineFunction &MF) const { 1509 // Debug info is never a scheduling boundary. It's necessary to be explicit 1510 // due to the special treatment of IT instructions below, otherwise a 1511 // dbg_value followed by an IT will result in the IT instruction being 1512 // considered a scheduling hazard, which is wrong. It should be the actual 1513 // instruction preceding the dbg_value instruction(s), just like it is 1514 // when debug info is not present. 1515 if (MI->isDebugValue()) 1516 return false; 1517 1518 // Terminators and labels can't be scheduled around. 1519 if (MI->isTerminator() || MI->isLabel()) 1520 return true; 1521 1522 // Treat the start of the IT block as a scheduling boundary, but schedule 1523 // t2IT along with all instructions following it. 1524 // FIXME: This is a big hammer. But the alternative is to add all potential 1525 // true and anti dependencies to IT block instructions as implicit operands 1526 // to the t2IT instruction. The added compile time and complexity does not 1527 // seem worth it. 1528 MachineBasicBlock::const_iterator I = MI; 1529 // Make sure to skip any dbg_value instructions 1530 while (++I != MBB->end() && I->isDebugValue()) 1531 ; 1532 if (I != MBB->end() && I->getOpcode() == ARM::t2IT) 1533 return true; 1534 1535 // Don't attempt to schedule around any instruction that defines 1536 // a stack-oriented pointer, as it's unlikely to be profitable. This 1537 // saves compile time, because it doesn't require every single 1538 // stack slot reference to depend on the instruction that does the 1539 // modification. 1540 // Calls don't actually change the stack pointer, even if they have imp-defs. 1541 // No ARM calling conventions change the stack pointer. (X86 calling 1542 // conventions sometimes do). 1543 if (!MI->isCall() && MI->definesRegister(ARM::SP)) 1544 return true; 1545 1546 return false; 1547} 1548 1549bool ARMBaseInstrInfo:: 1550isProfitableToIfCvt(MachineBasicBlock &MBB, 1551 unsigned NumCycles, unsigned ExtraPredCycles, 1552 const BranchProbability &Probability) const { 1553 if (!NumCycles) 1554 return false; 1555 1556 // Attempt to estimate the relative costs of predication versus branching. 1557 unsigned UnpredCost = Probability.getNumerator() * NumCycles; 1558 UnpredCost /= Probability.getDenominator(); 1559 UnpredCost += 1; // The branch itself 1560 UnpredCost += Subtarget.getMispredictionPenalty() / 10; 1561 1562 return (NumCycles + ExtraPredCycles) <= UnpredCost; 1563} 1564 1565bool ARMBaseInstrInfo:: 1566isProfitableToIfCvt(MachineBasicBlock &TMBB, 1567 unsigned TCycles, unsigned TExtra, 1568 MachineBasicBlock &FMBB, 1569 unsigned FCycles, unsigned FExtra, 1570 const BranchProbability &Probability) const { 1571 if (!TCycles || !FCycles) 1572 return false; 1573 1574 // Attempt to estimate the relative costs of predication versus branching. 1575 unsigned TUnpredCost = Probability.getNumerator() * TCycles; 1576 TUnpredCost /= Probability.getDenominator(); 1577 1578 uint32_t Comp = Probability.getDenominator() - Probability.getNumerator(); 1579 unsigned FUnpredCost = Comp * FCycles; 1580 FUnpredCost /= Probability.getDenominator(); 1581 1582 unsigned UnpredCost = TUnpredCost + FUnpredCost; 1583 UnpredCost += 1; // The branch itself 1584 UnpredCost += Subtarget.getMispredictionPenalty() / 10; 1585 1586 return (TCycles + FCycles + TExtra + FExtra) <= UnpredCost; 1587} 1588 1589bool 1590ARMBaseInstrInfo::isProfitableToUnpredicate(MachineBasicBlock &TMBB, 1591 MachineBasicBlock &FMBB) const { 1592 // Reduce false anti-dependencies to let Swift's out-of-order execution 1593 // engine do its thing. 1594 return Subtarget.isSwift(); 1595} 1596 1597/// getInstrPredicate - If instruction is predicated, returns its predicate 1598/// condition, otherwise returns AL. It also returns the condition code 1599/// register by reference. 1600ARMCC::CondCodes 1601llvm::getInstrPredicate(const MachineInstr *MI, unsigned &PredReg) { 1602 int PIdx = MI->findFirstPredOperandIdx(); 1603 if (PIdx == -1) { 1604 PredReg = 0; 1605 return ARMCC::AL; 1606 } 1607 1608 PredReg = MI->getOperand(PIdx+1).getReg(); 1609 return (ARMCC::CondCodes)MI->getOperand(PIdx).getImm(); 1610} 1611 1612 1613int llvm::getMatchingCondBranchOpcode(int Opc) { 1614 if (Opc == ARM::B) 1615 return ARM::Bcc; 1616 if (Opc == ARM::tB) 1617 return ARM::tBcc; 1618 if (Opc == ARM::t2B) 1619 return ARM::t2Bcc; 1620 1621 llvm_unreachable("Unknown unconditional branch opcode!"); 1622} 1623 1624/// commuteInstruction - Handle commutable instructions. 1625MachineInstr * 1626ARMBaseInstrInfo::commuteInstruction(MachineInstr *MI, bool NewMI) const { 1627 switch (MI->getOpcode()) { 1628 case ARM::MOVCCr: 1629 case ARM::t2MOVCCr: { 1630 // MOVCC can be commuted by inverting the condition. 1631 unsigned PredReg = 0; 1632 ARMCC::CondCodes CC = getInstrPredicate(MI, PredReg); 1633 // MOVCC AL can't be inverted. Shouldn't happen. 1634 if (CC == ARMCC::AL || PredReg != ARM::CPSR) 1635 return NULL; 1636 MI = TargetInstrInfo::commuteInstruction(MI, NewMI); 1637 if (!MI) 1638 return NULL; 1639 // After swapping the MOVCC operands, also invert the condition. 1640 MI->getOperand(MI->findFirstPredOperandIdx()) 1641 .setImm(ARMCC::getOppositeCondition(CC)); 1642 return MI; 1643 } 1644 } 1645 return TargetInstrInfo::commuteInstruction(MI, NewMI); 1646} 1647 1648/// Identify instructions that can be folded into a MOVCC instruction, and 1649/// return the defining instruction. 1650static MachineInstr *canFoldIntoMOVCC(unsigned Reg, 1651 const MachineRegisterInfo &MRI, 1652 const TargetInstrInfo *TII) { 1653 if (!TargetRegisterInfo::isVirtualRegister(Reg)) 1654 return 0; 1655 if (!MRI.hasOneNonDBGUse(Reg)) 1656 return 0; 1657 MachineInstr *MI = MRI.getVRegDef(Reg); 1658 if (!MI) 1659 return 0; 1660 // MI is folded into the MOVCC by predicating it. 1661 if (!MI->isPredicable()) 1662 return 0; 1663 // Check if MI has any non-dead defs or physreg uses. This also detects 1664 // predicated instructions which will be reading CPSR. 1665 for (unsigned i = 1, e = MI->getNumOperands(); i != e; ++i) { 1666 const MachineOperand &MO = MI->getOperand(i); 1667 // Reject frame index operands, PEI can't handle the predicated pseudos. 1668 if (MO.isFI() || MO.isCPI() || MO.isJTI()) 1669 return 0; 1670 if (!MO.isReg()) 1671 continue; 1672 // MI can't have any tied operands, that would conflict with predication. 1673 if (MO.isTied()) 1674 return 0; 1675 if (TargetRegisterInfo::isPhysicalRegister(MO.getReg())) 1676 return 0; 1677 if (MO.isDef() && !MO.isDead()) 1678 return 0; 1679 } 1680 bool DontMoveAcrossStores = true; 1681 if (!MI->isSafeToMove(TII, /* AliasAnalysis = */ 0, DontMoveAcrossStores)) 1682 return 0; 1683 return MI; 1684} 1685 1686bool ARMBaseInstrInfo::analyzeSelect(const MachineInstr *MI, 1687 SmallVectorImpl<MachineOperand> &Cond, 1688 unsigned &TrueOp, unsigned &FalseOp, 1689 bool &Optimizable) const { 1690 assert((MI->getOpcode() == ARM::MOVCCr || MI->getOpcode() == ARM::t2MOVCCr) && 1691 "Unknown select instruction"); 1692 // MOVCC operands: 1693 // 0: Def. 1694 // 1: True use. 1695 // 2: False use. 1696 // 3: Condition code. 1697 // 4: CPSR use. 1698 TrueOp = 1; 1699 FalseOp = 2; 1700 Cond.push_back(MI->getOperand(3)); 1701 Cond.push_back(MI->getOperand(4)); 1702 // We can always fold a def. 1703 Optimizable = true; 1704 return false; 1705} 1706 1707MachineInstr *ARMBaseInstrInfo::optimizeSelect(MachineInstr *MI, 1708 bool PreferFalse) const { 1709 assert((MI->getOpcode() == ARM::MOVCCr || MI->getOpcode() == ARM::t2MOVCCr) && 1710 "Unknown select instruction"); 1711 const MachineRegisterInfo &MRI = MI->getParent()->getParent()->getRegInfo(); 1712 MachineInstr *DefMI = canFoldIntoMOVCC(MI->getOperand(2).getReg(), MRI, this); 1713 bool Invert = !DefMI; 1714 if (!DefMI) 1715 DefMI = canFoldIntoMOVCC(MI->getOperand(1).getReg(), MRI, this); 1716 if (!DefMI) 1717 return 0; 1718 1719 // Create a new predicated version of DefMI. 1720 // Rfalse is the first use. 1721 MachineInstrBuilder NewMI = BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1722 DefMI->getDesc(), 1723 MI->getOperand(0).getReg()); 1724 1725 // Copy all the DefMI operands, excluding its (null) predicate. 1726 const MCInstrDesc &DefDesc = DefMI->getDesc(); 1727 for (unsigned i = 1, e = DefDesc.getNumOperands(); 1728 i != e && !DefDesc.OpInfo[i].isPredicate(); ++i) 1729 NewMI.addOperand(DefMI->getOperand(i)); 1730 1731 unsigned CondCode = MI->getOperand(3).getImm(); 1732 if (Invert) 1733 NewMI.addImm(ARMCC::getOppositeCondition(ARMCC::CondCodes(CondCode))); 1734 else 1735 NewMI.addImm(CondCode); 1736 NewMI.addOperand(MI->getOperand(4)); 1737 1738 // DefMI is not the -S version that sets CPSR, so add an optional %noreg. 1739 if (NewMI->hasOptionalDef()) 1740 AddDefaultCC(NewMI); 1741 1742 // The output register value when the predicate is false is an implicit 1743 // register operand tied to the first def. 1744 // The tie makes the register allocator ensure the FalseReg is allocated the 1745 // same register as operand 0. 1746 MachineOperand FalseReg = MI->getOperand(Invert ? 2 : 1); 1747 FalseReg.setImplicit(); 1748 NewMI.addOperand(FalseReg); 1749 NewMI->tieOperands(0, NewMI->getNumOperands() - 1); 1750 1751 // The caller will erase MI, but not DefMI. 1752 DefMI->eraseFromParent(); 1753 return NewMI; 1754} 1755 1756/// Map pseudo instructions that imply an 'S' bit onto real opcodes. Whether the 1757/// instruction is encoded with an 'S' bit is determined by the optional CPSR 1758/// def operand. 1759/// 1760/// This will go away once we can teach tblgen how to set the optional CPSR def 1761/// operand itself. 1762struct AddSubFlagsOpcodePair { 1763 uint16_t PseudoOpc; 1764 uint16_t MachineOpc; 1765}; 1766 1767static const AddSubFlagsOpcodePair AddSubFlagsOpcodeMap[] = { 1768 {ARM::ADDSri, ARM::ADDri}, 1769 {ARM::ADDSrr, ARM::ADDrr}, 1770 {ARM::ADDSrsi, ARM::ADDrsi}, 1771 {ARM::ADDSrsr, ARM::ADDrsr}, 1772 1773 {ARM::SUBSri, ARM::SUBri}, 1774 {ARM::SUBSrr, ARM::SUBrr}, 1775 {ARM::SUBSrsi, ARM::SUBrsi}, 1776 {ARM::SUBSrsr, ARM::SUBrsr}, 1777 1778 {ARM::RSBSri, ARM::RSBri}, 1779 {ARM::RSBSrsi, ARM::RSBrsi}, 1780 {ARM::RSBSrsr, ARM::RSBrsr}, 1781 1782 {ARM::t2ADDSri, ARM::t2ADDri}, 1783 {ARM::t2ADDSrr, ARM::t2ADDrr}, 1784 {ARM::t2ADDSrs, ARM::t2ADDrs}, 1785 1786 {ARM::t2SUBSri, ARM::t2SUBri}, 1787 {ARM::t2SUBSrr, ARM::t2SUBrr}, 1788 {ARM::t2SUBSrs, ARM::t2SUBrs}, 1789 1790 {ARM::t2RSBSri, ARM::t2RSBri}, 1791 {ARM::t2RSBSrs, ARM::t2RSBrs}, 1792}; 1793 1794unsigned llvm::convertAddSubFlagsOpcode(unsigned OldOpc) { 1795 for (unsigned i = 0, e = array_lengthof(AddSubFlagsOpcodeMap); i != e; ++i) 1796 if (OldOpc == AddSubFlagsOpcodeMap[i].PseudoOpc) 1797 return AddSubFlagsOpcodeMap[i].MachineOpc; 1798 return 0; 1799} 1800 1801void llvm::emitARMRegPlusImmediate(MachineBasicBlock &MBB, 1802 MachineBasicBlock::iterator &MBBI, DebugLoc dl, 1803 unsigned DestReg, unsigned BaseReg, int NumBytes, 1804 ARMCC::CondCodes Pred, unsigned PredReg, 1805 const ARMBaseInstrInfo &TII, unsigned MIFlags) { 1806 bool isSub = NumBytes < 0; 1807 if (isSub) NumBytes = -NumBytes; 1808 1809 while (NumBytes) { 1810 unsigned RotAmt = ARM_AM::getSOImmValRotate(NumBytes); 1811 unsigned ThisVal = NumBytes & ARM_AM::rotr32(0xFF, RotAmt); 1812 assert(ThisVal && "Didn't extract field correctly"); 1813 1814 // We will handle these bits from offset, clear them. 1815 NumBytes &= ~ThisVal; 1816 1817 assert(ARM_AM::getSOImmVal(ThisVal) != -1 && "Bit extraction didn't work?"); 1818 1819 // Build the new ADD / SUB. 1820 unsigned Opc = isSub ? ARM::SUBri : ARM::ADDri; 1821 BuildMI(MBB, MBBI, dl, TII.get(Opc), DestReg) 1822 .addReg(BaseReg, RegState::Kill).addImm(ThisVal) 1823 .addImm((unsigned)Pred).addReg(PredReg).addReg(0) 1824 .setMIFlags(MIFlags); 1825 BaseReg = DestReg; 1826 } 1827} 1828 1829bool llvm::rewriteARMFrameIndex(MachineInstr &MI, unsigned FrameRegIdx, 1830 unsigned FrameReg, int &Offset, 1831 const ARMBaseInstrInfo &TII) { 1832 unsigned Opcode = MI.getOpcode(); 1833 const MCInstrDesc &Desc = MI.getDesc(); 1834 unsigned AddrMode = (Desc.TSFlags & ARMII::AddrModeMask); 1835 bool isSub = false; 1836 1837 // Memory operands in inline assembly always use AddrMode2. 1838 if (Opcode == ARM::INLINEASM) 1839 AddrMode = ARMII::AddrMode2; 1840 1841 if (Opcode == ARM::ADDri) { 1842 Offset += MI.getOperand(FrameRegIdx+1).getImm(); 1843 if (Offset == 0) { 1844 // Turn it into a move. 1845 MI.setDesc(TII.get(ARM::MOVr)); 1846 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false); 1847 MI.RemoveOperand(FrameRegIdx+1); 1848 Offset = 0; 1849 return true; 1850 } else if (Offset < 0) { 1851 Offset = -Offset; 1852 isSub = true; 1853 MI.setDesc(TII.get(ARM::SUBri)); 1854 } 1855 1856 // Common case: small offset, fits into instruction. 1857 if (ARM_AM::getSOImmVal(Offset) != -1) { 1858 // Replace the FrameIndex with sp / fp 1859 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false); 1860 MI.getOperand(FrameRegIdx+1).ChangeToImmediate(Offset); 1861 Offset = 0; 1862 return true; 1863 } 1864 1865 // Otherwise, pull as much of the immedidate into this ADDri/SUBri 1866 // as possible. 1867 unsigned RotAmt = ARM_AM::getSOImmValRotate(Offset); 1868 unsigned ThisImmVal = Offset & ARM_AM::rotr32(0xFF, RotAmt); 1869 1870 // We will handle these bits from offset, clear them. 1871 Offset &= ~ThisImmVal; 1872 1873 // Get the properly encoded SOImmVal field. 1874 assert(ARM_AM::getSOImmVal(ThisImmVal) != -1 && 1875 "Bit extraction didn't work?"); 1876 MI.getOperand(FrameRegIdx+1).ChangeToImmediate(ThisImmVal); 1877 } else { 1878 unsigned ImmIdx = 0; 1879 int InstrOffs = 0; 1880 unsigned NumBits = 0; 1881 unsigned Scale = 1; 1882 switch (AddrMode) { 1883 case ARMII::AddrMode_i12: { 1884 ImmIdx = FrameRegIdx + 1; 1885 InstrOffs = MI.getOperand(ImmIdx).getImm(); 1886 NumBits = 12; 1887 break; 1888 } 1889 case ARMII::AddrMode2: { 1890 ImmIdx = FrameRegIdx+2; 1891 InstrOffs = ARM_AM::getAM2Offset(MI.getOperand(ImmIdx).getImm()); 1892 if (ARM_AM::getAM2Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub) 1893 InstrOffs *= -1; 1894 NumBits = 12; 1895 break; 1896 } 1897 case ARMII::AddrMode3: { 1898 ImmIdx = FrameRegIdx+2; 1899 InstrOffs = ARM_AM::getAM3Offset(MI.getOperand(ImmIdx).getImm()); 1900 if (ARM_AM::getAM3Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub) 1901 InstrOffs *= -1; 1902 NumBits = 8; 1903 break; 1904 } 1905 case ARMII::AddrMode4: 1906 case ARMII::AddrMode6: 1907 // Can't fold any offset even if it's zero. 1908 return false; 1909 case ARMII::AddrMode5: { 1910 ImmIdx = FrameRegIdx+1; 1911 InstrOffs = ARM_AM::getAM5Offset(MI.getOperand(ImmIdx).getImm()); 1912 if (ARM_AM::getAM5Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub) 1913 InstrOffs *= -1; 1914 NumBits = 8; 1915 Scale = 4; 1916 break; 1917 } 1918 default: 1919 llvm_unreachable("Unsupported addressing mode!"); 1920 } 1921 1922 Offset += InstrOffs * Scale; 1923 assert((Offset & (Scale-1)) == 0 && "Can't encode this offset!"); 1924 if (Offset < 0) { 1925 Offset = -Offset; 1926 isSub = true; 1927 } 1928 1929 // Attempt to fold address comp. if opcode has offset bits 1930 if (NumBits > 0) { 1931 // Common case: small offset, fits into instruction. 1932 MachineOperand &ImmOp = MI.getOperand(ImmIdx); 1933 int ImmedOffset = Offset / Scale; 1934 unsigned Mask = (1 << NumBits) - 1; 1935 if ((unsigned)Offset <= Mask * Scale) { 1936 // Replace the FrameIndex with sp 1937 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false); 1938 // FIXME: When addrmode2 goes away, this will simplify (like the 1939 // T2 version), as the LDR.i12 versions don't need the encoding 1940 // tricks for the offset value. 1941 if (isSub) { 1942 if (AddrMode == ARMII::AddrMode_i12) 1943 ImmedOffset = -ImmedOffset; 1944 else 1945 ImmedOffset |= 1 << NumBits; 1946 } 1947 ImmOp.ChangeToImmediate(ImmedOffset); 1948 Offset = 0; 1949 return true; 1950 } 1951 1952 // Otherwise, it didn't fit. Pull in what we can to simplify the immed. 1953 ImmedOffset = ImmedOffset & Mask; 1954 if (isSub) { 1955 if (AddrMode == ARMII::AddrMode_i12) 1956 ImmedOffset = -ImmedOffset; 1957 else 1958 ImmedOffset |= 1 << NumBits; 1959 } 1960 ImmOp.ChangeToImmediate(ImmedOffset); 1961 Offset &= ~(Mask*Scale); 1962 } 1963 } 1964 1965 Offset = (isSub) ? -Offset : Offset; 1966 return Offset == 0; 1967} 1968 1969/// analyzeCompare - For a comparison instruction, return the source registers 1970/// in SrcReg and SrcReg2 if having two register operands, and the value it 1971/// compares against in CmpValue. Return true if the comparison instruction 1972/// can be analyzed. 1973bool ARMBaseInstrInfo:: 1974analyzeCompare(const MachineInstr *MI, unsigned &SrcReg, unsigned &SrcReg2, 1975 int &CmpMask, int &CmpValue) const { 1976 switch (MI->getOpcode()) { 1977 default: break; 1978 case ARM::CMPri: 1979 case ARM::t2CMPri: 1980 SrcReg = MI->getOperand(0).getReg(); 1981 SrcReg2 = 0; 1982 CmpMask = ~0; 1983 CmpValue = MI->getOperand(1).getImm(); 1984 return true; 1985 case ARM::CMPrr: 1986 case ARM::t2CMPrr: 1987 SrcReg = MI->getOperand(0).getReg(); 1988 SrcReg2 = MI->getOperand(1).getReg(); 1989 CmpMask = ~0; 1990 CmpValue = 0; 1991 return true; 1992 case ARM::TSTri: 1993 case ARM::t2TSTri: 1994 SrcReg = MI->getOperand(0).getReg(); 1995 SrcReg2 = 0; 1996 CmpMask = MI->getOperand(1).getImm(); 1997 CmpValue = 0; 1998 return true; 1999 } 2000 2001 return false; 2002} 2003 2004/// isSuitableForMask - Identify a suitable 'and' instruction that 2005/// operates on the given source register and applies the same mask 2006/// as a 'tst' instruction. Provide a limited look-through for copies. 2007/// When successful, MI will hold the found instruction. 2008static bool isSuitableForMask(MachineInstr *&MI, unsigned SrcReg, 2009 int CmpMask, bool CommonUse) { 2010 switch (MI->getOpcode()) { 2011 case ARM::ANDri: 2012 case ARM::t2ANDri: 2013 if (CmpMask != MI->getOperand(2).getImm()) 2014 return false; 2015 if (SrcReg == MI->getOperand(CommonUse ? 1 : 0).getReg()) 2016 return true; 2017 break; 2018 case ARM::COPY: { 2019 // Walk down one instruction which is potentially an 'and'. 2020 const MachineInstr &Copy = *MI; 2021 MachineBasicBlock::iterator AND( 2022 llvm::next(MachineBasicBlock::iterator(MI))); 2023 if (AND == MI->getParent()->end()) return false; 2024 MI = AND; 2025 return isSuitableForMask(MI, Copy.getOperand(0).getReg(), 2026 CmpMask, true); 2027 } 2028 } 2029 2030 return false; 2031} 2032 2033/// getSwappedCondition - assume the flags are set by MI(a,b), return 2034/// the condition code if we modify the instructions such that flags are 2035/// set by MI(b,a). 2036inline static ARMCC::CondCodes getSwappedCondition(ARMCC::CondCodes CC) { 2037 switch (CC) { 2038 default: return ARMCC::AL; 2039 case ARMCC::EQ: return ARMCC::EQ; 2040 case ARMCC::NE: return ARMCC::NE; 2041 case ARMCC::HS: return ARMCC::LS; 2042 case ARMCC::LO: return ARMCC::HI; 2043 case ARMCC::HI: return ARMCC::LO; 2044 case ARMCC::LS: return ARMCC::HS; 2045 case ARMCC::GE: return ARMCC::LE; 2046 case ARMCC::LT: return ARMCC::GT; 2047 case ARMCC::GT: return ARMCC::LT; 2048 case ARMCC::LE: return ARMCC::GE; 2049 } 2050} 2051 2052/// isRedundantFlagInstr - check whether the first instruction, whose only 2053/// purpose is to update flags, can be made redundant. 2054/// CMPrr can be made redundant by SUBrr if the operands are the same. 2055/// CMPri can be made redundant by SUBri if the operands are the same. 2056/// This function can be extended later on. 2057inline static bool isRedundantFlagInstr(MachineInstr *CmpI, unsigned SrcReg, 2058 unsigned SrcReg2, int ImmValue, 2059 MachineInstr *OI) { 2060 if ((CmpI->getOpcode() == ARM::CMPrr || 2061 CmpI->getOpcode() == ARM::t2CMPrr) && 2062 (OI->getOpcode() == ARM::SUBrr || 2063 OI->getOpcode() == ARM::t2SUBrr) && 2064 ((OI->getOperand(1).getReg() == SrcReg && 2065 OI->getOperand(2).getReg() == SrcReg2) || 2066 (OI->getOperand(1).getReg() == SrcReg2 && 2067 OI->getOperand(2).getReg() == SrcReg))) 2068 return true; 2069 2070 if ((CmpI->getOpcode() == ARM::CMPri || 2071 CmpI->getOpcode() == ARM::t2CMPri) && 2072 (OI->getOpcode() == ARM::SUBri || 2073 OI->getOpcode() == ARM::t2SUBri) && 2074 OI->getOperand(1).getReg() == SrcReg && 2075 OI->getOperand(2).getImm() == ImmValue) 2076 return true; 2077 return false; 2078} 2079 2080/// optimizeCompareInstr - Convert the instruction supplying the argument to the 2081/// comparison into one that sets the zero bit in the flags register; 2082/// Remove a redundant Compare instruction if an earlier instruction can set the 2083/// flags in the same way as Compare. 2084/// E.g. SUBrr(r1,r2) and CMPrr(r1,r2). We also handle the case where two 2085/// operands are swapped: SUBrr(r1,r2) and CMPrr(r2,r1), by updating the 2086/// condition code of instructions which use the flags. 2087bool ARMBaseInstrInfo:: 2088optimizeCompareInstr(MachineInstr *CmpInstr, unsigned SrcReg, unsigned SrcReg2, 2089 int CmpMask, int CmpValue, 2090 const MachineRegisterInfo *MRI) const { 2091 // Get the unique definition of SrcReg. 2092 MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg); 2093 if (!MI) return false; 2094 2095 // Masked compares sometimes use the same register as the corresponding 'and'. 2096 if (CmpMask != ~0) { 2097 if (!isSuitableForMask(MI, SrcReg, CmpMask, false) || isPredicated(MI)) { 2098 MI = 0; 2099 for (MachineRegisterInfo::use_iterator UI = MRI->use_begin(SrcReg), 2100 UE = MRI->use_end(); UI != UE; ++UI) { 2101 if (UI->getParent() != CmpInstr->getParent()) continue; 2102 MachineInstr *PotentialAND = &*UI; 2103 if (!isSuitableForMask(PotentialAND, SrcReg, CmpMask, true) || 2104 isPredicated(PotentialAND)) 2105 continue; 2106 MI = PotentialAND; 2107 break; 2108 } 2109 if (!MI) return false; 2110 } 2111 } 2112 2113 // Get ready to iterate backward from CmpInstr. 2114 MachineBasicBlock::iterator I = CmpInstr, E = MI, 2115 B = CmpInstr->getParent()->begin(); 2116 2117 // Early exit if CmpInstr is at the beginning of the BB. 2118 if (I == B) return false; 2119 2120 // There are two possible candidates which can be changed to set CPSR: 2121 // One is MI, the other is a SUB instruction. 2122 // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1). 2123 // For CMPri(r1, CmpValue), we are looking for SUBri(r1, CmpValue). 2124 MachineInstr *Sub = NULL; 2125 if (SrcReg2 != 0) 2126 // MI is not a candidate for CMPrr. 2127 MI = NULL; 2128 else if (MI->getParent() != CmpInstr->getParent() || CmpValue != 0) { 2129 // Conservatively refuse to convert an instruction which isn't in the same 2130 // BB as the comparison. 2131 // For CMPri, we need to check Sub, thus we can't return here. 2132 if (CmpInstr->getOpcode() == ARM::CMPri || 2133 CmpInstr->getOpcode() == ARM::t2CMPri) 2134 MI = NULL; 2135 else 2136 return false; 2137 } 2138 2139 // Check that CPSR isn't set between the comparison instruction and the one we 2140 // want to change. At the same time, search for Sub. 2141 const TargetRegisterInfo *TRI = &getRegisterInfo(); 2142 --I; 2143 for (; I != E; --I) { 2144 const MachineInstr &Instr = *I; 2145 2146 if (Instr.modifiesRegister(ARM::CPSR, TRI) || 2147 Instr.readsRegister(ARM::CPSR, TRI)) 2148 // This instruction modifies or uses CPSR after the one we want to 2149 // change. We can't do this transformation. 2150 return false; 2151 2152 // Check whether CmpInstr can be made redundant by the current instruction. 2153 if (isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpValue, &*I)) { 2154 Sub = &*I; 2155 break; 2156 } 2157 2158 if (I == B) 2159 // The 'and' is below the comparison instruction. 2160 return false; 2161 } 2162 2163 // Return false if no candidates exist. 2164 if (!MI && !Sub) 2165 return false; 2166 2167 // The single candidate is called MI. 2168 if (!MI) MI = Sub; 2169 2170 // We can't use a predicated instruction - it doesn't always write the flags. 2171 if (isPredicated(MI)) 2172 return false; 2173 2174 switch (MI->getOpcode()) { 2175 default: break; 2176 case ARM::RSBrr: 2177 case ARM::RSBri: 2178 case ARM::RSCrr: 2179 case ARM::RSCri: 2180 case ARM::ADDrr: 2181 case ARM::ADDri: 2182 case ARM::ADCrr: 2183 case ARM::ADCri: 2184 case ARM::SUBrr: 2185 case ARM::SUBri: 2186 case ARM::SBCrr: 2187 case ARM::SBCri: 2188 case ARM::t2RSBri: 2189 case ARM::t2ADDrr: 2190 case ARM::t2ADDri: 2191 case ARM::t2ADCrr: 2192 case ARM::t2ADCri: 2193 case ARM::t2SUBrr: 2194 case ARM::t2SUBri: 2195 case ARM::t2SBCrr: 2196 case ARM::t2SBCri: 2197 case ARM::ANDrr: 2198 case ARM::ANDri: 2199 case ARM::t2ANDrr: 2200 case ARM::t2ANDri: 2201 case ARM::ORRrr: 2202 case ARM::ORRri: 2203 case ARM::t2ORRrr: 2204 case ARM::t2ORRri: 2205 case ARM::EORrr: 2206 case ARM::EORri: 2207 case ARM::t2EORrr: 2208 case ARM::t2EORri: { 2209 // Scan forward for the use of CPSR 2210 // When checking against MI: if it's a conditional code requires 2211 // checking of V bit, then this is not safe to do. 2212 // It is safe to remove CmpInstr if CPSR is redefined or killed. 2213 // If we are done with the basic block, we need to check whether CPSR is 2214 // live-out. 2215 SmallVector<std::pair<MachineOperand*, ARMCC::CondCodes>, 4> 2216 OperandsToUpdate; 2217 bool isSafe = false; 2218 I = CmpInstr; 2219 E = CmpInstr->getParent()->end(); 2220 while (!isSafe && ++I != E) { 2221 const MachineInstr &Instr = *I; 2222 for (unsigned IO = 0, EO = Instr.getNumOperands(); 2223 !isSafe && IO != EO; ++IO) { 2224 const MachineOperand &MO = Instr.getOperand(IO); 2225 if (MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR)) { 2226 isSafe = true; 2227 break; 2228 } 2229 if (!MO.isReg() || MO.getReg() != ARM::CPSR) 2230 continue; 2231 if (MO.isDef()) { 2232 isSafe = true; 2233 break; 2234 } 2235 // Condition code is after the operand before CPSR. 2236 ARMCC::CondCodes CC = (ARMCC::CondCodes)Instr.getOperand(IO-1).getImm(); 2237 if (Sub) { 2238 ARMCC::CondCodes NewCC = getSwappedCondition(CC); 2239 if (NewCC == ARMCC::AL) 2240 return false; 2241 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based 2242 // on CMP needs to be updated to be based on SUB. 2243 // Push the condition code operands to OperandsToUpdate. 2244 // If it is safe to remove CmpInstr, the condition code of these 2245 // operands will be modified. 2246 if (SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 && 2247 Sub->getOperand(2).getReg() == SrcReg) 2248 OperandsToUpdate.push_back(std::make_pair(&((*I).getOperand(IO-1)), 2249 NewCC)); 2250 } 2251 else 2252 switch (CC) { 2253 default: 2254 // CPSR can be used multiple times, we should continue. 2255 break; 2256 case ARMCC::VS: 2257 case ARMCC::VC: 2258 case ARMCC::GE: 2259 case ARMCC::LT: 2260 case ARMCC::GT: 2261 case ARMCC::LE: 2262 return false; 2263 } 2264 } 2265 } 2266 2267 // If CPSR is not killed nor re-defined, we should check whether it is 2268 // live-out. If it is live-out, do not optimize. 2269 if (!isSafe) { 2270 MachineBasicBlock *MBB = CmpInstr->getParent(); 2271 for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(), 2272 SE = MBB->succ_end(); SI != SE; ++SI) 2273 if ((*SI)->isLiveIn(ARM::CPSR)) 2274 return false; 2275 } 2276 2277 // Toggle the optional operand to CPSR. 2278 MI->getOperand(5).setReg(ARM::CPSR); 2279 MI->getOperand(5).setIsDef(true); 2280 assert(!isPredicated(MI) && "Can't use flags from predicated instruction"); 2281 CmpInstr->eraseFromParent(); 2282 2283 // Modify the condition code of operands in OperandsToUpdate. 2284 // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to 2285 // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc. 2286 for (unsigned i = 0, e = OperandsToUpdate.size(); i < e; i++) 2287 OperandsToUpdate[i].first->setImm(OperandsToUpdate[i].second); 2288 return true; 2289 } 2290 } 2291 2292 return false; 2293} 2294 2295bool ARMBaseInstrInfo::FoldImmediate(MachineInstr *UseMI, 2296 MachineInstr *DefMI, unsigned Reg, 2297 MachineRegisterInfo *MRI) const { 2298 // Fold large immediates into add, sub, or, xor. 2299 unsigned DefOpc = DefMI->getOpcode(); 2300 if (DefOpc != ARM::t2MOVi32imm && DefOpc != ARM::MOVi32imm) 2301 return false; 2302 if (!DefMI->getOperand(1).isImm()) 2303 // Could be t2MOVi32imm <ga:xx> 2304 return false; 2305 2306 if (!MRI->hasOneNonDBGUse(Reg)) 2307 return false; 2308 2309 const MCInstrDesc &DefMCID = DefMI->getDesc(); 2310 if (DefMCID.hasOptionalDef()) { 2311 unsigned NumOps = DefMCID.getNumOperands(); 2312 const MachineOperand &MO = DefMI->getOperand(NumOps-1); 2313 if (MO.getReg() == ARM::CPSR && !MO.isDead()) 2314 // If DefMI defines CPSR and it is not dead, it's obviously not safe 2315 // to delete DefMI. 2316 return false; 2317 } 2318 2319 const MCInstrDesc &UseMCID = UseMI->getDesc(); 2320 if (UseMCID.hasOptionalDef()) { 2321 unsigned NumOps = UseMCID.getNumOperands(); 2322 if (UseMI->getOperand(NumOps-1).getReg() == ARM::CPSR) 2323 // If the instruction sets the flag, do not attempt this optimization 2324 // since it may change the semantics of the code. 2325 return false; 2326 } 2327 2328 unsigned UseOpc = UseMI->getOpcode(); 2329 unsigned NewUseOpc = 0; 2330 uint32_t ImmVal = (uint32_t)DefMI->getOperand(1).getImm(); 2331 uint32_t SOImmValV1 = 0, SOImmValV2 = 0; 2332 bool Commute = false; 2333 switch (UseOpc) { 2334 default: return false; 2335 case ARM::SUBrr: 2336 case ARM::ADDrr: 2337 case ARM::ORRrr: 2338 case ARM::EORrr: 2339 case ARM::t2SUBrr: 2340 case ARM::t2ADDrr: 2341 case ARM::t2ORRrr: 2342 case ARM::t2EORrr: { 2343 Commute = UseMI->getOperand(2).getReg() != Reg; 2344 switch (UseOpc) { 2345 default: break; 2346 case ARM::SUBrr: { 2347 if (Commute) 2348 return false; 2349 ImmVal = -ImmVal; 2350 NewUseOpc = ARM::SUBri; 2351 // Fallthrough 2352 } 2353 case ARM::ADDrr: 2354 case ARM::ORRrr: 2355 case ARM::EORrr: { 2356 if (!ARM_AM::isSOImmTwoPartVal(ImmVal)) 2357 return false; 2358 SOImmValV1 = (uint32_t)ARM_AM::getSOImmTwoPartFirst(ImmVal); 2359 SOImmValV2 = (uint32_t)ARM_AM::getSOImmTwoPartSecond(ImmVal); 2360 switch (UseOpc) { 2361 default: break; 2362 case ARM::ADDrr: NewUseOpc = ARM::ADDri; break; 2363 case ARM::ORRrr: NewUseOpc = ARM::ORRri; break; 2364 case ARM::EORrr: NewUseOpc = ARM::EORri; break; 2365 } 2366 break; 2367 } 2368 case ARM::t2SUBrr: { 2369 if (Commute) 2370 return false; 2371 ImmVal = -ImmVal; 2372 NewUseOpc = ARM::t2SUBri; 2373 // Fallthrough 2374 } 2375 case ARM::t2ADDrr: 2376 case ARM::t2ORRrr: 2377 case ARM::t2EORrr: { 2378 if (!ARM_AM::isT2SOImmTwoPartVal(ImmVal)) 2379 return false; 2380 SOImmValV1 = (uint32_t)ARM_AM::getT2SOImmTwoPartFirst(ImmVal); 2381 SOImmValV2 = (uint32_t)ARM_AM::getT2SOImmTwoPartSecond(ImmVal); 2382 switch (UseOpc) { 2383 default: break; 2384 case ARM::t2ADDrr: NewUseOpc = ARM::t2ADDri; break; 2385 case ARM::t2ORRrr: NewUseOpc = ARM::t2ORRri; break; 2386 case ARM::t2EORrr: NewUseOpc = ARM::t2EORri; break; 2387 } 2388 break; 2389 } 2390 } 2391 } 2392 } 2393 2394 unsigned OpIdx = Commute ? 2 : 1; 2395 unsigned Reg1 = UseMI->getOperand(OpIdx).getReg(); 2396 bool isKill = UseMI->getOperand(OpIdx).isKill(); 2397 unsigned NewReg = MRI->createVirtualRegister(MRI->getRegClass(Reg)); 2398 AddDefaultCC(AddDefaultPred(BuildMI(*UseMI->getParent(), 2399 UseMI, UseMI->getDebugLoc(), 2400 get(NewUseOpc), NewReg) 2401 .addReg(Reg1, getKillRegState(isKill)) 2402 .addImm(SOImmValV1))); 2403 UseMI->setDesc(get(NewUseOpc)); 2404 UseMI->getOperand(1).setReg(NewReg); 2405 UseMI->getOperand(1).setIsKill(); 2406 UseMI->getOperand(2).ChangeToImmediate(SOImmValV2); 2407 DefMI->eraseFromParent(); 2408 return true; 2409} 2410 2411static unsigned getNumMicroOpsSwiftLdSt(const InstrItineraryData *ItinData, 2412 const MachineInstr *MI) { 2413 switch (MI->getOpcode()) { 2414 default: { 2415 const MCInstrDesc &Desc = MI->getDesc(); 2416 int UOps = ItinData->getNumMicroOps(Desc.getSchedClass()); 2417 assert(UOps >= 0 && "bad # UOps"); 2418 return UOps; 2419 } 2420 2421 case ARM::LDRrs: 2422 case ARM::LDRBrs: 2423 case ARM::STRrs: 2424 case ARM::STRBrs: { 2425 unsigned ShOpVal = MI->getOperand(3).getImm(); 2426 bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub; 2427 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 2428 if (!isSub && 2429 (ShImm == 0 || 2430 ((ShImm == 1 || ShImm == 2 || ShImm == 3) && 2431 ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))) 2432 return 1; 2433 return 2; 2434 } 2435 2436 case ARM::LDRH: 2437 case ARM::STRH: { 2438 if (!MI->getOperand(2).getReg()) 2439 return 1; 2440 2441 unsigned ShOpVal = MI->getOperand(3).getImm(); 2442 bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub; 2443 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 2444 if (!isSub && 2445 (ShImm == 0 || 2446 ((ShImm == 1 || ShImm == 2 || ShImm == 3) && 2447 ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))) 2448 return 1; 2449 return 2; 2450 } 2451 2452 case ARM::LDRSB: 2453 case ARM::LDRSH: 2454 return (ARM_AM::getAM3Op(MI->getOperand(3).getImm()) == ARM_AM::sub) ? 3:2; 2455 2456 case ARM::LDRSB_POST: 2457 case ARM::LDRSH_POST: { 2458 unsigned Rt = MI->getOperand(0).getReg(); 2459 unsigned Rm = MI->getOperand(3).getReg(); 2460 return (Rt == Rm) ? 4 : 3; 2461 } 2462 2463 case ARM::LDR_PRE_REG: 2464 case ARM::LDRB_PRE_REG: { 2465 unsigned Rt = MI->getOperand(0).getReg(); 2466 unsigned Rm = MI->getOperand(3).getReg(); 2467 if (Rt == Rm) 2468 return 3; 2469 unsigned ShOpVal = MI->getOperand(4).getImm(); 2470 bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub; 2471 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 2472 if (!isSub && 2473 (ShImm == 0 || 2474 ((ShImm == 1 || ShImm == 2 || ShImm == 3) && 2475 ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))) 2476 return 2; 2477 return 3; 2478 } 2479 2480 case ARM::STR_PRE_REG: 2481 case ARM::STRB_PRE_REG: { 2482 unsigned ShOpVal = MI->getOperand(4).getImm(); 2483 bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub; 2484 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 2485 if (!isSub && 2486 (ShImm == 0 || 2487 ((ShImm == 1 || ShImm == 2 || ShImm == 3) && 2488 ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))) 2489 return 2; 2490 return 3; 2491 } 2492 2493 case ARM::LDRH_PRE: 2494 case ARM::STRH_PRE: { 2495 unsigned Rt = MI->getOperand(0).getReg(); 2496 unsigned Rm = MI->getOperand(3).getReg(); 2497 if (!Rm) 2498 return 2; 2499 if (Rt == Rm) 2500 return 3; 2501 return (ARM_AM::getAM3Op(MI->getOperand(4).getImm()) == ARM_AM::sub) 2502 ? 3 : 2; 2503 } 2504 2505 case ARM::LDR_POST_REG: 2506 case ARM::LDRB_POST_REG: 2507 case ARM::LDRH_POST: { 2508 unsigned Rt = MI->getOperand(0).getReg(); 2509 unsigned Rm = MI->getOperand(3).getReg(); 2510 return (Rt == Rm) ? 3 : 2; 2511 } 2512 2513 case ARM::LDR_PRE_IMM: 2514 case ARM::LDRB_PRE_IMM: 2515 case ARM::LDR_POST_IMM: 2516 case ARM::LDRB_POST_IMM: 2517 case ARM::STRB_POST_IMM: 2518 case ARM::STRB_POST_REG: 2519 case ARM::STRB_PRE_IMM: 2520 case ARM::STRH_POST: 2521 case ARM::STR_POST_IMM: 2522 case ARM::STR_POST_REG: 2523 case ARM::STR_PRE_IMM: 2524 return 2; 2525 2526 case ARM::LDRSB_PRE: 2527 case ARM::LDRSH_PRE: { 2528 unsigned Rm = MI->getOperand(3).getReg(); 2529 if (Rm == 0) 2530 return 3; 2531 unsigned Rt = MI->getOperand(0).getReg(); 2532 if (Rt == Rm) 2533 return 4; 2534 unsigned ShOpVal = MI->getOperand(4).getImm(); 2535 bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub; 2536 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 2537 if (!isSub && 2538 (ShImm == 0 || 2539 ((ShImm == 1 || ShImm == 2 || ShImm == 3) && 2540 ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))) 2541 return 3; 2542 return 4; 2543 } 2544 2545 case ARM::LDRD: { 2546 unsigned Rt = MI->getOperand(0).getReg(); 2547 unsigned Rn = MI->getOperand(2).getReg(); 2548 unsigned Rm = MI->getOperand(3).getReg(); 2549 if (Rm) 2550 return (ARM_AM::getAM3Op(MI->getOperand(4).getImm()) == ARM_AM::sub) ?4:3; 2551 return (Rt == Rn) ? 3 : 2; 2552 } 2553 2554 case ARM::STRD: { 2555 unsigned Rm = MI->getOperand(3).getReg(); 2556 if (Rm) 2557 return (ARM_AM::getAM3Op(MI->getOperand(4).getImm()) == ARM_AM::sub) ?4:3; 2558 return 2; 2559 } 2560 2561 case ARM::LDRD_POST: 2562 case ARM::t2LDRD_POST: 2563 return 3; 2564 2565 case ARM::STRD_POST: 2566 case ARM::t2STRD_POST: 2567 return 4; 2568 2569 case ARM::LDRD_PRE: { 2570 unsigned Rt = MI->getOperand(0).getReg(); 2571 unsigned Rn = MI->getOperand(3).getReg(); 2572 unsigned Rm = MI->getOperand(4).getReg(); 2573 if (Rm) 2574 return (ARM_AM::getAM3Op(MI->getOperand(5).getImm()) == ARM_AM::sub) ?5:4; 2575 return (Rt == Rn) ? 4 : 3; 2576 } 2577 2578 case ARM::t2LDRD_PRE: { 2579 unsigned Rt = MI->getOperand(0).getReg(); 2580 unsigned Rn = MI->getOperand(3).getReg(); 2581 return (Rt == Rn) ? 4 : 3; 2582 } 2583 2584 case ARM::STRD_PRE: { 2585 unsigned Rm = MI->getOperand(4).getReg(); 2586 if (Rm) 2587 return (ARM_AM::getAM3Op(MI->getOperand(5).getImm()) == ARM_AM::sub) ?5:4; 2588 return 3; 2589 } 2590 2591 case ARM::t2STRD_PRE: 2592 return 3; 2593 2594 case ARM::t2LDR_POST: 2595 case ARM::t2LDRB_POST: 2596 case ARM::t2LDRB_PRE: 2597 case ARM::t2LDRSBi12: 2598 case ARM::t2LDRSBi8: 2599 case ARM::t2LDRSBpci: 2600 case ARM::t2LDRSBs: 2601 case ARM::t2LDRH_POST: 2602 case ARM::t2LDRH_PRE: 2603 case ARM::t2LDRSBT: 2604 case ARM::t2LDRSB_POST: 2605 case ARM::t2LDRSB_PRE: 2606 case ARM::t2LDRSH_POST: 2607 case ARM::t2LDRSH_PRE: 2608 case ARM::t2LDRSHi12: 2609 case ARM::t2LDRSHi8: 2610 case ARM::t2LDRSHpci: 2611 case ARM::t2LDRSHs: 2612 return 2; 2613 2614 case ARM::t2LDRDi8: { 2615 unsigned Rt = MI->getOperand(0).getReg(); 2616 unsigned Rn = MI->getOperand(2).getReg(); 2617 return (Rt == Rn) ? 3 : 2; 2618 } 2619 2620 case ARM::t2STRB_POST: 2621 case ARM::t2STRB_PRE: 2622 case ARM::t2STRBs: 2623 case ARM::t2STRDi8: 2624 case ARM::t2STRH_POST: 2625 case ARM::t2STRH_PRE: 2626 case ARM::t2STRHs: 2627 case ARM::t2STR_POST: 2628 case ARM::t2STR_PRE: 2629 case ARM::t2STRs: 2630 return 2; 2631 } 2632} 2633 2634// Return the number of 32-bit words loaded by LDM or stored by STM. If this 2635// can't be easily determined return 0 (missing MachineMemOperand). 2636// 2637// FIXME: The current MachineInstr design does not support relying on machine 2638// mem operands to determine the width of a memory access. Instead, we expect 2639// the target to provide this information based on the instruction opcode and 2640// operands. However, using MachineMemOperand is a the best solution now for 2641// two reasons: 2642// 2643// 1) getNumMicroOps tries to infer LDM memory width from the total number of MI 2644// operands. This is much more dangerous than using the MachineMemOperand 2645// sizes because CodeGen passes can insert/remove optional machine operands. In 2646// fact, it's totally incorrect for preRA passes and appears to be wrong for 2647// postRA passes as well. 2648// 2649// 2) getNumLDMAddresses is only used by the scheduling machine model and any 2650// machine model that calls this should handle the unknown (zero size) case. 2651// 2652// Long term, we should require a target hook that verifies MachineMemOperand 2653// sizes during MC lowering. That target hook should be local to MC lowering 2654// because we can't ensure that it is aware of other MI forms. Doing this will 2655// ensure that MachineMemOperands are correctly propagated through all passes. 2656unsigned ARMBaseInstrInfo::getNumLDMAddresses(const MachineInstr *MI) const { 2657 unsigned Size = 0; 2658 for (MachineInstr::mmo_iterator I = MI->memoperands_begin(), 2659 E = MI->memoperands_end(); I != E; ++I) { 2660 Size += (*I)->getSize(); 2661 } 2662 return Size / 4; 2663} 2664 2665unsigned 2666ARMBaseInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData, 2667 const MachineInstr *MI) const { 2668 if (!ItinData || ItinData->isEmpty()) 2669 return 1; 2670 2671 const MCInstrDesc &Desc = MI->getDesc(); 2672 unsigned Class = Desc.getSchedClass(); 2673 int ItinUOps = ItinData->getNumMicroOps(Class); 2674 if (ItinUOps >= 0) { 2675 if (Subtarget.isSwift() && (Desc.mayLoad() || Desc.mayStore())) 2676 return getNumMicroOpsSwiftLdSt(ItinData, MI); 2677 2678 return ItinUOps; 2679 } 2680 2681 unsigned Opc = MI->getOpcode(); 2682 switch (Opc) { 2683 default: 2684 llvm_unreachable("Unexpected multi-uops instruction!"); 2685 case ARM::VLDMQIA: 2686 case ARM::VSTMQIA: 2687 return 2; 2688 2689 // The number of uOps for load / store multiple are determined by the number 2690 // registers. 2691 // 2692 // On Cortex-A8, each pair of register loads / stores can be scheduled on the 2693 // same cycle. The scheduling for the first load / store must be done 2694 // separately by assuming the address is not 64-bit aligned. 2695 // 2696 // On Cortex-A9, the formula is simply (#reg / 2) + (#reg % 2). If the address 2697 // is not 64-bit aligned, then AGU would take an extra cycle. For VFP / NEON 2698 // load / store multiple, the formula is (#reg / 2) + (#reg % 2) + 1. 2699 case ARM::VLDMDIA: 2700 case ARM::VLDMDIA_UPD: 2701 case ARM::VLDMDDB_UPD: 2702 case ARM::VLDMSIA: 2703 case ARM::VLDMSIA_UPD: 2704 case ARM::VLDMSDB_UPD: 2705 case ARM::VSTMDIA: 2706 case ARM::VSTMDIA_UPD: 2707 case ARM::VSTMDDB_UPD: 2708 case ARM::VSTMSIA: 2709 case ARM::VSTMSIA_UPD: 2710 case ARM::VSTMSDB_UPD: { 2711 unsigned NumRegs = MI->getNumOperands() - Desc.getNumOperands(); 2712 return (NumRegs / 2) + (NumRegs % 2) + 1; 2713 } 2714 2715 case ARM::LDMIA_RET: 2716 case ARM::LDMIA: 2717 case ARM::LDMDA: 2718 case ARM::LDMDB: 2719 case ARM::LDMIB: 2720 case ARM::LDMIA_UPD: 2721 case ARM::LDMDA_UPD: 2722 case ARM::LDMDB_UPD: 2723 case ARM::LDMIB_UPD: 2724 case ARM::STMIA: 2725 case ARM::STMDA: 2726 case ARM::STMDB: 2727 case ARM::STMIB: 2728 case ARM::STMIA_UPD: 2729 case ARM::STMDA_UPD: 2730 case ARM::STMDB_UPD: 2731 case ARM::STMIB_UPD: 2732 case ARM::tLDMIA: 2733 case ARM::tLDMIA_UPD: 2734 case ARM::tSTMIA_UPD: 2735 case ARM::tPOP_RET: 2736 case ARM::tPOP: 2737 case ARM::tPUSH: 2738 case ARM::t2LDMIA_RET: 2739 case ARM::t2LDMIA: 2740 case ARM::t2LDMDB: 2741 case ARM::t2LDMIA_UPD: 2742 case ARM::t2LDMDB_UPD: 2743 case ARM::t2STMIA: 2744 case ARM::t2STMDB: 2745 case ARM::t2STMIA_UPD: 2746 case ARM::t2STMDB_UPD: { 2747 unsigned NumRegs = MI->getNumOperands() - Desc.getNumOperands() + 1; 2748 if (Subtarget.isSwift()) { 2749 int UOps = 1 + NumRegs; // One for address computation, one for each ld / st. 2750 switch (Opc) { 2751 default: break; 2752 case ARM::VLDMDIA_UPD: 2753 case ARM::VLDMDDB_UPD: 2754 case ARM::VLDMSIA_UPD: 2755 case ARM::VLDMSDB_UPD: 2756 case ARM::VSTMDIA_UPD: 2757 case ARM::VSTMDDB_UPD: 2758 case ARM::VSTMSIA_UPD: 2759 case ARM::VSTMSDB_UPD: 2760 case ARM::LDMIA_UPD: 2761 case ARM::LDMDA_UPD: 2762 case ARM::LDMDB_UPD: 2763 case ARM::LDMIB_UPD: 2764 case ARM::STMIA_UPD: 2765 case ARM::STMDA_UPD: 2766 case ARM::STMDB_UPD: 2767 case ARM::STMIB_UPD: 2768 case ARM::tLDMIA_UPD: 2769 case ARM::tSTMIA_UPD: 2770 case ARM::t2LDMIA_UPD: 2771 case ARM::t2LDMDB_UPD: 2772 case ARM::t2STMIA_UPD: 2773 case ARM::t2STMDB_UPD: 2774 ++UOps; // One for base register writeback. 2775 break; 2776 case ARM::LDMIA_RET: 2777 case ARM::tPOP_RET: 2778 case ARM::t2LDMIA_RET: 2779 UOps += 2; // One for base reg wb, one for write to pc. 2780 break; 2781 } 2782 return UOps; 2783 } else if (Subtarget.isCortexA8()) { 2784 if (NumRegs < 4) 2785 return 2; 2786 // 4 registers would be issued: 2, 2. 2787 // 5 registers would be issued: 2, 2, 1. 2788 int A8UOps = (NumRegs / 2); 2789 if (NumRegs % 2) 2790 ++A8UOps; 2791 return A8UOps; 2792 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) { 2793 int A9UOps = (NumRegs / 2); 2794 // If there are odd number of registers or if it's not 64-bit aligned, 2795 // then it takes an extra AGU (Address Generation Unit) cycle. 2796 if ((NumRegs % 2) || 2797 !MI->hasOneMemOperand() || 2798 (*MI->memoperands_begin())->getAlignment() < 8) 2799 ++A9UOps; 2800 return A9UOps; 2801 } else { 2802 // Assume the worst. 2803 return NumRegs; 2804 } 2805 } 2806 } 2807} 2808 2809int 2810ARMBaseInstrInfo::getVLDMDefCycle(const InstrItineraryData *ItinData, 2811 const MCInstrDesc &DefMCID, 2812 unsigned DefClass, 2813 unsigned DefIdx, unsigned DefAlign) const { 2814 int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1; 2815 if (RegNo <= 0) 2816 // Def is the address writeback. 2817 return ItinData->getOperandCycle(DefClass, DefIdx); 2818 2819 int DefCycle; 2820 if (Subtarget.isCortexA8()) { 2821 // (regno / 2) + (regno % 2) + 1 2822 DefCycle = RegNo / 2 + 1; 2823 if (RegNo % 2) 2824 ++DefCycle; 2825 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) { 2826 DefCycle = RegNo; 2827 bool isSLoad = false; 2828 2829 switch (DefMCID.getOpcode()) { 2830 default: break; 2831 case ARM::VLDMSIA: 2832 case ARM::VLDMSIA_UPD: 2833 case ARM::VLDMSDB_UPD: 2834 isSLoad = true; 2835 break; 2836 } 2837 2838 // If there are odd number of 'S' registers or if it's not 64-bit aligned, 2839 // then it takes an extra cycle. 2840 if ((isSLoad && (RegNo % 2)) || DefAlign < 8) 2841 ++DefCycle; 2842 } else { 2843 // Assume the worst. 2844 DefCycle = RegNo + 2; 2845 } 2846 2847 return DefCycle; 2848} 2849 2850int 2851ARMBaseInstrInfo::getLDMDefCycle(const InstrItineraryData *ItinData, 2852 const MCInstrDesc &DefMCID, 2853 unsigned DefClass, 2854 unsigned DefIdx, unsigned DefAlign) const { 2855 int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1; 2856 if (RegNo <= 0) 2857 // Def is the address writeback. 2858 return ItinData->getOperandCycle(DefClass, DefIdx); 2859 2860 int DefCycle; 2861 if (Subtarget.isCortexA8()) { 2862 // 4 registers would be issued: 1, 2, 1. 2863 // 5 registers would be issued: 1, 2, 2. 2864 DefCycle = RegNo / 2; 2865 if (DefCycle < 1) 2866 DefCycle = 1; 2867 // Result latency is issue cycle + 2: E2. 2868 DefCycle += 2; 2869 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) { 2870 DefCycle = (RegNo / 2); 2871 // If there are odd number of registers or if it's not 64-bit aligned, 2872 // then it takes an extra AGU (Address Generation Unit) cycle. 2873 if ((RegNo % 2) || DefAlign < 8) 2874 ++DefCycle; 2875 // Result latency is AGU cycles + 2. 2876 DefCycle += 2; 2877 } else { 2878 // Assume the worst. 2879 DefCycle = RegNo + 2; 2880 } 2881 2882 return DefCycle; 2883} 2884 2885int 2886ARMBaseInstrInfo::getVSTMUseCycle(const InstrItineraryData *ItinData, 2887 const MCInstrDesc &UseMCID, 2888 unsigned UseClass, 2889 unsigned UseIdx, unsigned UseAlign) const { 2890 int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1; 2891 if (RegNo <= 0) 2892 return ItinData->getOperandCycle(UseClass, UseIdx); 2893 2894 int UseCycle; 2895 if (Subtarget.isCortexA8()) { 2896 // (regno / 2) + (regno % 2) + 1 2897 UseCycle = RegNo / 2 + 1; 2898 if (RegNo % 2) 2899 ++UseCycle; 2900 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) { 2901 UseCycle = RegNo; 2902 bool isSStore = false; 2903 2904 switch (UseMCID.getOpcode()) { 2905 default: break; 2906 case ARM::VSTMSIA: 2907 case ARM::VSTMSIA_UPD: 2908 case ARM::VSTMSDB_UPD: 2909 isSStore = true; 2910 break; 2911 } 2912 2913 // If there are odd number of 'S' registers or if it's not 64-bit aligned, 2914 // then it takes an extra cycle. 2915 if ((isSStore && (RegNo % 2)) || UseAlign < 8) 2916 ++UseCycle; 2917 } else { 2918 // Assume the worst. 2919 UseCycle = RegNo + 2; 2920 } 2921 2922 return UseCycle; 2923} 2924 2925int 2926ARMBaseInstrInfo::getSTMUseCycle(const InstrItineraryData *ItinData, 2927 const MCInstrDesc &UseMCID, 2928 unsigned UseClass, 2929 unsigned UseIdx, unsigned UseAlign) const { 2930 int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1; 2931 if (RegNo <= 0) 2932 return ItinData->getOperandCycle(UseClass, UseIdx); 2933 2934 int UseCycle; 2935 if (Subtarget.isCortexA8()) { 2936 UseCycle = RegNo / 2; 2937 if (UseCycle < 2) 2938 UseCycle = 2; 2939 // Read in E3. 2940 UseCycle += 2; 2941 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) { 2942 UseCycle = (RegNo / 2); 2943 // If there are odd number of registers or if it's not 64-bit aligned, 2944 // then it takes an extra AGU (Address Generation Unit) cycle. 2945 if ((RegNo % 2) || UseAlign < 8) 2946 ++UseCycle; 2947 } else { 2948 // Assume the worst. 2949 UseCycle = 1; 2950 } 2951 return UseCycle; 2952} 2953 2954int 2955ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, 2956 const MCInstrDesc &DefMCID, 2957 unsigned DefIdx, unsigned DefAlign, 2958 const MCInstrDesc &UseMCID, 2959 unsigned UseIdx, unsigned UseAlign) const { 2960 unsigned DefClass = DefMCID.getSchedClass(); 2961 unsigned UseClass = UseMCID.getSchedClass(); 2962 2963 if (DefIdx < DefMCID.getNumDefs() && UseIdx < UseMCID.getNumOperands()) 2964 return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx); 2965 2966 // This may be a def / use of a variable_ops instruction, the operand 2967 // latency might be determinable dynamically. Let the target try to 2968 // figure it out. 2969 int DefCycle = -1; 2970 bool LdmBypass = false; 2971 switch (DefMCID.getOpcode()) { 2972 default: 2973 DefCycle = ItinData->getOperandCycle(DefClass, DefIdx); 2974 break; 2975 2976 case ARM::VLDMDIA: 2977 case ARM::VLDMDIA_UPD: 2978 case ARM::VLDMDDB_UPD: 2979 case ARM::VLDMSIA: 2980 case ARM::VLDMSIA_UPD: 2981 case ARM::VLDMSDB_UPD: 2982 DefCycle = getVLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign); 2983 break; 2984 2985 case ARM::LDMIA_RET: 2986 case ARM::LDMIA: 2987 case ARM::LDMDA: 2988 case ARM::LDMDB: 2989 case ARM::LDMIB: 2990 case ARM::LDMIA_UPD: 2991 case ARM::LDMDA_UPD: 2992 case ARM::LDMDB_UPD: 2993 case ARM::LDMIB_UPD: 2994 case ARM::tLDMIA: 2995 case ARM::tLDMIA_UPD: 2996 case ARM::tPUSH: 2997 case ARM::t2LDMIA_RET: 2998 case ARM::t2LDMIA: 2999 case ARM::t2LDMDB: 3000 case ARM::t2LDMIA_UPD: 3001 case ARM::t2LDMDB_UPD: 3002 LdmBypass = 1; 3003 DefCycle = getLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign); 3004 break; 3005 } 3006 3007 if (DefCycle == -1) 3008 // We can't seem to determine the result latency of the def, assume it's 2. 3009 DefCycle = 2; 3010 3011 int UseCycle = -1; 3012 switch (UseMCID.getOpcode()) { 3013 default: 3014 UseCycle = ItinData->getOperandCycle(UseClass, UseIdx); 3015 break; 3016 3017 case ARM::VSTMDIA: 3018 case ARM::VSTMDIA_UPD: 3019 case ARM::VSTMDDB_UPD: 3020 case ARM::VSTMSIA: 3021 case ARM::VSTMSIA_UPD: 3022 case ARM::VSTMSDB_UPD: 3023 UseCycle = getVSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign); 3024 break; 3025 3026 case ARM::STMIA: 3027 case ARM::STMDA: 3028 case ARM::STMDB: 3029 case ARM::STMIB: 3030 case ARM::STMIA_UPD: 3031 case ARM::STMDA_UPD: 3032 case ARM::STMDB_UPD: 3033 case ARM::STMIB_UPD: 3034 case ARM::tSTMIA_UPD: 3035 case ARM::tPOP_RET: 3036 case ARM::tPOP: 3037 case ARM::t2STMIA: 3038 case ARM::t2STMDB: 3039 case ARM::t2STMIA_UPD: 3040 case ARM::t2STMDB_UPD: 3041 UseCycle = getSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign); 3042 break; 3043 } 3044 3045 if (UseCycle == -1) 3046 // Assume it's read in the first stage. 3047 UseCycle = 1; 3048 3049 UseCycle = DefCycle - UseCycle + 1; 3050 if (UseCycle > 0) { 3051 if (LdmBypass) { 3052 // It's a variable_ops instruction so we can't use DefIdx here. Just use 3053 // first def operand. 3054 if (ItinData->hasPipelineForwarding(DefClass, DefMCID.getNumOperands()-1, 3055 UseClass, UseIdx)) 3056 --UseCycle; 3057 } else if (ItinData->hasPipelineForwarding(DefClass, DefIdx, 3058 UseClass, UseIdx)) { 3059 --UseCycle; 3060 } 3061 } 3062 3063 return UseCycle; 3064} 3065 3066static const MachineInstr *getBundledDefMI(const TargetRegisterInfo *TRI, 3067 const MachineInstr *MI, unsigned Reg, 3068 unsigned &DefIdx, unsigned &Dist) { 3069 Dist = 0; 3070 3071 MachineBasicBlock::const_iterator I = MI; ++I; 3072 MachineBasicBlock::const_instr_iterator II = 3073 llvm::prior(I.getInstrIterator()); 3074 assert(II->isInsideBundle() && "Empty bundle?"); 3075 3076 int Idx = -1; 3077 while (II->isInsideBundle()) { 3078 Idx = II->findRegisterDefOperandIdx(Reg, false, true, TRI); 3079 if (Idx != -1) 3080 break; 3081 --II; 3082 ++Dist; 3083 } 3084 3085 assert(Idx != -1 && "Cannot find bundled definition!"); 3086 DefIdx = Idx; 3087 return II; 3088} 3089 3090static const MachineInstr *getBundledUseMI(const TargetRegisterInfo *TRI, 3091 const MachineInstr *MI, unsigned Reg, 3092 unsigned &UseIdx, unsigned &Dist) { 3093 Dist = 0; 3094 3095 MachineBasicBlock::const_instr_iterator II = MI; ++II; 3096 assert(II->isInsideBundle() && "Empty bundle?"); 3097 MachineBasicBlock::const_instr_iterator E = MI->getParent()->instr_end(); 3098 3099 // FIXME: This doesn't properly handle multiple uses. 3100 int Idx = -1; 3101 while (II != E && II->isInsideBundle()) { 3102 Idx = II->findRegisterUseOperandIdx(Reg, false, TRI); 3103 if (Idx != -1) 3104 break; 3105 if (II->getOpcode() != ARM::t2IT) 3106 ++Dist; 3107 ++II; 3108 } 3109 3110 if (Idx == -1) { 3111 Dist = 0; 3112 return 0; 3113 } 3114 3115 UseIdx = Idx; 3116 return II; 3117} 3118 3119/// Return the number of cycles to add to (or subtract from) the static 3120/// itinerary based on the def opcode and alignment. The caller will ensure that 3121/// adjusted latency is at least one cycle. 3122static int adjustDefLatency(const ARMSubtarget &Subtarget, 3123 const MachineInstr *DefMI, 3124 const MCInstrDesc *DefMCID, unsigned DefAlign) { 3125 int Adjust = 0; 3126 if (Subtarget.isCortexA8() || Subtarget.isLikeA9()) { 3127 // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2] 3128 // variants are one cycle cheaper. 3129 switch (DefMCID->getOpcode()) { 3130 default: break; 3131 case ARM::LDRrs: 3132 case ARM::LDRBrs: { 3133 unsigned ShOpVal = DefMI->getOperand(3).getImm(); 3134 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 3135 if (ShImm == 0 || 3136 (ShImm == 2 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)) 3137 --Adjust; 3138 break; 3139 } 3140 case ARM::t2LDRs: 3141 case ARM::t2LDRBs: 3142 case ARM::t2LDRHs: 3143 case ARM::t2LDRSHs: { 3144 // Thumb2 mode: lsl only. 3145 unsigned ShAmt = DefMI->getOperand(3).getImm(); 3146 if (ShAmt == 0 || ShAmt == 2) 3147 --Adjust; 3148 break; 3149 } 3150 } 3151 } else if (Subtarget.isSwift()) { 3152 // FIXME: Properly handle all of the latency adjustments for address 3153 // writeback. 3154 switch (DefMCID->getOpcode()) { 3155 default: break; 3156 case ARM::LDRrs: 3157 case ARM::LDRBrs: { 3158 unsigned ShOpVal = DefMI->getOperand(3).getImm(); 3159 bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub; 3160 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 3161 if (!isSub && 3162 (ShImm == 0 || 3163 ((ShImm == 1 || ShImm == 2 || ShImm == 3) && 3164 ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))) 3165 Adjust -= 2; 3166 else if (!isSub && 3167 ShImm == 1 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsr) 3168 --Adjust; 3169 break; 3170 } 3171 case ARM::t2LDRs: 3172 case ARM::t2LDRBs: 3173 case ARM::t2LDRHs: 3174 case ARM::t2LDRSHs: { 3175 // Thumb2 mode: lsl only. 3176 unsigned ShAmt = DefMI->getOperand(3).getImm(); 3177 if (ShAmt == 0 || ShAmt == 1 || ShAmt == 2 || ShAmt == 3) 3178 Adjust -= 2; 3179 break; 3180 } 3181 } 3182 } 3183 3184 if (DefAlign < 8 && Subtarget.isLikeA9()) { 3185 switch (DefMCID->getOpcode()) { 3186 default: break; 3187 case ARM::VLD1q8: 3188 case ARM::VLD1q16: 3189 case ARM::VLD1q32: 3190 case ARM::VLD1q64: 3191 case ARM::VLD1q8wb_fixed: 3192 case ARM::VLD1q16wb_fixed: 3193 case ARM::VLD1q32wb_fixed: 3194 case ARM::VLD1q64wb_fixed: 3195 case ARM::VLD1q8wb_register: 3196 case ARM::VLD1q16wb_register: 3197 case ARM::VLD1q32wb_register: 3198 case ARM::VLD1q64wb_register: 3199 case ARM::VLD2d8: 3200 case ARM::VLD2d16: 3201 case ARM::VLD2d32: 3202 case ARM::VLD2q8: 3203 case ARM::VLD2q16: 3204 case ARM::VLD2q32: 3205 case ARM::VLD2d8wb_fixed: 3206 case ARM::VLD2d16wb_fixed: 3207 case ARM::VLD2d32wb_fixed: 3208 case ARM::VLD2q8wb_fixed: 3209 case ARM::VLD2q16wb_fixed: 3210 case ARM::VLD2q32wb_fixed: 3211 case ARM::VLD2d8wb_register: 3212 case ARM::VLD2d16wb_register: 3213 case ARM::VLD2d32wb_register: 3214 case ARM::VLD2q8wb_register: 3215 case ARM::VLD2q16wb_register: 3216 case ARM::VLD2q32wb_register: 3217 case ARM::VLD3d8: 3218 case ARM::VLD3d16: 3219 case ARM::VLD3d32: 3220 case ARM::VLD1d64T: 3221 case ARM::VLD3d8_UPD: 3222 case ARM::VLD3d16_UPD: 3223 case ARM::VLD3d32_UPD: 3224 case ARM::VLD1d64Twb_fixed: 3225 case ARM::VLD1d64Twb_register: 3226 case ARM::VLD3q8_UPD: 3227 case ARM::VLD3q16_UPD: 3228 case ARM::VLD3q32_UPD: 3229 case ARM::VLD4d8: 3230 case ARM::VLD4d16: 3231 case ARM::VLD4d32: 3232 case ARM::VLD1d64Q: 3233 case ARM::VLD4d8_UPD: 3234 case ARM::VLD4d16_UPD: 3235 case ARM::VLD4d32_UPD: 3236 case ARM::VLD1d64Qwb_fixed: 3237 case ARM::VLD1d64Qwb_register: 3238 case ARM::VLD4q8_UPD: 3239 case ARM::VLD4q16_UPD: 3240 case ARM::VLD4q32_UPD: 3241 case ARM::VLD1DUPq8: 3242 case ARM::VLD1DUPq16: 3243 case ARM::VLD1DUPq32: 3244 case ARM::VLD1DUPq8wb_fixed: 3245 case ARM::VLD1DUPq16wb_fixed: 3246 case ARM::VLD1DUPq32wb_fixed: 3247 case ARM::VLD1DUPq8wb_register: 3248 case ARM::VLD1DUPq16wb_register: 3249 case ARM::VLD1DUPq32wb_register: 3250 case ARM::VLD2DUPd8: 3251 case ARM::VLD2DUPd16: 3252 case ARM::VLD2DUPd32: 3253 case ARM::VLD2DUPd8wb_fixed: 3254 case ARM::VLD2DUPd16wb_fixed: 3255 case ARM::VLD2DUPd32wb_fixed: 3256 case ARM::VLD2DUPd8wb_register: 3257 case ARM::VLD2DUPd16wb_register: 3258 case ARM::VLD2DUPd32wb_register: 3259 case ARM::VLD4DUPd8: 3260 case ARM::VLD4DUPd16: 3261 case ARM::VLD4DUPd32: 3262 case ARM::VLD4DUPd8_UPD: 3263 case ARM::VLD4DUPd16_UPD: 3264 case ARM::VLD4DUPd32_UPD: 3265 case ARM::VLD1LNd8: 3266 case ARM::VLD1LNd16: 3267 case ARM::VLD1LNd32: 3268 case ARM::VLD1LNd8_UPD: 3269 case ARM::VLD1LNd16_UPD: 3270 case ARM::VLD1LNd32_UPD: 3271 case ARM::VLD2LNd8: 3272 case ARM::VLD2LNd16: 3273 case ARM::VLD2LNd32: 3274 case ARM::VLD2LNq16: 3275 case ARM::VLD2LNq32: 3276 case ARM::VLD2LNd8_UPD: 3277 case ARM::VLD2LNd16_UPD: 3278 case ARM::VLD2LNd32_UPD: 3279 case ARM::VLD2LNq16_UPD: 3280 case ARM::VLD2LNq32_UPD: 3281 case ARM::VLD4LNd8: 3282 case ARM::VLD4LNd16: 3283 case ARM::VLD4LNd32: 3284 case ARM::VLD4LNq16: 3285 case ARM::VLD4LNq32: 3286 case ARM::VLD4LNd8_UPD: 3287 case ARM::VLD4LNd16_UPD: 3288 case ARM::VLD4LNd32_UPD: 3289 case ARM::VLD4LNq16_UPD: 3290 case ARM::VLD4LNq32_UPD: 3291 // If the address is not 64-bit aligned, the latencies of these 3292 // instructions increases by one. 3293 ++Adjust; 3294 break; 3295 } 3296 } 3297 return Adjust; 3298} 3299 3300 3301 3302int 3303ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, 3304 const MachineInstr *DefMI, unsigned DefIdx, 3305 const MachineInstr *UseMI, 3306 unsigned UseIdx) const { 3307 // No operand latency. The caller may fall back to getInstrLatency. 3308 if (!ItinData || ItinData->isEmpty()) 3309 return -1; 3310 3311 const MachineOperand &DefMO = DefMI->getOperand(DefIdx); 3312 unsigned Reg = DefMO.getReg(); 3313 const MCInstrDesc *DefMCID = &DefMI->getDesc(); 3314 const MCInstrDesc *UseMCID = &UseMI->getDesc(); 3315 3316 unsigned DefAdj = 0; 3317 if (DefMI->isBundle()) { 3318 DefMI = getBundledDefMI(&getRegisterInfo(), DefMI, Reg, DefIdx, DefAdj); 3319 DefMCID = &DefMI->getDesc(); 3320 } 3321 if (DefMI->isCopyLike() || DefMI->isInsertSubreg() || 3322 DefMI->isRegSequence() || DefMI->isImplicitDef()) { 3323 return 1; 3324 } 3325 3326 unsigned UseAdj = 0; 3327 if (UseMI->isBundle()) { 3328 unsigned NewUseIdx; 3329 const MachineInstr *NewUseMI = getBundledUseMI(&getRegisterInfo(), UseMI, 3330 Reg, NewUseIdx, UseAdj); 3331 if (!NewUseMI) 3332 return -1; 3333 3334 UseMI = NewUseMI; 3335 UseIdx = NewUseIdx; 3336 UseMCID = &UseMI->getDesc(); 3337 } 3338 3339 if (Reg == ARM::CPSR) { 3340 if (DefMI->getOpcode() == ARM::FMSTAT) { 3341 // fpscr -> cpsr stalls over 20 cycles on A8 (and earlier?) 3342 return Subtarget.isLikeA9() ? 1 : 20; 3343 } 3344 3345 // CPSR set and branch can be paired in the same cycle. 3346 if (UseMI->isBranch()) 3347 return 0; 3348 3349 // Otherwise it takes the instruction latency (generally one). 3350 unsigned Latency = getInstrLatency(ItinData, DefMI); 3351 3352 // For Thumb2 and -Os, prefer scheduling CPSR setting instruction close to 3353 // its uses. Instructions which are otherwise scheduled between them may 3354 // incur a code size penalty (not able to use the CPSR setting 16-bit 3355 // instructions). 3356 if (Latency > 0 && Subtarget.isThumb2()) { 3357 const MachineFunction *MF = DefMI->getParent()->getParent(); 3358 if (MF->getFunction()->getAttributes(). 3359 hasAttribute(AttributeSet::FunctionIndex, 3360 Attribute::OptimizeForSize)) 3361 --Latency; 3362 } 3363 return Latency; 3364 } 3365 3366 if (DefMO.isImplicit() || UseMI->getOperand(UseIdx).isImplicit()) 3367 return -1; 3368 3369 unsigned DefAlign = DefMI->hasOneMemOperand() 3370 ? (*DefMI->memoperands_begin())->getAlignment() : 0; 3371 unsigned UseAlign = UseMI->hasOneMemOperand() 3372 ? (*UseMI->memoperands_begin())->getAlignment() : 0; 3373 3374 // Get the itinerary's latency if possible, and handle variable_ops. 3375 int Latency = getOperandLatency(ItinData, *DefMCID, DefIdx, DefAlign, 3376 *UseMCID, UseIdx, UseAlign); 3377 // Unable to find operand latency. The caller may resort to getInstrLatency. 3378 if (Latency < 0) 3379 return Latency; 3380 3381 // Adjust for IT block position. 3382 int Adj = DefAdj + UseAdj; 3383 3384 // Adjust for dynamic def-side opcode variants not captured by the itinerary. 3385 Adj += adjustDefLatency(Subtarget, DefMI, DefMCID, DefAlign); 3386 if (Adj >= 0 || (int)Latency > -Adj) { 3387 return Latency + Adj; 3388 } 3389 // Return the itinerary latency, which may be zero but not less than zero. 3390 return Latency; 3391} 3392 3393int 3394ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, 3395 SDNode *DefNode, unsigned DefIdx, 3396 SDNode *UseNode, unsigned UseIdx) const { 3397 if (!DefNode->isMachineOpcode()) 3398 return 1; 3399 3400 const MCInstrDesc &DefMCID = get(DefNode->getMachineOpcode()); 3401 3402 if (isZeroCost(DefMCID.Opcode)) 3403 return 0; 3404 3405 if (!ItinData || ItinData->isEmpty()) 3406 return DefMCID.mayLoad() ? 3 : 1; 3407 3408 if (!UseNode->isMachineOpcode()) { 3409 int Latency = ItinData->getOperandCycle(DefMCID.getSchedClass(), DefIdx); 3410 if (Subtarget.isLikeA9() || Subtarget.isSwift()) 3411 return Latency <= 2 ? 1 : Latency - 1; 3412 else 3413 return Latency <= 3 ? 1 : Latency - 2; 3414 } 3415 3416 const MCInstrDesc &UseMCID = get(UseNode->getMachineOpcode()); 3417 const MachineSDNode *DefMN = dyn_cast<MachineSDNode>(DefNode); 3418 unsigned DefAlign = !DefMN->memoperands_empty() 3419 ? (*DefMN->memoperands_begin())->getAlignment() : 0; 3420 const MachineSDNode *UseMN = dyn_cast<MachineSDNode>(UseNode); 3421 unsigned UseAlign = !UseMN->memoperands_empty() 3422 ? (*UseMN->memoperands_begin())->getAlignment() : 0; 3423 int Latency = getOperandLatency(ItinData, DefMCID, DefIdx, DefAlign, 3424 UseMCID, UseIdx, UseAlign); 3425 3426 if (Latency > 1 && 3427 (Subtarget.isCortexA8() || Subtarget.isLikeA9())) { 3428 // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2] 3429 // variants are one cycle cheaper. 3430 switch (DefMCID.getOpcode()) { 3431 default: break; 3432 case ARM::LDRrs: 3433 case ARM::LDRBrs: { 3434 unsigned ShOpVal = 3435 cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue(); 3436 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 3437 if (ShImm == 0 || 3438 (ShImm == 2 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)) 3439 --Latency; 3440 break; 3441 } 3442 case ARM::t2LDRs: 3443 case ARM::t2LDRBs: 3444 case ARM::t2LDRHs: 3445 case ARM::t2LDRSHs: { 3446 // Thumb2 mode: lsl only. 3447 unsigned ShAmt = 3448 cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue(); 3449 if (ShAmt == 0 || ShAmt == 2) 3450 --Latency; 3451 break; 3452 } 3453 } 3454 } else if (DefIdx == 0 && Latency > 2 && Subtarget.isSwift()) { 3455 // FIXME: Properly handle all of the latency adjustments for address 3456 // writeback. 3457 switch (DefMCID.getOpcode()) { 3458 default: break; 3459 case ARM::LDRrs: 3460 case ARM::LDRBrs: { 3461 unsigned ShOpVal = 3462 cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue(); 3463 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 3464 if (ShImm == 0 || 3465 ((ShImm == 1 || ShImm == 2 || ShImm == 3) && 3466 ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)) 3467 Latency -= 2; 3468 else if (ShImm == 1 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsr) 3469 --Latency; 3470 break; 3471 } 3472 case ARM::t2LDRs: 3473 case ARM::t2LDRBs: 3474 case ARM::t2LDRHs: 3475 case ARM::t2LDRSHs: { 3476 // Thumb2 mode: lsl 0-3 only. 3477 Latency -= 2; 3478 break; 3479 } 3480 } 3481 } 3482 3483 if (DefAlign < 8 && Subtarget.isLikeA9()) 3484 switch (DefMCID.getOpcode()) { 3485 default: break; 3486 case ARM::VLD1q8: 3487 case ARM::VLD1q16: 3488 case ARM::VLD1q32: 3489 case ARM::VLD1q64: 3490 case ARM::VLD1q8wb_register: 3491 case ARM::VLD1q16wb_register: 3492 case ARM::VLD1q32wb_register: 3493 case ARM::VLD1q64wb_register: 3494 case ARM::VLD1q8wb_fixed: 3495 case ARM::VLD1q16wb_fixed: 3496 case ARM::VLD1q32wb_fixed: 3497 case ARM::VLD1q64wb_fixed: 3498 case ARM::VLD2d8: 3499 case ARM::VLD2d16: 3500 case ARM::VLD2d32: 3501 case ARM::VLD2q8Pseudo: 3502 case ARM::VLD2q16Pseudo: 3503 case ARM::VLD2q32Pseudo: 3504 case ARM::VLD2d8wb_fixed: 3505 case ARM::VLD2d16wb_fixed: 3506 case ARM::VLD2d32wb_fixed: 3507 case ARM::VLD2q8PseudoWB_fixed: 3508 case ARM::VLD2q16PseudoWB_fixed: 3509 case ARM::VLD2q32PseudoWB_fixed: 3510 case ARM::VLD2d8wb_register: 3511 case ARM::VLD2d16wb_register: 3512 case ARM::VLD2d32wb_register: 3513 case ARM::VLD2q8PseudoWB_register: 3514 case ARM::VLD2q16PseudoWB_register: 3515 case ARM::VLD2q32PseudoWB_register: 3516 case ARM::VLD3d8Pseudo: 3517 case ARM::VLD3d16Pseudo: 3518 case ARM::VLD3d32Pseudo: 3519 case ARM::VLD1d64TPseudo: 3520 case ARM::VLD3d8Pseudo_UPD: 3521 case ARM::VLD3d16Pseudo_UPD: 3522 case ARM::VLD3d32Pseudo_UPD: 3523 case ARM::VLD3q8Pseudo_UPD: 3524 case ARM::VLD3q16Pseudo_UPD: 3525 case ARM::VLD3q32Pseudo_UPD: 3526 case ARM::VLD3q8oddPseudo: 3527 case ARM::VLD3q16oddPseudo: 3528 case ARM::VLD3q32oddPseudo: 3529 case ARM::VLD3q8oddPseudo_UPD: 3530 case ARM::VLD3q16oddPseudo_UPD: 3531 case ARM::VLD3q32oddPseudo_UPD: 3532 case ARM::VLD4d8Pseudo: 3533 case ARM::VLD4d16Pseudo: 3534 case ARM::VLD4d32Pseudo: 3535 case ARM::VLD1d64QPseudo: 3536 case ARM::VLD4d8Pseudo_UPD: 3537 case ARM::VLD4d16Pseudo_UPD: 3538 case ARM::VLD4d32Pseudo_UPD: 3539 case ARM::VLD4q8Pseudo_UPD: 3540 case ARM::VLD4q16Pseudo_UPD: 3541 case ARM::VLD4q32Pseudo_UPD: 3542 case ARM::VLD4q8oddPseudo: 3543 case ARM::VLD4q16oddPseudo: 3544 case ARM::VLD4q32oddPseudo: 3545 case ARM::VLD4q8oddPseudo_UPD: 3546 case ARM::VLD4q16oddPseudo_UPD: 3547 case ARM::VLD4q32oddPseudo_UPD: 3548 case ARM::VLD1DUPq8: 3549 case ARM::VLD1DUPq16: 3550 case ARM::VLD1DUPq32: 3551 case ARM::VLD1DUPq8wb_fixed: 3552 case ARM::VLD1DUPq16wb_fixed: 3553 case ARM::VLD1DUPq32wb_fixed: 3554 case ARM::VLD1DUPq8wb_register: 3555 case ARM::VLD1DUPq16wb_register: 3556 case ARM::VLD1DUPq32wb_register: 3557 case ARM::VLD2DUPd8: 3558 case ARM::VLD2DUPd16: 3559 case ARM::VLD2DUPd32: 3560 case ARM::VLD2DUPd8wb_fixed: 3561 case ARM::VLD2DUPd16wb_fixed: 3562 case ARM::VLD2DUPd32wb_fixed: 3563 case ARM::VLD2DUPd8wb_register: 3564 case ARM::VLD2DUPd16wb_register: 3565 case ARM::VLD2DUPd32wb_register: 3566 case ARM::VLD4DUPd8Pseudo: 3567 case ARM::VLD4DUPd16Pseudo: 3568 case ARM::VLD4DUPd32Pseudo: 3569 case ARM::VLD4DUPd8Pseudo_UPD: 3570 case ARM::VLD4DUPd16Pseudo_UPD: 3571 case ARM::VLD4DUPd32Pseudo_UPD: 3572 case ARM::VLD1LNq8Pseudo: 3573 case ARM::VLD1LNq16Pseudo: 3574 case ARM::VLD1LNq32Pseudo: 3575 case ARM::VLD1LNq8Pseudo_UPD: 3576 case ARM::VLD1LNq16Pseudo_UPD: 3577 case ARM::VLD1LNq32Pseudo_UPD: 3578 case ARM::VLD2LNd8Pseudo: 3579 case ARM::VLD2LNd16Pseudo: 3580 case ARM::VLD2LNd32Pseudo: 3581 case ARM::VLD2LNq16Pseudo: 3582 case ARM::VLD2LNq32Pseudo: 3583 case ARM::VLD2LNd8Pseudo_UPD: 3584 case ARM::VLD2LNd16Pseudo_UPD: 3585 case ARM::VLD2LNd32Pseudo_UPD: 3586 case ARM::VLD2LNq16Pseudo_UPD: 3587 case ARM::VLD2LNq32Pseudo_UPD: 3588 case ARM::VLD4LNd8Pseudo: 3589 case ARM::VLD4LNd16Pseudo: 3590 case ARM::VLD4LNd32Pseudo: 3591 case ARM::VLD4LNq16Pseudo: 3592 case ARM::VLD4LNq32Pseudo: 3593 case ARM::VLD4LNd8Pseudo_UPD: 3594 case ARM::VLD4LNd16Pseudo_UPD: 3595 case ARM::VLD4LNd32Pseudo_UPD: 3596 case ARM::VLD4LNq16Pseudo_UPD: 3597 case ARM::VLD4LNq32Pseudo_UPD: 3598 // If the address is not 64-bit aligned, the latencies of these 3599 // instructions increases by one. 3600 ++Latency; 3601 break; 3602 } 3603 3604 return Latency; 3605} 3606 3607unsigned ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, 3608 const MachineInstr *MI, 3609 unsigned *PredCost) const { 3610 if (MI->isCopyLike() || MI->isInsertSubreg() || 3611 MI->isRegSequence() || MI->isImplicitDef()) 3612 return 1; 3613 3614 // An instruction scheduler typically runs on unbundled instructions, however 3615 // other passes may query the latency of a bundled instruction. 3616 if (MI->isBundle()) { 3617 unsigned Latency = 0; 3618 MachineBasicBlock::const_instr_iterator I = MI; 3619 MachineBasicBlock::const_instr_iterator E = MI->getParent()->instr_end(); 3620 while (++I != E && I->isInsideBundle()) { 3621 if (I->getOpcode() != ARM::t2IT) 3622 Latency += getInstrLatency(ItinData, I, PredCost); 3623 } 3624 return Latency; 3625 } 3626 3627 const MCInstrDesc &MCID = MI->getDesc(); 3628 if (PredCost && (MCID.isCall() || MCID.hasImplicitDefOfPhysReg(ARM::CPSR))) { 3629 // When predicated, CPSR is an additional source operand for CPSR updating 3630 // instructions, this apparently increases their latencies. 3631 *PredCost = 1; 3632 } 3633 // Be sure to call getStageLatency for an empty itinerary in case it has a 3634 // valid MinLatency property. 3635 if (!ItinData) 3636 return MI->mayLoad() ? 3 : 1; 3637 3638 unsigned Class = MCID.getSchedClass(); 3639 3640 // For instructions with variable uops, use uops as latency. 3641 if (!ItinData->isEmpty() && ItinData->getNumMicroOps(Class) < 0) 3642 return getNumMicroOps(ItinData, MI); 3643 3644 // For the common case, fall back on the itinerary's latency. 3645 unsigned Latency = ItinData->getStageLatency(Class); 3646 3647 // Adjust for dynamic def-side opcode variants not captured by the itinerary. 3648 unsigned DefAlign = MI->hasOneMemOperand() 3649 ? (*MI->memoperands_begin())->getAlignment() : 0; 3650 int Adj = adjustDefLatency(Subtarget, MI, &MCID, DefAlign); 3651 if (Adj >= 0 || (int)Latency > -Adj) { 3652 return Latency + Adj; 3653 } 3654 return Latency; 3655} 3656 3657int ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, 3658 SDNode *Node) const { 3659 if (!Node->isMachineOpcode()) 3660 return 1; 3661 3662 if (!ItinData || ItinData->isEmpty()) 3663 return 1; 3664 3665 unsigned Opcode = Node->getMachineOpcode(); 3666 switch (Opcode) { 3667 default: 3668 return ItinData->getStageLatency(get(Opcode).getSchedClass()); 3669 case ARM::VLDMQIA: 3670 case ARM::VSTMQIA: 3671 return 2; 3672 } 3673} 3674 3675bool ARMBaseInstrInfo:: 3676hasHighOperandLatency(const InstrItineraryData *ItinData, 3677 const MachineRegisterInfo *MRI, 3678 const MachineInstr *DefMI, unsigned DefIdx, 3679 const MachineInstr *UseMI, unsigned UseIdx) const { 3680 unsigned DDomain = DefMI->getDesc().TSFlags & ARMII::DomainMask; 3681 unsigned UDomain = UseMI->getDesc().TSFlags & ARMII::DomainMask; 3682 if (Subtarget.isCortexA8() && 3683 (DDomain == ARMII::DomainVFP || UDomain == ARMII::DomainVFP)) 3684 // CortexA8 VFP instructions are not pipelined. 3685 return true; 3686 3687 // Hoist VFP / NEON instructions with 4 or higher latency. 3688 int Latency = computeOperandLatency(ItinData, DefMI, DefIdx, UseMI, UseIdx, 3689 /*FindMin=*/false); 3690 if (Latency < 0) 3691 Latency = getInstrLatency(ItinData, DefMI); 3692 if (Latency <= 3) 3693 return false; 3694 return DDomain == ARMII::DomainVFP || DDomain == ARMII::DomainNEON || 3695 UDomain == ARMII::DomainVFP || UDomain == ARMII::DomainNEON; 3696} 3697 3698bool ARMBaseInstrInfo:: 3699hasLowDefLatency(const InstrItineraryData *ItinData, 3700 const MachineInstr *DefMI, unsigned DefIdx) const { 3701 if (!ItinData || ItinData->isEmpty()) 3702 return false; 3703 3704 unsigned DDomain = DefMI->getDesc().TSFlags & ARMII::DomainMask; 3705 if (DDomain == ARMII::DomainGeneral) { 3706 unsigned DefClass = DefMI->getDesc().getSchedClass(); 3707 int DefCycle = ItinData->getOperandCycle(DefClass, DefIdx); 3708 return (DefCycle != -1 && DefCycle <= 2); 3709 } 3710 return false; 3711} 3712 3713bool ARMBaseInstrInfo::verifyInstruction(const MachineInstr *MI, 3714 StringRef &ErrInfo) const { 3715 if (convertAddSubFlagsOpcode(MI->getOpcode())) { 3716 ErrInfo = "Pseudo flag setting opcodes only exist in Selection DAG"; 3717 return false; 3718 } 3719 return true; 3720} 3721 3722bool 3723ARMBaseInstrInfo::isFpMLxInstruction(unsigned Opcode, unsigned &MulOpc, 3724 unsigned &AddSubOpc, 3725 bool &NegAcc, bool &HasLane) const { 3726 DenseMap<unsigned, unsigned>::const_iterator I = MLxEntryMap.find(Opcode); 3727 if (I == MLxEntryMap.end()) 3728 return false; 3729 3730 const ARM_MLxEntry &Entry = ARM_MLxTable[I->second]; 3731 MulOpc = Entry.MulOpc; 3732 AddSubOpc = Entry.AddSubOpc; 3733 NegAcc = Entry.NegAcc; 3734 HasLane = Entry.HasLane; 3735 return true; 3736} 3737 3738//===----------------------------------------------------------------------===// 3739// Execution domains. 3740//===----------------------------------------------------------------------===// 3741// 3742// Some instructions go down the NEON pipeline, some go down the VFP pipeline, 3743// and some can go down both. The vmov instructions go down the VFP pipeline, 3744// but they can be changed to vorr equivalents that are executed by the NEON 3745// pipeline. 3746// 3747// We use the following execution domain numbering: 3748// 3749enum ARMExeDomain { 3750 ExeGeneric = 0, 3751 ExeVFP = 1, 3752 ExeNEON = 2 3753}; 3754// 3755// Also see ARMInstrFormats.td and Domain* enums in ARMBaseInfo.h 3756// 3757std::pair<uint16_t, uint16_t> 3758ARMBaseInstrInfo::getExecutionDomain(const MachineInstr *MI) const { 3759 // VMOVD, VMOVRS and VMOVSR are VFP instructions, but can be changed to NEON 3760 // if they are not predicated. 3761 if (MI->getOpcode() == ARM::VMOVD && !isPredicated(MI)) 3762 return std::make_pair(ExeVFP, (1<<ExeVFP) | (1<<ExeNEON)); 3763 3764 // CortexA9 is particularly picky about mixing the two and wants these 3765 // converted. 3766 if (Subtarget.isCortexA9() && !isPredicated(MI) && 3767 (MI->getOpcode() == ARM::VMOVRS || 3768 MI->getOpcode() == ARM::VMOVSR || 3769 MI->getOpcode() == ARM::VMOVS)) 3770 return std::make_pair(ExeVFP, (1<<ExeVFP) | (1<<ExeNEON)); 3771 3772 // No other instructions can be swizzled, so just determine their domain. 3773 unsigned Domain = MI->getDesc().TSFlags & ARMII::DomainMask; 3774 3775 if (Domain & ARMII::DomainNEON) 3776 return std::make_pair(ExeNEON, 0); 3777 3778 // Certain instructions can go either way on Cortex-A8. 3779 // Treat them as NEON instructions. 3780 if ((Domain & ARMII::DomainNEONA8) && Subtarget.isCortexA8()) 3781 return std::make_pair(ExeNEON, 0); 3782 3783 if (Domain & ARMII::DomainVFP) 3784 return std::make_pair(ExeVFP, 0); 3785 3786 return std::make_pair(ExeGeneric, 0); 3787} 3788 3789static unsigned getCorrespondingDRegAndLane(const TargetRegisterInfo *TRI, 3790 unsigned SReg, unsigned &Lane) { 3791 unsigned DReg = TRI->getMatchingSuperReg(SReg, ARM::ssub_0, &ARM::DPRRegClass); 3792 Lane = 0; 3793 3794 if (DReg != ARM::NoRegister) 3795 return DReg; 3796 3797 Lane = 1; 3798 DReg = TRI->getMatchingSuperReg(SReg, ARM::ssub_1, &ARM::DPRRegClass); 3799 3800 assert(DReg && "S-register with no D super-register?"); 3801 return DReg; 3802} 3803 3804/// getImplicitSPRUseForDPRUse - Given a use of a DPR register and lane, 3805/// set ImplicitSReg to a register number that must be marked as implicit-use or 3806/// zero if no register needs to be defined as implicit-use. 3807/// 3808/// If the function cannot determine if an SPR should be marked implicit use or 3809/// not, it returns false. 3810/// 3811/// This function handles cases where an instruction is being modified from taking 3812/// an SPR to a DPR[Lane]. A use of the DPR is being added, which may conflict 3813/// with an earlier def of an SPR corresponding to DPR[Lane^1] (i.e. the other 3814/// lane of the DPR). 3815/// 3816/// If the other SPR is defined, an implicit-use of it should be added. Else, 3817/// (including the case where the DPR itself is defined), it should not. 3818/// 3819static bool getImplicitSPRUseForDPRUse(const TargetRegisterInfo *TRI, 3820 MachineInstr *MI, 3821 unsigned DReg, unsigned Lane, 3822 unsigned &ImplicitSReg) { 3823 // If the DPR is defined or used already, the other SPR lane will be chained 3824 // correctly, so there is nothing to be done. 3825 if (MI->definesRegister(DReg, TRI) || MI->readsRegister(DReg, TRI)) { 3826 ImplicitSReg = 0; 3827 return true; 3828 } 3829 3830 // Otherwise we need to go searching to see if the SPR is set explicitly. 3831 ImplicitSReg = TRI->getSubReg(DReg, 3832 (Lane & 1) ? ARM::ssub_0 : ARM::ssub_1); 3833 MachineBasicBlock::LivenessQueryResult LQR = 3834 MI->getParent()->computeRegisterLiveness(TRI, ImplicitSReg, MI); 3835 3836 if (LQR == MachineBasicBlock::LQR_Live) 3837 return true; 3838 else if (LQR == MachineBasicBlock::LQR_Unknown) 3839 return false; 3840 3841 // If the register is known not to be live, there is no need to add an 3842 // implicit-use. 3843 ImplicitSReg = 0; 3844 return true; 3845} 3846 3847void 3848ARMBaseInstrInfo::setExecutionDomain(MachineInstr *MI, unsigned Domain) const { 3849 unsigned DstReg, SrcReg, DReg; 3850 unsigned Lane; 3851 MachineInstrBuilder MIB(*MI->getParent()->getParent(), MI); 3852 const TargetRegisterInfo *TRI = &getRegisterInfo(); 3853 switch (MI->getOpcode()) { 3854 default: 3855 llvm_unreachable("cannot handle opcode!"); 3856 break; 3857 case ARM::VMOVD: 3858 if (Domain != ExeNEON) 3859 break; 3860 3861 // Zap the predicate operands. 3862 assert(!isPredicated(MI) && "Cannot predicate a VORRd"); 3863 3864 // Source instruction is %DDst = VMOVD %DSrc, 14, %noreg (; implicits) 3865 DstReg = MI->getOperand(0).getReg(); 3866 SrcReg = MI->getOperand(1).getReg(); 3867 3868 for (unsigned i = MI->getDesc().getNumOperands(); i; --i) 3869 MI->RemoveOperand(i-1); 3870 3871 // Change to a %DDst = VORRd %DSrc, %DSrc, 14, %noreg (; implicits) 3872 MI->setDesc(get(ARM::VORRd)); 3873 AddDefaultPred(MIB.addReg(DstReg, RegState::Define) 3874 .addReg(SrcReg) 3875 .addReg(SrcReg)); 3876 break; 3877 case ARM::VMOVRS: 3878 if (Domain != ExeNEON) 3879 break; 3880 assert(!isPredicated(MI) && "Cannot predicate a VGETLN"); 3881 3882 // Source instruction is %RDst = VMOVRS %SSrc, 14, %noreg (; implicits) 3883 DstReg = MI->getOperand(0).getReg(); 3884 SrcReg = MI->getOperand(1).getReg(); 3885 3886 for (unsigned i = MI->getDesc().getNumOperands(); i; --i) 3887 MI->RemoveOperand(i-1); 3888 3889 DReg = getCorrespondingDRegAndLane(TRI, SrcReg, Lane); 3890 3891 // Convert to %RDst = VGETLNi32 %DSrc, Lane, 14, %noreg (; imps) 3892 // Note that DSrc has been widened and the other lane may be undef, which 3893 // contaminates the entire register. 3894 MI->setDesc(get(ARM::VGETLNi32)); 3895 AddDefaultPred(MIB.addReg(DstReg, RegState::Define) 3896 .addReg(DReg, RegState::Undef) 3897 .addImm(Lane)); 3898 3899 // The old source should be an implicit use, otherwise we might think it 3900 // was dead before here. 3901 MIB.addReg(SrcReg, RegState::Implicit); 3902 break; 3903 case ARM::VMOVSR: { 3904 if (Domain != ExeNEON) 3905 break; 3906 assert(!isPredicated(MI) && "Cannot predicate a VSETLN"); 3907 3908 // Source instruction is %SDst = VMOVSR %RSrc, 14, %noreg (; implicits) 3909 DstReg = MI->getOperand(0).getReg(); 3910 SrcReg = MI->getOperand(1).getReg(); 3911 3912 DReg = getCorrespondingDRegAndLane(TRI, DstReg, Lane); 3913 3914 unsigned ImplicitSReg; 3915 if (!getImplicitSPRUseForDPRUse(TRI, MI, DReg, Lane, ImplicitSReg)) 3916 break; 3917 3918 for (unsigned i = MI->getDesc().getNumOperands(); i; --i) 3919 MI->RemoveOperand(i-1); 3920 3921 // Convert to %DDst = VSETLNi32 %DDst, %RSrc, Lane, 14, %noreg (; imps) 3922 // Again DDst may be undefined at the beginning of this instruction. 3923 MI->setDesc(get(ARM::VSETLNi32)); 3924 MIB.addReg(DReg, RegState::Define) 3925 .addReg(DReg, getUndefRegState(!MI->readsRegister(DReg, TRI))) 3926 .addReg(SrcReg) 3927 .addImm(Lane); 3928 AddDefaultPred(MIB); 3929 3930 // The narrower destination must be marked as set to keep previous chains 3931 // in place. 3932 MIB.addReg(DstReg, RegState::Define | RegState::Implicit); 3933 if (ImplicitSReg != 0) 3934 MIB.addReg(ImplicitSReg, RegState::Implicit); 3935 break; 3936 } 3937 case ARM::VMOVS: { 3938 if (Domain != ExeNEON) 3939 break; 3940 3941 // Source instruction is %SDst = VMOVS %SSrc, 14, %noreg (; implicits) 3942 DstReg = MI->getOperand(0).getReg(); 3943 SrcReg = MI->getOperand(1).getReg(); 3944 3945 unsigned DstLane = 0, SrcLane = 0, DDst, DSrc; 3946 DDst = getCorrespondingDRegAndLane(TRI, DstReg, DstLane); 3947 DSrc = getCorrespondingDRegAndLane(TRI, SrcReg, SrcLane); 3948 3949 unsigned ImplicitSReg; 3950 if (!getImplicitSPRUseForDPRUse(TRI, MI, DSrc, SrcLane, ImplicitSReg)) 3951 break; 3952 3953 for (unsigned i = MI->getDesc().getNumOperands(); i; --i) 3954 MI->RemoveOperand(i-1); 3955 3956 if (DSrc == DDst) { 3957 // Destination can be: 3958 // %DDst = VDUPLN32d %DDst, Lane, 14, %noreg (; implicits) 3959 MI->setDesc(get(ARM::VDUPLN32d)); 3960 MIB.addReg(DDst, RegState::Define) 3961 .addReg(DDst, getUndefRegState(!MI->readsRegister(DDst, TRI))) 3962 .addImm(SrcLane); 3963 AddDefaultPred(MIB); 3964 3965 // Neither the source or the destination are naturally represented any 3966 // more, so add them in manually. 3967 MIB.addReg(DstReg, RegState::Implicit | RegState::Define); 3968 MIB.addReg(SrcReg, RegState::Implicit); 3969 if (ImplicitSReg != 0) 3970 MIB.addReg(ImplicitSReg, RegState::Implicit); 3971 break; 3972 } 3973 3974 // In general there's no single instruction that can perform an S <-> S 3975 // move in NEON space, but a pair of VEXT instructions *can* do the 3976 // job. It turns out that the VEXTs needed will only use DSrc once, with 3977 // the position based purely on the combination of lane-0 and lane-1 3978 // involved. For example 3979 // vmov s0, s2 -> vext.32 d0, d0, d1, #1 vext.32 d0, d0, d0, #1 3980 // vmov s1, s3 -> vext.32 d0, d1, d0, #1 vext.32 d0, d0, d0, #1 3981 // vmov s0, s3 -> vext.32 d0, d0, d0, #1 vext.32 d0, d1, d0, #1 3982 // vmov s1, s2 -> vext.32 d0, d0, d0, #1 vext.32 d0, d0, d1, #1 3983 // 3984 // Pattern of the MachineInstrs is: 3985 // %DDst = VEXTd32 %DSrc1, %DSrc2, Lane, 14, %noreg (;implicits) 3986 MachineInstrBuilder NewMIB; 3987 NewMIB = BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 3988 get(ARM::VEXTd32), DDst); 3989 3990 // On the first instruction, both DSrc and DDst may be <undef> if present. 3991 // Specifically when the original instruction didn't have them as an 3992 // <imp-use>. 3993 unsigned CurReg = SrcLane == 1 && DstLane == 1 ? DSrc : DDst; 3994 bool CurUndef = !MI->readsRegister(CurReg, TRI); 3995 NewMIB.addReg(CurReg, getUndefRegState(CurUndef)); 3996 3997 CurReg = SrcLane == 0 && DstLane == 0 ? DSrc : DDst; 3998 CurUndef = !MI->readsRegister(CurReg, TRI); 3999 NewMIB.addReg(CurReg, getUndefRegState(CurUndef)); 4000 4001 NewMIB.addImm(1); 4002 AddDefaultPred(NewMIB); 4003 4004 if (SrcLane == DstLane) 4005 NewMIB.addReg(SrcReg, RegState::Implicit); 4006 4007 MI->setDesc(get(ARM::VEXTd32)); 4008 MIB.addReg(DDst, RegState::Define); 4009 4010 // On the second instruction, DDst has definitely been defined above, so 4011 // it is not <undef>. DSrc, if present, can be <undef> as above. 4012 CurReg = SrcLane == 1 && DstLane == 0 ? DSrc : DDst; 4013 CurUndef = CurReg == DSrc && !MI->readsRegister(CurReg, TRI); 4014 MIB.addReg(CurReg, getUndefRegState(CurUndef)); 4015 4016 CurReg = SrcLane == 0 && DstLane == 1 ? DSrc : DDst; 4017 CurUndef = CurReg == DSrc && !MI->readsRegister(CurReg, TRI); 4018 MIB.addReg(CurReg, getUndefRegState(CurUndef)); 4019 4020 MIB.addImm(1); 4021 AddDefaultPred(MIB); 4022 4023 if (SrcLane != DstLane) 4024 MIB.addReg(SrcReg, RegState::Implicit); 4025 4026 // As before, the original destination is no longer represented, add it 4027 // implicitly. 4028 MIB.addReg(DstReg, RegState::Define | RegState::Implicit); 4029 if (ImplicitSReg != 0) 4030 MIB.addReg(ImplicitSReg, RegState::Implicit); 4031 break; 4032 } 4033 } 4034 4035} 4036 4037//===----------------------------------------------------------------------===// 4038// Partial register updates 4039//===----------------------------------------------------------------------===// 4040// 4041// Swift renames NEON registers with 64-bit granularity. That means any 4042// instruction writing an S-reg implicitly reads the containing D-reg. The 4043// problem is mostly avoided by translating f32 operations to v2f32 operations 4044// on D-registers, but f32 loads are still a problem. 4045// 4046// These instructions can load an f32 into a NEON register: 4047// 4048// VLDRS - Only writes S, partial D update. 4049// VLD1LNd32 - Writes all D-regs, explicit partial D update, 2 uops. 4050// VLD1DUPd32 - Writes all D-regs, no partial reg update, 2 uops. 4051// 4052// FCONSTD can be used as a dependency-breaking instruction. 4053unsigned ARMBaseInstrInfo:: 4054getPartialRegUpdateClearance(const MachineInstr *MI, 4055 unsigned OpNum, 4056 const TargetRegisterInfo *TRI) const { 4057 if (!SwiftPartialUpdateClearance || 4058 !(Subtarget.isSwift() || Subtarget.isCortexA15())) 4059 return 0; 4060 4061 assert(TRI && "Need TRI instance"); 4062 4063 const MachineOperand &MO = MI->getOperand(OpNum); 4064 if (MO.readsReg()) 4065 return 0; 4066 unsigned Reg = MO.getReg(); 4067 int UseOp = -1; 4068 4069 switch(MI->getOpcode()) { 4070 // Normal instructions writing only an S-register. 4071 case ARM::VLDRS: 4072 case ARM::FCONSTS: 4073 case ARM::VMOVSR: 4074 case ARM::VMOVv8i8: 4075 case ARM::VMOVv4i16: 4076 case ARM::VMOVv2i32: 4077 case ARM::VMOVv2f32: 4078 case ARM::VMOVv1i64: 4079 UseOp = MI->findRegisterUseOperandIdx(Reg, false, TRI); 4080 break; 4081 4082 // Explicitly reads the dependency. 4083 case ARM::VLD1LNd32: 4084 UseOp = 3; 4085 break; 4086 default: 4087 return 0; 4088 } 4089 4090 // If this instruction actually reads a value from Reg, there is no unwanted 4091 // dependency. 4092 if (UseOp != -1 && MI->getOperand(UseOp).readsReg()) 4093 return 0; 4094 4095 // We must be able to clobber the whole D-reg. 4096 if (TargetRegisterInfo::isVirtualRegister(Reg)) { 4097 // Virtual register must be a foo:ssub_0<def,undef> operand. 4098 if (!MO.getSubReg() || MI->readsVirtualRegister(Reg)) 4099 return 0; 4100 } else if (ARM::SPRRegClass.contains(Reg)) { 4101 // Physical register: MI must define the full D-reg. 4102 unsigned DReg = TRI->getMatchingSuperReg(Reg, ARM::ssub_0, 4103 &ARM::DPRRegClass); 4104 if (!DReg || !MI->definesRegister(DReg, TRI)) 4105 return 0; 4106 } 4107 4108 // MI has an unwanted D-register dependency. 4109 // Avoid defs in the previous N instructrions. 4110 return SwiftPartialUpdateClearance; 4111} 4112 4113// Break a partial register dependency after getPartialRegUpdateClearance 4114// returned non-zero. 4115void ARMBaseInstrInfo:: 4116breakPartialRegDependency(MachineBasicBlock::iterator MI, 4117 unsigned OpNum, 4118 const TargetRegisterInfo *TRI) const { 4119 assert(MI && OpNum < MI->getDesc().getNumDefs() && "OpNum is not a def"); 4120 assert(TRI && "Need TRI instance"); 4121 4122 const MachineOperand &MO = MI->getOperand(OpNum); 4123 unsigned Reg = MO.getReg(); 4124 assert(TargetRegisterInfo::isPhysicalRegister(Reg) && 4125 "Can't break virtual register dependencies."); 4126 unsigned DReg = Reg; 4127 4128 // If MI defines an S-reg, find the corresponding D super-register. 4129 if (ARM::SPRRegClass.contains(Reg)) { 4130 DReg = ARM::D0 + (Reg - ARM::S0) / 2; 4131 assert(TRI->isSuperRegister(Reg, DReg) && "Register enums broken"); 4132 } 4133 4134 assert(ARM::DPRRegClass.contains(DReg) && "Can only break D-reg deps"); 4135 assert(MI->definesRegister(DReg, TRI) && "MI doesn't clobber full D-reg"); 4136 4137 // FIXME: In some cases, VLDRS can be changed to a VLD1DUPd32 which defines 4138 // the full D-register by loading the same value to both lanes. The 4139 // instruction is micro-coded with 2 uops, so don't do this until we can 4140 // properly schedule micro-coded instuctions. The dispatcher stalls cause 4141 // too big regressions. 4142 4143 // Insert the dependency-breaking FCONSTD before MI. 4144 // 96 is the encoding of 0.5, but the actual value doesn't matter here. 4145 AddDefaultPred(BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 4146 get(ARM::FCONSTD), DReg).addImm(96)); 4147 MI->addRegisterKilled(DReg, TRI, true); 4148} 4149 4150bool ARMBaseInstrInfo::hasNOP() const { 4151 return (Subtarget.getFeatureBits() & ARM::HasV6T2Ops) != 0; 4152} 4153 4154bool ARMBaseInstrInfo::isSwiftFastImmShift(const MachineInstr *MI) const { 4155 unsigned ShOpVal = MI->getOperand(3).getImm(); 4156 unsigned ShImm = ARM_AM::getSORegOffset(ShOpVal); 4157 // Swift supports faster shifts for: lsl 2, lsl 1, and lsr 1. 4158 if ((ShImm == 1 && ARM_AM::getSORegShOp(ShOpVal) == ARM_AM::lsr) || 4159 ((ShImm == 1 || ShImm == 2) && 4160 ARM_AM::getSORegShOp(ShOpVal) == ARM_AM::lsl)) 4161 return true; 4162 4163 return false; 4164} 4165