PrologEpilogInserter.cpp revision 6734c3a8ea14d2bbdf7704794f3df2e0a28aacb2
1//===-- PrologEpilogInserter.cpp - Insert Prolog/Epilog code in function --===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This pass is responsible for finalizing the functions frame layout, saving 11// callee saved registers, and for emitting prolog & epilog code for the 12// function. 13// 14// This pass must be run after register allocation. After this pass is 15// executed, it is illegal to construct MO_FrameIndex operands. 16// 17//===----------------------------------------------------------------------===// 18 19#include "llvm/CodeGen/Passes.h" 20#include "llvm/CodeGen/MachineFunctionPass.h" 21#include "llvm/CodeGen/MachineInstr.h" 22#include "llvm/CodeGen/MachineFrameInfo.h" 23#include "llvm/CodeGen/MachineModuleInfo.h" 24#include "llvm/CodeGen/MachineRegisterInfo.h" 25#include "llvm/CodeGen/RegisterScavenging.h" 26#include "llvm/Target/TargetMachine.h" 27#include "llvm/Target/TargetRegisterInfo.h" 28#include "llvm/Target/TargetFrameInfo.h" 29#include "llvm/Target/TargetInstrInfo.h" 30#include "llvm/Support/Compiler.h" 31#include "llvm/ADT/STLExtras.h" 32#include <climits> 33using namespace llvm; 34 35namespace { 36 struct VISIBILITY_HIDDEN PEI : public MachineFunctionPass { 37 static char ID; 38 PEI() : MachineFunctionPass((intptr_t)&ID) {} 39 40 const char *getPassName() const { 41 return "Prolog/Epilog Insertion & Frame Finalization"; 42 } 43 44 /// runOnMachineFunction - Insert prolog/epilog code and replace abstract 45 /// frame indexes with appropriate references. 46 /// 47 bool runOnMachineFunction(MachineFunction &Fn) { 48 const TargetRegisterInfo *TRI = Fn.getTarget().getRegisterInfo(); 49 RS = TRI->requiresRegisterScavenging(Fn) ? new RegScavenger() : NULL; 50 51 // Get MachineModuleInfo so that we can track the construction of the 52 // frame. 53 if (MachineModuleInfo *MMI = getAnalysisToUpdate<MachineModuleInfo>()) 54 Fn.getFrameInfo()->setMachineModuleInfo(MMI); 55 56 // Allow the target machine to make some adjustments to the function 57 // e.g. UsedPhysRegs before calculateCalleeSavedRegisters. 58 TRI->processFunctionBeforeCalleeSavedScan(Fn, RS); 59 60 // Scan the function for modified callee saved registers and insert spill 61 // code for any callee saved registers that are modified. Also calculate 62 // the MaxCallFrameSize and HasCalls variables for the function's frame 63 // information and eliminates call frame pseudo instructions. 64 calculateCalleeSavedRegisters(Fn); 65 66 // Add the code to save and restore the callee saved registers 67 saveCalleeSavedRegisters(Fn); 68 69 // Allow the target machine to make final modifications to the function 70 // before the frame layout is finalized. 71 Fn.getTarget().getRegisterInfo()->processFunctionBeforeFrameFinalized(Fn); 72 73 // Calculate actual frame offsets for all of the abstract stack objects... 74 calculateFrameObjectOffsets(Fn); 75 76 // Add prolog and epilog code to the function. This function is required 77 // to align the stack frame as necessary for any stack variables or 78 // called functions. Because of this, calculateCalleeSavedRegisters 79 // must be called before this function in order to set the HasCalls 80 // and MaxCallFrameSize variables. 81 insertPrologEpilogCode(Fn); 82 83 // Replace all MO_FrameIndex operands with physical register references 84 // and actual offsets. 85 // 86 replaceFrameIndices(Fn); 87 88 delete RS; 89 return true; 90 } 91 92 private: 93 RegScavenger *RS; 94 95 // MinCSFrameIndex, MaxCSFrameIndex - Keeps the range of callee saved 96 // stack frame indexes. 97 unsigned MinCSFrameIndex, MaxCSFrameIndex; 98 99 void calculateCalleeSavedRegisters(MachineFunction &Fn); 100 void saveCalleeSavedRegisters(MachineFunction &Fn); 101 void calculateFrameObjectOffsets(MachineFunction &Fn); 102 void replaceFrameIndices(MachineFunction &Fn); 103 void insertPrologEpilogCode(MachineFunction &Fn); 104 }; 105 char PEI::ID = 0; 106} 107 108 109/// createPrologEpilogCodeInserter - This function returns a pass that inserts 110/// prolog and epilog code, and eliminates abstract frame references. 111/// 112FunctionPass *llvm::createPrologEpilogCodeInserter() { return new PEI(); } 113 114 115/// calculateCalleeSavedRegisters - Scan the function for modified callee saved 116/// registers. Also calculate the MaxCallFrameSize and HasCalls variables for 117/// the function's frame information and eliminates call frame pseudo 118/// instructions. 119/// 120void PEI::calculateCalleeSavedRegisters(MachineFunction &Fn) { 121 const TargetRegisterInfo *RegInfo = Fn.getTarget().getRegisterInfo(); 122 const TargetFrameInfo *TFI = Fn.getTarget().getFrameInfo(); 123 124 // Get the callee saved register list... 125 const unsigned *CSRegs = RegInfo->getCalleeSavedRegs(&Fn); 126 127 // Get the function call frame set-up and tear-down instruction opcode 128 int FrameSetupOpcode = RegInfo->getCallFrameSetupOpcode(); 129 int FrameDestroyOpcode = RegInfo->getCallFrameDestroyOpcode(); 130 131 // These are used to keep track the callee-save area. Initialize them. 132 MinCSFrameIndex = INT_MAX; 133 MaxCSFrameIndex = 0; 134 135 // Early exit for targets which have no callee saved registers and no call 136 // frame setup/destroy pseudo instructions. 137 if ((CSRegs == 0 || CSRegs[0] == 0) && 138 FrameSetupOpcode == -1 && FrameDestroyOpcode == -1) 139 return; 140 141 unsigned MaxCallFrameSize = 0; 142 bool HasCalls = false; 143 144 std::vector<MachineBasicBlock::iterator> FrameSDOps; 145 for (MachineFunction::iterator BB = Fn.begin(), E = Fn.end(); BB != E; ++BB) 146 for (MachineBasicBlock::iterator I = BB->begin(); I != BB->end(); ++I) 147 if (I->getOpcode() == FrameSetupOpcode || 148 I->getOpcode() == FrameDestroyOpcode) { 149 assert(I->getNumOperands() >= 1 && "Call Frame Setup/Destroy Pseudo" 150 " instructions should have a single immediate argument!"); 151 unsigned Size = I->getOperand(0).getImm(); 152 if (Size > MaxCallFrameSize) MaxCallFrameSize = Size; 153 HasCalls = true; 154 FrameSDOps.push_back(I); 155 } 156 157 MachineFrameInfo *FFI = Fn.getFrameInfo(); 158 FFI->setHasCalls(HasCalls); 159 FFI->setMaxCallFrameSize(MaxCallFrameSize); 160 161 for (unsigned i = 0, e = FrameSDOps.size(); i != e; ++i) { 162 MachineBasicBlock::iterator I = FrameSDOps[i]; 163 // If call frames are not being included as part of the stack frame, 164 // and there is no dynamic allocation (therefore referencing frame slots 165 // off sp), leave the pseudo ops alone. We'll eliminate them later. 166 if (RegInfo->hasReservedCallFrame(Fn) || RegInfo->hasFP(Fn)) 167 RegInfo->eliminateCallFramePseudoInstr(Fn, *I->getParent(), I); 168 } 169 170 // Now figure out which *callee saved* registers are modified by the current 171 // function, thus needing to be saved and restored in the prolog/epilog. 172 // 173 const TargetRegisterClass* const *CSRegClasses = 174 RegInfo->getCalleeSavedRegClasses(&Fn); 175 std::vector<CalleeSavedInfo> CSI; 176 for (unsigned i = 0; CSRegs[i]; ++i) { 177 unsigned Reg = CSRegs[i]; 178 if (Fn.getRegInfo().isPhysRegUsed(Reg)) { 179 // If the reg is modified, save it! 180 CSI.push_back(CalleeSavedInfo(Reg, CSRegClasses[i])); 181 } else { 182 for (const unsigned *AliasSet = RegInfo->getAliasSet(Reg); 183 *AliasSet; ++AliasSet) { // Check alias registers too. 184 if (Fn.getRegInfo().isPhysRegUsed(*AliasSet)) { 185 CSI.push_back(CalleeSavedInfo(Reg, CSRegClasses[i])); 186 break; 187 } 188 } 189 } 190 } 191 192 if (CSI.empty()) 193 return; // Early exit if no callee saved registers are modified! 194 195 unsigned NumFixedSpillSlots; 196 const std::pair<unsigned,int> *FixedSpillSlots = 197 TFI->getCalleeSavedSpillSlots(NumFixedSpillSlots); 198 199 // Now that we know which registers need to be saved and restored, allocate 200 // stack slots for them. 201 for (unsigned i = 0, e = CSI.size(); i != e; ++i) { 202 unsigned Reg = CSI[i].getReg(); 203 const TargetRegisterClass *RC = CSI[i].getRegClass(); 204 205 // Check to see if this physreg must be spilled to a particular stack slot 206 // on this target. 207 const std::pair<unsigned,int> *FixedSlot = FixedSpillSlots; 208 while (FixedSlot != FixedSpillSlots+NumFixedSpillSlots && 209 FixedSlot->first != Reg) 210 ++FixedSlot; 211 212 int FrameIdx; 213 if (FixedSlot == FixedSpillSlots+NumFixedSpillSlots) { 214 // Nope, just spill it anywhere convenient. 215 unsigned Align = RC->getAlignment(); 216 unsigned StackAlign = TFI->getStackAlignment(); 217 // We may not be able to sastify the desired alignment specification of 218 // the TargetRegisterClass if the stack alignment is smaller. Use the min. 219 Align = std::min(Align, StackAlign); 220 FrameIdx = FFI->CreateStackObject(RC->getSize(), Align); 221 if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx; 222 if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx; 223 } else { 224 // Spill it to the stack where we must. 225 FrameIdx = FFI->CreateFixedObject(RC->getSize(), FixedSlot->second); 226 } 227 CSI[i].setFrameIdx(FrameIdx); 228 } 229 230 FFI->setCalleeSavedInfo(CSI); 231} 232 233/// saveCalleeSavedRegisters - Insert spill code for any callee saved registers 234/// that are modified in the function. 235/// 236void PEI::saveCalleeSavedRegisters(MachineFunction &Fn) { 237 // Get callee saved register information. 238 MachineFrameInfo *FFI = Fn.getFrameInfo(); 239 const std::vector<CalleeSavedInfo> &CSI = FFI->getCalleeSavedInfo(); 240 241 // Early exit if no callee saved registers are modified! 242 if (CSI.empty()) 243 return; 244 245 const TargetInstrInfo &TII = *Fn.getTarget().getInstrInfo(); 246 247 // Now that we have a stack slot for each register to be saved, insert spill 248 // code into the entry block. 249 MachineBasicBlock *MBB = Fn.begin(); 250 MachineBasicBlock::iterator I = MBB->begin(); 251 252 if (!TII.spillCalleeSavedRegisters(*MBB, I, CSI)) { 253 for (unsigned i = 0, e = CSI.size(); i != e; ++i) { 254 // Add the callee-saved register as live-in. It's killed at the spill. 255 MBB->addLiveIn(CSI[i].getReg()); 256 257 // Insert the spill to the stack frame. 258 TII.storeRegToStackSlot(*MBB, I, CSI[i].getReg(), true, 259 CSI[i].getFrameIdx(), CSI[i].getRegClass()); 260 } 261 } 262 263 // Add code to restore the callee-save registers in each exiting block. 264 for (MachineFunction::iterator FI = Fn.begin(), E = Fn.end(); FI != E; ++FI) 265 // If last instruction is a return instruction, add an epilogue. 266 if (!FI->empty() && FI->back().getDesc().isReturn()) { 267 MBB = FI; 268 I = MBB->end(); --I; 269 270 // Skip over all terminator instructions, which are part of the return 271 // sequence. 272 MachineBasicBlock::iterator I2 = I; 273 while (I2 != MBB->begin() && (--I2)->getDesc().isTerminator()) 274 I = I2; 275 276 bool AtStart = I == MBB->begin(); 277 MachineBasicBlock::iterator BeforeI = I; 278 if (!AtStart) 279 --BeforeI; 280 281 // Restore all registers immediately before the return and any terminators 282 // that preceed it. 283 if (!TII.restoreCalleeSavedRegisters(*MBB, I, CSI)) { 284 for (unsigned i = 0, e = CSI.size(); i != e; ++i) { 285 TII.loadRegFromStackSlot(*MBB, I, CSI[i].getReg(), 286 CSI[i].getFrameIdx(), 287 CSI[i].getRegClass()); 288 assert(I != MBB->begin() && 289 "loadRegFromStackSlot didn't insert any code!"); 290 // Insert in reverse order. loadRegFromStackSlot can insert multiple 291 // instructions. 292 if (AtStart) 293 I = MBB->begin(); 294 else { 295 I = BeforeI; 296 ++I; 297 } 298 } 299 } 300 } 301} 302 303 304/// calculateFrameObjectOffsets - Calculate actual frame offsets for all of the 305/// abstract stack objects. 306/// 307void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { 308 const TargetFrameInfo &TFI = *Fn.getTarget().getFrameInfo(); 309 310 bool StackGrowsDown = 311 TFI.getStackGrowthDirection() == TargetFrameInfo::StackGrowsDown; 312 313 // Loop over all of the stack objects, assigning sequential addresses... 314 MachineFrameInfo *FFI = Fn.getFrameInfo(); 315 316 unsigned MaxAlign = 0; 317 318 // Start at the beginning of the local area. 319 // The Offset is the distance from the stack top in the direction 320 // of stack growth -- so it's always positive. 321 int64_t Offset = TFI.getOffsetOfLocalArea(); 322 if (StackGrowsDown) 323 Offset = -Offset; 324 assert(Offset >= 0 325 && "Local area offset should be in direction of stack growth"); 326 327 // If there are fixed sized objects that are preallocated in the local area, 328 // non-fixed objects can't be allocated right at the start of local area. 329 // We currently don't support filling in holes in between fixed sized objects, 330 // so we adjust 'Offset' to point to the end of last fixed sized 331 // preallocated object. 332 for (int i = FFI->getObjectIndexBegin(); i != 0; ++i) { 333 int64_t FixedOff; 334 if (StackGrowsDown) { 335 // The maximum distance from the stack pointer is at lower address of 336 // the object -- which is given by offset. For down growing stack 337 // the offset is negative, so we negate the offset to get the distance. 338 FixedOff = -FFI->getObjectOffset(i); 339 } else { 340 // The maximum distance from the start pointer is at the upper 341 // address of the object. 342 FixedOff = FFI->getObjectOffset(i) + FFI->getObjectSize(i); 343 } 344 if (FixedOff > Offset) Offset = FixedOff; 345 } 346 347 // First assign frame offsets to stack objects that are used to spill 348 // callee saved registers. 349 if (StackGrowsDown) { 350 for (unsigned i = MinCSFrameIndex; i <= MaxCSFrameIndex; ++i) { 351 // If stack grows down, we need to add size of find the lowest 352 // address of the object. 353 Offset += FFI->getObjectSize(i); 354 355 unsigned Align = FFI->getObjectAlignment(i); 356 // If the alignment of this object is greater than that of the stack, then 357 // increase the stack alignment to match. 358 MaxAlign = std::max(MaxAlign, Align); 359 // Adjust to alignment boundary 360 Offset = (Offset+Align-1)/Align*Align; 361 362 FFI->setObjectOffset(i, -Offset); // Set the computed offset 363 } 364 } else { 365 for (unsigned i = MaxCSFrameIndex; i >= MinCSFrameIndex; --i) { 366 unsigned Align = FFI->getObjectAlignment(i); 367 // If the alignment of this object is greater than that of the stack, then 368 // increase the stack alignment to match. 369 MaxAlign = std::max(MaxAlign, Align); 370 // Adjust to alignment boundary 371 Offset = (Offset+Align-1)/Align*Align; 372 373 FFI->setObjectOffset(i, Offset); 374 Offset += FFI->getObjectSize(i); 375 } 376 } 377 378 // Make sure the special register scavenging spill slot is closest to the 379 // frame pointer if a frame pointer is required. 380 const TargetRegisterInfo *RegInfo = Fn.getTarget().getRegisterInfo(); 381 if (RS && RegInfo->hasFP(Fn)) { 382 int SFI = RS->getScavengingFrameIndex(); 383 if (SFI >= 0) { 384 // If stack grows down, we need to add size of the lowest 385 // address of the object. 386 if (StackGrowsDown) 387 Offset += FFI->getObjectSize(SFI); 388 389 unsigned Align = FFI->getObjectAlignment(SFI); 390 // Adjust to alignment boundary 391 Offset = (Offset+Align-1)/Align*Align; 392 393 if (StackGrowsDown) { 394 FFI->setObjectOffset(SFI, -Offset); // Set the computed offset 395 } else { 396 FFI->setObjectOffset(SFI, Offset); 397 Offset += FFI->getObjectSize(SFI); 398 } 399 } 400 } 401 402 // Then assign frame offsets to stack objects that are not used to spill 403 // callee saved registers. 404 for (unsigned i = 0, e = FFI->getObjectIndexEnd(); i != e; ++i) { 405 if (i >= MinCSFrameIndex && i <= MaxCSFrameIndex) 406 continue; 407 if (RS && (int)i == RS->getScavengingFrameIndex()) 408 continue; 409 if (FFI->isDeadObjectIndex(i)) 410 continue; 411 412 // If stack grows down, we need to add size of find the lowest 413 // address of the object. 414 if (StackGrowsDown) 415 Offset += FFI->getObjectSize(i); 416 417 unsigned Align = FFI->getObjectAlignment(i); 418 // If the alignment of this object is greater than that of the stack, then 419 // increase the stack alignment to match. 420 MaxAlign = std::max(MaxAlign, Align); 421 // Adjust to alignment boundary 422 Offset = (Offset+Align-1)/Align*Align; 423 424 if (StackGrowsDown) { 425 FFI->setObjectOffset(i, -Offset); // Set the computed offset 426 } else { 427 FFI->setObjectOffset(i, Offset); 428 Offset += FFI->getObjectSize(i); 429 } 430 } 431 432 // Make sure the special register scavenging spill slot is closest to the 433 // stack pointer. 434 if (RS && !RegInfo->hasFP(Fn)) { 435 int SFI = RS->getScavengingFrameIndex(); 436 if (SFI >= 0) { 437 // If stack grows down, we need to add size of find the lowest 438 // address of the object. 439 if (StackGrowsDown) 440 Offset += FFI->getObjectSize(SFI); 441 442 unsigned Align = FFI->getObjectAlignment(SFI); 443 // If the alignment of this object is greater than that of the 444 // stack, then increase the stack alignment to match. 445 MaxAlign = std::max(MaxAlign, Align); 446 // Adjust to alignment boundary 447 Offset = (Offset+Align-1)/Align*Align; 448 449 if (StackGrowsDown) { 450 FFI->setObjectOffset(SFI, -Offset); // Set the computed offset 451 } else { 452 FFI->setObjectOffset(SFI, Offset); 453 Offset += FFI->getObjectSize(SFI); 454 } 455 } 456 } 457 458 // Round up the size to a multiple of the alignment, but only if there are 459 // calls or alloca's in the function. This ensures that any calls to 460 // subroutines have their stack frames suitable aligned. 461 if (!RegInfo->targetHandlesStackFrameRounding() && 462 (FFI->hasCalls() || FFI->hasVarSizedObjects())) { 463 // If we have reserved argument space for call sites in the function 464 // immediately on entry to the current function, count it as part of the 465 // overall stack size. 466 if (RegInfo->hasReservedCallFrame(Fn)) 467 Offset += FFI->getMaxCallFrameSize(); 468 469 unsigned AlignMask = TFI.getStackAlignment() - 1; 470 Offset = (Offset + AlignMask) & ~uint64_t(AlignMask); 471 } 472 473 // Update frame info to pretend that this is part of the stack... 474 FFI->setStackSize(Offset+TFI.getOffsetOfLocalArea()); 475 476 // Remember the required stack alignment in case targets need it to perform 477 // dynamic stack alignment. 478 FFI->setMaxAlignment(MaxAlign); 479} 480 481 482/// insertPrologEpilogCode - Scan the function for modified callee saved 483/// registers, insert spill code for these callee saved registers, then add 484/// prolog and epilog code to the function. 485/// 486void PEI::insertPrologEpilogCode(MachineFunction &Fn) { 487 // Add prologue to the function... 488 Fn.getTarget().getRegisterInfo()->emitPrologue(Fn); 489 490 // Add epilogue to restore the callee-save registers in each exiting block 491 for (MachineFunction::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) { 492 // If last instruction is a return instruction, add an epilogue 493 if (!I->empty() && I->back().getDesc().isReturn()) 494 Fn.getTarget().getRegisterInfo()->emitEpilogue(Fn, *I); 495 } 496} 497 498 499/// replaceFrameIndices - Replace all MO_FrameIndex operands with physical 500/// register references and actual offsets. 501/// 502void PEI::replaceFrameIndices(MachineFunction &Fn) { 503 if (!Fn.getFrameInfo()->hasStackObjects()) return; // Nothing to do? 504 505 const TargetMachine &TM = Fn.getTarget(); 506 assert(TM.getRegisterInfo() && "TM::getRegisterInfo() must be implemented!"); 507 const TargetRegisterInfo &TRI = *TM.getRegisterInfo(); 508 const TargetFrameInfo *TFI = TM.getFrameInfo(); 509 bool StackGrowsDown = 510 TFI->getStackGrowthDirection() == TargetFrameInfo::StackGrowsDown; 511 int FrameSetupOpcode = TRI.getCallFrameSetupOpcode(); 512 int FrameDestroyOpcode = TRI.getCallFrameDestroyOpcode(); 513 514 for (MachineFunction::iterator BB = Fn.begin(), E = Fn.end(); BB != E; ++BB) { 515 int SPAdj = 0; // SP offset due to call frame setup / destroy. 516 if (RS) RS->enterBasicBlock(BB); 517 for (MachineBasicBlock::iterator I = BB->begin(); I != BB->end(); ) { 518 MachineInstr *MI = I; 519 520 if (I->getOpcode() == TargetInstrInfo::DECLARE) { 521 // Ignore it. 522 ++I; 523 continue; 524 } 525 526 if (I->getOpcode() == FrameSetupOpcode || 527 I->getOpcode() == FrameDestroyOpcode) { 528 // Remember how much SP has been adjusted to create the call 529 // frame. 530 int Size = I->getOperand(0).getImm(); 531 532 if ((!StackGrowsDown && I->getOpcode() == FrameSetupOpcode) || 533 (StackGrowsDown && I->getOpcode() == FrameDestroyOpcode)) 534 Size = -Size; 535 536 SPAdj += Size; 537 538 MachineBasicBlock::iterator PrevI = prior(I); 539 TRI.eliminateCallFramePseudoInstr(Fn, *BB, I); 540 541 // Visit the instructions created by eliminateCallFramePseudoInstr(). 542 I = next(PrevI); 543 continue; 544 } 545 546 bool DoIncr = true; 547 548 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) 549 if (MI->getOperand(i).isFrameIndex()) { 550 // Some instructions (e.g. inline asm instructions) can have 551 // multiple frame indices and/or cause eliminateFrameIndex 552 // to insert more than one instruction. We need the register 553 // scavenger to go through all of these instructions so that 554 // it can update its register information. We keep the 555 // iterator at the point before insertion so that we can 556 // revisit them in full. 557 bool AtBeginning = (I == BB->begin()); 558 if (!AtBeginning) --I; 559 560 // If this instruction has a FrameIndex operand, we need to 561 // use that target machine register info object to eliminate 562 // it. 563 TRI.eliminateFrameIndex(MI, SPAdj, RS); 564 565 // Reset the iterator if we were at the beginning of the BB. 566 if (AtBeginning) { 567 I = BB->begin(); 568 DoIncr = false; 569 } 570 571 MI = 0; 572 break; 573 } 574 575 if (DoIncr) ++I; 576 577 // Update register states. 578 if (RS && MI) RS->forward(MI); 579 } 580 581 assert(SPAdj == 0 && "Unbalanced call frame setup / destroy pairs?"); 582 } 583} 584