131d157ae1ac2cd9c787dc3c1d28e64c682803844Jia Liu//===-- ARMLoadStoreOptimizer.cpp - ARM load / store opt. pass ------------===// 2a8e2989ece6dc46df59b0768184028257f913843Evan Cheng// 3a8e2989ece6dc46df59b0768184028257f913843Evan Cheng// The LLVM Compiler Infrastructure 4a8e2989ece6dc46df59b0768184028257f913843Evan Cheng// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 7a8e2989ece6dc46df59b0768184028257f913843Evan Cheng// 8a8e2989ece6dc46df59b0768184028257f913843Evan Cheng//===----------------------------------------------------------------------===// 9a8e2989ece6dc46df59b0768184028257f913843Evan Cheng// 10a8e2989ece6dc46df59b0768184028257f913843Evan Cheng// This file contains a pass that performs load / store related peephole 11a8e2989ece6dc46df59b0768184028257f913843Evan Cheng// optimizations. This pass should be run after register allocation. 12a8e2989ece6dc46df59b0768184028257f913843Evan Cheng// 13a8e2989ece6dc46df59b0768184028257f913843Evan Cheng//===----------------------------------------------------------------------===// 14a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 15a8e2989ece6dc46df59b0768184028257f913843Evan Cheng#define DEBUG_TYPE "arm-ldst-opt" 16a8e2989ece6dc46df59b0768184028257f913843Evan Cheng#include "ARM.h" 178fb903604e83dfd63659c919042bf2bfed3c940fEvan Cheng#include "ARMBaseInstrInfo.h" 180e5233a9e5ee9385c6a940e3985194d77bee0bbbCraig Topper#include "ARMBaseRegisterInfo.h" 19603b83ebcdfb9c27e44c1da16f2799755e3e3022Evan Cheng#include "ARMMachineFunctionInfo.h" 20ee04a6d3a40c3017124e3fd89a0db473a2824498Evan Cheng#include "MCTargetDesc/ARMAddressingModes.h" 21d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/DenseMap.h" 22d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/STLExtras.h" 23d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/SmallPtrSet.h" 24d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/SmallSet.h" 25d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/SmallVector.h" 26d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/Statistic.h" 27a8e2989ece6dc46df59b0768184028257f913843Evan Cheng#include "llvm/CodeGen/MachineBasicBlock.h" 28a8e2989ece6dc46df59b0768184028257f913843Evan Cheng#include "llvm/CodeGen/MachineFunctionPass.h" 29a8e2989ece6dc46df59b0768184028257f913843Evan Cheng#include "llvm/CodeGen/MachineInstr.h" 30a8e2989ece6dc46df59b0768184028257f913843Evan Cheng#include "llvm/CodeGen/MachineInstrBuilder.h" 31e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng#include "llvm/CodeGen/MachineRegisterInfo.h" 32cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng#include "llvm/CodeGen/RegisterScavenging.h" 33ee04a6d3a40c3017124e3fd89a0db473a2824498Evan Cheng#include "llvm/CodeGen/SelectionDAGNodes.h" 340b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/DataLayout.h" 350b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/DerivedTypes.h" 360b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Function.h" 37d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Support/Debug.h" 38d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Support/ErrorHandling.h" 39d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Support/raw_ostream.h" 40a8e2989ece6dc46df59b0768184028257f913843Evan Cheng#include "llvm/Target/TargetInstrInfo.h" 41a8e2989ece6dc46df59b0768184028257f913843Evan Cheng#include "llvm/Target/TargetMachine.h" 42358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng#include "llvm/Target/TargetRegisterInfo.h" 43a8e2989ece6dc46df59b0768184028257f913843Evan Chengusing namespace llvm; 44a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 45a8e2989ece6dc46df59b0768184028257f913843Evan ChengSTATISTIC(NumLDMGened , "Number of ldm instructions generated"); 46a8e2989ece6dc46df59b0768184028257f913843Evan ChengSTATISTIC(NumSTMGened , "Number of stm instructions generated"); 47e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim GrosbachSTATISTIC(NumVLDMGened, "Number of vldm instructions generated"); 48e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim GrosbachSTATISTIC(NumVSTMGened, "Number of vstm instructions generated"); 49e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan ChengSTATISTIC(NumLdStMoved, "Number of load / store instructions moved"); 50f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan ChengSTATISTIC(NumLDRDFormed,"Number of ldrd created before allocation"); 51f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan ChengSTATISTIC(NumSTRDFormed,"Number of strd created before allocation"); 52f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan ChengSTATISTIC(NumLDRD2LDM, "Number of ldrd instructions turned back into ldm"); 53f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan ChengSTATISTIC(NumSTRD2STM, "Number of strd instructions turned back into stm"); 54f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan ChengSTATISTIC(NumLDRD2LDR, "Number of ldrd instructions turned back into ldr's"); 55f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan ChengSTATISTIC(NumSTRD2STR, "Number of strd instructions turned back into str's"); 56e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 57e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng/// ARMAllocLoadStoreOpt - Post- register allocation pass the combine 58e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng/// load / store instructions to form ldm / stm instructions. 59a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 60a8e2989ece6dc46df59b0768184028257f913843Evan Chengnamespace { 616726b6d75a8b679068a58cb954ba97cf9d1690baNick Lewycky struct ARMLoadStoreOpt : public MachineFunctionPass { 621997473cf72957d0e70322e2fe6fe2ab141c58a6Devang Patel static char ID; 6390c579de5a383cee278acc3f7e7b9d0a656e6a35Owen Anderson ARMLoadStoreOpt() : MachineFunctionPass(ID) {} 64794fd75c67a2cdc128d67342c6d88a504d186896Devang Patel 65a8e2989ece6dc46df59b0768184028257f913843Evan Cheng const TargetInstrInfo *TII; 666f0d024a534af18d9e60b3ea757376cd8a3a980eDan Gohman const TargetRegisterInfo *TRI; 673568a1051efb9a9edbd4914b04b44e9d7bc1b004Evan Cheng const ARMSubtarget *STI; 68603b83ebcdfb9c27e44c1da16f2799755e3e3022Evan Cheng ARMFunctionInfo *AFI; 69cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng RegScavenger *RS; 7045032f28013aa69d07bf859da9d976947910f059Evan Cheng bool isThumb2; 71a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 72a8e2989ece6dc46df59b0768184028257f913843Evan Cheng virtual bool runOnMachineFunction(MachineFunction &Fn); 73a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 74a8e2989ece6dc46df59b0768184028257f913843Evan Cheng virtual const char *getPassName() const { 75a8e2989ece6dc46df59b0768184028257f913843Evan Cheng return "ARM load / store optimization pass"; 76a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 77a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 78a8e2989ece6dc46df59b0768184028257f913843Evan Cheng private: 79a8e2989ece6dc46df59b0768184028257f913843Evan Cheng struct MemOpQueueEntry { 80a8e2989ece6dc46df59b0768184028257f913843Evan Cheng int Offset; 81d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng unsigned Reg; 82d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng bool isKill; 83a8e2989ece6dc46df59b0768184028257f913843Evan Cheng unsigned Position; 84a8e2989ece6dc46df59b0768184028257f913843Evan Cheng MachineBasicBlock::iterator MBBI; 85a8e2989ece6dc46df59b0768184028257f913843Evan Cheng bool Merged; 86848b0c39b11801614c47e460248b60e8d40eb257Owen Anderson MemOpQueueEntry(int o, unsigned r, bool k, unsigned p, 87d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng MachineBasicBlock::iterator i) 88d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng : Offset(o), Reg(r), isKill(k), Position(p), MBBI(i), Merged(false) {} 89a8e2989ece6dc46df59b0768184028257f913843Evan Cheng }; 90a8e2989ece6dc46df59b0768184028257f913843Evan Cheng typedef SmallVector<MemOpQueueEntry,8> MemOpQueue; 91a8e2989ece6dc46df59b0768184028257f913843Evan Cheng typedef MemOpQueue::iterator MemOpQueueIter; 92a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 93925492279ae7d93180ebdd689c87cd58522e68f5Evan Cheng bool MergeOps(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, 9487d59e49e9008767896c4c8c80efdc172f1cbd18Evan Cheng int Offset, unsigned Base, bool BaseKill, int Opcode, 9587d59e49e9008767896c4c8c80efdc172f1cbd18Evan Cheng ARMCC::CondCodes Pred, unsigned PredReg, unsigned Scratch, 96dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen DebugLoc dl, 97dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen ArrayRef<std::pair<unsigned, bool> > Regs, 98dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen ArrayRef<unsigned> ImpDefs); 99f8e33e513f2620ea30fda7f17bc227729b7621b8Jakob Stoklund Olesen void MergeOpsUpdate(MachineBasicBlock &MBB, 1006528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen MemOpQueue &MemOps, 1016528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen unsigned memOpsBegin, 1026528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen unsigned memOpsEnd, 1036528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen unsigned insertAfter, 104f8e33e513f2620ea30fda7f17bc227729b7621b8Jakob Stoklund Olesen int Offset, 105f8e33e513f2620ea30fda7f17bc227729b7621b8Jakob Stoklund Olesen unsigned Base, 106f8e33e513f2620ea30fda7f17bc227729b7621b8Jakob Stoklund Olesen bool BaseKill, 107f8e33e513f2620ea30fda7f17bc227729b7621b8Jakob Stoklund Olesen int Opcode, 108f8e33e513f2620ea30fda7f17bc227729b7621b8Jakob Stoklund Olesen ARMCC::CondCodes Pred, 109f8e33e513f2620ea30fda7f17bc227729b7621b8Jakob Stoklund Olesen unsigned PredReg, 110f8e33e513f2620ea30fda7f17bc227729b7621b8Jakob Stoklund Olesen unsigned Scratch, 111f8e33e513f2620ea30fda7f17bc227729b7621b8Jakob Stoklund Olesen DebugLoc dl, 112f8e33e513f2620ea30fda7f17bc227729b7621b8Jakob Stoklund Olesen SmallVector<MachineBasicBlock::iterator, 4> &Merges); 1135ba71887f918a9da82140210494608df020dcbd5Evan Cheng void MergeLDR_STR(MachineBasicBlock &MBB, unsigned SIndex, unsigned Base, 1145ba71887f918a9da82140210494608df020dcbd5Evan Cheng int Opcode, unsigned Size, 1155ba71887f918a9da82140210494608df020dcbd5Evan Cheng ARMCC::CondCodes Pred, unsigned PredReg, 1165ba71887f918a9da82140210494608df020dcbd5Evan Cheng unsigned Scratch, MemOpQueue &MemOps, 1175ba71887f918a9da82140210494608df020dcbd5Evan Cheng SmallVector<MachineBasicBlock::iterator, 4> &Merges); 118a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 11911788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng void AdvanceRS(MachineBasicBlock &MBB, MemOpQueue &MemOps); 120358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng bool FixInvalidRegPairOp(MachineBasicBlock &MBB, 121358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng MachineBasicBlock::iterator &MBBI); 12245032f28013aa69d07bf859da9d976947910f059Evan Cheng bool MergeBaseUpdateLoadStore(MachineBasicBlock &MBB, 12345032f28013aa69d07bf859da9d976947910f059Evan Cheng MachineBasicBlock::iterator MBBI, 12445032f28013aa69d07bf859da9d976947910f059Evan Cheng const TargetInstrInfo *TII, 12545032f28013aa69d07bf859da9d976947910f059Evan Cheng bool &Advance, 12645032f28013aa69d07bf859da9d976947910f059Evan Cheng MachineBasicBlock::iterator &I); 12745032f28013aa69d07bf859da9d976947910f059Evan Cheng bool MergeBaseUpdateLSMultiple(MachineBasicBlock &MBB, 12845032f28013aa69d07bf859da9d976947910f059Evan Cheng MachineBasicBlock::iterator MBBI, 12945032f28013aa69d07bf859da9d976947910f059Evan Cheng bool &Advance, 13045032f28013aa69d07bf859da9d976947910f059Evan Cheng MachineBasicBlock::iterator &I); 131a8e2989ece6dc46df59b0768184028257f913843Evan Cheng bool LoadStoreMultipleOpti(MachineBasicBlock &MBB); 132a8e2989ece6dc46df59b0768184028257f913843Evan Cheng bool MergeReturnIntoLDM(MachineBasicBlock &MBB); 133a8e2989ece6dc46df59b0768184028257f913843Evan Cheng }; 1341997473cf72957d0e70322e2fe6fe2ab141c58a6Devang Patel char ARMLoadStoreOpt::ID = 0; 135a8e2989ece6dc46df59b0768184028257f913843Evan Cheng} 136a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 13773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendlingstatic int getLoadStoreMultipleOpcode(int Opcode, ARM_AM::AMSubMode Mode) { 138a8e2989ece6dc46df59b0768184028257f913843Evan Cheng switch (Opcode) { 13973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled opcode!"); 1403e5561247202bae994dd259a2d8dc4eff8f799f3Jim Grosbach case ARM::LDRi12: 141fe60104ac97f3a8736dcfbfdf9547c7b7cc7b951Dan Gohman ++NumLDMGened; 14273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Mode) { 14373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled submode!"); 14473fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ia: return ARM::LDMIA; 14573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::da: return ARM::LDMDA; 14673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::db: return ARM::LDMDB; 14773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ib: return ARM::LDMIB; 14873fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 1497e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach case ARM::STRi12: 150fe60104ac97f3a8736dcfbfdf9547c7b7cc7b951Dan Gohman ++NumSTMGened; 15173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Mode) { 15273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled submode!"); 15373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ia: return ARM::STMIA; 15473fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::da: return ARM::STMDA; 15573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::db: return ARM::STMDB; 15673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ib: return ARM::STMIB; 15773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 15845032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2LDRi8: 15945032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2LDRi12: 160fe60104ac97f3a8736dcfbfdf9547c7b7cc7b951Dan Gohman ++NumLDMGened; 16173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Mode) { 16273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled submode!"); 16373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ia: return ARM::t2LDMIA; 16473fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::db: return ARM::t2LDMDB; 16573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 16645032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2STRi8: 16745032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2STRi12: 168fe60104ac97f3a8736dcfbfdf9547c7b7cc7b951Dan Gohman ++NumSTMGened; 16973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Mode) { 17073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled submode!"); 17173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ia: return ARM::t2STMIA; 17273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::db: return ARM::t2STMDB; 17373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 174e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim Grosbach case ARM::VLDRS: 175fe60104ac97f3a8736dcfbfdf9547c7b7cc7b951Dan Gohman ++NumVLDMGened; 17673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Mode) { 17773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled submode!"); 17873fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ia: return ARM::VLDMSIA; 179848b0c39b11801614c47e460248b60e8d40eb257Owen Anderson case ARM_AM::db: return 0; // Only VLDMSDB_UPD exists. 18073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 181e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim Grosbach case ARM::VSTRS: 182fe60104ac97f3a8736dcfbfdf9547c7b7cc7b951Dan Gohman ++NumVSTMGened; 18373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Mode) { 18473fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled submode!"); 18573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ia: return ARM::VSTMSIA; 186848b0c39b11801614c47e460248b60e8d40eb257Owen Anderson case ARM_AM::db: return 0; // Only VSTMSDB_UPD exists. 18773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 188e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim Grosbach case ARM::VLDRD: 189fe60104ac97f3a8736dcfbfdf9547c7b7cc7b951Dan Gohman ++NumVLDMGened; 19073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Mode) { 19173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled submode!"); 19273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ia: return ARM::VLDMDIA; 193848b0c39b11801614c47e460248b60e8d40eb257Owen Anderson case ARM_AM::db: return 0; // Only VLDMDDB_UPD exists. 19473fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 195e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim Grosbach case ARM::VSTRD: 196fe60104ac97f3a8736dcfbfdf9547c7b7cc7b951Dan Gohman ++NumVSTMGened; 19773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Mode) { 19873fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled submode!"); 19973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ia: return ARM::VSTMDIA; 200848b0c39b11801614c47e460248b60e8d40eb257Owen Anderson case ARM_AM::db: return 0; // Only VSTMDDB_UPD exists. 20173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 202a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 203a8e2989ece6dc46df59b0768184028257f913843Evan Cheng} 204a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 2052567eec4233d58a2a0cbdcafca9420452689b395Bill Wendlingnamespace llvm { 2062567eec4233d58a2a0cbdcafca9420452689b395Bill Wendling namespace ARM_AM { 2072567eec4233d58a2a0cbdcafca9420452689b395Bill Wendling 2082567eec4233d58a2a0cbdcafca9420452689b395Bill WendlingAMSubMode getLoadStoreMultipleSubMode(int Opcode) { 20973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Opcode) { 21073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled opcode!"); 211707120047e0107cb15dd4bbb31613df129b13c7aBill Wendling case ARM::LDMIA_RET: 21273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::LDMIA: 213df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::LDMIA_UPD: 21473fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::STMIA: 215df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::STMIA_UPD: 216707120047e0107cb15dd4bbb31613df129b13c7aBill Wendling case ARM::t2LDMIA_RET: 21773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::t2LDMIA: 218df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::t2LDMIA_UPD: 21973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::t2STMIA: 220df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::t2STMIA_UPD: 22173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VLDMSIA: 222df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::VLDMSIA_UPD: 22373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VSTMSIA: 224df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::VSTMSIA_UPD: 22573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VLDMDIA: 226df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::VLDMDIA_UPD: 22773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VSTMDIA: 228df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::VSTMDIA_UPD: 22973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling return ARM_AM::ia; 23073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling 23173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::LDMDA: 232df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::LDMDA_UPD: 23373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::STMDA: 234df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::STMDA_UPD: 23573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling return ARM_AM::da; 23673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling 23773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::LDMDB: 238df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::LDMDB_UPD: 23973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::STMDB: 240df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::STMDB_UPD: 24173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::t2LDMDB: 242df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::t2LDMDB_UPD: 24373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::t2STMDB: 244df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::t2STMDB_UPD: 245df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::VLDMSDB_UPD: 246df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::VSTMSDB_UPD: 247df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::VLDMDDB_UPD: 248df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::VSTMDDB_UPD: 24973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling return ARM_AM::db; 25073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling 25173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::LDMIB: 252df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::LDMIB_UPD: 25373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::STMIB: 254df8d94da01ccae58c93b03a5fb93fa1f05799c2fBill Wendling case ARM::STMIB_UPD: 25573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling return ARM_AM::ib; 25673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 25773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling} 25873fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling 2592567eec4233d58a2a0cbdcafca9420452689b395Bill Wendling } // end namespace ARM_AM 2602567eec4233d58a2a0cbdcafca9420452689b395Bill Wendling} // end namespace llvm 2612567eec4233d58a2a0cbdcafca9420452689b395Bill Wendling 26227934da97bcb7a6a4949dfc0c449d99f43077e98Evan Chengstatic bool isT2i32Load(unsigned Opc) { 26327934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng return Opc == ARM::t2LDRi12 || Opc == ARM::t2LDRi8; 26427934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng} 26527934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng 26645032f28013aa69d07bf859da9d976947910f059Evan Chengstatic bool isi32Load(unsigned Opc) { 2673e5561247202bae994dd259a2d8dc4eff8f799f3Jim Grosbach return Opc == ARM::LDRi12 || isT2i32Load(Opc); 26827934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng} 26927934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng 27027934da97bcb7a6a4949dfc0c449d99f43077e98Evan Chengstatic bool isT2i32Store(unsigned Opc) { 27127934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng return Opc == ARM::t2STRi12 || Opc == ARM::t2STRi8; 27245032f28013aa69d07bf859da9d976947910f059Evan Cheng} 27345032f28013aa69d07bf859da9d976947910f059Evan Cheng 27445032f28013aa69d07bf859da9d976947910f059Evan Chengstatic bool isi32Store(unsigned Opc) { 2757e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach return Opc == ARM::STRi12 || isT2i32Store(Opc); 27645032f28013aa69d07bf859da9d976947910f059Evan Cheng} 27745032f28013aa69d07bf859da9d976947910f059Evan Cheng 278925492279ae7d93180ebdd689c87cd58522e68f5Evan Cheng/// MergeOps - Create and insert a LDM or STM with Base as base register and 279a8e2989ece6dc46df59b0768184028257f913843Evan Cheng/// registers in Regs as the register operands that would be loaded / stored. 280764ab52dd80310a205c9888bf166d09dab858f90Jim Grosbach/// It returns true if the transformation is done. 28187d59e49e9008767896c4c8c80efdc172f1cbd18Evan Chengbool 282925492279ae7d93180ebdd689c87cd58522e68f5Evan ChengARMLoadStoreOpt::MergeOps(MachineBasicBlock &MBB, 28387d59e49e9008767896c4c8c80efdc172f1cbd18Evan Cheng MachineBasicBlock::iterator MBBI, 28487d59e49e9008767896c4c8c80efdc172f1cbd18Evan Cheng int Offset, unsigned Base, bool BaseKill, 28587d59e49e9008767896c4c8c80efdc172f1cbd18Evan Cheng int Opcode, ARMCC::CondCodes Pred, 28687d59e49e9008767896c4c8c80efdc172f1cbd18Evan Cheng unsigned PredReg, unsigned Scratch, DebugLoc dl, 287dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen ArrayRef<std::pair<unsigned, bool> > Regs, 288dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen ArrayRef<unsigned> ImpDefs) { 289a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // Only a single register to load / store. Don't bother. 290a8e2989ece6dc46df59b0768184028257f913843Evan Cheng unsigned NumRegs = Regs.size(); 291a8e2989ece6dc46df59b0768184028257f913843Evan Cheng if (NumRegs <= 1) 292a8e2989ece6dc46df59b0768184028257f913843Evan Cheng return false; 293a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 294a8e2989ece6dc46df59b0768184028257f913843Evan Cheng ARM_AM::AMSubMode Mode = ARM_AM::ia; 29514805e2afd5aa754e13fd5bb99365ffe972e9696Bob Wilson // VFP and Thumb2 do not support IB or DA modes. 296d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson bool isNotVFP = isi32Load(Opcode) || isi32Store(Opcode); 29714805e2afd5aa754e13fd5bb99365ffe972e9696Bob Wilson bool haveIBAndDA = isNotVFP && !isThumb2; 29814805e2afd5aa754e13fd5bb99365ffe972e9696Bob Wilson if (Offset == 4 && haveIBAndDA) 299a8e2989ece6dc46df59b0768184028257f913843Evan Cheng Mode = ARM_AM::ib; 30014805e2afd5aa754e13fd5bb99365ffe972e9696Bob Wilson else if (Offset == -4 * (int)NumRegs + 4 && haveIBAndDA) 301a8e2989ece6dc46df59b0768184028257f913843Evan Cheng Mode = ARM_AM::da; 30214805e2afd5aa754e13fd5bb99365ffe972e9696Bob Wilson else if (Offset == -4 * (int)NumRegs && isNotVFP) 30314805e2afd5aa754e13fd5bb99365ffe972e9696Bob Wilson // VLDM/VSTM do not support DB mode without also updating the base reg. 304a8e2989ece6dc46df59b0768184028257f913843Evan Cheng Mode = ARM_AM::db; 30514805e2afd5aa754e13fd5bb99365ffe972e9696Bob Wilson else if (Offset != 0) { 306d0cfc99b308a805ffc130cde7317a0b0c626348fOwen Anderson // Check if this is a supported opcode before we insert instructions to 307d0cfc99b308a805ffc130cde7317a0b0c626348fOwen Anderson // calculate a new base register. 308d0cfc99b308a805ffc130cde7317a0b0c626348fOwen Anderson if (!getLoadStoreMultipleOpcode(Opcode, Mode)) return false; 309d0cfc99b308a805ffc130cde7317a0b0c626348fOwen Anderson 310a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // If starting offset isn't zero, insert a MI to materialize a new base. 311a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // But only do so if it is cost effective, i.e. merging more than two 312a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // loads / stores. 313a8e2989ece6dc46df59b0768184028257f913843Evan Cheng if (NumRegs <= 2) 314a8e2989ece6dc46df59b0768184028257f913843Evan Cheng return false; 315a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 316a8e2989ece6dc46df59b0768184028257f913843Evan Cheng unsigned NewBase; 31745032f28013aa69d07bf859da9d976947910f059Evan Cheng if (isi32Load(Opcode)) 318a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // If it is a load, then just use one of the destination register to 319a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // use as the new base. 320a90f3408b3aca71cd438efa2c539af041430e059Evan Cheng NewBase = Regs[NumRegs-1].first; 321a8e2989ece6dc46df59b0768184028257f913843Evan Cheng else { 3220ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng // Use the scratch register to use as a new base. 3230ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng NewBase = Scratch; 324a90f3408b3aca71cd438efa2c539af041430e059Evan Cheng if (NewBase == 0) 325a90f3408b3aca71cd438efa2c539af041430e059Evan Cheng return false; 326a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 327f6fd90910a552ad9883f031350ae517e26dfdb44Jim Grosbach int BaseOpc = !isThumb2 ? ARM::ADDri : ARM::t2ADDri; 328a8e2989ece6dc46df59b0768184028257f913843Evan Cheng if (Offset < 0) { 329f6fd90910a552ad9883f031350ae517e26dfdb44Jim Grosbach BaseOpc = !isThumb2 ? ARM::SUBri : ARM::t2SUBri; 330a8e2989ece6dc46df59b0768184028257f913843Evan Cheng Offset = - Offset; 331a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 33245032f28013aa69d07bf859da9d976947910f059Evan Cheng int ImmedOffset = isThumb2 33345032f28013aa69d07bf859da9d976947910f059Evan Cheng ? ARM_AM::getT2SOImmVal(Offset) : ARM_AM::getSOImmVal(Offset); 33445032f28013aa69d07bf859da9d976947910f059Evan Cheng if (ImmedOffset == -1) 33545032f28013aa69d07bf859da9d976947910f059Evan Cheng // FIXME: Try t2ADDri12 or t2SUBri12? 336a8e2989ece6dc46df59b0768184028257f913843Evan Cheng return false; // Probably not worth it then. 337a90f3408b3aca71cd438efa2c539af041430e059Evan Cheng 338b67284057ee130114055309eabe0bcd1af13777dDale Johannesen BuildMI(MBB, MBBI, dl, TII->get(BaseOpc), NewBase) 339e7cbe4118b7ddf05032ff8772a98c51e1637bb5cEvan Cheng .addReg(Base, getKillRegState(BaseKill)).addImm(Offset) 34013ab020ea08826f1b87db6ec3da63889a12e3d9dEvan Cheng .addImm(Pred).addReg(PredReg).addReg(0); 341a8e2989ece6dc46df59b0768184028257f913843Evan Cheng Base = NewBase; 342a90f3408b3aca71cd438efa2c539af041430e059Evan Cheng BaseKill = true; // New base is always killed right its use. 343a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 344a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 3458d95e0be13079339c2b84f245fb1665953e2bc77Bob Wilson bool isDef = (isi32Load(Opcode) || Opcode == ARM::VLDRS || 3468d95e0be13079339c2b84f245fb1665953e2bc77Bob Wilson Opcode == ARM::VLDRD); 34773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling Opcode = getLoadStoreMultipleOpcode(Opcode, Mode); 3489eae80051b6f6f5564b725221b2163a1f0d83672Owen Anderson if (!Opcode) return false; 349d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII->get(Opcode)) 350d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson .addReg(Base, getKillRegState(BaseKill)) 35173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling .addImm(Pred).addReg(PredReg); 352a8e2989ece6dc46df59b0768184028257f913843Evan Cheng for (unsigned i = 0; i != NumRegs; ++i) 353587daedce2d6c2b2d380b6a5843a6f8b6cfc79e4Bill Wendling MIB = MIB.addReg(Regs[i].first, getDefRegState(isDef) 354587daedce2d6c2b2d380b6a5843a6f8b6cfc79e4Bill Wendling | getKillRegState(Regs[i].second)); 355a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 356dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen // Add implicit defs for super-registers. 357dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen for (unsigned i = 0, e = ImpDefs.size(); i != e; ++i) 358dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen MIB.addReg(ImpDefs[i], RegState::ImplicitDefine); 359dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen 360a8e2989ece6dc46df59b0768184028257f913843Evan Cheng return true; 361a8e2989ece6dc46df59b0768184028257f913843Evan Cheng} 362a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 363f8e33e513f2620ea30fda7f17bc227729b7621b8Jakob Stoklund Olesen// MergeOpsUpdate - call MergeOps and update MemOps and merges accordingly on 364f8e33e513f2620ea30fda7f17bc227729b7621b8Jakob Stoklund Olesen// success. 365d95ea2da28901ca6e81645e444e528d66350d781Evan Chengvoid ARMLoadStoreOpt::MergeOpsUpdate(MachineBasicBlock &MBB, 366d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng MemOpQueue &memOps, 367d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng unsigned memOpsBegin, unsigned memOpsEnd, 368d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng unsigned insertAfter, int Offset, 369d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng unsigned Base, bool BaseKill, 370d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng int Opcode, 371d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng ARMCC::CondCodes Pred, unsigned PredReg, 372d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng unsigned Scratch, 373d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng DebugLoc dl, 374d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng SmallVector<MachineBasicBlock::iterator, 4> &Merges) { 3753063aed8d29cf2418fc1c3022a3dd9c8de0e4922Jakob Stoklund Olesen // First calculate which of the registers should be killed by the merged 3763063aed8d29cf2418fc1c3022a3dd9c8de0e4922Jakob Stoklund Olesen // instruction. 3771dbc38f52e2d6436ba79048013e42d96187993d8Jakob Stoklund Olesen const unsigned insertPos = memOps[insertAfter].Position; 378d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng SmallSet<unsigned, 4> KilledRegs; 379d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng DenseMap<unsigned, unsigned> Killer; 38079bb6dd363fd0a23040910b32d69a282063521bdJakob Stoklund Olesen for (unsigned i = 0, e = memOps.size(); i != e; ++i) { 38179bb6dd363fd0a23040910b32d69a282063521bdJakob Stoklund Olesen if (i == memOpsBegin) { 38279bb6dd363fd0a23040910b32d69a282063521bdJakob Stoklund Olesen i = memOpsEnd; 38379bb6dd363fd0a23040910b32d69a282063521bdJakob Stoklund Olesen if (i == e) 38479bb6dd363fd0a23040910b32d69a282063521bdJakob Stoklund Olesen break; 385d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng } 386d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng if (memOps[i].Position < insertPos && memOps[i].isKill) { 387d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng unsigned Reg = memOps[i].Reg; 388d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng KilledRegs.insert(Reg); 389d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng Killer[Reg] = i; 390d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng } 391d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng } 392d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng 393d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng SmallVector<std::pair<unsigned, bool>, 8> Regs; 394dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen SmallVector<unsigned, 8> ImpDefs; 3956528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen for (unsigned i = memOpsBegin; i < memOpsEnd; ++i) { 396d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng unsigned Reg = memOps[i].Reg; 39779bb6dd363fd0a23040910b32d69a282063521bdJakob Stoklund Olesen // If we are inserting the merged operation after an operation that 3981dbc38f52e2d6436ba79048013e42d96187993d8Jakob Stoklund Olesen // uses the same register, make sure to transfer any kill flag. 399d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng bool isKill = memOps[i].isKill || KilledRegs.count(Reg); 4001dbc38f52e2d6436ba79048013e42d96187993d8Jakob Stoklund Olesen Regs.push_back(std::make_pair(Reg, isKill)); 401dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen 402dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen // Collect any implicit defs of super-registers. They must be preserved. 403dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen for (MIOperands MO(memOps[i].MBBI); MO.isValid(); ++MO) { 404dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen if (!MO->isReg() || !MO->isDef() || !MO->isImplicit() || MO->isDead()) 405dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen continue; 406dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen unsigned DefReg = MO->getReg(); 407dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen if (std::find(ImpDefs.begin(), ImpDefs.end(), DefReg) == ImpDefs.end()) 408dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen ImpDefs.push_back(DefReg); 409dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen } 4103063aed8d29cf2418fc1c3022a3dd9c8de0e4922Jakob Stoklund Olesen } 4113063aed8d29cf2418fc1c3022a3dd9c8de0e4922Jakob Stoklund Olesen 4126528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen // Try to do the merge. 4136528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen MachineBasicBlock::iterator Loc = memOps[insertAfter].MBBI; 414fe60104ac97f3a8736dcfbfdf9547c7b7cc7b951Dan Gohman ++Loc; 4156528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen if (!MergeOps(MBB, Loc, Offset, Base, BaseKill, Opcode, 416dc909bf46ba94bf123bf5c7273fa867bddffc4a2Jakob Stoklund Olesen Pred, PredReg, Scratch, dl, Regs, ImpDefs)) 417f8e33e513f2620ea30fda7f17bc227729b7621b8Jakob Stoklund Olesen return; 4183063aed8d29cf2418fc1c3022a3dd9c8de0e4922Jakob Stoklund Olesen 4193063aed8d29cf2418fc1c3022a3dd9c8de0e4922Jakob Stoklund Olesen // Merge succeeded, update records. 4206528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen Merges.push_back(prior(Loc)); 4216528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen for (unsigned i = memOpsBegin; i < memOpsEnd; ++i) { 42279bb6dd363fd0a23040910b32d69a282063521bdJakob Stoklund Olesen // Remove kill flags from any memops that come before insertPos. 423d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng if (Regs[i-memOpsBegin].second) { 424d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng unsigned Reg = Regs[i-memOpsBegin].first; 425d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng if (KilledRegs.count(Reg)) { 426d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng unsigned j = Killer[Reg]; 42779bb6dd363fd0a23040910b32d69a282063521bdJakob Stoklund Olesen int Idx = memOps[j].MBBI->findRegisterUseOperandIdx(Reg, true); 42879bb6dd363fd0a23040910b32d69a282063521bdJakob Stoklund Olesen assert(Idx >= 0 && "Cannot find killing operand"); 42979bb6dd363fd0a23040910b32d69a282063521bdJakob Stoklund Olesen memOps[j].MBBI->getOperand(Idx).setIsKill(false); 430253627967454246d347a5038ec971f6738f77f07Jakob Stoklund Olesen memOps[j].isKill = false; 431d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng } 43279bb6dd363fd0a23040910b32d69a282063521bdJakob Stoklund Olesen memOps[i].isKill = true; 433d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng } 4346528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen MBB.erase(memOps[i].MBBI); 43579bb6dd363fd0a23040910b32d69a282063521bdJakob Stoklund Olesen // Update this memop to refer to the merged instruction. 43679bb6dd363fd0a23040910b32d69a282063521bdJakob Stoklund Olesen // We may need to move kill flags again. 4376528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen memOps[i].Merged = true; 43879bb6dd363fd0a23040910b32d69a282063521bdJakob Stoklund Olesen memOps[i].MBBI = Merges.back(); 43979bb6dd363fd0a23040910b32d69a282063521bdJakob Stoklund Olesen memOps[i].Position = insertPos; 440f8e33e513f2620ea30fda7f17bc227729b7621b8Jakob Stoklund Olesen } 441f8e33e513f2620ea30fda7f17bc227729b7621b8Jakob Stoklund Olesen} 442f8e33e513f2620ea30fda7f17bc227729b7621b8Jakob Stoklund Olesen 443a90f3408b3aca71cd438efa2c539af041430e059Evan Cheng/// MergeLDR_STR - Merge a number of load / store instructions into one or more 444a90f3408b3aca71cd438efa2c539af041430e059Evan Cheng/// load / store multiple instructions. 4455ba71887f918a9da82140210494608df020dcbd5Evan Chengvoid 4460ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan ChengARMLoadStoreOpt::MergeLDR_STR(MachineBasicBlock &MBB, unsigned SIndex, 4475ba71887f918a9da82140210494608df020dcbd5Evan Cheng unsigned Base, int Opcode, unsigned Size, 4485ba71887f918a9da82140210494608df020dcbd5Evan Cheng ARMCC::CondCodes Pred, unsigned PredReg, 4495ba71887f918a9da82140210494608df020dcbd5Evan Cheng unsigned Scratch, MemOpQueue &MemOps, 4505ba71887f918a9da82140210494608df020dcbd5Evan Cheng SmallVector<MachineBasicBlock::iterator, 4> &Merges) { 451d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson bool isNotVFP = isi32Load(Opcode) || isi32Store(Opcode); 452a8e2989ece6dc46df59b0768184028257f913843Evan Cheng int Offset = MemOps[SIndex].Offset; 453a8e2989ece6dc46df59b0768184028257f913843Evan Cheng int SOffset = Offset; 4546528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen unsigned insertAfter = SIndex; 455a8e2989ece6dc46df59b0768184028257f913843Evan Cheng MachineBasicBlock::iterator Loc = MemOps[SIndex].MBBI; 45687d59e49e9008767896c4c8c80efdc172f1cbd18Evan Cheng DebugLoc dl = Loc->getDebugLoc(); 457158a2263bd221a6920c6c7f7f96d6272344efb8fJakob Stoklund Olesen const MachineOperand &PMO = Loc->getOperand(0); 458158a2263bd221a6920c6c7f7f96d6272344efb8fJakob Stoklund Olesen unsigned PReg = PMO.getReg(); 459df1c637ac4b6f6587c037be55cafed665c732d8fEric Christopher unsigned PRegNum = PMO.isUndef() ? UINT_MAX : TRI->getEncodingValue(PReg); 4609a52d0c352c852dc9517430442afc54f53e1d4ddJim Grosbach unsigned Count = 1; 46161f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson unsigned Limit = ~0U; 46261f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson 46361f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson // vldm / vstm limit are 32 for S variants, 16 for D variants. 46461f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson 46561f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson switch (Opcode) { 46661f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson default: break; 46761f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson case ARM::VSTRS: 46861f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson Limit = 32; 46961f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson break; 47061f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson case ARM::VSTRD: 47161f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson Limit = 16; 47261f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson break; 47361f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson case ARM::VLDRD: 47461f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson Limit = 16; 47561f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson break; 47661f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson case ARM::VLDRS: 47761f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson Limit = 32; 47861f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson break; 47961f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson } 48044bec52b1b7e9a3ac1efbae90db240b8c1ca2ad4Evan Cheng 481a8e2989ece6dc46df59b0768184028257f913843Evan Cheng for (unsigned i = SIndex+1, e = MemOps.size(); i != e; ++i) { 482a8e2989ece6dc46df59b0768184028257f913843Evan Cheng int NewOffset = MemOps[i].Offset; 483158a2263bd221a6920c6c7f7f96d6272344efb8fJakob Stoklund Olesen const MachineOperand &MO = MemOps[i].MBBI->getOperand(0); 484158a2263bd221a6920c6c7f7f96d6272344efb8fJakob Stoklund Olesen unsigned Reg = MO.getReg(); 485df1c637ac4b6f6587c037be55cafed665c732d8fEric Christopher unsigned RegNum = MO.isUndef() ? UINT_MAX : TRI->getEncodingValue(Reg); 48661f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson // Register numbers must be in ascending order. For VFP / NEON load and 48761f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson // store multiples, the registers must also be consecutive and within the 48861f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson // limit on the number of registers per instruction. 4893f7aa79c2a46d525cf0468ad74ef2395246a309fEvan Cheng if (Reg != ARM::SP && 4903f7aa79c2a46d525cf0468ad74ef2395246a309fEvan Cheng NewOffset == Offset + (int)Size && 49161f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson ((isNotVFP && RegNum > PRegNum) || 49261f3cf3bc9c4db657eda5a4b9f4f8079e65aba8fBob Wilson ((Count < Limit) && RegNum == PRegNum+1))) { 493a8e2989ece6dc46df59b0768184028257f913843Evan Cheng Offset += Size; 494a8e2989ece6dc46df59b0768184028257f913843Evan Cheng PRegNum = RegNum; 4959a52d0c352c852dc9517430442afc54f53e1d4ddJim Grosbach ++Count; 496a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } else { 497a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // Can't merge this in. Try merge the earlier ones first. 4986528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen MergeOpsUpdate(MBB, MemOps, SIndex, i, insertAfter, SOffset, 4996528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen Base, false, Opcode, Pred, PredReg, Scratch, dl, Merges); 5005ba71887f918a9da82140210494608df020dcbd5Evan Cheng MergeLDR_STR(MBB, i, Base, Opcode, Size, Pred, PredReg, Scratch, 5015ba71887f918a9da82140210494608df020dcbd5Evan Cheng MemOps, Merges); 5025ba71887f918a9da82140210494608df020dcbd5Evan Cheng return; 503a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 504a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 5056528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen if (MemOps[i].Position > MemOps[insertAfter].Position) 5066528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen insertAfter = i; 507a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 508a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 509faa510726f4b40aa4495e60e4d341c6467e3fb01Evan Cheng bool BaseKill = Loc->findRegisterUseOperandIdx(Base, true) != -1; 5106528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen MergeOpsUpdate(MBB, MemOps, SIndex, MemOps.size(), insertAfter, SOffset, 5116528966eaeaf38244ce165ef7c80326aa34a55caJakob Stoklund Olesen Base, BaseKill, Opcode, Pred, PredReg, Scratch, dl, Merges); 5125ba71887f918a9da82140210494608df020dcbd5Evan Cheng return; 513a8e2989ece6dc46df59b0768184028257f913843Evan Cheng} 514a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 5154ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Chengstatic bool definesCPSR(MachineInstr *MI) { 5164ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { 5174ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng const MachineOperand &MO = MI->getOperand(i); 5184ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng if (!MO.isReg()) 5194ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng continue; 5204ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng if (MO.isDef() && MO.getReg() == ARM::CPSR && !MO.isDead()) 5214ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng // If the instruction has live CPSR def, then it's not safe to fold it 5224ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng // into load / store. 5234ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng return true; 5244ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng } 5254ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng 5264ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng return false; 5274ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng} 5284ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng 5294ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Chengstatic bool isMatchingDecrement(MachineInstr *MI, unsigned Base, 5304ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng unsigned Bytes, unsigned Limit, 5314ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng ARMCC::CondCodes Pred, unsigned PredReg) { 5320e1d37904abbf7e8c4c478408253dd995f3040f4Evan Cheng unsigned MyPredReg = 0; 53345032f28013aa69d07bf859da9d976947910f059Evan Cheng if (!MI) 53445032f28013aa69d07bf859da9d976947910f059Evan Cheng return false; 5354ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng 5364ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng bool CheckCPSRDef = false; 5374ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng switch (MI->getOpcode()) { 5384ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng default: return false; 5394ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng case ARM::t2SUBri: 5404ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng case ARM::SUBri: 5414ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng CheckCPSRDef = true; 5424ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng // fallthrough 5434ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng case ARM::tSUBspi: 5444ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng break; 5454ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng } 54627934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng 54727934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng // Make sure the offset fits in 8 bits. 5483d38e8364a542abab6ca3d3aed658cf60d3112b4Bob Wilson if (Bytes == 0 || (Limit && Bytes >= Limit)) 54927934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng return false; 55045032f28013aa69d07bf859da9d976947910f059Evan Cheng 551861986401e05e437cb33bfd8320d510b956fe41eEvan Cheng unsigned Scale = (MI->getOpcode() == ARM::tSUBspi) ? 4 : 1; // FIXME 5524ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng if (!(MI->getOperand(0).getReg() == Base && 5534ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng MI->getOperand(1).getReg() == Base && 5544ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng (MI->getOperand(2).getImm()*Scale) == Bytes && 555c89c744b69cecac576317a98322fd295e36e9886Craig Topper getInstrPredicate(MI, MyPredReg) == Pred && 5564ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng MyPredReg == PredReg)) 5574ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng return false; 5584ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng 5594ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng return CheckCPSRDef ? !definesCPSR(MI) : true; 560a8e2989ece6dc46df59b0768184028257f913843Evan Cheng} 561a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 5624ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Chengstatic bool isMatchingIncrement(MachineInstr *MI, unsigned Base, 5634ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng unsigned Bytes, unsigned Limit, 5644ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng ARMCC::CondCodes Pred, unsigned PredReg) { 5650e1d37904abbf7e8c4c478408253dd995f3040f4Evan Cheng unsigned MyPredReg = 0; 56645032f28013aa69d07bf859da9d976947910f059Evan Cheng if (!MI) 56745032f28013aa69d07bf859da9d976947910f059Evan Cheng return false; 5684ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng 5694ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng bool CheckCPSRDef = false; 5704ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng switch (MI->getOpcode()) { 5714ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng default: return false; 5724ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng case ARM::t2ADDri: 5734ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng case ARM::ADDri: 5744ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng CheckCPSRDef = true; 5754ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng // fallthrough 5764ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng case ARM::tADDspi: 5774ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng break; 5784ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng } 57927934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng 5803d38e8364a542abab6ca3d3aed658cf60d3112b4Bob Wilson if (Bytes == 0 || (Limit && Bytes >= Limit)) 58145032f28013aa69d07bf859da9d976947910f059Evan Cheng // Make sure the offset fits in 8 bits. 58227934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng return false; 58345032f28013aa69d07bf859da9d976947910f059Evan Cheng 584861986401e05e437cb33bfd8320d510b956fe41eEvan Cheng unsigned Scale = (MI->getOpcode() == ARM::tADDspi) ? 4 : 1; // FIXME 5854ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng if (!(MI->getOperand(0).getReg() == Base && 5864ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng MI->getOperand(1).getReg() == Base && 5874ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng (MI->getOperand(2).getImm()*Scale) == Bytes && 588c89c744b69cecac576317a98322fd295e36e9886Craig Topper getInstrPredicate(MI, MyPredReg) == Pred && 5894ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng MyPredReg == PredReg)) 5904ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng return false; 5914ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng 5924ee1c5cf6c18db0fa4946437f0bca3dbb313e21bEvan Cheng return CheckCPSRDef ? !definesCPSR(MI) : true; 593a8e2989ece6dc46df59b0768184028257f913843Evan Cheng} 594a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 595a8e2989ece6dc46df59b0768184028257f913843Evan Chengstatic inline unsigned getLSMultipleTransferSize(MachineInstr *MI) { 596a8e2989ece6dc46df59b0768184028257f913843Evan Cheng switch (MI->getOpcode()) { 597a8e2989ece6dc46df59b0768184028257f913843Evan Cheng default: return 0; 5983e5561247202bae994dd259a2d8dc4eff8f799f3Jim Grosbach case ARM::LDRi12: 5997e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach case ARM::STRi12: 60045032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2LDRi8: 60145032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2LDRi12: 60245032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2STRi8: 60345032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2STRi12: 604e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim Grosbach case ARM::VLDRS: 605e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim Grosbach case ARM::VSTRS: 606a8e2989ece6dc46df59b0768184028257f913843Evan Cheng return 4; 607e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim Grosbach case ARM::VLDRD: 608e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim Grosbach case ARM::VSTRD: 609a8e2989ece6dc46df59b0768184028257f913843Evan Cheng return 8; 61073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::LDMIA: 61173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::LDMDA: 61273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::LDMDB: 61373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::LDMIB: 61473fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::STMIA: 61573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::STMDA: 61673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::STMDB: 61773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::STMIB: 61873fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::t2LDMIA: 61973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::t2LDMDB: 62073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::t2STMIA: 62173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::t2STMDB: 62273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VLDMSIA: 62373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VSTMSIA: 624979927ab26a1832ee2ecd1edb17d50184756f474Bob Wilson return (MI->getNumOperands() - MI->getDesc().getNumOperands() + 1) * 4; 62573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VLDMDIA: 62673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VSTMDIA: 627979927ab26a1832ee2ecd1edb17d50184756f474Bob Wilson return (MI->getNumOperands() - MI->getDesc().getNumOperands() + 1) * 8; 628a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 629a8e2989ece6dc46df59b0768184028257f913843Evan Cheng} 630a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 63173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendlingstatic unsigned getUpdatingLSMultipleOpcode(unsigned Opc, 63273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling ARM_AM::AMSubMode Mode) { 633815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson switch (Opc) { 634815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson default: llvm_unreachable("Unhandled opcode!"); 63573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::LDMIA: 63673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::LDMDA: 63773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::LDMDB: 63873fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::LDMIB: 63973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Mode) { 64073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled submode!"); 64173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ia: return ARM::LDMIA_UPD; 64273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ib: return ARM::LDMIB_UPD; 64373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::da: return ARM::LDMDA_UPD; 64473fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::db: return ARM::LDMDB_UPD; 64573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 64673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::STMIA: 64773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::STMDA: 64873fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::STMDB: 64973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::STMIB: 65073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Mode) { 65173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled submode!"); 65273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ia: return ARM::STMIA_UPD; 65373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ib: return ARM::STMIB_UPD; 65473fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::da: return ARM::STMDA_UPD; 65573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::db: return ARM::STMDB_UPD; 65673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 65773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::t2LDMIA: 65873fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::t2LDMDB: 65973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Mode) { 66073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled submode!"); 66173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ia: return ARM::t2LDMIA_UPD; 66273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::db: return ARM::t2LDMDB_UPD; 66373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 66473fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::t2STMIA: 66573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::t2STMDB: 66673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Mode) { 66773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled submode!"); 66873fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ia: return ARM::t2STMIA_UPD; 66973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::db: return ARM::t2STMDB_UPD; 67073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 67173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VLDMSIA: 67273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Mode) { 67373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled submode!"); 67473fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ia: return ARM::VLDMSIA_UPD; 67573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::db: return ARM::VLDMSDB_UPD; 67673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 67773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VLDMDIA: 67873fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Mode) { 67973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled submode!"); 68073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ia: return ARM::VLDMDIA_UPD; 68173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::db: return ARM::VLDMDDB_UPD; 68273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 68373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VSTMSIA: 68473fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Mode) { 68573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled submode!"); 68673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ia: return ARM::VSTMSIA_UPD; 68773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::db: return ARM::VSTMSDB_UPD; 68873fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 68973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VSTMDIA: 69073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling switch (Mode) { 69173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling default: llvm_unreachable("Unhandled submode!"); 69273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::ia: return ARM::VSTMDIA_UPD; 69373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM_AM::db: return ARM::VSTMDDB_UPD; 69473fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling } 695815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson } 696815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson} 697815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson 69845032f28013aa69d07bf859da9d976947910f059Evan Cheng/// MergeBaseUpdateLSMultiple - Fold proceeding/trailing inc/dec of base 699e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim Grosbach/// register into the LDM/STM/VLDM{D|S}/VSTM{D|S} op when possible: 700a8e2989ece6dc46df59b0768184028257f913843Evan Cheng/// 701a8e2989ece6dc46df59b0768184028257f913843Evan Cheng/// stmia rn, <ra, rb, rc> 702a8e2989ece6dc46df59b0768184028257f913843Evan Cheng/// rn := rn + 4 * 3; 703a8e2989ece6dc46df59b0768184028257f913843Evan Cheng/// => 704a8e2989ece6dc46df59b0768184028257f913843Evan Cheng/// stmia rn!, <ra, rb, rc> 705a8e2989ece6dc46df59b0768184028257f913843Evan Cheng/// 706a8e2989ece6dc46df59b0768184028257f913843Evan Cheng/// rn := rn - 4 * 3; 707a8e2989ece6dc46df59b0768184028257f913843Evan Cheng/// ldmia rn, <ra, rb, rc> 708a8e2989ece6dc46df59b0768184028257f913843Evan Cheng/// => 709a8e2989ece6dc46df59b0768184028257f913843Evan Cheng/// ldmdb rn!, <ra, rb, rc> 71045032f28013aa69d07bf859da9d976947910f059Evan Chengbool ARMLoadStoreOpt::MergeBaseUpdateLSMultiple(MachineBasicBlock &MBB, 71145032f28013aa69d07bf859da9d976947910f059Evan Cheng MachineBasicBlock::iterator MBBI, 71245032f28013aa69d07bf859da9d976947910f059Evan Cheng bool &Advance, 71345032f28013aa69d07bf859da9d976947910f059Evan Cheng MachineBasicBlock::iterator &I) { 714a8e2989ece6dc46df59b0768184028257f913843Evan Cheng MachineInstr *MI = MBBI; 715a8e2989ece6dc46df59b0768184028257f913843Evan Cheng unsigned Base = MI->getOperand(0).getReg(); 716815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson bool BaseKill = MI->getOperand(0).isKill(); 717a8e2989ece6dc46df59b0768184028257f913843Evan Cheng unsigned Bytes = getLSMultipleTransferSize(MI); 7180e1d37904abbf7e8c4c478408253dd995f3040f4Evan Cheng unsigned PredReg = 0; 719c89c744b69cecac576317a98322fd295e36e9886Craig Topper ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg); 720a8e2989ece6dc46df59b0768184028257f913843Evan Cheng int Opcode = MI->getOpcode(); 721815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson DebugLoc dl = MI->getDebugLoc(); 722a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 723d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson // Can't use an updating ld/st if the base register is also a dest 724d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson // register. e.g. ldmdb r0!, {r0, r1, r2}. The behavior is undefined. 72573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling for (unsigned i = 2, e = MI->getNumOperands(); i != e; ++i) 726d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson if (MI->getOperand(i).getReg() == Base) 727d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson return false; 72873fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling 72973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling bool DoMerge = false; 7302567eec4233d58a2a0cbdcafca9420452689b395Bill Wendling ARM_AM::AMSubMode Mode = ARM_AM::getLoadStoreMultipleSubMode(Opcode); 731a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 732815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson // Try merging with the previous instruction. 7333de755bb407f91c494ae675fa6408a29b1954a52Jim Grosbach MachineBasicBlock::iterator BeginMBBI = MBB.begin(); 7343de755bb407f91c494ae675fa6408a29b1954a52Jim Grosbach if (MBBI != BeginMBBI) { 735815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson MachineBasicBlock::iterator PrevMBBI = prior(MBBI); 7363de755bb407f91c494ae675fa6408a29b1954a52Jim Grosbach while (PrevMBBI != BeginMBBI && PrevMBBI->isDebugValue()) 7373de755bb407f91c494ae675fa6408a29b1954a52Jim Grosbach --PrevMBBI; 738d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson if (Mode == ARM_AM::ia && 739d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson isMatchingDecrement(PrevMBBI, Base, Bytes, 0, Pred, PredReg)) { 740d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson Mode = ARM_AM::db; 741d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson DoMerge = true; 742d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson } else if (Mode == ARM_AM::ib && 743d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson isMatchingDecrement(PrevMBBI, Base, Bytes, 0, Pred, PredReg)) { 744d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson Mode = ARM_AM::da; 745d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson DoMerge = true; 746a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 747815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson if (DoMerge) 748815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson MBB.erase(PrevMBBI); 749815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson } 750a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 751815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson // Try merging with the next instruction. 7523de755bb407f91c494ae675fa6408a29b1954a52Jim Grosbach MachineBasicBlock::iterator EndMBBI = MBB.end(); 7533de755bb407f91c494ae675fa6408a29b1954a52Jim Grosbach if (!DoMerge && MBBI != EndMBBI) { 754815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson MachineBasicBlock::iterator NextMBBI = llvm::next(MBBI); 7553de755bb407f91c494ae675fa6408a29b1954a52Jim Grosbach while (NextMBBI != EndMBBI && NextMBBI->isDebugValue()) 7563de755bb407f91c494ae675fa6408a29b1954a52Jim Grosbach ++NextMBBI; 757d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson if ((Mode == ARM_AM::ia || Mode == ARM_AM::ib) && 758d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson isMatchingIncrement(NextMBBI, Base, Bytes, 0, Pred, PredReg)) { 759d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson DoMerge = true; 760d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson } else if ((Mode == ARM_AM::da || Mode == ARM_AM::db) && 761d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson isMatchingDecrement(NextMBBI, Base, Bytes, 0, Pred, PredReg)) { 762d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson DoMerge = true; 763a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 764815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson if (DoMerge) { 765815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson if (NextMBBI == I) { 766815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson Advance = true; 767815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson ++I; 768a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 769815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson MBB.erase(NextMBBI); 770a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 771a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 772a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 773815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson if (!DoMerge) 774815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson return false; 775815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson 77673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling unsigned NewOpc = getUpdatingLSMultipleOpcode(Opcode, Mode); 777815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII->get(NewOpc)) 778815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson .addReg(Base, getDefRegState(true)) // WB base register 779d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson .addReg(Base, getKillRegState(BaseKill)) 780d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson .addImm(Pred).addReg(PredReg); 78173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling 782815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson // Transfer the rest of operands. 78373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling for (unsigned OpNum = 3, e = MI->getNumOperands(); OpNum != e; ++OpNum) 784815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson MIB.addOperand(MI->getOperand(OpNum)); 78573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling 786815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson // Transfer memoperands. 787d7d030a44796adc73a6eaa939cd17e52047734c1Chris Lattner MIB->setMemRefs(MI->memoperands_begin(), MI->memoperands_end()); 788815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson 789815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson MBB.erase(MBBI); 790815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson return true; 791a8e2989ece6dc46df59b0768184028257f913843Evan Cheng} 792a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 79373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendlingstatic unsigned getPreIndexedLoadStoreOpcode(unsigned Opc, 79473fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling ARM_AM::AddrOpc Mode) { 795a8e2989ece6dc46df59b0768184028257f913843Evan Cheng switch (Opc) { 79673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::LDRi12: 7979ab0f25fc194b4315db1b87d38d4024054120bf6Owen Anderson return ARM::LDR_PRE_IMM; 79873fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::STRi12: 799793e79601f0fd68ba082fa2016018f80b2379460Owen Anderson return ARM::STR_PRE_IMM; 80073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VLDRS: 80173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling return Mode == ARM_AM::add ? ARM::VLDMSIA_UPD : ARM::VLDMSDB_UPD; 80273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VLDRD: 80373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling return Mode == ARM_AM::add ? ARM::VLDMDIA_UPD : ARM::VLDMDDB_UPD; 80473fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VSTRS: 80573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling return Mode == ARM_AM::add ? ARM::VSTMSIA_UPD : ARM::VSTMSDB_UPD; 80673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VSTRD: 80773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling return Mode == ARM_AM::add ? ARM::VSTMDIA_UPD : ARM::VSTMDDB_UPD; 80845032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2LDRi8: 80945032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2LDRi12: 81045032f28013aa69d07bf859da9d976947910f059Evan Cheng return ARM::t2LDR_PRE; 81145032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2STRi8: 81245032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2STRi12: 81345032f28013aa69d07bf859da9d976947910f059Evan Cheng return ARM::t2STR_PRE; 814c23197a26f34f559ea9797de51e187087c039c42Torok Edwin default: llvm_unreachable("Unhandled opcode!"); 815a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 816a8e2989ece6dc46df59b0768184028257f913843Evan Cheng} 817a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 81873fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendlingstatic unsigned getPostIndexedLoadStoreOpcode(unsigned Opc, 81973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling ARM_AM::AddrOpc Mode) { 820a8e2989ece6dc46df59b0768184028257f913843Evan Cheng switch (Opc) { 82173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::LDRi12: 822793e79601f0fd68ba082fa2016018f80b2379460Owen Anderson return ARM::LDR_POST_IMM; 82373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::STRi12: 824793e79601f0fd68ba082fa2016018f80b2379460Owen Anderson return ARM::STR_POST_IMM; 82573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VLDRS: 82673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling return Mode == ARM_AM::add ? ARM::VLDMSIA_UPD : ARM::VLDMSDB_UPD; 82773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VLDRD: 82873fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling return Mode == ARM_AM::add ? ARM::VLDMDIA_UPD : ARM::VLDMDDB_UPD; 82973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VSTRS: 83073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling return Mode == ARM_AM::add ? ARM::VSTMSIA_UPD : ARM::VSTMSDB_UPD; 83173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling case ARM::VSTRD: 83273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling return Mode == ARM_AM::add ? ARM::VSTMDIA_UPD : ARM::VSTMDDB_UPD; 83345032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2LDRi8: 83445032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2LDRi12: 83545032f28013aa69d07bf859da9d976947910f059Evan Cheng return ARM::t2LDR_POST; 83645032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2STRi8: 83745032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2STRi12: 83845032f28013aa69d07bf859da9d976947910f059Evan Cheng return ARM::t2STR_POST; 839c23197a26f34f559ea9797de51e187087c039c42Torok Edwin default: llvm_unreachable("Unhandled opcode!"); 840a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 841a8e2989ece6dc46df59b0768184028257f913843Evan Cheng} 842a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 84345032f28013aa69d07bf859da9d976947910f059Evan Cheng/// MergeBaseUpdateLoadStore - Fold proceeding/trailing inc/dec of base 844a8e2989ece6dc46df59b0768184028257f913843Evan Cheng/// register into the LDR/STR/FLD{D|S}/FST{D|S} op when possible: 84545032f28013aa69d07bf859da9d976947910f059Evan Chengbool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineBasicBlock &MBB, 84645032f28013aa69d07bf859da9d976947910f059Evan Cheng MachineBasicBlock::iterator MBBI, 84745032f28013aa69d07bf859da9d976947910f059Evan Cheng const TargetInstrInfo *TII, 84845032f28013aa69d07bf859da9d976947910f059Evan Cheng bool &Advance, 84945032f28013aa69d07bf859da9d976947910f059Evan Cheng MachineBasicBlock::iterator &I) { 850a8e2989ece6dc46df59b0768184028257f913843Evan Cheng MachineInstr *MI = MBBI; 851a8e2989ece6dc46df59b0768184028257f913843Evan Cheng unsigned Base = MI->getOperand(1).getReg(); 852a90f3408b3aca71cd438efa2c539af041430e059Evan Cheng bool BaseKill = MI->getOperand(1).isKill(); 853a8e2989ece6dc46df59b0768184028257f913843Evan Cheng unsigned Bytes = getLSMultipleTransferSize(MI); 854a8e2989ece6dc46df59b0768184028257f913843Evan Cheng int Opcode = MI->getOpcode(); 855b67284057ee130114055309eabe0bcd1af13777dDale Johannesen DebugLoc dl = MI->getDebugLoc(); 856e4193b20fd30e59c389814835ed2f6b3bc3b225cBob Wilson bool isAM5 = (Opcode == ARM::VLDRD || Opcode == ARM::VLDRS || 857e4193b20fd30e59c389814835ed2f6b3bc3b225cBob Wilson Opcode == ARM::VSTRD || Opcode == ARM::VSTRS); 8587e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach bool isAM2 = (Opcode == ARM::LDRi12 || Opcode == ARM::STRi12); 8597e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach if (isi32Load(Opcode) || isi32Store(Opcode)) 8603e5561247202bae994dd259a2d8dc4eff8f799f3Jim Grosbach if (MI->getOperand(2).getImm() != 0) 8613e5561247202bae994dd259a2d8dc4eff8f799f3Jim Grosbach return false; 862e4193b20fd30e59c389814835ed2f6b3bc3b225cBob Wilson if (isAM5 && ARM_AM::getAM5Offset(MI->getOperand(2).getImm()) != 0) 863a8e2989ece6dc46df59b0768184028257f913843Evan Cheng return false; 864a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 865e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim Grosbach bool isLd = isi32Load(Opcode) || Opcode == ARM::VLDRS || Opcode == ARM::VLDRD; 866a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // Can't do the merge if the destination register is the same as the would-be 867a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // writeback register. 868a8e2989ece6dc46df59b0768184028257f913843Evan Cheng if (isLd && MI->getOperand(0).getReg() == Base) 869a8e2989ece6dc46df59b0768184028257f913843Evan Cheng return false; 870a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 8710e1d37904abbf7e8c4c478408253dd995f3040f4Evan Cheng unsigned PredReg = 0; 872c89c744b69cecac576317a98322fd295e36e9886Craig Topper ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg); 873a8e2989ece6dc46df59b0768184028257f913843Evan Cheng bool DoMerge = false; 874a8e2989ece6dc46df59b0768184028257f913843Evan Cheng ARM_AM::AddrOpc AddSub = ARM_AM::add; 875a8e2989ece6dc46df59b0768184028257f913843Evan Cheng unsigned NewOpc = 0; 87627934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng // AM2 - 12 bits, thumb2 - 8 bits. 87727934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng unsigned Limit = isAM5 ? 0 : (isAM2 ? 0x1000 : 0x100); 878e4193b20fd30e59c389814835ed2f6b3bc3b225cBob Wilson 879e4193b20fd30e59c389814835ed2f6b3bc3b225cBob Wilson // Try merging with the previous instruction. 8803de755bb407f91c494ae675fa6408a29b1954a52Jim Grosbach MachineBasicBlock::iterator BeginMBBI = MBB.begin(); 8813de755bb407f91c494ae675fa6408a29b1954a52Jim Grosbach if (MBBI != BeginMBBI) { 882a8e2989ece6dc46df59b0768184028257f913843Evan Cheng MachineBasicBlock::iterator PrevMBBI = prior(MBBI); 8833de755bb407f91c494ae675fa6408a29b1954a52Jim Grosbach while (PrevMBBI != BeginMBBI && PrevMBBI->isDebugValue()) 8843de755bb407f91c494ae675fa6408a29b1954a52Jim Grosbach --PrevMBBI; 88527934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng if (isMatchingDecrement(PrevMBBI, Base, Bytes, Limit, Pred, PredReg)) { 886a8e2989ece6dc46df59b0768184028257f913843Evan Cheng DoMerge = true; 887a8e2989ece6dc46df59b0768184028257f913843Evan Cheng AddSub = ARM_AM::sub; 88827934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng } else if (!isAM5 && 88927934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng isMatchingIncrement(PrevMBBI, Base, Bytes, Limit,Pred,PredReg)) { 890a8e2989ece6dc46df59b0768184028257f913843Evan Cheng DoMerge = true; 891a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 892e4193b20fd30e59c389814835ed2f6b3bc3b225cBob Wilson if (DoMerge) { 89373fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling NewOpc = getPreIndexedLoadStoreOpcode(Opcode, AddSub); 894a8e2989ece6dc46df59b0768184028257f913843Evan Cheng MBB.erase(PrevMBBI); 895e4193b20fd30e59c389814835ed2f6b3bc3b225cBob Wilson } 896a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 897a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 898e4193b20fd30e59c389814835ed2f6b3bc3b225cBob Wilson // Try merging with the next instruction. 8996335ac67b60c69c7314b1fb16721ae4a51043fc0Jim Grosbach MachineBasicBlock::iterator EndMBBI = MBB.end(); 9003de755bb407f91c494ae675fa6408a29b1954a52Jim Grosbach if (!DoMerge && MBBI != EndMBBI) { 9017896c9f436a4eda5ec15e882a7505ba482a2fcd0Chris Lattner MachineBasicBlock::iterator NextMBBI = llvm::next(MBBI); 9023de755bb407f91c494ae675fa6408a29b1954a52Jim Grosbach while (NextMBBI != EndMBBI && NextMBBI->isDebugValue()) 9033de755bb407f91c494ae675fa6408a29b1954a52Jim Grosbach ++NextMBBI; 90427934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng if (!isAM5 && 90527934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng isMatchingDecrement(NextMBBI, Base, Bytes, Limit, Pred, PredReg)) { 906a8e2989ece6dc46df59b0768184028257f913843Evan Cheng DoMerge = true; 907a8e2989ece6dc46df59b0768184028257f913843Evan Cheng AddSub = ARM_AM::sub; 90827934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng } else if (isMatchingIncrement(NextMBBI, Base, Bytes, Limit,Pred,PredReg)) { 909a8e2989ece6dc46df59b0768184028257f913843Evan Cheng DoMerge = true; 910a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 911e71bff7405392ad5904f986724a65f965c0686e8Evan Cheng if (DoMerge) { 91273fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling NewOpc = getPostIndexedLoadStoreOpcode(Opcode, AddSub); 913e71bff7405392ad5904f986724a65f965c0686e8Evan Cheng if (NextMBBI == I) { 914e71bff7405392ad5904f986724a65f965c0686e8Evan Cheng Advance = true; 915e71bff7405392ad5904f986724a65f965c0686e8Evan Cheng ++I; 916e71bff7405392ad5904f986724a65f965c0686e8Evan Cheng } 917a8e2989ece6dc46df59b0768184028257f913843Evan Cheng MBB.erase(NextMBBI); 918e71bff7405392ad5904f986724a65f965c0686e8Evan Cheng } 919a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 920a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 921a8e2989ece6dc46df59b0768184028257f913843Evan Cheng if (!DoMerge) 922a8e2989ece6dc46df59b0768184028257f913843Evan Cheng return false; 923a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 9243943ac38c946aaac21dbe686978d098770d7679eBob Wilson if (isAM5) { 925815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson // VLDM[SD}_UPD, VSTM[SD]_UPD 926d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson // (There are no base-updating versions of VLDR/VSTR instructions, but the 927d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson // updating load/store-multiple instructions can be used with only one 928d4bfd54ec2947e73ab152c3c548e4dd4beb700baBob Wilson // register.) 9293943ac38c946aaac21dbe686978d098770d7679eBob Wilson MachineOperand &MO = MI->getOperand(0); 9303943ac38c946aaac21dbe686978d098770d7679eBob Wilson BuildMI(MBB, MBBI, dl, TII->get(NewOpc)) 931815baebe1c8dc02accf128ae10dff9a1742d3244Bob Wilson .addReg(Base, getDefRegState(true)) // WB base register 9323943ac38c946aaac21dbe686978d098770d7679eBob Wilson .addReg(Base, getKillRegState(isLd ? BaseKill : false)) 9333943ac38c946aaac21dbe686978d098770d7679eBob Wilson .addImm(Pred).addReg(PredReg) 9343943ac38c946aaac21dbe686978d098770d7679eBob Wilson .addReg(MO.getReg(), (isLd ? getDefRegState(true) : 9353943ac38c946aaac21dbe686978d098770d7679eBob Wilson getKillRegState(MO.isKill()))); 9363943ac38c946aaac21dbe686978d098770d7679eBob Wilson } else if (isLd) { 93710342123adec62151bf9060493dd13583c67ae52Jim Grosbach if (isAM2) { 93807700d486ec790327723d2a81fe9c66b2fb52016Owen Anderson // LDR_PRE, LDR_POST 93907700d486ec790327723d2a81fe9c66b2fb52016Owen Anderson if (NewOpc == ARM::LDR_PRE_IMM || NewOpc == ARM::LDRB_PRE_IMM) { 940acb274baacbc104d7ef3b826b443847f11548760Owen Anderson int Offset = AddSub == ARM_AM::sub ? -Bytes : Bytes; 94107700d486ec790327723d2a81fe9c66b2fb52016Owen Anderson BuildMI(MBB, MBBI, dl, TII->get(NewOpc), MI->getOperand(0).getReg()) 94207700d486ec790327723d2a81fe9c66b2fb52016Owen Anderson .addReg(Base, RegState::Define) 94307700d486ec790327723d2a81fe9c66b2fb52016Owen Anderson .addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg); 94407700d486ec790327723d2a81fe9c66b2fb52016Owen Anderson } else { 945acb274baacbc104d7ef3b826b443847f11548760Owen Anderson int Offset = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift); 94607700d486ec790327723d2a81fe9c66b2fb52016Owen Anderson BuildMI(MBB, MBBI, dl, TII->get(NewOpc), MI->getOperand(0).getReg()) 94707700d486ec790327723d2a81fe9c66b2fb52016Owen Anderson .addReg(Base, RegState::Define) 94807700d486ec790327723d2a81fe9c66b2fb52016Owen Anderson .addReg(Base).addReg(0).addImm(Offset).addImm(Pred).addReg(PredReg); 94907700d486ec790327723d2a81fe9c66b2fb52016Owen Anderson } 95010342123adec62151bf9060493dd13583c67ae52Jim Grosbach } else { 95110342123adec62151bf9060493dd13583c67ae52Jim Grosbach int Offset = AddSub == ARM_AM::sub ? -Bytes : Bytes; 95227934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng // t2LDR_PRE, t2LDR_POST 95327934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng BuildMI(MBB, MBBI, dl, TII->get(NewOpc), MI->getOperand(0).getReg()) 95427934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng .addReg(Base, RegState::Define) 95527934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng .addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg); 95610342123adec62151bf9060493dd13583c67ae52Jim Grosbach } 95727934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng } else { 95827934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng MachineOperand &MO = MI->getOperand(0); 95919dec207fcc0f04902b7f097b7771ba7abba43fbJim Grosbach // FIXME: post-indexed stores use am2offset_imm, which still encodes 96019dec207fcc0f04902b7f097b7771ba7abba43fbJim Grosbach // the vestigal zero-reg offset register. When that's fixed, this clause 96119dec207fcc0f04902b7f097b7771ba7abba43fbJim Grosbach // can be removed entirely. 96210342123adec62151bf9060493dd13583c67ae52Jim Grosbach if (isAM2 && NewOpc == ARM::STR_POST_IMM) { 96310342123adec62151bf9060493dd13583c67ae52Jim Grosbach int Offset = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift); 96427934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng // STR_PRE, STR_POST 96527934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng BuildMI(MBB, MBBI, dl, TII->get(NewOpc), Base) 96627934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng .addReg(MO.getReg(), getKillRegState(MO.isKill())) 96727934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng .addReg(Base).addReg(0).addImm(Offset).addImm(Pred).addReg(PredReg); 96810342123adec62151bf9060493dd13583c67ae52Jim Grosbach } else { 96910342123adec62151bf9060493dd13583c67ae52Jim Grosbach int Offset = AddSub == ARM_AM::sub ? -Bytes : Bytes; 97027934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng // t2STR_PRE, t2STR_POST 97127934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng BuildMI(MBB, MBBI, dl, TII->get(NewOpc), Base) 97227934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng .addReg(MO.getReg(), getKillRegState(MO.isKill())) 97327934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng .addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg); 97410342123adec62151bf9060493dd13583c67ae52Jim Grosbach } 975a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 976a8e2989ece6dc46df59b0768184028257f913843Evan Cheng MBB.erase(MBBI); 977a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 978a8e2989ece6dc46df59b0768184028257f913843Evan Cheng return true; 979a8e2989ece6dc46df59b0768184028257f913843Evan Cheng} 980a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 9817bb1c4054986e51682ff89170c740cc16b921236Eric Christopher/// isMemoryOp - Returns true if instruction is a memory operation that this 9827bb1c4054986e51682ff89170c740cc16b921236Eric Christopher/// pass is capable of operating on. 98345032f28013aa69d07bf859da9d976947910f059Evan Chengstatic bool isMemoryOp(const MachineInstr *MI) { 984628a79771ba6d293ba7b6f2615f022e1ae9c7c49Jakob Stoklund Olesen // When no memory operands are present, conservatively assume unaligned, 985628a79771ba6d293ba7b6f2615f022e1ae9c7c49Jakob Stoklund Olesen // volatile, unfoldable. 986628a79771ba6d293ba7b6f2615f022e1ae9c7c49Jakob Stoklund Olesen if (!MI->hasOneMemOperand()) 987628a79771ba6d293ba7b6f2615f022e1ae9c7c49Jakob Stoklund Olesen return false; 988069e100f9a79a63db177a521fd790f4d77d1209cJakob Stoklund Olesen 989628a79771ba6d293ba7b6f2615f022e1ae9c7c49Jakob Stoklund Olesen const MachineMemOperand *MMO = *MI->memoperands_begin(); 990069e100f9a79a63db177a521fd790f4d77d1209cJakob Stoklund Olesen 991628a79771ba6d293ba7b6f2615f022e1ae9c7c49Jakob Stoklund Olesen // Don't touch volatile memory accesses - we may be changing their order. 992628a79771ba6d293ba7b6f2615f022e1ae9c7c49Jakob Stoklund Olesen if (MMO->isVolatile()) 993628a79771ba6d293ba7b6f2615f022e1ae9c7c49Jakob Stoklund Olesen return false; 994628a79771ba6d293ba7b6f2615f022e1ae9c7c49Jakob Stoklund Olesen 995628a79771ba6d293ba7b6f2615f022e1ae9c7c49Jakob Stoklund Olesen // Unaligned ldr/str is emulated by some kernels, but unaligned ldm/stm is 996628a79771ba6d293ba7b6f2615f022e1ae9c7c49Jakob Stoklund Olesen // not. 997628a79771ba6d293ba7b6f2615f022e1ae9c7c49Jakob Stoklund Olesen if (MMO->getAlignment() < 4) 998628a79771ba6d293ba7b6f2615f022e1ae9c7c49Jakob Stoklund Olesen return false; 999069e100f9a79a63db177a521fd790f4d77d1209cJakob Stoklund Olesen 10009e6396d05ee43858f04b0f52eb7da6240845f530Jakob Stoklund Olesen // str <undef> could probably be eliminated entirely, but for now we just want 10019e6396d05ee43858f04b0f52eb7da6240845f530Jakob Stoklund Olesen // to avoid making a mess of it. 10029e6396d05ee43858f04b0f52eb7da6240845f530Jakob Stoklund Olesen // FIXME: Use str <undef> as a wildcard to enable better stm folding. 10039e6396d05ee43858f04b0f52eb7da6240845f530Jakob Stoklund Olesen if (MI->getNumOperands() > 0 && MI->getOperand(0).isReg() && 10049e6396d05ee43858f04b0f52eb7da6240845f530Jakob Stoklund Olesen MI->getOperand(0).isUndef()) 10059e6396d05ee43858f04b0f52eb7da6240845f530Jakob Stoklund Olesen return false; 10069e6396d05ee43858f04b0f52eb7da6240845f530Jakob Stoklund Olesen 1007bbf39b0fd9c83f1d46ca5f858e66de66fb64ec98Bob Wilson // Likewise don't mess with references to undefined addresses. 1008bbf39b0fd9c83f1d46ca5f858e66de66fb64ec98Bob Wilson if (MI->getNumOperands() > 1 && MI->getOperand(1).isReg() && 1009bbf39b0fd9c83f1d46ca5f858e66de66fb64ec98Bob Wilson MI->getOperand(1).isUndef()) 1010bbf39b0fd9c83f1d46ca5f858e66de66fb64ec98Bob Wilson return false; 1011bbf39b0fd9c83f1d46ca5f858e66de66fb64ec98Bob Wilson 1012cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng int Opcode = MI->getOpcode(); 1013cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng switch (Opcode) { 1014cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng default: break; 1015e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim Grosbach case ARM::VLDRS: 1016e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim Grosbach case ARM::VSTRS: 1017d735b8019b0f297d7c14b55adcd887af24d8e602Dan Gohman return MI->getOperand(1).isReg(); 1018e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim Grosbach case ARM::VLDRD: 1019e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim Grosbach case ARM::VSTRD: 1020d735b8019b0f297d7c14b55adcd887af24d8e602Dan Gohman return MI->getOperand(1).isReg(); 10213e5561247202bae994dd259a2d8dc4eff8f799f3Jim Grosbach case ARM::LDRi12: 10227e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach case ARM::STRi12: 102345032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2LDRi8: 102445032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2LDRi12: 102545032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2STRi8: 102645032f28013aa69d07bf859da9d976947910f059Evan Cheng case ARM::t2STRi12: 1027e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng return MI->getOperand(1).isReg(); 1028cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng } 1029cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng return false; 1030cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng} 1031cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng 103211788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng/// AdvanceRS - Advance register scavenger to just before the earliest memory 103311788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng/// op that is being merged. 103411788fde93fb74636aa333b2910d606d2c19ba9eEvan Chengvoid ARMLoadStoreOpt::AdvanceRS(MachineBasicBlock &MBB, MemOpQueue &MemOps) { 103511788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng MachineBasicBlock::iterator Loc = MemOps[0].MBBI; 103611788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng unsigned Position = MemOps[0].Position; 103711788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng for (unsigned i = 1, e = MemOps.size(); i != e; ++i) { 103811788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng if (MemOps[i].Position < Position) { 103911788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng Position = MemOps[i].Position; 104011788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng Loc = MemOps[i].MBBI; 104111788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng } 104211788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng } 104311788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng 104411788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng if (Loc != MBB.begin()) 104511788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng RS->forward(prior(Loc)); 104611788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng} 104711788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng 1048e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Chengstatic int getMemoryOpOffset(const MachineInstr *MI) { 1049e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng int Opcode = MI->getOpcode(); 1050358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng bool isAM3 = Opcode == ARM::LDRD || Opcode == ARM::STRD; 1051e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng unsigned NumOperands = MI->getDesc().getNumOperands(); 1052e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng unsigned OffField = MI->getOperand(NumOperands-3).getImm(); 105345032f28013aa69d07bf859da9d976947910f059Evan Cheng 105445032f28013aa69d07bf859da9d976947910f059Evan Cheng if (Opcode == ARM::t2LDRi12 || Opcode == ARM::t2LDRi8 || 105545032f28013aa69d07bf859da9d976947910f059Evan Cheng Opcode == ARM::t2STRi12 || Opcode == ARM::t2STRi8 || 10563e5561247202bae994dd259a2d8dc4eff8f799f3Jim Grosbach Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8 || 10577e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach Opcode == ARM::LDRi12 || Opcode == ARM::STRi12) 105845032f28013aa69d07bf859da9d976947910f059Evan Cheng return OffField; 105945032f28013aa69d07bf859da9d976947910f059Evan Cheng 10607e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach int Offset = isAM3 ? ARM_AM::getAM3Offset(OffField) 10617e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach : ARM_AM::getAM5Offset(OffField) * 4; 10627e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach if (isAM3) { 1063358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng if (ARM_AM::getAM3Op(OffField) == ARM_AM::sub) 1064358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng Offset = -Offset; 1065e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } else { 1066e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (ARM_AM::getAM5Op(OffField) == ARM_AM::sub) 1067e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng Offset = -Offset; 1068e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1069e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng return Offset; 1070e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng} 1071e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1072358dec51804ee52e47ea3a47c9248086e458ad7cEvan Chengstatic void InsertLDR_STR(MachineBasicBlock &MBB, 1073358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng MachineBasicBlock::iterator &MBBI, 10747e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach int Offset, bool isDef, 1075358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng DebugLoc dl, unsigned NewOpc, 1076e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng unsigned Reg, bool RegDeadKill, bool RegUndef, 1077e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng unsigned BaseReg, bool BaseKill, bool BaseUndef, 10787e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach bool OffKill, bool OffUndef, 1079358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng ARMCC::CondCodes Pred, unsigned PredReg, 1080e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng const TargetInstrInfo *TII, bool isT2) { 1081e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng if (isDef) { 1082e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MBBI->getDebugLoc(), 1083e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng TII->get(NewOpc)) 1084974fe5d69187bdf33b0e111ff72e965431df4191Evan Cheng .addReg(Reg, getDefRegState(true) | getDeadRegState(RegDeadKill)) 1085e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng .addReg(BaseReg, getKillRegState(BaseKill)|getUndefRegState(BaseUndef)); 1086e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng MIB.addImm(Offset).addImm(Pred).addReg(PredReg); 1087e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng } else { 1088e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MBBI->getDebugLoc(), 1089e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng TII->get(NewOpc)) 1090e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng .addReg(Reg, getKillRegState(RegDeadKill) | getUndefRegState(RegUndef)) 1091e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng .addReg(BaseReg, getKillRegState(BaseKill)|getUndefRegState(BaseUndef)); 1092e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng MIB.addImm(Offset).addImm(Pred).addReg(PredReg); 1093e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng } 1094358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng} 1095358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng 1096358dec51804ee52e47ea3a47c9248086e458ad7cEvan Chengbool ARMLoadStoreOpt::FixInvalidRegPairOp(MachineBasicBlock &MBB, 1097358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng MachineBasicBlock::iterator &MBBI) { 1098358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng MachineInstr *MI = &*MBBI; 1099358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng unsigned Opcode = MI->getOpcode(); 1100e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng if (Opcode == ARM::LDRD || Opcode == ARM::STRD || 1101e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8) { 11023568a1051efb9a9edbd4914b04b44e9d7bc1b004Evan Cheng const MachineOperand &BaseOp = MI->getOperand(2); 11033568a1051efb9a9edbd4914b04b44e9d7bc1b004Evan Cheng unsigned BaseReg = BaseOp.getReg(); 1104358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng unsigned EvenReg = MI->getOperand(0).getReg(); 1105358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng unsigned OddReg = MI->getOperand(1).getReg(); 1106358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng unsigned EvenRegNum = TRI->getDwarfRegNum(EvenReg, false); 1107358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng unsigned OddRegNum = TRI->getDwarfRegNum(OddReg, false); 11083568a1051efb9a9edbd4914b04b44e9d7bc1b004Evan Cheng // ARM errata 602117: LDRD with base in list may result in incorrect base 11093568a1051efb9a9edbd4914b04b44e9d7bc1b004Evan Cheng // register when interrupted or faulted. 111044ee4714a8c245d4fdfd03840efcf58c3f66c6bcEvan Cheng bool Errata602117 = EvenReg == BaseReg && STI->isCortexM3(); 11113568a1051efb9a9edbd4914b04b44e9d7bc1b004Evan Cheng if (!Errata602117 && 11123568a1051efb9a9edbd4914b04b44e9d7bc1b004Evan Cheng ((EvenRegNum & 1) == 0 && (EvenRegNum + 1) == OddRegNum)) 1113358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng return false; 1114358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng 1115d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng MachineBasicBlock::iterator NewBBI = MBBI; 1116e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng bool isT2 = Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8; 1117e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng bool isLd = Opcode == ARM::LDRD || Opcode == ARM::t2LDRDi8; 1118974fe5d69187bdf33b0e111ff72e965431df4191Evan Cheng bool EvenDeadKill = isLd ? 1119974fe5d69187bdf33b0e111ff72e965431df4191Evan Cheng MI->getOperand(0).isDead() : MI->getOperand(0).isKill(); 1120e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng bool EvenUndef = MI->getOperand(0).isUndef(); 1121974fe5d69187bdf33b0e111ff72e965431df4191Evan Cheng bool OddDeadKill = isLd ? 1122974fe5d69187bdf33b0e111ff72e965431df4191Evan Cheng MI->getOperand(1).isDead() : MI->getOperand(1).isKill(); 1123e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng bool OddUndef = MI->getOperand(1).isUndef(); 1124358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng bool BaseKill = BaseOp.isKill(); 1125e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng bool BaseUndef = BaseOp.isUndef(); 1126e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng bool OffKill = isT2 ? false : MI->getOperand(3).isKill(); 1127e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng bool OffUndef = isT2 ? false : MI->getOperand(3).isUndef(); 1128358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng int OffImm = getMemoryOpOffset(MI); 1129358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng unsigned PredReg = 0; 1130c89c744b69cecac576317a98322fd295e36e9886Craig Topper ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg); 1131358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng 11327e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach if (OddRegNum > EvenRegNum && OffImm == 0) { 1133358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng // Ascending register numbers and no offset. It's safe to change it to a 1134358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng // ldm or stm. 1135e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng unsigned NewOpc = (isLd) 113673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling ? (isT2 ? ARM::t2LDMIA : ARM::LDMIA) 113773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling : (isT2 ? ARM::t2STMIA : ARM::STMIA); 1138f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng if (isLd) { 1139f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(NewOpc)) 1140f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng .addReg(BaseReg, getKillRegState(BaseKill)) 1141f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng .addImm(Pred).addReg(PredReg) 1142974fe5d69187bdf33b0e111ff72e965431df4191Evan Cheng .addReg(EvenReg, getDefRegState(isLd) | getDeadRegState(EvenDeadKill)) 1143d20d6586759fe7a53ec8b1dde80622cda49e31b8Evan Cheng .addReg(OddReg, getDefRegState(isLd) | getDeadRegState(OddDeadKill)); 1144f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng ++NumLDRD2LDM; 1145f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng } else { 1146f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(NewOpc)) 1147f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng .addReg(BaseReg, getKillRegState(BaseKill)) 1148f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng .addImm(Pred).addReg(PredReg) 1149e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng .addReg(EvenReg, 1150e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng getKillRegState(EvenDeadKill) | getUndefRegState(EvenUndef)) 1151e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng .addReg(OddReg, 1152d20d6586759fe7a53ec8b1dde80622cda49e31b8Evan Cheng getKillRegState(OddDeadKill) | getUndefRegState(OddUndef)); 1153f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng ++NumSTRD2STM; 1154f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng } 1155d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng NewBBI = llvm::prior(MBBI); 1156358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng } else { 1157358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng // Split into two instructions. 1158e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng unsigned NewOpc = (isLd) 11593e5561247202bae994dd259a2d8dc4eff8f799f3Jim Grosbach ? (isT2 ? (OffImm < 0 ? ARM::t2LDRi8 : ARM::t2LDRi12) : ARM::LDRi12) 11607e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach : (isT2 ? (OffImm < 0 ? ARM::t2STRi8 : ARM::t2STRi12) : ARM::STRi12); 11612d620c571cb53993b788893b6ef73a6e2805411fJim Grosbach // Be extra careful for thumb2. t2LDRi8 can't reference a zero offset, 11622d620c571cb53993b788893b6ef73a6e2805411fJim Grosbach // so adjust and use t2LDRi12 here for that. 11632d620c571cb53993b788893b6ef73a6e2805411fJim Grosbach unsigned NewOpc2 = (isLd) 11642d620c571cb53993b788893b6ef73a6e2805411fJim Grosbach ? (isT2 ? (OffImm+4 < 0 ? ARM::t2LDRi8 : ARM::t2LDRi12) : ARM::LDRi12) 11652d620c571cb53993b788893b6ef73a6e2805411fJim Grosbach : (isT2 ? (OffImm+4 < 0 ? ARM::t2STRi8 : ARM::t2STRi12) : ARM::STRi12); 1166358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng DebugLoc dl = MBBI->getDebugLoc(); 1167358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng // If this is a load and base register is killed, it may have been 1168358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng // re-defed by the load, make sure the first load does not clobber it. 1169f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng if (isLd && 1170358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng (BaseKill || OffKill) && 11717e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach (TRI->regsOverlap(EvenReg, BaseReg))) { 11727e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach assert(!TRI->regsOverlap(OddReg, BaseReg)); 11732d620c571cb53993b788893b6ef73a6e2805411fJim Grosbach InsertLDR_STR(MBB, MBBI, OffImm+4, isLd, dl, NewOpc2, 1174e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng OddReg, OddDeadKill, false, 11757e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach BaseReg, false, BaseUndef, false, OffUndef, 1176e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng Pred, PredReg, TII, isT2); 1177d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng NewBBI = llvm::prior(MBBI); 1178e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng InsertLDR_STR(MBB, MBBI, OffImm, isLd, dl, NewOpc, 1179e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng EvenReg, EvenDeadKill, false, 11807e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach BaseReg, BaseKill, BaseUndef, OffKill, OffUndef, 1181e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng Pred, PredReg, TII, isT2); 1182358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng } else { 11830cd22dd7383111192571884eb941ac2ccb668025Evan Cheng if (OddReg == EvenReg && EvenDeadKill) { 118418f30e6f5e80787808fe1455742452a5210afe07Jim Grosbach // If the two source operands are the same, the kill marker is 118518f30e6f5e80787808fe1455742452a5210afe07Jim Grosbach // probably on the first one. e.g. 11860cd22dd7383111192571884eb941ac2ccb668025Evan Cheng // t2STRDi8 %R5<kill>, %R5, %R9<kill>, 0, 14, %reg0 11870cd22dd7383111192571884eb941ac2ccb668025Evan Cheng EvenDeadKill = false; 11880cd22dd7383111192571884eb941ac2ccb668025Evan Cheng OddDeadKill = true; 11890cd22dd7383111192571884eb941ac2ccb668025Evan Cheng } 1190892143ff70b5e7d0cb06e7c4596c232347806b17Jakob Stoklund Olesen // Never kill the base register in the first instruction. 1191892143ff70b5e7d0cb06e7c4596c232347806b17Jakob Stoklund Olesen if (EvenReg == BaseReg) 1192892143ff70b5e7d0cb06e7c4596c232347806b17Jakob Stoklund Olesen EvenDeadKill = false; 1193974fe5d69187bdf33b0e111ff72e965431df4191Evan Cheng InsertLDR_STR(MBB, MBBI, OffImm, isLd, dl, NewOpc, 1194e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng EvenReg, EvenDeadKill, EvenUndef, 11957e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach BaseReg, false, BaseUndef, false, OffUndef, 1196e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng Pred, PredReg, TII, isT2); 1197d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng NewBBI = llvm::prior(MBBI); 11982d620c571cb53993b788893b6ef73a6e2805411fJim Grosbach InsertLDR_STR(MBB, MBBI, OffImm+4, isLd, dl, NewOpc2, 1199e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng OddReg, OddDeadKill, OddUndef, 12007e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach BaseReg, BaseKill, BaseUndef, OffKill, OffUndef, 1201e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng Pred, PredReg, TII, isT2); 1202358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng } 1203f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng if (isLd) 1204f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng ++NumLDRD2LDR; 1205f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng else 1206f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng ++NumSTRD2STR; 1207358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng } 1208358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng 1209358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng MBB.erase(MI); 1210d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng MBBI = NewBBI; 1211d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng return true; 1212358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng } 1213358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng return false; 1214358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng} 1215358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng 1216a8e2989ece6dc46df59b0768184028257f913843Evan Cheng/// LoadStoreMultipleOpti - An optimization pass to turn multiple LDR / STR 1217a8e2989ece6dc46df59b0768184028257f913843Evan Cheng/// ops of the same base and incrementing offset into LDM / STM ops. 1218a8e2989ece6dc46df59b0768184028257f913843Evan Chengbool ARMLoadStoreOpt::LoadStoreMultipleOpti(MachineBasicBlock &MBB) { 1219a8e2989ece6dc46df59b0768184028257f913843Evan Cheng unsigned NumMerges = 0; 1220a8e2989ece6dc46df59b0768184028257f913843Evan Cheng unsigned NumMemOps = 0; 1221a8e2989ece6dc46df59b0768184028257f913843Evan Cheng MemOpQueue MemOps; 1222a8e2989ece6dc46df59b0768184028257f913843Evan Cheng unsigned CurrBase = 0; 1223a8e2989ece6dc46df59b0768184028257f913843Evan Cheng int CurrOpc = -1; 1224a8e2989ece6dc46df59b0768184028257f913843Evan Cheng unsigned CurrSize = 0; 122544bec52b1b7e9a3ac1efbae90db240b8c1ca2ad4Evan Cheng ARMCC::CondCodes CurrPred = ARMCC::AL; 12260e1d37904abbf7e8c4c478408253dd995f3040f4Evan Cheng unsigned CurrPredReg = 0; 1227a8e2989ece6dc46df59b0768184028257f913843Evan Cheng unsigned Position = 0; 12285ba71887f918a9da82140210494608df020dcbd5Evan Cheng SmallVector<MachineBasicBlock::iterator,4> Merges; 1229cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng 12300ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng RS->enterBasicBlock(&MBB); 1231a8e2989ece6dc46df59b0768184028257f913843Evan Cheng MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); 1232a8e2989ece6dc46df59b0768184028257f913843Evan Cheng while (MBBI != E) { 1233358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng if (FixInvalidRegPairOp(MBB, MBBI)) 1234358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng continue; 1235358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng 1236a8e2989ece6dc46df59b0768184028257f913843Evan Cheng bool Advance = false; 1237a8e2989ece6dc46df59b0768184028257f913843Evan Cheng bool TryMerge = false; 1238a8e2989ece6dc46df59b0768184028257f913843Evan Cheng bool Clobber = false; 1239a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 1240cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng bool isMemOp = isMemoryOp(MBBI); 1241a8e2989ece6dc46df59b0768184028257f913843Evan Cheng if (isMemOp) { 1242cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng int Opcode = MBBI->getOpcode(); 1243cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng unsigned Size = getLSMultipleTransferSize(MBBI); 1244d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng const MachineOperand &MO = MBBI->getOperand(0); 1245d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng unsigned Reg = MO.getReg(); 1246d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng bool isKill = MO.isDef() ? false : MO.isKill(); 1247a8e2989ece6dc46df59b0768184028257f913843Evan Cheng unsigned Base = MBBI->getOperand(1).getReg(); 12480e1d37904abbf7e8c4c478408253dd995f3040f4Evan Cheng unsigned PredReg = 0; 1249c89c744b69cecac576317a98322fd295e36e9886Craig Topper ARMCC::CondCodes Pred = getInstrPredicate(MBBI, PredReg); 1250e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng int Offset = getMemoryOpOffset(MBBI); 1251a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // Watch out for: 1252a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // r4 := ldr [r5] 1253a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // r5 := ldr [r5, #4] 1254a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // r6 := ldr [r5, #8] 1255a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // 1256a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // The second ldr has effectively broken the chain even though it 1257a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // looks like the later ldr(s) use the same base register. Try to 1258a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // merge the ldr's so far, including this one. But don't try to 1259a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // combine the following ldr(s). 126045032f28013aa69d07bf859da9d976947910f059Evan Cheng Clobber = (isi32Load(Opcode) && Base == MBBI->getOperand(0).getReg()); 1261a8e2989ece6dc46df59b0768184028257f913843Evan Cheng if (CurrBase == 0 && !Clobber) { 1262a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // Start of a new chain. 1263a8e2989ece6dc46df59b0768184028257f913843Evan Cheng CurrBase = Base; 1264a8e2989ece6dc46df59b0768184028257f913843Evan Cheng CurrOpc = Opcode; 1265a8e2989ece6dc46df59b0768184028257f913843Evan Cheng CurrSize = Size; 126644bec52b1b7e9a3ac1efbae90db240b8c1ca2ad4Evan Cheng CurrPred = Pred; 12670e1d37904abbf7e8c4c478408253dd995f3040f4Evan Cheng CurrPredReg = PredReg; 1268d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng MemOps.push_back(MemOpQueueEntry(Offset, Reg, isKill, Position, MBBI)); 1269fe60104ac97f3a8736dcfbfdf9547c7b7cc7b951Dan Gohman ++NumMemOps; 1270a8e2989ece6dc46df59b0768184028257f913843Evan Cheng Advance = true; 1271a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } else { 1272a8e2989ece6dc46df59b0768184028257f913843Evan Cheng if (Clobber) { 1273a8e2989ece6dc46df59b0768184028257f913843Evan Cheng TryMerge = true; 1274a8e2989ece6dc46df59b0768184028257f913843Evan Cheng Advance = true; 1275a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 1276a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 127744bec52b1b7e9a3ac1efbae90db240b8c1ca2ad4Evan Cheng if (CurrOpc == Opcode && CurrBase == Base && CurrPred == Pred) { 12780e1d37904abbf7e8c4c478408253dd995f3040f4Evan Cheng // No need to match PredReg. 1279a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // Continue adding to the queue. 1280a8e2989ece6dc46df59b0768184028257f913843Evan Cheng if (Offset > MemOps.back().Offset) { 1281d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng MemOps.push_back(MemOpQueueEntry(Offset, Reg, isKill, 1282d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng Position, MBBI)); 1283fe60104ac97f3a8736dcfbfdf9547c7b7cc7b951Dan Gohman ++NumMemOps; 1284a8e2989ece6dc46df59b0768184028257f913843Evan Cheng Advance = true; 1285a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } else { 1286a8e2989ece6dc46df59b0768184028257f913843Evan Cheng for (MemOpQueueIter I = MemOps.begin(), E = MemOps.end(); 1287a8e2989ece6dc46df59b0768184028257f913843Evan Cheng I != E; ++I) { 1288a8e2989ece6dc46df59b0768184028257f913843Evan Cheng if (Offset < I->Offset) { 1289d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng MemOps.insert(I, MemOpQueueEntry(Offset, Reg, isKill, 1290d95ea2da28901ca6e81645e444e528d66350d781Evan Cheng Position, MBBI)); 1291fe60104ac97f3a8736dcfbfdf9547c7b7cc7b951Dan Gohman ++NumMemOps; 1292a8e2989ece6dc46df59b0768184028257f913843Evan Cheng Advance = true; 1293a8e2989ece6dc46df59b0768184028257f913843Evan Cheng break; 1294a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } else if (Offset == I->Offset) { 1295a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // Collision! This can't be merged! 1296a8e2989ece6dc46df59b0768184028257f913843Evan Cheng break; 1297a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 1298a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 1299a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 1300a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 1301a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 1302a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 1303a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 1304db03adb34615331c6ef55ebbd80d8bc750deefe0Jim Grosbach if (MBBI->isDebugValue()) { 1305db03adb34615331c6ef55ebbd80d8bc750deefe0Jim Grosbach ++MBBI; 1306db03adb34615331c6ef55ebbd80d8bc750deefe0Jim Grosbach if (MBBI == E) 1307db03adb34615331c6ef55ebbd80d8bc750deefe0Jim Grosbach // Reach the end of the block, try merging the memory instructions. 1308db03adb34615331c6ef55ebbd80d8bc750deefe0Jim Grosbach TryMerge = true; 1309db03adb34615331c6ef55ebbd80d8bc750deefe0Jim Grosbach } else if (Advance) { 1310a8e2989ece6dc46df59b0768184028257f913843Evan Cheng ++Position; 1311a8e2989ece6dc46df59b0768184028257f913843Evan Cheng ++MBBI; 1312faf93aa23390389375b8e52f0dd1c3727ed07ee8Evan Cheng if (MBBI == E) 1313faf93aa23390389375b8e52f0dd1c3727ed07ee8Evan Cheng // Reach the end of the block, try merging the memory instructions. 1314faf93aa23390389375b8e52f0dd1c3727ed07ee8Evan Cheng TryMerge = true; 1315a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } else 1316a8e2989ece6dc46df59b0768184028257f913843Evan Cheng TryMerge = true; 1317a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 1318a8e2989ece6dc46df59b0768184028257f913843Evan Cheng if (TryMerge) { 1319a8e2989ece6dc46df59b0768184028257f913843Evan Cheng if (NumMemOps > 1) { 13200ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng // Try to find a free register to use as a new base in case it's needed. 13210ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng // First advance to the instruction just before the start of the chain. 132211788fde93fb74636aa333b2910d606d2c19ba9eEvan Cheng AdvanceRS(MBB, MemOps); 1323c0823fe7c679ca8f7d1667a310c2fca97b9402d5Jakob Stoklund Olesen // Find a scratch register. 1324420761a0f193e87d08ee1c51b26bba23ab4bac7fCraig Topper unsigned Scratch = RS->FindUnusedReg(&ARM::GPRRegClass); 13250ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng // Process the load / store instructions. 13260ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng RS->forward(prior(MBBI)); 13270ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng 13280ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng // Merge ops. 13295ba71887f918a9da82140210494608df020dcbd5Evan Cheng Merges.clear(); 13305ba71887f918a9da82140210494608df020dcbd5Evan Cheng MergeLDR_STR(MBB, 0, CurrBase, CurrOpc, CurrSize, 13315ba71887f918a9da82140210494608df020dcbd5Evan Cheng CurrPred, CurrPredReg, Scratch, MemOps, Merges); 13320ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng 13337a2bdde0a0eebcd2125055e0eacaca040f0b766cChris Lattner // Try folding preceding/trailing base inc/dec into the generated 1334a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // LDM/STM ops. 13355ba71887f918a9da82140210494608df020dcbd5Evan Cheng for (unsigned i = 0, e = Merges.size(); i < e; ++i) 133645032f28013aa69d07bf859da9d976947910f059Evan Cheng if (MergeBaseUpdateLSMultiple(MBB, Merges[i], Advance, MBBI)) 13379d5fb981b0ebc00e068b9bcb4df7388a8ea94b71Evan Cheng ++NumMerges; 13385ba71887f918a9da82140210494608df020dcbd5Evan Cheng NumMerges += Merges.size(); 1339a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 13407a2bdde0a0eebcd2125055e0eacaca040f0b766cChris Lattner // Try folding preceding/trailing base inc/dec into those load/store 13410ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng // that were not merged to form LDM/STM ops. 13420ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng for (unsigned i = 0; i != NumMemOps; ++i) 13430ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng if (!MemOps[i].Merged) 134445032f28013aa69d07bf859da9d976947910f059Evan Cheng if (MergeBaseUpdateLoadStore(MBB, MemOps[i].MBBI, TII,Advance,MBBI)) 13459d5fb981b0ebc00e068b9bcb4df7388a8ea94b71Evan Cheng ++NumMerges; 13460ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng 1347764ab52dd80310a205c9888bf166d09dab858f90Jim Grosbach // RS may be pointing to an instruction that's deleted. 13480ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng RS->skipTo(prior(MBBI)); 13491488326156741063fa7a23e1638c13e81d167e22Evan Cheng } else if (NumMemOps == 1) { 13507a2bdde0a0eebcd2125055e0eacaca040f0b766cChris Lattner // Try folding preceding/trailing base inc/dec into the single 13511488326156741063fa7a23e1638c13e81d167e22Evan Cheng // load/store. 135245032f28013aa69d07bf859da9d976947910f059Evan Cheng if (MergeBaseUpdateLoadStore(MBB, MemOps[0].MBBI, TII, Advance, MBBI)) { 13531488326156741063fa7a23e1638c13e81d167e22Evan Cheng ++NumMerges; 13541488326156741063fa7a23e1638c13e81d167e22Evan Cheng RS->forward(prior(MBBI)); 13551488326156741063fa7a23e1638c13e81d167e22Evan Cheng } 13560ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng } 1357a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 1358a8e2989ece6dc46df59b0768184028257f913843Evan Cheng CurrBase = 0; 1359a8e2989ece6dc46df59b0768184028257f913843Evan Cheng CurrOpc = -1; 136044bec52b1b7e9a3ac1efbae90db240b8c1ca2ad4Evan Cheng CurrSize = 0; 136144bec52b1b7e9a3ac1efbae90db240b8c1ca2ad4Evan Cheng CurrPred = ARMCC::AL; 13620e1d37904abbf7e8c4c478408253dd995f3040f4Evan Cheng CurrPredReg = 0; 1363a8e2989ece6dc46df59b0768184028257f913843Evan Cheng if (NumMemOps) { 1364a8e2989ece6dc46df59b0768184028257f913843Evan Cheng MemOps.clear(); 1365a8e2989ece6dc46df59b0768184028257f913843Evan Cheng NumMemOps = 0; 1366a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 1367a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 1368a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // If iterator hasn't been advanced and this is not a memory op, skip it. 1369a8e2989ece6dc46df59b0768184028257f913843Evan Cheng // It can't start a new chain anyway. 1370a8e2989ece6dc46df59b0768184028257f913843Evan Cheng if (!Advance && !isMemOp && MBBI != E) { 1371a8e2989ece6dc46df59b0768184028257f913843Evan Cheng ++Position; 1372a8e2989ece6dc46df59b0768184028257f913843Evan Cheng ++MBBI; 1373a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 1374a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 1375a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 1376a8e2989ece6dc46df59b0768184028257f913843Evan Cheng return NumMerges > 0; 1377a8e2989ece6dc46df59b0768184028257f913843Evan Cheng} 1378a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 1379c88d0722931b8442cd347bd530e2182d164c82c6Bob Wilson/// MergeReturnIntoLDM - If this is a exit BB, try merging the return ops 13807a2bdde0a0eebcd2125055e0eacaca040f0b766cChris Lattner/// ("bx lr" and "mov pc, lr") into the preceding stack restore so it 1381c88d0722931b8442cd347bd530e2182d164c82c6Bob Wilson/// directly restore the value of LR into pc. 1382c88d0722931b8442cd347bd530e2182d164c82c6Bob Wilson/// ldmfd sp!, {..., lr} 1383a8e2989ece6dc46df59b0768184028257f913843Evan Cheng/// bx lr 1384c88d0722931b8442cd347bd530e2182d164c82c6Bob Wilson/// or 1385c88d0722931b8442cd347bd530e2182d164c82c6Bob Wilson/// ldmfd sp!, {..., lr} 1386c88d0722931b8442cd347bd530e2182d164c82c6Bob Wilson/// mov pc, lr 1387a8e2989ece6dc46df59b0768184028257f913843Evan Cheng/// => 1388c88d0722931b8442cd347bd530e2182d164c82c6Bob Wilson/// ldmfd sp!, {..., pc} 1389a8e2989ece6dc46df59b0768184028257f913843Evan Chengbool ARMLoadStoreOpt::MergeReturnIntoLDM(MachineBasicBlock &MBB) { 1390a8e2989ece6dc46df59b0768184028257f913843Evan Cheng if (MBB.empty()) return false; 1391a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 1392f7ca976e74eafeeab0e9097f0fb07d6bb447415bJakob Stoklund Olesen MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr(); 139345032f28013aa69d07bf859da9d976947910f059Evan Cheng if (MBBI != MBB.begin() && 1394c88d0722931b8442cd347bd530e2182d164c82c6Bob Wilson (MBBI->getOpcode() == ARM::BX_RET || 1395c88d0722931b8442cd347bd530e2182d164c82c6Bob Wilson MBBI->getOpcode() == ARM::tBX_RET || 1396c88d0722931b8442cd347bd530e2182d164c82c6Bob Wilson MBBI->getOpcode() == ARM::MOVPCLR)) { 1397a8e2989ece6dc46df59b0768184028257f913843Evan Cheng MachineInstr *PrevMI = prior(MBBI); 139873fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling unsigned Opcode = PrevMI->getOpcode(); 139973fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling if (Opcode == ARM::LDMIA_UPD || Opcode == ARM::LDMDA_UPD || 140073fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling Opcode == ARM::LDMDB_UPD || Opcode == ARM::LDMIB_UPD || 140173fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling Opcode == ARM::t2LDMIA_UPD || Opcode == ARM::t2LDMDB_UPD) { 1402a8e2989ece6dc46df59b0768184028257f913843Evan Cheng MachineOperand &MO = PrevMI->getOperand(PrevMI->getNumOperands()-1); 140327934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng if (MO.getReg() != ARM::LR) 140427934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng return false; 140573fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling unsigned NewOpc = (isThumb2 ? ARM::t2LDMIA_RET : ARM::LDMIA_RET); 140673fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling assert(((isThumb2 && Opcode == ARM::t2LDMIA_UPD) || 140773fe34a3ee866867d5028f4a9afa2c3b8efebcbaBill Wendling Opcode == ARM::LDMIA_UPD) && "Unsupported multiple load-return!"); 140827934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng PrevMI->setDesc(TII->get(NewOpc)); 140927934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng MO.setReg(ARM::PC); 1410be06aacaa9a270384599bbfa850b967e9996b9fbJakob Stoklund Olesen PrevMI->copyImplicitOps(*MBB.getParent(), &*MBBI); 141127934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng MBB.erase(MBBI); 141227934da97bcb7a6a4949dfc0c449d99f43077e98Evan Cheng return true; 1413a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 1414a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 1415a8e2989ece6dc46df59b0768184028257f913843Evan Cheng return false; 1416a8e2989ece6dc46df59b0768184028257f913843Evan Cheng} 1417a8e2989ece6dc46df59b0768184028257f913843Evan Cheng 1418a8e2989ece6dc46df59b0768184028257f913843Evan Chengbool ARMLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) { 1419cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng const TargetMachine &TM = Fn.getTarget(); 1420603b83ebcdfb9c27e44c1da16f2799755e3e3022Evan Cheng AFI = Fn.getInfo<ARMFunctionInfo>(); 1421cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng TII = TM.getInstrInfo(); 14226f0d024a534af18d9e60b3ea757376cd8a3a980eDan Gohman TRI = TM.getRegisterInfo(); 14233568a1051efb9a9edbd4914b04b44e9d7bc1b004Evan Cheng STI = &TM.getSubtarget<ARMSubtarget>(); 14240ea12ec8484c45ca1394f3b2dcd39c9a34cf2ab9Evan Cheng RS = new RegScavenger(); 142545032f28013aa69d07bf859da9d976947910f059Evan Cheng isThumb2 = AFI->isThumb2Function(); 1426cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng 1427a8e2989ece6dc46df59b0768184028257f913843Evan Cheng bool Modified = false; 1428a8e2989ece6dc46df59b0768184028257f913843Evan Cheng for (MachineFunction::iterator MFI = Fn.begin(), E = Fn.end(); MFI != E; 1429a8e2989ece6dc46df59b0768184028257f913843Evan Cheng ++MFI) { 1430a8e2989ece6dc46df59b0768184028257f913843Evan Cheng MachineBasicBlock &MBB = *MFI; 1431a8e2989ece6dc46df59b0768184028257f913843Evan Cheng Modified |= LoadStoreMultipleOpti(MBB); 14326819dbb6f86f519130edf22ac3a30fa53b01fa45Bob Wilson if (TM.getSubtarget<ARMSubtarget>().hasV5TOps()) 14336819dbb6f86f519130edf22ac3a30fa53b01fa45Bob Wilson Modified |= MergeReturnIntoLDM(MBB); 1434a8e2989ece6dc46df59b0768184028257f913843Evan Cheng } 1435cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng 1436cc1c427266817e0f41cbb9d0dc97a52890182040Evan Cheng delete RS; 1437a8e2989ece6dc46df59b0768184028257f913843Evan Cheng return Modified; 1438a8e2989ece6dc46df59b0768184028257f913843Evan Cheng} 1439e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1440e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1441e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng/// ARMPreAllocLoadStoreOpt - Pre- register allocation pass that move 1442e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng/// load / stores from consecutive locations close to make it more 1443e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng/// likely they will be combined later. 1444e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1445e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Chengnamespace { 14466726b6d75a8b679068a58cb954ba97cf9d1690baNick Lewycky struct ARMPreAllocLoadStoreOpt : public MachineFunctionPass{ 1447e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng static char ID; 144890c579de5a383cee278acc3f7e7b9d0a656e6a35Owen Anderson ARMPreAllocLoadStoreOpt() : MachineFunctionPass(ID) {} 1449e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 14503574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow const DataLayout *TD; 1451e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng const TargetInstrInfo *TII; 1452e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng const TargetRegisterInfo *TRI; 1453358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng const ARMSubtarget *STI; 1454e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng MachineRegisterInfo *MRI; 1455eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng MachineFunction *MF; 1456e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1457e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng virtual bool runOnMachineFunction(MachineFunction &Fn); 1458e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1459e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng virtual const char *getPassName() const { 1460e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng return "ARM pre- register allocation load / store optimization pass"; 1461e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1462e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1463e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng private: 1464d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng bool CanFormLdStDWord(MachineInstr *Op0, MachineInstr *Op1, DebugLoc &dl, 1465d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng unsigned &NewOpc, unsigned &EvenReg, 1466d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng unsigned &OddReg, unsigned &BaseReg, 14677e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach int &Offset, 1468eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng unsigned &PredReg, ARMCC::CondCodes &Pred, 1469eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng bool &isT2); 1470e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng bool RescheduleOps(MachineBasicBlock *MBB, 1471e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng SmallVector<MachineInstr*, 4> &Ops, 1472e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng unsigned Base, bool isLd, 1473e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng DenseMap<MachineInstr*, unsigned> &MI2LocMap); 1474e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng bool RescheduleLoadStoreInstrs(MachineBasicBlock *MBB); 1475e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng }; 1476e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng char ARMPreAllocLoadStoreOpt::ID = 0; 1477e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng} 1478e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1479e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Chengbool ARMPreAllocLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) { 14803574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow TD = Fn.getTarget().getDataLayout(); 1481e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng TII = Fn.getTarget().getInstrInfo(); 1482e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng TRI = Fn.getTarget().getRegisterInfo(); 1483358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng STI = &Fn.getTarget().getSubtarget<ARMSubtarget>(); 1484e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng MRI = &Fn.getRegInfo(); 1485eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng MF = &Fn; 1486e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1487e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng bool Modified = false; 1488e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng for (MachineFunction::iterator MFI = Fn.begin(), E = Fn.end(); MFI != E; 1489e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng ++MFI) 1490e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng Modified |= RescheduleLoadStoreInstrs(MFI); 1491e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1492e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng return Modified; 1493e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng} 1494e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1495ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Chengstatic bool IsSafeAndProfitableToMove(bool isLd, unsigned Base, 1496ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng MachineBasicBlock::iterator I, 1497ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng MachineBasicBlock::iterator E, 1498ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng SmallPtrSet<MachineInstr*, 4> &MemOps, 1499ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng SmallSet<unsigned, 4> &MemRegs, 1500ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng const TargetRegisterInfo *TRI) { 1501e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // Are there stores / loads / calls between them? 1502e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // FIXME: This is overly conservative. We should make use of alias information 1503e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // some day. 1504ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng SmallSet<unsigned, 4> AddedRegPressure; 1505e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng while (++I != E) { 1506958e4e1967838766d327f1112e5b4900be939275Jim Grosbach if (I->isDebugValue() || MemOps.count(&*I)) 1507ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng continue; 15085a96b3dad2f634c9081c8b2b6c2575441dc5a2bdEvan Cheng if (I->isCall() || I->isTerminator() || I->hasUnmodeledSideEffects()) 1509e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng return false; 15105a96b3dad2f634c9081c8b2b6c2575441dc5a2bdEvan Cheng if (isLd && I->mayStore()) 1511e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng return false; 1512e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (!isLd) { 15135a96b3dad2f634c9081c8b2b6c2575441dc5a2bdEvan Cheng if (I->mayLoad()) 1514e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng return false; 1515e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // It's not safe to move the first 'str' down. 1516e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // str r1, [r0] 1517e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // strh r5, [r0] 1518e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // str r4, [r0, #+4] 15195a96b3dad2f634c9081c8b2b6c2575441dc5a2bdEvan Cheng if (I->mayStore()) 1520e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng return false; 1521e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1522e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng for (unsigned j = 0, NumOps = I->getNumOperands(); j != NumOps; ++j) { 1523e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng MachineOperand &MO = I->getOperand(j); 1524ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng if (!MO.isReg()) 1525ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng continue; 1526ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng unsigned Reg = MO.getReg(); 1527ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng if (MO.isDef() && TRI->regsOverlap(Reg, Base)) 1528e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng return false; 1529ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng if (Reg != Base && !MemRegs.count(Reg)) 1530ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng AddedRegPressure.insert(Reg); 1531e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1532e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1533ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng 1534ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng // Estimate register pressure increase due to the transformation. 1535ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng if (MemRegs.size() <= 4) 1536ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng // Ok if we are moving small number of instructions. 1537ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng return true; 1538ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng return AddedRegPressure.size() <= MemRegs.size() * 2; 1539e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng} 1540e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 154195bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick 154295bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick/// Copy Op0 and Op1 operands into a new array assigned to MI. 154395bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trickstatic void concatenateMemOperands(MachineInstr *MI, MachineInstr *Op0, 154495bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick MachineInstr *Op1) { 154595bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick assert(MI->memoperands_empty() && "expected a new machineinstr"); 154695bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick size_t numMemRefs = (Op0->memoperands_end() - Op0->memoperands_begin()) 154795bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick + (Op1->memoperands_end() - Op1->memoperands_begin()); 154895bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick 154995bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick MachineFunction *MF = MI->getParent()->getParent(); 155095bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick MachineSDNode::mmo_iterator MemBegin = MF->allocateMemRefsArray(numMemRefs); 155195bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick MachineSDNode::mmo_iterator MemEnd = 155295bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick std::copy(Op0->memoperands_begin(), Op0->memoperands_end(), MemBegin); 155395bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick MemEnd = 155495bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick std::copy(Op1->memoperands_begin(), Op1->memoperands_end(), MemEnd); 155595bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick MI->setMemRefs(MemBegin, MemEnd); 155695bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick} 155795bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick 1558d780f357941fc587d36d141bab3d78d6ff972dd4Evan Chengbool 1559d780f357941fc587d36d141bab3d78d6ff972dd4Evan ChengARMPreAllocLoadStoreOpt::CanFormLdStDWord(MachineInstr *Op0, MachineInstr *Op1, 1560d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng DebugLoc &dl, 1561d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng unsigned &NewOpc, unsigned &EvenReg, 1562d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng unsigned &OddReg, unsigned &BaseReg, 15637e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach int &Offset, unsigned &PredReg, 1564eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng ARMCC::CondCodes &Pred, 1565eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng bool &isT2) { 1566fa1be5d99153a4709740ce5aabba3793e9f77982Evan Cheng // Make sure we're allowed to generate LDRD/STRD. 1567fa1be5d99153a4709740ce5aabba3793e9f77982Evan Cheng if (!STI->hasV5TEOps()) 1568fa1be5d99153a4709740ce5aabba3793e9f77982Evan Cheng return false; 1569fa1be5d99153a4709740ce5aabba3793e9f77982Evan Cheng 1570e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim Grosbach // FIXME: VLDRS / VSTRS -> VLDRD / VSTRD 1571eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng unsigned Scale = 1; 1572d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng unsigned Opcode = Op0->getOpcode(); 15733e5561247202bae994dd259a2d8dc4eff8f799f3Jim Grosbach if (Opcode == ARM::LDRi12) 1574d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng NewOpc = ARM::LDRD; 15757e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach else if (Opcode == ARM::STRi12) 1576d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng NewOpc = ARM::STRD; 1577eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng else if (Opcode == ARM::t2LDRi8 || Opcode == ARM::t2LDRi12) { 1578eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng NewOpc = ARM::t2LDRDi8; 1579eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng Scale = 4; 1580eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng isT2 = true; 1581eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng } else if (Opcode == ARM::t2STRi8 || Opcode == ARM::t2STRi12) { 1582eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng NewOpc = ARM::t2STRDi8; 1583eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng Scale = 4; 1584eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng isT2 = true; 1585eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng } else 1586eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng return false; 1587eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng 15880eb7d06ab12bd66d04ea2329f71ceb2e8022f83aJim Grosbach // Make sure the base address satisfies i64 ld / st alignment requirement. 1589d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng if (!Op0->hasOneMemOperand() || 1590c76909abfec876c6b751d693ebd3df07df686aa0Dan Gohman !(*Op0->memoperands_begin())->getValue() || 1591c76909abfec876c6b751d693ebd3df07df686aa0Dan Gohman (*Op0->memoperands_begin())->isVolatile()) 1592358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng return false; 1593358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng 1594c76909abfec876c6b751d693ebd3df07df686aa0Dan Gohman unsigned Align = (*Op0->memoperands_begin())->getAlignment(); 1595ae541aad5c36cb3e4256514447d1f81e253079c7Dan Gohman const Function *Func = MF->getFunction(); 1596358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng unsigned ReqAlign = STI->hasV6Ops() 15977e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach ? TD->getABITypeAlignment(Type::getInt64Ty(Func->getContext())) 1598eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng : 8; // Pre-v6 need 8-byte align 1599d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng if (Align < ReqAlign) 1600d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng return false; 1601d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng 1602d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng // Then make sure the immediate offset fits. 1603d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng int OffImm = getMemoryOpOffset(Op0); 1604e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng if (isT2) { 1605019195229693c1829e6ed5a3f57209728fdcba8fEvan Cheng int Limit = (1 << 8) * Scale; 1606019195229693c1829e6ed5a3f57209728fdcba8fEvan Cheng if (OffImm >= Limit || (OffImm <= -Limit) || (OffImm & (Scale-1))) 1607019195229693c1829e6ed5a3f57209728fdcba8fEvan Cheng return false; 1608eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng Offset = OffImm; 1609e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng } else { 1610e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng ARM_AM::AddrOpc AddSub = ARM_AM::add; 1611e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng if (OffImm < 0) { 1612e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng AddSub = ARM_AM::sub; 1613e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng OffImm = - OffImm; 1614e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng } 1615e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng int Limit = (1 << 8) * Scale; 1616e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng if (OffImm >= Limit || (OffImm & (Scale-1))) 1617e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng return false; 1618eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng Offset = ARM_AM::getAM3Opc(AddSub, OffImm); 1619e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng } 1620d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng EvenReg = Op0->getOperand(0).getReg(); 1621675860758ec8926f9803840615366931aca7f8d8Evan Cheng OddReg = Op1->getOperand(0).getReg(); 1622d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng if (EvenReg == OddReg) 1623d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng return false; 1624d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng BaseReg = Op0->getOperand(1).getReg(); 1625c89c744b69cecac576317a98322fd295e36e9886Craig Topper Pred = getInstrPredicate(Op0, PredReg); 1626d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng dl = Op0->getDebugLoc(); 1627d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng return true; 1628358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng} 1629358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng 16304e97e8ee5748a92326078d744f5fc2b581a7b5b0Bob Wilsonnamespace { 16314e97e8ee5748a92326078d744f5fc2b581a7b5b0Bob Wilson struct OffsetCompare { 16324e97e8ee5748a92326078d744f5fc2b581a7b5b0Bob Wilson bool operator()(const MachineInstr *LHS, const MachineInstr *RHS) const { 16334e97e8ee5748a92326078d744f5fc2b581a7b5b0Bob Wilson int LOffset = getMemoryOpOffset(LHS); 16344e97e8ee5748a92326078d744f5fc2b581a7b5b0Bob Wilson int ROffset = getMemoryOpOffset(RHS); 16354e97e8ee5748a92326078d744f5fc2b581a7b5b0Bob Wilson assert(LHS == RHS || LOffset != ROffset); 16364e97e8ee5748a92326078d744f5fc2b581a7b5b0Bob Wilson return LOffset > ROffset; 16374e97e8ee5748a92326078d744f5fc2b581a7b5b0Bob Wilson } 16384e97e8ee5748a92326078d744f5fc2b581a7b5b0Bob Wilson }; 16394e97e8ee5748a92326078d744f5fc2b581a7b5b0Bob Wilson} 16404e97e8ee5748a92326078d744f5fc2b581a7b5b0Bob Wilson 1641e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Chengbool ARMPreAllocLoadStoreOpt::RescheduleOps(MachineBasicBlock *MBB, 1642e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng SmallVector<MachineInstr*, 4> &Ops, 1643e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng unsigned Base, bool isLd, 1644e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng DenseMap<MachineInstr*, unsigned> &MI2LocMap) { 1645e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng bool RetVal = false; 1646e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1647e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // Sort by offset (in reverse order). 1648e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng std::sort(Ops.begin(), Ops.end(), OffsetCompare()); 1649e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1650e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // The loads / stores of the same base are in order. Scan them from first to 1651d089a7ac70397ea41fe6128639dc54b8e273ed60Jim Grosbach // last and check for the following: 1652e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // 1. Any def of base. 1653e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // 2. Any gaps. 1654e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng while (Ops.size() > 1) { 1655e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng unsigned FirstLoc = ~0U; 1656e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng unsigned LastLoc = 0; 1657e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng MachineInstr *FirstOp = 0; 1658e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng MachineInstr *LastOp = 0; 1659e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng int LastOffset = 0; 1660f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng unsigned LastOpcode = 0; 1661e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng unsigned LastBytes = 0; 1662e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng unsigned NumMove = 0; 1663e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng for (int i = Ops.size() - 1; i >= 0; --i) { 1664e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng MachineInstr *Op = Ops[i]; 1665e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng unsigned Loc = MI2LocMap[Op]; 1666e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (Loc <= FirstLoc) { 1667e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng FirstLoc = Loc; 1668e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng FirstOp = Op; 1669e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1670e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (Loc >= LastLoc) { 1671e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng LastLoc = Loc; 1672e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng LastOp = Op; 1673e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1674e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 167508c66642d70879cc479b502c048df4f5fdeaefaeAndrew Trick unsigned LSMOpcode 167608c66642d70879cc479b502c048df4f5fdeaefaeAndrew Trick = getLoadStoreMultipleOpcode(Op->getOpcode(), ARM_AM::ia); 167708c66642d70879cc479b502c048df4f5fdeaefaeAndrew Trick if (LastOpcode && LSMOpcode != LastOpcode) 1678f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng break; 1679f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng 1680e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng int Offset = getMemoryOpOffset(Op); 1681e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng unsigned Bytes = getLSMultipleTransferSize(Op); 1682e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (LastBytes) { 1683e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (Bytes != LastBytes || Offset != (LastOffset + (int)Bytes)) 1684e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng break; 1685e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1686e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng LastOffset = Offset; 1687e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng LastBytes = Bytes; 168808c66642d70879cc479b502c048df4f5fdeaefaeAndrew Trick LastOpcode = LSMOpcode; 1689eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng if (++NumMove == 8) // FIXME: Tune this limit. 1690e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng break; 1691e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1692e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1693e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (NumMove <= 1) 1694e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng Ops.pop_back(); 1695e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng else { 1696ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng SmallPtrSet<MachineInstr*, 4> MemOps; 1697ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng SmallSet<unsigned, 4> MemRegs; 1698ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng for (int i = NumMove-1; i >= 0; --i) { 1699ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng MemOps.insert(Ops[i]); 1700ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng MemRegs.insert(Ops[i]->getOperand(0).getReg()); 1701ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng } 1702e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1703e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // Be conservative, if the instructions are too far apart, don't 1704e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // move them. We want to limit the increase of register pressure. 1705ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng bool DoMove = (LastLoc - FirstLoc) <= NumMove*4; // FIXME: Tune this. 1706e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (DoMove) 1707ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng DoMove = IsSafeAndProfitableToMove(isLd, Base, FirstOp, LastOp, 1708ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng MemOps, MemRegs, TRI); 1709e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (!DoMove) { 1710e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng for (unsigned i = 0; i != NumMove; ++i) 1711e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng Ops.pop_back(); 1712e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } else { 1713e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // This is the new location for the loads / stores. 1714e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng MachineBasicBlock::iterator InsertPos = isLd ? FirstOp : LastOp; 1715400c95fe3802821815c69077e48c8fd276ec6494Jim Grosbach while (InsertPos != MBB->end() 1716400c95fe3802821815c69077e48c8fd276ec6494Jim Grosbach && (MemOps.count(InsertPos) || InsertPos->isDebugValue())) 1717e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng ++InsertPos; 1718358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng 1719358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng // If we are moving a pair of loads / stores, see if it makes sense 1720358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng // to try to allocate a pair of registers that can form register pairs. 1721d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng MachineInstr *Op0 = Ops.back(); 1722d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng MachineInstr *Op1 = Ops[Ops.size()-2]; 1723d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng unsigned EvenReg = 0, OddReg = 0; 17247e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach unsigned BaseReg = 0, PredReg = 0; 1725d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng ARMCC::CondCodes Pred = ARMCC::AL; 1726eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng bool isT2 = false; 1727d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng unsigned NewOpc = 0; 1728e298ab26b11cf6e278b4876bbc5b890e234d4029Evan Cheng int Offset = 0; 1729d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng DebugLoc dl; 1730d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng if (NumMove == 2 && CanFormLdStDWord(Op0, Op1, dl, NewOpc, 17317e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach EvenReg, OddReg, BaseReg, 1732eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng Offset, PredReg, Pred, isT2)) { 1733d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng Ops.pop_back(); 1734d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng Ops.pop_back(); 1735358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng 1736e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng const MCInstrDesc &MCID = TII->get(NewOpc); 1737397fc4874efe9c17e737d4c5c50bd19dc3bf27f5Jakob Stoklund Olesen const TargetRegisterClass *TRC = TII->getRegClass(MCID, 0, TRI, *MF); 1738955db42568ae61c9f381a5af3e354d4270d06d92Cameron Zwarich MRI->constrainRegClass(EvenReg, TRC); 1739955db42568ae61c9f381a5af3e354d4270d06d92Cameron Zwarich MRI->constrainRegClass(OddReg, TRC); 1740955db42568ae61c9f381a5af3e354d4270d06d92Cameron Zwarich 1741d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng // Form the pair instruction. 1742f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng if (isLd) { 1743e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng MachineInstrBuilder MIB = BuildMI(*MBB, InsertPos, dl, MCID) 1744358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng .addReg(EvenReg, RegState::Define) 1745358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng .addReg(OddReg, RegState::Define) 1746eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng .addReg(BaseReg); 17477e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach // FIXME: We're converting from LDRi12 to an insn that still 17483e5561247202bae994dd259a2d8dc4eff8f799f3Jim Grosbach // uses addrmode2, so we need an explicit offset reg. It should 17497e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach // always by reg0 since we're transforming LDRi12s. 1750eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng if (!isT2) 17513e5561247202bae994dd259a2d8dc4eff8f799f3Jim Grosbach MIB.addReg(0); 1752eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng MIB.addImm(Offset).addImm(Pred).addReg(PredReg); 175395bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick concatenateMemOperands(MIB, Op0, Op1); 175495bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick DEBUG(dbgs() << "Formed " << *MIB << "\n"); 1755f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng ++NumLDRDFormed; 1756f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng } else { 1757e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng MachineInstrBuilder MIB = BuildMI(*MBB, InsertPos, dl, MCID) 1758358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng .addReg(EvenReg) 1759358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng .addReg(OddReg) 1760eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng .addReg(BaseReg); 17617e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach // FIXME: We're converting from LDRi12 to an insn that still 17627e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach // uses addrmode2, so we need an explicit offset reg. It should 17637e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach // always by reg0 since we're transforming STRi12s. 1764eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng if (!isT2) 17657e3383c007f53b3a00675af225e428cb66ddf404Jim Grosbach MIB.addReg(0); 1766eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng MIB.addImm(Offset).addImm(Pred).addReg(PredReg); 176795bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick concatenateMemOperands(MIB, Op0, Op1); 176895bc85e4eefdfc1aabfde85daf752f05d2a60701Andrew Trick DEBUG(dbgs() << "Formed " << *MIB << "\n"); 1769f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng ++NumSTRDFormed; 1770f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng } 1771f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng MBB->erase(Op0); 1772f9f1da17f8bd6ffb2df62ce3cb933f0ee63f8da4Evan Cheng MBB->erase(Op1); 1773358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng 1774358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng // Add register allocation hints to form register pairs. 1775358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng MRI->setRegAllocationHint(EvenReg, ARMRI::RegPairEven, OddReg); 1776358dec51804ee52e47ea3a47c9248086e458ad7cEvan Cheng MRI->setRegAllocationHint(OddReg, ARMRI::RegPairOdd, EvenReg); 1777d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng } else { 1778d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng for (unsigned i = 0; i != NumMove; ++i) { 1779d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng MachineInstr *Op = Ops.back(); 1780d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng Ops.pop_back(); 1781d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng MBB->splice(InsertPos, MBB, Op); 1782d780f357941fc587d36d141bab3d78d6ff972dd4Evan Cheng } 1783e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1784e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1785e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng NumLdStMoved += NumMove; 1786e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng RetVal = true; 1787e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1788e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1789e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1790e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1791e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng return RetVal; 1792e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng} 1793e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1794e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Chengbool 1795e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan ChengARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) { 1796e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng bool RetVal = false; 1797e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1798e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng DenseMap<MachineInstr*, unsigned> MI2LocMap; 1799e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng DenseMap<unsigned, SmallVector<MachineInstr*, 4> > Base2LdsMap; 1800e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng DenseMap<unsigned, SmallVector<MachineInstr*, 4> > Base2StsMap; 1801e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng SmallVector<unsigned, 4> LdBases; 1802e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng SmallVector<unsigned, 4> StBases; 1803e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1804e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng unsigned Loc = 0; 1805e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng MachineBasicBlock::iterator MBBI = MBB->begin(); 1806e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng MachineBasicBlock::iterator E = MBB->end(); 1807e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng while (MBBI != E) { 1808e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng for (; MBBI != E; ++MBBI) { 1809e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng MachineInstr *MI = MBBI; 18105a96b3dad2f634c9081c8b2b6c2575441dc5a2bdEvan Cheng if (MI->isCall() || MI->isTerminator()) { 1811e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // Stop at barriers. 1812e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng ++MBBI; 1813e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng break; 1814e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1815e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1816958e4e1967838766d327f1112e5b4900be939275Jim Grosbach if (!MI->isDebugValue()) 1817958e4e1967838766d327f1112e5b4900be939275Jim Grosbach MI2LocMap[MI] = ++Loc; 1818958e4e1967838766d327f1112e5b4900be939275Jim Grosbach 1819e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (!isMemoryOp(MI)) 1820e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng continue; 1821e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng unsigned PredReg = 0; 1822c89c744b69cecac576317a98322fd295e36e9886Craig Topper if (getInstrPredicate(MI, PredReg) != ARMCC::AL) 1823e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng continue; 1824e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1825eef490f45919c8f279e40b8b5ef09f612ec6ce2cEvan Cheng int Opc = MI->getOpcode(); 1826e5165490b7ba24bb2f3043399e0d60e7f3bcf8a5Jim Grosbach bool isLd = isi32Load(Opc) || Opc == ARM::VLDRS || Opc == ARM::VLDRD; 1827e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng unsigned Base = MI->getOperand(1).getReg(); 1828e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng int Offset = getMemoryOpOffset(MI); 1829e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1830e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng bool StopHere = false; 1831e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (isLd) { 1832e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng DenseMap<unsigned, SmallVector<MachineInstr*, 4> >::iterator BI = 1833e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng Base2LdsMap.find(Base); 1834e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (BI != Base2LdsMap.end()) { 1835e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng for (unsigned i = 0, e = BI->second.size(); i != e; ++i) { 1836e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (Offset == getMemoryOpOffset(BI->second[i])) { 1837e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng StopHere = true; 1838e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng break; 1839e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1840e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1841e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (!StopHere) 1842e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng BI->second.push_back(MI); 1843e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } else { 1844e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng SmallVector<MachineInstr*, 4> MIs; 1845e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng MIs.push_back(MI); 1846e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng Base2LdsMap[Base] = MIs; 1847e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng LdBases.push_back(Base); 1848e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1849e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } else { 1850e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng DenseMap<unsigned, SmallVector<MachineInstr*, 4> >::iterator BI = 1851e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng Base2StsMap.find(Base); 1852e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (BI != Base2StsMap.end()) { 1853e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng for (unsigned i = 0, e = BI->second.size(); i != e; ++i) { 1854e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (Offset == getMemoryOpOffset(BI->second[i])) { 1855e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng StopHere = true; 1856e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng break; 1857e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1858e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1859e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (!StopHere) 1860e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng BI->second.push_back(MI); 1861e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } else { 1862e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng SmallVector<MachineInstr*, 4> MIs; 1863e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng MIs.push_back(MI); 1864e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng Base2StsMap[Base] = MIs; 1865e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng StBases.push_back(Base); 1866e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1867e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1868e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1869e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (StopHere) { 1870ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng // Found a duplicate (a base+offset combination that's seen earlier). 1871ae69a2a12bd0af3fa81957f7896d1a54ad69dbb2Evan Cheng // Backtrack. 1872e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng --Loc; 1873e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng break; 1874e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1875e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1876e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1877e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // Re-schedule loads. 1878e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng for (unsigned i = 0, e = LdBases.size(); i != e; ++i) { 1879e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng unsigned Base = LdBases[i]; 1880e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng SmallVector<MachineInstr*, 4> &Lds = Base2LdsMap[Base]; 1881e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (Lds.size() > 1) 1882e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng RetVal |= RescheduleOps(MBB, Lds, Base, true, MI2LocMap); 1883e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1884e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1885e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng // Re-schedule stores. 1886e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng for (unsigned i = 0, e = StBases.size(); i != e; ++i) { 1887e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng unsigned Base = StBases[i]; 1888e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng SmallVector<MachineInstr*, 4> &Sts = Base2StsMap[Base]; 1889e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (Sts.size() > 1) 1890e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng RetVal |= RescheduleOps(MBB, Sts, Base, false, MI2LocMap); 1891e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1892e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1893e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (MBBI != E) { 1894e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng Base2LdsMap.clear(); 1895e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng Base2StsMap.clear(); 1896e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng LdBases.clear(); 1897e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng StBases.clear(); 1898e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1899e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng } 1900e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1901e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng return RetVal; 1902e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng} 1903e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1904e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng 1905e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng/// createARMLoadStoreOptimizationPass - returns an instance of the load / store 1906e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng/// optimization pass. 1907e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan ChengFunctionPass *llvm::createARMLoadStoreOptimizationPass(bool PreAlloc) { 1908e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng if (PreAlloc) 1909e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng return new ARMPreAllocLoadStoreOpt(); 1910e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng return new ARMLoadStoreOpt(); 1911e7d6df73530a98a5cc5f69ddfd17073b464caa57Evan Cheng} 1912