ScheduleDAGRRList.cpp revision 24312230ada6f4cfa8776351dafb12eea8a81b33
104f4f4f447806cd92a2fb6f4b66d11f6d5003a82Dan Gohman//===----- ScheduleDAGRRList.cpp - Reg pressure reduction list scheduler --===// 2e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 3e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// The LLVM Compiler Infrastructure 4e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 7e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 8e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 9e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 10e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// This implements bottom-up and top-down register pressure reduction list 11e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// schedulers, using standard algorithms. The basic approach uses a priority 12e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// queue of available nodes to schedule. One at a time, nodes are taken from 13e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// the priority queue (thus in priority order), checked for legality to 14e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// schedule, and emitted if legal. 15e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 16e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 17e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 18e7e7d0d7e39d0c7c659d26b97e8081fce0fcd749Dale Johannesen#define DEBUG_TYPE "pre-RA-sched" 1984fbac580941548a6ab1121ed3b0ffdc4e2bc080Dan Gohman#include "ScheduleDAGSDNodes.h" 20decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner#include "llvm/InlineAsm.h" 21eb577ba3b815a1fa4627b060dd2345d17abf672dJim Laskey#include "llvm/CodeGen/SchedulerRegistry.h" 2279ce276083ced01256a0eb7d80731e4948ca6e87Dan Gohman#include "llvm/CodeGen/SelectionDAGISel.h" 232da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick#include "llvm/CodeGen/ScheduleHazardRecognizer.h" 246f0d024a534af18d9e60b3ea757376cd8a3a980eDan Gohman#include "llvm/Target/TargetRegisterInfo.h" 2507000c6f01d8f57170f2d4c77a86d934bdc5c696Owen Anderson#include "llvm/Target/TargetData.h" 26e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#include "llvm/Target/TargetMachine.h" 27e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#include "llvm/Target/TargetInstrInfo.h" 284f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng#include "llvm/Target/TargetLowering.h" 29a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng#include "llvm/ADT/SmallSet.h" 30e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#include "llvm/ADT/Statistic.h" 31a0201d52049be8dcefffe4304a49690a831bcb34Roman Levenstein#include "llvm/ADT/STLExtras.h" 32decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner#include "llvm/Support/Debug.h" 33decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner#include "llvm/Support/ErrorHandling.h" 34bbbfa99d3d18fe9f20265305e833666645ada528Chris Lattner#include "llvm/Support/raw_ostream.h" 35e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#include <climits> 36e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengusing namespace llvm; 37e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 38cffbd2562a5e3ba435dd2b622710ec272c634da5Dan GohmanSTATISTIC(NumBacktracks, "Number of times scheduler backtracked"); 39f10c973797cf79da802f9b0118543cbd50954c9cEvan ChengSTATISTIC(NumUnfolds, "Number of nodes unfolded"); 40a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan ChengSTATISTIC(NumDups, "Number of duplicated nodes"); 41c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan ChengSTATISTIC(NumPRCopies, "Number of physical register copies"); 42a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng 4313ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskeystatic RegisterScheduler 4413ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskey burrListDAGScheduler("list-burr", 45b8cab9227a0f6ffbdaae33e3c64268e265008a6aDan Gohman "Bottom-up register reduction list scheduling", 4613ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskey createBURRListDAGScheduler); 4713ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskeystatic RegisterScheduler 4813ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskey tdrListrDAGScheduler("list-tdrr", 49b8cab9227a0f6ffbdaae33e3c64268e265008a6aDan Gohman "Top-down register reduction list scheduling", 5013ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskey createTDRRListDAGScheduler); 51187361b056823df4ff292561fe47468dad956872Bill Wendlingstatic RegisterScheduler 52187361b056823df4ff292561fe47468dad956872Bill Wendling sourceListDAGScheduler("source", 53187361b056823df4ff292561fe47468dad956872Bill Wendling "Similar to list-burr but schedules in source " 54187361b056823df4ff292561fe47468dad956872Bill Wendling "order when possible", 55187361b056823df4ff292561fe47468dad956872Bill Wendling createSourceListDAGScheduler); 5613ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskey 5715a16def6e70c8f7df1023da80ceb89887203b40Evan Chengstatic RegisterScheduler 58b11ac950d69c7a238de0a22fd23fbfcd994f57eeEvan Cheng hybridListDAGScheduler("list-hybrid", 5970017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng "Bottom-up register pressure aware list scheduling " 6070017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng "which tries to balance latency and register pressure", 6115a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng createHybridListDAGScheduler); 6215a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 6370017e44cdba1946cc478ce1856a3e855a767e28Evan Chengstatic RegisterScheduler 6470017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng ILPListDAGScheduler("list-ilp", 6570017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng "Bottom-up register pressure aware list scheduling " 6670017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng "which tries to balance ILP and register pressure", 6770017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng createILPListDAGScheduler); 6870017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 692da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickstatic cl::opt<bool> EnableSchedCycles( 702da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick "enable-sched-cycles", 712da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick cl::desc("Enable cycle-level precision during preRA scheduling"), 722da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick cl::init(false), cl::Hidden); 732da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 74e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengnamespace { 75e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 76e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// ScheduleDAGRRList - The actual register reduction list scheduler 77e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// implementation. This supports both top-down and bottom-up scheduling. 78e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// 796726b6d75a8b679068a58cb954ba97cf9d1690baNick Lewyckyclass ScheduleDAGRRList : public ScheduleDAGSDNodes { 80e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengprivate: 81e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng /// isBottomUp - This is true if the scheduling problem is bottom-up, false if 82e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng /// it is top-down. 83e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng bool isBottomUp; 844576f6d7a9c0f2c6a3b6c5d4d8a3063bbf763ae5Evan Cheng 8515a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng /// NeedLatency - True if the scheduler will make use of latency information. 8615a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng /// 8715a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng bool NeedLatency; 8815a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 89e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng /// AvailableQueue - The priority queue to use for the available SUnits. 90e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng SchedulingPriorityQueue *AvailableQueue; 91e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 922da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// PendingQueue - This contains all of the instructions whose operands have 932da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// been issued, but their results are not ready yet (due to the latency of 942da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// the operation). Once the operands becomes available, the instruction is 952da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// added to the AvailableQueue. 962da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick std::vector<SUnit*> PendingQueue; 972da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 982da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// HazardRec - The hazard recognizer to use. 992da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleHazardRecognizer *HazardRec; 1002da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1012902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick /// CurCycle - The current scheduler state corresponds to this cycle. 1022902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick unsigned CurCycle; 1032902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 1042da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// MinAvailableCycle - Cycle of the soonest available instruction. 1052da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned MinAvailableCycle; 1062da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 107086ec9976ff6cee083de618429c78473491d5713Dan Gohman /// LiveRegDefs - A set of physical registers and their definition 108a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng /// that are "live". These nodes must be scheduled before any other nodes that 109a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng /// modifies the registers can be scheduled. 110086ec9976ff6cee083de618429c78473491d5713Dan Gohman unsigned NumLiveRegs; 111a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng std::vector<SUnit*> LiveRegDefs; 1123d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick std::vector<SUnit*> LiveRegGens; 113a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 11421d9003087c9a707e6cd95460136b499df358fb8Dan Gohman /// Topo - A topological ordering for SUnits which permits fast IsReachable 11521d9003087c9a707e6cd95460136b499df358fb8Dan Gohman /// and similar queries. 11621d9003087c9a707e6cd95460136b499df358fb8Dan Gohman ScheduleDAGTopologicalSort Topo; 11721d9003087c9a707e6cd95460136b499df358fb8Dan Gohman 118e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengpublic: 1192da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleDAGRRList(MachineFunction &mf, bool needlatency, 1202da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SchedulingPriorityQueue *availqueue, 1212da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CodeGenOpt::Level OptLevel) 1222da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick : ScheduleDAGSDNodes(mf), isBottomUp(availqueue->isBottomUp()), 1232da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick NeedLatency(needlatency), AvailableQueue(availqueue), CurCycle(0), 1242da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick Topo(SUnits) { 1252da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1262da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick const TargetMachine &tm = mf.getTarget(); 1272da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (EnableSchedCycles && OptLevel != CodeGenOpt::None) 1282da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec = tm.getInstrInfo()->CreateTargetHazardRecognizer(&tm, this); 1292da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick else 1302da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec = new ScheduleHazardRecognizer(); 1312da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 132e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 133e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng ~ScheduleDAGRRList() { 1342da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick delete HazardRec; 135e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng delete AvailableQueue; 136e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 137e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 138e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng void Schedule(); 139e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1408dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// IsReachable - Checks if SU is reachable from TargetSU. 14121d9003087c9a707e6cd95460136b499df358fb8Dan Gohman bool IsReachable(const SUnit *SU, const SUnit *TargetSU) { 14221d9003087c9a707e6cd95460136b499df358fb8Dan Gohman return Topo.IsReachable(SU, TargetSU); 14321d9003087c9a707e6cd95460136b499df358fb8Dan Gohman } 144e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 1451cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohman /// WillCreateCycle - Returns true if adding an edge from SU to TargetSU will 146e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein /// create a cycle. 14721d9003087c9a707e6cd95460136b499df358fb8Dan Gohman bool WillCreateCycle(SUnit *SU, SUnit *TargetSU) { 14821d9003087c9a707e6cd95460136b499df358fb8Dan Gohman return Topo.WillCreateCycle(SU, TargetSU); 14921d9003087c9a707e6cd95460136b499df358fb8Dan Gohman } 150e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 15154e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman /// AddPred - adds a predecessor edge to SUnit SU. 1528dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// This returns true if this is a new predecessor. 1538dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// Updates the topological ordering if required. 154ffa391272bad598d73fd5404dadf3686b69f2a63Dan Gohman void AddPred(SUnit *SU, const SDep &D) { 15554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman Topo.AddPred(SU, D.getSUnit()); 156ffa391272bad598d73fd5404dadf3686b69f2a63Dan Gohman SU->addPred(D); 15721d9003087c9a707e6cd95460136b499df358fb8Dan Gohman } 158e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 15954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman /// RemovePred - removes a predecessor edge from SUnit SU. 16054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman /// This returns true if an edge was removed. 16154e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman /// Updates the topological ordering if required. 162ffa391272bad598d73fd5404dadf3686b69f2a63Dan Gohman void RemovePred(SUnit *SU, const SDep &D) { 16354e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman Topo.RemovePred(SU, D.getSUnit()); 164ffa391272bad598d73fd5404dadf3686b69f2a63Dan Gohman SU->removePred(D); 16521d9003087c9a707e6cd95460136b499df358fb8Dan Gohman } 166e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 167e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengprivate: 1682da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick bool isReady(SUnit *SU) { 1692da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return !EnableSchedCycles || !AvailableQueue->hasReadyFilter() || 1702da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->isReady(SU); 1712da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 1722da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1731cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohman void ReleasePred(SUnit *SU, const SDep *PredEdge); 1743d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick void ReleasePredecessors(SUnit *SU); 1751cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohman void ReleaseSucc(SUnit *SU, const SDep *SuccEdge); 1769e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman void ReleaseSuccessors(SUnit *SU); 1772da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void ReleasePending(); 1782da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void AdvanceToCycle(unsigned NextCycle); 1792da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void AdvancePastStalls(SUnit *SU); 1802da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void EmitNode(SUnit *SU); 1812902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick void ScheduleNodeBottomUp(SUnit*); 1822da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void CapturePred(SDep *PredEdge); 18342d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng void UnscheduleNodeBottomUp(SUnit*); 1842da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void RestoreHazardCheckerBottomUp(); 1852da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void BacktrackBottomUp(SUnit*, SUnit*); 18642d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng SUnit *CopyAndMoveSuccessors(SUnit*); 187c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng void InsertCopiesAndMoveSuccs(SUnit*, unsigned, 188c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng const TargetRegisterClass*, 189c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng const TargetRegisterClass*, 190c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng SmallVector<SUnit*, 2>&); 191a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng bool DelayForLiveRegsBottomUp(SUnit*, SmallVector<unsigned, 4>&); 1922da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1932902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *PickNodeToScheduleBottomUp(); 194e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng void ListScheduleBottomUp(); 195e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 1962902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick void ScheduleNodeTopDown(SUnit*); 1972902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick void ListScheduleTopDown(); 1982902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 199e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 200e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein /// CreateNewSUnit - Creates a new SUnit and returns a pointer to it. 2018dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// Updates the topological ordering if required. 202e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *CreateNewSUnit(SDNode *N) { 20321d9003087c9a707e6cd95460136b499df358fb8Dan Gohman unsigned NumSUnits = SUnits.size(); 204e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *NewNode = NewSUnit(N); 2058dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein // Update the topological ordering. 20621d9003087c9a707e6cd95460136b499df358fb8Dan Gohman if (NewNode->NodeNum >= NumSUnits) 20721d9003087c9a707e6cd95460136b499df358fb8Dan Gohman Topo.InitDAGTopologicalSorting(); 208e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein return NewNode; 209e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein } 210e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 2118dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// CreateClone - Creates a new SUnit from an existing one. 2128dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// Updates the topological ordering if required. 213e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *CreateClone(SUnit *N) { 21421d9003087c9a707e6cd95460136b499df358fb8Dan Gohman unsigned NumSUnits = SUnits.size(); 215e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *NewNode = Clone(N); 2168dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein // Update the topological ordering. 21721d9003087c9a707e6cd95460136b499df358fb8Dan Gohman if (NewNode->NodeNum >= NumSUnits) 21821d9003087c9a707e6cd95460136b499df358fb8Dan Gohman Topo.InitDAGTopologicalSorting(); 219e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein return NewNode; 220e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein } 2213f23744df4809eba94284e601e81489212c974d4Dan Gohman 22215a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng /// ForceUnitLatencies - Register-pressure-reducing scheduling doesn't 22315a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng /// need actual latency information but the hybrid scheduler does. 22415a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng bool ForceUnitLatencies() const { 22515a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng return !NeedLatency; 22615a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng } 227e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng}; 228e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} // end anonymous namespace 229e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 230e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 231e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// Schedule - Schedule the DAG using list scheduling. 232e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengvoid ScheduleDAGRRList::Schedule() { 2334f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng DEBUG(dbgs() 2344f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng << "********** List Scheduling BB#" << BB->getNumber() 235089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng << " '" << BB->getName() << "' **********\n"); 236a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 2372902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurCycle = 0; 2382da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick MinAvailableCycle = EnableSchedCycles ? UINT_MAX : 0; 239086ec9976ff6cee083de618429c78473491d5713Dan Gohman NumLiveRegs = 0; 24038036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick LiveRegDefs.resize(TRI->getNumRegs(), NULL); 2413d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick LiveRegGens.resize(TRI->getNumRegs(), NULL); 242a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 243c9a5b9e38b442c2ae6b115213a07df3fcd14708dDan Gohman // Build the scheduling graph. 24498976e4dcd18adbbe676048c0069e67346eb4adeDan Gohman BuildSchedGraph(NULL); 245e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 246e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su) 2473cc6243ddfdba3ad64035b919c88b09773a60880Dan Gohman SUnits[su].dumpAll(this)); 24821d9003087c9a707e6cd95460136b499df358fb8Dan Gohman Topo.InitDAGTopologicalSorting(); 249e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 25094d7a5f8156e62532870fbaf197377b34e52ff2aDan Gohman AvailableQueue->initNodes(SUnits); 25138036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 2522da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->Reset(); 2532da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 254e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // Execute the actual scheduling loop Top-Down or Bottom-Up as appropriate. 255e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng if (isBottomUp) 256e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng ListScheduleBottomUp(); 257e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng else 258e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng ListScheduleTopDown(); 25938036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 260e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng AvailableQueue->releaseState(); 26113d41b9d721f98372b97d2ec119e6c91932ab0aeEvan Cheng} 262e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 263e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 264e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// Bottom-Up Scheduling 265e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 266e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 267e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// ReleasePred - Decrement the NumSuccsLeft count of a predecessor. Add it to 2688d1bfad00b1ebff5b140b6e1bd7e26bad697d6e1Dan Gohman/// the AvailableQueue if the count reaches zero. Also update its cycle bound. 2691cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohmanvoid ScheduleDAGRRList::ReleasePred(SUnit *SU, const SDep *PredEdge) { 27054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *PredSU = PredEdge->getSUnit(); 2713a90c9b8edb53ea1ea57d94f7ab256608b5b30c6Reid Kleckner 272e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#ifndef NDEBUG 2733a90c9b8edb53ea1ea57d94f7ab256608b5b30c6Reid Kleckner if (PredSU->NumSuccsLeft == 0) { 274e492ae13edd83b120d665c0503cf4de2925b5e56David Greene dbgs() << "*** Scheduling failed! ***\n"; 2753cc6243ddfdba3ad64035b919c88b09773a60880Dan Gohman PredSU->dump(this); 276e492ae13edd83b120d665c0503cf4de2925b5e56David Greene dbgs() << " has been released too many times!\n"; 277c23197a26f34f559ea9797de51e187087c039c42Torok Edwin llvm_unreachable(0); 278e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 279e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#endif 2803a90c9b8edb53ea1ea57d94f7ab256608b5b30c6Reid Kleckner --PredSU->NumSuccsLeft; 2813a90c9b8edb53ea1ea57d94f7ab256608b5b30c6Reid Kleckner 28215a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng if (!ForceUnitLatencies()) { 28315a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng // Updating predecessor's height. This is now the cycle when the 28415a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng // predecessor can be scheduled without causing a pipeline stall. 28515a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng PredSU->setHeightToAtLeast(SU->getHeight() + PredEdge->getLatency()); 28615a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng } 28715a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 2889e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman // If all the node's successors are scheduled, this node is ready 2899e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman // to be scheduled. Ignore the special EntrySU node. 2909e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman if (PredSU->NumSuccsLeft == 0 && PredSU != &EntrySU) { 29180792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman PredSU->isAvailable = true; 2922da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 2932da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned Height = PredSU->getHeight(); 2942da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (Height < MinAvailableCycle) 2952da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick MinAvailableCycle = Height; 2962da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 2972da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (isReady(SU)) { 2982da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->push(PredSU); 2992da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 3002da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // CapturePred and others may have left the node in the pending queue, avoid 3012da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // adding it twice. 3022da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick else if (!PredSU->isPending) { 3032da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PredSU->isPending = true; 3042da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PendingQueue.push_back(PredSU); 3052da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 306e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 307e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 308e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 3091b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// Call ReleasePred for each predecessor, then update register live def/gen. 3101b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// Always update LiveRegDefs for a register dependence even if the current SU 3111b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// also defines the register. This effectively create one large live range 3121b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// across a sequence of two-address node. This is important because the 3131b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// entire chain must be scheduled together. Example: 3141b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// 3151b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// flags = (3) add 3161b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// flags = (2) addc flags 3171b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// flags = (1) addc flags 3181b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// 3191b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// results in 3201b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// 3211b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// LiveRegDefs[flags] = 3 3223d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick/// LiveRegGens[flags] = 1 3231b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// 3241b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// If (2) addc is unscheduled, then (1) addc must also be unscheduled to avoid 3251b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// interference on flags. 3263d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trickvoid ScheduleDAGRRList::ReleasePredecessors(SUnit *SU) { 327e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // Bottom up: release predecessors 328228a18e0f220fb85ee06fd5bfa29304e57047ff1Chris Lattner for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 329a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 33054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman ReleasePred(SU, &*I); 33154e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isAssignedRegDep()) { 332a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // This is a physical register dependency and it's impossible or 33338036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick // expensive to copy the register. Make sure nothing that can 334a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // clobber the register is scheduled between the predecessor and 335a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // this node. 3363d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick SUnit *RegDef = LiveRegDefs[I->getReg()]; (void)RegDef; 3371b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick assert((!RegDef || RegDef == SU || RegDef == I->getSUnit()) && 3381b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick "interference on register dependence"); 3393d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick LiveRegDefs[I->getReg()] = I->getSUnit(); 3403d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick if (!LiveRegGens[I->getReg()]) { 341086ec9976ff6cee083de618429c78473491d5713Dan Gohman ++NumLiveRegs; 3423d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick LiveRegGens[I->getReg()] = SU; 343a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 344a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 345a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 3469e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman} 3479e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman 3482da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// Check to see if any of the pending instructions are ready to issue. If 3492da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// so, add them to the available queue. 3502da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::ReleasePending() { 3512da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick assert(!EnableSchedCycles && "requires --enable-sched-cycles" ); 3522da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 3532da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // If the available queue is empty, it is safe to reset MinAvailableCycle. 3542da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (AvailableQueue->empty()) 3552da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick MinAvailableCycle = UINT_MAX; 3562da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 3572da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Check to see if any of the pending instructions are ready to issue. If 3582da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // so, add them to the available queue. 3592da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick for (unsigned i = 0, e = PendingQueue.size(); i != e; ++i) { 3602da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned ReadyCycle = 3612da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick isBottomUp ? PendingQueue[i]->getHeight() : PendingQueue[i]->getDepth(); 3622da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (ReadyCycle < MinAvailableCycle) 3632da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick MinAvailableCycle = ReadyCycle; 3642da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 3652da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (PendingQueue[i]->isAvailable) { 3662da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (!isReady(PendingQueue[i])) 3672da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick continue; 3682da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->push(PendingQueue[i]); 3692da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 3702da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PendingQueue[i]->isPending = false; 3712da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PendingQueue[i] = PendingQueue.back(); 3722da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PendingQueue.pop_back(); 3732da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick --i; --e; 3742da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 3752da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 3762da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 3772da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// Move the scheduler state forward by the specified number of Cycles. 3782da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::AdvanceToCycle(unsigned NextCycle) { 3792da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (NextCycle <= CurCycle) 3802da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 3812da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 3822da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->setCurCycle(NextCycle); 3832da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (HazardRec->getMaxLookAhead() == 0) { 3842da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Bypass lots of virtual calls in case of long latency. 3852da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CurCycle = NextCycle; 3862da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 3872da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick else { 3882da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick for (; CurCycle != NextCycle; ++CurCycle) { 3892da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (isBottomUp) 3902da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->RecedeCycle(); 3912da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick else 3922da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->AdvanceCycle(); 3932da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 3942da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 3952da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // FIXME: Instead of visiting the pending Q each time, set a dirty flag on the 3962da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // available Q to release pending nodes at least once before popping. 3972da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ReleasePending(); 3982da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 3992da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 4002da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// Move the scheduler state forward until the specified node's dependents are 4012da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// ready and can be scheduled with no resource conflicts. 4022da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::AdvancePastStalls(SUnit *SU) { 4032da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (!EnableSchedCycles) 4042da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 4052da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 4062da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned ReadyCycle = isBottomUp ? SU->getHeight() : SU->getDepth(); 4072da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 4082da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Bump CurCycle to account for latency. We assume the latency of other 4092da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // available instructions may be hidden by the stall (not a full pipe stall). 4102da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // This updates the hazard recognizer's cycle before reserving resources for 4112da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // this instruction. 4122da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AdvanceToCycle(ReadyCycle); 4132da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 4142da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Calls are scheduled in their preceding cycle, so don't conflict with 4152da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // hazards from instructions after the call. EmitNode will reset the 4162da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // scoreboard state before emitting the call. 4172da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (isBottomUp && SU->isCall) 4182da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 4192da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 4202da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // FIXME: For resource conflicts in very long non-pipelined stages, we 4212da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // should probably skip ahead here to avoid useless scoreboard checks. 4222da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick int Stalls = 0; 4232da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick while (true) { 4242da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleHazardRecognizer::HazardType HT = 4252da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->getHazardType(SU, isBottomUp ? -Stalls : Stalls); 4262da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 4272da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (HT == ScheduleHazardRecognizer::NoHazard) 4282da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick break; 4292da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 4302da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ++Stalls; 4312da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 4322da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AdvanceToCycle(CurCycle + Stalls); 4332da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 4342da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 4352da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// Record this SUnit in the HazardRecognizer. 4362da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// Does not update CurCycle. 4372da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::EmitNode(SUnit *SU) { 43824312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick if (!EnableSchedCycles || HazardRec->getMaxLookAhead() == 0) 43924312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick return; 44024312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick 44124312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick // Check for phys reg copy. 44224312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick if (!SU->getNode()) 44324312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick return; 44424312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick 4452da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick switch (SU->getNode()->getOpcode()) { 4462da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick default: 4472da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick assert(SU->getNode()->isMachineOpcode() && 4482da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick "This target-independent node should not be scheduled."); 4492da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick break; 4502da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::MERGE_VALUES: 4512da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::TokenFactor: 4522da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::CopyToReg: 4532da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::CopyFromReg: 4542da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::EH_LABEL: 4552da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Noops don't affect the scoreboard state. Copies are likely to be 4562da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // removed. 4572da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 4582da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::INLINEASM: 4592da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // For inline asm, clear the pipeline state. 4602da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->Reset(); 4612da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 4622da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 4632da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (isBottomUp && SU->isCall) { 4642da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Calls are scheduled with their preceding instructions. For bottom-up 4652da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // scheduling, clear the pipeline state before emitting. 4662da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->Reset(); 4672da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 4682da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 4692da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->EmitInstruction(SU); 4702da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 4712da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (!isBottomUp && SU->isCall) { 4722da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->Reset(); 4732da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 4742da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 4752da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 4769e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman/// ScheduleNodeBottomUp - Add the node to the schedule. Decrement the pending 4779e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman/// count of its predecessors. If a predecessor pending count is zero, add it to 4789e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman/// the Available queue. 4792902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trickvoid ScheduleDAGRRList::ScheduleNodeBottomUp(SUnit *SU) { 48015a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng DEBUG(dbgs() << "\n*** Scheduling [" << CurCycle << "]: "); 4819e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman DEBUG(SU->dump(this)); 4829e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman 48315a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng#ifndef NDEBUG 48415a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng if (CurCycle < SU->getHeight()) 48515a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng DEBUG(dbgs() << " Height [" << SU->getHeight() << "] pipeline stall!\n"); 48615a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng#endif 48715a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 4882da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // FIXME: Do not modify node height. It may interfere with 4892da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // backtracking. Instead add a "ready cycle" to SUnit. Before scheduling the 4902da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // node it's ready cycle can aid heuristics, and after scheduling it can 4912da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // indicate the scheduled cycle. 4929e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman SU->setHeightToAtLeast(CurCycle); 4932da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 4942da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Reserve resources for the scheduled intruction. 4952da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick EmitNode(SU); 4962da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 4979e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman Sequence.push_back(SU); 4989e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman 4994a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng AvailableQueue->ScheduledNode(SU); 50037944985a569f8c2b0d75dafd9e2739a9887ac5dChris Lattner 5011b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick // Update liveness of predecessors before successors to avoid treating a 5021b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick // two-address node as a live range def. 5033d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick ReleasePredecessors(SU); 504a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 505a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // Release all the implicit physical register defs that are live. 506a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 507a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 5081b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick // LiveRegDegs[I->getReg()] != SU when SU is a two-address node. 5091b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick if (I->isAssignedRegDep() && LiveRegDefs[I->getReg()] == SU) { 5101b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!"); 5111b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick --NumLiveRegs; 5121b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick LiveRegDefs[I->getReg()] = NULL; 5133d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick LiveRegGens[I->getReg()] = NULL; 514a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 515a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 516a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 517e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng SU->isScheduled = true; 5182da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 5192da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Conditions under which the scheduler should eagerly advance the cycle: 5202da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // (1) No available instructions 5212da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // (2) All pipelines full, so available instructions must have hazards. 5222da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // 5232da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // If SchedCycles is disabled, count each inst as one cycle. 5242da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (!EnableSchedCycles || 5252da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->empty() || HazardRec->atIssueLimit()) 5262da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AdvanceToCycle(CurCycle + 1); 527e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 528e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 529a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// CapturePred - This does the opposite of ReleasePred. Since SU is being 530a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// unscheduled, incrcease the succ left count of its predecessors. Remove 531a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// them from AvailableQueue if necessary. 53238036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trickvoid ScheduleDAGRRList::CapturePred(SDep *PredEdge) { 53354e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *PredSU = PredEdge->getSUnit(); 534a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng if (PredSU->isAvailable) { 535a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng PredSU->isAvailable = false; 536a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng if (!PredSU->isPending) 537a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng AvailableQueue->remove(PredSU); 538a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 539a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 540c277ab08a24d2dbe9b4ff1a9154ea6115ed6a4e3Reid Kleckner assert(PredSU->NumSuccsLeft < UINT_MAX && "NumSuccsLeft will overflow!"); 54174d2fd8dd847e0ebccef30e2c5907ff09495d518Evan Cheng ++PredSU->NumSuccsLeft; 542a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng} 543a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 544a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// UnscheduleNodeBottomUp - Remove the node from the schedule, update its and 545a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// its predecessor states to reflect the change. 546a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Chengvoid ScheduleDAGRRList::UnscheduleNodeBottomUp(SUnit *SU) { 547e492ae13edd83b120d665c0503cf4de2925b5e56David Greene DEBUG(dbgs() << "*** Unscheduling [" << SU->getHeight() << "]: "); 5483cc6243ddfdba3ad64035b919c88b09773a60880Dan Gohman DEBUG(SU->dump(this)); 549a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 550a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 551a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 55254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman CapturePred(&*I); 5533d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick if (I->isAssignedRegDep() && SU == LiveRegGens[I->getReg()]){ 554086ec9976ff6cee083de618429c78473491d5713Dan Gohman assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!"); 55554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman assert(LiveRegDefs[I->getReg()] == I->getSUnit() && 556a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng "Physical register dependency violated?"); 557086ec9976ff6cee083de618429c78473491d5713Dan Gohman --NumLiveRegs; 55854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman LiveRegDefs[I->getReg()] = NULL; 5593d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick LiveRegGens[I->getReg()] = NULL; 560a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 561a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 562a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 563a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 564a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 56554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isAssignedRegDep()) { 5661b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick // This becomes the nearest def. Note that an earlier def may still be 5671b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick // pending if this is a two-address node. 5681b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick LiveRegDefs[I->getReg()] = SU; 56954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (!LiveRegDefs[I->getReg()]) { 570086ec9976ff6cee083de618429c78473491d5713Dan Gohman ++NumLiveRegs; 571a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 5723d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick if (LiveRegGens[I->getReg()] == NULL || 5733d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick I->getSUnit()->getHeight() < LiveRegGens[I->getReg()]->getHeight()) 5743d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick LiveRegGens[I->getReg()] = I->getSUnit(); 575a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 576a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 5772da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (SU->getHeight() < MinAvailableCycle) 5782da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick MinAvailableCycle = SU->getHeight(); 579a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 5803f23744df4809eba94284e601e81489212c974d4Dan Gohman SU->setHeightDirty(); 581a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng SU->isScheduled = false; 582a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng SU->isAvailable = true; 5832da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (EnableSchedCycles && AvailableQueue->hasReadyFilter()) { 5842da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Don't make available until backtracking is complete. 5852da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SU->isPending = true; 5862da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PendingQueue.push_back(SU); 5872da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 5882da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick else { 5892da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->push(SU); 5902da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 5914a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng AvailableQueue->UnscheduledNode(SU); 592a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng} 593a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 5942da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// After backtracking, the hazard checker needs to be restored to a state 5952da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// corresponding the the current cycle. 5962da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::RestoreHazardCheckerBottomUp() { 5972da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->Reset(); 5982da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 5992da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned LookAhead = std::min((unsigned)Sequence.size(), 6002da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->getMaxLookAhead()); 6012da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (LookAhead == 0) 6022da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 6032da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6042da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick std::vector<SUnit*>::const_iterator I = (Sequence.end() - LookAhead); 6052da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned HazardCycle = (*I)->getHeight(); 6062da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick for (std::vector<SUnit*>::const_iterator E = Sequence.end(); I != E; ++I) { 6072da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SUnit *SU = *I; 6082da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick for (; SU->getHeight() > HazardCycle; ++HazardCycle) { 6092da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->RecedeCycle(); 6102da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 6112da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick EmitNode(SU); 6122da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 6132da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 6142da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 61542d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng/// BacktrackBottomUp - Backtrack scheduling to a previous cycle specified in 6161cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohman/// BTCycle in order to schedule a specific node. 6172da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::BacktrackBottomUp(SUnit *SU, SUnit *BtSU) { 6182da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SUnit *OldSU = Sequence.back(); 6192da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick while (true) { 620a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng Sequence.pop_back(); 621a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng if (SU->isSucc(OldSU)) 62242d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng // Don't try to remove SU from AvailableQueue. 62342d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng SU->isAvailable = false; 6242da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // FIXME: use ready cycle instead of height 6252da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CurCycle = OldSU->getHeight(); 626a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng UnscheduleNodeBottomUp(OldSU); 62715a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng AvailableQueue->setCurCycle(CurCycle); 6282da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (OldSU == BtSU) 6292da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick break; 6302da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick OldSU = Sequence.back(); 631a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 632a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 6331cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohman assert(!SU->isSucc(OldSU) && "Something is wrong!"); 634a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng 6352da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick RestoreHazardCheckerBottomUp(); 6362da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6372da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (EnableSchedCycles) 6382da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ReleasePending(); 6392da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 640a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng ++NumBacktracks; 641a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng} 642a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 6435ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Chengstatic bool isOperandOf(const SUnit *SU, SDNode *N) { 6445ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng for (const SDNode *SUNode = SU->getNode(); SUNode; 64529d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner SUNode = SUNode->getGluedNode()) { 6465ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng if (SUNode->isOperandOf(N)) 6475ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng return true; 6485ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng } 6495ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng return false; 6505ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng} 6515ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng 652f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng/// CopyAndMoveSuccessors - Clone the specified node and move its scheduled 653f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng/// successors to the newly created node. 654f10c973797cf79da802f9b0118543cbd50954c9cEvan ChengSUnit *ScheduleDAGRRList::CopyAndMoveSuccessors(SUnit *SU) { 655550f5afb68ce8f034991863cac65bef22a6554daDan Gohman SDNode *N = SU->getNode(); 65642d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng if (!N) 657f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng return NULL; 65842d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 65924312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick if (SU->getNode()->getGluedNode()) 66024312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick return NULL; 66124312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick 662f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SUnit *NewSU; 663f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng bool TryUnfold = false; 664d5cb5a462b6fd91bf54116c3eefc3b046489c414Evan Cheng for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) { 665e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = N->getValueType(i); 666f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner if (VT == MVT::Glue) 667d5cb5a462b6fd91bf54116c3eefc3b046489c414Evan Cheng return NULL; 668825b72b0571821bf2d378749f69d6c4cfb52d2f9Owen Anderson else if (VT == MVT::Other) 669d5cb5a462b6fd91bf54116c3eefc3b046489c414Evan Cheng TryUnfold = true; 670d5cb5a462b6fd91bf54116c3eefc3b046489c414Evan Cheng } 671a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 672475871a144eb604ddaf37503397ba0941442e5fbDan Gohman const SDValue &Op = N->getOperand(i); 673e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = Op.getNode()->getValueType(Op.getResNo()); 674f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner if (VT == MVT::Glue) 675f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng return NULL; 676a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 677a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 678f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng if (TryUnfold) { 6794c8c83022b501759d8559e224c84ae2a9921ba41Dan Gohman SmallVector<SDNode*, 2> NewNodes; 680a23b3b803e3c65e84d6cadaa221de8b256cbe28dDan Gohman if (!TII->unfoldMemoryOperand(*DAG, N, NewNodes)) 681f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng return NULL; 682f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 68315a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng DEBUG(dbgs() << "Unfolding SU #" << SU->NodeNum << "\n"); 684f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng assert(NewNodes.size() == 2 && "Expected a load folding node!"); 685f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 686f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng N = NewNodes[1]; 687f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SDNode *LoadNode = NewNodes[0]; 688f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng unsigned NumVals = N->getNumValues(); 689550f5afb68ce8f034991863cac65bef22a6554daDan Gohman unsigned OldNumVals = SU->getNode()->getNumValues(); 690f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (unsigned i = 0; i != NumVals; ++i) 691550f5afb68ce8f034991863cac65bef22a6554daDan Gohman DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), i), SDValue(N, i)); 692550f5afb68ce8f034991863cac65bef22a6554daDan Gohman DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), OldNumVals-1), 693a23b3b803e3c65e84d6cadaa221de8b256cbe28dDan Gohman SDValue(LoadNode, 1)); 694f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 695b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman // LoadNode may already exist. This can happen when there is another 696b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman // load from the same location and producing the same type of value 697b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman // but it has different alignment or volatileness. 698b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman bool isNewLoad = true; 699b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman SUnit *LoadSU; 700b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman if (LoadNode->getNodeId() != -1) { 701b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman LoadSU = &SUnits[LoadNode->getNodeId()]; 702b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman isNewLoad = false; 703b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman } else { 704b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman LoadSU = CreateNewSUnit(LoadNode); 705b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman LoadNode->setNodeId(LoadSU->NodeNum); 706b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman ComputeLatency(LoadSU); 707b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman } 708b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman 709e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *NewSU = CreateNewSUnit(N); 71094d7a5f8156e62532870fbaf197377b34e52ff2aDan Gohman assert(N->getNodeId() == -1 && "Node already inserted!"); 71194d7a5f8156e62532870fbaf197377b34e52ff2aDan Gohman N->setNodeId(NewSU->NodeNum); 71238036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 713e8be6c63915e0389f1eef6b53c64300d13b2ce99Dan Gohman const TargetInstrDesc &TID = TII->get(N->getMachineOpcode()); 71494ebde1d45dcd7e209663c49a1cf1a4589191df1Dan Gohman for (unsigned i = 0; i != TID.getNumOperands(); ++i) { 7153db805ea80eeec9084a1b86273d93804d233d938Chris Lattner if (TID.getOperandConstraint(i, TOI::TIED_TO) != -1) { 716f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng NewSU->isTwoAddress = true; 717f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng break; 718f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 719f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 7203db805ea80eeec9084a1b86273d93804d233d938Chris Lattner if (TID.isCommutable()) 721f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng NewSU->isCommutable = true; 722f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng ComputeLatency(NewSU); 723f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 724fa9afef7eaa0865fb9f3489a68a04332b232ed82Dan Gohman // Record all the edges to and from the old SU, by category. 72516e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman SmallVector<SDep, 4> ChainPreds; 726f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SmallVector<SDep, 4> ChainSuccs; 727f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SmallVector<SDep, 4> LoadPreds; 728f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SmallVector<SDep, 4> NodePreds; 729f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SmallVector<SDep, 4> NodeSuccs; 730f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 731f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng I != E; ++I) { 73254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isCtrl()) 73316e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman ChainPreds.push_back(*I); 7345ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng else if (isOperandOf(I->getSUnit(), LoadNode)) 73554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman LoadPreds.push_back(*I); 736f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng else 73754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman NodePreds.push_back(*I); 738f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 739f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 740f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng I != E; ++I) { 74154e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isCtrl()) 74254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman ChainSuccs.push_back(*I); 743f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng else 74454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman NodeSuccs.push_back(*I); 745f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 74642d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 747fa9afef7eaa0865fb9f3489a68a04332b232ed82Dan Gohman // Now assign edges to the newly-created nodes. 74816e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman for (unsigned i = 0, e = ChainPreds.size(); i != e; ++i) { 74916e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman const SDep &Pred = ChainPreds[i]; 75016e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman RemovePred(SU, Pred); 75180792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman if (isNewLoad) 75216e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman AddPred(LoadSU, Pred); 753e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein } 754f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (unsigned i = 0, e = LoadPreds.size(); i != e; ++i) { 75554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman const SDep &Pred = LoadPreds[i]; 75654e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(SU, Pred); 75716e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman if (isNewLoad) 75854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(LoadSU, Pred); 759f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 760f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (unsigned i = 0, e = NodePreds.size(); i != e; ++i) { 76154e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman const SDep &Pred = NodePreds[i]; 76254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(SU, Pred); 76354e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(NewSU, Pred); 764f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 765f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (unsigned i = 0, e = NodeSuccs.size(); i != e; ++i) { 76654e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SDep D = NodeSuccs[i]; 76754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *SuccDep = D.getSUnit(); 76854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(SU); 76954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(SuccDep, D); 77054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(NewSU); 77154e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(SuccDep, D); 772f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 773f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (unsigned i = 0, e = ChainSuccs.size(); i != e; ++i) { 77454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SDep D = ChainSuccs[i]; 77554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *SuccDep = D.getSUnit(); 77654e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(SU); 77754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(SuccDep, D); 778e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein if (isNewLoad) { 77954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(LoadSU); 78054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(SuccDep, D); 781e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein } 78238036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick } 783fa9afef7eaa0865fb9f3489a68a04332b232ed82Dan Gohman 784fa9afef7eaa0865fb9f3489a68a04332b232ed82Dan Gohman // Add a data dependency to reflect that NewSU reads the value defined 785fa9afef7eaa0865fb9f3489a68a04332b232ed82Dan Gohman // by LoadSU. 786fa9afef7eaa0865fb9f3489a68a04332b232ed82Dan Gohman AddPred(NewSU, SDep(LoadSU, SDep::Data, LoadSU->Latency)); 787f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 788beec823d4bba22b1c0c6658d2b3e71cd64a70e2eEvan Cheng if (isNewLoad) 789beec823d4bba22b1c0c6658d2b3e71cd64a70e2eEvan Cheng AvailableQueue->addNode(LoadSU); 790f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng AvailableQueue->addNode(NewSU); 791f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 792f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng ++NumUnfolds; 793f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 794f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng if (NewSU->NumSuccsLeft == 0) { 795f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng NewSU->isAvailable = true; 796f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng return NewSU; 797beec823d4bba22b1c0c6658d2b3e71cd64a70e2eEvan Cheng } 798beec823d4bba22b1c0c6658d2b3e71cd64a70e2eEvan Cheng SU = NewSU; 799f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 800f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 80115a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng DEBUG(dbgs() << " Duplicating SU #" << SU->NodeNum << "\n"); 802e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein NewSU = CreateClone(SU); 803a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 804a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // New SUnit has the exact same predecessors. 805a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 806a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) 8073f23744df4809eba94284e601e81489212c974d4Dan Gohman if (!I->isArtificial()) 80854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(NewSU, *I); 809a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 810a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // Only copy scheduled successors. Cut them from old node's successor 811a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // list and move them over. 81254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SmallVector<std::pair<SUnit *, SDep>, 4> DelDeps; 813a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 814a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 81554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isArtificial()) 816a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng continue; 81754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *SuccSU = I->getSUnit(); 81854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (SuccSU->isScheduled) { 81954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SDep D = *I; 82054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(NewSU); 82154e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(SuccSU, D); 82254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(SU); 82354e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman DelDeps.push_back(std::make_pair(SuccSU, D)); 824a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 825a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 8263f23744df4809eba94284e601e81489212c974d4Dan Gohman for (unsigned i = 0, e = DelDeps.size(); i != e; ++i) 82754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(DelDeps[i].first, DelDeps[i].second); 828a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 829a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng AvailableQueue->updateNode(SU); 830a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng AvailableQueue->addNode(NewSU); 831a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 832a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng ++NumDups; 833a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng return NewSU; 834a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng} 835a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 836c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng/// InsertCopiesAndMoveSuccs - Insert register copies and move all 837c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng/// scheduled successors of the given SUnit to the last copy. 838c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Chengvoid ScheduleDAGRRList::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg, 839c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng const TargetRegisterClass *DestRC, 840c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng const TargetRegisterClass *SrcRC, 841a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng SmallVector<SUnit*, 2> &Copies) { 842e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *CopyFromSU = CreateNewSUnit(NULL); 84342d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng CopyFromSU->CopySrcRC = SrcRC; 84442d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng CopyFromSU->CopyDstRC = DestRC; 84542d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 846e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *CopyToSU = CreateNewSUnit(NULL); 84742d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng CopyToSU->CopySrcRC = DestRC; 84842d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng CopyToSU->CopyDstRC = SrcRC; 84942d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 85042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng // Only copy scheduled successors. Cut them from old node's successor 85142d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng // list and move them over. 85254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SmallVector<std::pair<SUnit *, SDep>, 4> DelDeps; 85342d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 85442d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng I != E; ++I) { 85554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isArtificial()) 85642d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng continue; 85754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *SuccSU = I->getSUnit(); 85854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (SuccSU->isScheduled) { 85954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SDep D = *I; 86054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(CopyToSU); 86154e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(SuccSU, D); 86254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman DelDeps.push_back(std::make_pair(SuccSU, *I)); 86342d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng } 86442d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng } 865c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng for (unsigned i = 0, e = DelDeps.size(); i != e; ++i) 86654e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(DelDeps[i].first, DelDeps[i].second); 86742d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 86854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(CopyFromSU, SDep(SU, SDep::Data, SU->Latency, Reg)); 86954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(CopyToSU, SDep(CopyFromSU, SDep::Data, CopyFromSU->Latency, 0)); 87042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 87142d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng AvailableQueue->updateNode(SU); 87242d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng AvailableQueue->addNode(CopyFromSU); 87342d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng AvailableQueue->addNode(CopyToSU); 874a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng Copies.push_back(CopyFromSU); 875a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng Copies.push_back(CopyToSU); 87642d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 877c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng ++NumPRCopies; 87842d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng} 87942d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 88042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng/// getPhysicalRegisterVT - Returns the ValueType of the physical register 88142d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng/// definition of the specified node. 88242d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng/// FIXME: Move to SelectionDAG? 883e50ed30282bb5b4a9ed952580523f2dda16215acOwen Andersonstatic EVT getPhysicalRegisterVT(SDNode *N, unsigned Reg, 88483ec4b6711980242ef3c55a4fa36b2d7a39c1bfbDuncan Sands const TargetInstrInfo *TII) { 885e8be6c63915e0389f1eef6b53c64300d13b2ce99Dan Gohman const TargetInstrDesc &TID = TII->get(N->getMachineOpcode()); 88642d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng assert(TID.ImplicitDefs && "Physical reg def must be in implicit def list!"); 887349c4952009525b27383e2120a6b3c998f39bd09Chris Lattner unsigned NumRes = TID.getNumDefs(); 888349c4952009525b27383e2120a6b3c998f39bd09Chris Lattner for (const unsigned *ImpDef = TID.getImplicitDefs(); *ImpDef; ++ImpDef) { 88942d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng if (Reg == *ImpDef) 89042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng break; 89142d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng ++NumRes; 89242d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng } 89342d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng return N->getValueType(NumRes); 89442d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng} 89542d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 896599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng/// CheckForLiveRegDef - Return true and update live register vector if the 897599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng/// specified register def of the specified SUnit clobbers any "live" registers. 898142d21c861c0b686e38a515b1271f4157cd24004Chris Lattnerstatic void CheckForLiveRegDef(SUnit *SU, unsigned Reg, 899599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng std::vector<SUnit*> &LiveRegDefs, 900599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng SmallSet<unsigned, 4> &RegAdded, 901599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng SmallVector<unsigned, 4> &LRegs, 902599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng const TargetRegisterInfo *TRI) { 903cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick for (const unsigned *AliasI = TRI->getOverlaps(Reg); *AliasI; ++AliasI) { 904cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick 905cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick // Check if Ref is live. 906cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick if (!LiveRegDefs[Reg]) continue; 907cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick 908cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick // Allow multiple uses of the same def. 909cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick if (LiveRegDefs[Reg] == SU) continue; 910cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick 911cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick // Add Reg to the set of interfering live regs. 912142d21c861c0b686e38a515b1271f4157cd24004Chris Lattner if (RegAdded.insert(Reg)) 913599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng LRegs.push_back(Reg); 914599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng } 915599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng} 916599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng 917a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// DelayForLiveRegsBottomUp - Returns true if it is necessary to delay 918a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// scheduling of the given node to satisfy live physical register dependencies. 919a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// If the specific node is the last one that's available to schedule, do 920a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// whatever is necessary (i.e. backtracking or cloning) to make it possible. 921142d21c861c0b686e38a515b1271f4157cd24004Chris Lattnerbool ScheduleDAGRRList:: 922142d21c861c0b686e38a515b1271f4157cd24004Chris LattnerDelayForLiveRegsBottomUp(SUnit *SU, SmallVector<unsigned, 4> &LRegs) { 923086ec9976ff6cee083de618429c78473491d5713Dan Gohman if (NumLiveRegs == 0) 924a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng return false; 925a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 926cd1c00cc6521c265784ffc7a1b5baf4ef64d80bcEvan Cheng SmallSet<unsigned, 4> RegAdded; 927a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // If this node would clobber any "live" register, then it's not ready. 928feac09801b5c03412d452e685570baff6eb84c88Andrew Trick // 929feac09801b5c03412d452e685570baff6eb84c88Andrew Trick // If SU is the currently live definition of the same register that it uses, 930feac09801b5c03412d452e685570baff6eb84c88Andrew Trick // then we are free to schedule it. 931a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 932a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 933feac09801b5c03412d452e685570baff6eb84c88Andrew Trick if (I->isAssignedRegDep() && LiveRegDefs[I->getReg()] != SU) 934599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng CheckForLiveRegDef(I->getSUnit(), I->getReg(), LiveRegDefs, 935599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng RegAdded, LRegs, TRI); 936a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 937a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 93829d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) { 939599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng if (Node->getOpcode() == ISD::INLINEASM) { 940599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng // Inline asm can clobber physical defs. 941599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng unsigned NumOps = Node->getNumOperands(); 942f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue) 94329d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner --NumOps; // Ignore the glue operand. 944599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng 945decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) { 946599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng unsigned Flags = 947599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue(); 948decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags); 949599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng 950599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng ++i; // Skip the ID value. 951decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner if (InlineAsm::isRegDefKind(Flags) || 952decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner InlineAsm::isRegDefEarlyClobberKind(Flags)) { 953599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng // Check for def of register or earlyclobber register. 954599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng for (; NumVals; --NumVals, ++i) { 955599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg(); 956599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng if (TargetRegisterInfo::isPhysicalRegister(Reg)) 957599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng CheckForLiveRegDef(SU, Reg, LiveRegDefs, RegAdded, LRegs, TRI); 958599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng } 959599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng } else 960599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng i += NumVals; 961599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng } 962599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng continue; 963599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng } 964599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng 965d23e0f81bc76902052e9198cad3a0d87a412a632Dan Gohman if (!Node->isMachineOpcode()) 966a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng continue; 967e8be6c63915e0389f1eef6b53c64300d13b2ce99Dan Gohman const TargetInstrDesc &TID = TII->get(Node->getMachineOpcode()); 968a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng if (!TID.ImplicitDefs) 969a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng continue; 970599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng for (const unsigned *Reg = TID.ImplicitDefs; *Reg; ++Reg) 971599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng CheckForLiveRegDef(SU, *Reg, LiveRegDefs, RegAdded, LRegs, TRI); 972a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 97338036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 974a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng return !LRegs.empty(); 975e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 976e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 9772902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick/// Return a node that can be scheduled in this cycle. Requirements: 9782902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick/// (1) Ready: latency has been satisfied 9792da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// (2) No Hazards: resources are available 9802902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick/// (3) No Interferences: may unschedule to break register interferences. 9812902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew TrickSUnit *ScheduleDAGRRList::PickNodeToScheduleBottomUp() { 9822902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SmallVector<SUnit*, 4> Interferences; 9832902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick DenseMap<SUnit*, SmallVector<unsigned, 4> > LRegsMap; 9842902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 9852902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *CurSU = AvailableQueue->pop(); 9862902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick while (CurSU) { 9872902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SmallVector<unsigned, 4> LRegs; 9882902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick if (!DelayForLiveRegsBottomUp(CurSU, LRegs)) 9892902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick break; 9902902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick LRegsMap.insert(std::make_pair(CurSU, LRegs)); 9912902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 9922902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurSU->isPending = true; // This SU is not in AvailableQueue right now. 9932902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick Interferences.push_back(CurSU); 9942902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurSU = AvailableQueue->pop(); 9952902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 9962902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick if (CurSU) { 9972902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Add the nodes that aren't ready back onto the available list. 9982902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick for (unsigned i = 0, e = Interferences.size(); i != e; ++i) { 9992902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick Interferences[i]->isPending = false; 10002902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick assert(Interferences[i]->isAvailable && "must still be available"); 10012902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick AvailableQueue->push(Interferences[i]); 10022902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 10032902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick return CurSU; 10042902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 10052902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 10062902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // All candidates are delayed due to live physical reg dependencies. 10072902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Try backtracking, code duplication, or inserting cross class copies 10082902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // to resolve it. 10092902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick for (unsigned i = 0, e = Interferences.size(); i != e; ++i) { 10102902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *TrySU = Interferences[i]; 10112902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SmallVector<unsigned, 4> &LRegs = LRegsMap[TrySU]; 10122902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 10132902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Try unscheduling up to the point where it's safe to schedule 10142902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // this node. 10152da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SUnit *BtSU = NULL; 10162da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned LiveCycle = UINT_MAX; 10172902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick for (unsigned j = 0, ee = LRegs.size(); j != ee; ++j) { 10182902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick unsigned Reg = LRegs[j]; 10192da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (LiveRegGens[Reg]->getHeight() < LiveCycle) { 10202da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick BtSU = LiveRegGens[Reg]; 10212da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick LiveCycle = BtSU->getHeight(); 10222da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 10232902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 10242da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (!WillCreateCycle(TrySU, BtSU)) { 10252da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick BacktrackBottomUp(TrySU, BtSU); 10262902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 10272902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Force the current node to be scheduled before the node that 10282902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // requires the physical reg dep. 10292da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (BtSU->isAvailable) { 10302da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick BtSU->isAvailable = false; 10312da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (!BtSU->isPending) 10322da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->remove(BtSU); 10332902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 10342da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AddPred(TrySU, SDep(BtSU, SDep::Order, /*Latency=*/1, 10352902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick /*Reg=*/0, /*isNormalMemory=*/false, 10362902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick /*isMustAlias=*/false, /*isArtificial=*/true)); 10372902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 10382902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // If one or more successors has been unscheduled, then the current 10392902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // node is no longer avaialable. Schedule a successor that's now 10402902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // available instead. 10412902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick if (!TrySU->isAvailable) { 10422902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurSU = AvailableQueue->pop(); 10432902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 10442902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick else { 10452902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurSU = TrySU; 10462902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick TrySU->isPending = false; 10472902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick Interferences.erase(Interferences.begin()+i); 10482902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 10492902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick break; 10502902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 10512902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 10522902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 10532902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick if (!CurSU) { 10542902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Can't backtrack. If it's too expensive to copy the value, then try 10552902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // duplicate the nodes that produces these "too expensive to copy" 10562902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // values to break the dependency. In case even that doesn't work, 10572902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // insert cross class copies. 10582902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // If it's not too expensive, i.e. cost != -1, issue copies. 10592902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *TrySU = Interferences[0]; 10602902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SmallVector<unsigned, 4> &LRegs = LRegsMap[TrySU]; 10612902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick assert(LRegs.size() == 1 && "Can't handle this yet!"); 10622902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick unsigned Reg = LRegs[0]; 10632902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *LRDef = LiveRegDefs[Reg]; 10642902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick EVT VT = getPhysicalRegisterVT(LRDef->getNode(), Reg, TII); 10652902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick const TargetRegisterClass *RC = 10662902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick TRI->getMinimalPhysRegClass(Reg, VT); 10672902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick const TargetRegisterClass *DestRC = TRI->getCrossCopyRegClass(RC); 10682902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 10692902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // If cross copy register class is null, then it must be possible copy 10702902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // the value directly. Do not try duplicate the def. 10712902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *NewDef = 0; 10722902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick if (DestRC) 10732902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick NewDef = CopyAndMoveSuccessors(LRDef); 10742902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick else 10752902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick DestRC = RC; 10762902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick if (!NewDef) { 10772902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Issue copies, these can be expensive cross register class copies. 10782902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SmallVector<SUnit*, 2> Copies; 10792902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies); 10802902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick DEBUG(dbgs() << " Adding an edge from SU #" << TrySU->NodeNum 10812902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick << " to SU #" << Copies.front()->NodeNum << "\n"); 10822902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick AddPred(TrySU, SDep(Copies.front(), SDep::Order, /*Latency=*/1, 10832902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick /*Reg=*/0, /*isNormalMemory=*/false, 10842902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick /*isMustAlias=*/false, 10852902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick /*isArtificial=*/true)); 10862902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick NewDef = Copies.back(); 10872902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 10882902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 10892902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick DEBUG(dbgs() << " Adding an edge from SU #" << NewDef->NodeNum 10902902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick << " to SU #" << TrySU->NodeNum << "\n"); 10912902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick LiveRegDefs[Reg] = NewDef; 10922902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick AddPred(NewDef, SDep(TrySU, SDep::Order, /*Latency=*/1, 10932902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick /*Reg=*/0, /*isNormalMemory=*/false, 10942902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick /*isMustAlias=*/false, 10952902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick /*isArtificial=*/true)); 10962902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick TrySU->isAvailable = false; 10972902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurSU = NewDef; 10982902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 10992902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 11002902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick assert(CurSU && "Unable to resolve live physical register dependencies!"); 11012902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 11022902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Add the nodes that aren't ready back onto the available list. 11032902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick for (unsigned i = 0, e = Interferences.size(); i != e; ++i) { 11042902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick Interferences[i]->isPending = false; 11052902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // May no longer be available due to backtracking. 11062902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick if (Interferences[i]->isAvailable) { 11072902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick AvailableQueue->push(Interferences[i]); 11082902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 11092902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 11102902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick return CurSU; 11112902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick} 1112a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng 1113e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// ListScheduleBottomUp - The main loop of list scheduling for bottom-up 1114e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// schedulers. 1115e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengvoid ScheduleDAGRRList::ListScheduleBottomUp() { 11169e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman // Release any predecessors of the special Exit node. 11173d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick ReleasePredecessors(&ExitSU); 11189e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman 1119e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // Add root to Available queue. 112080792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman if (!SUnits.empty()) { 1121a23b3b803e3c65e84d6cadaa221de8b256cbe28dDan Gohman SUnit *RootSU = &SUnits[DAG->getRoot().getNode()->getNodeId()]; 112280792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman assert(RootSU->Succs.empty() && "Graph root shouldn't have successors!"); 112380792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman RootSU->isAvailable = true; 112480792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman AvailableQueue->push(RootSU); 112580792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman } 1126e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1127e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // While Available queue is not empty, grab the node with the highest 11288d1bfad00b1ebff5b140b6e1bd7e26bad697d6e1Dan Gohman // priority. If it is not ready put it back. Schedule the node. 11294c8c83022b501759d8559e224c84ae2a9921ba41Dan Gohman Sequence.reserve(SUnits.size()); 1130e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng while (!AvailableQueue->empty()) { 11312da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick DEBUG(dbgs() << "\n*** Examining Available\n"; 11322da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->dump(this)); 11332da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 11342902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Pick the best node to schedule taking all constraints into 11352902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // consideration. 11362902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *SU = PickNodeToScheduleBottomUp(); 1137a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng 11382da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AdvancePastStalls(SU); 1139a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng 11402da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleNodeBottomUp(SU); 11412da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 11422da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick while (AvailableQueue->empty() && !PendingQueue.empty()) { 11432da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Advance the cycle to free resources. Skip ahead to the next ready SU. 11442da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick assert(MinAvailableCycle < UINT_MAX && "MinAvailableCycle uninitialized"); 11452da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AdvanceToCycle(std::max(CurCycle + 1, MinAvailableCycle)); 11462da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 1147e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 1148e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1149e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // Reverse the order if it is bottom up. 1150e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng std::reverse(Sequence.begin(), Sequence.end()); 115138036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1152e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#ifndef NDEBUG 1153a1e6d363e5efa9eb1a2e7ac21a0394c870bef5adDan Gohman VerifySchedule(isBottomUp); 1154e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#endif 1155e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 1156e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1157e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 1158e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// Top-Down Scheduling 1159e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 1160e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1161e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// ReleaseSucc - Decrement the NumPredsLeft count of a successor. Add it to 11628d1bfad00b1ebff5b140b6e1bd7e26bad697d6e1Dan Gohman/// the AvailableQueue if the count reaches zero. Also update its cycle bound. 11631cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohmanvoid ScheduleDAGRRList::ReleaseSucc(SUnit *SU, const SDep *SuccEdge) { 116454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *SuccSU = SuccEdge->getSUnit(); 1165c277ab08a24d2dbe9b4ff1a9154ea6115ed6a4e3Reid Kleckner 1166e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#ifndef NDEBUG 1167c277ab08a24d2dbe9b4ff1a9154ea6115ed6a4e3Reid Kleckner if (SuccSU->NumPredsLeft == 0) { 1168e492ae13edd83b120d665c0503cf4de2925b5e56David Greene dbgs() << "*** Scheduling failed! ***\n"; 11693cc6243ddfdba3ad64035b919c88b09773a60880Dan Gohman SuccSU->dump(this); 1170e492ae13edd83b120d665c0503cf4de2925b5e56David Greene dbgs() << " has been released too many times!\n"; 1171c23197a26f34f559ea9797de51e187087c039c42Torok Edwin llvm_unreachable(0); 1172e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 1173e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#endif 1174c277ab08a24d2dbe9b4ff1a9154ea6115ed6a4e3Reid Kleckner --SuccSU->NumPredsLeft; 1175c277ab08a24d2dbe9b4ff1a9154ea6115ed6a4e3Reid Kleckner 11769e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman // If all the node's predecessors are scheduled, this node is ready 11779e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman // to be scheduled. Ignore the special ExitSU node. 11789e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman if (SuccSU->NumPredsLeft == 0 && SuccSU != &ExitSU) { 1179e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng SuccSU->isAvailable = true; 1180e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng AvailableQueue->push(SuccSU); 1181e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 1182e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 1183e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 11849e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohmanvoid ScheduleDAGRRList::ReleaseSuccessors(SUnit *SU) { 11859e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman // Top down: release successors 11869e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 11879e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman I != E; ++I) { 11889e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman assert(!I->isAssignedRegDep() && 11899e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman "The list-tdrr scheduler doesn't yet support physreg dependencies!"); 11909e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman 11919e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman ReleaseSucc(SU, &*I); 11929e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman } 11939e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman} 11949e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman 1195e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// ScheduleNodeTopDown - Add the node to the schedule. Decrement the pending 1196e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// count of its successors. If a successor pending count is zero, add it to 1197e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// the Available queue. 11982902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trickvoid ScheduleDAGRRList::ScheduleNodeTopDown(SUnit *SU) { 1199e492ae13edd83b120d665c0503cf4de2925b5e56David Greene DEBUG(dbgs() << "*** Scheduling [" << CurCycle << "]: "); 12003cc6243ddfdba3ad64035b919c88b09773a60880Dan Gohman DEBUG(SU->dump(this)); 1201e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 12023f23744df4809eba94284e601e81489212c974d4Dan Gohman assert(CurCycle >= SU->getDepth() && "Node scheduled above its depth!"); 12033f23744df4809eba94284e601e81489212c974d4Dan Gohman SU->setDepthToAtLeast(CurCycle); 12048123419f2be881ca77a897918f28514aa4e91765Dan Gohman Sequence.push_back(SU); 1205e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 12069e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman ReleaseSuccessors(SU); 1207e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng SU->isScheduled = true; 12088123419f2be881ca77a897918f28514aa4e91765Dan Gohman AvailableQueue->ScheduledNode(SU); 1209e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 1210e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 12118d1bfad00b1ebff5b140b6e1bd7e26bad697d6e1Dan Gohman/// ListScheduleTopDown - The main loop of list scheduling for top-down 12128d1bfad00b1ebff5b140b6e1bd7e26bad697d6e1Dan Gohman/// schedulers. 1213e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengvoid ScheduleDAGRRList::ListScheduleTopDown() { 121415a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng AvailableQueue->setCurCycle(CurCycle); 1215e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 12169e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman // Release any successors of the special Entry node. 12179e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman ReleaseSuccessors(&EntrySU); 12189e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman 1219e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // All leaves to Available queue. 1220e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng for (unsigned i = 0, e = SUnits.size(); i != e; ++i) { 1221e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // It is available if it has no predecessors. 122280792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman if (SUnits[i].Preds.empty()) { 1223e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng AvailableQueue->push(&SUnits[i]); 1224e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng SUnits[i].isAvailable = true; 1225e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 1226e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 122738036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1228e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // While Available queue is not empty, grab the node with the highest 12298d1bfad00b1ebff5b140b6e1bd7e26bad697d6e1Dan Gohman // priority. If it is not ready put it back. Schedule the node. 12304c8c83022b501759d8559e224c84ae2a9921ba41Dan Gohman Sequence.reserve(SUnits.size()); 1231e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng while (!AvailableQueue->empty()) { 1232a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng SUnit *CurSU = AvailableQueue->pop(); 123338036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 123447d1a214a7013d12140a0c4972d7ba761150dfd4Dan Gohman if (CurSU) 12352902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick ScheduleNodeTopDown(CurSU); 123680792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman ++CurCycle; 123715a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng AvailableQueue->setCurCycle(CurCycle); 1238e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 123938036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1240e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#ifndef NDEBUG 1241a1e6d363e5efa9eb1a2e7ac21a0394c870bef5adDan Gohman VerifySchedule(isBottomUp); 1242e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#endif 1243e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 1244e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1245e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1246e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 1247e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// RegReductionPriorityQueue Implementation 1248e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 1249e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 1250e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// This is a SchedulingPriorityQueue that schedules using Sethi Ullman numbers 1251e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// to reduce register pressure. 125238036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick// 1253e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengnamespace { 1254e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng template<class SF> 1255e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng class RegReductionPriorityQueue; 125638036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 12572da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick struct queue_sort : public std::binary_function<SUnit*, SUnit*, bool> { 12582da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick bool isReady(SUnit* SU, unsigned CurCycle) const { return true; } 12592da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick }; 12602da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1261e28ed16d2507633893269670d289006f3e0b351eEvan Cheng /// bu_ls_rr_sort - Priority function for bottom up register pressure 1262e28ed16d2507633893269670d289006f3e0b351eEvan Cheng // reduction scheduler. 12632da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick struct bu_ls_rr_sort : public queue_sort { 12642da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick enum { 12652da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick IsBottomUp = true, 12662da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HasReadyFilter = false 12672da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick }; 12682da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1269e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng RegReductionPriorityQueue<bu_ls_rr_sort> *SPQ; 1270e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng bu_ls_rr_sort(RegReductionPriorityQueue<bu_ls_rr_sort> *spq) : SPQ(spq) {} 1271e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng bu_ls_rr_sort(const bu_ls_rr_sort &RHS) : SPQ(RHS.SPQ) {} 127238036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1273e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng bool operator()(const SUnit* left, const SUnit* right) const; 1274e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng }; 1275e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1276e28ed16d2507633893269670d289006f3e0b351eEvan Cheng // td_ls_rr_sort - Priority function for top down register pressure reduction 1277e28ed16d2507633893269670d289006f3e0b351eEvan Cheng // scheduler. 12782da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick struct td_ls_rr_sort : public queue_sort { 12792da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick enum { 12802da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick IsBottomUp = false, 12812da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HasReadyFilter = false 12822da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick }; 12832da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 12842da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick RegReductionPriorityQueue<td_ls_rr_sort> *SPQ; 1285e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng td_ls_rr_sort(RegReductionPriorityQueue<td_ls_rr_sort> *spq) : SPQ(spq) {} 1286e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng td_ls_rr_sort(const td_ls_rr_sort &RHS) : SPQ(RHS.SPQ) {} 128738036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1288e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng bool operator()(const SUnit* left, const SUnit* right) const; 1289e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng }; 1290187361b056823df4ff292561fe47468dad956872Bill Wendling 1291e28ed16d2507633893269670d289006f3e0b351eEvan Cheng // src_ls_rr_sort - Priority function for source order scheduler. 12922da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick struct src_ls_rr_sort : public queue_sort { 12932da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick enum { 12942da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick IsBottomUp = true, 12952da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HasReadyFilter = false 12962da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick }; 12972da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1298187361b056823df4ff292561fe47468dad956872Bill Wendling RegReductionPriorityQueue<src_ls_rr_sort> *SPQ; 1299187361b056823df4ff292561fe47468dad956872Bill Wendling src_ls_rr_sort(RegReductionPriorityQueue<src_ls_rr_sort> *spq) 1300187361b056823df4ff292561fe47468dad956872Bill Wendling : SPQ(spq) {} 1301187361b056823df4ff292561fe47468dad956872Bill Wendling src_ls_rr_sort(const src_ls_rr_sort &RHS) 1302187361b056823df4ff292561fe47468dad956872Bill Wendling : SPQ(RHS.SPQ) {} 130338036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1304187361b056823df4ff292561fe47468dad956872Bill Wendling bool operator()(const SUnit* left, const SUnit* right) const; 1305187361b056823df4ff292561fe47468dad956872Bill Wendling }; 130615a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 1307e28ed16d2507633893269670d289006f3e0b351eEvan Cheng // hybrid_ls_rr_sort - Priority function for hybrid scheduler. 13082da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick struct hybrid_ls_rr_sort : public queue_sort { 13092da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick enum { 13102da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick IsBottomUp = true, 13112da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HasReadyFilter = false 13122da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick }; 13132da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 131415a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng RegReductionPriorityQueue<hybrid_ls_rr_sort> *SPQ; 131515a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng hybrid_ls_rr_sort(RegReductionPriorityQueue<hybrid_ls_rr_sort> *spq) 131615a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng : SPQ(spq) {} 131715a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng hybrid_ls_rr_sort(const hybrid_ls_rr_sort &RHS) 131815a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng : SPQ(RHS.SPQ) {} 13194f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 132015a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng bool operator()(const SUnit* left, const SUnit* right) const; 132115a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng }; 132270017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 1323e28ed16d2507633893269670d289006f3e0b351eEvan Cheng // ilp_ls_rr_sort - Priority function for ILP (instruction level parallelism) 1324e28ed16d2507633893269670d289006f3e0b351eEvan Cheng // scheduler. 13252da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick struct ilp_ls_rr_sort : public queue_sort { 13262da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick enum { 13272da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick IsBottomUp = true, 13282da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HasReadyFilter = true 13292da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick }; 13302da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 133170017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng RegReductionPriorityQueue<ilp_ls_rr_sort> *SPQ; 133270017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng ilp_ls_rr_sort(RegReductionPriorityQueue<ilp_ls_rr_sort> *spq) 133370017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng : SPQ(spq) {} 133470017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng ilp_ls_rr_sort(const ilp_ls_rr_sort &RHS) 133570017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng : SPQ(RHS.SPQ) {} 133670017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 13372da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick bool isReady(SUnit *SU, unsigned CurCycle) const; 13382da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 133970017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng bool operator()(const SUnit* left, const SUnit* right) const; 134070017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng }; 1341e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} // end anonymous namespace 1342e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1343117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman/// CalcNodeSethiUllmanNumber - Compute Sethi Ullman number. 1344117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman/// Smaller number is the higher priority. 1345c6be777208f4539af400ac694d9d1dc8b992bc80Evan Chengstatic unsigned 1346117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan GohmanCalcNodeSethiUllmanNumber(const SUnit *SU, std::vector<unsigned> &SUNumbers) { 1347c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng unsigned &SethiUllmanNumber = SUNumbers[SU->NodeNum]; 1348c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng if (SethiUllmanNumber != 0) 1349c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng return SethiUllmanNumber; 1350c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng 1351c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng unsigned Extra = 0; 1352c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 1353c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng I != E; ++I) { 135454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isCtrl()) continue; // ignore chain preds 135554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *PredSU = I->getSUnit(); 1356117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman unsigned PredSethiUllman = CalcNodeSethiUllmanNumber(PredSU, SUNumbers); 1357c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng if (PredSethiUllman > SethiUllmanNumber) { 1358c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng SethiUllmanNumber = PredSethiUllman; 1359c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng Extra = 0; 13608182347d70413174f2e80ea429801e887aee5cc3Evan Cheng } else if (PredSethiUllman == SethiUllmanNumber) 1361c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng ++Extra; 1362c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng } 1363c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng 1364c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng SethiUllmanNumber += Extra; 1365c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng 1366c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng if (SethiUllmanNumber == 0) 1367c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng SethiUllmanNumber = 1; 136838036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1369c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng return SethiUllmanNumber; 1370c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng} 1371c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng 1372e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengnamespace { 1373e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng template<class SF> 13746726b6d75a8b679068a58cb954ba97cf9d1690baNick Lewycky class RegReductionPriorityQueue : public SchedulingPriorityQueue { 13752da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick static SUnit *popFromQueue(std::vector<SUnit*> &Q, SF &Picker) { 13762da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick std::vector<SUnit *>::iterator Best = Q.begin(); 13772da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick for (std::vector<SUnit *>::iterator I = llvm::next(Q.begin()), 13782da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick E = Q.end(); I != E; ++I) 13792da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (Picker(*Best, *I)) 13802da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick Best = I; 13812da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SUnit *V = *Best; 13822da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (Best != prior(Q.end())) 13832da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick std::swap(*Best, Q.back()); 13842da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick Q.pop_back(); 13852da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return V; 13862da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 13872da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 138893d343357944beb701d425fc7ef00dd7b0a32bd7Dan Gohman std::vector<SUnit*> Queue; 138993d343357944beb701d425fc7ef00dd7b0a32bd7Dan Gohman SF Picker; 139015a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng unsigned CurQueueId; 139189ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng bool TracksRegPressure; 1392e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 13936be2ee431f44e3eb4d87bb3779a7e97a766c7a3eDan Gohman protected: 13946be2ee431f44e3eb4d87bb3779a7e97a766c7a3eDan Gohman // SUnits - The SUnits for the current graph. 13956be2ee431f44e3eb4d87bb3779a7e97a766c7a3eDan Gohman std::vector<SUnit> *SUnits; 13964f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 13974f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng MachineFunction &MF; 13986be2ee431f44e3eb4d87bb3779a7e97a766c7a3eDan Gohman const TargetInstrInfo *TII; 13996be2ee431f44e3eb4d87bb3779a7e97a766c7a3eDan Gohman const TargetRegisterInfo *TRI; 14004f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng const TargetLowering *TLI; 14016be2ee431f44e3eb4d87bb3779a7e97a766c7a3eDan Gohman ScheduleDAGRRList *scheduleDAG; 14026be2ee431f44e3eb4d87bb3779a7e97a766c7a3eDan Gohman 1403117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman // SethiUllmanNumbers - The SethiUllman number for each node. 1404117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman std::vector<unsigned> SethiUllmanNumbers; 1405117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman 14064f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng /// RegPressure - Tracking current reg pressure per register class. 14074f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng /// 14084a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng std::vector<unsigned> RegPressure; 14094f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 14104f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng /// RegLimit - Tracking the number of allocatable registers per register 14114f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng /// class. 14124a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng std::vector<unsigned> RegLimit; 14134f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 1414e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng public: 14154f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng RegReductionPriorityQueue(MachineFunction &mf, 141689ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng bool tracksrp, 14174f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng const TargetInstrInfo *tii, 14184f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng const TargetRegisterInfo *tri, 14194f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng const TargetLowering *tli) 14202da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick : SchedulingPriorityQueue(SF::HasReadyFilter), Picker(this), 14212da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CurQueueId(0), TracksRegPressure(tracksrp), 14224f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng MF(mf), TII(tii), TRI(tri), TLI(tli), scheduleDAG(NULL) { 142389ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng if (TracksRegPressure) { 142489ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng unsigned NumRC = TRI->getNumRegClasses(); 142589ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng RegLimit.resize(NumRC); 142689ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng RegPressure.resize(NumRC); 142789ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng std::fill(RegLimit.begin(), RegLimit.end(), 0); 142889ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng std::fill(RegPressure.begin(), RegPressure.end(), 0); 142989ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng for (TargetRegisterInfo::regclass_iterator I = TRI->regclass_begin(), 143089ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng E = TRI->regclass_end(); I != E; ++I) 14313144687df78731ac4ddbc716a24b951678a73f57Evan Cheng RegLimit[(*I)->getID()] = tli->getRegPressureLimit(*I, MF); 143289ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng } 14334f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 143438036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 14352da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick bool isBottomUp() const { return SF::IsBottomUp; } 14362da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 14376be2ee431f44e3eb4d87bb3779a7e97a766c7a3eDan Gohman void initNodes(std::vector<SUnit> &sunits) { 14386be2ee431f44e3eb4d87bb3779a7e97a766c7a3eDan Gohman SUnits = &sunits; 1439e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // Add pseudo dependency edges for two-address nodes. 1440c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng AddPseudoTwoAddrDeps(); 1441002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Reroute edges to nodes with multiple uses. 1442002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman PrescheduleNodesWithMultipleUses(); 1443e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // Calculate node priorities. 1444c8edc64188399437f5476d7fa45f714a92f2cb93Evan Cheng CalculateSethiUllmanNumbers(); 1445e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 1446e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1447a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng void addNode(const SUnit *SU) { 1448c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng unsigned SUSize = SethiUllmanNumbers.size(); 1449c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng if (SUnits->size() > SUSize) 1450c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng SethiUllmanNumbers.resize(SUSize*2, 0); 1451117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman CalcNodeSethiUllmanNumber(SU, SethiUllmanNumbers); 1452a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 1453a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1454a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng void updateNode(const SUnit *SU) { 1455a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng SethiUllmanNumbers[SU->NodeNum] = 0; 1456117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman CalcNodeSethiUllmanNumber(SU, SethiUllmanNumbers); 1457a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 1458a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1459e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng void releaseState() { 1460117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman SUnits = 0; 1461e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng SethiUllmanNumbers.clear(); 14624f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng std::fill(RegPressure.begin(), RegPressure.end(), 0); 1463e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 1464e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1465c8edc64188399437f5476d7fa45f714a92f2cb93Evan Cheng unsigned getNodePriority(const SUnit *SU) const { 1466c62d4bb6952a1459f10aa93579e1b881d42a33eaEvan Cheng assert(SU->NodeNum < SethiUllmanNumbers.size()); 1467550f5afb68ce8f034991863cac65bef22a6554daDan Gohman unsigned Opc = SU->getNode() ? SU->getNode()->getOpcode() : 0; 146825fd4037a7f698877eedb69bf32d8591344e86eeDan Gohman if (Opc == ISD::TokenFactor || Opc == ISD::CopyToReg) 1469c62d4bb6952a1459f10aa93579e1b881d42a33eaEvan Cheng // CopyToReg should be close to its uses to facilitate coalescing and 1470c62d4bb6952a1459f10aa93579e1b881d42a33eaEvan Cheng // avoid spilling. 1471c62d4bb6952a1459f10aa93579e1b881d42a33eaEvan Cheng return 0; 1472518bb53485df640d7b7e3f6b0544099020c42aa7Chris Lattner if (Opc == TargetOpcode::EXTRACT_SUBREG || 1473518bb53485df640d7b7e3f6b0544099020c42aa7Chris Lattner Opc == TargetOpcode::SUBREG_TO_REG || 1474518bb53485df640d7b7e3f6b0544099020c42aa7Chris Lattner Opc == TargetOpcode::INSERT_SUBREG) 14758af808a3d64dfdcc5d126e6ad762f57b1483671cDan Gohman // EXTRACT_SUBREG, INSERT_SUBREG, and SUBREG_TO_REG nodes should be 14768af808a3d64dfdcc5d126e6ad762f57b1483671cDan Gohman // close to their uses to facilitate coalescing. 147732dfbeada7292167bb488f36a71a5a6a519ddaffEvan Cheng return 0; 1478c8db34cb07fea88c4b8f3e0f095fd8aed568b28eDan Gohman if (SU->NumSuccs == 0 && SU->NumPreds != 0) 1479c8db34cb07fea88c4b8f3e0f095fd8aed568b28eDan Gohman // If SU does not have a register use, i.e. it doesn't produce a value 1480c8db34cb07fea88c4b8f3e0f095fd8aed568b28eDan Gohman // that would be consumed (e.g. store), then it terminates a chain of 1481c8db34cb07fea88c4b8f3e0f095fd8aed568b28eDan Gohman // computation. Give it a large SethiUllman number so it will be 1482c8db34cb07fea88c4b8f3e0f095fd8aed568b28eDan Gohman // scheduled right before its predecessors that it doesn't lengthen 1483c8db34cb07fea88c4b8f3e0f095fd8aed568b28eDan Gohman // their live ranges. 1484c62d4bb6952a1459f10aa93579e1b881d42a33eaEvan Cheng return 0xffff; 1485c8db34cb07fea88c4b8f3e0f095fd8aed568b28eDan Gohman if (SU->NumPreds == 0 && SU->NumSuccs != 0) 1486c8db34cb07fea88c4b8f3e0f095fd8aed568b28eDan Gohman // If SU does not have a register def, schedule it close to its uses 1487c8db34cb07fea88c4b8f3e0f095fd8aed568b28eDan Gohman // because it does not lengthen any live ranges. 1488c62d4bb6952a1459f10aa93579e1b881d42a33eaEvan Cheng return 0; 148925fd4037a7f698877eedb69bf32d8591344e86eeDan Gohman return SethiUllmanNumbers[SU->NodeNum]; 1490e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 1491247fb4ebd363323b6642ce2bb08e53db705ca094Bill Wendling 1492247fb4ebd363323b6642ce2bb08e53db705ca094Bill Wendling unsigned getNodeOrdering(const SUnit *SU) const { 1493247fb4ebd363323b6642ce2bb08e53db705ca094Bill Wendling return scheduleDAG->DAG->GetOrdering(SU->getNode()); 1494247fb4ebd363323b6642ce2bb08e53db705ca094Bill Wendling } 149515a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 1496117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman bool empty() const { return Queue.empty(); } 149738036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 14982da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick bool isReady(SUnit *U) const { 14992da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return Picker.HasReadyFilter && Picker.isReady(U, getCurCycle()); 15002da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 15012da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1502117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman void push(SUnit *U) { 1503117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman assert(!U->NodeQueueId && "Node in the queue already"); 150415a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng U->NodeQueueId = ++CurQueueId; 150593d343357944beb701d425fc7ef00dd7b0a32bd7Dan Gohman Queue.push_back(U); 1506e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 1507e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1508117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman SUnit *pop() { 15092da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (Queue.empty()) return NULL; 15102da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 15112da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SUnit *V = popFromQueue(Queue, Picker); 1512117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman V->NodeQueueId = 0; 1513117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman return V; 1514a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 1515a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1516117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman void remove(SUnit *SU) { 1517117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman assert(!Queue.empty() && "Queue is empty!"); 1518117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman assert(SU->NodeQueueId != 0 && "Not in queue!"); 151993d343357944beb701d425fc7ef00dd7b0a32bd7Dan Gohman std::vector<SUnit *>::iterator I = std::find(Queue.begin(), Queue.end(), 152093d343357944beb701d425fc7ef00dd7b0a32bd7Dan Gohman SU); 152193d343357944beb701d425fc7ef00dd7b0a32bd7Dan Gohman if (I != prior(Queue.end())) 152293d343357944beb701d425fc7ef00dd7b0a32bd7Dan Gohman std::swap(*I, Queue.back()); 152393d343357944beb701d425fc7ef00dd7b0a32bd7Dan Gohman Queue.pop_back(); 1524117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman SU->NodeQueueId = 0; 1525e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 1526e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1527e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng bool HighRegPressure(const SUnit *SU) const { 15284f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng if (!TLI) 15294a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng return false; 15304f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 15314f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng for (SUnit::const_pred_iterator I = SU->Preds.begin(),E = SU->Preds.end(); 15324f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng I != E; ++I) { 15334f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng if (I->isCtrl()) 15344f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng continue; 15354f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng SUnit *PredSU = I->getSUnit(); 15364a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng const SDNode *PN = PredSU->getNode(); 15374a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng if (!PN->isMachineOpcode()) { 15383144687df78731ac4ddbc716a24b951678a73f57Evan Cheng if (PN->getOpcode() == ISD::CopyFromReg) { 15393144687df78731ac4ddbc716a24b951678a73f57Evan Cheng EVT VT = PN->getValueType(0); 15404a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 15414a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng unsigned Cost = TLI->getRepRegClassCostFor(VT); 1542e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng if ((RegPressure[RCId] + Cost) >= RegLimit[RCId]) 1543e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng return true; 15443144687df78731ac4ddbc716a24b951678a73f57Evan Cheng } 15453144687df78731ac4ddbc716a24b951678a73f57Evan Cheng continue; 15463144687df78731ac4ddbc716a24b951678a73f57Evan Cheng } 15473144687df78731ac4ddbc716a24b951678a73f57Evan Cheng unsigned POpc = PN->getMachineOpcode(); 15483144687df78731ac4ddbc716a24b951678a73f57Evan Cheng if (POpc == TargetOpcode::IMPLICIT_DEF) 15493144687df78731ac4ddbc716a24b951678a73f57Evan Cheng continue; 15503144687df78731ac4ddbc716a24b951678a73f57Evan Cheng if (POpc == TargetOpcode::EXTRACT_SUBREG) { 15513144687df78731ac4ddbc716a24b951678a73f57Evan Cheng EVT VT = PN->getOperand(0).getValueType(); 15523144687df78731ac4ddbc716a24b951678a73f57Evan Cheng unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 15533144687df78731ac4ddbc716a24b951678a73f57Evan Cheng unsigned Cost = TLI->getRepRegClassCostFor(VT); 15543144687df78731ac4ddbc716a24b951678a73f57Evan Cheng // Check if this increases register pressure of the specific register 15553144687df78731ac4ddbc716a24b951678a73f57Evan Cheng // class to the point where it would cause spills. 1556e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng if ((RegPressure[RCId] + Cost) >= RegLimit[RCId]) 1557e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng return true; 155838036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick continue; 15593144687df78731ac4ddbc716a24b951678a73f57Evan Cheng } else if (POpc == TargetOpcode::INSERT_SUBREG || 15603144687df78731ac4ddbc716a24b951678a73f57Evan Cheng POpc == TargetOpcode::SUBREG_TO_REG) { 15613144687df78731ac4ddbc716a24b951678a73f57Evan Cheng EVT VT = PN->getValueType(0); 15623144687df78731ac4ddbc716a24b951678a73f57Evan Cheng unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 15633144687df78731ac4ddbc716a24b951678a73f57Evan Cheng unsigned Cost = TLI->getRepRegClassCostFor(VT); 15643144687df78731ac4ddbc716a24b951678a73f57Evan Cheng // Check if this increases register pressure of the specific register 15653144687df78731ac4ddbc716a24b951678a73f57Evan Cheng // class to the point where it would cause spills. 1566e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng if ((RegPressure[RCId] + Cost) >= RegLimit[RCId]) 1567e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng return true; 15684f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng continue; 15694a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng } 15704a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng unsigned NumDefs = TII->get(PN->getMachineOpcode()).getNumDefs(); 15714f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng for (unsigned i = 0; i != NumDefs; ++i) { 15724a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng EVT VT = PN->getValueType(i); 15734f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 1574e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng if (RegPressure[RCId] >= RegLimit[RCId]) 1575e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng return true; // Reg pressure already high. 15764f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng unsigned Cost = TLI->getRepRegClassCostFor(VT); 1577e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng if (!PN->hasAnyUseOfValue(i)) 1578e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng continue; 15794f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng // Check if this increases register pressure of the specific register 15804f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng // class to the point where it would cause spills. 1581e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng if ((RegPressure[RCId] + Cost) >= RegLimit[RCId]) 1582e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng return true; 15834f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 15844f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 15854f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 1586e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng return false; 15874f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 15884f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 158989ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng void ScheduledNode(SUnit *SU) { 159089ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng if (!TracksRegPressure) 159189ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng return; 159289ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng 15934f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng const SDNode *N = SU->getNode(); 15943144687df78731ac4ddbc716a24b951678a73f57Evan Cheng if (!N->isMachineOpcode()) { 15953144687df78731ac4ddbc716a24b951678a73f57Evan Cheng if (N->getOpcode() != ISD::CopyToReg) 15963144687df78731ac4ddbc716a24b951678a73f57Evan Cheng return; 15973144687df78731ac4ddbc716a24b951678a73f57Evan Cheng } else { 15983144687df78731ac4ddbc716a24b951678a73f57Evan Cheng unsigned Opc = N->getMachineOpcode(); 15993144687df78731ac4ddbc716a24b951678a73f57Evan Cheng if (Opc == TargetOpcode::EXTRACT_SUBREG || 16003144687df78731ac4ddbc716a24b951678a73f57Evan Cheng Opc == TargetOpcode::INSERT_SUBREG || 16013144687df78731ac4ddbc716a24b951678a73f57Evan Cheng Opc == TargetOpcode::SUBREG_TO_REG || 16023144687df78731ac4ddbc716a24b951678a73f57Evan Cheng Opc == TargetOpcode::REG_SEQUENCE || 16033144687df78731ac4ddbc716a24b951678a73f57Evan Cheng Opc == TargetOpcode::IMPLICIT_DEF) 16043144687df78731ac4ddbc716a24b951678a73f57Evan Cheng return; 16053144687df78731ac4ddbc716a24b951678a73f57Evan Cheng } 16064f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 16074f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 16084f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng I != E; ++I) { 16094f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng if (I->isCtrl()) 16104f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng continue; 16114f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng SUnit *PredSU = I->getSUnit(); 16124a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng if (PredSU->NumSuccsLeft != PredSU->NumSuccs) 16134f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng continue; 16144f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng const SDNode *PN = PredSU->getNode(); 16154a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng if (!PN->isMachineOpcode()) { 16163144687df78731ac4ddbc716a24b951678a73f57Evan Cheng if (PN->getOpcode() == ISD::CopyFromReg) { 16173144687df78731ac4ddbc716a24b951678a73f57Evan Cheng EVT VT = PN->getValueType(0); 16184a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 16194a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); 16204a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng } 16214a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng continue; 16224a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng } 16234a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng unsigned POpc = PN->getMachineOpcode(); 16244a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng if (POpc == TargetOpcode::IMPLICIT_DEF) 16254f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng continue; 16263144687df78731ac4ddbc716a24b951678a73f57Evan Cheng if (POpc == TargetOpcode::EXTRACT_SUBREG) { 16273144687df78731ac4ddbc716a24b951678a73f57Evan Cheng EVT VT = PN->getOperand(0).getValueType(); 16283144687df78731ac4ddbc716a24b951678a73f57Evan Cheng unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 16293144687df78731ac4ddbc716a24b951678a73f57Evan Cheng RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); 163038036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick continue; 16313144687df78731ac4ddbc716a24b951678a73f57Evan Cheng } else if (POpc == TargetOpcode::INSERT_SUBREG || 16323144687df78731ac4ddbc716a24b951678a73f57Evan Cheng POpc == TargetOpcode::SUBREG_TO_REG) { 16333144687df78731ac4ddbc716a24b951678a73f57Evan Cheng EVT VT = PN->getValueType(0); 16343144687df78731ac4ddbc716a24b951678a73f57Evan Cheng unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 16353144687df78731ac4ddbc716a24b951678a73f57Evan Cheng RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); 16363144687df78731ac4ddbc716a24b951678a73f57Evan Cheng continue; 16373144687df78731ac4ddbc716a24b951678a73f57Evan Cheng } 16384f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng unsigned NumDefs = TII->get(PN->getMachineOpcode()).getNumDefs(); 16394f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng for (unsigned i = 0; i != NumDefs; ++i) { 16404f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng EVT VT = PN->getValueType(i); 16414f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng if (!PN->hasAnyUseOfValue(i)) 16424f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng continue; 16434f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 16444f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); 16454f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 16464f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 16474f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 1648e28ed16d2507633893269670d289006f3e0b351eEvan Cheng // Check for isMachineOpcode() as PrescheduleNodesWithMultipleUses() 1649e28ed16d2507633893269670d289006f3e0b351eEvan Cheng // may transfer data dependencies to CopyToReg. 1650e28ed16d2507633893269670d289006f3e0b351eEvan Cheng if (SU->NumSuccs && N->isMachineOpcode()) { 16513144687df78731ac4ddbc716a24b951678a73f57Evan Cheng unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs(); 16523144687df78731ac4ddbc716a24b951678a73f57Evan Cheng for (unsigned i = 0; i != NumDefs; ++i) { 16533144687df78731ac4ddbc716a24b951678a73f57Evan Cheng EVT VT = N->getValueType(i); 16543144687df78731ac4ddbc716a24b951678a73f57Evan Cheng if (!N->hasAnyUseOfValue(i)) 16553144687df78731ac4ddbc716a24b951678a73f57Evan Cheng continue; 16563144687df78731ac4ddbc716a24b951678a73f57Evan Cheng unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 16573144687df78731ac4ddbc716a24b951678a73f57Evan Cheng if (RegPressure[RCId] < TLI->getRepRegClassCostFor(VT)) 16583144687df78731ac4ddbc716a24b951678a73f57Evan Cheng // Register pressure tracking is imprecise. This can happen. 16593144687df78731ac4ddbc716a24b951678a73f57Evan Cheng RegPressure[RCId] = 0; 16603144687df78731ac4ddbc716a24b951678a73f57Evan Cheng else 16613144687df78731ac4ddbc716a24b951678a73f57Evan Cheng RegPressure[RCId] -= TLI->getRepRegClassCostFor(VT); 16623144687df78731ac4ddbc716a24b951678a73f57Evan Cheng } 16634f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 166489ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng 166589ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng dumpRegPressure(); 16664f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 16674f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 166889ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng void UnscheduledNode(SUnit *SU) { 166989ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng if (!TracksRegPressure) 167089ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng return; 167189ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng 16724f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng const SDNode *N = SU->getNode(); 16733144687df78731ac4ddbc716a24b951678a73f57Evan Cheng if (!N->isMachineOpcode()) { 16743144687df78731ac4ddbc716a24b951678a73f57Evan Cheng if (N->getOpcode() != ISD::CopyToReg) 16753144687df78731ac4ddbc716a24b951678a73f57Evan Cheng return; 167670017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng } else { 167770017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng unsigned Opc = N->getMachineOpcode(); 167870017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng if (Opc == TargetOpcode::EXTRACT_SUBREG || 167970017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng Opc == TargetOpcode::INSERT_SUBREG || 168070017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng Opc == TargetOpcode::SUBREG_TO_REG || 168170017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng Opc == TargetOpcode::REG_SEQUENCE || 168270017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng Opc == TargetOpcode::IMPLICIT_DEF) 168370017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng return; 16843144687df78731ac4ddbc716a24b951678a73f57Evan Cheng } 16854f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 16864f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 16874f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng I != E; ++I) { 16884f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng if (I->isCtrl()) 16894f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng continue; 16904f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng SUnit *PredSU = I->getSUnit(); 16914a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng if (PredSU->NumSuccsLeft != PredSU->NumSuccs) 16924f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng continue; 16934f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng const SDNode *PN = PredSU->getNode(); 16944a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng if (!PN->isMachineOpcode()) { 16953144687df78731ac4ddbc716a24b951678a73f57Evan Cheng if (PN->getOpcode() == ISD::CopyFromReg) { 16963144687df78731ac4ddbc716a24b951678a73f57Evan Cheng EVT VT = PN->getValueType(0); 16974a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 16984a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); 16994a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng } 17004a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng continue; 17014a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng } 17024a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng unsigned POpc = PN->getMachineOpcode(); 17034a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng if (POpc == TargetOpcode::IMPLICIT_DEF) 17044f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng continue; 17053144687df78731ac4ddbc716a24b951678a73f57Evan Cheng if (POpc == TargetOpcode::EXTRACT_SUBREG) { 17063144687df78731ac4ddbc716a24b951678a73f57Evan Cheng EVT VT = PN->getOperand(0).getValueType(); 17073144687df78731ac4ddbc716a24b951678a73f57Evan Cheng unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 17083144687df78731ac4ddbc716a24b951678a73f57Evan Cheng RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); 170938036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick continue; 17103144687df78731ac4ddbc716a24b951678a73f57Evan Cheng } else if (POpc == TargetOpcode::INSERT_SUBREG || 17113144687df78731ac4ddbc716a24b951678a73f57Evan Cheng POpc == TargetOpcode::SUBREG_TO_REG) { 17123144687df78731ac4ddbc716a24b951678a73f57Evan Cheng EVT VT = PN->getValueType(0); 17133144687df78731ac4ddbc716a24b951678a73f57Evan Cheng unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 17143144687df78731ac4ddbc716a24b951678a73f57Evan Cheng RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); 17153144687df78731ac4ddbc716a24b951678a73f57Evan Cheng continue; 17163144687df78731ac4ddbc716a24b951678a73f57Evan Cheng } 17174f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng unsigned NumDefs = TII->get(PN->getMachineOpcode()).getNumDefs(); 17184f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng for (unsigned i = 0; i != NumDefs; ++i) { 17194f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng EVT VT = PN->getValueType(i); 17204f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng if (!PN->hasAnyUseOfValue(i)) 17214f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng continue; 17224f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 17234a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng if (RegPressure[RCId] < TLI->getRepRegClassCostFor(VT)) 17244f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng // Register pressure tracking is imprecise. This can happen. 17254f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng RegPressure[RCId] = 0; 17264a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng else 17274a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng RegPressure[RCId] -= TLI->getRepRegClassCostFor(VT); 17284f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 17294f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 17304f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 1731e28ed16d2507633893269670d289006f3e0b351eEvan Cheng // Check for isMachineOpcode() as PrescheduleNodesWithMultipleUses() 1732e28ed16d2507633893269670d289006f3e0b351eEvan Cheng // may transfer data dependencies to CopyToReg. 1733e28ed16d2507633893269670d289006f3e0b351eEvan Cheng if (SU->NumSuccs && N->isMachineOpcode()) { 17343144687df78731ac4ddbc716a24b951678a73f57Evan Cheng unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs(); 17353144687df78731ac4ddbc716a24b951678a73f57Evan Cheng for (unsigned i = NumDefs, e = N->getNumValues(); i != e; ++i) { 17363144687df78731ac4ddbc716a24b951678a73f57Evan Cheng EVT VT = N->getValueType(i); 1737f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner if (VT == MVT::Glue || VT == MVT::Other) 17383144687df78731ac4ddbc716a24b951678a73f57Evan Cheng continue; 17393144687df78731ac4ddbc716a24b951678a73f57Evan Cheng if (!N->hasAnyUseOfValue(i)) 17403144687df78731ac4ddbc716a24b951678a73f57Evan Cheng continue; 17413144687df78731ac4ddbc716a24b951678a73f57Evan Cheng unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 17423144687df78731ac4ddbc716a24b951678a73f57Evan Cheng RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); 17433144687df78731ac4ddbc716a24b951678a73f57Evan Cheng } 17444f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 17454f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 17464f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng dumpRegPressure(); 17474f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 17484f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 174938036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick void setScheduleDAG(ScheduleDAGRRList *scheduleDag) { 175038036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick scheduleDAG = scheduleDag; 1751e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 1752e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 17534f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng void dumpRegPressure() const { 17544f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng for (TargetRegisterInfo::regclass_iterator I = TRI->regclass_begin(), 17554f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng E = TRI->regclass_end(); I != E; ++I) { 17564f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng const TargetRegisterClass *RC = *I; 17574f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng unsigned Id = RC->getID(); 17584f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng unsigned RP = RegPressure[Id]; 17594f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng if (!RP) continue; 17604f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng DEBUG(dbgs() << RC->getName() << ": " << RP << " / " << RegLimit[Id] 17614f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng << '\n'); 17624f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 17634f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 17644f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 17652da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void dump(ScheduleDAG *DAG) const { 17662da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Emulate pop() without clobbering NodeQueueIds. 17672da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick std::vector<SUnit*> DumpQueue = Queue; 17682da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SF DumpPicker = Picker; 17692da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick while (!DumpQueue.empty()) { 17702da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SUnit *SU = popFromQueue(DumpQueue, DumpPicker); 17712da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (isBottomUp()) 17722da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick dbgs() << "Height " << SU->getHeight() << ": "; 17732da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick else 17742da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick dbgs() << "Depth " << SU->getDepth() << ": "; 17752da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SU->dump(DAG); 17762da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 17772da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 17782da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1779117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman protected: 1780117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman bool canClobber(const SUnit *SU, const SUnit *Op); 1781117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman void AddPseudoTwoAddrDeps(); 1782002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman void PrescheduleNodesWithMultipleUses(); 1783c8edc64188399437f5476d7fa45f714a92f2cb93Evan Cheng void CalculateSethiUllmanNumbers(); 1784e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng }; 1785117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman 1786117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman typedef RegReductionPriorityQueue<bu_ls_rr_sort> 1787117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman BURegReductionPriorityQueue; 1788117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman 1789117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman typedef RegReductionPriorityQueue<td_ls_rr_sort> 1790117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman TDRegReductionPriorityQueue; 1791187361b056823df4ff292561fe47468dad956872Bill Wendling 1792187361b056823df4ff292561fe47468dad956872Bill Wendling typedef RegReductionPriorityQueue<src_ls_rr_sort> 1793187361b056823df4ff292561fe47468dad956872Bill Wendling SrcRegReductionPriorityQueue; 179415a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 179515a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng typedef RegReductionPriorityQueue<hybrid_ls_rr_sort> 179615a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng HybridBURRPriorityQueue; 179770017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 179870017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng typedef RegReductionPriorityQueue<ilp_ls_rr_sort> 179970017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng ILPBURRPriorityQueue; 1800e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 1801e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1802c6deb3d44707de57e82e16642ab845bc8b9e9e01Evan Cheng/// closestSucc - Returns the scheduled cycle of the successor which is 1803b398fca15bcca895526699f20336fa6c6a624013Dan Gohman/// closest to the current cycle. 180461230d18d21a5dca1378e994f43934e4b314e595Evan Chengstatic unsigned closestSucc(const SUnit *SU) { 18053f23744df4809eba94284e601e81489212c974d4Dan Gohman unsigned MaxHeight = 0; 180661230d18d21a5dca1378e994f43934e4b314e595Evan Cheng for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 1807c6deb3d44707de57e82e16642ab845bc8b9e9e01Evan Cheng I != E; ++I) { 1808f0e366a929a1acb4bc14df5ef831cce74607a967Evan Cheng if (I->isCtrl()) continue; // ignore chain succs 18093f23744df4809eba94284e601e81489212c974d4Dan Gohman unsigned Height = I->getSUnit()->getHeight(); 1810c6deb3d44707de57e82e16642ab845bc8b9e9e01Evan Cheng // If there are bunch of CopyToRegs stacked up, they should be considered 1811c6deb3d44707de57e82e16642ab845bc8b9e9e01Evan Cheng // to be at the same position. 181254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->getSUnit()->getNode() && 181354e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman I->getSUnit()->getNode()->getOpcode() == ISD::CopyToReg) 18143f23744df4809eba94284e601e81489212c974d4Dan Gohman Height = closestSucc(I->getSUnit())+1; 18153f23744df4809eba94284e601e81489212c974d4Dan Gohman if (Height > MaxHeight) 18163f23744df4809eba94284e601e81489212c974d4Dan Gohman MaxHeight = Height; 1817c6deb3d44707de57e82e16642ab845bc8b9e9e01Evan Cheng } 18183f23744df4809eba94284e601e81489212c974d4Dan Gohman return MaxHeight; 181961230d18d21a5dca1378e994f43934e4b314e595Evan Cheng} 182061230d18d21a5dca1378e994f43934e4b314e595Evan Cheng 1821d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng/// calcMaxScratches - Returns an cost estimate of the worse case requirement 18228182347d70413174f2e80ea429801e887aee5cc3Evan Cheng/// for scratch registers, i.e. number of data dependencies. 1823d6c0758944b31bb5316b36cad37f4610a77f784dEvan Chengstatic unsigned calcMaxScratches(const SUnit *SU) { 1824d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng unsigned Scratches = 0; 1825d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 1826f2b14715d11e52adbb17a5860d1ce42f82f85a0cEvan Cheng I != E; ++I) { 182754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isCtrl()) continue; // ignore chain preds 1828f2b14715d11e52adbb17a5860d1ce42f82f85a0cEvan Cheng Scratches++; 1829f2b14715d11e52adbb17a5860d1ce42f82f85a0cEvan Cheng } 1830d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng return Scratches; 1831d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng} 1832d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng 1833089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng/// hasOnlyLiveOutUse - Return true if SU has a single value successor that is a 1834089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng/// CopyToReg to a virtual register. This SU def is probably a liveout and 1835089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng/// it has no other use. It should be scheduled closer to the terminator. 1836089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Chengstatic bool hasOnlyLiveOutUses(const SUnit *SU) { 1837089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng bool RetVal = false; 1838089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 1839089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng I != E; ++I) { 1840089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng if (I->isCtrl()) continue; 1841089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng const SUnit *SuccSU = I->getSUnit(); 1842089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng if (SuccSU->getNode() && SuccSU->getNode()->getOpcode() == ISD::CopyToReg) { 1843089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng unsigned Reg = 1844089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng cast<RegisterSDNode>(SuccSU->getNode()->getOperand(1))->getReg(); 1845089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng if (TargetRegisterInfo::isVirtualRegister(Reg)) { 1846089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng RetVal = true; 1847089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng continue; 1848089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng } 1849089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng } 1850089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng return false; 1851089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng } 1852089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng return RetVal; 1853089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng} 1854089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng 1855089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng/// UnitsSharePred - Return true if the two scheduling units share a common 1856089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng/// data predecessor. 1857089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Chengstatic bool UnitsSharePred(const SUnit *left, const SUnit *right) { 1858089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng SmallSet<const SUnit*, 4> Preds; 1859089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng for (SUnit::const_pred_iterator I = left->Preds.begin(),E = left->Preds.end(); 1860089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng I != E; ++I) { 1861089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng if (I->isCtrl()) continue; // ignore chain preds 1862089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng Preds.insert(I->getSUnit()); 1863089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng } 1864089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng for (SUnit::const_pred_iterator I = right->Preds.begin(),E = right->Preds.end(); 1865089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng I != E; ++I) { 1866089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng if (I->isCtrl()) continue; // ignore chain preds 1867089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng if (Preds.count(I->getSUnit())) 1868089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng return true; 1869089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng } 1870089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng return false; 1871089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng} 1872089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng 1873187361b056823df4ff292561fe47468dad956872Bill Wendlingtemplate <typename RRSort> 1874187361b056823df4ff292561fe47468dad956872Bill Wendlingstatic bool BURRSort(const SUnit *left, const SUnit *right, 1875187361b056823df4ff292561fe47468dad956872Bill Wendling const RegReductionPriorityQueue<RRSort> *SPQ) { 1876c8edc64188399437f5476d7fa45f714a92f2cb93Evan Cheng unsigned LPriority = SPQ->getNodePriority(left); 1877c8edc64188399437f5476d7fa45f714a92f2cb93Evan Cheng unsigned RPriority = SPQ->getNodePriority(right); 187884d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng if (LPriority != RPriority) 187984d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng return LPriority > RPriority; 188084d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng 188184d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // Try schedule def + use closer when Sethi-Ullman numbers are the same. 188284d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // e.g. 188384d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t1 = op t2, c1 188484d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t3 = op t4, c2 188584d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // 188684d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // and the following instructions are both ready. 188784d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t2 = op c3 188884d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t4 = op c4 188984d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // 189084d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // Then schedule t2 = op first. 189184d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // i.e. 189284d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t4 = op c4 189384d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t2 = op c3 189484d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t1 = op t2, c1 189584d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t3 = op t4, c2 189684d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // 189784d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // This creates more short live intervals. 189884d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng unsigned LDist = closestSucc(left); 189984d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng unsigned RDist = closestSucc(right); 190084d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng if (LDist != RDist) 190184d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng return LDist < RDist; 190284d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng 19038182347d70413174f2e80ea429801e887aee5cc3Evan Cheng // How many registers becomes live when the node is scheduled. 190484d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng unsigned LScratch = calcMaxScratches(left); 190584d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng unsigned RScratch = calcMaxScratches(right); 190684d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng if (LScratch != RScratch) 190784d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng return LScratch > RScratch; 190884d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng 19092da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Note: with a bottom-up ready filter, the height check may be redundant. 19103f23744df4809eba94284e601e81489212c974d4Dan Gohman if (left->getHeight() != right->getHeight()) 19113f23744df4809eba94284e601e81489212c974d4Dan Gohman return left->getHeight() > right->getHeight(); 191238036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 19133f23744df4809eba94284e601e81489212c974d4Dan Gohman if (left->getDepth() != right->getDepth()) 19143f23744df4809eba94284e601e81489212c974d4Dan Gohman return left->getDepth() < right->getDepth(); 191584d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng 191638036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick assert(left->NodeQueueId && right->NodeQueueId && 1917c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng "NodeQueueId cannot be zero"); 1918c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng return (left->NodeQueueId > right->NodeQueueId); 1919c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng} 1920c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng 1921187361b056823df4ff292561fe47468dad956872Bill Wendling// Bottom up 1922187361b056823df4ff292561fe47468dad956872Bill Wendlingbool bu_ls_rr_sort::operator()(const SUnit *left, const SUnit *right) const { 1923187361b056823df4ff292561fe47468dad956872Bill Wendling return BURRSort(left, right, SPQ); 1924187361b056823df4ff292561fe47468dad956872Bill Wendling} 1925187361b056823df4ff292561fe47468dad956872Bill Wendling 1926187361b056823df4ff292561fe47468dad956872Bill Wendling// Source order, otherwise bottom up. 192715a16def6e70c8f7df1023da80ceb89887203b40Evan Chengbool src_ls_rr_sort::operator()(const SUnit *left, const SUnit *right) const { 1928187361b056823df4ff292561fe47468dad956872Bill Wendling unsigned LOrder = SPQ->getNodeOrdering(left); 1929187361b056823df4ff292561fe47468dad956872Bill Wendling unsigned ROrder = SPQ->getNodeOrdering(right); 1930187361b056823df4ff292561fe47468dad956872Bill Wendling 1931187361b056823df4ff292561fe47468dad956872Bill Wendling // Prefer an ordering where the lower the non-zero order number, the higher 1932187361b056823df4ff292561fe47468dad956872Bill Wendling // the preference. 1933187361b056823df4ff292561fe47468dad956872Bill Wendling if ((LOrder || ROrder) && LOrder != ROrder) 1934187361b056823df4ff292561fe47468dad956872Bill Wendling return LOrder != 0 && (LOrder < ROrder || ROrder == 0); 1935187361b056823df4ff292561fe47468dad956872Bill Wendling 1936187361b056823df4ff292561fe47468dad956872Bill Wendling return BURRSort(left, right, SPQ); 1937187361b056823df4ff292561fe47468dad956872Bill Wendling} 1938187361b056823df4ff292561fe47468dad956872Bill Wendling 193915a16def6e70c8f7df1023da80ceb89887203b40Evan Chengbool hybrid_ls_rr_sort::operator()(const SUnit *left, const SUnit *right) const{ 19408239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng if (left->isCall || right->isCall) 19418239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng // No way to compute latency of calls. 19428239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng return BURRSort(left, right, SPQ); 19438239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng 1944e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng bool LHigh = SPQ->HighRegPressure(left); 1945e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng bool RHigh = SPQ->HighRegPressure(right); 194670017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng // Avoid causing spills. If register pressure is high, schedule for 194770017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng // register pressure reduction. 19484a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng if (LHigh && !RHigh) 19494a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng return true; 19504a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng else if (!LHigh && RHigh) 19514a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng return false; 1952e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng else if (!LHigh && !RHigh) { 1953089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng // If the two nodes share an operand and one of them has a single 1954089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng // use that is a live out copy, favor the one that is live out. Otherwise 1955089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng // it will be difficult to eliminate the copy if the instruction is a 1956089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng // loop induction variable update. e.g. 1957089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng // BB: 1958089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng // sub r1, r3, #1 1959089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng // str r0, [r2, r3] 1960089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng // mov r3, r1 1961089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng // cmp 1962089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng // bne BB 1963089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng bool SharePred = UnitsSharePred(left, right); 1964089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng // FIXME: Only adjust if BB is a loop back edge. 1965089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng // FIXME: What's the cost of a copy? 1966089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng int LBonus = (SharePred && hasOnlyLiveOutUses(left)) ? 1 : 0; 1967089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng int RBonus = (SharePred && hasOnlyLiveOutUses(right)) ? 1 : 0; 1968089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng int LHeight = (int)left->getHeight() - LBonus; 1969089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng int RHeight = (int)right->getHeight() - RBonus; 1970089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng 19714a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng // Low register pressure situation, schedule for latency if possible. 19724a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng bool LStall = left->SchedulingPref == Sched::Latency && 1973089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng (int)SPQ->getCurCycle() < LHeight; 19744a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng bool RStall = right->SchedulingPref == Sched::Latency && 1975089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng (int)SPQ->getCurCycle() < RHeight; 19764a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng // If scheduling one of the node will cause a pipeline stall, delay it. 19774a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng // If scheduling either one of the node will cause a pipeline stall, sort 19784a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng // them according to their height. 19794a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng if (LStall) { 19804a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng if (!RStall) 19814a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng return true; 1982089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng if (LHeight != RHeight) 1983089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng return LHeight > RHeight; 19844a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng } else if (RStall) 198515a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng return false; 1986046fa3f90a31ebfa10df89ae348f478d492709a9Evan Cheng 1987089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng // If either node is scheduling for latency, sort them by height 1988089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng // and latency. 19894a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng if (left->SchedulingPref == Sched::Latency || 19904a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng right->SchedulingPref == Sched::Latency) { 1991089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng if (LHeight != RHeight) 1992089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng return LHeight > RHeight; 19934a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng if (left->Latency != right->Latency) 19944a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng return left->Latency > right->Latency; 19954a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng } 1996046fa3f90a31ebfa10df89ae348f478d492709a9Evan Cheng } 1997046fa3f90a31ebfa10df89ae348f478d492709a9Evan Cheng 199815a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng return BURRSort(left, right, SPQ); 199915a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng} 200015a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 20012da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick// Schedule as many instructions in each cycle as possible. So don't make an 20022da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick// instruction available unless it is ready in the current cycle. 20032da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickbool ilp_ls_rr_sort::isReady(SUnit *SU, unsigned CurCycle) const { 20042da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return SU->getHeight() <= CurCycle; 20052da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 20062da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 200770017e44cdba1946cc478ce1856a3e855a767e28Evan Chengbool ilp_ls_rr_sort::operator()(const SUnit *left, 200870017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng const SUnit *right) const { 20098239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng if (left->isCall || right->isCall) 20108239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng // No way to compute latency of calls. 20118239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng return BURRSort(left, right, SPQ); 20128239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng 2013e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng bool LHigh = SPQ->HighRegPressure(left); 2014e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng bool RHigh = SPQ->HighRegPressure(right); 201570017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng // Avoid causing spills. If register pressure is high, schedule for 201670017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng // register pressure reduction. 201770017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng if (LHigh && !RHigh) 201870017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng return true; 201970017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng else if (!LHigh && RHigh) 202070017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng return false; 2021e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng else if (!LHigh && !RHigh) { 2022e28ed16d2507633893269670d289006f3e0b351eEvan Cheng // Low register pressure situation, schedule to maximize instruction level 2023e28ed16d2507633893269670d289006f3e0b351eEvan Cheng // parallelism. 202470017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng if (left->NumPreds > right->NumPreds) 202570017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng return false; 202670017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng else if (left->NumPreds < right->NumPreds) 202770017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng return false; 202870017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng } 202970017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 203070017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng return BURRSort(left, right, SPQ); 203170017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng} 203270017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 20336be2ee431f44e3eb4d87bb3779a7e97a766c7a3eDan Gohmantemplate<class SF> 2034c6be777208f4539af400ac694d9d1dc8b992bc80Evan Chengbool 20356be2ee431f44e3eb4d87bb3779a7e97a766c7a3eDan GohmanRegReductionPriorityQueue<SF>::canClobber(const SUnit *SU, const SUnit *Op) { 203695f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng if (SU->isTwoAddress) { 2037550f5afb68ce8f034991863cac65bef22a6554daDan Gohman unsigned Opc = SU->getNode()->getMachineOpcode(); 2038749c6f6b5ed301c84aac562e414486549d7b98ebChris Lattner const TargetInstrDesc &TID = TII->get(Opc); 20393db805ea80eeec9084a1b86273d93804d233d938Chris Lattner unsigned NumRes = TID.getNumDefs(); 20403b66555c53eb8921b2dd50335e0b278ddf80d220Dan Gohman unsigned NumOps = TID.getNumOperands() - NumRes; 204195f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng for (unsigned i = 0; i != NumOps; ++i) { 20423db805ea80eeec9084a1b86273d93804d233d938Chris Lattner if (TID.getOperandConstraint(i+NumRes, TOI::TIED_TO) != -1) { 2043550f5afb68ce8f034991863cac65bef22a6554daDan Gohman SDNode *DU = SU->getNode()->getOperand(i).getNode(); 204494d7a5f8156e62532870fbaf197377b34e52ff2aDan Gohman if (DU->getNodeId() != -1 && 204594d7a5f8156e62532870fbaf197377b34e52ff2aDan Gohman Op->OrigNode == &(*SUnits)[DU->getNodeId()]) 204695f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng return true; 204795f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 204895f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 2049e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 2050e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng return false; 2051e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 2052e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 2053180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng/// canClobberPhysRegDefs - True if SU would clobber one of SuccSU's 20542f1d3108e481758da66662f72673741da86312daDan Gohman/// physical register defs. 2055430b8a22e2717d3dfb6b4f096bc23c9538fd7959Dan Gohmanstatic bool canClobberPhysRegDefs(const SUnit *SuccSU, const SUnit *SU, 2056180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng const TargetInstrInfo *TII, 20576f0d024a534af18d9e60b3ea757376cd8a3a980eDan Gohman const TargetRegisterInfo *TRI) { 2058550f5afb68ce8f034991863cac65bef22a6554daDan Gohman SDNode *N = SuccSU->getNode(); 2059e8be6c63915e0389f1eef6b53c64300d13b2ce99Dan Gohman unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs(); 2060e8be6c63915e0389f1eef6b53c64300d13b2ce99Dan Gohman const unsigned *ImpDefs = TII->get(N->getMachineOpcode()).getImplicitDefs(); 20612f1d3108e481758da66662f72673741da86312daDan Gohman assert(ImpDefs && "Caller should check hasPhysRegDefs"); 2062a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman for (const SDNode *SUNode = SU->getNode(); SUNode; 206329d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner SUNode = SUNode->getGluedNode()) { 2064a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman if (!SUNode->isMachineOpcode()) 206559932584a812685b16ad6a53a023b2bb3fc21819Dan Gohman continue; 2066a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman const unsigned *SUImpDefs = 2067a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman TII->get(SUNode->getMachineOpcode()).getImplicitDefs(); 2068a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman if (!SUImpDefs) 2069a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman return false; 2070a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman for (unsigned i = NumDefs, e = N->getNumValues(); i != e; ++i) { 2071e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = N->getValueType(i); 2072f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner if (VT == MVT::Glue || VT == MVT::Other) 2073a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman continue; 2074a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman if (!N->hasAnyUseOfValue(i)) 2075a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman continue; 2076a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman unsigned Reg = ImpDefs[i - NumDefs]; 2077a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman for (;*SUImpDefs; ++SUImpDefs) { 2078a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman unsigned SUReg = *SUImpDefs; 2079a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman if (TRI->regsOverlap(Reg, SUReg)) 2080a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman return true; 2081a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman } 2082180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng } 2083180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng } 2084180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng return false; 2085180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng} 2086180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng 2087002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// PrescheduleNodesWithMultipleUses - Nodes with multiple uses 2088002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// are not handled well by the general register pressure reduction 2089002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// heuristics. When presented with code like this: 2090002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2091002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// N 2092002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// / | 2093002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// / | 2094002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// U store 2095002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// | 2096002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// ... 2097002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2098002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// the heuristics tend to push the store up, but since the 2099002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// operand of the store has another use (U), this would increase 2100002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// the length of that other use (the U->N edge). 2101002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2102002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// This function transforms code like the above to route U's 2103002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// dependence through the store when possible, like this: 2104002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2105002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// N 2106002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// || 2107002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// || 2108002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// store 2109002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// | 2110002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// U 2111002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// | 2112002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// ... 2113002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2114002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// This results in the store being scheduled immediately 2115002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// after N, which shortens the U->N live range, reducing 2116002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// register pressure. 2117002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2118002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohmantemplate<class SF> 2119002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohmanvoid RegReductionPriorityQueue<SF>::PrescheduleNodesWithMultipleUses() { 2120002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Visit all the nodes in topological order, working top-down. 2121002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman for (unsigned i = 0, e = SUnits->size(); i != e; ++i) { 2122002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman SUnit *SU = &(*SUnits)[i]; 2123002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // For now, only look at nodes with no data successors, such as stores. 2124002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // These are especially important, due to the heuristics in 2125002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // getNodePriority for nodes with no data successors. 2126002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SU->NumSuccs != 0) 2127002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2128002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // For now, only look at nodes with exactly one data predecessor. 2129002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SU->NumPreds != 1) 2130002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2131002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Avoid prescheduling copies to virtual registers, which don't behave 2132002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // like other nodes from the perspective of scheduling heuristics. 2133002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SDNode *N = SU->getNode()) 2134002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (N->getOpcode() == ISD::CopyToReg && 2135002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman TargetRegisterInfo::isVirtualRegister 2136002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman (cast<RegisterSDNode>(N->getOperand(1))->getReg())) 2137002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2138002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman 2139002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Locate the single data predecessor. 2140002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman SUnit *PredSU = 0; 2141002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman for (SUnit::const_pred_iterator II = SU->Preds.begin(), 2142002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman EE = SU->Preds.end(); II != EE; ++II) 2143002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (!II->isCtrl()) { 2144002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman PredSU = II->getSUnit(); 2145002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman break; 2146002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman } 2147002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman assert(PredSU); 2148002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman 2149002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Don't rewrite edges that carry physregs, because that requires additional 2150002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // support infrastructure. 2151002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (PredSU->hasPhysRegDefs) 2152002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2153002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Short-circuit the case where SU is PredSU's only data successor. 2154002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (PredSU->NumSuccs == 1) 2155002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2156002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Avoid prescheduling to copies from virtual registers, which don't behave 2157002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // like other nodes from the perspective of scheduling // heuristics. 2158002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SDNode *N = SU->getNode()) 2159002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (N->getOpcode() == ISD::CopyFromReg && 2160002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman TargetRegisterInfo::isVirtualRegister 2161002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman (cast<RegisterSDNode>(N->getOperand(1))->getReg())) 2162002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2163002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman 2164002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Perform checks on the successors of PredSU. 2165002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman for (SUnit::const_succ_iterator II = PredSU->Succs.begin(), 2166002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman EE = PredSU->Succs.end(); II != EE; ++II) { 2167002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman SUnit *PredSuccSU = II->getSUnit(); 2168002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (PredSuccSU == SU) continue; 2169002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // If PredSU has another successor with no data successors, for 2170002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // now don't attempt to choose either over the other. 2171002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (PredSuccSU->NumSuccs == 0) 2172002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman goto outer_loop_continue; 2173002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Don't break physical register dependencies. 2174002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SU->hasPhysRegClobbers && PredSuccSU->hasPhysRegDefs) 2175002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (canClobberPhysRegDefs(PredSuccSU, SU, TII, TRI)) 2176002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman goto outer_loop_continue; 2177002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Don't introduce graph cycles. 2178002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (scheduleDAG->IsReachable(SU, PredSuccSU)) 2179002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman goto outer_loop_continue; 2180002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman } 2181002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman 2182002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Ok, the transformation is safe and the heuristics suggest it is 2183002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // profitable. Update the graph. 218415a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng DEBUG(dbgs() << " Prescheduling SU #" << SU->NodeNum 218515a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng << " next to PredSU #" << PredSU->NodeNum 2186bbbfa99d3d18fe9f20265305e833666645ada528Chris Lattner << " to guide scheduling in the presence of multiple uses\n"); 2187002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman for (unsigned i = 0; i != PredSU->Succs.size(); ++i) { 2188002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman SDep Edge = PredSU->Succs[i]; 2189002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman assert(!Edge.isAssignedRegDep()); 2190002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman SUnit *SuccSU = Edge.getSUnit(); 2191002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SuccSU != SU) { 2192002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman Edge.setSUnit(PredSU); 2193002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman scheduleDAG->RemovePred(SuccSU, Edge); 2194002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman scheduleDAG->AddPred(SU, Edge); 2195002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman Edge.setSUnit(SU); 2196002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman scheduleDAG->AddPred(SuccSU, Edge); 2197002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman --i; 2198002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman } 2199002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman } 2200002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman outer_loop_continue:; 2201002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman } 2202002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman} 2203002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman 2204e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// AddPseudoTwoAddrDeps - If two nodes share an operand and one of them uses 2205e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// it as a def&use operand. Add a pseudo control edge from it to the other 2206e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// node (if it won't create a cycle) so the two-address one will be scheduled 220722a529990bb4bb86bdb2ae1cfce7340320a6ca7fEvan Cheng/// first (lower in the schedule). If both nodes are two-address, favor the 220822a529990bb4bb86bdb2ae1cfce7340320a6ca7fEvan Cheng/// one that has a CopyToReg use (more likely to be a loop induction update). 220922a529990bb4bb86bdb2ae1cfce7340320a6ca7fEvan Cheng/// If both are two-address, but one is commutable while the other is not 221022a529990bb4bb86bdb2ae1cfce7340320a6ca7fEvan Cheng/// commutable, favor the one that's not commutable. 22116be2ee431f44e3eb4d87bb3779a7e97a766c7a3eDan Gohmantemplate<class SF> 22126be2ee431f44e3eb4d87bb3779a7e97a766c7a3eDan Gohmanvoid RegReductionPriorityQueue<SF>::AddPseudoTwoAddrDeps() { 221395f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng for (unsigned i = 0, e = SUnits->size(); i != e; ++i) { 2214430b8a22e2717d3dfb6b4f096bc23c9538fd7959Dan Gohman SUnit *SU = &(*SUnits)[i]; 221595f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng if (!SU->isTwoAddress) 221695f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng continue; 221795f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng 2218550f5afb68ce8f034991863cac65bef22a6554daDan Gohman SDNode *Node = SU->getNode(); 221929d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner if (!Node || !Node->isMachineOpcode() || SU->getNode()->getGluedNode()) 222095f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng continue; 222195f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng 2222089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng bool isLiveOut = hasOnlyLiveOutUses(SU); 2223e8be6c63915e0389f1eef6b53c64300d13b2ce99Dan Gohman unsigned Opc = Node->getMachineOpcode(); 2224749c6f6b5ed301c84aac562e414486549d7b98ebChris Lattner const TargetInstrDesc &TID = TII->get(Opc); 22253db805ea80eeec9084a1b86273d93804d233d938Chris Lattner unsigned NumRes = TID.getNumDefs(); 22263b66555c53eb8921b2dd50335e0b278ddf80d220Dan Gohman unsigned NumOps = TID.getNumOperands() - NumRes; 222795f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng for (unsigned j = 0; j != NumOps; ++j) { 2228c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if (TID.getOperandConstraint(j+NumRes, TOI::TIED_TO) == -1) 2229c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman continue; 2230c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman SDNode *DU = SU->getNode()->getOperand(j).getNode(); 2231c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if (DU->getNodeId() == -1) 2232c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman continue; 2233c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman const SUnit *DUSU = &(*SUnits)[DU->getNodeId()]; 2234c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if (!DUSU) continue; 2235c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman for (SUnit::const_succ_iterator I = DUSU->Succs.begin(), 2236c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman E = DUSU->Succs.end(); I != E; ++I) { 223754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isCtrl()) continue; 223854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *SuccSU = I->getSUnit(); 2239c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if (SuccSU == SU) 22407da8f399bf09e9a03fe8bdd8c8eef6e5a7d87327Evan Cheng continue; 2241c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman // Be conservative. Ignore if nodes aren't at roughly the same 2242c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman // depth and height. 22433f23744df4809eba94284e601e81489212c974d4Dan Gohman if (SuccSU->getHeight() < SU->getHeight() && 22443f23744df4809eba94284e601e81489212c974d4Dan Gohman (SU->getHeight() - SuccSU->getHeight()) > 1) 2245c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman continue; 22460e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman // Skip past COPY_TO_REGCLASS nodes, so that the pseudo edge 22470e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman // constrains whatever is using the copy, instead of the copy 22480e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman // itself. In the case that the copy is coalesced, this 22490e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman // preserves the intent of the pseudo two-address heurietics. 22500e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman while (SuccSU->Succs.size() == 1 && 22510e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman SuccSU->getNode()->isMachineOpcode() && 22520e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman SuccSU->getNode()->getMachineOpcode() == 2253518bb53485df640d7b7e3f6b0544099020c42aa7Chris Lattner TargetOpcode::COPY_TO_REGCLASS) 22540e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman SuccSU = SuccSU->Succs.front().getSUnit(); 22550e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman // Don't constrain non-instruction nodes. 2256c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if (!SuccSU->getNode() || !SuccSU->getNode()->isMachineOpcode()) 2257c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman continue; 2258c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman // Don't constrain nodes with physical register defs if the 2259c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman // predecessor can clobber them. 22608f4aa333d02d0f48f90f4604d894a73ee53edcb5Dan Gohman if (SuccSU->hasPhysRegDefs && SU->hasPhysRegClobbers) { 2261c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if (canClobberPhysRegDefs(SuccSU, SU, TII, TRI)) 226232dfbeada7292167bb488f36a71a5a6a519ddaffEvan Cheng continue; 2263c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman } 22648af808a3d64dfdcc5d126e6ad762f57b1483671cDan Gohman // Don't constrain EXTRACT_SUBREG, INSERT_SUBREG, and SUBREG_TO_REG; 22658af808a3d64dfdcc5d126e6ad762f57b1483671cDan Gohman // these may be coalesced away. We want them close to their uses. 2266c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman unsigned SuccOpc = SuccSU->getNode()->getMachineOpcode(); 2267518bb53485df640d7b7e3f6b0544099020c42aa7Chris Lattner if (SuccOpc == TargetOpcode::EXTRACT_SUBREG || 2268518bb53485df640d7b7e3f6b0544099020c42aa7Chris Lattner SuccOpc == TargetOpcode::INSERT_SUBREG || 2269518bb53485df640d7b7e3f6b0544099020c42aa7Chris Lattner SuccOpc == TargetOpcode::SUBREG_TO_REG) 2270c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman continue; 2271c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if ((!canClobber(SuccSU, DUSU) || 2272089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng (isLiveOut && !hasOnlyLiveOutUses(SuccSU)) || 2273c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman (!SU->isCommutable && SuccSU->isCommutable)) && 2274c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman !scheduleDAG->IsReachable(SuccSU, SU)) { 227515a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng DEBUG(dbgs() << " Adding a pseudo-two-addr edge from SU #" 2276bbbfa99d3d18fe9f20265305e833666645ada528Chris Lattner << SU->NodeNum << " to SU #" << SuccSU->NodeNum << "\n"); 2277fd2163bcf77df6b3e58868483c089bd3869b01d6Dan Gohman scheduleDAG->AddPred(SU, SDep(SuccSU, SDep::Order, /*Latency=*/0, 2278ce0d4b7a77da716f234d42edd0472e97b3ba5f57Dan Gohman /*Reg=*/0, /*isNormalMemory=*/false, 2279ce0d4b7a77da716f234d42edd0472e97b3ba5f57Dan Gohman /*isMustAlias=*/false, 228054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman /*isArtificial=*/true)); 228195f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 228295f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 228395f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 228495f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 2285e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 2286e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 2287c8edc64188399437f5476d7fa45f714a92f2cb93Evan Cheng/// CalculateSethiUllmanNumbers - Calculate Sethi-Ullman numbers of all 2288c8edc64188399437f5476d7fa45f714a92f2cb93Evan Cheng/// scheduling units. 2289117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohmantemplate<class SF> 2290117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohmanvoid RegReductionPriorityQueue<SF>::CalculateSethiUllmanNumbers() { 2291e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng SethiUllmanNumbers.assign(SUnits->size(), 0); 229238036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 2293e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng for (unsigned i = 0, e = SUnits->size(); i != e; ++i) 2294117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman CalcNodeSethiUllmanNumber(&(*SUnits)[i], SethiUllmanNumbers); 2295c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng} 2296e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 2297d7d3ea00c0a26e2545d4ba01825d8358075264e7Roman Levenstein/// LimitedSumOfUnscheduledPredsOfSuccs - Compute the sum of the unscheduled 229895d4184e7218d7ec21b5b8d693dd3b14146eefdcRoman Levenstein/// predecessors of the successors of the SUnit SU. Stop when the provided 229995d4184e7218d7ec21b5b8d693dd3b14146eefdcRoman Levenstein/// limit is exceeded. 230038036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trickstatic unsigned LimitedSumOfUnscheduledPredsOfSuccs(const SUnit *SU, 230195d4184e7218d7ec21b5b8d693dd3b14146eefdcRoman Levenstein unsigned Limit) { 230295d4184e7218d7ec21b5b8d693dd3b14146eefdcRoman Levenstein unsigned Sum = 0; 230395d4184e7218d7ec21b5b8d693dd3b14146eefdcRoman Levenstein for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 230495d4184e7218d7ec21b5b8d693dd3b14146eefdcRoman Levenstein I != E; ++I) { 230554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman const SUnit *SuccSU = I->getSUnit(); 230695d4184e7218d7ec21b5b8d693dd3b14146eefdcRoman Levenstein for (SUnit::const_pred_iterator II = SuccSU->Preds.begin(), 230795d4184e7218d7ec21b5b8d693dd3b14146eefdcRoman Levenstein EE = SuccSU->Preds.end(); II != EE; ++II) { 230854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *PredSU = II->getSUnit(); 2309fd5da6c991ad0bb5c0b6ce797d56c49ad3f73803Evan Cheng if (!PredSU->isScheduled) 2310fd5da6c991ad0bb5c0b6ce797d56c49ad3f73803Evan Cheng if (++Sum > Limit) 2311fd5da6c991ad0bb5c0b6ce797d56c49ad3f73803Evan Cheng return Sum; 231295d4184e7218d7ec21b5b8d693dd3b14146eefdcRoman Levenstein } 231395d4184e7218d7ec21b5b8d693dd3b14146eefdcRoman Levenstein } 231495d4184e7218d7ec21b5b8d693dd3b14146eefdcRoman Levenstein return Sum; 231595d4184e7218d7ec21b5b8d693dd3b14146eefdcRoman Levenstein} 231695d4184e7218d7ec21b5b8d693dd3b14146eefdcRoman Levenstein 2317e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 2318e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// Top down 2319e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengbool td_ls_rr_sort::operator()(const SUnit *left, const SUnit *right) const { 2320c8edc64188399437f5476d7fa45f714a92f2cb93Evan Cheng unsigned LPriority = SPQ->getNodePriority(left); 2321c8edc64188399437f5476d7fa45f714a92f2cb93Evan Cheng unsigned RPriority = SPQ->getNodePriority(right); 2322550f5afb68ce8f034991863cac65bef22a6554daDan Gohman bool LIsTarget = left->getNode() && left->getNode()->isMachineOpcode(); 2323550f5afb68ce8f034991863cac65bef22a6554daDan Gohman bool RIsTarget = right->getNode() && right->getNode()->isMachineOpcode(); 2324e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng bool LIsFloater = LIsTarget && left->NumPreds == 0; 2325e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng bool RIsFloater = RIsTarget && right->NumPreds == 0; 232695d4184e7218d7ec21b5b8d693dd3b14146eefdcRoman Levenstein unsigned LBonus = (LimitedSumOfUnscheduledPredsOfSuccs(left,1) == 1) ? 2 : 0; 232795d4184e7218d7ec21b5b8d693dd3b14146eefdcRoman Levenstein unsigned RBonus = (LimitedSumOfUnscheduledPredsOfSuccs(right,1) == 1) ? 2 : 0; 2328e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 2329e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng if (left->NumSuccs == 0 && right->NumSuccs != 0) 2330e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng return false; 2331e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng else if (left->NumSuccs != 0 && right->NumSuccs == 0) 2332e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng return true; 2333e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 2334e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng if (LIsFloater) 2335e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng LBonus -= 2; 2336e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng if (RIsFloater) 2337e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng RBonus -= 2; 2338e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng if (left->NumSuccs == 1) 2339e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng LBonus += 2; 2340e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng if (right->NumSuccs == 1) 2341e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng RBonus += 2; 2342e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 234384d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng if (LPriority+LBonus != RPriority+RBonus) 234484d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng return LPriority+LBonus < RPriority+RBonus; 234584d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng 23463f23744df4809eba94284e601e81489212c974d4Dan Gohman if (left->getDepth() != right->getDepth()) 23473f23744df4809eba94284e601e81489212c974d4Dan Gohman return left->getDepth() < right->getDepth(); 234884d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng 234984d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng if (left->NumSuccsLeft != right->NumSuccsLeft) 235084d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng return left->NumSuccsLeft > right->NumSuccsLeft; 235184d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng 235238036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick assert(left->NodeQueueId && right->NodeQueueId && 2353a0201d52049be8dcefffe4304a49690a831bcb34Roman Levenstein "NodeQueueId cannot be zero"); 2354a0201d52049be8dcefffe4304a49690a831bcb34Roman Levenstein return (left->NodeQueueId > right->NodeQueueId); 2355e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 2356e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 2357e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 2358e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// Public Constructor Functions 2359e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 2360e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 236147ac0f0c7c39289f5970688154e385be22b7f293Dan Gohmanllvm::ScheduleDAGSDNodes * 23622da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickllvm::createBURRListDAGScheduler(SelectionDAGISel *IS, 23632da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CodeGenOpt::Level OptLevel) { 236479ce276083ced01256a0eb7d80731e4948ca6e87Dan Gohman const TargetMachine &TM = IS->TM; 236579ce276083ced01256a0eb7d80731e4948ca6e87Dan Gohman const TargetInstrInfo *TII = TM.getInstrInfo(); 236679ce276083ced01256a0eb7d80731e4948ca6e87Dan Gohman const TargetRegisterInfo *TRI = TM.getRegisterInfo(); 236738036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 23684f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng BURegReductionPriorityQueue *PQ = 236989ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng new BURegReductionPriorityQueue(*IS->MF, false, TII, TRI, 0); 23702da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, false, PQ, OptLevel); 2371c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng PQ->setScheduleDAG(SD); 237238036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick return SD; 2373e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 2374e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 237547ac0f0c7c39289f5970688154e385be22b7f293Dan Gohmanllvm::ScheduleDAGSDNodes * 23762da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickllvm::createTDRRListDAGScheduler(SelectionDAGISel *IS, 23772da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CodeGenOpt::Level OptLevel) { 237879ce276083ced01256a0eb7d80731e4948ca6e87Dan Gohman const TargetMachine &TM = IS->TM; 237979ce276083ced01256a0eb7d80731e4948ca6e87Dan Gohman const TargetInstrInfo *TII = TM.getInstrInfo(); 238079ce276083ced01256a0eb7d80731e4948ca6e87Dan Gohman const TargetRegisterInfo *TRI = TM.getRegisterInfo(); 238138036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 23824f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng TDRegReductionPriorityQueue *PQ = 23834f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng new TDRegReductionPriorityQueue(*IS->MF, false, TII, TRI, 0); 23842da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, false, PQ, OptLevel); 23856be2ee431f44e3eb4d87bb3779a7e97a766c7a3eDan Gohman PQ->setScheduleDAG(SD); 23866be2ee431f44e3eb4d87bb3779a7e97a766c7a3eDan Gohman return SD; 2387e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 2388187361b056823df4ff292561fe47468dad956872Bill Wendling 2389187361b056823df4ff292561fe47468dad956872Bill Wendlingllvm::ScheduleDAGSDNodes * 23902da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickllvm::createSourceListDAGScheduler(SelectionDAGISel *IS, 23912da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CodeGenOpt::Level OptLevel) { 2392187361b056823df4ff292561fe47468dad956872Bill Wendling const TargetMachine &TM = IS->TM; 2393187361b056823df4ff292561fe47468dad956872Bill Wendling const TargetInstrInfo *TII = TM.getInstrInfo(); 2394187361b056823df4ff292561fe47468dad956872Bill Wendling const TargetRegisterInfo *TRI = TM.getRegisterInfo(); 239538036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 23964f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng SrcRegReductionPriorityQueue *PQ = 239789ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng new SrcRegReductionPriorityQueue(*IS->MF, false, TII, TRI, 0); 23982da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, false, PQ, OptLevel); 239915a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng PQ->setScheduleDAG(SD); 240038036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick return SD; 240115a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng} 240215a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 240315a16def6e70c8f7df1023da80ceb89887203b40Evan Chengllvm::ScheduleDAGSDNodes * 24042da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickllvm::createHybridListDAGScheduler(SelectionDAGISel *IS, 24052da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CodeGenOpt::Level OptLevel) { 240615a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng const TargetMachine &TM = IS->TM; 240715a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng const TargetInstrInfo *TII = TM.getInstrInfo(); 240815a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng const TargetRegisterInfo *TRI = TM.getRegisterInfo(); 24094f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng const TargetLowering *TLI = &IS->getTargetLowering(); 241038036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 24114f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng HybridBURRPriorityQueue *PQ = 24123144687df78731ac4ddbc716a24b951678a73f57Evan Cheng new HybridBURRPriorityQueue(*IS->MF, true, TII, TRI, TLI); 24132da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 24142da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, PQ, OptLevel); 2415187361b056823df4ff292561fe47468dad956872Bill Wendling PQ->setScheduleDAG(SD); 241638036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick return SD; 2417187361b056823df4ff292561fe47468dad956872Bill Wendling} 241870017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 241970017e44cdba1946cc478ce1856a3e855a767e28Evan Chengllvm::ScheduleDAGSDNodes * 24202da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickllvm::createILPListDAGScheduler(SelectionDAGISel *IS, 24212da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CodeGenOpt::Level OptLevel) { 242270017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng const TargetMachine &TM = IS->TM; 242370017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng const TargetInstrInfo *TII = TM.getInstrInfo(); 242470017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng const TargetRegisterInfo *TRI = TM.getRegisterInfo(); 242570017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng const TargetLowering *TLI = &IS->getTargetLowering(); 242638036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 242770017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng ILPBURRPriorityQueue *PQ = 242870017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng new ILPBURRPriorityQueue(*IS->MF, true, TII, TRI, TLI); 24292da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, PQ, OptLevel); 243070017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng PQ->setScheduleDAG(SD); 243138036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick return SD; 243270017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng} 2433