ScheduleDAGRRList.cpp revision d04a8d4b33ff316ca4cf961e06c9e312eff8e64f
104f4f4f447806cd92a2fb6f4b66d11f6d5003a82Dan Gohman//===----- ScheduleDAGRRList.cpp - Reg pressure reduction list scheduler --===// 2e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 3e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// The LLVM Compiler Infrastructure 4e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 7e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 8e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 9e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 10e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// This implements bottom-up and top-down register pressure reduction list 11e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// schedulers, using standard algorithms. The basic approach uses a priority 12e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// queue of available nodes to schedule. One at a time, nodes are taken from 13e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// the priority queue (thus in priority order), checked for legality to 14e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// schedule, and emitted if legal. 15e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 16e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 17e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 18e7e7d0d7e39d0c7c659d26b97e8081fce0fcd749Dale Johannesen#define DEBUG_TYPE "pre-RA-sched" 19eb577ba3b815a1fa4627b060dd2345d17abf672dJim Laskey#include "llvm/CodeGen/SchedulerRegistry.h" 20d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "ScheduleDAGSDNodes.h" 21d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/STLExtras.h" 22a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng#include "llvm/ADT/SmallSet.h" 23e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#include "llvm/ADT/Statistic.h" 24d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/CodeGen/ScheduleHazardRecognizer.h" 25d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/CodeGen/SelectionDAGISel.h" 26d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/DataLayout.h" 27d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/InlineAsm.h" 28decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner#include "llvm/Support/Debug.h" 29decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner#include "llvm/Support/ErrorHandling.h" 30bbbfa99d3d18fe9f20265305e833666645ada528Chris Lattner#include "llvm/Support/raw_ostream.h" 31d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Target/TargetInstrInfo.h" 32d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Target/TargetLowering.h" 33d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Target/TargetMachine.h" 34d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Target/TargetRegisterInfo.h" 35e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#include <climits> 36e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengusing namespace llvm; 37e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 38cffbd2562a5e3ba435dd2b622710ec272c634da5Dan GohmanSTATISTIC(NumBacktracks, "Number of times scheduler backtracked"); 39f10c973797cf79da802f9b0118543cbd50954c9cEvan ChengSTATISTIC(NumUnfolds, "Number of nodes unfolded"); 40a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan ChengSTATISTIC(NumDups, "Number of duplicated nodes"); 41c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan ChengSTATISTIC(NumPRCopies, "Number of physical register copies"); 42a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng 4313ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskeystatic RegisterScheduler 4413ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskey burrListDAGScheduler("list-burr", 45b8cab9227a0f6ffbdaae33e3c64268e265008a6aDan Gohman "Bottom-up register reduction list scheduling", 4613ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskey createBURRListDAGScheduler); 4713ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskeystatic RegisterScheduler 48187361b056823df4ff292561fe47468dad956872Bill Wendling sourceListDAGScheduler("source", 49187361b056823df4ff292561fe47468dad956872Bill Wendling "Similar to list-burr but schedules in source " 50187361b056823df4ff292561fe47468dad956872Bill Wendling "order when possible", 51187361b056823df4ff292561fe47468dad956872Bill Wendling createSourceListDAGScheduler); 5213ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskey 5315a16def6e70c8f7df1023da80ceb89887203b40Evan Chengstatic RegisterScheduler 54b11ac950d69c7a238de0a22fd23fbfcd994f57eeEvan Cheng hybridListDAGScheduler("list-hybrid", 5570017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng "Bottom-up register pressure aware list scheduling " 5670017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng "which tries to balance latency and register pressure", 5715a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng createHybridListDAGScheduler); 5815a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 5970017e44cdba1946cc478ce1856a3e855a767e28Evan Chengstatic RegisterScheduler 6070017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng ILPListDAGScheduler("list-ilp", 6170017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng "Bottom-up register pressure aware list scheduling " 6270017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng "which tries to balance ILP and register pressure", 6370017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng createILPListDAGScheduler); 6470017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 65c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trickstatic cl::opt<bool> DisableSchedCycles( 66d1dace8aea073716daf0055ad07fde1164b2a472Andrew Trick "disable-sched-cycles", cl::Hidden, cl::init(false), 67c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick cl::desc("Disable cycle-level precision during preRA scheduling")); 682da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 69e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick// Temporary sched=list-ilp flags until the heuristics are robust. 7012f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick// Some options are also available under sched=list-hybrid. 71e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickstatic cl::opt<bool> DisableSchedRegPressure( 72e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick "disable-sched-reg-pressure", cl::Hidden, cl::init(false), 73e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick cl::desc("Disable regpressure priority in sched=list-ilp")); 74e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickstatic cl::opt<bool> DisableSchedLiveUses( 753c6e49504e9a57a4818750fd2520967f84634eacAndrew Trick "disable-sched-live-uses", cl::Hidden, cl::init(true), 76e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick cl::desc("Disable live use priority in sched=list-ilp")); 7754699765064842fd08d1466adc93453660bc2a85Andrew Trickstatic cl::opt<bool> DisableSchedVRegCycle( 7854699765064842fd08d1466adc93453660bc2a85Andrew Trick "disable-sched-vrcycle", cl::Hidden, cl::init(false), 7954699765064842fd08d1466adc93453660bc2a85Andrew Trick cl::desc("Disable virtual register cycle interference checks")); 8012f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trickstatic cl::opt<bool> DisableSchedPhysRegJoin( 8112f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick "disable-sched-physreg-join", cl::Hidden, cl::init(false), 8212f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick cl::desc("Disable physreg def-use affinity")); 83e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickstatic cl::opt<bool> DisableSchedStalls( 843c6e49504e9a57a4818750fd2520967f84634eacAndrew Trick "disable-sched-stalls", cl::Hidden, cl::init(true), 85e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick cl::desc("Disable no-stall priority in sched=list-ilp")); 86e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickstatic cl::opt<bool> DisableSchedCriticalPath( 87e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick "disable-sched-critical-path", cl::Hidden, cl::init(false), 88e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick cl::desc("Disable critical path priority in sched=list-ilp")); 89e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickstatic cl::opt<bool> DisableSchedHeight( 90e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick "disable-sched-height", cl::Hidden, cl::init(false), 91e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick cl::desc("Disable scheduled-height priority in sched=list-ilp")); 92623a7e146bd86747dc46a6f8bb9993fc217d6b78Evan Chengstatic cl::opt<bool> Disable2AddrHack( 93623a7e146bd86747dc46a6f8bb9993fc217d6b78Evan Cheng "disable-2addr-hack", cl::Hidden, cl::init(true), 94623a7e146bd86747dc46a6f8bb9993fc217d6b78Evan Cheng cl::desc("Disable scheduler's two-address hack")); 95e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 96e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickstatic cl::opt<int> MaxReorderWindow( 97e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick "max-sched-reorder", cl::Hidden, cl::init(6), 98e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick cl::desc("Number of instructions to allow ahead of the critical path " 99e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick "in sched=list-ilp")); 100e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 101e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickstatic cl::opt<unsigned> AvgIPC( 102e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick "sched-avg-ipc", cl::Hidden, cl::init(1), 103e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick cl::desc("Average inst/cycle whan no target itinerary exists.")); 104e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 105e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengnamespace { 106e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 107e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// ScheduleDAGRRList - The actual register reduction list scheduler 108e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// implementation. This supports both top-down and bottom-up scheduling. 109e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// 1106726b6d75a8b679068a58cb954ba97cf9d1690baNick Lewyckyclass ScheduleDAGRRList : public ScheduleDAGSDNodes { 111e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengprivate: 11215a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng /// NeedLatency - True if the scheduler will make use of latency information. 11315a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng /// 11415a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng bool NeedLatency; 11515a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 116e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng /// AvailableQueue - The priority queue to use for the available SUnits. 117e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng SchedulingPriorityQueue *AvailableQueue; 118e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1192da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// PendingQueue - This contains all of the instructions whose operands have 1202da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// been issued, but their results are not ready yet (due to the latency of 1212da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// the operation). Once the operands becomes available, the instruction is 1222da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// added to the AvailableQueue. 1232da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick std::vector<SUnit*> PendingQueue; 1242da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1252da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// HazardRec - The hazard recognizer to use. 1262da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleHazardRecognizer *HazardRec; 1272da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1282902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick /// CurCycle - The current scheduler state corresponds to this cycle. 1292902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick unsigned CurCycle; 1302902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 1312da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// MinAvailableCycle - Cycle of the soonest available instruction. 1322da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned MinAvailableCycle; 1332da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 134e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick /// IssueCount - Count instructions issued in this cycle 135e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick /// Currently valid only for bottom-up scheduling. 136e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick unsigned IssueCount; 137e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 138086ec9976ff6cee083de618429c78473491d5713Dan Gohman /// LiveRegDefs - A set of physical registers and their definition 139a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng /// that are "live". These nodes must be scheduled before any other nodes that 140a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng /// modifies the registers can be scheduled. 141086ec9976ff6cee083de618429c78473491d5713Dan Gohman unsigned NumLiveRegs; 142a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng std::vector<SUnit*> LiveRegDefs; 1433d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick std::vector<SUnit*> LiveRegGens; 144a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 14521d9003087c9a707e6cd95460136b499df358fb8Dan Gohman /// Topo - A topological ordering for SUnits which permits fast IsReachable 14621d9003087c9a707e6cd95460136b499df358fb8Dan Gohman /// and similar queries. 14721d9003087c9a707e6cd95460136b499df358fb8Dan Gohman ScheduleDAGTopologicalSort Topo; 14821d9003087c9a707e6cd95460136b499df358fb8Dan Gohman 1490e6307f6423a7ee39b80f8dc3caef00ad11e0266Eli Friedman // Hack to keep track of the inverse of FindCallSeqStart without more crazy 1500e6307f6423a7ee39b80f8dc3caef00ad11e0266Eli Friedman // DAG crawling. 1510e6307f6423a7ee39b80f8dc3caef00ad11e0266Eli Friedman DenseMap<SUnit*, SUnit*> CallSeqEndForStart; 1520e6307f6423a7ee39b80f8dc3caef00ad11e0266Eli Friedman 153e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengpublic: 1542da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleDAGRRList(MachineFunction &mf, bool needlatency, 1552da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SchedulingPriorityQueue *availqueue, 1562da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CodeGenOpt::Level OptLevel) 157ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman : ScheduleDAGSDNodes(mf), 1582da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick NeedLatency(needlatency), AvailableQueue(availqueue), CurCycle(0), 159ae692f2baedf53504af2715993b166950e185a55Andrew Trick Topo(SUnits, NULL) { 1602da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1612da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick const TargetMachine &tm = mf.getTarget(); 162c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick if (DisableSchedCycles || !NeedLatency) 1632da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec = new ScheduleHazardRecognizer(); 164c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick else 165c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick HazardRec = tm.getInstrInfo()->CreateTargetHazardRecognizer(&tm, this); 1662da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 167e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 168e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng ~ScheduleDAGRRList() { 1692da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick delete HazardRec; 170e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng delete AvailableQueue; 171e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 172e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 173e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng void Schedule(); 174e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 175f697c8a19adf962a933b055383952e72789a0e20Andrew Trick ScheduleHazardRecognizer *getHazardRec() { return HazardRec; } 176f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1778dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// IsReachable - Checks if SU is reachable from TargetSU. 17821d9003087c9a707e6cd95460136b499df358fb8Dan Gohman bool IsReachable(const SUnit *SU, const SUnit *TargetSU) { 17921d9003087c9a707e6cd95460136b499df358fb8Dan Gohman return Topo.IsReachable(SU, TargetSU); 18021d9003087c9a707e6cd95460136b499df358fb8Dan Gohman } 181e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 1821cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohman /// WillCreateCycle - Returns true if adding an edge from SU to TargetSU will 183e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein /// create a cycle. 18421d9003087c9a707e6cd95460136b499df358fb8Dan Gohman bool WillCreateCycle(SUnit *SU, SUnit *TargetSU) { 18521d9003087c9a707e6cd95460136b499df358fb8Dan Gohman return Topo.WillCreateCycle(SU, TargetSU); 18621d9003087c9a707e6cd95460136b499df358fb8Dan Gohman } 187e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 18854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman /// AddPred - adds a predecessor edge to SUnit SU. 1898dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// This returns true if this is a new predecessor. 1908dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// Updates the topological ordering if required. 191ffa391272bad598d73fd5404dadf3686b69f2a63Dan Gohman void AddPred(SUnit *SU, const SDep &D) { 19254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman Topo.AddPred(SU, D.getSUnit()); 193ffa391272bad598d73fd5404dadf3686b69f2a63Dan Gohman SU->addPred(D); 19421d9003087c9a707e6cd95460136b499df358fb8Dan Gohman } 195e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 19654e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman /// RemovePred - removes a predecessor edge from SUnit SU. 19754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman /// This returns true if an edge was removed. 19854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman /// Updates the topological ordering if required. 199ffa391272bad598d73fd5404dadf3686b69f2a63Dan Gohman void RemovePred(SUnit *SU, const SDep &D) { 20054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman Topo.RemovePred(SU, D.getSUnit()); 201ffa391272bad598d73fd5404dadf3686b69f2a63Dan Gohman SU->removePred(D); 20221d9003087c9a707e6cd95460136b499df358fb8Dan Gohman } 203e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 204e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengprivate: 2052da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick bool isReady(SUnit *SU) { 206c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick return DisableSchedCycles || !AvailableQueue->hasReadyFilter() || 2072da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->isReady(SU); 2082da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 2092da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 2101cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohman void ReleasePred(SUnit *SU, const SDep *PredEdge); 2113d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick void ReleasePredecessors(SUnit *SU); 2122da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void ReleasePending(); 2132da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void AdvanceToCycle(unsigned NextCycle); 2142da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void AdvancePastStalls(SUnit *SU); 2152da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void EmitNode(SUnit *SU); 2162902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick void ScheduleNodeBottomUp(SUnit*); 2172da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void CapturePred(SDep *PredEdge); 21842d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng void UnscheduleNodeBottomUp(SUnit*); 2192da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void RestoreHazardCheckerBottomUp(); 2202da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void BacktrackBottomUp(SUnit*, SUnit*); 22142d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng SUnit *CopyAndMoveSuccessors(SUnit*); 222c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng void InsertCopiesAndMoveSuccs(SUnit*, unsigned, 223c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng const TargetRegisterClass*, 224c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng const TargetRegisterClass*, 225c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng SmallVector<SUnit*, 2>&); 226a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng bool DelayForLiveRegsBottomUp(SUnit*, SmallVector<unsigned, 4>&); 2272da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 2282902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *PickNodeToScheduleBottomUp(); 229e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng void ListScheduleBottomUp(); 230e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 231e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein /// CreateNewSUnit - Creates a new SUnit and returns a pointer to it. 2328dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// Updates the topological ordering if required. 233e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *CreateNewSUnit(SDNode *N) { 23421d9003087c9a707e6cd95460136b499df358fb8Dan Gohman unsigned NumSUnits = SUnits.size(); 235953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick SUnit *NewNode = newSUnit(N); 2368dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein // Update the topological ordering. 23721d9003087c9a707e6cd95460136b499df358fb8Dan Gohman if (NewNode->NodeNum >= NumSUnits) 23821d9003087c9a707e6cd95460136b499df358fb8Dan Gohman Topo.InitDAGTopologicalSorting(); 239e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein return NewNode; 240e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein } 241e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 2428dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// CreateClone - Creates a new SUnit from an existing one. 2438dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// Updates the topological ordering if required. 244e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *CreateClone(SUnit *N) { 24521d9003087c9a707e6cd95460136b499df358fb8Dan Gohman unsigned NumSUnits = SUnits.size(); 246e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *NewNode = Clone(N); 2478dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein // Update the topological ordering. 24821d9003087c9a707e6cd95460136b499df358fb8Dan Gohman if (NewNode->NodeNum >= NumSUnits) 24921d9003087c9a707e6cd95460136b499df358fb8Dan Gohman Topo.InitDAGTopologicalSorting(); 250e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein return NewNode; 251e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein } 2523f23744df4809eba94284e601e81489212c974d4Dan Gohman 253953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick /// forceUnitLatencies - Register-pressure-reducing scheduling doesn't 25415a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng /// need actual latency information but the hybrid scheduler does. 255953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick bool forceUnitLatencies() const { 25615a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng return !NeedLatency; 25715a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng } 258e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng}; 259e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} // end anonymous namespace 260e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 26177b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson/// GetCostForDef - Looks up the register class and cost for a given definition. 26277b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson/// Typically this just means looking up the representative register class, 26399aa14ff64c92eab347d23696e358361d3bd90eaOwen Anderson/// but for untyped values (MVT::Untyped) it means inspecting the node's 26477b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson/// opcode to determine what register class is being generated. 26577b4b13c2a525faf646a6784b24692cf0459b75eOwen Andersonstatic void GetCostForDef(const ScheduleDAGSDNodes::RegDefIter &RegDefPos, 26677b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson const TargetLowering *TLI, 26777b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson const TargetInstrInfo *TII, 26877b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson const TargetRegisterInfo *TRI, 269397fc4874efe9c17e737d4c5c50bd19dc3bf27f5Jakob Stoklund Olesen unsigned &RegClass, unsigned &Cost, 270397fc4874efe9c17e737d4c5c50bd19dc3bf27f5Jakob Stoklund Olesen const MachineFunction &MF) { 27177b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson EVT VT = RegDefPos.GetValue(); 27277b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson 27377b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson // Special handling for untyped values. These values can only come from 27477b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson // the expansion of custom DAG-to-DAG patterns. 27599aa14ff64c92eab347d23696e358361d3bd90eaOwen Anderson if (VT == MVT::Untyped) { 276109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson const SDNode *Node = RegDefPos.GetNode(); 277109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson unsigned Opcode = Node->getMachineOpcode(); 278109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson 279109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson if (Opcode == TargetOpcode::REG_SEQUENCE) { 280109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson unsigned DstRCIdx = cast<ConstantSDNode>(Node->getOperand(0))->getZExtValue(); 281109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson const TargetRegisterClass *RC = TRI->getRegClass(DstRCIdx); 282109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson RegClass = RC->getID(); 283109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson Cost = 1; 284109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson return; 285109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson } 286109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson 28777b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson unsigned Idx = RegDefPos.GetIdx(); 288e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng const MCInstrDesc Desc = TII->get(Opcode); 289397fc4874efe9c17e737d4c5c50bd19dc3bf27f5Jakob Stoklund Olesen const TargetRegisterClass *RC = TII->getRegClass(Desc, Idx, TRI, MF); 29077b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson RegClass = RC->getID(); 29177b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson // FIXME: Cost arbitrarily set to 1 because there doesn't seem to be a 29277b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson // better way to determine it. 29377b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson Cost = 1; 29477b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson } else { 29577b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson RegClass = TLI->getRepRegClassFor(VT)->getID(); 29677b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson Cost = TLI->getRepRegClassCostFor(VT); 29777b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson } 29877b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson} 299e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 300e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// Schedule - Schedule the DAG using list scheduling. 301e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengvoid ScheduleDAGRRList::Schedule() { 3024f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng DEBUG(dbgs() 3034f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng << "********** List Scheduling BB#" << BB->getNumber() 304089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng << " '" << BB->getName() << "' **********\n"); 305a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 3062902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurCycle = 0; 307e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick IssueCount = 0; 308c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick MinAvailableCycle = DisableSchedCycles ? 0 : UINT_MAX; 309086ec9976ff6cee083de618429c78473491d5713Dan Gohman NumLiveRegs = 0; 31065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Allocate slots for each physical register, plus one for a special register 31165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // to track the virtual resource of a calling sequence. 31265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegDefs.resize(TRI->getNumRegs() + 1, NULL); 31365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegGens.resize(TRI->getNumRegs() + 1, NULL); 3140e6307f6423a7ee39b80f8dc3caef00ad11e0266Eli Friedman CallSeqEndForStart.clear(); 315a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 316c9a5b9e38b442c2ae6b115213a07df3fcd14708dDan Gohman // Build the scheduling graph. 31798976e4dcd18adbbe676048c0069e67346eb4adeDan Gohman BuildSchedGraph(NULL); 318e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 319e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su) 3203cc6243ddfdba3ad64035b919c88b09773a60880Dan Gohman SUnits[su].dumpAll(this)); 32121d9003087c9a707e6cd95460136b499df358fb8Dan Gohman Topo.InitDAGTopologicalSorting(); 322e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 32394d7a5f8156e62532870fbaf197377b34e52ff2aDan Gohman AvailableQueue->initNodes(SUnits); 32438036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 3252da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->Reset(); 3262da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 327ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman // Execute the actual scheduling loop. 328ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman ListScheduleBottomUp(); 32938036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 330e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng AvailableQueue->releaseState(); 33173ba69b6843f7f23345b1e8745cb328952cae0d8Andrew Trick 33273ba69b6843f7f23345b1e8745cb328952cae0d8Andrew Trick DEBUG({ 33373ba69b6843f7f23345b1e8745cb328952cae0d8Andrew Trick dbgs() << "*** Final schedule ***\n"; 33473ba69b6843f7f23345b1e8745cb328952cae0d8Andrew Trick dumpSchedule(); 33573ba69b6843f7f23345b1e8745cb328952cae0d8Andrew Trick dbgs() << '\n'; 33673ba69b6843f7f23345b1e8745cb328952cae0d8Andrew Trick }); 33713d41b9d721f98372b97d2ec119e6c91932ab0aeEvan Cheng} 338e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 339e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 340e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// Bottom-Up Scheduling 341e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 342e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 343e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// ReleasePred - Decrement the NumSuccsLeft count of a predecessor. Add it to 3448d1bfad00b1ebff5b140b6e1bd7e26bad697d6e1Dan Gohman/// the AvailableQueue if the count reaches zero. Also update its cycle bound. 3451cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohmanvoid ScheduleDAGRRList::ReleasePred(SUnit *SU, const SDep *PredEdge) { 34654e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *PredSU = PredEdge->getSUnit(); 3473a90c9b8edb53ea1ea57d94f7ab256608b5b30c6Reid Kleckner 348e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#ifndef NDEBUG 3493a90c9b8edb53ea1ea57d94f7ab256608b5b30c6Reid Kleckner if (PredSU->NumSuccsLeft == 0) { 350e492ae13edd83b120d665c0503cf4de2925b5e56David Greene dbgs() << "*** Scheduling failed! ***\n"; 3513cc6243ddfdba3ad64035b919c88b09773a60880Dan Gohman PredSU->dump(this); 352e492ae13edd83b120d665c0503cf4de2925b5e56David Greene dbgs() << " has been released too many times!\n"; 353c23197a26f34f559ea9797de51e187087c039c42Torok Edwin llvm_unreachable(0); 354e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 355e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#endif 3563a90c9b8edb53ea1ea57d94f7ab256608b5b30c6Reid Kleckner --PredSU->NumSuccsLeft; 3573a90c9b8edb53ea1ea57d94f7ab256608b5b30c6Reid Kleckner 358953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick if (!forceUnitLatencies()) { 35915a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng // Updating predecessor's height. This is now the cycle when the 36015a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng // predecessor can be scheduled without causing a pipeline stall. 36115a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng PredSU->setHeightToAtLeast(SU->getHeight() + PredEdge->getLatency()); 36215a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng } 36315a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 3649e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman // If all the node's successors are scheduled, this node is ready 3659e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman // to be scheduled. Ignore the special EntrySU node. 3669e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman if (PredSU->NumSuccsLeft == 0 && PredSU != &EntrySU) { 36780792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman PredSU->isAvailable = true; 3682da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 3692da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned Height = PredSU->getHeight(); 3702da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (Height < MinAvailableCycle) 3712da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick MinAvailableCycle = Height; 3722da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 373a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick if (isReady(PredSU)) { 3742da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->push(PredSU); 3752da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 3762da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // CapturePred and others may have left the node in the pending queue, avoid 3772da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // adding it twice. 3782da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick else if (!PredSU->isPending) { 3792da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PredSU->isPending = true; 3802da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PendingQueue.push_back(PredSU); 3812da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 382e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 383e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 384e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 38565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// IsChainDependent - Test if Outer is reachable from Inner through 38665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// chain dependencies. 38765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohmanstatic bool IsChainDependent(SDNode *Outer, SDNode *Inner, 38865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned NestLevel, 38965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman const TargetInstrInfo *TII) { 39065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SDNode *N = Outer; 39165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (;;) { 39265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N == Inner) 39365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return true; 39465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // For a TokenFactor, examine each operand. There may be multiple ways 39565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // to get to the CALLSEQ_BEGIN, but we need to find the path with the 39665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // most nesting in order to ensure that we find the corresponding match. 39765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->getOpcode() == ISD::TokenFactor) { 39865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 39965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (IsChainDependent(N->getOperand(i).getNode(), Inner, NestLevel, TII)) 40065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return true; 40165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return false; 40265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 40365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Check for a lowered CALLSEQ_BEGIN or CALLSEQ_END. 40465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->isMachineOpcode()) { 40565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->getMachineOpcode() == 40665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman (unsigned)TII->getCallFrameDestroyOpcode()) { 40765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman ++NestLevel; 40865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } else if (N->getMachineOpcode() == 40965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman (unsigned)TII->getCallFrameSetupOpcode()) { 41065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (NestLevel == 0) 41165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return false; 41265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman --NestLevel; 41365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 41465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 41565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Otherwise, find the chain and continue climbing. 41665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 41765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->getOperand(i).getValueType() == MVT::Other) { 41865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman N = N->getOperand(i).getNode(); 41965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman goto found_chain_operand; 42065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 42165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return false; 42265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman found_chain_operand:; 42365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->getOpcode() == ISD::EntryToken) 42465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return false; 42565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 42665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman} 42765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman 42865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// FindCallSeqStart - Starting from the (lowered) CALLSEQ_END node, locate 42965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// the corresponding (lowered) CALLSEQ_BEGIN node. 43065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// 43165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// NestLevel and MaxNested are used in recursion to indcate the current level 43265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// of nesting of CALLSEQ_BEGIN and CALLSEQ_END pairs, as well as the maximum 43365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// level seen so far. 43465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// 43565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// TODO: It would be better to give CALLSEQ_END an explicit operand to point 43665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// to the corresponding CALLSEQ_BEGIN to avoid needing to search for it. 43765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohmanstatic SDNode * 43865fd6564b8aedd053845c81ede1ac594acb470e4Dan GohmanFindCallSeqStart(SDNode *N, unsigned &NestLevel, unsigned &MaxNest, 43965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman const TargetInstrInfo *TII) { 44065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (;;) { 44165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // For a TokenFactor, examine each operand. There may be multiple ways 44265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // to get to the CALLSEQ_BEGIN, but we need to find the path with the 44365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // most nesting in order to ensure that we find the corresponding match. 44465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->getOpcode() == ISD::TokenFactor) { 44565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SDNode *Best = 0; 44665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned BestMaxNest = MaxNest; 44765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 44865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned MyNestLevel = NestLevel; 44965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned MyMaxNest = MaxNest; 45065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (SDNode *New = FindCallSeqStart(N->getOperand(i).getNode(), 45165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman MyNestLevel, MyMaxNest, TII)) 45265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (!Best || (MyMaxNest > BestMaxNest)) { 45365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman Best = New; 45465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman BestMaxNest = MyMaxNest; 45565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 45665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 45765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman assert(Best); 45865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman MaxNest = BestMaxNest; 45965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return Best; 46065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 46165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Check for a lowered CALLSEQ_BEGIN or CALLSEQ_END. 46265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->isMachineOpcode()) { 46365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->getMachineOpcode() == 46465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman (unsigned)TII->getCallFrameDestroyOpcode()) { 46565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman ++NestLevel; 46665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman MaxNest = std::max(MaxNest, NestLevel); 46765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } else if (N->getMachineOpcode() == 46865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman (unsigned)TII->getCallFrameSetupOpcode()) { 46965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman assert(NestLevel != 0); 47065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman --NestLevel; 47165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (NestLevel == 0) 47265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return N; 47365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 47465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 47565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Otherwise, find the chain and continue climbing. 47665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 47765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->getOperand(i).getValueType() == MVT::Other) { 47865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman N = N->getOperand(i).getNode(); 47965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman goto found_chain_operand; 48065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 48165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return 0; 48265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman found_chain_operand:; 48365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->getOpcode() == ISD::EntryToken) 48465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return 0; 48565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 48665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman} 48765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman 4881b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// Call ReleasePred for each predecessor, then update register live def/gen. 4891b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// Always update LiveRegDefs for a register dependence even if the current SU 4901b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// also defines the register. This effectively create one large live range 4911b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// across a sequence of two-address node. This is important because the 4921b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// entire chain must be scheduled together. Example: 4931b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// 4941b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// flags = (3) add 4951b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// flags = (2) addc flags 4961b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// flags = (1) addc flags 4971b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// 4981b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// results in 4991b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// 5001b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// LiveRegDefs[flags] = 3 5013d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick/// LiveRegGens[flags] = 1 5021b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// 5031b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// If (2) addc is unscheduled, then (1) addc must also be unscheduled to avoid 5041b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// interference on flags. 5053d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trickvoid ScheduleDAGRRList::ReleasePredecessors(SUnit *SU) { 506e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // Bottom up: release predecessors 507228a18e0f220fb85ee06fd5bfa29304e57047ff1Chris Lattner for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 508a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 50954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman ReleasePred(SU, &*I); 51054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isAssignedRegDep()) { 511a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // This is a physical register dependency and it's impossible or 51238036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick // expensive to copy the register. Make sure nothing that can 513a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // clobber the register is scheduled between the predecessor and 514a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // this node. 5153d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick SUnit *RegDef = LiveRegDefs[I->getReg()]; (void)RegDef; 5161b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick assert((!RegDef || RegDef == SU || RegDef == I->getSUnit()) && 5171b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick "interference on register dependence"); 5183d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick LiveRegDefs[I->getReg()] = I->getSUnit(); 5193d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick if (!LiveRegGens[I->getReg()]) { 520086ec9976ff6cee083de618429c78473491d5713Dan Gohman ++NumLiveRegs; 5213d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick LiveRegGens[I->getReg()] = SU; 522a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 523a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 524a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 52565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman 52665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // If we're scheduling a lowered CALLSEQ_END, find the corresponding 52765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // CALLSEQ_BEGIN. Inject an artificial physical register dependence between 52865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // these nodes, to prevent other calls from being interscheduled with them. 52965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned CallResource = TRI->getNumRegs(); 53065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (!LiveRegDefs[CallResource]) 53165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) 53265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (Node->isMachineOpcode() && 53365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman Node->getMachineOpcode() == (unsigned)TII->getCallFrameDestroyOpcode()) { 53465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned NestLevel = 0; 53565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned MaxNest = 0; 53665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SDNode *N = FindCallSeqStart(Node, NestLevel, MaxNest, TII); 53765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman 53865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SUnit *Def = &SUnits[N->getNodeId()]; 5390e6307f6423a7ee39b80f8dc3caef00ad11e0266Eli Friedman CallSeqEndForStart[Def] = SU; 5400e6307f6423a7ee39b80f8dc3caef00ad11e0266Eli Friedman 54165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman ++NumLiveRegs; 54265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegDefs[CallResource] = Def; 54365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegGens[CallResource] = SU; 54465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman break; 54565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 5469e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman} 5479e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman 5482da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// Check to see if any of the pending instructions are ready to issue. If 5492da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// so, add them to the available queue. 5502da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::ReleasePending() { 551c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick if (DisableSchedCycles) { 552a75ce9f5d2236d93c117e861e60e6f3f748c9555Andrew Trick assert(PendingQueue.empty() && "pending instrs not allowed in this mode"); 553a75ce9f5d2236d93c117e861e60e6f3f748c9555Andrew Trick return; 554a75ce9f5d2236d93c117e861e60e6f3f748c9555Andrew Trick } 5552da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 5562da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // If the available queue is empty, it is safe to reset MinAvailableCycle. 5572da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (AvailableQueue->empty()) 5582da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick MinAvailableCycle = UINT_MAX; 5592da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 5602da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Check to see if any of the pending instructions are ready to issue. If 5612da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // so, add them to the available queue. 5622da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick for (unsigned i = 0, e = PendingQueue.size(); i != e; ++i) { 563ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman unsigned ReadyCycle = PendingQueue[i]->getHeight(); 5642da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (ReadyCycle < MinAvailableCycle) 5652da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick MinAvailableCycle = ReadyCycle; 5662da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 5672da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (PendingQueue[i]->isAvailable) { 5682da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (!isReady(PendingQueue[i])) 5692da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick continue; 5702da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->push(PendingQueue[i]); 5712da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 5722da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PendingQueue[i]->isPending = false; 5732da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PendingQueue[i] = PendingQueue.back(); 5742da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PendingQueue.pop_back(); 5752da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick --i; --e; 5762da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 5772da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 5782da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 5792da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// Move the scheduler state forward by the specified number of Cycles. 5802da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::AdvanceToCycle(unsigned NextCycle) { 5812da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (NextCycle <= CurCycle) 5822da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 5832da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 584e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick IssueCount = 0; 5852da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->setCurCycle(NextCycle); 586c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick if (!HazardRec->isEnabled()) { 5872da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Bypass lots of virtual calls in case of long latency. 5882da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CurCycle = NextCycle; 5892da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 5902da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick else { 5912da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick for (; CurCycle != NextCycle; ++CurCycle) { 592ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman HazardRec->RecedeCycle(); 5932da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 5942da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 5952da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // FIXME: Instead of visiting the pending Q each time, set a dirty flag on the 5962da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // available Q to release pending nodes at least once before popping. 5972da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ReleasePending(); 5982da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 5992da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6002da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// Move the scheduler state forward until the specified node's dependents are 6012da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// ready and can be scheduled with no resource conflicts. 6022da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::AdvancePastStalls(SUnit *SU) { 603c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick if (DisableSchedCycles) 6042da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 6052da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 60687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // FIXME: Nodes such as CopyFromReg probably should not advance the current 60787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // cycle. Otherwise, we can wrongly mask real stalls. If the non-machine node 60887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // has predecessors the cycle will be advanced when they are scheduled. 60987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // But given the crude nature of modeling latency though such nodes, we 61087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // currently need to treat these nodes like real instructions. 61187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // if (!SU->getNode() || !SU->getNode()->isMachineOpcode()) return; 61287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 613ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman unsigned ReadyCycle = SU->getHeight(); 6142da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6152da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Bump CurCycle to account for latency. We assume the latency of other 6162da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // available instructions may be hidden by the stall (not a full pipe stall). 6172da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // This updates the hazard recognizer's cycle before reserving resources for 6182da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // this instruction. 6192da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AdvanceToCycle(ReadyCycle); 6202da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6212da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Calls are scheduled in their preceding cycle, so don't conflict with 6222da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // hazards from instructions after the call. EmitNode will reset the 6232da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // scoreboard state before emitting the call. 624ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman if (SU->isCall) 6252da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 6262da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6272da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // FIXME: For resource conflicts in very long non-pipelined stages, we 6282da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // should probably skip ahead here to avoid useless scoreboard checks. 6292da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick int Stalls = 0; 6302da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick while (true) { 6312da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleHazardRecognizer::HazardType HT = 632ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman HazardRec->getHazardType(SU, -Stalls); 6332da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6342da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (HT == ScheduleHazardRecognizer::NoHazard) 6352da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick break; 6362da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6372da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ++Stalls; 6382da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 6392da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AdvanceToCycle(CurCycle + Stalls); 6402da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 6412da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6422da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// Record this SUnit in the HazardRecognizer. 6432da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// Does not update CurCycle. 6442da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::EmitNode(SUnit *SU) { 645c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick if (!HazardRec->isEnabled()) 64624312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick return; 64724312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick 64824312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick // Check for phys reg copy. 64924312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick if (!SU->getNode()) 65024312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick return; 65124312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick 6522da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick switch (SU->getNode()->getOpcode()) { 6532da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick default: 6542da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick assert(SU->getNode()->isMachineOpcode() && 6552da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick "This target-independent node should not be scheduled."); 6562da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick break; 6572da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::MERGE_VALUES: 6582da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::TokenFactor: 659c05d30601ced172b55be81bb529df6be91d6ae15Nadav Rotem case ISD::LIFETIME_START: 660c05d30601ced172b55be81bb529df6be91d6ae15Nadav Rotem case ISD::LIFETIME_END: 6612da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::CopyToReg: 6622da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::CopyFromReg: 6632da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::EH_LABEL: 6642da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Noops don't affect the scoreboard state. Copies are likely to be 6652da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // removed. 6662da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 6672da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::INLINEASM: 6682da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // For inline asm, clear the pipeline state. 6692da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->Reset(); 6702da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 6712da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 672ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman if (SU->isCall) { 6732da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Calls are scheduled with their preceding instructions. For bottom-up 6742da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // scheduling, clear the pipeline state before emitting. 6752da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->Reset(); 6762da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 6772da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6782da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->EmitInstruction(SU); 6792da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 6802da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 68187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trickstatic void resetVRegCycle(SUnit *SU); 68287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 6839e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman/// ScheduleNodeBottomUp - Add the node to the schedule. Decrement the pending 6849e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman/// count of its predecessors. If a predecessor pending count is zero, add it to 6859e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman/// the Available queue. 6862902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trickvoid ScheduleDAGRRList::ScheduleNodeBottomUp(SUnit *SU) { 687c558bf397257f5ef902bdb45a28e622ee2b5b4f2Andrew Trick DEBUG(dbgs() << "\n*** Scheduling [" << CurCycle << "]: "); 6889e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman DEBUG(SU->dump(this)); 6899e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman 69015a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng#ifndef NDEBUG 69115a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng if (CurCycle < SU->getHeight()) 69287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick DEBUG(dbgs() << " Height [" << SU->getHeight() 69387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick << "] pipeline stall!\n"); 69415a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng#endif 69515a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 6962da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // FIXME: Do not modify node height. It may interfere with 6972da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // backtracking. Instead add a "ready cycle" to SUnit. Before scheduling the 69828ed90b95db5f14b60b2cb532a62d407d4faf5e5Eric Christopher // node its ready cycle can aid heuristics, and after scheduling it can 6992da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // indicate the scheduled cycle. 7009e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman SU->setHeightToAtLeast(CurCycle); 7012da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 7022da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Reserve resources for the scheduled intruction. 7032da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick EmitNode(SU); 7042da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 7059e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman Sequence.push_back(SU); 7069e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman 707953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick AvailableQueue->scheduledNode(SU); 70837944985a569f8c2b0d75dafd9e2739a9887ac5dChris Lattner 709e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick // If HazardRec is disabled, and each inst counts as one cycle, then 71087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // advance CurCycle before ReleasePredecessors to avoid useless pushes to 711a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick // PendingQueue for schedulers that implement HasReadyFilter. 712e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (!HazardRec->isEnabled() && AvgIPC < 2) 713a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick AdvanceToCycle(CurCycle + 1); 714a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick 7151b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick // Update liveness of predecessors before successors to avoid treating a 7161b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick // two-address node as a live range def. 7173d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick ReleasePredecessors(SU); 718a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 719a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // Release all the implicit physical register defs that are live. 720a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 721a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 7221b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick // LiveRegDegs[I->getReg()] != SU when SU is a two-address node. 7231b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick if (I->isAssignedRegDep() && LiveRegDefs[I->getReg()] == SU) { 7241b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!"); 7251b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick --NumLiveRegs; 7261b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick LiveRegDefs[I->getReg()] = NULL; 7273d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick LiveRegGens[I->getReg()] = NULL; 728a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 729a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 73065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Release the special call resource dependence, if this is the beginning 73165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // of a call. 73265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned CallResource = TRI->getNumRegs(); 73365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (LiveRegDefs[CallResource] == SU) 73465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (const SDNode *SUNode = SU->getNode(); SUNode; 73565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SUNode = SUNode->getGluedNode()) { 73665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (SUNode->isMachineOpcode() && 73765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SUNode->getMachineOpcode() == (unsigned)TII->getCallFrameSetupOpcode()) { 73865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!"); 73965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman --NumLiveRegs; 74065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegDefs[CallResource] = NULL; 74165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegGens[CallResource] = NULL; 74265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 74365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 744a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 74587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick resetVRegCycle(SU); 74687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 747e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng SU->isScheduled = true; 7482da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 7492da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Conditions under which the scheduler should eagerly advance the cycle: 7502da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // (1) No available instructions 7512da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // (2) All pipelines full, so available instructions must have hazards. 7522da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // 75387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // If HazardRec is disabled, the cycle was pre-advanced before calling 75487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // ReleasePredecessors. In that case, IssueCount should remain 0. 755a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick // 756a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick // Check AvailableQueue after ReleasePredecessors in case of zero latency. 75787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (HazardRec->isEnabled() || AvgIPC > 1) { 75887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (SU->getNode() && SU->getNode()->isMachineOpcode()) 75987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick ++IssueCount; 76087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if ((HazardRec->isEnabled() && HazardRec->atIssueLimit()) 76187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick || (!HazardRec->isEnabled() && IssueCount == AvgIPC)) 76287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick AdvanceToCycle(CurCycle + 1); 76387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick } 764e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 765e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 766a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// CapturePred - This does the opposite of ReleasePred. Since SU is being 767a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// unscheduled, incrcease the succ left count of its predecessors. Remove 768a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// them from AvailableQueue if necessary. 76938036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trickvoid ScheduleDAGRRList::CapturePred(SDep *PredEdge) { 77054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *PredSU = PredEdge->getSUnit(); 771a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng if (PredSU->isAvailable) { 772a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng PredSU->isAvailable = false; 773a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng if (!PredSU->isPending) 774a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng AvailableQueue->remove(PredSU); 775a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 776a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 777c277ab08a24d2dbe9b4ff1a9154ea6115ed6a4e3Reid Kleckner assert(PredSU->NumSuccsLeft < UINT_MAX && "NumSuccsLeft will overflow!"); 77874d2fd8dd847e0ebccef30e2c5907ff09495d518Evan Cheng ++PredSU->NumSuccsLeft; 779a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng} 780a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 781a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// UnscheduleNodeBottomUp - Remove the node from the schedule, update its and 782a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// its predecessor states to reflect the change. 783a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Chengvoid ScheduleDAGRRList::UnscheduleNodeBottomUp(SUnit *SU) { 784e492ae13edd83b120d665c0503cf4de2925b5e56David Greene DEBUG(dbgs() << "*** Unscheduling [" << SU->getHeight() << "]: "); 7853cc6243ddfdba3ad64035b919c88b09773a60880Dan Gohman DEBUG(SU->dump(this)); 786a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 787a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 788a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 78954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman CapturePred(&*I); 7903d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick if (I->isAssignedRegDep() && SU == LiveRegGens[I->getReg()]){ 791086ec9976ff6cee083de618429c78473491d5713Dan Gohman assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!"); 79254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman assert(LiveRegDefs[I->getReg()] == I->getSUnit() && 793a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng "Physical register dependency violated?"); 794086ec9976ff6cee083de618429c78473491d5713Dan Gohman --NumLiveRegs; 79554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman LiveRegDefs[I->getReg()] = NULL; 7963d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick LiveRegGens[I->getReg()] = NULL; 797a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 798a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 799a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 80065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Reclaim the special call resource dependence, if this is the beginning 80165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // of a call. 80265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned CallResource = TRI->getNumRegs(); 80365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (const SDNode *SUNode = SU->getNode(); SUNode; 80465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SUNode = SUNode->getGluedNode()) { 80565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (SUNode->isMachineOpcode() && 80665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SUNode->getMachineOpcode() == (unsigned)TII->getCallFrameSetupOpcode()) { 80765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman ++NumLiveRegs; 80865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegDefs[CallResource] = SU; 8090e6307f6423a7ee39b80f8dc3caef00ad11e0266Eli Friedman LiveRegGens[CallResource] = CallSeqEndForStart[SU]; 81065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 81165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 81265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman 81365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Release the special call resource dependence, if this is the end 81465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // of a call. 81565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (LiveRegGens[CallResource] == SU) 81665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (const SDNode *SUNode = SU->getNode(); SUNode; 81765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SUNode = SUNode->getGluedNode()) { 81865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (SUNode->isMachineOpcode() && 81965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SUNode->getMachineOpcode() == (unsigned)TII->getCallFrameDestroyOpcode()) { 82065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!"); 82165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman --NumLiveRegs; 82265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegDefs[CallResource] = NULL; 82365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegGens[CallResource] = NULL; 82465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 82565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 82665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman 827a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 828a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 82954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isAssignedRegDep()) { 83030c44e18bf28c3f5feda56e97695e2b72de7fedbEli Friedman if (!LiveRegDefs[I->getReg()]) 83130c44e18bf28c3f5feda56e97695e2b72de7fedbEli Friedman ++NumLiveRegs; 8321b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick // This becomes the nearest def. Note that an earlier def may still be 8331b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick // pending if this is a two-address node. 8341b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick LiveRegDefs[I->getReg()] = SU; 8353d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick if (LiveRegGens[I->getReg()] == NULL || 8363d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick I->getSUnit()->getHeight() < LiveRegGens[I->getReg()]->getHeight()) 8373d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick LiveRegGens[I->getReg()] = I->getSUnit(); 838a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 839a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 8402da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (SU->getHeight() < MinAvailableCycle) 8412da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick MinAvailableCycle = SU->getHeight(); 842a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 8433f23744df4809eba94284e601e81489212c974d4Dan Gohman SU->setHeightDirty(); 844a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng SU->isScheduled = false; 845a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng SU->isAvailable = true; 846c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick if (!DisableSchedCycles && AvailableQueue->hasReadyFilter()) { 8472da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Don't make available until backtracking is complete. 8482da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SU->isPending = true; 8492da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PendingQueue.push_back(SU); 8502da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 8512da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick else { 8522da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->push(SU); 8532da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 854953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick AvailableQueue->unscheduledNode(SU); 855a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng} 856a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 8572da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// After backtracking, the hazard checker needs to be restored to a state 858c8e41c591741b3da1077f7000274ad040bef8002Sylvestre Ledru/// corresponding the current cycle. 8592da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::RestoreHazardCheckerBottomUp() { 8602da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->Reset(); 8612da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 8622da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned LookAhead = std::min((unsigned)Sequence.size(), 8632da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->getMaxLookAhead()); 8642da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (LookAhead == 0) 8652da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 8662da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 8672da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick std::vector<SUnit*>::const_iterator I = (Sequence.end() - LookAhead); 8682da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned HazardCycle = (*I)->getHeight(); 8692da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick for (std::vector<SUnit*>::const_iterator E = Sequence.end(); I != E; ++I) { 8702da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SUnit *SU = *I; 8712da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick for (; SU->getHeight() > HazardCycle; ++HazardCycle) { 8722da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->RecedeCycle(); 8732da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 8742da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick EmitNode(SU); 8752da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 8762da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 8772da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 87842d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng/// BacktrackBottomUp - Backtrack scheduling to a previous cycle specified in 8791cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohman/// BTCycle in order to schedule a specific node. 8802da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::BacktrackBottomUp(SUnit *SU, SUnit *BtSU) { 8812da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SUnit *OldSU = Sequence.back(); 8822da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick while (true) { 883a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng Sequence.pop_back(); 884a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng if (SU->isSucc(OldSU)) 88542d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng // Don't try to remove SU from AvailableQueue. 88642d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng SU->isAvailable = false; 8872da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // FIXME: use ready cycle instead of height 8882da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CurCycle = OldSU->getHeight(); 889a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng UnscheduleNodeBottomUp(OldSU); 89015a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng AvailableQueue->setCurCycle(CurCycle); 8912da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (OldSU == BtSU) 8922da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick break; 8932da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick OldSU = Sequence.back(); 894a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 895a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 8961cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohman assert(!SU->isSucc(OldSU) && "Something is wrong!"); 897a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng 8982da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick RestoreHazardCheckerBottomUp(); 8992da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 900a75ce9f5d2236d93c117e861e60e6f3f748c9555Andrew Trick ReleasePending(); 9012da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 902a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng ++NumBacktracks; 903a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng} 904a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 9055ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Chengstatic bool isOperandOf(const SUnit *SU, SDNode *N) { 9065ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng for (const SDNode *SUNode = SU->getNode(); SUNode; 90729d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner SUNode = SUNode->getGluedNode()) { 9085ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng if (SUNode->isOperandOf(N)) 9095ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng return true; 9105ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng } 9115ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng return false; 9125ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng} 9135ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng 914f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng/// CopyAndMoveSuccessors - Clone the specified node and move its scheduled 915f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng/// successors to the newly created node. 916f10c973797cf79da802f9b0118543cbd50954c9cEvan ChengSUnit *ScheduleDAGRRList::CopyAndMoveSuccessors(SUnit *SU) { 917550f5afb68ce8f034991863cac65bef22a6554daDan Gohman SDNode *N = SU->getNode(); 91842d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng if (!N) 919f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng return NULL; 92042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 92124312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick if (SU->getNode()->getGluedNode()) 92224312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick return NULL; 92324312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick 924f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SUnit *NewSU; 925f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng bool TryUnfold = false; 926d5cb5a462b6fd91bf54116c3eefc3b046489c414Evan Cheng for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) { 927e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = N->getValueType(i); 928f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner if (VT == MVT::Glue) 929d5cb5a462b6fd91bf54116c3eefc3b046489c414Evan Cheng return NULL; 930825b72b0571821bf2d378749f69d6c4cfb52d2f9Owen Anderson else if (VT == MVT::Other) 931d5cb5a462b6fd91bf54116c3eefc3b046489c414Evan Cheng TryUnfold = true; 932d5cb5a462b6fd91bf54116c3eefc3b046489c414Evan Cheng } 933a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 934475871a144eb604ddaf37503397ba0941442e5fbDan Gohman const SDValue &Op = N->getOperand(i); 935e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = Op.getNode()->getValueType(Op.getResNo()); 936f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner if (VT == MVT::Glue) 937f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng return NULL; 938a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 939a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 940f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng if (TryUnfold) { 9414c8c83022b501759d8559e224c84ae2a9921ba41Dan Gohman SmallVector<SDNode*, 2> NewNodes; 942a23b3b803e3c65e84d6cadaa221de8b256cbe28dDan Gohman if (!TII->unfoldMemoryOperand(*DAG, N, NewNodes)) 943f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng return NULL; 9442d496897934adfc0e3e1563dd64d9a7122971e22Pete Cooper 9452d496897934adfc0e3e1563dd64d9a7122971e22Pete Cooper // unfolding an x86 DEC64m operation results in store, dec, load which 9462d496897934adfc0e3e1563dd64d9a7122971e22Pete Cooper // can't be handled here so quit 9472d496897934adfc0e3e1563dd64d9a7122971e22Pete Cooper if (NewNodes.size() == 3) 9482d496897934adfc0e3e1563dd64d9a7122971e22Pete Cooper return NULL; 949f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 95015a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng DEBUG(dbgs() << "Unfolding SU #" << SU->NodeNum << "\n"); 951f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng assert(NewNodes.size() == 2 && "Expected a load folding node!"); 952f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 953f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng N = NewNodes[1]; 954f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SDNode *LoadNode = NewNodes[0]; 955f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng unsigned NumVals = N->getNumValues(); 956550f5afb68ce8f034991863cac65bef22a6554daDan Gohman unsigned OldNumVals = SU->getNode()->getNumValues(); 957f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (unsigned i = 0; i != NumVals; ++i) 958550f5afb68ce8f034991863cac65bef22a6554daDan Gohman DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), i), SDValue(N, i)); 959550f5afb68ce8f034991863cac65bef22a6554daDan Gohman DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), OldNumVals-1), 960a23b3b803e3c65e84d6cadaa221de8b256cbe28dDan Gohman SDValue(LoadNode, 1)); 961f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 962b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman // LoadNode may already exist. This can happen when there is another 963b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman // load from the same location and producing the same type of value 964b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman // but it has different alignment or volatileness. 965b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman bool isNewLoad = true; 966b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman SUnit *LoadSU; 967b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman if (LoadNode->getNodeId() != -1) { 968b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman LoadSU = &SUnits[LoadNode->getNodeId()]; 969b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman isNewLoad = false; 970b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman } else { 971b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman LoadSU = CreateNewSUnit(LoadNode); 972b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman LoadNode->setNodeId(LoadSU->NodeNum); 97392e946630d5f9bb092853b93501387dd216899b9Andrew Trick 97492e946630d5f9bb092853b93501387dd216899b9Andrew Trick InitNumRegDefsLeft(LoadSU); 975953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick computeLatency(LoadSU); 976b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman } 977b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman 978e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *NewSU = CreateNewSUnit(N); 97994d7a5f8156e62532870fbaf197377b34e52ff2aDan Gohman assert(N->getNodeId() == -1 && "Node already inserted!"); 98094d7a5f8156e62532870fbaf197377b34e52ff2aDan Gohman N->setNodeId(NewSU->NodeNum); 98138036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 982e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng const MCInstrDesc &MCID = TII->get(N->getMachineOpcode()); 983e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng for (unsigned i = 0; i != MCID.getNumOperands(); ++i) { 984e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng if (MCID.getOperandConstraint(i, MCOI::TIED_TO) != -1) { 985f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng NewSU->isTwoAddress = true; 986f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng break; 987f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 988f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 989e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng if (MCID.isCommutable()) 990f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng NewSU->isCommutable = true; 99192e946630d5f9bb092853b93501387dd216899b9Andrew Trick 99292e946630d5f9bb092853b93501387dd216899b9Andrew Trick InitNumRegDefsLeft(NewSU); 993953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick computeLatency(NewSU); 994f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 995fa9afef7eaa0865fb9f3489a68a04332b232ed82Dan Gohman // Record all the edges to and from the old SU, by category. 99616e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman SmallVector<SDep, 4> ChainPreds; 997f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SmallVector<SDep, 4> ChainSuccs; 998f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SmallVector<SDep, 4> LoadPreds; 999f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SmallVector<SDep, 4> NodePreds; 1000f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SmallVector<SDep, 4> NodeSuccs; 1001f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 1002f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng I != E; ++I) { 100354e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isCtrl()) 100416e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman ChainPreds.push_back(*I); 10055ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng else if (isOperandOf(I->getSUnit(), LoadNode)) 100654e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman LoadPreds.push_back(*I); 1007f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng else 100854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman NodePreds.push_back(*I); 1009f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 1010f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 1011f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng I != E; ++I) { 101254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isCtrl()) 101354e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman ChainSuccs.push_back(*I); 1014f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng else 101554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman NodeSuccs.push_back(*I); 1016f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 101742d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 1018fa9afef7eaa0865fb9f3489a68a04332b232ed82Dan Gohman // Now assign edges to the newly-created nodes. 101916e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman for (unsigned i = 0, e = ChainPreds.size(); i != e; ++i) { 102016e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman const SDep &Pred = ChainPreds[i]; 102116e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman RemovePred(SU, Pred); 102280792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman if (isNewLoad) 102316e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman AddPred(LoadSU, Pred); 1024e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein } 1025f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (unsigned i = 0, e = LoadPreds.size(); i != e; ++i) { 102654e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman const SDep &Pred = LoadPreds[i]; 102754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(SU, Pred); 102816e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman if (isNewLoad) 102954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(LoadSU, Pred); 1030f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 1031f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (unsigned i = 0, e = NodePreds.size(); i != e; ++i) { 103254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman const SDep &Pred = NodePreds[i]; 103354e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(SU, Pred); 103454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(NewSU, Pred); 1035f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 1036f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (unsigned i = 0, e = NodeSuccs.size(); i != e; ++i) { 103754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SDep D = NodeSuccs[i]; 103854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *SuccDep = D.getSUnit(); 103954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(SU); 104054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(SuccDep, D); 104154e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(NewSU); 104254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(SuccDep, D); 104392e946630d5f9bb092853b93501387dd216899b9Andrew Trick // Balance register pressure. 104492e946630d5f9bb092853b93501387dd216899b9Andrew Trick if (AvailableQueue->tracksRegPressure() && SuccDep->isScheduled 104592e946630d5f9bb092853b93501387dd216899b9Andrew Trick && !D.isCtrl() && NewSU->NumRegDefsLeft > 0) 104692e946630d5f9bb092853b93501387dd216899b9Andrew Trick --NewSU->NumRegDefsLeft; 1047f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 1048f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (unsigned i = 0, e = ChainSuccs.size(); i != e; ++i) { 104954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SDep D = ChainSuccs[i]; 105054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *SuccDep = D.getSUnit(); 105154e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(SU); 105254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(SuccDep, D); 1053e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein if (isNewLoad) { 105454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(LoadSU); 105554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(SuccDep, D); 1056e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein } 105738036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick } 1058fa9afef7eaa0865fb9f3489a68a04332b232ed82Dan Gohman 1059fa9afef7eaa0865fb9f3489a68a04332b232ed82Dan Gohman // Add a data dependency to reflect that NewSU reads the value defined 1060fa9afef7eaa0865fb9f3489a68a04332b232ed82Dan Gohman // by LoadSU. 1061a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick SDep D(LoadSU, SDep::Data, 0); 1062a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick D.setLatency(LoadSU->Latency); 1063a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick AddPred(NewSU, D); 1064f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 1065beec823d4bba22b1c0c6658d2b3e71cd64a70e2eEvan Cheng if (isNewLoad) 1066beec823d4bba22b1c0c6658d2b3e71cd64a70e2eEvan Cheng AvailableQueue->addNode(LoadSU); 1067f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng AvailableQueue->addNode(NewSU); 1068f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 1069f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng ++NumUnfolds; 1070f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 1071f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng if (NewSU->NumSuccsLeft == 0) { 1072f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng NewSU->isAvailable = true; 1073f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng return NewSU; 1074beec823d4bba22b1c0c6658d2b3e71cd64a70e2eEvan Cheng } 1075beec823d4bba22b1c0c6658d2b3e71cd64a70e2eEvan Cheng SU = NewSU; 1076f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 1077f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 107815a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng DEBUG(dbgs() << " Duplicating SU #" << SU->NodeNum << "\n"); 1079e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein NewSU = CreateClone(SU); 1080a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1081a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // New SUnit has the exact same predecessors. 1082a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 1083a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) 10843f23744df4809eba94284e601e81489212c974d4Dan Gohman if (!I->isArtificial()) 108554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(NewSU, *I); 1086a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1087a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // Only copy scheduled successors. Cut them from old node's successor 1088a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // list and move them over. 108954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SmallVector<std::pair<SUnit *, SDep>, 4> DelDeps; 1090a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 1091a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 109254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isArtificial()) 1093a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng continue; 109454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *SuccSU = I->getSUnit(); 109554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (SuccSU->isScheduled) { 109654e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SDep D = *I; 109754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(NewSU); 109854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(SuccSU, D); 109954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(SU); 110054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman DelDeps.push_back(std::make_pair(SuccSU, D)); 1101a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 1102a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 11033f23744df4809eba94284e601e81489212c974d4Dan Gohman for (unsigned i = 0, e = DelDeps.size(); i != e; ++i) 110454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(DelDeps[i].first, DelDeps[i].second); 1105a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1106a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng AvailableQueue->updateNode(SU); 1107a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng AvailableQueue->addNode(NewSU); 1108a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1109a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng ++NumDups; 1110a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng return NewSU; 1111a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng} 1112a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1113c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng/// InsertCopiesAndMoveSuccs - Insert register copies and move all 1114c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng/// scheduled successors of the given SUnit to the last copy. 1115c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Chengvoid ScheduleDAGRRList::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg, 1116c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng const TargetRegisterClass *DestRC, 1117c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng const TargetRegisterClass *SrcRC, 1118a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng SmallVector<SUnit*, 2> &Copies) { 1119e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *CopyFromSU = CreateNewSUnit(NULL); 112042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng CopyFromSU->CopySrcRC = SrcRC; 112142d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng CopyFromSU->CopyDstRC = DestRC; 112242d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 1123e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *CopyToSU = CreateNewSUnit(NULL); 112442d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng CopyToSU->CopySrcRC = DestRC; 112542d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng CopyToSU->CopyDstRC = SrcRC; 112642d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 112742d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng // Only copy scheduled successors. Cut them from old node's successor 112842d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng // list and move them over. 112954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SmallVector<std::pair<SUnit *, SDep>, 4> DelDeps; 113042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 113142d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng I != E; ++I) { 113254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isArtificial()) 113342d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng continue; 113454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *SuccSU = I->getSUnit(); 113554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (SuccSU->isScheduled) { 113654e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SDep D = *I; 113754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(CopyToSU); 113854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(SuccSU, D); 113954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman DelDeps.push_back(std::make_pair(SuccSU, *I)); 114042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng } 1141bd47a4a307a54dab25956430442cf4ac103e4158Andrew Trick else { 1142bd47a4a307a54dab25956430442cf4ac103e4158Andrew Trick // Avoid scheduling the def-side copy before other successors. Otherwise 1143bd47a4a307a54dab25956430442cf4ac103e4158Andrew Trick // we could introduce another physreg interference on the copy and 1144bd47a4a307a54dab25956430442cf4ac103e4158Andrew Trick // continue inserting copies indefinitely. 1145a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick AddPred(SuccSU, SDep(CopyFromSU, SDep::Artificial)); 1146bd47a4a307a54dab25956430442cf4ac103e4158Andrew Trick } 114742d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng } 1148c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng for (unsigned i = 0, e = DelDeps.size(); i != e; ++i) 114954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(DelDeps[i].first, DelDeps[i].second); 115042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 1151a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick SDep FromDep(SU, SDep::Data, Reg); 1152a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick FromDep.setLatency(SU->Latency); 1153a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick AddPred(CopyFromSU, FromDep); 1154a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick SDep ToDep(CopyFromSU, SDep::Data, 0); 1155a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick ToDep.setLatency(CopyFromSU->Latency); 1156a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick AddPred(CopyToSU, ToDep); 115742d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 115842d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng AvailableQueue->updateNode(SU); 115942d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng AvailableQueue->addNode(CopyFromSU); 116042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng AvailableQueue->addNode(CopyToSU); 1161a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng Copies.push_back(CopyFromSU); 1162a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng Copies.push_back(CopyToSU); 116342d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 1164c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng ++NumPRCopies; 116542d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng} 116642d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 116742d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng/// getPhysicalRegisterVT - Returns the ValueType of the physical register 116842d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng/// definition of the specified node. 116942d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng/// FIXME: Move to SelectionDAG? 1170e50ed30282bb5b4a9ed952580523f2dda16215acOwen Andersonstatic EVT getPhysicalRegisterVT(SDNode *N, unsigned Reg, 117183ec4b6711980242ef3c55a4fa36b2d7a39c1bfbDuncan Sands const TargetInstrInfo *TII) { 1172e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng const MCInstrDesc &MCID = TII->get(N->getMachineOpcode()); 1173e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng assert(MCID.ImplicitDefs && "Physical reg def must be in implicit def list!"); 1174e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng unsigned NumRes = MCID.getNumDefs(); 1175fac259814923d091942b230e7bd002a8d1130bc3Craig Topper for (const uint16_t *ImpDef = MCID.getImplicitDefs(); *ImpDef; ++ImpDef) { 117642d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng if (Reg == *ImpDef) 117742d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng break; 117842d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng ++NumRes; 117942d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng } 118042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng return N->getValueType(NumRes); 118142d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng} 118242d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 1183599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng/// CheckForLiveRegDef - Return true and update live register vector if the 1184599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng/// specified register def of the specified SUnit clobbers any "live" registers. 1185142d21c861c0b686e38a515b1271f4157cd24004Chris Lattnerstatic void CheckForLiveRegDef(SUnit *SU, unsigned Reg, 1186599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng std::vector<SUnit*> &LiveRegDefs, 1187599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng SmallSet<unsigned, 4> &RegAdded, 1188599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng SmallVector<unsigned, 4> &LRegs, 1189599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng const TargetRegisterInfo *TRI) { 1190396618b43a85e12d290a90b181c6af5d7c0c5f11Jakob Stoklund Olesen for (MCRegAliasIterator AliasI(Reg, TRI, true); AliasI.isValid(); ++AliasI) { 1191cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick 1192cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick // Check if Ref is live. 11939d507aec07c7e22c6ba83dfd75e23c8630cd25cdAndrew Trick if (!LiveRegDefs[*AliasI]) continue; 1194cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick 1195cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick // Allow multiple uses of the same def. 11969d507aec07c7e22c6ba83dfd75e23c8630cd25cdAndrew Trick if (LiveRegDefs[*AliasI] == SU) continue; 1197cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick 1198cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick // Add Reg to the set of interfering live regs. 11999d507aec07c7e22c6ba83dfd75e23c8630cd25cdAndrew Trick if (RegAdded.insert(*AliasI)) { 12009d507aec07c7e22c6ba83dfd75e23c8630cd25cdAndrew Trick LRegs.push_back(*AliasI); 12019d507aec07c7e22c6ba83dfd75e23c8630cd25cdAndrew Trick } 1202599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng } 1203599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng} 1204599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng 120516a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen/// CheckForLiveRegDefMasked - Check for any live physregs that are clobbered 120616a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen/// by RegMask, and add them to LRegs. 120716a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesenstatic void CheckForLiveRegDefMasked(SUnit *SU, const uint32_t *RegMask, 120816a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen std::vector<SUnit*> &LiveRegDefs, 120916a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen SmallSet<unsigned, 4> &RegAdded, 121016a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen SmallVector<unsigned, 4> &LRegs) { 121116a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen // Look at all live registers. Skip Reg0 and the special CallResource. 121216a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen for (unsigned i = 1, e = LiveRegDefs.size()-1; i != e; ++i) { 121316a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (!LiveRegDefs[i]) continue; 121416a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (LiveRegDefs[i] == SU) continue; 121516a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (!MachineOperand::clobbersPhysReg(RegMask, i)) continue; 121616a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (RegAdded.insert(i)) 121716a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen LRegs.push_back(i); 121816a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen } 121916a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen} 122016a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen 122116a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen/// getNodeRegMask - Returns the register mask attached to an SDNode, if any. 122216a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesenstatic const uint32_t *getNodeRegMask(const SDNode *N) { 122316a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 122416a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (const RegisterMaskSDNode *Op = 122516a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen dyn_cast<RegisterMaskSDNode>(N->getOperand(i).getNode())) 122616a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen return Op->getRegMask(); 122716a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen return NULL; 122816a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen} 122916a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen 1230a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// DelayForLiveRegsBottomUp - Returns true if it is necessary to delay 1231a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// scheduling of the given node to satisfy live physical register dependencies. 1232a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// If the specific node is the last one that's available to schedule, do 1233a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// whatever is necessary (i.e. backtracking or cloning) to make it possible. 1234142d21c861c0b686e38a515b1271f4157cd24004Chris Lattnerbool ScheduleDAGRRList:: 1235142d21c861c0b686e38a515b1271f4157cd24004Chris LattnerDelayForLiveRegsBottomUp(SUnit *SU, SmallVector<unsigned, 4> &LRegs) { 1236086ec9976ff6cee083de618429c78473491d5713Dan Gohman if (NumLiveRegs == 0) 1237a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng return false; 1238a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1239cd1c00cc6521c265784ffc7a1b5baf4ef64d80bcEvan Cheng SmallSet<unsigned, 4> RegAdded; 1240a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // If this node would clobber any "live" register, then it's not ready. 1241feac09801b5c03412d452e685570baff6eb84c88Andrew Trick // 1242feac09801b5c03412d452e685570baff6eb84c88Andrew Trick // If SU is the currently live definition of the same register that it uses, 1243feac09801b5c03412d452e685570baff6eb84c88Andrew Trick // then we are free to schedule it. 1244a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 1245a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 1246feac09801b5c03412d452e685570baff6eb84c88Andrew Trick if (I->isAssignedRegDep() && LiveRegDefs[I->getReg()] != SU) 1247599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng CheckForLiveRegDef(I->getSUnit(), I->getReg(), LiveRegDefs, 1248599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng RegAdded, LRegs, TRI); 1249a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 1250a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 125129d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) { 1252599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng if (Node->getOpcode() == ISD::INLINEASM) { 1253599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng // Inline asm can clobber physical defs. 1254599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng unsigned NumOps = Node->getNumOperands(); 1255f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue) 125629d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner --NumOps; // Ignore the glue operand. 1257599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng 1258decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) { 1259599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng unsigned Flags = 1260599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue(); 1261decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags); 1262599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng 1263599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng ++i; // Skip the ID value. 1264decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner if (InlineAsm::isRegDefKind(Flags) || 1265f792fa90f1125553008659c743cba85b9b5d2e5eJakob Stoklund Olesen InlineAsm::isRegDefEarlyClobberKind(Flags) || 1266f792fa90f1125553008659c743cba85b9b5d2e5eJakob Stoklund Olesen InlineAsm::isClobberKind(Flags)) { 1267599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng // Check for def of register or earlyclobber register. 1268599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng for (; NumVals; --NumVals, ++i) { 1269599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg(); 1270599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng if (TargetRegisterInfo::isPhysicalRegister(Reg)) 1271599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng CheckForLiveRegDef(SU, Reg, LiveRegDefs, RegAdded, LRegs, TRI); 1272599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng } 1273599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng } else 1274599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng i += NumVals; 1275599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng } 1276599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng continue; 1277599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng } 1278599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng 1279d23e0f81bc76902052e9198cad3a0d87a412a632Dan Gohman if (!Node->isMachineOpcode()) 1280a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng continue; 128165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // If we're in the middle of scheduling a call, don't begin scheduling 128265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // another call. Also, don't allow any physical registers to be live across 128365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // the call. 128465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (Node->getMachineOpcode() == (unsigned)TII->getCallFrameDestroyOpcode()) { 128565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Check the special calling-sequence resource. 128665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned CallResource = TRI->getNumRegs(); 128765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (LiveRegDefs[CallResource]) { 128865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SDNode *Gen = LiveRegGens[CallResource]->getNode(); 128965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman while (SDNode *Glued = Gen->getGluedNode()) 129065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman Gen = Glued; 129165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (!IsChainDependent(Gen, Node, 0, TII) && RegAdded.insert(CallResource)) 129265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LRegs.push_back(CallResource); 129365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 129465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 129516a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (const uint32_t *RegMask = getNodeRegMask(Node)) 129616a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen CheckForLiveRegDefMasked(SU, RegMask, LiveRegDefs, RegAdded, LRegs); 129716a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen 1298e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng const MCInstrDesc &MCID = TII->get(Node->getMachineOpcode()); 1299e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng if (!MCID.ImplicitDefs) 1300a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng continue; 1301fac259814923d091942b230e7bd002a8d1130bc3Craig Topper for (const uint16_t *Reg = MCID.getImplicitDefs(); *Reg; ++Reg) 1302599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng CheckForLiveRegDef(SU, *Reg, LiveRegDefs, RegAdded, LRegs, TRI); 1303a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 130438036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1305a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng return !LRegs.empty(); 1306e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 1307e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 13082902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick/// Return a node that can be scheduled in this cycle. Requirements: 13092902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick/// (1) Ready: latency has been satisfied 13102da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// (2) No Hazards: resources are available 13112902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick/// (3) No Interferences: may unschedule to break register interferences. 13122902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew TrickSUnit *ScheduleDAGRRList::PickNodeToScheduleBottomUp() { 13132902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SmallVector<SUnit*, 4> Interferences; 13142902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick DenseMap<SUnit*, SmallVector<unsigned, 4> > LRegsMap; 13152902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 13162902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *CurSU = AvailableQueue->pop(); 13172902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick while (CurSU) { 13182902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SmallVector<unsigned, 4> LRegs; 13192902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick if (!DelayForLiveRegsBottomUp(CurSU, LRegs)) 13202902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick break; 13212902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick LRegsMap.insert(std::make_pair(CurSU, LRegs)); 13222902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 13232902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurSU->isPending = true; // This SU is not in AvailableQueue right now. 13242902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick Interferences.push_back(CurSU); 13252902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurSU = AvailableQueue->pop(); 13262902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 13272902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick if (CurSU) { 13282902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Add the nodes that aren't ready back onto the available list. 13292902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick for (unsigned i = 0, e = Interferences.size(); i != e; ++i) { 13302902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick Interferences[i]->isPending = false; 13312902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick assert(Interferences[i]->isAvailable && "must still be available"); 13322902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick AvailableQueue->push(Interferences[i]); 13332902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 13342902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick return CurSU; 13352902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 13362902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 13372902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // All candidates are delayed due to live physical reg dependencies. 13382902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Try backtracking, code duplication, or inserting cross class copies 13392902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // to resolve it. 13402902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick for (unsigned i = 0, e = Interferences.size(); i != e; ++i) { 13412902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *TrySU = Interferences[i]; 13422902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SmallVector<unsigned, 4> &LRegs = LRegsMap[TrySU]; 13432902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 13442902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Try unscheduling up to the point where it's safe to schedule 13452902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // this node. 13462da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SUnit *BtSU = NULL; 13472da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned LiveCycle = UINT_MAX; 13482902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick for (unsigned j = 0, ee = LRegs.size(); j != ee; ++j) { 13492902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick unsigned Reg = LRegs[j]; 13502da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (LiveRegGens[Reg]->getHeight() < LiveCycle) { 13512da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick BtSU = LiveRegGens[Reg]; 13522da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick LiveCycle = BtSU->getHeight(); 13532da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 13542902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 13552da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (!WillCreateCycle(TrySU, BtSU)) { 13562da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick BacktrackBottomUp(TrySU, BtSU); 13572902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 13582902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Force the current node to be scheduled before the node that 13592902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // requires the physical reg dep. 13602da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (BtSU->isAvailable) { 13612da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick BtSU->isAvailable = false; 13622da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (!BtSU->isPending) 13632da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->remove(BtSU); 13642902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 1365a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick AddPred(TrySU, SDep(BtSU, SDep::Artificial)); 13662902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 13672902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // If one or more successors has been unscheduled, then the current 13682902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // node is no longer avaialable. Schedule a successor that's now 13692902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // available instead. 13702902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick if (!TrySU->isAvailable) { 13712902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurSU = AvailableQueue->pop(); 13722902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 13732902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick else { 13742902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurSU = TrySU; 13752902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick TrySU->isPending = false; 13762902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick Interferences.erase(Interferences.begin()+i); 13772902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 13782902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick break; 13792902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 13802902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 13812902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 13822902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick if (!CurSU) { 13832902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Can't backtrack. If it's too expensive to copy the value, then try 13842902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // duplicate the nodes that produces these "too expensive to copy" 13852902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // values to break the dependency. In case even that doesn't work, 13862902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // insert cross class copies. 13872902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // If it's not too expensive, i.e. cost != -1, issue copies. 13882902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *TrySU = Interferences[0]; 13892902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SmallVector<unsigned, 4> &LRegs = LRegsMap[TrySU]; 13902902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick assert(LRegs.size() == 1 && "Can't handle this yet!"); 13912902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick unsigned Reg = LRegs[0]; 13922902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *LRDef = LiveRegDefs[Reg]; 13932902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick EVT VT = getPhysicalRegisterVT(LRDef->getNode(), Reg, TII); 13942902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick const TargetRegisterClass *RC = 13952902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick TRI->getMinimalPhysRegClass(Reg, VT); 13962902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick const TargetRegisterClass *DestRC = TRI->getCrossCopyRegClass(RC); 13972902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 1398b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng // If cross copy register class is the same as RC, then it must be possible 1399b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng // copy the value directly. Do not try duplicate the def. 1400b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng // If cross copy register class is not the same as RC, then it's possible to 1401b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng // copy the value but it require cross register class copies and it is 1402b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng // expensive. 1403b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng // If cross copy register class is null, then it's not possible to copy 1404b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng // the value at all. 14052902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *NewDef = 0; 1406b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng if (DestRC != RC) { 14072902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick NewDef = CopyAndMoveSuccessors(LRDef); 1408b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng if (!DestRC && !NewDef) 1409b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng report_fatal_error("Can't handle live physical register dependency!"); 1410b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng } 14112902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick if (!NewDef) { 14122902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Issue copies, these can be expensive cross register class copies. 14132902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SmallVector<SUnit*, 2> Copies; 14142902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies); 14152902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick DEBUG(dbgs() << " Adding an edge from SU #" << TrySU->NodeNum 14162902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick << " to SU #" << Copies.front()->NodeNum << "\n"); 1417a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick AddPred(TrySU, SDep(Copies.front(), SDep::Artificial)); 14182902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick NewDef = Copies.back(); 14192902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 14202902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 14212902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick DEBUG(dbgs() << " Adding an edge from SU #" << NewDef->NodeNum 14222902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick << " to SU #" << TrySU->NodeNum << "\n"); 14232902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick LiveRegDefs[Reg] = NewDef; 1424a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick AddPred(NewDef, SDep(TrySU, SDep::Artificial)); 14252902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick TrySU->isAvailable = false; 14262902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurSU = NewDef; 14272902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 14282902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 14292902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick assert(CurSU && "Unable to resolve live physical register dependencies!"); 14302902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 14312902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Add the nodes that aren't ready back onto the available list. 14322902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick for (unsigned i = 0, e = Interferences.size(); i != e; ++i) { 14332902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick Interferences[i]->isPending = false; 14342902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // May no longer be available due to backtracking. 14352902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick if (Interferences[i]->isAvailable) { 14362902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick AvailableQueue->push(Interferences[i]); 14372902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 14382902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 14392902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick return CurSU; 14402902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick} 1441a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng 1442e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// ListScheduleBottomUp - The main loop of list scheduling for bottom-up 1443e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// schedulers. 1444e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengvoid ScheduleDAGRRList::ListScheduleBottomUp() { 14459e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman // Release any predecessors of the special Exit node. 14463d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick ReleasePredecessors(&ExitSU); 14479e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman 1448e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // Add root to Available queue. 144980792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman if (!SUnits.empty()) { 1450a23b3b803e3c65e84d6cadaa221de8b256cbe28dDan Gohman SUnit *RootSU = &SUnits[DAG->getRoot().getNode()->getNodeId()]; 145180792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman assert(RootSU->Succs.empty() && "Graph root shouldn't have successors!"); 145280792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman RootSU->isAvailable = true; 145380792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman AvailableQueue->push(RootSU); 145480792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman } 1455e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1456e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // While Available queue is not empty, grab the node with the highest 14578d1bfad00b1ebff5b140b6e1bd7e26bad697d6e1Dan Gohman // priority. If it is not ready put it back. Schedule the node. 14584c8c83022b501759d8559e224c84ae2a9921ba41Dan Gohman Sequence.reserve(SUnits.size()); 1459e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng while (!AvailableQueue->empty()) { 146087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick DEBUG(dbgs() << "\nExamining Available:\n"; 14612da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->dump(this)); 14622da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 14632902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Pick the best node to schedule taking all constraints into 14642902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // consideration. 14652902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *SU = PickNodeToScheduleBottomUp(); 1466a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng 14672da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AdvancePastStalls(SU); 1468a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng 14692da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleNodeBottomUp(SU); 14702da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 14712da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick while (AvailableQueue->empty() && !PendingQueue.empty()) { 14722da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Advance the cycle to free resources. Skip ahead to the next ready SU. 14732da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick assert(MinAvailableCycle < UINT_MAX && "MinAvailableCycle uninitialized"); 14742da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AdvanceToCycle(std::max(CurCycle + 1, MinAvailableCycle)); 14752da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 1476e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 1477e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1478e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // Reverse the order if it is bottom up. 1479e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng std::reverse(Sequence.begin(), Sequence.end()); 148038036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1481e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#ifndef NDEBUG 14824c727204271067f3dbf50bd23098b2df8e1cc47aAndrew Trick VerifyScheduledSequence(/*isBottomUp=*/true); 1483e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#endif 1484e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 1485e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1486e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 1487f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// RegReductionPriorityQueue Definition 1488e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 1489e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 1490e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// This is a SchedulingPriorityQueue that schedules using Sethi Ullman numbers 1491e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// to reduce register pressure. 149238036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick// 1493e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengnamespace { 1494f697c8a19adf962a933b055383952e72789a0e20Andrew Trickclass RegReductionPQBase; 1495f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1496f697c8a19adf962a933b055383952e72789a0e20Andrew Trickstruct queue_sort : public std::binary_function<SUnit*, SUnit*, bool> { 1497f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool isReady(SUnit* SU, unsigned CurCycle) const { return true; } 1498f697c8a19adf962a933b055383952e72789a0e20Andrew Trick}; 149938036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 15004cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick#ifndef NDEBUG 15014cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Tricktemplate<class SF> 15024cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trickstruct reverse_sort : public queue_sort { 15034cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick SF &SortFunc; 15044cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick reverse_sort(SF &sf) : SortFunc(sf) {} 15054cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick reverse_sort(const reverse_sort &RHS) : SortFunc(RHS.SortFunc) {} 15064cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick 15074cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick bool operator()(SUnit* left, SUnit* right) const { 15084cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick // reverse left/right rather than simply !SortFunc(left, right) 15094cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick // to expose different paths in the comparison logic. 15104cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick return SortFunc(right, left); 15114cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick } 15124cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick}; 15134cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick#endif // NDEBUG 15144cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick 1515f697c8a19adf962a933b055383952e72789a0e20Andrew Trick/// bu_ls_rr_sort - Priority function for bottom up register pressure 1516f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// reduction scheduler. 1517f697c8a19adf962a933b055383952e72789a0e20Andrew Trickstruct bu_ls_rr_sort : public queue_sort { 1518f697c8a19adf962a933b055383952e72789a0e20Andrew Trick enum { 1519f697c8a19adf962a933b055383952e72789a0e20Andrew Trick IsBottomUp = true, 1520f697c8a19adf962a933b055383952e72789a0e20Andrew Trick HasReadyFilter = false 15212da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick }; 15222da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1523f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegReductionPQBase *SPQ; 1524f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bu_ls_rr_sort(RegReductionPQBase *spq) : SPQ(spq) {} 1525f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bu_ls_rr_sort(const bu_ls_rr_sort &RHS) : SPQ(RHS.SPQ) {} 15262da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1527f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool operator()(SUnit* left, SUnit* right) const; 1528f697c8a19adf962a933b055383952e72789a0e20Andrew Trick}; 152938036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1530f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// src_ls_rr_sort - Priority function for source order scheduler. 1531f697c8a19adf962a933b055383952e72789a0e20Andrew Trickstruct src_ls_rr_sort : public queue_sort { 1532f697c8a19adf962a933b055383952e72789a0e20Andrew Trick enum { 1533f697c8a19adf962a933b055383952e72789a0e20Andrew Trick IsBottomUp = true, 1534f697c8a19adf962a933b055383952e72789a0e20Andrew Trick HasReadyFilter = false 1535e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng }; 1536187361b056823df4ff292561fe47468dad956872Bill Wendling 1537f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegReductionPQBase *SPQ; 1538f697c8a19adf962a933b055383952e72789a0e20Andrew Trick src_ls_rr_sort(RegReductionPQBase *spq) 1539f697c8a19adf962a933b055383952e72789a0e20Andrew Trick : SPQ(spq) {} 1540f697c8a19adf962a933b055383952e72789a0e20Andrew Trick src_ls_rr_sort(const src_ls_rr_sort &RHS) 1541f697c8a19adf962a933b055383952e72789a0e20Andrew Trick : SPQ(RHS.SPQ) {} 15422da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1543f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool operator()(SUnit* left, SUnit* right) const; 1544f697c8a19adf962a933b055383952e72789a0e20Andrew Trick}; 154538036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1546f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// hybrid_ls_rr_sort - Priority function for hybrid scheduler. 1547f697c8a19adf962a933b055383952e72789a0e20Andrew Trickstruct hybrid_ls_rr_sort : public queue_sort { 1548f697c8a19adf962a933b055383952e72789a0e20Andrew Trick enum { 1549f697c8a19adf962a933b055383952e72789a0e20Andrew Trick IsBottomUp = true, 1550a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick HasReadyFilter = false 1551187361b056823df4ff292561fe47468dad956872Bill Wendling }; 155215a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 1553f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegReductionPQBase *SPQ; 1554f697c8a19adf962a933b055383952e72789a0e20Andrew Trick hybrid_ls_rr_sort(RegReductionPQBase *spq) 1555f697c8a19adf962a933b055383952e72789a0e20Andrew Trick : SPQ(spq) {} 1556f697c8a19adf962a933b055383952e72789a0e20Andrew Trick hybrid_ls_rr_sort(const hybrid_ls_rr_sort &RHS) 1557f697c8a19adf962a933b055383952e72789a0e20Andrew Trick : SPQ(RHS.SPQ) {} 1558f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1559f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool isReady(SUnit *SU, unsigned CurCycle) const; 15602da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1561f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool operator()(SUnit* left, SUnit* right) const; 1562f697c8a19adf962a933b055383952e72789a0e20Andrew Trick}; 15634f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 1564f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// ilp_ls_rr_sort - Priority function for ILP (instruction level parallelism) 1565f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// scheduler. 1566f697c8a19adf962a933b055383952e72789a0e20Andrew Trickstruct ilp_ls_rr_sort : public queue_sort { 1567f697c8a19adf962a933b055383952e72789a0e20Andrew Trick enum { 1568f697c8a19adf962a933b055383952e72789a0e20Andrew Trick IsBottomUp = true, 1569a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick HasReadyFilter = false 157015a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng }; 157170017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 1572f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegReductionPQBase *SPQ; 1573f697c8a19adf962a933b055383952e72789a0e20Andrew Trick ilp_ls_rr_sort(RegReductionPQBase *spq) 1574f697c8a19adf962a933b055383952e72789a0e20Andrew Trick : SPQ(spq) {} 1575f697c8a19adf962a933b055383952e72789a0e20Andrew Trick ilp_ls_rr_sort(const ilp_ls_rr_sort &RHS) 1576f697c8a19adf962a933b055383952e72789a0e20Andrew Trick : SPQ(RHS.SPQ) {} 1577f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1578f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool isReady(SUnit *SU, unsigned CurCycle) const; 15792da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1580f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool operator()(SUnit* left, SUnit* right) const; 1581f697c8a19adf962a933b055383952e72789a0e20Andrew Trick}; 158270017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 1583f697c8a19adf962a933b055383952e72789a0e20Andrew Trickclass RegReductionPQBase : public SchedulingPriorityQueue { 1584f697c8a19adf962a933b055383952e72789a0e20Andrew Trickprotected: 1585f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::vector<SUnit*> Queue; 1586f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned CurQueueId; 1587f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool TracksRegPressure; 1588479389a4da53ce72226366cc6d1cad13da158909Evan Cheng bool SrcOrder; 15892da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1590f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // SUnits - The SUnits for the current graph. 1591f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::vector<SUnit> *SUnits; 1592f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1593f697c8a19adf962a933b055383952e72789a0e20Andrew Trick MachineFunction &MF; 1594f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetInstrInfo *TII; 1595f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetRegisterInfo *TRI; 1596f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetLowering *TLI; 1597f697c8a19adf962a933b055383952e72789a0e20Andrew Trick ScheduleDAGRRList *scheduleDAG; 1598f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1599f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // SethiUllmanNumbers - The SethiUllman number for each node. 1600f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::vector<unsigned> SethiUllmanNumbers; 1601f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1602f697c8a19adf962a933b055383952e72789a0e20Andrew Trick /// RegPressure - Tracking current reg pressure per register class. 1603f697c8a19adf962a933b055383952e72789a0e20Andrew Trick /// 1604f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::vector<unsigned> RegPressure; 1605f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1606f697c8a19adf962a933b055383952e72789a0e20Andrew Trick /// RegLimit - Tracking the number of allocatable registers per register 1607f697c8a19adf962a933b055383952e72789a0e20Andrew Trick /// class. 1608f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::vector<unsigned> RegLimit; 1609f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1610f697c8a19adf962a933b055383952e72789a0e20Andrew Trickpublic: 1611f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegReductionPQBase(MachineFunction &mf, 1612f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool hasReadyFilter, 1613f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool tracksrp, 1614479389a4da53ce72226366cc6d1cad13da158909Evan Cheng bool srcorder, 1615f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetInstrInfo *tii, 1616f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetRegisterInfo *tri, 1617f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetLowering *tli) 1618f697c8a19adf962a933b055383952e72789a0e20Andrew Trick : SchedulingPriorityQueue(hasReadyFilter), 1619479389a4da53ce72226366cc6d1cad13da158909Evan Cheng CurQueueId(0), TracksRegPressure(tracksrp), SrcOrder(srcorder), 1620f697c8a19adf962a933b055383952e72789a0e20Andrew Trick MF(mf), TII(tii), TRI(tri), TLI(tli), scheduleDAG(NULL) { 1621f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (TracksRegPressure) { 1622f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned NumRC = TRI->getNumRegClasses(); 1623f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegLimit.resize(NumRC); 1624f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegPressure.resize(NumRC); 1625f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::fill(RegLimit.begin(), RegLimit.end(), 0); 1626f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::fill(RegPressure.begin(), RegPressure.end(), 0); 1627f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (TargetRegisterInfo::regclass_iterator I = TRI->regclass_begin(), 1628f697c8a19adf962a933b055383952e72789a0e20Andrew Trick E = TRI->regclass_end(); I != E; ++I) 1629be2119e8e2bc7006cfd638a24367acbfda625d16Cameron Zwarich RegLimit[(*I)->getID()] = tri->getRegPressureLimit(*I, MF); 1630f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1631f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1632f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1633f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void setScheduleDAG(ScheduleDAGRRList *scheduleDag) { 1634f697c8a19adf962a933b055383952e72789a0e20Andrew Trick scheduleDAG = scheduleDag; 1635f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1636f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1637f697c8a19adf962a933b055383952e72789a0e20Andrew Trick ScheduleHazardRecognizer* getHazardRec() { 1638f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return scheduleDAG->getHazardRec(); 1639f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1640f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1641f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void initNodes(std::vector<SUnit> &sunits); 1642f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1643f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void addNode(const SUnit *SU); 1644f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1645f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void updateNode(const SUnit *SU); 1646f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1647f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void releaseState() { 1648f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SUnits = 0; 1649f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SethiUllmanNumbers.clear(); 1650f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::fill(RegPressure.begin(), RegPressure.end(), 0); 1651f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1652f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1653f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned getNodePriority(const SUnit *SU) const; 1654f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1655f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned getNodeOrdering(const SUnit *SU) const { 1656336298cf2cc68d9af163992b9f9cafddd4bb3c8aAndrew Trick if (!SU->getNode()) return 0; 1657336298cf2cc68d9af163992b9f9cafddd4bb3c8aAndrew Trick 1658f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return scheduleDAG->DAG->GetOrdering(SU->getNode()); 1659f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1660f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1661f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool empty() const { return Queue.empty(); } 1662f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1663f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void push(SUnit *U) { 1664f697c8a19adf962a933b055383952e72789a0e20Andrew Trick assert(!U->NodeQueueId && "Node in the queue already"); 1665f697c8a19adf962a933b055383952e72789a0e20Andrew Trick U->NodeQueueId = ++CurQueueId; 1666f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Queue.push_back(U); 1667f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1668f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1669f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void remove(SUnit *SU) { 1670f697c8a19adf962a933b055383952e72789a0e20Andrew Trick assert(!Queue.empty() && "Queue is empty!"); 1671f697c8a19adf962a933b055383952e72789a0e20Andrew Trick assert(SU->NodeQueueId != 0 && "Not in queue!"); 1672f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::vector<SUnit *>::iterator I = std::find(Queue.begin(), Queue.end(), 1673f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SU); 1674f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (I != prior(Queue.end())) 1675f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::swap(*I, Queue.back()); 1676f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Queue.pop_back(); 1677f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SU->NodeQueueId = 0; 1678f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1679f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 168092e946630d5f9bb092853b93501387dd216899b9Andrew Trick bool tracksRegPressure() const { return TracksRegPressure; } 168192e946630d5f9bb092853b93501387dd216899b9Andrew Trick 1682f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void dumpRegPressure() const; 1683f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1684f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool HighRegPressure(const SUnit *SU) const; 1685f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1686e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick bool MayReduceRegPressure(SUnit *SU) const; 1687e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 1688e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick int RegPressureDiff(SUnit *SU, unsigned &LiveUses) const; 1689f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1690953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick void scheduledNode(SUnit *SU); 1691f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1692953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick void unscheduledNode(SUnit *SU); 1693f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1694f697c8a19adf962a933b055383952e72789a0e20Andrew Trickprotected: 1695f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool canClobber(const SUnit *SU, const SUnit *Op); 1696ef0b3ca3a8935b5390633dc7bb4adcdb99e0c26aDuncan Sands void AddPseudoTwoAddrDeps(); 1697f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void PrescheduleNodesWithMultipleUses(); 1698f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void CalculateSethiUllmanNumbers(); 1699f697c8a19adf962a933b055383952e72789a0e20Andrew Trick}; 1700f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1701f697c8a19adf962a933b055383952e72789a0e20Andrew Tricktemplate<class SF> 17024cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trickstatic SUnit *popFromQueueImpl(std::vector<SUnit*> &Q, SF &Picker) { 17034cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick std::vector<SUnit *>::iterator Best = Q.begin(); 17044cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick for (std::vector<SUnit *>::iterator I = llvm::next(Q.begin()), 17054cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick E = Q.end(); I != E; ++I) 17064cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick if (Picker(*Best, *I)) 17074cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick Best = I; 17084cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick SUnit *V = *Best; 17094cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick if (Best != prior(Q.end())) 17104cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick std::swap(*Best, Q.back()); 17114cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick Q.pop_back(); 17124cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick return V; 17134cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick} 17144cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick 17154cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Tricktemplate<class SF> 17164cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew TrickSUnit *popFromQueue(std::vector<SUnit*> &Q, SF &Picker, ScheduleDAG *DAG) { 17174cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick#ifndef NDEBUG 17184cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick if (DAG->StressSched) { 17194cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick reverse_sort<SF> RPicker(Picker); 17204cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick return popFromQueueImpl(Q, RPicker); 1721f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 17224cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick#endif 17234cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick (void)DAG; 17244cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick return popFromQueueImpl(Q, Picker); 17254cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick} 1726f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 17274cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Tricktemplate<class SF> 17284cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trickclass RegReductionPriorityQueue : public RegReductionPQBase { 1729f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SF Picker; 1730f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1731f697c8a19adf962a933b055383952e72789a0e20Andrew Trickpublic: 1732f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegReductionPriorityQueue(MachineFunction &mf, 1733f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool tracksrp, 1734479389a4da53ce72226366cc6d1cad13da158909Evan Cheng bool srcorder, 1735f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetInstrInfo *tii, 1736f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetRegisterInfo *tri, 1737f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetLowering *tli) 1738479389a4da53ce72226366cc6d1cad13da158909Evan Cheng : RegReductionPQBase(mf, SF::HasReadyFilter, tracksrp, srcorder, 1739479389a4da53ce72226366cc6d1cad13da158909Evan Cheng tii, tri, tli), 1740f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Picker(this) {} 1741f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1742f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool isBottomUp() const { return SF::IsBottomUp; } 1743f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1744f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool isReady(SUnit *U) const { 1745f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return Picker.HasReadyFilter && Picker.isReady(U, getCurCycle()); 1746f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1747f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1748f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SUnit *pop() { 1749f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (Queue.empty()) return NULL; 1750f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 17514cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick SUnit *V = popFromQueue(Queue, Picker, scheduleDAG); 1752f697c8a19adf962a933b055383952e72789a0e20Andrew Trick V->NodeQueueId = 0; 1753f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return V; 1754f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1755f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1756b720be6a50f4e1b3280d2b029ee38dda14577525Manman Ren#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 1757f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void dump(ScheduleDAG *DAG) const { 1758f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // Emulate pop() without clobbering NodeQueueIds. 1759f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::vector<SUnit*> DumpQueue = Queue; 1760f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SF DumpPicker = Picker; 1761f697c8a19adf962a933b055383952e72789a0e20Andrew Trick while (!DumpQueue.empty()) { 17624cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick SUnit *SU = popFromQueue(DumpQueue, DumpPicker, scheduleDAG); 1763ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman dbgs() << "Height " << SU->getHeight() << ": "; 1764f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SU->dump(DAG); 1765f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1766f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 176777e300e8f0b8db8eec448cae9c87d7c5bfad9757Manman Ren#endif 1768f697c8a19adf962a933b055383952e72789a0e20Andrew Trick}; 1769f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1770f697c8a19adf962a933b055383952e72789a0e20Andrew Tricktypedef RegReductionPriorityQueue<bu_ls_rr_sort> 1771f697c8a19adf962a933b055383952e72789a0e20Andrew TrickBURegReductionPriorityQueue; 1772f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1773f697c8a19adf962a933b055383952e72789a0e20Andrew Tricktypedef RegReductionPriorityQueue<src_ls_rr_sort> 1774f697c8a19adf962a933b055383952e72789a0e20Andrew TrickSrcRegReductionPriorityQueue; 1775f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1776f697c8a19adf962a933b055383952e72789a0e20Andrew Tricktypedef RegReductionPriorityQueue<hybrid_ls_rr_sort> 1777f697c8a19adf962a933b055383952e72789a0e20Andrew TrickHybridBURRPriorityQueue; 1778f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1779f697c8a19adf962a933b055383952e72789a0e20Andrew Tricktypedef RegReductionPriorityQueue<ilp_ls_rr_sort> 1780f697c8a19adf962a933b055383952e72789a0e20Andrew TrickILPBURRPriorityQueue; 1781f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} // end anonymous namespace 1782f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1783f697c8a19adf962a933b055383952e72789a0e20Andrew Trick//===----------------------------------------------------------------------===// 1784f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Static Node Priority for Register Pressure Reduction 1785f697c8a19adf962a933b055383952e72789a0e20Andrew Trick//===----------------------------------------------------------------------===// 1786e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 178712f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick// Check for special nodes that bypass scheduling heuristics. 178812f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick// Currently this pushes TokenFactor nodes down, but may be used for other 178912f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick// pseudo-ops as well. 179012f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick// 179112f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick// Return -1 to schedule right above left, 1 for left above right. 179212f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick// Return 0 if no bias exists. 179312f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trickstatic int checkSpecialNodes(const SUnit *left, const SUnit *right) { 179412f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick bool LSchedLow = left->isScheduleLow; 179512f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick bool RSchedLow = right->isScheduleLow; 179612f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick if (LSchedLow != RSchedLow) 179712f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick return LSchedLow < RSchedLow ? 1 : -1; 179812f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick return 0; 179912f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick} 180012f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick 1801117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman/// CalcNodeSethiUllmanNumber - Compute Sethi Ullman number. 1802117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman/// Smaller number is the higher priority. 1803c6be777208f4539af400ac694d9d1dc8b992bc80Evan Chengstatic unsigned 1804117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan GohmanCalcNodeSethiUllmanNumber(const SUnit *SU, std::vector<unsigned> &SUNumbers) { 1805c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng unsigned &SethiUllmanNumber = SUNumbers[SU->NodeNum]; 1806c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng if (SethiUllmanNumber != 0) 1807c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng return SethiUllmanNumber; 1808c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng 1809c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng unsigned Extra = 0; 1810c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 1811c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng I != E; ++I) { 181254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isCtrl()) continue; // ignore chain preds 181354e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *PredSU = I->getSUnit(); 1814117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman unsigned PredSethiUllman = CalcNodeSethiUllmanNumber(PredSU, SUNumbers); 1815c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng if (PredSethiUllman > SethiUllmanNumber) { 1816c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng SethiUllmanNumber = PredSethiUllman; 1817c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng Extra = 0; 18188182347d70413174f2e80ea429801e887aee5cc3Evan Cheng } else if (PredSethiUllman == SethiUllmanNumber) 1819c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng ++Extra; 1820c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng } 1821c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng 1822c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng SethiUllmanNumber += Extra; 1823c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng 1824c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng if (SethiUllmanNumber == 0) 1825c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng SethiUllmanNumber = 1; 182638036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1827c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng return SethiUllmanNumber; 1828c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng} 1829c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng 1830f697c8a19adf962a933b055383952e72789a0e20Andrew Trick/// CalculateSethiUllmanNumbers - Calculate Sethi-Ullman numbers of all 1831f697c8a19adf962a933b055383952e72789a0e20Andrew Trick/// scheduling units. 1832f697c8a19adf962a933b055383952e72789a0e20Andrew Trickvoid RegReductionPQBase::CalculateSethiUllmanNumbers() { 1833f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SethiUllmanNumbers.assign(SUnits->size(), 0); 183438036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1835f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (unsigned i = 0, e = SUnits->size(); i != e; ++i) 1836f697c8a19adf962a933b055383952e72789a0e20Andrew Trick CalcNodeSethiUllmanNumber(&(*SUnits)[i], SethiUllmanNumbers); 1837f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 18382da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1839f697c8a19adf962a933b055383952e72789a0e20Andrew Trickvoid RegReductionPQBase::addNode(const SUnit *SU) { 1840f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned SUSize = SethiUllmanNumbers.size(); 1841f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SUnits->size() > SUSize) 1842f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SethiUllmanNumbers.resize(SUSize*2, 0); 1843f697c8a19adf962a933b055383952e72789a0e20Andrew Trick CalcNodeSethiUllmanNumber(SU, SethiUllmanNumbers); 1844f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 1845a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1846f697c8a19adf962a933b055383952e72789a0e20Andrew Trickvoid RegReductionPQBase::updateNode(const SUnit *SU) { 1847f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SethiUllmanNumbers[SU->NodeNum] = 0; 1848f697c8a19adf962a933b055383952e72789a0e20Andrew Trick CalcNodeSethiUllmanNumber(SU, SethiUllmanNumbers); 1849f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 1850a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 18510bc308600b0069f07ba722b472c68588573ebd28Andrew Trick// Lower priority means schedule further down. For bottom-up scheduling, lower 18520bc308600b0069f07ba722b472c68588573ebd28Andrew Trick// priority SUs are scheduled before higher priority SUs. 1853f697c8a19adf962a933b055383952e72789a0e20Andrew Trickunsigned RegReductionPQBase::getNodePriority(const SUnit *SU) const { 1854f697c8a19adf962a933b055383952e72789a0e20Andrew Trick assert(SU->NodeNum < SethiUllmanNumbers.size()); 1855f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned Opc = SU->getNode() ? SU->getNode()->getOpcode() : 0; 1856f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (Opc == ISD::TokenFactor || Opc == ISD::CopyToReg) 1857f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // CopyToReg should be close to its uses to facilitate coalescing and 1858f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // avoid spilling. 1859f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return 0; 1860f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (Opc == TargetOpcode::EXTRACT_SUBREG || 1861f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Opc == TargetOpcode::SUBREG_TO_REG || 1862f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Opc == TargetOpcode::INSERT_SUBREG) 1863f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // EXTRACT_SUBREG, INSERT_SUBREG, and SUBREG_TO_REG nodes should be 1864f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // close to their uses to facilitate coalescing. 1865f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return 0; 1866f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SU->NumSuccs == 0 && SU->NumPreds != 0) 1867f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // If SU does not have a register use, i.e. it doesn't produce a value 1868f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // that would be consumed (e.g. store), then it terminates a chain of 1869f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // computation. Give it a large SethiUllman number so it will be 1870f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // scheduled right before its predecessors that it doesn't lengthen 1871f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // their live ranges. 1872f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return 0xffff; 1873f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SU->NumPreds == 0 && SU->NumSuccs != 0) 1874f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // If SU does not have a register def, schedule it close to its uses 1875f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // because it does not lengthen any live ranges. 1876f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return 0; 1877554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng#if 1 1878f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return SethiUllmanNumbers[SU->NodeNum]; 1879554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng#else 1880554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng unsigned Priority = SethiUllmanNumbers[SU->NodeNum]; 1881554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng if (SU->isCallOp) { 1882554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // FIXME: This assumes all of the defs are used as call operands. 1883554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng int NP = (int)Priority - SU->getNode()->getNumValues(); 1884554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng return (NP > 0) ? NP : 0; 1885554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng } 1886554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng return Priority; 1887554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng#endif 1888f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 1889e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1890f697c8a19adf962a933b055383952e72789a0e20Andrew Trick//===----------------------------------------------------------------------===// 1891f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Register Pressure Tracking 1892f697c8a19adf962a933b055383952e72789a0e20Andrew Trick//===----------------------------------------------------------------------===// 1893247fb4ebd363323b6642ce2bb08e53db705ca094Bill Wendling 1894f697c8a19adf962a933b055383952e72789a0e20Andrew Trickvoid RegReductionPQBase::dumpRegPressure() const { 1895b720be6a50f4e1b3280d2b029ee38dda14577525Manman Ren#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 1896f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (TargetRegisterInfo::regclass_iterator I = TRI->regclass_begin(), 1897f697c8a19adf962a933b055383952e72789a0e20Andrew Trick E = TRI->regclass_end(); I != E; ++I) { 1898f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetRegisterClass *RC = *I; 1899f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned Id = RC->getID(); 1900f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned RP = RegPressure[Id]; 1901f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!RP) continue; 1902f697c8a19adf962a933b055383952e72789a0e20Andrew Trick DEBUG(dbgs() << RC->getName() << ": " << RP << " / " << RegLimit[Id] 1903f697c8a19adf962a933b055383952e72789a0e20Andrew Trick << '\n'); 1904f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 190577e300e8f0b8db8eec448cae9c87d7c5bfad9757Manman Ren#endif 1906f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 190715a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 1908f697c8a19adf962a933b055383952e72789a0e20Andrew Trickbool RegReductionPQBase::HighRegPressure(const SUnit *SU) const { 1909f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!TLI) 1910f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return false; 191138036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1912f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (SUnit::const_pred_iterator I = SU->Preds.begin(),E = SU->Preds.end(); 1913f697c8a19adf962a933b055383952e72789a0e20Andrew Trick I != E; ++I) { 1914f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (I->isCtrl()) 1915f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 1916f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SUnit *PredSU = I->getSUnit(); 191792e946630d5f9bb092853b93501387dd216899b9Andrew Trick // NumRegDefsLeft is zero when enough uses of this node have been scheduled 191892e946630d5f9bb092853b93501387dd216899b9Andrew Trick // to cover the number of registers defined (they are all live). 191992e946630d5f9bb092853b93501387dd216899b9Andrew Trick if (PredSU->NumRegDefsLeft == 0) { 1920f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 1921e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 192292e946630d5f9bb092853b93501387dd216899b9Andrew Trick for (ScheduleDAGSDNodes::RegDefIter RegDefPos(PredSU, scheduleDAG); 192392e946630d5f9bb092853b93501387dd216899b9Andrew Trick RegDefPos.IsValid(); RegDefPos.Advance()) { 192477b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson unsigned RCId, Cost; 1925397fc4874efe9c17e737d4c5c50bd19dc3bf27f5Jakob Stoklund Olesen GetCostForDef(RegDefPos, TLI, TII, TRI, RCId, Cost, MF); 192677b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson 1927f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if ((RegPressure[RCId] + Cost) >= RegLimit[RCId]) 1928f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return true; 1929a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 1930f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1931f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return false; 1932f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 1933e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1934e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickbool RegReductionPQBase::MayReduceRegPressure(SUnit *SU) const { 1935f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const SDNode *N = SU->getNode(); 19364f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 1937f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!N->isMachineOpcode() || !SU->NumSuccs) 1938f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return false; 19394f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 1940f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs(); 1941f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (unsigned i = 0; i != NumDefs; ++i) { 1942f697c8a19adf962a933b055383952e72789a0e20Andrew Trick EVT VT = N->getValueType(i); 1943f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!N->hasAnyUseOfValue(i)) 1944f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 1945f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 1946f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (RegPressure[RCId] >= RegLimit[RCId]) 1947f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return true; 1948f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1949f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return false; 1950f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 19514f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 1952e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick// Compute the register pressure contribution by this instruction by count up 1953e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick// for uses that are not live and down for defs. Only count register classes 1954e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick// that are already under high pressure. As a side effect, compute the number of 1955e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick// uses of registers that are already live. 1956e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick// 1957e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick// FIXME: This encompasses the logic in HighRegPressure and MayReduceRegPressure 1958e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick// so could probably be factored. 1959e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickint RegReductionPQBase::RegPressureDiff(SUnit *SU, unsigned &LiveUses) const { 1960e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick LiveUses = 0; 1961e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick int PDiff = 0; 1962e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick for (SUnit::const_pred_iterator I = SU->Preds.begin(),E = SU->Preds.end(); 1963e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick I != E; ++I) { 1964e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (I->isCtrl()) 1965e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick continue; 1966e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick SUnit *PredSU = I->getSUnit(); 1967e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick // NumRegDefsLeft is zero when enough uses of this node have been scheduled 1968e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick // to cover the number of registers defined (they are all live). 1969e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (PredSU->NumRegDefsLeft == 0) { 1970e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (PredSU->getNode()->isMachineOpcode()) 1971e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick ++LiveUses; 1972e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick continue; 1973e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick } 1974e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick for (ScheduleDAGSDNodes::RegDefIter RegDefPos(PredSU, scheduleDAG); 1975e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick RegDefPos.IsValid(); RegDefPos.Advance()) { 1976e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick EVT VT = RegDefPos.GetValue(); 1977e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 1978e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (RegPressure[RCId] >= RegLimit[RCId]) 1979e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick ++PDiff; 1980e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick } 1981e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick } 1982e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick const SDNode *N = SU->getNode(); 1983e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 198429449448b0f0420dfcf52e278fc01adbf1690d70Eric Christopher if (!N || !N->isMachineOpcode() || !SU->NumSuccs) 1985e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick return PDiff; 1986e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 1987e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs(); 1988e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick for (unsigned i = 0; i != NumDefs; ++i) { 1989e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick EVT VT = N->getValueType(i); 1990e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (!N->hasAnyUseOfValue(i)) 1991e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick continue; 1992e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 1993e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (RegPressure[RCId] >= RegLimit[RCId]) 1994e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick --PDiff; 1995e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick } 1996e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick return PDiff; 1997e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick} 1998e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 1999953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trickvoid RegReductionPQBase::scheduledNode(SUnit *SU) { 2000f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!TracksRegPressure) 2001f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return; 20024f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 200329449448b0f0420dfcf52e278fc01adbf1690d70Eric Christopher if (!SU->getNode()) 200429449448b0f0420dfcf52e278fc01adbf1690d70Eric Christopher return; 20050d93a110e31b384f59d91d6be27388d8ded5f03cAndrew Trick 2006f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 2007f697c8a19adf962a933b055383952e72789a0e20Andrew Trick I != E; ++I) { 2008f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (I->isCtrl()) 2009f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 2010f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SUnit *PredSU = I->getSUnit(); 201192e946630d5f9bb092853b93501387dd216899b9Andrew Trick // NumRegDefsLeft is zero when enough uses of this node have been scheduled 201292e946630d5f9bb092853b93501387dd216899b9Andrew Trick // to cover the number of registers defined (they are all live). 201392e946630d5f9bb092853b93501387dd216899b9Andrew Trick if (PredSU->NumRegDefsLeft == 0) { 2014f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 2015f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 201692e946630d5f9bb092853b93501387dd216899b9Andrew Trick // FIXME: The ScheduleDAG currently loses information about which of a 201792e946630d5f9bb092853b93501387dd216899b9Andrew Trick // node's values is consumed by each dependence. Consequently, if the node 201892e946630d5f9bb092853b93501387dd216899b9Andrew Trick // defines multiple register classes, we don't know which to pressurize 201992e946630d5f9bb092853b93501387dd216899b9Andrew Trick // here. Instead the following loop consumes the register defs in an 202092e946630d5f9bb092853b93501387dd216899b9Andrew Trick // arbitrary order. At least it handles the common case of clustered loads 202192e946630d5f9bb092853b93501387dd216899b9Andrew Trick // to the same class. For precise liveness, each SDep needs to indicate the 202292e946630d5f9bb092853b93501387dd216899b9Andrew Trick // result number. But that tightly couples the ScheduleDAG with the 202392e946630d5f9bb092853b93501387dd216899b9Andrew Trick // SelectionDAG making updates tricky. A simpler hack would be to attach a 202492e946630d5f9bb092853b93501387dd216899b9Andrew Trick // value type or register class to SDep. 202592e946630d5f9bb092853b93501387dd216899b9Andrew Trick // 202692e946630d5f9bb092853b93501387dd216899b9Andrew Trick // The most important aspect of register tracking is balancing the increase 202792e946630d5f9bb092853b93501387dd216899b9Andrew Trick // here with the reduction further below. Note that this SU may use multiple 202892e946630d5f9bb092853b93501387dd216899b9Andrew Trick // defs in PredSU. The can't be determined here, but we've already 202992e946630d5f9bb092853b93501387dd216899b9Andrew Trick // compensated by reducing NumRegDefsLeft in PredSU during 203092e946630d5f9bb092853b93501387dd216899b9Andrew Trick // ScheduleDAGSDNodes::AddSchedEdges. 203192e946630d5f9bb092853b93501387dd216899b9Andrew Trick --PredSU->NumRegDefsLeft; 203292e946630d5f9bb092853b93501387dd216899b9Andrew Trick unsigned SkipRegDefs = PredSU->NumRegDefsLeft; 203392e946630d5f9bb092853b93501387dd216899b9Andrew Trick for (ScheduleDAGSDNodes::RegDefIter RegDefPos(PredSU, scheduleDAG); 203492e946630d5f9bb092853b93501387dd216899b9Andrew Trick RegDefPos.IsValid(); RegDefPos.Advance(), --SkipRegDefs) { 203592e946630d5f9bb092853b93501387dd216899b9Andrew Trick if (SkipRegDefs) 2036f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 203777b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson 203877b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson unsigned RCId, Cost; 2039397fc4874efe9c17e737d4c5c50bd19dc3bf27f5Jakob Stoklund Olesen GetCostForDef(RegDefPos, TLI, TII, TRI, RCId, Cost, MF); 204077b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson RegPressure[RCId] += Cost; 204192e946630d5f9bb092853b93501387dd216899b9Andrew Trick break; 2042f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 2043f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 204489ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng 204592e946630d5f9bb092853b93501387dd216899b9Andrew Trick // We should have this assert, but there may be dead SDNodes that never 204692e946630d5f9bb092853b93501387dd216899b9Andrew Trick // materialize as SUnits, so they don't appear to generate liveness. 204792e946630d5f9bb092853b93501387dd216899b9Andrew Trick //assert(SU->NumRegDefsLeft == 0 && "not all regdefs have scheduled uses"); 204892e946630d5f9bb092853b93501387dd216899b9Andrew Trick int SkipRegDefs = (int)SU->NumRegDefsLeft; 204992e946630d5f9bb092853b93501387dd216899b9Andrew Trick for (ScheduleDAGSDNodes::RegDefIter RegDefPos(SU, scheduleDAG); 205092e946630d5f9bb092853b93501387dd216899b9Andrew Trick RegDefPos.IsValid(); RegDefPos.Advance(), --SkipRegDefs) { 205192e946630d5f9bb092853b93501387dd216899b9Andrew Trick if (SkipRegDefs > 0) 205292e946630d5f9bb092853b93501387dd216899b9Andrew Trick continue; 205377b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson unsigned RCId, Cost; 2054397fc4874efe9c17e737d4c5c50bd19dc3bf27f5Jakob Stoklund Olesen GetCostForDef(RegDefPos, TLI, TII, TRI, RCId, Cost, MF); 205577b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson if (RegPressure[RCId] < Cost) { 205692e946630d5f9bb092853b93501387dd216899b9Andrew Trick // Register pressure tracking is imprecise. This can happen. But we try 205792e946630d5f9bb092853b93501387dd216899b9Andrew Trick // hard not to let it happen because it likely results in poor scheduling. 205892e946630d5f9bb092853b93501387dd216899b9Andrew Trick DEBUG(dbgs() << " SU(" << SU->NodeNum << ") has too many regdefs\n"); 205992e946630d5f9bb092853b93501387dd216899b9Andrew Trick RegPressure[RCId] = 0; 206092e946630d5f9bb092853b93501387dd216899b9Andrew Trick } 206192e946630d5f9bb092853b93501387dd216899b9Andrew Trick else { 206277b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson RegPressure[RCId] -= Cost; 20634f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 2064f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 2065f697c8a19adf962a933b055383952e72789a0e20Andrew Trick dumpRegPressure(); 2066f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 20674f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 2068953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trickvoid RegReductionPQBase::unscheduledNode(SUnit *SU) { 2069f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!TracksRegPressure) 2070f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return; 20714f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 2072f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const SDNode *N = SU->getNode(); 207329449448b0f0420dfcf52e278fc01adbf1690d70Eric Christopher if (!N) return; 20740d93a110e31b384f59d91d6be27388d8ded5f03cAndrew Trick 2075f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!N->isMachineOpcode()) { 2076f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (N->getOpcode() != ISD::CopyToReg) 2077f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return; 2078f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } else { 2079f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned Opc = N->getMachineOpcode(); 2080f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (Opc == TargetOpcode::EXTRACT_SUBREG || 2081f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Opc == TargetOpcode::INSERT_SUBREG || 2082f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Opc == TargetOpcode::SUBREG_TO_REG || 2083f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Opc == TargetOpcode::REG_SEQUENCE || 2084f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Opc == TargetOpcode::IMPLICIT_DEF) 2085f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return; 2086f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 20874f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 2088f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 2089f697c8a19adf962a933b055383952e72789a0e20Andrew Trick I != E; ++I) { 2090f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (I->isCtrl()) 2091f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 2092f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SUnit *PredSU = I->getSUnit(); 20930bc308600b0069f07ba722b472c68588573ebd28Andrew Trick // NumSuccsLeft counts all deps. Don't compare it with NumSuccs which only 20940bc308600b0069f07ba722b472c68588573ebd28Andrew Trick // counts data deps. 20950bc308600b0069f07ba722b472c68588573ebd28Andrew Trick if (PredSU->NumSuccsLeft != PredSU->Succs.size()) 2096f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 2097f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const SDNode *PN = PredSU->getNode(); 2098f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!PN->isMachineOpcode()) { 2099f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (PN->getOpcode() == ISD::CopyFromReg) { 2100f697c8a19adf962a933b055383952e72789a0e20Andrew Trick EVT VT = PN->getValueType(0); 2101f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 2102f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); 2103f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 2104f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 21054f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 2106f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned POpc = PN->getMachineOpcode(); 2107f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (POpc == TargetOpcode::IMPLICIT_DEF) 2108f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 21094ef4c171dba8e479f5f3fe7acb22e9fe97a4d6f8Andrew Trick if (POpc == TargetOpcode::EXTRACT_SUBREG || 21104ef4c171dba8e479f5f3fe7acb22e9fe97a4d6f8Andrew Trick POpc == TargetOpcode::INSERT_SUBREG || 21114ef4c171dba8e479f5f3fe7acb22e9fe97a4d6f8Andrew Trick POpc == TargetOpcode::SUBREG_TO_REG) { 2112f697c8a19adf962a933b055383952e72789a0e20Andrew Trick EVT VT = PN->getValueType(0); 2113f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 2114f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); 2115f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 2116e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 2117f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned NumDefs = TII->get(PN->getMachineOpcode()).getNumDefs(); 2118f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (unsigned i = 0; i != NumDefs; ++i) { 2119f697c8a19adf962a933b055383952e72789a0e20Andrew Trick EVT VT = PN->getValueType(i); 2120f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!PN->hasAnyUseOfValue(i)) 2121f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 2122f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 2123f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (RegPressure[RCId] < TLI->getRepRegClassCostFor(VT)) 2124f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // Register pressure tracking is imprecise. This can happen. 2125f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegPressure[RCId] = 0; 2126f697c8a19adf962a933b055383952e72789a0e20Andrew Trick else 2127f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegPressure[RCId] -= TLI->getRepRegClassCostFor(VT); 21284f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 2129f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 21304f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 2131f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // Check for isMachineOpcode() as PrescheduleNodesWithMultipleUses() 2132f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // may transfer data dependencies to CopyToReg. 2133f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SU->NumSuccs && N->isMachineOpcode()) { 2134f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs(); 2135f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (unsigned i = NumDefs, e = N->getNumValues(); i != e; ++i) { 2136f697c8a19adf962a933b055383952e72789a0e20Andrew Trick EVT VT = N->getValueType(i); 2137f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (VT == MVT::Glue || VT == MVT::Other) 2138f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 2139f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!N->hasAnyUseOfValue(i)) 2140f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 2141f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 2142f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); 21432da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 2144f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 21452da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 2146f697c8a19adf962a933b055383952e72789a0e20Andrew Trick dumpRegPressure(); 2147e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 2148e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 2149f697c8a19adf962a933b055383952e72789a0e20Andrew Trick//===----------------------------------------------------------------------===// 2150f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Dynamic Node Priority for Register Pressure Reduction 2151f697c8a19adf962a933b055383952e72789a0e20Andrew Trick//===----------------------------------------------------------------------===// 2152f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2153c6deb3d44707de57e82e16642ab845bc8b9e9e01Evan Cheng/// closestSucc - Returns the scheduled cycle of the successor which is 2154b398fca15bcca895526699f20336fa6c6a624013Dan Gohman/// closest to the current cycle. 215561230d18d21a5dca1378e994f43934e4b314e595Evan Chengstatic unsigned closestSucc(const SUnit *SU) { 21563f23744df4809eba94284e601e81489212c974d4Dan Gohman unsigned MaxHeight = 0; 215761230d18d21a5dca1378e994f43934e4b314e595Evan Cheng for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 2158c6deb3d44707de57e82e16642ab845bc8b9e9e01Evan Cheng I != E; ++I) { 2159f0e366a929a1acb4bc14df5ef831cce74607a967Evan Cheng if (I->isCtrl()) continue; // ignore chain succs 21603f23744df4809eba94284e601e81489212c974d4Dan Gohman unsigned Height = I->getSUnit()->getHeight(); 2161c6deb3d44707de57e82e16642ab845bc8b9e9e01Evan Cheng // If there are bunch of CopyToRegs stacked up, they should be considered 2162c6deb3d44707de57e82e16642ab845bc8b9e9e01Evan Cheng // to be at the same position. 216354e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->getSUnit()->getNode() && 216454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman I->getSUnit()->getNode()->getOpcode() == ISD::CopyToReg) 21653f23744df4809eba94284e601e81489212c974d4Dan Gohman Height = closestSucc(I->getSUnit())+1; 21663f23744df4809eba94284e601e81489212c974d4Dan Gohman if (Height > MaxHeight) 21673f23744df4809eba94284e601e81489212c974d4Dan Gohman MaxHeight = Height; 2168c6deb3d44707de57e82e16642ab845bc8b9e9e01Evan Cheng } 21693f23744df4809eba94284e601e81489212c974d4Dan Gohman return MaxHeight; 217061230d18d21a5dca1378e994f43934e4b314e595Evan Cheng} 217161230d18d21a5dca1378e994f43934e4b314e595Evan Cheng 2172d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng/// calcMaxScratches - Returns an cost estimate of the worse case requirement 21738182347d70413174f2e80ea429801e887aee5cc3Evan Cheng/// for scratch registers, i.e. number of data dependencies. 2174d6c0758944b31bb5316b36cad37f4610a77f784dEvan Chengstatic unsigned calcMaxScratches(const SUnit *SU) { 2175d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng unsigned Scratches = 0; 2176d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 2177f2b14715d11e52adbb17a5860d1ce42f82f85a0cEvan Cheng I != E; ++I) { 217854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isCtrl()) continue; // ignore chain preds 2179f2b14715d11e52adbb17a5860d1ce42f82f85a0cEvan Cheng Scratches++; 2180f2b14715d11e52adbb17a5860d1ce42f82f85a0cEvan Cheng } 2181d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng return Scratches; 2182d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng} 2183d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng 218487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick/// hasOnlyLiveInOpers - Return true if SU has only value predecessors that are 218587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick/// CopyFromReg from a virtual register. 218687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trickstatic bool hasOnlyLiveInOpers(const SUnit *SU) { 218787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick bool RetVal = false; 218887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 218987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick I != E; ++I) { 219087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (I->isCtrl()) continue; 219187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick const SUnit *PredSU = I->getSUnit(); 219287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (PredSU->getNode() && 219387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick PredSU->getNode()->getOpcode() == ISD::CopyFromReg) { 219487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick unsigned Reg = 219587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick cast<RegisterSDNode>(PredSU->getNode()->getOperand(1))->getReg(); 219687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (TargetRegisterInfo::isVirtualRegister(Reg)) { 219787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick RetVal = true; 219887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick continue; 219987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick } 220087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick } 220187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick return false; 220287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick } 220387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick return RetVal; 220487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick} 220587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 220687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick/// hasOnlyLiveOutUses - Return true if SU has only value successors that are 2207089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng/// CopyToReg to a virtual register. This SU def is probably a liveout and 2208089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng/// it has no other use. It should be scheduled closer to the terminator. 2209089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Chengstatic bool hasOnlyLiveOutUses(const SUnit *SU) { 2210089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng bool RetVal = false; 2211089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 2212089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng I != E; ++I) { 2213089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng if (I->isCtrl()) continue; 2214089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng const SUnit *SuccSU = I->getSUnit(); 2215089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng if (SuccSU->getNode() && SuccSU->getNode()->getOpcode() == ISD::CopyToReg) { 2216089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng unsigned Reg = 2217089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng cast<RegisterSDNode>(SuccSU->getNode()->getOperand(1))->getReg(); 2218089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng if (TargetRegisterInfo::isVirtualRegister(Reg)) { 2219089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng RetVal = true; 2220089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng continue; 2221089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng } 2222089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng } 2223089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng return false; 2224089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng } 2225089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng return RetVal; 2226089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng} 2227089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng 222887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// Set isVRegCycle for a node with only live in opers and live out uses. Also 222987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// set isVRegCycle for its CopyFromReg operands. 223087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// 223187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// This is only relevant for single-block loops, in which case the VRegCycle 223287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// node is likely an induction variable in which the operand and target virtual 223387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// registers should be coalesced (e.g. pre/post increment values). Setting the 223487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// isVRegCycle flag helps the scheduler prioritize other uses of the same 223587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// CopyFromReg so that this node becomes the virtual register "kill". This 223687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// avoids interference between the values live in and out of the block and 223787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// eliminates a copy inside the loop. 223887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trickstatic void initVRegCycle(SUnit *SU) { 223987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (DisableSchedVRegCycle) 224087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick return; 224187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 224287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (!hasOnlyLiveInOpers(SU) || !hasOnlyLiveOutUses(SU)) 224387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick return; 224487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 224587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick DEBUG(dbgs() << "VRegCycle: SU(" << SU->NodeNum << ")\n"); 224687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 224787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick SU->isVRegCycle = true; 224887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 224987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 2250089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng I != E; ++I) { 225187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (I->isCtrl()) continue; 225287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick I->getSUnit()->isVRegCycle = true; 2253089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng } 225487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick} 225587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 225687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// After scheduling the definition of a VRegCycle, clear the isVRegCycle flag of 225787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// CopyFromReg operands. We should no longer penalize other uses of this VReg. 225887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trickstatic void resetVRegCycle(SUnit *SU) { 225987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (!SU->isVRegCycle) 226087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick return; 226187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 226287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick for (SUnit::const_pred_iterator I = SU->Preds.begin(),E = SU->Preds.end(); 2263089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng I != E; ++I) { 2264089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng if (I->isCtrl()) continue; // ignore chain preds 226587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick SUnit *PredSU = I->getSUnit(); 226687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (PredSU->isVRegCycle) { 226787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick assert(PredSU->getNode()->getOpcode() == ISD::CopyFromReg && 226887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick "VRegCycle def must be CopyFromReg"); 226987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick I->getSUnit()->isVRegCycle = 0; 227087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick } 2271089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng } 2272089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng} 2273089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng 227487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// Return true if this SUnit uses a CopyFromReg node marked as a VRegCycle. This 227587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// means a node that defines the VRegCycle has not been scheduled yet. 227687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trickstatic bool hasVRegCycleUse(const SUnit *SU) { 227787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // If this SU also defines the VReg, don't hoist it as a "use". 227887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (SU->isVRegCycle) 227987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick return false; 228087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 228187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick for (SUnit::const_pred_iterator I = SU->Preds.begin(),E = SU->Preds.end(); 228287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick I != E; ++I) { 228354699765064842fd08d1466adc93453660bc2a85Andrew Trick if (I->isCtrl()) continue; // ignore chain preds 228487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (I->getSUnit()->isVRegCycle && 228587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick I->getSUnit()->getNode()->getOpcode() == ISD::CopyFromReg) { 228687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick DEBUG(dbgs() << " VReg cycle use: SU (" << SU->NodeNum << ")\n"); 228787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick return true; 228854699765064842fd08d1466adc93453660bc2a85Andrew Trick } 228954699765064842fd08d1466adc93453660bc2a85Andrew Trick } 229054699765064842fd08d1466adc93453660bc2a85Andrew Trick return false; 229154699765064842fd08d1466adc93453660bc2a85Andrew Trick} 229254699765064842fd08d1466adc93453660bc2a85Andrew Trick 2293f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Check for either a dependence (latency) or resource (hazard) stall. 2294f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// 2295f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Note: The ScheduleHazardRecognizer interface requires a non-const SU. 2296f697c8a19adf962a933b055383952e72789a0e20Andrew Trickstatic bool BUHasStall(SUnit *SU, int Height, RegReductionPQBase *SPQ) { 2297f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if ((int)SPQ->getCurCycle() < Height) return true; 2298f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SPQ->getHazardRec()->getHazardType(SU, 0) 2299f697c8a19adf962a933b055383952e72789a0e20Andrew Trick != ScheduleHazardRecognizer::NoHazard) 2300f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return true; 2301f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return false; 2302f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 2303f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2304f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Return -1 if left has higher priority, 1 if right has higher priority. 2305f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Return 0 if latency-based priority is equivalent. 2306f697c8a19adf962a933b055383952e72789a0e20Andrew Trickstatic int BUCompareLatency(SUnit *left, SUnit *right, bool checkPref, 2307f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegReductionPQBase *SPQ) { 230887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // Scheduling an instruction that uses a VReg whose postincrement has not yet 230987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // been scheduled will induce a copy. Model this as an extra cycle of latency. 231087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick int LPenalty = hasVRegCycleUse(left) ? 1 : 0; 231187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick int RPenalty = hasVRegCycleUse(right) ? 1 : 0; 231287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick int LHeight = (int)left->getHeight() + LPenalty; 231387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick int RHeight = (int)right->getHeight() + RPenalty; 2314f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2315692c1d85353249124caa1885cfeda513146c6d81Dan Gohman bool LStall = (!checkPref || left->SchedulingPref == Sched::ILP) && 2316f697c8a19adf962a933b055383952e72789a0e20Andrew Trick BUHasStall(left, LHeight, SPQ); 2317692c1d85353249124caa1885cfeda513146c6d81Dan Gohman bool RStall = (!checkPref || right->SchedulingPref == Sched::ILP) && 2318f697c8a19adf962a933b055383952e72789a0e20Andrew Trick BUHasStall(right, RHeight, SPQ); 2319f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2320f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // If scheduling one of the node will cause a pipeline stall, delay it. 2321f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // If scheduling either one of the node will cause a pipeline stall, sort 2322f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // them according to their height. 2323f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (LStall) { 2324e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (!RStall) 2325f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return 1; 2326e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (LHeight != RHeight) 2327f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return LHeight > RHeight ? 1 : -1; 2328e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky } else if (RStall) 2329f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return -1; 2330f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2331c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick // If either node is scheduling for latency, sort them by height/depth 2332f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // and latency. 2333692c1d85353249124caa1885cfeda513146c6d81Dan Gohman if (!checkPref || (left->SchedulingPref == Sched::ILP || 2334692c1d85353249124caa1885cfeda513146c6d81Dan Gohman right->SchedulingPref == Sched::ILP)) { 23354eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick // If neither instruction stalls (!LStall && !RStall) and HazardRecognizer 23364eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick // is enabled, grouping instructions by cycle, then its height is already 23374eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick // covered so only its depth matters. We also reach this point if both stall 23384eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick // but have the same height. 23394eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick if (!SPQ->getHazardRec()->isEnabled()) { 2340e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (LHeight != RHeight) 2341c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick return LHeight > RHeight ? 1 : -1; 2342c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick } 23434eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick int LDepth = left->getDepth() - LPenalty; 23444eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick int RDepth = right->getDepth() - RPenalty; 23454eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick if (LDepth != RDepth) { 23464eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick DEBUG(dbgs() << " Comparing latency of SU (" << left->NodeNum 23474eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick << ") depth " << LDepth << " vs SU (" << right->NodeNum 23484eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick << ") depth " << RDepth << "\n"); 23494eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick return LDepth < RDepth ? 1 : -1; 2350f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 2351e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (left->Latency != right->Latency) 2352f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return left->Latency > right->Latency ? 1 : -1; 2353f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 2354f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return 0; 2355f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 2356f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2357f697c8a19adf962a933b055383952e72789a0e20Andrew Trickstatic bool BURRSort(SUnit *left, SUnit *right, RegReductionPQBase *SPQ) { 235812f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick // Schedule physical register definitions close to their use. This is 235912f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick // motivated by microarchitectures that can fuse cmp+jump macro-ops. But as 236012f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick // long as shortening physreg live ranges is generally good, we can defer 236112f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick // creating a subtarget hook. 236212f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick if (!DisableSchedPhysRegJoin) { 236312f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick bool LHasPhysReg = left->hasPhysRegDefs; 236412f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick bool RHasPhysReg = right->hasPhysRegDefs; 236512f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick if (LHasPhysReg != RHasPhysReg) { 236612f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick #ifndef NDEBUG 2367e32981048244ecfa67d0bdc211af1bac2020a555Craig Topper const char *const PhysRegMsg[] = {" has no physreg"," defines a physreg"}; 236812f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick #endif 236912f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick DEBUG(dbgs() << " SU (" << left->NodeNum << ") " 237012f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick << PhysRegMsg[LHasPhysReg] << " SU(" << right->NodeNum << ") " 237112f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick << PhysRegMsg[RHasPhysReg] << "\n"); 237212f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick return LHasPhysReg < RHasPhysReg; 237312f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick } 237412f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick } 237512f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick 2376b16d06f88a81a5163e0caffad4bb268a8e1d0204Evan Cheng // Prioritize by Sethi-Ulmann number and push CopyToReg nodes down. 2377c8edc64188399437f5476d7fa45f714a92f2cb93Evan Cheng unsigned LPriority = SPQ->getNodePriority(left); 2378c8edc64188399437f5476d7fa45f714a92f2cb93Evan Cheng unsigned RPriority = SPQ->getNodePriority(right); 2379554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng 2380554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // Be really careful about hoisting call operands above previous calls. 2381554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // Only allows it if it would reduce register pressure. 2382554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng if (left->isCall && right->isCallOp) { 2383554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng unsigned RNumVals = right->getNode()->getNumValues(); 2384554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng RPriority = (RPriority > RNumVals) ? (RPriority - RNumVals) : 0; 2385554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng } 2386554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng if (right->isCall && left->isCallOp) { 2387554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng unsigned LNumVals = left->getNode()->getNumValues(); 2388554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng LPriority = (LPriority > LNumVals) ? (LPriority - LNumVals) : 0; 2389554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng } 2390554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng 2391e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (LPriority != RPriority) 239284d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng return LPriority > RPriority; 23930bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick 2394554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // One or both of the nodes are calls and their sethi-ullman numbers are the 2395554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // same, then keep source order. 2396554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng if (left->isCall || right->isCall) { 2397554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng unsigned LOrder = SPQ->getNodeOrdering(left); 2398554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng unsigned ROrder = SPQ->getNodeOrdering(right); 2399554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng 2400554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // Prefer an ordering where the lower the non-zero order number, the higher 2401554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // the preference. 2402554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng if ((LOrder || ROrder) && LOrder != ROrder) 2403554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng return LOrder != 0 && (LOrder < ROrder || ROrder == 0); 2404554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng } 2405554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng 240684d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // Try schedule def + use closer when Sethi-Ullman numbers are the same. 240784d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // e.g. 240884d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t1 = op t2, c1 240984d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t3 = op t4, c2 241084d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // 241184d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // and the following instructions are both ready. 241284d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t2 = op c3 241384d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t4 = op c4 241484d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // 241584d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // Then schedule t2 = op first. 241684d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // i.e. 241784d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t4 = op c4 241884d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t2 = op c3 241984d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t1 = op t2, c1 242084d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t3 = op t4, c2 242184d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // 242284d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // This creates more short live intervals. 242384d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng unsigned LDist = closestSucc(left); 242484d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng unsigned RDist = closestSucc(right); 2425e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (LDist != RDist) 242684d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng return LDist < RDist; 242784d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng 24288182347d70413174f2e80ea429801e887aee5cc3Evan Cheng // How many registers becomes live when the node is scheduled. 242984d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng unsigned LScratch = calcMaxScratches(left); 243084d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng unsigned RScratch = calcMaxScratches(right); 2431e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (LScratch != RScratch) 243284d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng return LScratch > RScratch; 243384d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng 2434554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // Comparing latency against a call makes little sense unless the node 2435554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // is register pressure-neutral. 2436554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng if ((left->isCall && RPriority > 0) || (right->isCall && LPriority > 0)) 2437554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng return (left->NodeQueueId > right->NodeQueueId); 2438554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng 2439554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // Do not compare latencies when one or both of the nodes are calls. 2440554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng if (!DisableSchedCycles && 2441554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng !(left->isCall || right->isCall)) { 2442f697c8a19adf962a933b055383952e72789a0e20Andrew Trick int result = BUCompareLatency(left, right, false /*checkPref*/, SPQ); 2443f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (result != 0) 2444f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return result > 0; 2445f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 2446f697c8a19adf962a933b055383952e72789a0e20Andrew Trick else { 2447e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (left->getHeight() != right->getHeight()) 2448f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return left->getHeight() > right->getHeight(); 244938036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 2450e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (left->getDepth() != right->getDepth()) 2451f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return left->getDepth() < right->getDepth(); 2452f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 245384d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng 245438036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick assert(left->NodeQueueId && right->NodeQueueId && 2455c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng "NodeQueueId cannot be zero"); 2456c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng return (left->NodeQueueId > right->NodeQueueId); 2457c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng} 2458c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng 2459187361b056823df4ff292561fe47468dad956872Bill Wendling// Bottom up 2460f697c8a19adf962a933b055383952e72789a0e20Andrew Trickbool bu_ls_rr_sort::operator()(SUnit *left, SUnit *right) const { 246112f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick if (int res = checkSpecialNodes(left, right)) 246212f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick return res > 0; 246312f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick 2464187361b056823df4ff292561fe47468dad956872Bill Wendling return BURRSort(left, right, SPQ); 2465187361b056823df4ff292561fe47468dad956872Bill Wendling} 2466187361b056823df4ff292561fe47468dad956872Bill Wendling 2467187361b056823df4ff292561fe47468dad956872Bill Wendling// Source order, otherwise bottom up. 2468f697c8a19adf962a933b055383952e72789a0e20Andrew Trickbool src_ls_rr_sort::operator()(SUnit *left, SUnit *right) const { 246912f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick if (int res = checkSpecialNodes(left, right)) 247012f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick return res > 0; 247112f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick 2472187361b056823df4ff292561fe47468dad956872Bill Wendling unsigned LOrder = SPQ->getNodeOrdering(left); 2473187361b056823df4ff292561fe47468dad956872Bill Wendling unsigned ROrder = SPQ->getNodeOrdering(right); 2474187361b056823df4ff292561fe47468dad956872Bill Wendling 2475187361b056823df4ff292561fe47468dad956872Bill Wendling // Prefer an ordering where the lower the non-zero order number, the higher 2476187361b056823df4ff292561fe47468dad956872Bill Wendling // the preference. 2477187361b056823df4ff292561fe47468dad956872Bill Wendling if ((LOrder || ROrder) && LOrder != ROrder) 2478187361b056823df4ff292561fe47468dad956872Bill Wendling return LOrder != 0 && (LOrder < ROrder || ROrder == 0); 2479187361b056823df4ff292561fe47468dad956872Bill Wendling 2480187361b056823df4ff292561fe47468dad956872Bill Wendling return BURRSort(left, right, SPQ); 2481187361b056823df4ff292561fe47468dad956872Bill Wendling} 2482187361b056823df4ff292561fe47468dad956872Bill Wendling 2483f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// If the time between now and when the instruction will be ready can cover 2484f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// the spill code, then avoid adding it to the ready queue. This gives long 2485f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// stalls highest priority and allows hoisting across calls. It should also 2486f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// speed up processing the available queue. 2487f697c8a19adf962a933b055383952e72789a0e20Andrew Trickbool hybrid_ls_rr_sort::isReady(SUnit *SU, unsigned CurCycle) const { 2488f697c8a19adf962a933b055383952e72789a0e20Andrew Trick static const unsigned ReadyDelay = 3; 2489f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2490f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SPQ->MayReduceRegPressure(SU)) return true; 2491f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2492f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SU->getHeight() > (CurCycle + ReadyDelay)) return false; 2493f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2494f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SPQ->getHazardRec()->getHazardType(SU, -ReadyDelay) 2495f697c8a19adf962a933b055383952e72789a0e20Andrew Trick != ScheduleHazardRecognizer::NoHazard) 2496f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return false; 2497f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2498f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return true; 2499f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 2500f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2501f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Return true if right should be scheduled with higher priority than left. 2502f697c8a19adf962a933b055383952e72789a0e20Andrew Trickbool hybrid_ls_rr_sort::operator()(SUnit *left, SUnit *right) const { 250312f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick if (int res = checkSpecialNodes(left, right)) 250412f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick return res > 0; 250512f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick 25068239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng if (left->isCall || right->isCall) 25078239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng // No way to compute latency of calls. 25088239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng return BURRSort(left, right, SPQ); 25098239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng 2510e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng bool LHigh = SPQ->HighRegPressure(left); 2511e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng bool RHigh = SPQ->HighRegPressure(right); 251270017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng // Avoid causing spills. If register pressure is high, schedule for 251370017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng // register pressure reduction. 25140bc308600b0069f07ba722b472c68588573ebd28Andrew Trick if (LHigh && !RHigh) { 25150bc308600b0069f07ba722b472c68588573ebd28Andrew Trick DEBUG(dbgs() << " pressure SU(" << left->NodeNum << ") > SU(" 25160bc308600b0069f07ba722b472c68588573ebd28Andrew Trick << right->NodeNum << ")\n"); 25174a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng return true; 25180bc308600b0069f07ba722b472c68588573ebd28Andrew Trick } 25190bc308600b0069f07ba722b472c68588573ebd28Andrew Trick else if (!LHigh && RHigh) { 25200bc308600b0069f07ba722b472c68588573ebd28Andrew Trick DEBUG(dbgs() << " pressure SU(" << right->NodeNum << ") > SU(" 25210bc308600b0069f07ba722b472c68588573ebd28Andrew Trick << left->NodeNum << ")\n"); 25224a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng return false; 25230bc308600b0069f07ba722b472c68588573ebd28Andrew Trick } 252487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (!LHigh && !RHigh) { 252587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick int result = BUCompareLatency(left, right, true /*checkPref*/, SPQ); 252687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (result != 0) 252787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick return result > 0; 252854699765064842fd08d1466adc93453660bc2a85Andrew Trick } 252915a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng return BURRSort(left, right, SPQ); 253015a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng} 253115a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 25322da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick// Schedule as many instructions in each cycle as possible. So don't make an 25332da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick// instruction available unless it is ready in the current cycle. 25342da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickbool ilp_ls_rr_sort::isReady(SUnit *SU, unsigned CurCycle) const { 2535f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SU->getHeight() > CurCycle) return false; 2536f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2537f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SPQ->getHazardRec()->getHazardType(SU, 0) 2538f697c8a19adf962a933b055383952e72789a0e20Andrew Trick != ScheduleHazardRecognizer::NoHazard) 2539f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return false; 2540f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2541a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick return true; 25422da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 25432da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 2544d7cdc3e1f9df50be73e239c130dc92239c0e5d32Benjamin Kramerstatic bool canEnableCoalescing(SUnit *SU) { 25450bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick unsigned Opc = SU->getNode() ? SU->getNode()->getOpcode() : 0; 25460bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (Opc == ISD::TokenFactor || Opc == ISD::CopyToReg) 25470bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick // CopyToReg should be close to its uses to facilitate coalescing and 25480bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick // avoid spilling. 25490bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick return true; 25500bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick 25510bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (Opc == TargetOpcode::EXTRACT_SUBREG || 25520bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick Opc == TargetOpcode::SUBREG_TO_REG || 25530bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick Opc == TargetOpcode::INSERT_SUBREG) 25540bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick // EXTRACT_SUBREG, INSERT_SUBREG, and SUBREG_TO_REG nodes should be 25550bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick // close to their uses to facilitate coalescing. 25560bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick return true; 25570bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick 25580bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (SU->NumPreds == 0 && SU->NumSuccs != 0) 25590bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick // If SU does not have a register def, schedule it close to its uses 25600bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick // because it does not lengthen any live ranges. 25610bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick return true; 25620bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick 25630bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick return false; 25640bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick} 25650bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick 2566ab2e3e2d7074207e2a4bb15e2913fa83795bb1caAndrew Trick// list-ilp is currently an experimental scheduler that allows various 2567ab2e3e2d7074207e2a4bb15e2913fa83795bb1caAndrew Trick// heuristics to be enabled prior to the normal register reduction logic. 2568f697c8a19adf962a933b055383952e72789a0e20Andrew Trickbool ilp_ls_rr_sort::operator()(SUnit *left, SUnit *right) const { 256912f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick if (int res = checkSpecialNodes(left, right)) 257012f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick return res > 0; 257112f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick 25728239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng if (left->isCall || right->isCall) 25738239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng // No way to compute latency of calls. 25748239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng return BURRSort(left, right, SPQ); 25758239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng 25760bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick unsigned LLiveUses = 0, RLiveUses = 0; 25770bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick int LPDiff = 0, RPDiff = 0; 25780bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (!DisableSchedRegPressure || !DisableSchedLiveUses) { 25790bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick LPDiff = SPQ->RegPressureDiff(left, LLiveUses); 25800bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick RPDiff = SPQ->RegPressureDiff(right, RLiveUses); 25810bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick } 2582e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (!DisableSchedRegPressure && LPDiff != RPDiff) { 25830bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick DEBUG(dbgs() << "RegPressureDiff SU(" << left->NodeNum << "): " << LPDiff 25840bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick << " != SU(" << right->NodeNum << "): " << RPDiff << "\n"); 2585e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick return LPDiff > RPDiff; 2586e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick } 2587e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 25880bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (!DisableSchedRegPressure && (LPDiff > 0 || RPDiff > 0)) { 2589d7cdc3e1f9df50be73e239c130dc92239c0e5d32Benjamin Kramer bool LReduce = canEnableCoalescing(left); 2590d7cdc3e1f9df50be73e239c130dc92239c0e5d32Benjamin Kramer bool RReduce = canEnableCoalescing(right); 25910bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (LReduce && !RReduce) return false; 25920bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (RReduce && !LReduce) return true; 25930bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick } 25940bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick 25950bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (!DisableSchedLiveUses && (LLiveUses != RLiveUses)) { 25960bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick DEBUG(dbgs() << "Live uses SU(" << left->NodeNum << "): " << LLiveUses 25970bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick << " != SU(" << right->NodeNum << "): " << RLiveUses << "\n"); 2598e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick return LLiveUses < RLiveUses; 2599e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick } 2600e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 26010bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (!DisableSchedStalls) { 26020bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick bool LStall = BUHasStall(left, left->getHeight(), SPQ); 26030bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick bool RStall = BUHasStall(right, right->getHeight(), SPQ); 2604e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (LStall != RStall) 26050bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick return left->getHeight() > right->getHeight(); 2606e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick } 2607e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 2608afc7d235e91a769f74d87bbe745558ed1b692ff7Andrew Trick if (!DisableSchedCriticalPath) { 2609afc7d235e91a769f74d87bbe745558ed1b692ff7Andrew Trick int spread = (int)left->getDepth() - (int)right->getDepth(); 2610afc7d235e91a769f74d87bbe745558ed1b692ff7Andrew Trick if (std::abs(spread) > MaxReorderWindow) { 26110bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick DEBUG(dbgs() << "Depth of SU(" << left->NodeNum << "): " 26120bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick << left->getDepth() << " != SU(" << right->NodeNum << "): " 26130bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick << right->getDepth() << "\n"); 2614afc7d235e91a769f74d87bbe745558ed1b692ff7Andrew Trick return left->getDepth() < right->getDepth(); 2615afc7d235e91a769f74d87bbe745558ed1b692ff7Andrew Trick } 2616e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick } 2617e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 2618e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (!DisableSchedHeight && left->getHeight() != right->getHeight()) { 26190bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick int spread = (int)left->getHeight() - (int)right->getHeight(); 2620e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (std::abs(spread) > MaxReorderWindow) 26210bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick return left->getHeight() > right->getHeight(); 262270017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng } 262370017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 262470017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng return BURRSort(left, right, SPQ); 262570017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng} 262670017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 262787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trickvoid RegReductionPQBase::initNodes(std::vector<SUnit> &sunits) { 262887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick SUnits = &sunits; 262987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // Add pseudo dependency edges for two-address nodes. 2630623a7e146bd86747dc46a6f8bb9993fc217d6b78Evan Cheng if (!Disable2AddrHack) 2631623a7e146bd86747dc46a6f8bb9993fc217d6b78Evan Cheng AddPseudoTwoAddrDeps(); 263287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // Reroute edges to nodes with multiple uses. 2633479389a4da53ce72226366cc6d1cad13da158909Evan Cheng if (!TracksRegPressure && !SrcOrder) 263487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick PrescheduleNodesWithMultipleUses(); 263587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // Calculate node priorities. 263687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick CalculateSethiUllmanNumbers(); 263787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 263887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // For single block loops, mark nodes that look like canonical IV increments. 263987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (scheduleDAG->BB->isSuccessor(scheduleDAG->BB)) { 264087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick for (unsigned i = 0, e = sunits.size(); i != e; ++i) { 264187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick initVRegCycle(&sunits[i]); 264287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick } 264387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick } 264487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick} 264587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 2646f697c8a19adf962a933b055383952e72789a0e20Andrew Trick//===----------------------------------------------------------------------===// 2647f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Preschedule for Register Pressure 2648f697c8a19adf962a933b055383952e72789a0e20Andrew Trick//===----------------------------------------------------------------------===// 2649f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2650f697c8a19adf962a933b055383952e72789a0e20Andrew Trickbool RegReductionPQBase::canClobber(const SUnit *SU, const SUnit *Op) { 265195f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng if (SU->isTwoAddress) { 2652550f5afb68ce8f034991863cac65bef22a6554daDan Gohman unsigned Opc = SU->getNode()->getMachineOpcode(); 2653e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng const MCInstrDesc &MCID = TII->get(Opc); 2654e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng unsigned NumRes = MCID.getNumDefs(); 2655e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng unsigned NumOps = MCID.getNumOperands() - NumRes; 265695f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng for (unsigned i = 0; i != NumOps; ++i) { 2657e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng if (MCID.getOperandConstraint(i+NumRes, MCOI::TIED_TO) != -1) { 2658550f5afb68ce8f034991863cac65bef22a6554daDan Gohman SDNode *DU = SU->getNode()->getOperand(i).getNode(); 265994d7a5f8156e62532870fbaf197377b34e52ff2aDan Gohman if (DU->getNodeId() != -1 && 266094d7a5f8156e62532870fbaf197377b34e52ff2aDan Gohman Op->OrigNode == &(*SUnits)[DU->getNodeId()]) 266195f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng return true; 266295f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 266395f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 2664e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 2665e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng return false; 2666e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 2667e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 2668340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick/// canClobberReachingPhysRegUse - True if SU would clobber one of it's 2669340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick/// successor's explicit physregs whose definition can reach DepSU. 2670340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick/// i.e. DepSU should not be scheduled above SU. 2671340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trickstatic bool canClobberReachingPhysRegUse(const SUnit *DepSU, const SUnit *SU, 2672340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick ScheduleDAGRRList *scheduleDAG, 2673340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick const TargetInstrInfo *TII, 2674340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick const TargetRegisterInfo *TRI) { 2675fac259814923d091942b230e7bd002a8d1130bc3Craig Topper const uint16_t *ImpDefs 2676340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick = TII->get(SU->getNode()->getMachineOpcode()).getImplicitDefs(); 267716a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen const uint32_t *RegMask = getNodeRegMask(SU->getNode()); 267816a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if(!ImpDefs && !RegMask) 2679340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick return false; 2680340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick 2681340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick for (SUnit::const_succ_iterator SI = SU->Succs.begin(), SE = SU->Succs.end(); 2682340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick SI != SE; ++SI) { 2683340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick SUnit *SuccSU = SI->getSUnit(); 2684340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick for (SUnit::const_pred_iterator PI = SuccSU->Preds.begin(), 2685340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick PE = SuccSU->Preds.end(); PI != PE; ++PI) { 2686340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick if (!PI->isAssignedRegDep()) 2687340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick continue; 2688340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick 268916a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (RegMask && MachineOperand::clobbersPhysReg(RegMask, PI->getReg()) && 269016a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen scheduleDAG->IsReachable(DepSU, PI->getSUnit())) 269116a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen return true; 269216a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen 269316a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (ImpDefs) 2694fac259814923d091942b230e7bd002a8d1130bc3Craig Topper for (const uint16_t *ImpDef = ImpDefs; *ImpDef; ++ImpDef) 269516a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen // Return true if SU clobbers this physical register use and the 269616a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen // definition of the register reaches from DepSU. IsReachable queries 269716a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen // a topological forward sort of the DAG (following the successors). 269816a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (TRI->regsOverlap(*ImpDef, PI->getReg()) && 269916a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen scheduleDAG->IsReachable(DepSU, PI->getSUnit())) 270016a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen return true; 2701340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick } 2702340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick } 2703340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick return false; 2704340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick} 2705340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick 2706180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng/// canClobberPhysRegDefs - True if SU would clobber one of SuccSU's 27072f1d3108e481758da66662f72673741da86312daDan Gohman/// physical register defs. 2708430b8a22e2717d3dfb6b4f096bc23c9538fd7959Dan Gohmanstatic bool canClobberPhysRegDefs(const SUnit *SuccSU, const SUnit *SU, 2709180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng const TargetInstrInfo *TII, 27106f0d024a534af18d9e60b3ea757376cd8a3a980eDan Gohman const TargetRegisterInfo *TRI) { 2711550f5afb68ce8f034991863cac65bef22a6554daDan Gohman SDNode *N = SuccSU->getNode(); 2712e8be6c63915e0389f1eef6b53c64300d13b2ce99Dan Gohman unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs(); 2713fac259814923d091942b230e7bd002a8d1130bc3Craig Topper const uint16_t *ImpDefs = TII->get(N->getMachineOpcode()).getImplicitDefs(); 27142f1d3108e481758da66662f72673741da86312daDan Gohman assert(ImpDefs && "Caller should check hasPhysRegDefs"); 2715a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman for (const SDNode *SUNode = SU->getNode(); SUNode; 271629d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner SUNode = SUNode->getGluedNode()) { 2717a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman if (!SUNode->isMachineOpcode()) 271859932584a812685b16ad6a53a023b2bb3fc21819Dan Gohman continue; 2719fac259814923d091942b230e7bd002a8d1130bc3Craig Topper const uint16_t *SUImpDefs = 2720a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman TII->get(SUNode->getMachineOpcode()).getImplicitDefs(); 272116a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen const uint32_t *SURegMask = getNodeRegMask(SUNode); 272216a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (!SUImpDefs && !SURegMask) 272316a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen continue; 2724a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman for (unsigned i = NumDefs, e = N->getNumValues(); i != e; ++i) { 2725e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = N->getValueType(i); 2726f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner if (VT == MVT::Glue || VT == MVT::Other) 2727a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman continue; 2728a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman if (!N->hasAnyUseOfValue(i)) 2729a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman continue; 2730a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman unsigned Reg = ImpDefs[i - NumDefs]; 273116a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (SURegMask && MachineOperand::clobbersPhysReg(SURegMask, Reg)) 273216a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen return true; 273316a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (!SUImpDefs) 273416a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen continue; 2735a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman for (;*SUImpDefs; ++SUImpDefs) { 2736a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman unsigned SUReg = *SUImpDefs; 2737a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman if (TRI->regsOverlap(Reg, SUReg)) 2738a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman return true; 2739a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman } 2740180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng } 2741180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng } 2742180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng return false; 2743180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng} 2744180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng 2745002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// PrescheduleNodesWithMultipleUses - Nodes with multiple uses 2746002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// are not handled well by the general register pressure reduction 2747002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// heuristics. When presented with code like this: 2748002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2749002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// N 2750002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// / | 2751002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// / | 2752002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// U store 2753002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// | 2754002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// ... 2755002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2756002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// the heuristics tend to push the store up, but since the 2757002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// operand of the store has another use (U), this would increase 2758002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// the length of that other use (the U->N edge). 2759002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2760002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// This function transforms code like the above to route U's 2761002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// dependence through the store when possible, like this: 2762002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2763002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// N 2764002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// || 2765002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// || 2766002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// store 2767002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// | 2768002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// U 2769002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// | 2770002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// ... 2771002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2772002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// This results in the store being scheduled immediately 2773002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// after N, which shortens the U->N live range, reducing 2774002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// register pressure. 2775002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2776f697c8a19adf962a933b055383952e72789a0e20Andrew Trickvoid RegReductionPQBase::PrescheduleNodesWithMultipleUses() { 2777002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Visit all the nodes in topological order, working top-down. 2778002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman for (unsigned i = 0, e = SUnits->size(); i != e; ++i) { 2779002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman SUnit *SU = &(*SUnits)[i]; 2780002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // For now, only look at nodes with no data successors, such as stores. 2781002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // These are especially important, due to the heuristics in 2782002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // getNodePriority for nodes with no data successors. 2783002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SU->NumSuccs != 0) 2784002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2785002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // For now, only look at nodes with exactly one data predecessor. 2786002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SU->NumPreds != 1) 2787002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2788002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Avoid prescheduling copies to virtual registers, which don't behave 2789002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // like other nodes from the perspective of scheduling heuristics. 2790002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SDNode *N = SU->getNode()) 2791002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (N->getOpcode() == ISD::CopyToReg && 2792002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman TargetRegisterInfo::isVirtualRegister 2793002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman (cast<RegisterSDNode>(N->getOperand(1))->getReg())) 2794002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2795002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman 2796002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Locate the single data predecessor. 2797002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman SUnit *PredSU = 0; 2798002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman for (SUnit::const_pred_iterator II = SU->Preds.begin(), 2799002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman EE = SU->Preds.end(); II != EE; ++II) 2800002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (!II->isCtrl()) { 2801002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman PredSU = II->getSUnit(); 2802002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman break; 2803002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman } 2804002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman assert(PredSU); 2805002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman 2806002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Don't rewrite edges that carry physregs, because that requires additional 2807002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // support infrastructure. 2808002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (PredSU->hasPhysRegDefs) 2809002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2810002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Short-circuit the case where SU is PredSU's only data successor. 2811002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (PredSU->NumSuccs == 1) 2812002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2813002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Avoid prescheduling to copies from virtual registers, which don't behave 281492e946630d5f9bb092853b93501387dd216899b9Andrew Trick // like other nodes from the perspective of scheduling heuristics. 2815002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SDNode *N = SU->getNode()) 2816002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (N->getOpcode() == ISD::CopyFromReg && 2817002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman TargetRegisterInfo::isVirtualRegister 2818002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman (cast<RegisterSDNode>(N->getOperand(1))->getReg())) 2819002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2820002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman 2821002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Perform checks on the successors of PredSU. 2822002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman for (SUnit::const_succ_iterator II = PredSU->Succs.begin(), 2823002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman EE = PredSU->Succs.end(); II != EE; ++II) { 2824002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman SUnit *PredSuccSU = II->getSUnit(); 2825002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (PredSuccSU == SU) continue; 2826002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // If PredSU has another successor with no data successors, for 2827002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // now don't attempt to choose either over the other. 2828002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (PredSuccSU->NumSuccs == 0) 2829002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman goto outer_loop_continue; 2830002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Don't break physical register dependencies. 2831002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SU->hasPhysRegClobbers && PredSuccSU->hasPhysRegDefs) 2832002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (canClobberPhysRegDefs(PredSuccSU, SU, TII, TRI)) 2833002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman goto outer_loop_continue; 2834002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Don't introduce graph cycles. 2835002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (scheduleDAG->IsReachable(SU, PredSuccSU)) 2836002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman goto outer_loop_continue; 2837002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman } 2838002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman 2839002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Ok, the transformation is safe and the heuristics suggest it is 2840002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // profitable. Update the graph. 284115a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng DEBUG(dbgs() << " Prescheduling SU #" << SU->NodeNum 284215a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng << " next to PredSU #" << PredSU->NodeNum 2843bbbfa99d3d18fe9f20265305e833666645ada528Chris Lattner << " to guide scheduling in the presence of multiple uses\n"); 2844002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman for (unsigned i = 0; i != PredSU->Succs.size(); ++i) { 2845002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman SDep Edge = PredSU->Succs[i]; 2846002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman assert(!Edge.isAssignedRegDep()); 2847002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman SUnit *SuccSU = Edge.getSUnit(); 2848002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SuccSU != SU) { 2849002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman Edge.setSUnit(PredSU); 2850002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman scheduleDAG->RemovePred(SuccSU, Edge); 2851002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman scheduleDAG->AddPred(SU, Edge); 2852002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman Edge.setSUnit(SU); 2853002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman scheduleDAG->AddPred(SuccSU, Edge); 2854002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman --i; 2855002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman } 2856002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman } 2857002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman outer_loop_continue:; 2858002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman } 2859002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman} 2860002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman 2861e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// AddPseudoTwoAddrDeps - If two nodes share an operand and one of them uses 2862e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// it as a def&use operand. Add a pseudo control edge from it to the other 2863e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// node (if it won't create a cycle) so the two-address one will be scheduled 286422a529990bb4bb86bdb2ae1cfce7340320a6ca7fEvan Cheng/// first (lower in the schedule). If both nodes are two-address, favor the 286522a529990bb4bb86bdb2ae1cfce7340320a6ca7fEvan Cheng/// one that has a CopyToReg use (more likely to be a loop induction update). 286622a529990bb4bb86bdb2ae1cfce7340320a6ca7fEvan Cheng/// If both are two-address, but one is commutable while the other is not 286722a529990bb4bb86bdb2ae1cfce7340320a6ca7fEvan Cheng/// commutable, favor the one that's not commutable. 2868ef0b3ca3a8935b5390633dc7bb4adcdb99e0c26aDuncan Sandsvoid RegReductionPQBase::AddPseudoTwoAddrDeps() { 286995f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng for (unsigned i = 0, e = SUnits->size(); i != e; ++i) { 2870430b8a22e2717d3dfb6b4f096bc23c9538fd7959Dan Gohman SUnit *SU = &(*SUnits)[i]; 287195f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng if (!SU->isTwoAddress) 287295f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng continue; 287395f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng 2874550f5afb68ce8f034991863cac65bef22a6554daDan Gohman SDNode *Node = SU->getNode(); 287529d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner if (!Node || !Node->isMachineOpcode() || SU->getNode()->getGluedNode()) 287695f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng continue; 287795f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng 2878089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng bool isLiveOut = hasOnlyLiveOutUses(SU); 2879e8be6c63915e0389f1eef6b53c64300d13b2ce99Dan Gohman unsigned Opc = Node->getMachineOpcode(); 2880e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng const MCInstrDesc &MCID = TII->get(Opc); 2881e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng unsigned NumRes = MCID.getNumDefs(); 2882e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng unsigned NumOps = MCID.getNumOperands() - NumRes; 288395f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng for (unsigned j = 0; j != NumOps; ++j) { 2884e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng if (MCID.getOperandConstraint(j+NumRes, MCOI::TIED_TO) == -1) 2885c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman continue; 2886c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman SDNode *DU = SU->getNode()->getOperand(j).getNode(); 2887c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if (DU->getNodeId() == -1) 2888c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman continue; 2889c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman const SUnit *DUSU = &(*SUnits)[DU->getNodeId()]; 2890c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if (!DUSU) continue; 2891c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman for (SUnit::const_succ_iterator I = DUSU->Succs.begin(), 2892c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman E = DUSU->Succs.end(); I != E; ++I) { 289354e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isCtrl()) continue; 289454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *SuccSU = I->getSUnit(); 2895c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if (SuccSU == SU) 28967da8f399bf09e9a03fe8bdd8c8eef6e5a7d87327Evan Cheng continue; 2897c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman // Be conservative. Ignore if nodes aren't at roughly the same 2898c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman // depth and height. 28993f23744df4809eba94284e601e81489212c974d4Dan Gohman if (SuccSU->getHeight() < SU->getHeight() && 29003f23744df4809eba94284e601e81489212c974d4Dan Gohman (SU->getHeight() - SuccSU->getHeight()) > 1) 2901c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman continue; 29020e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman // Skip past COPY_TO_REGCLASS nodes, so that the pseudo edge 29030e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman // constrains whatever is using the copy, instead of the copy 29040e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman // itself. In the case that the copy is coalesced, this 29050e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman // preserves the intent of the pseudo two-address heurietics. 29060e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman while (SuccSU->Succs.size() == 1 && 29070e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman SuccSU->getNode()->isMachineOpcode() && 29080e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman SuccSU->getNode()->getMachineOpcode() == 2909518bb53485df640d7b7e3f6b0544099020c42aa7Chris Lattner TargetOpcode::COPY_TO_REGCLASS) 29100e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman SuccSU = SuccSU->Succs.front().getSUnit(); 29110e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman // Don't constrain non-instruction nodes. 2912c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if (!SuccSU->getNode() || !SuccSU->getNode()->isMachineOpcode()) 2913c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman continue; 2914c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman // Don't constrain nodes with physical register defs if the 2915c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman // predecessor can clobber them. 29168f4aa333d02d0f48f90f4604d894a73ee53edcb5Dan Gohman if (SuccSU->hasPhysRegDefs && SU->hasPhysRegClobbers) { 2917c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if (canClobberPhysRegDefs(SuccSU, SU, TII, TRI)) 291832dfbeada7292167bb488f36a71a5a6a519ddaffEvan Cheng continue; 2919c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman } 29208af808a3d64dfdcc5d126e6ad762f57b1483671cDan Gohman // Don't constrain EXTRACT_SUBREG, INSERT_SUBREG, and SUBREG_TO_REG; 29218af808a3d64dfdcc5d126e6ad762f57b1483671cDan Gohman // these may be coalesced away. We want them close to their uses. 2922c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman unsigned SuccOpc = SuccSU->getNode()->getMachineOpcode(); 2923518bb53485df640d7b7e3f6b0544099020c42aa7Chris Lattner if (SuccOpc == TargetOpcode::EXTRACT_SUBREG || 2924518bb53485df640d7b7e3f6b0544099020c42aa7Chris Lattner SuccOpc == TargetOpcode::INSERT_SUBREG || 2925518bb53485df640d7b7e3f6b0544099020c42aa7Chris Lattner SuccOpc == TargetOpcode::SUBREG_TO_REG) 2926c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman continue; 2927340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick if (!canClobberReachingPhysRegUse(SuccSU, SU, scheduleDAG, TII, TRI) && 2928340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick (!canClobber(SuccSU, DUSU) || 2929089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng (isLiveOut && !hasOnlyLiveOutUses(SuccSU)) || 2930c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman (!SU->isCommutable && SuccSU->isCommutable)) && 2931c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman !scheduleDAG->IsReachable(SuccSU, SU)) { 293215a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng DEBUG(dbgs() << " Adding a pseudo-two-addr edge from SU #" 2933bbbfa99d3d18fe9f20265305e833666645ada528Chris Lattner << SU->NodeNum << " to SU #" << SuccSU->NodeNum << "\n"); 2934a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick scheduleDAG->AddPred(SU, SDep(SuccSU, SDep::Artificial)); 293595f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 293695f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 293795f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 293895f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 2939e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 2940e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 2941e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 2942e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// Public Constructor Functions 2943e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 2944e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 294547ac0f0c7c39289f5970688154e385be22b7f293Dan Gohmanllvm::ScheduleDAGSDNodes * 29462da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickllvm::createBURRListDAGScheduler(SelectionDAGISel *IS, 29472da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CodeGenOpt::Level OptLevel) { 294879ce276083ced01256a0eb7d80731e4948ca6e87Dan Gohman const TargetMachine &TM = IS->TM; 294979ce276083ced01256a0eb7d80731e4948ca6e87Dan Gohman const TargetInstrInfo *TII = TM.getInstrInfo(); 295079ce276083ced01256a0eb7d80731e4948ca6e87Dan Gohman const TargetRegisterInfo *TRI = TM.getRegisterInfo(); 295138036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 29524f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng BURegReductionPriorityQueue *PQ = 2953479389a4da53ce72226366cc6d1cad13da158909Evan Cheng new BURegReductionPriorityQueue(*IS->MF, false, false, TII, TRI, 0); 29542da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, false, PQ, OptLevel); 2955c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng PQ->setScheduleDAG(SD); 295638036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick return SD; 2957e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 2958e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 295947ac0f0c7c39289f5970688154e385be22b7f293Dan Gohmanllvm::ScheduleDAGSDNodes * 29602da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickllvm::createSourceListDAGScheduler(SelectionDAGISel *IS, 29612da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CodeGenOpt::Level OptLevel) { 2962187361b056823df4ff292561fe47468dad956872Bill Wendling const TargetMachine &TM = IS->TM; 2963187361b056823df4ff292561fe47468dad956872Bill Wendling const TargetInstrInfo *TII = TM.getInstrInfo(); 2964187361b056823df4ff292561fe47468dad956872Bill Wendling const TargetRegisterInfo *TRI = TM.getRegisterInfo(); 296538036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 29664f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng SrcRegReductionPriorityQueue *PQ = 2967479389a4da53ce72226366cc6d1cad13da158909Evan Cheng new SrcRegReductionPriorityQueue(*IS->MF, false, true, TII, TRI, 0); 29682da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, false, PQ, OptLevel); 296915a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng PQ->setScheduleDAG(SD); 297038036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick return SD; 297115a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng} 297215a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 297315a16def6e70c8f7df1023da80ceb89887203b40Evan Chengllvm::ScheduleDAGSDNodes * 29742da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickllvm::createHybridListDAGScheduler(SelectionDAGISel *IS, 29752da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CodeGenOpt::Level OptLevel) { 297615a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng const TargetMachine &TM = IS->TM; 297715a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng const TargetInstrInfo *TII = TM.getInstrInfo(); 297815a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng const TargetRegisterInfo *TRI = TM.getRegisterInfo(); 29794f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng const TargetLowering *TLI = &IS->getTargetLowering(); 298038036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 29814f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng HybridBURRPriorityQueue *PQ = 2982479389a4da53ce72226366cc6d1cad13da158909Evan Cheng new HybridBURRPriorityQueue(*IS->MF, true, false, TII, TRI, TLI); 29832da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 29842da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, PQ, OptLevel); 2985187361b056823df4ff292561fe47468dad956872Bill Wendling PQ->setScheduleDAG(SD); 298638036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick return SD; 2987187361b056823df4ff292561fe47468dad956872Bill Wendling} 298870017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 298970017e44cdba1946cc478ce1856a3e855a767e28Evan Chengllvm::ScheduleDAGSDNodes * 29902da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickllvm::createILPListDAGScheduler(SelectionDAGISel *IS, 29912da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CodeGenOpt::Level OptLevel) { 299270017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng const TargetMachine &TM = IS->TM; 299370017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng const TargetInstrInfo *TII = TM.getInstrInfo(); 299470017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng const TargetRegisterInfo *TRI = TM.getRegisterInfo(); 299570017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng const TargetLowering *TLI = &IS->getTargetLowering(); 299638036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 299770017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng ILPBURRPriorityQueue *PQ = 2998479389a4da53ce72226366cc6d1cad13da158909Evan Cheng new ILPBURRPriorityQueue(*IS->MF, true, false, TII, TRI, TLI); 29992da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, PQ, OptLevel); 300070017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng PQ->setScheduleDAG(SD); 300138036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick return SD; 300270017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng} 3003