104f4f4f447806cd92a2fb6f4b66d11f6d5003a82Dan Gohman//===----- ScheduleDAGRRList.cpp - Reg pressure reduction list scheduler --===// 2e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 3e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// The LLVM Compiler Infrastructure 4e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 7e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 8e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 9e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 10e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// This implements bottom-up and top-down register pressure reduction list 11e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// schedulers, using standard algorithms. The basic approach uses a priority 12e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// queue of available nodes to schedule. One at a time, nodes are taken from 13e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// the priority queue (thus in priority order), checked for legality to 14e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// schedule, and emitted if legal. 15e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 16e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 17e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 18e7e7d0d7e39d0c7c659d26b97e8081fce0fcd749Dale Johannesen#define DEBUG_TYPE "pre-RA-sched" 19eb577ba3b815a1fa4627b060dd2345d17abf672dJim Laskey#include "llvm/CodeGen/SchedulerRegistry.h" 20d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "ScheduleDAGSDNodes.h" 21d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/STLExtras.h" 22a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng#include "llvm/ADT/SmallSet.h" 23e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#include "llvm/ADT/Statistic.h" 24f662a59b8b031bd43e43e0282b58bef920f0793dWeiming Zhao#include "llvm/CodeGen/MachineRegisterInfo.h" 25d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/CodeGen/ScheduleHazardRecognizer.h" 26d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/CodeGen/SelectionDAGISel.h" 270b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/DataLayout.h" 280b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/InlineAsm.h" 29decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner#include "llvm/Support/Debug.h" 30decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner#include "llvm/Support/ErrorHandling.h" 31bbbfa99d3d18fe9f20265305e833666645ada528Chris Lattner#include "llvm/Support/raw_ostream.h" 32d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Target/TargetInstrInfo.h" 33d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Target/TargetLowering.h" 34d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Target/TargetMachine.h" 35d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Target/TargetRegisterInfo.h" 36e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#include <climits> 37e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengusing namespace llvm; 38e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 39cffbd2562a5e3ba435dd2b622710ec272c634da5Dan GohmanSTATISTIC(NumBacktracks, "Number of times scheduler backtracked"); 40f10c973797cf79da802f9b0118543cbd50954c9cEvan ChengSTATISTIC(NumUnfolds, "Number of nodes unfolded"); 41a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan ChengSTATISTIC(NumDups, "Number of duplicated nodes"); 42c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan ChengSTATISTIC(NumPRCopies, "Number of physical register copies"); 43a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng 4413ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskeystatic RegisterScheduler 4513ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskey burrListDAGScheduler("list-burr", 46b8cab9227a0f6ffbdaae33e3c64268e265008a6aDan Gohman "Bottom-up register reduction list scheduling", 4713ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskey createBURRListDAGScheduler); 4813ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskeystatic RegisterScheduler 49187361b056823df4ff292561fe47468dad956872Bill Wendling sourceListDAGScheduler("source", 50187361b056823df4ff292561fe47468dad956872Bill Wendling "Similar to list-burr but schedules in source " 51187361b056823df4ff292561fe47468dad956872Bill Wendling "order when possible", 52187361b056823df4ff292561fe47468dad956872Bill Wendling createSourceListDAGScheduler); 5313ec702c430b91ee49b9e6d9581cd95412f216c8Jim Laskey 5415a16def6e70c8f7df1023da80ceb89887203b40Evan Chengstatic RegisterScheduler 55b11ac950d69c7a238de0a22fd23fbfcd994f57eeEvan Cheng hybridListDAGScheduler("list-hybrid", 5670017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng "Bottom-up register pressure aware list scheduling " 5770017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng "which tries to balance latency and register pressure", 5815a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng createHybridListDAGScheduler); 5915a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 6070017e44cdba1946cc478ce1856a3e855a767e28Evan Chengstatic RegisterScheduler 6170017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng ILPListDAGScheduler("list-ilp", 6270017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng "Bottom-up register pressure aware list scheduling " 6370017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng "which tries to balance ILP and register pressure", 6470017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng createILPListDAGScheduler); 6570017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 66c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trickstatic cl::opt<bool> DisableSchedCycles( 67d1dace8aea073716daf0055ad07fde1164b2a472Andrew Trick "disable-sched-cycles", cl::Hidden, cl::init(false), 68c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick cl::desc("Disable cycle-level precision during preRA scheduling")); 692da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 70e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick// Temporary sched=list-ilp flags until the heuristics are robust. 7112f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick// Some options are also available under sched=list-hybrid. 72e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickstatic cl::opt<bool> DisableSchedRegPressure( 73e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick "disable-sched-reg-pressure", cl::Hidden, cl::init(false), 74e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick cl::desc("Disable regpressure priority in sched=list-ilp")); 75e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickstatic cl::opt<bool> DisableSchedLiveUses( 763c6e49504e9a57a4818750fd2520967f84634eacAndrew Trick "disable-sched-live-uses", cl::Hidden, cl::init(true), 77e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick cl::desc("Disable live use priority in sched=list-ilp")); 7854699765064842fd08d1466adc93453660bc2a85Andrew Trickstatic cl::opt<bool> DisableSchedVRegCycle( 7954699765064842fd08d1466adc93453660bc2a85Andrew Trick "disable-sched-vrcycle", cl::Hidden, cl::init(false), 8054699765064842fd08d1466adc93453660bc2a85Andrew Trick cl::desc("Disable virtual register cycle interference checks")); 8112f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trickstatic cl::opt<bool> DisableSchedPhysRegJoin( 8212f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick "disable-sched-physreg-join", cl::Hidden, cl::init(false), 8312f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick cl::desc("Disable physreg def-use affinity")); 84e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickstatic cl::opt<bool> DisableSchedStalls( 853c6e49504e9a57a4818750fd2520967f84634eacAndrew Trick "disable-sched-stalls", cl::Hidden, cl::init(true), 86e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick cl::desc("Disable no-stall priority in sched=list-ilp")); 87e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickstatic cl::opt<bool> DisableSchedCriticalPath( 88e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick "disable-sched-critical-path", cl::Hidden, cl::init(false), 89e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick cl::desc("Disable critical path priority in sched=list-ilp")); 90e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickstatic cl::opt<bool> DisableSchedHeight( 91e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick "disable-sched-height", cl::Hidden, cl::init(false), 92e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick cl::desc("Disable scheduled-height priority in sched=list-ilp")); 93623a7e146bd86747dc46a6f8bb9993fc217d6b78Evan Chengstatic cl::opt<bool> Disable2AddrHack( 94623a7e146bd86747dc46a6f8bb9993fc217d6b78Evan Cheng "disable-2addr-hack", cl::Hidden, cl::init(true), 95623a7e146bd86747dc46a6f8bb9993fc217d6b78Evan Cheng cl::desc("Disable scheduler's two-address hack")); 96e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 97e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickstatic cl::opt<int> MaxReorderWindow( 98e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick "max-sched-reorder", cl::Hidden, cl::init(6), 99e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick cl::desc("Number of instructions to allow ahead of the critical path " 100e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick "in sched=list-ilp")); 101e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 102e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickstatic cl::opt<unsigned> AvgIPC( 103e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick "sched-avg-ipc", cl::Hidden, cl::init(1), 104e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick cl::desc("Average inst/cycle whan no target itinerary exists.")); 105e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 106e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengnamespace { 107e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 108e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// ScheduleDAGRRList - The actual register reduction list scheduler 109e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// implementation. This supports both top-down and bottom-up scheduling. 110e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// 1116726b6d75a8b679068a58cb954ba97cf9d1690baNick Lewyckyclass ScheduleDAGRRList : public ScheduleDAGSDNodes { 112e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengprivate: 11315a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng /// NeedLatency - True if the scheduler will make use of latency information. 11415a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng /// 11515a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng bool NeedLatency; 11615a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 117e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng /// AvailableQueue - The priority queue to use for the available SUnits. 118e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng SchedulingPriorityQueue *AvailableQueue; 119e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1202da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// PendingQueue - This contains all of the instructions whose operands have 1212da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// been issued, but their results are not ready yet (due to the latency of 1222da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// the operation). Once the operands becomes available, the instruction is 1232da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// added to the AvailableQueue. 1242da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick std::vector<SUnit*> PendingQueue; 1252da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1262da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// HazardRec - The hazard recognizer to use. 1272da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleHazardRecognizer *HazardRec; 1282da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1292902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick /// CurCycle - The current scheduler state corresponds to this cycle. 1302902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick unsigned CurCycle; 1312902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 1322da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick /// MinAvailableCycle - Cycle of the soonest available instruction. 1332da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned MinAvailableCycle; 1342da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 135e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick /// IssueCount - Count instructions issued in this cycle 136e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick /// Currently valid only for bottom-up scheduling. 137e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick unsigned IssueCount; 138e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 139086ec9976ff6cee083de618429c78473491d5713Dan Gohman /// LiveRegDefs - A set of physical registers and their definition 140a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng /// that are "live". These nodes must be scheduled before any other nodes that 141a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng /// modifies the registers can be scheduled. 142086ec9976ff6cee083de618429c78473491d5713Dan Gohman unsigned NumLiveRegs; 143a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng std::vector<SUnit*> LiveRegDefs; 1443d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick std::vector<SUnit*> LiveRegGens; 145a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 146029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick // Collect interferences between physical register use/defs. 147029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick // Each interference is an SUnit and set of physical registers. 148029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick SmallVector<SUnit*, 4> Interferences; 149029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick typedef DenseMap<SUnit*, SmallVector<unsigned, 4> > LRegsMapT; 150029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick LRegsMapT LRegsMap; 151029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick 15221d9003087c9a707e6cd95460136b499df358fb8Dan Gohman /// Topo - A topological ordering for SUnits which permits fast IsReachable 15321d9003087c9a707e6cd95460136b499df358fb8Dan Gohman /// and similar queries. 15421d9003087c9a707e6cd95460136b499df358fb8Dan Gohman ScheduleDAGTopologicalSort Topo; 15521d9003087c9a707e6cd95460136b499df358fb8Dan Gohman 1560e6307f6423a7ee39b80f8dc3caef00ad11e0266Eli Friedman // Hack to keep track of the inverse of FindCallSeqStart without more crazy 1570e6307f6423a7ee39b80f8dc3caef00ad11e0266Eli Friedman // DAG crawling. 1580e6307f6423a7ee39b80f8dc3caef00ad11e0266Eli Friedman DenseMap<SUnit*, SUnit*> CallSeqEndForStart; 1590e6307f6423a7ee39b80f8dc3caef00ad11e0266Eli Friedman 160e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengpublic: 1612da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleDAGRRList(MachineFunction &mf, bool needlatency, 1622da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SchedulingPriorityQueue *availqueue, 1632da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CodeGenOpt::Level OptLevel) 164ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman : ScheduleDAGSDNodes(mf), 1652da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick NeedLatency(needlatency), AvailableQueue(availqueue), CurCycle(0), 166ae692f2baedf53504af2715993b166950e185a55Andrew Trick Topo(SUnits, NULL) { 1672da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1682da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick const TargetMachine &tm = mf.getTarget(); 169c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick if (DisableSchedCycles || !NeedLatency) 1702da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec = new ScheduleHazardRecognizer(); 171c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick else 172c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick HazardRec = tm.getInstrInfo()->CreateTargetHazardRecognizer(&tm, this); 1732da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 174e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 175e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng ~ScheduleDAGRRList() { 1762da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick delete HazardRec; 177e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng delete AvailableQueue; 178e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 179e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 180e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng void Schedule(); 181e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 182f697c8a19adf962a933b055383952e72789a0e20Andrew Trick ScheduleHazardRecognizer *getHazardRec() { return HazardRec; } 183f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1848dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// IsReachable - Checks if SU is reachable from TargetSU. 18521d9003087c9a707e6cd95460136b499df358fb8Dan Gohman bool IsReachable(const SUnit *SU, const SUnit *TargetSU) { 18621d9003087c9a707e6cd95460136b499df358fb8Dan Gohman return Topo.IsReachable(SU, TargetSU); 18721d9003087c9a707e6cd95460136b499df358fb8Dan Gohman } 188e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 1891cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohman /// WillCreateCycle - Returns true if adding an edge from SU to TargetSU will 190e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein /// create a cycle. 19121d9003087c9a707e6cd95460136b499df358fb8Dan Gohman bool WillCreateCycle(SUnit *SU, SUnit *TargetSU) { 19221d9003087c9a707e6cd95460136b499df358fb8Dan Gohman return Topo.WillCreateCycle(SU, TargetSU); 19321d9003087c9a707e6cd95460136b499df358fb8Dan Gohman } 194e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 19554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman /// AddPred - adds a predecessor edge to SUnit SU. 1968dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// This returns true if this is a new predecessor. 1978dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// Updates the topological ordering if required. 198ffa391272bad598d73fd5404dadf3686b69f2a63Dan Gohman void AddPred(SUnit *SU, const SDep &D) { 19954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman Topo.AddPred(SU, D.getSUnit()); 200ffa391272bad598d73fd5404dadf3686b69f2a63Dan Gohman SU->addPred(D); 20121d9003087c9a707e6cd95460136b499df358fb8Dan Gohman } 202e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 20354e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman /// RemovePred - removes a predecessor edge from SUnit SU. 20454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman /// This returns true if an edge was removed. 20554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman /// Updates the topological ordering if required. 206ffa391272bad598d73fd5404dadf3686b69f2a63Dan Gohman void RemovePred(SUnit *SU, const SDep &D) { 20754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman Topo.RemovePred(SU, D.getSUnit()); 208ffa391272bad598d73fd5404dadf3686b69f2a63Dan Gohman SU->removePred(D); 20921d9003087c9a707e6cd95460136b499df358fb8Dan Gohman } 210e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 211e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengprivate: 2122da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick bool isReady(SUnit *SU) { 213c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick return DisableSchedCycles || !AvailableQueue->hasReadyFilter() || 2142da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->isReady(SU); 2152da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 2162da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 2171cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohman void ReleasePred(SUnit *SU, const SDep *PredEdge); 2183d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick void ReleasePredecessors(SUnit *SU); 2192da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void ReleasePending(); 2202da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void AdvanceToCycle(unsigned NextCycle); 2212da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void AdvancePastStalls(SUnit *SU); 2222da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void EmitNode(SUnit *SU); 2232902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick void ScheduleNodeBottomUp(SUnit*); 2242da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void CapturePred(SDep *PredEdge); 22542d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng void UnscheduleNodeBottomUp(SUnit*); 2262da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void RestoreHazardCheckerBottomUp(); 2272da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick void BacktrackBottomUp(SUnit*, SUnit*); 22842d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng SUnit *CopyAndMoveSuccessors(SUnit*); 229c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng void InsertCopiesAndMoveSuccs(SUnit*, unsigned, 230c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng const TargetRegisterClass*, 231c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng const TargetRegisterClass*, 232c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng SmallVector<SUnit*, 2>&); 233a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng bool DelayForLiveRegsBottomUp(SUnit*, SmallVector<unsigned, 4>&); 2342da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 235029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick void releaseInterferences(unsigned Reg = 0); 236029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick 2372902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *PickNodeToScheduleBottomUp(); 238e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng void ListScheduleBottomUp(); 239e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 240e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein /// CreateNewSUnit - Creates a new SUnit and returns a pointer to it. 2418dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// Updates the topological ordering if required. 242e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *CreateNewSUnit(SDNode *N) { 24321d9003087c9a707e6cd95460136b499df358fb8Dan Gohman unsigned NumSUnits = SUnits.size(); 244953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick SUnit *NewNode = newSUnit(N); 2458dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein // Update the topological ordering. 24621d9003087c9a707e6cd95460136b499df358fb8Dan Gohman if (NewNode->NodeNum >= NumSUnits) 24721d9003087c9a707e6cd95460136b499df358fb8Dan Gohman Topo.InitDAGTopologicalSorting(); 248e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein return NewNode; 249e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein } 250e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein 2518dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// CreateClone - Creates a new SUnit from an existing one. 2528dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein /// Updates the topological ordering if required. 253e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *CreateClone(SUnit *N) { 25421d9003087c9a707e6cd95460136b499df358fb8Dan Gohman unsigned NumSUnits = SUnits.size(); 255e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *NewNode = Clone(N); 2568dba9afd086f72db920db81a3d73c7297390cda7Roman Levenstein // Update the topological ordering. 25721d9003087c9a707e6cd95460136b499df358fb8Dan Gohman if (NewNode->NodeNum >= NumSUnits) 25821d9003087c9a707e6cd95460136b499df358fb8Dan Gohman Topo.InitDAGTopologicalSorting(); 259e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein return NewNode; 260e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein } 2613f23744df4809eba94284e601e81489212c974d4Dan Gohman 262953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick /// forceUnitLatencies - Register-pressure-reducing scheduling doesn't 26315a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng /// need actual latency information but the hybrid scheduler does. 264953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick bool forceUnitLatencies() const { 26515a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng return !NeedLatency; 26615a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng } 267e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng}; 268e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} // end anonymous namespace 269e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 27077b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson/// GetCostForDef - Looks up the register class and cost for a given definition. 27177b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson/// Typically this just means looking up the representative register class, 27299aa14ff64c92eab347d23696e358361d3bd90eaOwen Anderson/// but for untyped values (MVT::Untyped) it means inspecting the node's 27377b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson/// opcode to determine what register class is being generated. 27477b4b13c2a525faf646a6784b24692cf0459b75eOwen Andersonstatic void GetCostForDef(const ScheduleDAGSDNodes::RegDefIter &RegDefPos, 27577b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson const TargetLowering *TLI, 27677b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson const TargetInstrInfo *TII, 27777b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson const TargetRegisterInfo *TRI, 278397fc4874efe9c17e737d4c5c50bd19dc3bf27f5Jakob Stoklund Olesen unsigned &RegClass, unsigned &Cost, 279397fc4874efe9c17e737d4c5c50bd19dc3bf27f5Jakob Stoklund Olesen const MachineFunction &MF) { 280860e7cdab9d5eceda5ac52ae0ddfb4bdab0067f2Patrik Hagglund MVT VT = RegDefPos.GetValue(); 28177b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson 28277b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson // Special handling for untyped values. These values can only come from 28377b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson // the expansion of custom DAG-to-DAG patterns. 28499aa14ff64c92eab347d23696e358361d3bd90eaOwen Anderson if (VT == MVT::Untyped) { 285109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson const SDNode *Node = RegDefPos.GetNode(); 286109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson 287f662a59b8b031bd43e43e0282b58bef920f0793dWeiming Zhao // Special handling for CopyFromReg of untyped values. 288f662a59b8b031bd43e43e0282b58bef920f0793dWeiming Zhao if (!Node->isMachineOpcode() && Node->getOpcode() == ISD::CopyFromReg) { 289f662a59b8b031bd43e43e0282b58bef920f0793dWeiming Zhao unsigned Reg = cast<RegisterSDNode>(Node->getOperand(1))->getReg(); 290f662a59b8b031bd43e43e0282b58bef920f0793dWeiming Zhao const TargetRegisterClass *RC = MF.getRegInfo().getRegClass(Reg); 291f662a59b8b031bd43e43e0282b58bef920f0793dWeiming Zhao RegClass = RC->getID(); 292f662a59b8b031bd43e43e0282b58bef920f0793dWeiming Zhao Cost = 1; 293f662a59b8b031bd43e43e0282b58bef920f0793dWeiming Zhao return; 294f662a59b8b031bd43e43e0282b58bef920f0793dWeiming Zhao } 295f662a59b8b031bd43e43e0282b58bef920f0793dWeiming Zhao 296f662a59b8b031bd43e43e0282b58bef920f0793dWeiming Zhao unsigned Opcode = Node->getMachineOpcode(); 297109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson if (Opcode == TargetOpcode::REG_SEQUENCE) { 298109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson unsigned DstRCIdx = cast<ConstantSDNode>(Node->getOperand(0))->getZExtValue(); 299109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson const TargetRegisterClass *RC = TRI->getRegClass(DstRCIdx); 300109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson RegClass = RC->getID(); 301109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson Cost = 1; 302109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson return; 303109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson } 304109c22c06232358597afec5d8b7a6b6fd24e19b1Owen Anderson 30577b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson unsigned Idx = RegDefPos.GetIdx(); 306e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng const MCInstrDesc Desc = TII->get(Opcode); 307397fc4874efe9c17e737d4c5c50bd19dc3bf27f5Jakob Stoklund Olesen const TargetRegisterClass *RC = TII->getRegClass(Desc, Idx, TRI, MF); 30877b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson RegClass = RC->getID(); 30977b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson // FIXME: Cost arbitrarily set to 1 because there doesn't seem to be a 31077b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson // better way to determine it. 31177b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson Cost = 1; 31277b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson } else { 31377b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson RegClass = TLI->getRepRegClassFor(VT)->getID(); 31477b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson Cost = TLI->getRepRegClassCostFor(VT); 31577b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson } 31677b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson} 317e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 318e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// Schedule - Schedule the DAG using list scheduling. 319e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengvoid ScheduleDAGRRList::Schedule() { 3204f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng DEBUG(dbgs() 3214f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng << "********** List Scheduling BB#" << BB->getNumber() 322089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng << " '" << BB->getName() << "' **********\n"); 323a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 3242902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurCycle = 0; 325e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick IssueCount = 0; 326c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick MinAvailableCycle = DisableSchedCycles ? 0 : UINT_MAX; 327086ec9976ff6cee083de618429c78473491d5713Dan Gohman NumLiveRegs = 0; 32865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Allocate slots for each physical register, plus one for a special register 32965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // to track the virtual resource of a calling sequence. 33065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegDefs.resize(TRI->getNumRegs() + 1, NULL); 33165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegGens.resize(TRI->getNumRegs() + 1, NULL); 3320e6307f6423a7ee39b80f8dc3caef00ad11e0266Eli Friedman CallSeqEndForStart.clear(); 333029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick assert(Interferences.empty() && LRegsMap.empty() && "stale Interferences"); 334a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 335c9a5b9e38b442c2ae6b115213a07df3fcd14708dDan Gohman // Build the scheduling graph. 33698976e4dcd18adbbe676048c0069e67346eb4adeDan Gohman BuildSchedGraph(NULL); 337e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 338e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su) 3393cc6243ddfdba3ad64035b919c88b09773a60880Dan Gohman SUnits[su].dumpAll(this)); 34021d9003087c9a707e6cd95460136b499df358fb8Dan Gohman Topo.InitDAGTopologicalSorting(); 341e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 34294d7a5f8156e62532870fbaf197377b34e52ff2aDan Gohman AvailableQueue->initNodes(SUnits); 34338036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 3442da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->Reset(); 3452da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 346ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman // Execute the actual scheduling loop. 347ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman ListScheduleBottomUp(); 34838036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 349e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng AvailableQueue->releaseState(); 35073ba69b6843f7f23345b1e8745cb328952cae0d8Andrew Trick 35173ba69b6843f7f23345b1e8745cb328952cae0d8Andrew Trick DEBUG({ 35273ba69b6843f7f23345b1e8745cb328952cae0d8Andrew Trick dbgs() << "*** Final schedule ***\n"; 35373ba69b6843f7f23345b1e8745cb328952cae0d8Andrew Trick dumpSchedule(); 35473ba69b6843f7f23345b1e8745cb328952cae0d8Andrew Trick dbgs() << '\n'; 35573ba69b6843f7f23345b1e8745cb328952cae0d8Andrew Trick }); 35613d41b9d721f98372b97d2ec119e6c91932ab0aeEvan Cheng} 357e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 358e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 359e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// Bottom-Up Scheduling 360e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 361e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 362e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// ReleasePred - Decrement the NumSuccsLeft count of a predecessor. Add it to 3638d1bfad00b1ebff5b140b6e1bd7e26bad697d6e1Dan Gohman/// the AvailableQueue if the count reaches zero. Also update its cycle bound. 3641cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohmanvoid ScheduleDAGRRList::ReleasePred(SUnit *SU, const SDep *PredEdge) { 36554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *PredSU = PredEdge->getSUnit(); 3663a90c9b8edb53ea1ea57d94f7ab256608b5b30c6Reid Kleckner 367e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#ifndef NDEBUG 3683a90c9b8edb53ea1ea57d94f7ab256608b5b30c6Reid Kleckner if (PredSU->NumSuccsLeft == 0) { 369e492ae13edd83b120d665c0503cf4de2925b5e56David Greene dbgs() << "*** Scheduling failed! ***\n"; 3703cc6243ddfdba3ad64035b919c88b09773a60880Dan Gohman PredSU->dump(this); 371e492ae13edd83b120d665c0503cf4de2925b5e56David Greene dbgs() << " has been released too many times!\n"; 372c23197a26f34f559ea9797de51e187087c039c42Torok Edwin llvm_unreachable(0); 373e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 374e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#endif 3753a90c9b8edb53ea1ea57d94f7ab256608b5b30c6Reid Kleckner --PredSU->NumSuccsLeft; 3763a90c9b8edb53ea1ea57d94f7ab256608b5b30c6Reid Kleckner 377953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick if (!forceUnitLatencies()) { 37815a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng // Updating predecessor's height. This is now the cycle when the 37915a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng // predecessor can be scheduled without causing a pipeline stall. 38015a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng PredSU->setHeightToAtLeast(SU->getHeight() + PredEdge->getLatency()); 38115a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng } 38215a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 3839e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman // If all the node's successors are scheduled, this node is ready 3849e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman // to be scheduled. Ignore the special EntrySU node. 3859e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman if (PredSU->NumSuccsLeft == 0 && PredSU != &EntrySU) { 38680792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman PredSU->isAvailable = true; 3872da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 3882da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned Height = PredSU->getHeight(); 3892da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (Height < MinAvailableCycle) 3902da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick MinAvailableCycle = Height; 3912da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 392a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick if (isReady(PredSU)) { 3932da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->push(PredSU); 3942da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 3952da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // CapturePred and others may have left the node in the pending queue, avoid 3962da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // adding it twice. 3972da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick else if (!PredSU->isPending) { 3982da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PredSU->isPending = true; 3992da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PendingQueue.push_back(PredSU); 4002da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 401e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 402e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 403e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 40465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// IsChainDependent - Test if Outer is reachable from Inner through 40565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// chain dependencies. 40665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohmanstatic bool IsChainDependent(SDNode *Outer, SDNode *Inner, 40765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned NestLevel, 40865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman const TargetInstrInfo *TII) { 40965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SDNode *N = Outer; 41065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (;;) { 41165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N == Inner) 41265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return true; 41365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // For a TokenFactor, examine each operand. There may be multiple ways 41465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // to get to the CALLSEQ_BEGIN, but we need to find the path with the 41565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // most nesting in order to ensure that we find the corresponding match. 41665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->getOpcode() == ISD::TokenFactor) { 41765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 41865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (IsChainDependent(N->getOperand(i).getNode(), Inner, NestLevel, TII)) 41965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return true; 42065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return false; 42165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 42265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Check for a lowered CALLSEQ_BEGIN or CALLSEQ_END. 42365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->isMachineOpcode()) { 42465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->getMachineOpcode() == 42565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman (unsigned)TII->getCallFrameDestroyOpcode()) { 42665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman ++NestLevel; 42765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } else if (N->getMachineOpcode() == 42865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman (unsigned)TII->getCallFrameSetupOpcode()) { 42965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (NestLevel == 0) 43065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return false; 43165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman --NestLevel; 43265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 43365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 43465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Otherwise, find the chain and continue climbing. 43565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 43665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->getOperand(i).getValueType() == MVT::Other) { 43765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman N = N->getOperand(i).getNode(); 43865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman goto found_chain_operand; 43965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 44065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return false; 44165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman found_chain_operand:; 44265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->getOpcode() == ISD::EntryToken) 44365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return false; 44465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 44565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman} 44665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman 44765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// FindCallSeqStart - Starting from the (lowered) CALLSEQ_END node, locate 44865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// the corresponding (lowered) CALLSEQ_BEGIN node. 44965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// 45065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// NestLevel and MaxNested are used in recursion to indcate the current level 45165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// of nesting of CALLSEQ_BEGIN and CALLSEQ_END pairs, as well as the maximum 45265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// level seen so far. 45365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// 45465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// TODO: It would be better to give CALLSEQ_END an explicit operand to point 45565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman/// to the corresponding CALLSEQ_BEGIN to avoid needing to search for it. 45665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohmanstatic SDNode * 45765fd6564b8aedd053845c81ede1ac594acb470e4Dan GohmanFindCallSeqStart(SDNode *N, unsigned &NestLevel, unsigned &MaxNest, 45865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman const TargetInstrInfo *TII) { 45965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (;;) { 46065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // For a TokenFactor, examine each operand. There may be multiple ways 46165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // to get to the CALLSEQ_BEGIN, but we need to find the path with the 46265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // most nesting in order to ensure that we find the corresponding match. 46365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->getOpcode() == ISD::TokenFactor) { 46465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SDNode *Best = 0; 46565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned BestMaxNest = MaxNest; 46665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 46765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned MyNestLevel = NestLevel; 46865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned MyMaxNest = MaxNest; 46965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (SDNode *New = FindCallSeqStart(N->getOperand(i).getNode(), 47065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman MyNestLevel, MyMaxNest, TII)) 47165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (!Best || (MyMaxNest > BestMaxNest)) { 47265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman Best = New; 47365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman BestMaxNest = MyMaxNest; 47465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 47565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 47665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman assert(Best); 47765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman MaxNest = BestMaxNest; 47865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return Best; 47965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 48065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Check for a lowered CALLSEQ_BEGIN or CALLSEQ_END. 48165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->isMachineOpcode()) { 48265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->getMachineOpcode() == 48365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman (unsigned)TII->getCallFrameDestroyOpcode()) { 48465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman ++NestLevel; 48565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman MaxNest = std::max(MaxNest, NestLevel); 48665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } else if (N->getMachineOpcode() == 48765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman (unsigned)TII->getCallFrameSetupOpcode()) { 48865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman assert(NestLevel != 0); 48965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman --NestLevel; 49065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (NestLevel == 0) 49165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return N; 49265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 49365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 49465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Otherwise, find the chain and continue climbing. 49565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 49665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->getOperand(i).getValueType() == MVT::Other) { 49765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman N = N->getOperand(i).getNode(); 49865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman goto found_chain_operand; 49965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 50065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return 0; 50165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman found_chain_operand:; 50265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (N->getOpcode() == ISD::EntryToken) 50365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman return 0; 50465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 50565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman} 50665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman 5071b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// Call ReleasePred for each predecessor, then update register live def/gen. 5081b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// Always update LiveRegDefs for a register dependence even if the current SU 5091b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// also defines the register. This effectively create one large live range 5101b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// across a sequence of two-address node. This is important because the 5111b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// entire chain must be scheduled together. Example: 5121b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// 5131b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// flags = (3) add 5141b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// flags = (2) addc flags 5151b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// flags = (1) addc flags 5161b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// 5171b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// results in 5181b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// 5191b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// LiveRegDefs[flags] = 3 5203d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick/// LiveRegGens[flags] = 1 5211b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// 5221b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// If (2) addc is unscheduled, then (1) addc must also be unscheduled to avoid 5231b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick/// interference on flags. 5243d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trickvoid ScheduleDAGRRList::ReleasePredecessors(SUnit *SU) { 525e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // Bottom up: release predecessors 526228a18e0f220fb85ee06fd5bfa29304e57047ff1Chris Lattner for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 527a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 52854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman ReleasePred(SU, &*I); 52954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isAssignedRegDep()) { 530a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // This is a physical register dependency and it's impossible or 53138036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick // expensive to copy the register. Make sure nothing that can 532a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // clobber the register is scheduled between the predecessor and 533a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // this node. 5343d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick SUnit *RegDef = LiveRegDefs[I->getReg()]; (void)RegDef; 5351b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick assert((!RegDef || RegDef == SU || RegDef == I->getSUnit()) && 5361b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick "interference on register dependence"); 5373d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick LiveRegDefs[I->getReg()] = I->getSUnit(); 5383d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick if (!LiveRegGens[I->getReg()]) { 539086ec9976ff6cee083de618429c78473491d5713Dan Gohman ++NumLiveRegs; 5403d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick LiveRegGens[I->getReg()] = SU; 541a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 542a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 543a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 54465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman 54565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // If we're scheduling a lowered CALLSEQ_END, find the corresponding 54665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // CALLSEQ_BEGIN. Inject an artificial physical register dependence between 54765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // these nodes, to prevent other calls from being interscheduled with them. 54865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned CallResource = TRI->getNumRegs(); 54965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (!LiveRegDefs[CallResource]) 55065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) 55165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (Node->isMachineOpcode() && 55265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman Node->getMachineOpcode() == (unsigned)TII->getCallFrameDestroyOpcode()) { 55365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned NestLevel = 0; 55465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned MaxNest = 0; 55565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SDNode *N = FindCallSeqStart(Node, NestLevel, MaxNest, TII); 55665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman 55765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SUnit *Def = &SUnits[N->getNodeId()]; 5580e6307f6423a7ee39b80f8dc3caef00ad11e0266Eli Friedman CallSeqEndForStart[Def] = SU; 5590e6307f6423a7ee39b80f8dc3caef00ad11e0266Eli Friedman 56065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman ++NumLiveRegs; 56165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegDefs[CallResource] = Def; 56265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegGens[CallResource] = SU; 56365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman break; 56465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 5659e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman} 5669e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman 5672da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// Check to see if any of the pending instructions are ready to issue. If 5682da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// so, add them to the available queue. 5692da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::ReleasePending() { 570c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick if (DisableSchedCycles) { 571a75ce9f5d2236d93c117e861e60e6f3f748c9555Andrew Trick assert(PendingQueue.empty() && "pending instrs not allowed in this mode"); 572a75ce9f5d2236d93c117e861e60e6f3f748c9555Andrew Trick return; 573a75ce9f5d2236d93c117e861e60e6f3f748c9555Andrew Trick } 5742da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 5752da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // If the available queue is empty, it is safe to reset MinAvailableCycle. 5762da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (AvailableQueue->empty()) 5772da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick MinAvailableCycle = UINT_MAX; 5782da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 5792da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Check to see if any of the pending instructions are ready to issue. If 5802da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // so, add them to the available queue. 5812da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick for (unsigned i = 0, e = PendingQueue.size(); i != e; ++i) { 582ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman unsigned ReadyCycle = PendingQueue[i]->getHeight(); 5832da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (ReadyCycle < MinAvailableCycle) 5842da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick MinAvailableCycle = ReadyCycle; 5852da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 5862da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (PendingQueue[i]->isAvailable) { 5872da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (!isReady(PendingQueue[i])) 5882da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick continue; 5892da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->push(PendingQueue[i]); 5902da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 5912da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PendingQueue[i]->isPending = false; 5922da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PendingQueue[i] = PendingQueue.back(); 5932da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PendingQueue.pop_back(); 5942da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick --i; --e; 5952da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 5962da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 5972da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 5982da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// Move the scheduler state forward by the specified number of Cycles. 5992da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::AdvanceToCycle(unsigned NextCycle) { 6002da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (NextCycle <= CurCycle) 6012da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 6022da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 603e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick IssueCount = 0; 6042da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->setCurCycle(NextCycle); 605c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick if (!HazardRec->isEnabled()) { 6062da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Bypass lots of virtual calls in case of long latency. 6072da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CurCycle = NextCycle; 6082da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 6092da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick else { 6102da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick for (; CurCycle != NextCycle; ++CurCycle) { 611ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman HazardRec->RecedeCycle(); 6122da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 6132da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 6142da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // FIXME: Instead of visiting the pending Q each time, set a dirty flag on the 6152da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // available Q to release pending nodes at least once before popping. 6162da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ReleasePending(); 6172da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 6182da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6192da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// Move the scheduler state forward until the specified node's dependents are 6202da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// ready and can be scheduled with no resource conflicts. 6212da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::AdvancePastStalls(SUnit *SU) { 622c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick if (DisableSchedCycles) 6232da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 6242da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 62587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // FIXME: Nodes such as CopyFromReg probably should not advance the current 62687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // cycle. Otherwise, we can wrongly mask real stalls. If the non-machine node 62787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // has predecessors the cycle will be advanced when they are scheduled. 62887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // But given the crude nature of modeling latency though such nodes, we 62987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // currently need to treat these nodes like real instructions. 63087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // if (!SU->getNode() || !SU->getNode()->isMachineOpcode()) return; 63187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 632ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman unsigned ReadyCycle = SU->getHeight(); 6332da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6342da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Bump CurCycle to account for latency. We assume the latency of other 6352da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // available instructions may be hidden by the stall (not a full pipe stall). 6362da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // This updates the hazard recognizer's cycle before reserving resources for 6372da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // this instruction. 6382da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AdvanceToCycle(ReadyCycle); 6392da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6402da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Calls are scheduled in their preceding cycle, so don't conflict with 6412da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // hazards from instructions after the call. EmitNode will reset the 6422da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // scoreboard state before emitting the call. 643ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman if (SU->isCall) 6442da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 6452da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6462da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // FIXME: For resource conflicts in very long non-pipelined stages, we 6472da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // should probably skip ahead here to avoid useless scoreboard checks. 6482da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick int Stalls = 0; 6492da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick while (true) { 6502da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleHazardRecognizer::HazardType HT = 651ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman HazardRec->getHazardType(SU, -Stalls); 6522da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6532da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (HT == ScheduleHazardRecognizer::NoHazard) 6542da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick break; 6552da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6562da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ++Stalls; 6572da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 6582da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AdvanceToCycle(CurCycle + Stalls); 6592da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 6602da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6612da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// Record this SUnit in the HazardRecognizer. 6622da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// Does not update CurCycle. 6632da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::EmitNode(SUnit *SU) { 664c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick if (!HazardRec->isEnabled()) 66524312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick return; 66624312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick 66724312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick // Check for phys reg copy. 66824312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick if (!SU->getNode()) 66924312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick return; 67024312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick 6712da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick switch (SU->getNode()->getOpcode()) { 6722da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick default: 6732da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick assert(SU->getNode()->isMachineOpcode() && 6742da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick "This target-independent node should not be scheduled."); 6752da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick break; 6762da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::MERGE_VALUES: 6772da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::TokenFactor: 678c05d30601ced172b55be81bb529df6be91d6ae15Nadav Rotem case ISD::LIFETIME_START: 679c05d30601ced172b55be81bb529df6be91d6ae15Nadav Rotem case ISD::LIFETIME_END: 6802da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::CopyToReg: 6812da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::CopyFromReg: 6822da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::EH_LABEL: 6832da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Noops don't affect the scoreboard state. Copies are likely to be 6842da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // removed. 6852da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 6862da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick case ISD::INLINEASM: 6872da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // For inline asm, clear the pipeline state. 6882da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->Reset(); 6892da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 6902da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 691ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman if (SU->isCall) { 6922da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Calls are scheduled with their preceding instructions. For bottom-up 6932da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // scheduling, clear the pipeline state before emitting. 6942da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->Reset(); 6952da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 6962da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 6972da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->EmitInstruction(SU); 6982da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 6992da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 70087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trickstatic void resetVRegCycle(SUnit *SU); 70187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 7029e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman/// ScheduleNodeBottomUp - Add the node to the schedule. Decrement the pending 7039e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman/// count of its predecessors. If a predecessor pending count is zero, add it to 7049e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman/// the Available queue. 7052902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trickvoid ScheduleDAGRRList::ScheduleNodeBottomUp(SUnit *SU) { 706c558bf397257f5ef902bdb45a28e622ee2b5b4f2Andrew Trick DEBUG(dbgs() << "\n*** Scheduling [" << CurCycle << "]: "); 7079e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman DEBUG(SU->dump(this)); 7089e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman 70915a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng#ifndef NDEBUG 71015a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng if (CurCycle < SU->getHeight()) 71187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick DEBUG(dbgs() << " Height [" << SU->getHeight() 71287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick << "] pipeline stall!\n"); 71315a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng#endif 71415a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 7152da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // FIXME: Do not modify node height. It may interfere with 7162da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // backtracking. Instead add a "ready cycle" to SUnit. Before scheduling the 71728ed90b95db5f14b60b2cb532a62d407d4faf5e5Eric Christopher // node its ready cycle can aid heuristics, and after scheduling it can 7182da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // indicate the scheduled cycle. 7199e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman SU->setHeightToAtLeast(CurCycle); 7202da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 7212da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Reserve resources for the scheduled intruction. 7222da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick EmitNode(SU); 7232da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 7249e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman Sequence.push_back(SU); 7259e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman 726953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick AvailableQueue->scheduledNode(SU); 72737944985a569f8c2b0d75dafd9e2739a9887ac5dChris Lattner 728e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick // If HazardRec is disabled, and each inst counts as one cycle, then 72987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // advance CurCycle before ReleasePredecessors to avoid useless pushes to 730a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick // PendingQueue for schedulers that implement HasReadyFilter. 731e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (!HazardRec->isEnabled() && AvgIPC < 2) 732a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick AdvanceToCycle(CurCycle + 1); 733a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick 7341b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick // Update liveness of predecessors before successors to avoid treating a 7351b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick // two-address node as a live range def. 7363d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick ReleasePredecessors(SU); 737a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 738a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // Release all the implicit physical register defs that are live. 739a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 740a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 7411b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick // LiveRegDegs[I->getReg()] != SU when SU is a two-address node. 7421b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick if (I->isAssignedRegDep() && LiveRegDefs[I->getReg()] == SU) { 7431b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!"); 7441b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick --NumLiveRegs; 7451b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick LiveRegDefs[I->getReg()] = NULL; 7463d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick LiveRegGens[I->getReg()] = NULL; 747029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick releaseInterferences(I->getReg()); 748a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 749a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 75065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Release the special call resource dependence, if this is the beginning 75165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // of a call. 75265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned CallResource = TRI->getNumRegs(); 75365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (LiveRegDefs[CallResource] == SU) 75465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (const SDNode *SUNode = SU->getNode(); SUNode; 75565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SUNode = SUNode->getGluedNode()) { 75665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (SUNode->isMachineOpcode() && 75765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SUNode->getMachineOpcode() == (unsigned)TII->getCallFrameSetupOpcode()) { 75865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!"); 75965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman --NumLiveRegs; 76065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegDefs[CallResource] = NULL; 76165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegGens[CallResource] = NULL; 762029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick releaseInterferences(CallResource); 76365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 76465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 765a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 76687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick resetVRegCycle(SU); 76787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 768e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng SU->isScheduled = true; 7692da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 7702da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Conditions under which the scheduler should eagerly advance the cycle: 7712da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // (1) No available instructions 7722da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // (2) All pipelines full, so available instructions must have hazards. 7732da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // 77487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // If HazardRec is disabled, the cycle was pre-advanced before calling 77587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // ReleasePredecessors. In that case, IssueCount should remain 0. 776a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick // 777a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick // Check AvailableQueue after ReleasePredecessors in case of zero latency. 77887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (HazardRec->isEnabled() || AvgIPC > 1) { 77987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (SU->getNode() && SU->getNode()->isMachineOpcode()) 78087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick ++IssueCount; 78187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if ((HazardRec->isEnabled() && HazardRec->atIssueLimit()) 78287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick || (!HazardRec->isEnabled() && IssueCount == AvgIPC)) 78387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick AdvanceToCycle(CurCycle + 1); 78487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick } 785e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 786e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 787a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// CapturePred - This does the opposite of ReleasePred. Since SU is being 788a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// unscheduled, incrcease the succ left count of its predecessors. Remove 789a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// them from AvailableQueue if necessary. 79038036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trickvoid ScheduleDAGRRList::CapturePred(SDep *PredEdge) { 79154e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *PredSU = PredEdge->getSUnit(); 792a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng if (PredSU->isAvailable) { 793a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng PredSU->isAvailable = false; 794a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng if (!PredSU->isPending) 795a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng AvailableQueue->remove(PredSU); 796a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 797a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 798c277ab08a24d2dbe9b4ff1a9154ea6115ed6a4e3Reid Kleckner assert(PredSU->NumSuccsLeft < UINT_MAX && "NumSuccsLeft will overflow!"); 79974d2fd8dd847e0ebccef30e2c5907ff09495d518Evan Cheng ++PredSU->NumSuccsLeft; 800a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng} 801a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 802a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// UnscheduleNodeBottomUp - Remove the node from the schedule, update its and 803a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// its predecessor states to reflect the change. 804a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Chengvoid ScheduleDAGRRList::UnscheduleNodeBottomUp(SUnit *SU) { 805e492ae13edd83b120d665c0503cf4de2925b5e56David Greene DEBUG(dbgs() << "*** Unscheduling [" << SU->getHeight() << "]: "); 8063cc6243ddfdba3ad64035b919c88b09773a60880Dan Gohman DEBUG(SU->dump(this)); 807a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 808a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 809a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 81054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman CapturePred(&*I); 8113d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick if (I->isAssignedRegDep() && SU == LiveRegGens[I->getReg()]){ 812086ec9976ff6cee083de618429c78473491d5713Dan Gohman assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!"); 81354e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman assert(LiveRegDefs[I->getReg()] == I->getSUnit() && 814a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng "Physical register dependency violated?"); 815086ec9976ff6cee083de618429c78473491d5713Dan Gohman --NumLiveRegs; 81654e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman LiveRegDefs[I->getReg()] = NULL; 8173d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick LiveRegGens[I->getReg()] = NULL; 818029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick releaseInterferences(I->getReg()); 819a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 820a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 821a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 82265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Reclaim the special call resource dependence, if this is the beginning 82365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // of a call. 82465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned CallResource = TRI->getNumRegs(); 82565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (const SDNode *SUNode = SU->getNode(); SUNode; 82665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SUNode = SUNode->getGluedNode()) { 82765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (SUNode->isMachineOpcode() && 82865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SUNode->getMachineOpcode() == (unsigned)TII->getCallFrameSetupOpcode()) { 82965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman ++NumLiveRegs; 83065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegDefs[CallResource] = SU; 8310e6307f6423a7ee39b80f8dc3caef00ad11e0266Eli Friedman LiveRegGens[CallResource] = CallSeqEndForStart[SU]; 83265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 83365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 83465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman 83565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Release the special call resource dependence, if this is the end 83665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // of a call. 83765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (LiveRegGens[CallResource] == SU) 83865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman for (const SDNode *SUNode = SU->getNode(); SUNode; 83965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SUNode = SUNode->getGluedNode()) { 84065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (SUNode->isMachineOpcode() && 84165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SUNode->getMachineOpcode() == (unsigned)TII->getCallFrameDestroyOpcode()) { 84265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!"); 84365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman --NumLiveRegs; 84465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegDefs[CallResource] = NULL; 84565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LiveRegGens[CallResource] = NULL; 846029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick releaseInterferences(CallResource); 84765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 84865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 84965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman 850a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 851a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 85254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isAssignedRegDep()) { 85330c44e18bf28c3f5feda56e97695e2b72de7fedbEli Friedman if (!LiveRegDefs[I->getReg()]) 85430c44e18bf28c3f5feda56e97695e2b72de7fedbEli Friedman ++NumLiveRegs; 8551b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick // This becomes the nearest def. Note that an earlier def may still be 8561b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick // pending if this is a two-address node. 8571b1658741aa4bc88fc488ef217d9603294e16c58Andrew Trick LiveRegDefs[I->getReg()] = SU; 8583d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick if (LiveRegGens[I->getReg()] == NULL || 8593d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick I->getSUnit()->getHeight() < LiveRegGens[I->getReg()]->getHeight()) 8603d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick LiveRegGens[I->getReg()] = I->getSUnit(); 861a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 862a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 8632da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (SU->getHeight() < MinAvailableCycle) 8642da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick MinAvailableCycle = SU->getHeight(); 865a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 8663f23744df4809eba94284e601e81489212c974d4Dan Gohman SU->setHeightDirty(); 867a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng SU->isScheduled = false; 868a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng SU->isAvailable = true; 869c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick if (!DisableSchedCycles && AvailableQueue->hasReadyFilter()) { 8702da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Don't make available until backtracking is complete. 8712da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SU->isPending = true; 8722da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick PendingQueue.push_back(SU); 8732da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 8742da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick else { 8752da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->push(SU); 8762da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 877953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick AvailableQueue->unscheduledNode(SU); 878a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng} 879a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 8802da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// After backtracking, the hazard checker needs to be restored to a state 881c8e41c591741b3da1077f7000274ad040bef8002Sylvestre Ledru/// corresponding the current cycle. 8822da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::RestoreHazardCheckerBottomUp() { 8832da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->Reset(); 8842da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 8852da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned LookAhead = std::min((unsigned)Sequence.size(), 8862da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->getMaxLookAhead()); 8872da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (LookAhead == 0) 8882da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick return; 8892da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 8902da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick std::vector<SUnit*>::const_iterator I = (Sequence.end() - LookAhead); 8912da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned HazardCycle = (*I)->getHeight(); 8922da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick for (std::vector<SUnit*>::const_iterator E = Sequence.end(); I != E; ++I) { 8932da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SUnit *SU = *I; 8942da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick for (; SU->getHeight() > HazardCycle; ++HazardCycle) { 8952da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick HazardRec->RecedeCycle(); 8962da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 8972da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick EmitNode(SU); 8982da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 8992da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 9002da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 90142d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng/// BacktrackBottomUp - Backtrack scheduling to a previous cycle specified in 9021cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohman/// BTCycle in order to schedule a specific node. 9032da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickvoid ScheduleDAGRRList::BacktrackBottomUp(SUnit *SU, SUnit *BtSU) { 9042da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SUnit *OldSU = Sequence.back(); 9052da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick while (true) { 906a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng Sequence.pop_back(); 9072da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // FIXME: use ready cycle instead of height 9082da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CurCycle = OldSU->getHeight(); 909a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng UnscheduleNodeBottomUp(OldSU); 91015a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng AvailableQueue->setCurCycle(CurCycle); 9112da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (OldSU == BtSU) 9122da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick break; 9132da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick OldSU = Sequence.back(); 914a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 915a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 9161cc6b8ee787c183a7c88fec8fbb96c1404cf34e8Dan Gohman assert(!SU->isSucc(OldSU) && "Something is wrong!"); 917a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng 9182da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick RestoreHazardCheckerBottomUp(); 9192da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 920a75ce9f5d2236d93c117e861e60e6f3f748c9555Andrew Trick ReleasePending(); 9212da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 922a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng ++NumBacktracks; 923a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng} 924a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 9255ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Chengstatic bool isOperandOf(const SUnit *SU, SDNode *N) { 9265ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng for (const SDNode *SUNode = SU->getNode(); SUNode; 92729d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner SUNode = SUNode->getGluedNode()) { 9285ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng if (SUNode->isOperandOf(N)) 9295ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng return true; 9305ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng } 9315ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng return false; 9325ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng} 9335ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng 934f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng/// CopyAndMoveSuccessors - Clone the specified node and move its scheduled 935f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng/// successors to the newly created node. 936f10c973797cf79da802f9b0118543cbd50954c9cEvan ChengSUnit *ScheduleDAGRRList::CopyAndMoveSuccessors(SUnit *SU) { 937550f5afb68ce8f034991863cac65bef22a6554daDan Gohman SDNode *N = SU->getNode(); 93842d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng if (!N) 939f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng return NULL; 94042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 94124312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick if (SU->getNode()->getGluedNode()) 94224312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick return NULL; 94324312230ada6f4cfa8776351dafb12eea8a81b33Andrew Trick 944f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SUnit *NewSU; 945f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng bool TryUnfold = false; 946d5cb5a462b6fd91bf54116c3eefc3b046489c414Evan Cheng for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) { 947e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = N->getValueType(i); 948f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner if (VT == MVT::Glue) 949d5cb5a462b6fd91bf54116c3eefc3b046489c414Evan Cheng return NULL; 950825b72b0571821bf2d378749f69d6c4cfb52d2f9Owen Anderson else if (VT == MVT::Other) 951d5cb5a462b6fd91bf54116c3eefc3b046489c414Evan Cheng TryUnfold = true; 952d5cb5a462b6fd91bf54116c3eefc3b046489c414Evan Cheng } 953a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 954475871a144eb604ddaf37503397ba0941442e5fbDan Gohman const SDValue &Op = N->getOperand(i); 955e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = Op.getNode()->getValueType(Op.getResNo()); 956f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner if (VT == MVT::Glue) 957f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng return NULL; 958a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 959a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 960f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng if (TryUnfold) { 9614c8c83022b501759d8559e224c84ae2a9921ba41Dan Gohman SmallVector<SDNode*, 2> NewNodes; 962a23b3b803e3c65e84d6cadaa221de8b256cbe28dDan Gohman if (!TII->unfoldMemoryOperand(*DAG, N, NewNodes)) 963f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng return NULL; 9642d496897934adfc0e3e1563dd64d9a7122971e22Pete Cooper 9652d496897934adfc0e3e1563dd64d9a7122971e22Pete Cooper // unfolding an x86 DEC64m operation results in store, dec, load which 9662d496897934adfc0e3e1563dd64d9a7122971e22Pete Cooper // can't be handled here so quit 9672d496897934adfc0e3e1563dd64d9a7122971e22Pete Cooper if (NewNodes.size() == 3) 9682d496897934adfc0e3e1563dd64d9a7122971e22Pete Cooper return NULL; 969f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 97015a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng DEBUG(dbgs() << "Unfolding SU #" << SU->NodeNum << "\n"); 971f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng assert(NewNodes.size() == 2 && "Expected a load folding node!"); 972f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 973f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng N = NewNodes[1]; 974f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SDNode *LoadNode = NewNodes[0]; 975f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng unsigned NumVals = N->getNumValues(); 976550f5afb68ce8f034991863cac65bef22a6554daDan Gohman unsigned OldNumVals = SU->getNode()->getNumValues(); 977f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (unsigned i = 0; i != NumVals; ++i) 978550f5afb68ce8f034991863cac65bef22a6554daDan Gohman DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), i), SDValue(N, i)); 979550f5afb68ce8f034991863cac65bef22a6554daDan Gohman DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), OldNumVals-1), 980a23b3b803e3c65e84d6cadaa221de8b256cbe28dDan Gohman SDValue(LoadNode, 1)); 981f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 982b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman // LoadNode may already exist. This can happen when there is another 983b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman // load from the same location and producing the same type of value 984b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman // but it has different alignment or volatileness. 985b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman bool isNewLoad = true; 986b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman SUnit *LoadSU; 987b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman if (LoadNode->getNodeId() != -1) { 988b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman LoadSU = &SUnits[LoadNode->getNodeId()]; 989b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman isNewLoad = false; 990b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman } else { 991b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman LoadSU = CreateNewSUnit(LoadNode); 992b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman LoadNode->setNodeId(LoadSU->NodeNum); 99392e946630d5f9bb092853b93501387dd216899b9Andrew Trick 99492e946630d5f9bb092853b93501387dd216899b9Andrew Trick InitNumRegDefsLeft(LoadSU); 995953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick computeLatency(LoadSU); 996b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman } 997b13af2f2ec2fd8dc215136cf8783d70225b59f66Dan Gohman 998e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *NewSU = CreateNewSUnit(N); 99994d7a5f8156e62532870fbaf197377b34e52ff2aDan Gohman assert(N->getNodeId() == -1 && "Node already inserted!"); 100094d7a5f8156e62532870fbaf197377b34e52ff2aDan Gohman N->setNodeId(NewSU->NodeNum); 100138036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1002e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng const MCInstrDesc &MCID = TII->get(N->getMachineOpcode()); 1003e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng for (unsigned i = 0; i != MCID.getNumOperands(); ++i) { 1004e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng if (MCID.getOperandConstraint(i, MCOI::TIED_TO) != -1) { 1005f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng NewSU->isTwoAddress = true; 1006f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng break; 1007f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 1008f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 1009e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng if (MCID.isCommutable()) 1010f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng NewSU->isCommutable = true; 101192e946630d5f9bb092853b93501387dd216899b9Andrew Trick 101292e946630d5f9bb092853b93501387dd216899b9Andrew Trick InitNumRegDefsLeft(NewSU); 1013953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick computeLatency(NewSU); 1014f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 1015fa9afef7eaa0865fb9f3489a68a04332b232ed82Dan Gohman // Record all the edges to and from the old SU, by category. 101616e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman SmallVector<SDep, 4> ChainPreds; 1017f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SmallVector<SDep, 4> ChainSuccs; 1018f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SmallVector<SDep, 4> LoadPreds; 1019f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SmallVector<SDep, 4> NodePreds; 1020f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng SmallVector<SDep, 4> NodeSuccs; 1021f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 1022f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng I != E; ++I) { 102354e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isCtrl()) 102416e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman ChainPreds.push_back(*I); 10255ba8bf6d28ad350114a4b02f01d70ebe08612cfeEvan Cheng else if (isOperandOf(I->getSUnit(), LoadNode)) 102654e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman LoadPreds.push_back(*I); 1027f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng else 102854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman NodePreds.push_back(*I); 1029f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 1030f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 1031f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng I != E; ++I) { 103254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isCtrl()) 103354e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman ChainSuccs.push_back(*I); 1034f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng else 103554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman NodeSuccs.push_back(*I); 1036f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 103742d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 1038fa9afef7eaa0865fb9f3489a68a04332b232ed82Dan Gohman // Now assign edges to the newly-created nodes. 103916e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman for (unsigned i = 0, e = ChainPreds.size(); i != e; ++i) { 104016e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman const SDep &Pred = ChainPreds[i]; 104116e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman RemovePred(SU, Pred); 104280792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman if (isNewLoad) 104316e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman AddPred(LoadSU, Pred); 1044e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein } 1045f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (unsigned i = 0, e = LoadPreds.size(); i != e; ++i) { 104654e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman const SDep &Pred = LoadPreds[i]; 104754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(SU, Pred); 104816e8eda4b8b0825d38ba37d31c99edba0fb25482Dan Gohman if (isNewLoad) 104954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(LoadSU, Pred); 1050f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 1051f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (unsigned i = 0, e = NodePreds.size(); i != e; ++i) { 105254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman const SDep &Pred = NodePreds[i]; 105354e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(SU, Pred); 105454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(NewSU, Pred); 1055f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 1056f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (unsigned i = 0, e = NodeSuccs.size(); i != e; ++i) { 105754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SDep D = NodeSuccs[i]; 105854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *SuccDep = D.getSUnit(); 105954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(SU); 106054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(SuccDep, D); 106154e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(NewSU); 106254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(SuccDep, D); 106392e946630d5f9bb092853b93501387dd216899b9Andrew Trick // Balance register pressure. 106492e946630d5f9bb092853b93501387dd216899b9Andrew Trick if (AvailableQueue->tracksRegPressure() && SuccDep->isScheduled 106592e946630d5f9bb092853b93501387dd216899b9Andrew Trick && !D.isCtrl() && NewSU->NumRegDefsLeft > 0) 106692e946630d5f9bb092853b93501387dd216899b9Andrew Trick --NewSU->NumRegDefsLeft; 1067f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 1068f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng for (unsigned i = 0, e = ChainSuccs.size(); i != e; ++i) { 106954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SDep D = ChainSuccs[i]; 107054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *SuccDep = D.getSUnit(); 107154e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(SU); 107254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(SuccDep, D); 1073e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein if (isNewLoad) { 107454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(LoadSU); 107554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(SuccDep, D); 1076e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein } 107738036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick } 1078fa9afef7eaa0865fb9f3489a68a04332b232ed82Dan Gohman 1079fa9afef7eaa0865fb9f3489a68a04332b232ed82Dan Gohman // Add a data dependency to reflect that NewSU reads the value defined 1080fa9afef7eaa0865fb9f3489a68a04332b232ed82Dan Gohman // by LoadSU. 1081a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick SDep D(LoadSU, SDep::Data, 0); 1082a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick D.setLatency(LoadSU->Latency); 1083a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick AddPred(NewSU, D); 1084f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 1085beec823d4bba22b1c0c6658d2b3e71cd64a70e2eEvan Cheng if (isNewLoad) 1086beec823d4bba22b1c0c6658d2b3e71cd64a70e2eEvan Cheng AvailableQueue->addNode(LoadSU); 1087f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng AvailableQueue->addNode(NewSU); 1088f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 1089f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng ++NumUnfolds; 1090f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 1091f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng if (NewSU->NumSuccsLeft == 0) { 1092f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng NewSU->isAvailable = true; 1093f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng return NewSU; 1094beec823d4bba22b1c0c6658d2b3e71cd64a70e2eEvan Cheng } 1095beec823d4bba22b1c0c6658d2b3e71cd64a70e2eEvan Cheng SU = NewSU; 1096f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng } 1097f10c973797cf79da802f9b0118543cbd50954c9cEvan Cheng 109815a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng DEBUG(dbgs() << " Duplicating SU #" << SU->NodeNum << "\n"); 1099e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein NewSU = CreateClone(SU); 1100a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1101a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // New SUnit has the exact same predecessors. 1102a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 1103a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) 11043f23744df4809eba94284e601e81489212c974d4Dan Gohman if (!I->isArtificial()) 110554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(NewSU, *I); 1106a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1107a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // Only copy scheduled successors. Cut them from old node's successor 1108a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // list and move them over. 110954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SmallVector<std::pair<SUnit *, SDep>, 4> DelDeps; 1110a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 1111a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 111254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isArtificial()) 1113a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng continue; 111454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *SuccSU = I->getSUnit(); 111554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (SuccSU->isScheduled) { 111654e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SDep D = *I; 111754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(NewSU); 111854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(SuccSU, D); 111954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(SU); 112054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman DelDeps.push_back(std::make_pair(SuccSU, D)); 1121a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 1122a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 11233f23744df4809eba94284e601e81489212c974d4Dan Gohman for (unsigned i = 0, e = DelDeps.size(); i != e; ++i) 112454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(DelDeps[i].first, DelDeps[i].second); 1125a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1126a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng AvailableQueue->updateNode(SU); 1127a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng AvailableQueue->addNode(NewSU); 1128a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1129a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng ++NumDups; 1130a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng return NewSU; 1131a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng} 1132a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1133c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng/// InsertCopiesAndMoveSuccs - Insert register copies and move all 1134c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng/// scheduled successors of the given SUnit to the last copy. 1135c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Chengvoid ScheduleDAGRRList::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg, 1136c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng const TargetRegisterClass *DestRC, 1137c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng const TargetRegisterClass *SrcRC, 1138a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng SmallVector<SUnit*, 2> &Copies) { 1139e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *CopyFromSU = CreateNewSUnit(NULL); 114042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng CopyFromSU->CopySrcRC = SrcRC; 114142d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng CopyFromSU->CopyDstRC = DestRC; 114242d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 1143e513ba49589bcf8fdf7dad658e20db21d6ef4758Roman Levenstein SUnit *CopyToSU = CreateNewSUnit(NULL); 114442d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng CopyToSU->CopySrcRC = DestRC; 114542d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng CopyToSU->CopyDstRC = SrcRC; 114642d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 114742d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng // Only copy scheduled successors. Cut them from old node's successor 114842d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng // list and move them over. 114954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SmallVector<std::pair<SUnit *, SDep>, 4> DelDeps; 115042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 115142d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng I != E; ++I) { 115254e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isArtificial()) 115342d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng continue; 115454e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *SuccSU = I->getSUnit(); 115554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (SuccSU->isScheduled) { 115654e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SDep D = *I; 115754e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman D.setSUnit(CopyToSU); 115854e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman AddPred(SuccSU, D); 115954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman DelDeps.push_back(std::make_pair(SuccSU, *I)); 116042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng } 1161bd47a4a307a54dab25956430442cf4ac103e4158Andrew Trick else { 1162bd47a4a307a54dab25956430442cf4ac103e4158Andrew Trick // Avoid scheduling the def-side copy before other successors. Otherwise 1163bd47a4a307a54dab25956430442cf4ac103e4158Andrew Trick // we could introduce another physreg interference on the copy and 1164bd47a4a307a54dab25956430442cf4ac103e4158Andrew Trick // continue inserting copies indefinitely. 1165a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick AddPred(SuccSU, SDep(CopyFromSU, SDep::Artificial)); 1166bd47a4a307a54dab25956430442cf4ac103e4158Andrew Trick } 116742d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng } 1168c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng for (unsigned i = 0, e = DelDeps.size(); i != e; ++i) 116954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman RemovePred(DelDeps[i].first, DelDeps[i].second); 117042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 1171a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick SDep FromDep(SU, SDep::Data, Reg); 1172a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick FromDep.setLatency(SU->Latency); 1173a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick AddPred(CopyFromSU, FromDep); 1174a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick SDep ToDep(CopyFromSU, SDep::Data, 0); 1175a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick ToDep.setLatency(CopyFromSU->Latency); 1176a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick AddPred(CopyToSU, ToDep); 117742d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 117842d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng AvailableQueue->updateNode(SU); 117942d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng AvailableQueue->addNode(CopyFromSU); 118042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng AvailableQueue->addNode(CopyToSU); 1181a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng Copies.push_back(CopyFromSU); 1182a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng Copies.push_back(CopyToSU); 118342d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 1184c29a56dedbe4297dad94b9bf2e19035c5903fd1fEvan Cheng ++NumPRCopies; 118542d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng} 118642d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 118742d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng/// getPhysicalRegisterVT - Returns the ValueType of the physical register 118842d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng/// definition of the specified node. 118942d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng/// FIXME: Move to SelectionDAG? 1190e50ed30282bb5b4a9ed952580523f2dda16215acOwen Andersonstatic EVT getPhysicalRegisterVT(SDNode *N, unsigned Reg, 119183ec4b6711980242ef3c55a4fa36b2d7a39c1bfbDuncan Sands const TargetInstrInfo *TII) { 1192e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng const MCInstrDesc &MCID = TII->get(N->getMachineOpcode()); 1193e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng assert(MCID.ImplicitDefs && "Physical reg def must be in implicit def list!"); 1194e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng unsigned NumRes = MCID.getNumDefs(); 1195fac259814923d091942b230e7bd002a8d1130bc3Craig Topper for (const uint16_t *ImpDef = MCID.getImplicitDefs(); *ImpDef; ++ImpDef) { 119642d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng if (Reg == *ImpDef) 119742d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng break; 119842d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng ++NumRes; 119942d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng } 120042d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng return N->getValueType(NumRes); 120142d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng} 120242d60274eaa70f8cdbed76d04d25d7a8fc1237cbEvan Cheng 1203599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng/// CheckForLiveRegDef - Return true and update live register vector if the 1204599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng/// specified register def of the specified SUnit clobbers any "live" registers. 1205142d21c861c0b686e38a515b1271f4157cd24004Chris Lattnerstatic void CheckForLiveRegDef(SUnit *SU, unsigned Reg, 1206599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng std::vector<SUnit*> &LiveRegDefs, 1207599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng SmallSet<unsigned, 4> &RegAdded, 1208599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng SmallVector<unsigned, 4> &LRegs, 1209599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng const TargetRegisterInfo *TRI) { 1210396618b43a85e12d290a90b181c6af5d7c0c5f11Jakob Stoklund Olesen for (MCRegAliasIterator AliasI(Reg, TRI, true); AliasI.isValid(); ++AliasI) { 1211cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick 1212cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick // Check if Ref is live. 12139d507aec07c7e22c6ba83dfd75e23c8630cd25cdAndrew Trick if (!LiveRegDefs[*AliasI]) continue; 1214cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick 1215cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick // Allow multiple uses of the same def. 12169d507aec07c7e22c6ba83dfd75e23c8630cd25cdAndrew Trick if (LiveRegDefs[*AliasI] == SU) continue; 1217cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick 1218cb7947b81be4cb55fc647ea3722eb7a8613545cbAndrew Trick // Add Reg to the set of interfering live regs. 12199d507aec07c7e22c6ba83dfd75e23c8630cd25cdAndrew Trick if (RegAdded.insert(*AliasI)) { 12209d507aec07c7e22c6ba83dfd75e23c8630cd25cdAndrew Trick LRegs.push_back(*AliasI); 12219d507aec07c7e22c6ba83dfd75e23c8630cd25cdAndrew Trick } 1222599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng } 1223599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng} 1224599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng 122516a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen/// CheckForLiveRegDefMasked - Check for any live physregs that are clobbered 122616a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen/// by RegMask, and add them to LRegs. 122716a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesenstatic void CheckForLiveRegDefMasked(SUnit *SU, const uint32_t *RegMask, 122816a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen std::vector<SUnit*> &LiveRegDefs, 122916a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen SmallSet<unsigned, 4> &RegAdded, 123016a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen SmallVector<unsigned, 4> &LRegs) { 123116a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen // Look at all live registers. Skip Reg0 and the special CallResource. 123216a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen for (unsigned i = 1, e = LiveRegDefs.size()-1; i != e; ++i) { 123316a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (!LiveRegDefs[i]) continue; 123416a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (LiveRegDefs[i] == SU) continue; 123516a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (!MachineOperand::clobbersPhysReg(RegMask, i)) continue; 123616a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (RegAdded.insert(i)) 123716a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen LRegs.push_back(i); 123816a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen } 123916a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen} 124016a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen 124116a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen/// getNodeRegMask - Returns the register mask attached to an SDNode, if any. 124216a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesenstatic const uint32_t *getNodeRegMask(const SDNode *N) { 124316a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 124416a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (const RegisterMaskSDNode *Op = 124516a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen dyn_cast<RegisterMaskSDNode>(N->getOperand(i).getNode())) 124616a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen return Op->getRegMask(); 124716a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen return NULL; 124816a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen} 124916a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen 1250a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// DelayForLiveRegsBottomUp - Returns true if it is necessary to delay 1251a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// scheduling of the given node to satisfy live physical register dependencies. 1252a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// If the specific node is the last one that's available to schedule, do 1253a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng/// whatever is necessary (i.e. backtracking or cloning) to make it possible. 1254142d21c861c0b686e38a515b1271f4157cd24004Chris Lattnerbool ScheduleDAGRRList:: 1255142d21c861c0b686e38a515b1271f4157cd24004Chris LattnerDelayForLiveRegsBottomUp(SUnit *SU, SmallVector<unsigned, 4> &LRegs) { 1256086ec9976ff6cee083de618429c78473491d5713Dan Gohman if (NumLiveRegs == 0) 1257a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng return false; 1258a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1259cd1c00cc6521c265784ffc7a1b5baf4ef64d80bcEvan Cheng SmallSet<unsigned, 4> RegAdded; 1260a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng // If this node would clobber any "live" register, then it's not ready. 1261feac09801b5c03412d452e685570baff6eb84c88Andrew Trick // 1262feac09801b5c03412d452e685570baff6eb84c88Andrew Trick // If SU is the currently live definition of the same register that it uses, 1263feac09801b5c03412d452e685570baff6eb84c88Andrew Trick // then we are free to schedule it. 1264a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 1265a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng I != E; ++I) { 1266feac09801b5c03412d452e685570baff6eb84c88Andrew Trick if (I->isAssignedRegDep() && LiveRegDefs[I->getReg()] != SU) 1267599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng CheckForLiveRegDef(I->getSUnit(), I->getReg(), LiveRegDefs, 1268599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng RegAdded, LRegs, TRI); 1269a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 1270a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 127129d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) { 1272599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng if (Node->getOpcode() == ISD::INLINEASM) { 1273599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng // Inline asm can clobber physical defs. 1274599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng unsigned NumOps = Node->getNumOperands(); 1275f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue) 127629d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner --NumOps; // Ignore the glue operand. 1277599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng 1278decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) { 1279599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng unsigned Flags = 1280599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue(); 1281decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags); 1282599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng 1283599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng ++i; // Skip the ID value. 1284decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner if (InlineAsm::isRegDefKind(Flags) || 1285f792fa90f1125553008659c743cba85b9b5d2e5eJakob Stoklund Olesen InlineAsm::isRegDefEarlyClobberKind(Flags) || 1286f792fa90f1125553008659c743cba85b9b5d2e5eJakob Stoklund Olesen InlineAsm::isClobberKind(Flags)) { 1287599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng // Check for def of register or earlyclobber register. 1288599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng for (; NumVals; --NumVals, ++i) { 1289599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg(); 1290599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng if (TargetRegisterInfo::isPhysicalRegister(Reg)) 1291599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng CheckForLiveRegDef(SU, Reg, LiveRegDefs, RegAdded, LRegs, TRI); 1292599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng } 1293599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng } else 1294599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng i += NumVals; 1295599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng } 1296599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng continue; 1297599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng } 1298599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng 1299d23e0f81bc76902052e9198cad3a0d87a412a632Dan Gohman if (!Node->isMachineOpcode()) 1300a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng continue; 130165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // If we're in the middle of scheduling a call, don't begin scheduling 130265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // another call. Also, don't allow any physical registers to be live across 130365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // the call. 130465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (Node->getMachineOpcode() == (unsigned)TII->getCallFrameDestroyOpcode()) { 130565fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman // Check the special calling-sequence resource. 130665fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman unsigned CallResource = TRI->getNumRegs(); 130765fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (LiveRegDefs[CallResource]) { 130865fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman SDNode *Gen = LiveRegGens[CallResource]->getNode(); 130965fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman while (SDNode *Glued = Gen->getGluedNode()) 131065fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman Gen = Glued; 131165fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman if (!IsChainDependent(Gen, Node, 0, TII) && RegAdded.insert(CallResource)) 131265fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman LRegs.push_back(CallResource); 131365fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 131465fd6564b8aedd053845c81ede1ac594acb470e4Dan Gohman } 131516a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (const uint32_t *RegMask = getNodeRegMask(Node)) 131616a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen CheckForLiveRegDefMasked(SU, RegMask, LiveRegDefs, RegAdded, LRegs); 131716a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen 1318e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng const MCInstrDesc &MCID = TII->get(Node->getMachineOpcode()); 1319e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng if (!MCID.ImplicitDefs) 1320a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng continue; 1321fac259814923d091942b230e7bd002a8d1130bc3Craig Topper for (const uint16_t *Reg = MCID.getImplicitDefs(); *Reg; ++Reg) 1322599a6a88ce1925a6349ac7af9a9638aad1d832ccEvan Cheng CheckForLiveRegDef(SU, *Reg, LiveRegDefs, RegAdded, LRegs, TRI); 1323a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 132438036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1325a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng return !LRegs.empty(); 1326e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 1327e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1328029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trickvoid ScheduleDAGRRList::releaseInterferences(unsigned Reg) { 1329029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick // Add the nodes that aren't ready back onto the available list. 1330029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick for (unsigned i = Interferences.size(); i > 0; --i) { 1331029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick SUnit *SU = Interferences[i-1]; 1332029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick LRegsMapT::iterator LRegsPos = LRegsMap.find(SU); 1333029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick if (Reg) { 1334029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick SmallVector<unsigned, 4> &LRegs = LRegsPos->second; 1335029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick if (std::find(LRegs.begin(), LRegs.end(), Reg) == LRegs.end()) 1336029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick continue; 1337029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick } 1338029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick SU->isPending = false; 1339029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick // The interfering node may no longer be available due to backtracking. 1340029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick // Furthermore, it may have been made available again, in which case it is 1341029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick // now already in the AvailableQueue. 1342029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick if (SU->isAvailable && !SU->NodeQueueId) { 1343029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick DEBUG(dbgs() << " Repushing SU #" << SU->NodeNum << '\n'); 1344029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick AvailableQueue->push(SU); 1345029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick } 1346029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick if (i < Interferences.size()) 1347029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick Interferences[i-1] = Interferences.back(); 1348029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick Interferences.pop_back(); 1349029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick LRegsMap.erase(LRegsPos); 1350029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick } 1351029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick} 1352029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick 13532902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick/// Return a node that can be scheduled in this cycle. Requirements: 13542902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick/// (1) Ready: latency has been satisfied 13552da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick/// (2) No Hazards: resources are available 13562902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick/// (3) No Interferences: may unschedule to break register interferences. 13572902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew TrickSUnit *ScheduleDAGRRList::PickNodeToScheduleBottomUp() { 1358029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick SUnit *CurSU = AvailableQueue->empty() ? 0 : AvailableQueue->pop(); 13592902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick while (CurSU) { 13602902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SmallVector<unsigned, 4> LRegs; 13612902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick if (!DelayForLiveRegsBottomUp(CurSU, LRegs)) 13622902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick break; 1363877fcf52d17f1cadc38112ca988cdd4e666bac24Andrew Trick DEBUG(dbgs() << " Interfering reg " << 1364877fcf52d17f1cadc38112ca988cdd4e666bac24Andrew Trick (LRegs[0] == TRI->getNumRegs() ? "CallResource" 1365877fcf52d17f1cadc38112ca988cdd4e666bac24Andrew Trick : TRI->getName(LRegs[0])) 1366877fcf52d17f1cadc38112ca988cdd4e666bac24Andrew Trick << " SU #" << CurSU->NodeNum << '\n'); 1367029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick std::pair<LRegsMapT::iterator, bool> LRegsPair = 1368029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick LRegsMap.insert(std::make_pair(CurSU, LRegs)); 1369029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick if (LRegsPair.second) { 1370029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick CurSU->isPending = true; // This SU is not in AvailableQueue right now. 1371029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick Interferences.push_back(CurSU); 1372029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick } 1373029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick else { 1374029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick assert(CurSU->isPending && "Intereferences are pending"); 1375029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick // Update the interference with current live regs. 1376029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick LRegsPair.first->second = LRegs; 1377029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick } 13782902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurSU = AvailableQueue->pop(); 13792902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 1380029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick if (CurSU) 13812902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick return CurSU; 13822902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 13832902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // All candidates are delayed due to live physical reg dependencies. 13842902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Try backtracking, code duplication, or inserting cross class copies 13852902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // to resolve it. 13862902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick for (unsigned i = 0, e = Interferences.size(); i != e; ++i) { 13872902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *TrySU = Interferences[i]; 13882902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SmallVector<unsigned, 4> &LRegs = LRegsMap[TrySU]; 13892902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 13902902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Try unscheduling up to the point where it's safe to schedule 13912902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // this node. 13922da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick SUnit *BtSU = NULL; 13932da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick unsigned LiveCycle = UINT_MAX; 13942902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick for (unsigned j = 0, ee = LRegs.size(); j != ee; ++j) { 13952902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick unsigned Reg = LRegs[j]; 13962da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (LiveRegGens[Reg]->getHeight() < LiveCycle) { 13972da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick BtSU = LiveRegGens[Reg]; 13982da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick LiveCycle = BtSU->getHeight(); 13992da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 14002902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 14012da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (!WillCreateCycle(TrySU, BtSU)) { 1402029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick // BacktrackBottomUp mutates Interferences! 14032da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick BacktrackBottomUp(TrySU, BtSU); 14042902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 14052902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Force the current node to be scheduled before the node that 14062902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // requires the physical reg dep. 14072da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (BtSU->isAvailable) { 14082da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick BtSU->isAvailable = false; 14092da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick if (!BtSU->isPending) 14102da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->remove(BtSU); 14112902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 1412029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick DEBUG(dbgs() << "ARTIFICIAL edge from SU(" << BtSU->NodeNum << ") to SU(" 1413029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick << TrySU->NodeNum << ")\n"); 1414a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick AddPred(TrySU, SDep(BtSU, SDep::Artificial)); 14152902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 14162902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // If one or more successors has been unscheduled, then the current 1417029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick // node is no longer available. 1418029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick if (!TrySU->isAvailable) 14192902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurSU = AvailableQueue->pop(); 14202902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick else { 1421029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick AvailableQueue->remove(TrySU); 14222902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurSU = TrySU; 14232902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 1424029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick // Interferences has been mutated. We must break. 14252902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick break; 14262902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 14272902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 14282902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 14292902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick if (!CurSU) { 14302902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Can't backtrack. If it's too expensive to copy the value, then try 14312902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // duplicate the nodes that produces these "too expensive to copy" 14322902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // values to break the dependency. In case even that doesn't work, 14332902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // insert cross class copies. 14342902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // If it's not too expensive, i.e. cost != -1, issue copies. 14352902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *TrySU = Interferences[0]; 14362902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SmallVector<unsigned, 4> &LRegs = LRegsMap[TrySU]; 14372902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick assert(LRegs.size() == 1 && "Can't handle this yet!"); 14382902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick unsigned Reg = LRegs[0]; 14392902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *LRDef = LiveRegDefs[Reg]; 14402902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick EVT VT = getPhysicalRegisterVT(LRDef->getNode(), Reg, TII); 14412902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick const TargetRegisterClass *RC = 14422902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick TRI->getMinimalPhysRegClass(Reg, VT); 14432902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick const TargetRegisterClass *DestRC = TRI->getCrossCopyRegClass(RC); 14442902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 1445b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng // If cross copy register class is the same as RC, then it must be possible 1446b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng // copy the value directly. Do not try duplicate the def. 1447b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng // If cross copy register class is not the same as RC, then it's possible to 1448b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng // copy the value but it require cross register class copies and it is 1449b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng // expensive. 1450b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng // If cross copy register class is null, then it's not possible to copy 1451b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng // the value at all. 14522902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *NewDef = 0; 1453b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng if (DestRC != RC) { 14542902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick NewDef = CopyAndMoveSuccessors(LRDef); 1455b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng if (!DestRC && !NewDef) 1456b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng report_fatal_error("Can't handle live physical register dependency!"); 1457b0519e15f70cef7ba16b712f258d4782ade17e13Evan Cheng } 14582902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick if (!NewDef) { 14592902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Issue copies, these can be expensive cross register class copies. 14602902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SmallVector<SUnit*, 2> Copies; 14612902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies); 14622902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick DEBUG(dbgs() << " Adding an edge from SU #" << TrySU->NodeNum 14632902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick << " to SU #" << Copies.front()->NodeNum << "\n"); 1464a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick AddPred(TrySU, SDep(Copies.front(), SDep::Artificial)); 14652902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick NewDef = Copies.back(); 14662902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 14672902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick 14682902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick DEBUG(dbgs() << " Adding an edge from SU #" << NewDef->NodeNum 14692902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick << " to SU #" << TrySU->NodeNum << "\n"); 14702902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick LiveRegDefs[Reg] = NewDef; 1471a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick AddPred(NewDef, SDep(TrySU, SDep::Artificial)); 14722902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick TrySU->isAvailable = false; 14732902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick CurSU = NewDef; 14742902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick } 14752902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick assert(CurSU && "Unable to resolve live physical register dependencies!"); 14762902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick return CurSU; 14772902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick} 1478a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng 1479e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// ListScheduleBottomUp - The main loop of list scheduling for bottom-up 1480e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// schedulers. 1481e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengvoid ScheduleDAGRRList::ListScheduleBottomUp() { 14829e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman // Release any predecessors of the special Exit node. 14833d420cb2fee925d0888cd3a60a222a19e75cd890Andrew Trick ReleasePredecessors(&ExitSU); 14849e64bbb322417c09f27afdf08e3946287c9df5aaDan Gohman 1485e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // Add root to Available queue. 148680792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman if (!SUnits.empty()) { 1487a23b3b803e3c65e84d6cadaa221de8b256cbe28dDan Gohman SUnit *RootSU = &SUnits[DAG->getRoot().getNode()->getNodeId()]; 148880792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman assert(RootSU->Succs.empty() && "Graph root shouldn't have successors!"); 148980792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman RootSU->isAvailable = true; 149080792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman AvailableQueue->push(RootSU); 149180792f3ddec43aff7f0758c9096f8cb53dcc1e40Dan Gohman } 1492e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1493e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // While Available queue is not empty, grab the node with the highest 14948d1bfad00b1ebff5b140b6e1bd7e26bad697d6e1Dan Gohman // priority. If it is not ready put it back. Schedule the node. 14954c8c83022b501759d8559e224c84ae2a9921ba41Dan Gohman Sequence.reserve(SUnits.size()); 1496029f4fd2ff539ed143b83c140349df2c064965d2Andrew Trick while (!AvailableQueue->empty() || !Interferences.empty()) { 149787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick DEBUG(dbgs() << "\nExamining Available:\n"; 14982da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AvailableQueue->dump(this)); 14992da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 15002902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // Pick the best node to schedule taking all constraints into 15012902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick // consideration. 15022902736a507df1c6fdc0daa4e8f0e385bb5f7820Andrew Trick SUnit *SU = PickNodeToScheduleBottomUp(); 1503a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng 15042da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AdvancePastStalls(SU); 1505a2ee2756f7f80d24312d6ac41b4f2ae548441cacEvan Cheng 15062da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleNodeBottomUp(SU); 15072da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 15082da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick while (AvailableQueue->empty() && !PendingQueue.empty()) { 15092da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick // Advance the cycle to free resources. Skip ahead to the next ready SU. 15102da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick assert(MinAvailableCycle < UINT_MAX && "MinAvailableCycle uninitialized"); 15112da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick AdvanceToCycle(std::max(CurCycle + 1, MinAvailableCycle)); 15122da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 1513e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 1514e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1515e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng // Reverse the order if it is bottom up. 1516e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng std::reverse(Sequence.begin(), Sequence.end()); 151738036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1518e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#ifndef NDEBUG 15194c727204271067f3dbf50bd23098b2df8e1cc47aAndrew Trick VerifyScheduledSequence(/*isBottomUp=*/true); 1520e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng#endif 1521e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 1522e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1523e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 1524f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// RegReductionPriorityQueue Definition 1525e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 1526e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// 1527e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// This is a SchedulingPriorityQueue that schedules using Sethi Ullman numbers 1528e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// to reduce register pressure. 152938036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick// 1530e165a78551a91d8420cd8f074d97701e8788f8b5Evan Chengnamespace { 1531f697c8a19adf962a933b055383952e72789a0e20Andrew Trickclass RegReductionPQBase; 1532f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1533f697c8a19adf962a933b055383952e72789a0e20Andrew Trickstruct queue_sort : public std::binary_function<SUnit*, SUnit*, bool> { 1534f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool isReady(SUnit* SU, unsigned CurCycle) const { return true; } 1535f697c8a19adf962a933b055383952e72789a0e20Andrew Trick}; 153638036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 15374cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick#ifndef NDEBUG 15384cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Tricktemplate<class SF> 15394cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trickstruct reverse_sort : public queue_sort { 15404cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick SF &SortFunc; 15414cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick reverse_sort(SF &sf) : SortFunc(sf) {} 15424cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick reverse_sort(const reverse_sort &RHS) : SortFunc(RHS.SortFunc) {} 15434cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick 15444cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick bool operator()(SUnit* left, SUnit* right) const { 15454cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick // reverse left/right rather than simply !SortFunc(left, right) 15464cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick // to expose different paths in the comparison logic. 15474cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick return SortFunc(right, left); 15484cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick } 15494cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick}; 15504cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick#endif // NDEBUG 15514cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick 1552f697c8a19adf962a933b055383952e72789a0e20Andrew Trick/// bu_ls_rr_sort - Priority function for bottom up register pressure 1553f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// reduction scheduler. 1554f697c8a19adf962a933b055383952e72789a0e20Andrew Trickstruct bu_ls_rr_sort : public queue_sort { 1555f697c8a19adf962a933b055383952e72789a0e20Andrew Trick enum { 1556f697c8a19adf962a933b055383952e72789a0e20Andrew Trick IsBottomUp = true, 1557f697c8a19adf962a933b055383952e72789a0e20Andrew Trick HasReadyFilter = false 15582da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick }; 15592da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1560f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegReductionPQBase *SPQ; 1561f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bu_ls_rr_sort(RegReductionPQBase *spq) : SPQ(spq) {} 1562f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bu_ls_rr_sort(const bu_ls_rr_sort &RHS) : SPQ(RHS.SPQ) {} 15632da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1564f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool operator()(SUnit* left, SUnit* right) const; 1565f697c8a19adf962a933b055383952e72789a0e20Andrew Trick}; 156638036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1567f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// src_ls_rr_sort - Priority function for source order scheduler. 1568f697c8a19adf962a933b055383952e72789a0e20Andrew Trickstruct src_ls_rr_sort : public queue_sort { 1569f697c8a19adf962a933b055383952e72789a0e20Andrew Trick enum { 1570f697c8a19adf962a933b055383952e72789a0e20Andrew Trick IsBottomUp = true, 1571f697c8a19adf962a933b055383952e72789a0e20Andrew Trick HasReadyFilter = false 1572e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng }; 1573187361b056823df4ff292561fe47468dad956872Bill Wendling 1574f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegReductionPQBase *SPQ; 1575f697c8a19adf962a933b055383952e72789a0e20Andrew Trick src_ls_rr_sort(RegReductionPQBase *spq) 1576f697c8a19adf962a933b055383952e72789a0e20Andrew Trick : SPQ(spq) {} 1577f697c8a19adf962a933b055383952e72789a0e20Andrew Trick src_ls_rr_sort(const src_ls_rr_sort &RHS) 1578f697c8a19adf962a933b055383952e72789a0e20Andrew Trick : SPQ(RHS.SPQ) {} 15792da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1580f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool operator()(SUnit* left, SUnit* right) const; 1581f697c8a19adf962a933b055383952e72789a0e20Andrew Trick}; 158238036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1583f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// hybrid_ls_rr_sort - Priority function for hybrid scheduler. 1584f697c8a19adf962a933b055383952e72789a0e20Andrew Trickstruct hybrid_ls_rr_sort : public queue_sort { 1585f697c8a19adf962a933b055383952e72789a0e20Andrew Trick enum { 1586f697c8a19adf962a933b055383952e72789a0e20Andrew Trick IsBottomUp = true, 1587a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick HasReadyFilter = false 1588187361b056823df4ff292561fe47468dad956872Bill Wendling }; 158915a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 1590f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegReductionPQBase *SPQ; 1591f697c8a19adf962a933b055383952e72789a0e20Andrew Trick hybrid_ls_rr_sort(RegReductionPQBase *spq) 1592f697c8a19adf962a933b055383952e72789a0e20Andrew Trick : SPQ(spq) {} 1593f697c8a19adf962a933b055383952e72789a0e20Andrew Trick hybrid_ls_rr_sort(const hybrid_ls_rr_sort &RHS) 1594f697c8a19adf962a933b055383952e72789a0e20Andrew Trick : SPQ(RHS.SPQ) {} 1595f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1596f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool isReady(SUnit *SU, unsigned CurCycle) const; 15972da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1598f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool operator()(SUnit* left, SUnit* right) const; 1599f697c8a19adf962a933b055383952e72789a0e20Andrew Trick}; 16004f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 1601f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// ilp_ls_rr_sort - Priority function for ILP (instruction level parallelism) 1602f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// scheduler. 1603f697c8a19adf962a933b055383952e72789a0e20Andrew Trickstruct ilp_ls_rr_sort : public queue_sort { 1604f697c8a19adf962a933b055383952e72789a0e20Andrew Trick enum { 1605f697c8a19adf962a933b055383952e72789a0e20Andrew Trick IsBottomUp = true, 1606a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick HasReadyFilter = false 160715a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng }; 160870017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 1609f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegReductionPQBase *SPQ; 1610f697c8a19adf962a933b055383952e72789a0e20Andrew Trick ilp_ls_rr_sort(RegReductionPQBase *spq) 1611f697c8a19adf962a933b055383952e72789a0e20Andrew Trick : SPQ(spq) {} 1612f697c8a19adf962a933b055383952e72789a0e20Andrew Trick ilp_ls_rr_sort(const ilp_ls_rr_sort &RHS) 1613f697c8a19adf962a933b055383952e72789a0e20Andrew Trick : SPQ(RHS.SPQ) {} 1614f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1615f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool isReady(SUnit *SU, unsigned CurCycle) const; 16162da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1617f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool operator()(SUnit* left, SUnit* right) const; 1618f697c8a19adf962a933b055383952e72789a0e20Andrew Trick}; 161970017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 1620f697c8a19adf962a933b055383952e72789a0e20Andrew Trickclass RegReductionPQBase : public SchedulingPriorityQueue { 1621f697c8a19adf962a933b055383952e72789a0e20Andrew Trickprotected: 1622f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::vector<SUnit*> Queue; 1623f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned CurQueueId; 1624f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool TracksRegPressure; 1625479389a4da53ce72226366cc6d1cad13da158909Evan Cheng bool SrcOrder; 16262da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1627f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // SUnits - The SUnits for the current graph. 1628f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::vector<SUnit> *SUnits; 1629f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1630f697c8a19adf962a933b055383952e72789a0e20Andrew Trick MachineFunction &MF; 1631f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetInstrInfo *TII; 1632f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetRegisterInfo *TRI; 1633f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetLowering *TLI; 1634f697c8a19adf962a933b055383952e72789a0e20Andrew Trick ScheduleDAGRRList *scheduleDAG; 1635f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1636f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // SethiUllmanNumbers - The SethiUllman number for each node. 1637f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::vector<unsigned> SethiUllmanNumbers; 1638f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1639f697c8a19adf962a933b055383952e72789a0e20Andrew Trick /// RegPressure - Tracking current reg pressure per register class. 1640f697c8a19adf962a933b055383952e72789a0e20Andrew Trick /// 1641f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::vector<unsigned> RegPressure; 1642f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1643f697c8a19adf962a933b055383952e72789a0e20Andrew Trick /// RegLimit - Tracking the number of allocatable registers per register 1644f697c8a19adf962a933b055383952e72789a0e20Andrew Trick /// class. 1645f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::vector<unsigned> RegLimit; 1646f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1647f697c8a19adf962a933b055383952e72789a0e20Andrew Trickpublic: 1648f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegReductionPQBase(MachineFunction &mf, 1649f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool hasReadyFilter, 1650f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool tracksrp, 1651479389a4da53ce72226366cc6d1cad13da158909Evan Cheng bool srcorder, 1652f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetInstrInfo *tii, 1653f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetRegisterInfo *tri, 1654f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetLowering *tli) 1655f697c8a19adf962a933b055383952e72789a0e20Andrew Trick : SchedulingPriorityQueue(hasReadyFilter), 1656479389a4da53ce72226366cc6d1cad13da158909Evan Cheng CurQueueId(0), TracksRegPressure(tracksrp), SrcOrder(srcorder), 1657f697c8a19adf962a933b055383952e72789a0e20Andrew Trick MF(mf), TII(tii), TRI(tri), TLI(tli), scheduleDAG(NULL) { 1658f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (TracksRegPressure) { 1659f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned NumRC = TRI->getNumRegClasses(); 1660f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegLimit.resize(NumRC); 1661f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegPressure.resize(NumRC); 1662f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::fill(RegLimit.begin(), RegLimit.end(), 0); 1663f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::fill(RegPressure.begin(), RegPressure.end(), 0); 1664f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (TargetRegisterInfo::regclass_iterator I = TRI->regclass_begin(), 1665f697c8a19adf962a933b055383952e72789a0e20Andrew Trick E = TRI->regclass_end(); I != E; ++I) 1666be2119e8e2bc7006cfd638a24367acbfda625d16Cameron Zwarich RegLimit[(*I)->getID()] = tri->getRegPressureLimit(*I, MF); 1667f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1668f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1669f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1670f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void setScheduleDAG(ScheduleDAGRRList *scheduleDag) { 1671f697c8a19adf962a933b055383952e72789a0e20Andrew Trick scheduleDAG = scheduleDag; 1672f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1673f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1674f697c8a19adf962a933b055383952e72789a0e20Andrew Trick ScheduleHazardRecognizer* getHazardRec() { 1675f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return scheduleDAG->getHazardRec(); 1676f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1677f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1678f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void initNodes(std::vector<SUnit> &sunits); 1679f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1680f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void addNode(const SUnit *SU); 1681f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1682f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void updateNode(const SUnit *SU); 1683f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1684f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void releaseState() { 1685f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SUnits = 0; 1686f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SethiUllmanNumbers.clear(); 1687f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::fill(RegPressure.begin(), RegPressure.end(), 0); 1688f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1689f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1690f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned getNodePriority(const SUnit *SU) const; 1691f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1692f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned getNodeOrdering(const SUnit *SU) const { 1693336298cf2cc68d9af163992b9f9cafddd4bb3c8aAndrew Trick if (!SU->getNode()) return 0; 1694336298cf2cc68d9af163992b9f9cafddd4bb3c8aAndrew Trick 1695f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return scheduleDAG->DAG->GetOrdering(SU->getNode()); 1696f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1697f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1698f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool empty() const { return Queue.empty(); } 1699f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1700f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void push(SUnit *U) { 1701f697c8a19adf962a933b055383952e72789a0e20Andrew Trick assert(!U->NodeQueueId && "Node in the queue already"); 1702f697c8a19adf962a933b055383952e72789a0e20Andrew Trick U->NodeQueueId = ++CurQueueId; 1703f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Queue.push_back(U); 1704f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1705f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1706f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void remove(SUnit *SU) { 1707f697c8a19adf962a933b055383952e72789a0e20Andrew Trick assert(!Queue.empty() && "Queue is empty!"); 1708f697c8a19adf962a933b055383952e72789a0e20Andrew Trick assert(SU->NodeQueueId != 0 && "Not in queue!"); 1709f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::vector<SUnit *>::iterator I = std::find(Queue.begin(), Queue.end(), 1710f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SU); 1711f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (I != prior(Queue.end())) 1712f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::swap(*I, Queue.back()); 1713f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Queue.pop_back(); 1714f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SU->NodeQueueId = 0; 1715f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1716f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 171792e946630d5f9bb092853b93501387dd216899b9Andrew Trick bool tracksRegPressure() const { return TracksRegPressure; } 171892e946630d5f9bb092853b93501387dd216899b9Andrew Trick 1719f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void dumpRegPressure() const; 1720f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1721f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool HighRegPressure(const SUnit *SU) const; 1722f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1723e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick bool MayReduceRegPressure(SUnit *SU) const; 1724e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 1725e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick int RegPressureDiff(SUnit *SU, unsigned &LiveUses) const; 1726f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1727953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick void scheduledNode(SUnit *SU); 1728f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1729953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trick void unscheduledNode(SUnit *SU); 1730f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1731f697c8a19adf962a933b055383952e72789a0e20Andrew Trickprotected: 1732f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool canClobber(const SUnit *SU, const SUnit *Op); 1733ef0b3ca3a8935b5390633dc7bb4adcdb99e0c26aDuncan Sands void AddPseudoTwoAddrDeps(); 1734f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void PrescheduleNodesWithMultipleUses(); 1735f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void CalculateSethiUllmanNumbers(); 1736f697c8a19adf962a933b055383952e72789a0e20Andrew Trick}; 1737f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1738f697c8a19adf962a933b055383952e72789a0e20Andrew Tricktemplate<class SF> 17394cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trickstatic SUnit *popFromQueueImpl(std::vector<SUnit*> &Q, SF &Picker) { 17404cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick std::vector<SUnit *>::iterator Best = Q.begin(); 17414cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick for (std::vector<SUnit *>::iterator I = llvm::next(Q.begin()), 17424cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick E = Q.end(); I != E; ++I) 17434cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick if (Picker(*Best, *I)) 17444cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick Best = I; 17454cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick SUnit *V = *Best; 17464cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick if (Best != prior(Q.end())) 17474cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick std::swap(*Best, Q.back()); 17484cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick Q.pop_back(); 17494cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick return V; 17504cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick} 17514cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick 17524cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Tricktemplate<class SF> 17534cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew TrickSUnit *popFromQueue(std::vector<SUnit*> &Q, SF &Picker, ScheduleDAG *DAG) { 17544cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick#ifndef NDEBUG 17554cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick if (DAG->StressSched) { 17564cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick reverse_sort<SF> RPicker(Picker); 17574cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick return popFromQueueImpl(Q, RPicker); 1758f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 17594cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick#endif 17604cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick (void)DAG; 17614cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick return popFromQueueImpl(Q, Picker); 17624cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick} 1763f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 17644cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Tricktemplate<class SF> 17654cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trickclass RegReductionPriorityQueue : public RegReductionPQBase { 1766f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SF Picker; 1767f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1768f697c8a19adf962a933b055383952e72789a0e20Andrew Trickpublic: 1769f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegReductionPriorityQueue(MachineFunction &mf, 1770f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool tracksrp, 1771479389a4da53ce72226366cc6d1cad13da158909Evan Cheng bool srcorder, 1772f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetInstrInfo *tii, 1773f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetRegisterInfo *tri, 1774f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetLowering *tli) 1775479389a4da53ce72226366cc6d1cad13da158909Evan Cheng : RegReductionPQBase(mf, SF::HasReadyFilter, tracksrp, srcorder, 1776479389a4da53ce72226366cc6d1cad13da158909Evan Cheng tii, tri, tli), 1777f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Picker(this) {} 1778f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1779f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool isBottomUp() const { return SF::IsBottomUp; } 1780f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1781f697c8a19adf962a933b055383952e72789a0e20Andrew Trick bool isReady(SUnit *U) const { 1782f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return Picker.HasReadyFilter && Picker.isReady(U, getCurCycle()); 1783f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1784f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1785f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SUnit *pop() { 1786f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (Queue.empty()) return NULL; 1787f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 17884cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick SUnit *V = popFromQueue(Queue, Picker, scheduleDAG); 1789f697c8a19adf962a933b055383952e72789a0e20Andrew Trick V->NodeQueueId = 0; 1790f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return V; 1791f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1792f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1793b720be6a50f4e1b3280d2b029ee38dda14577525Manman Ren#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 1794f697c8a19adf962a933b055383952e72789a0e20Andrew Trick void dump(ScheduleDAG *DAG) const { 1795f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // Emulate pop() without clobbering NodeQueueIds. 1796f697c8a19adf962a933b055383952e72789a0e20Andrew Trick std::vector<SUnit*> DumpQueue = Queue; 1797f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SF DumpPicker = Picker; 1798f697c8a19adf962a933b055383952e72789a0e20Andrew Trick while (!DumpQueue.empty()) { 17994cb971ce1c8b254f29365c988b55f6dcfe86d21eAndrew Trick SUnit *SU = popFromQueue(DumpQueue, DumpPicker, scheduleDAG); 1800ff764815e6531be6b2d944bd6a3f1fcfc682db01Dan Gohman dbgs() << "Height " << SU->getHeight() << ": "; 1801f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SU->dump(DAG); 1802f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1803f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 180477e300e8f0b8db8eec448cae9c87d7c5bfad9757Manman Ren#endif 1805f697c8a19adf962a933b055383952e72789a0e20Andrew Trick}; 1806f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1807f697c8a19adf962a933b055383952e72789a0e20Andrew Tricktypedef RegReductionPriorityQueue<bu_ls_rr_sort> 1808f697c8a19adf962a933b055383952e72789a0e20Andrew TrickBURegReductionPriorityQueue; 1809f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1810f697c8a19adf962a933b055383952e72789a0e20Andrew Tricktypedef RegReductionPriorityQueue<src_ls_rr_sort> 1811f697c8a19adf962a933b055383952e72789a0e20Andrew TrickSrcRegReductionPriorityQueue; 1812f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1813f697c8a19adf962a933b055383952e72789a0e20Andrew Tricktypedef RegReductionPriorityQueue<hybrid_ls_rr_sort> 1814f697c8a19adf962a933b055383952e72789a0e20Andrew TrickHybridBURRPriorityQueue; 1815f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1816f697c8a19adf962a933b055383952e72789a0e20Andrew Tricktypedef RegReductionPriorityQueue<ilp_ls_rr_sort> 1817f697c8a19adf962a933b055383952e72789a0e20Andrew TrickILPBURRPriorityQueue; 1818f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} // end anonymous namespace 1819f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 1820f697c8a19adf962a933b055383952e72789a0e20Andrew Trick//===----------------------------------------------------------------------===// 1821f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Static Node Priority for Register Pressure Reduction 1822f697c8a19adf962a933b055383952e72789a0e20Andrew Trick//===----------------------------------------------------------------------===// 1823e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 182412f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick// Check for special nodes that bypass scheduling heuristics. 182512f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick// Currently this pushes TokenFactor nodes down, but may be used for other 182612f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick// pseudo-ops as well. 182712f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick// 182812f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick// Return -1 to schedule right above left, 1 for left above right. 182912f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick// Return 0 if no bias exists. 183012f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trickstatic int checkSpecialNodes(const SUnit *left, const SUnit *right) { 183112f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick bool LSchedLow = left->isScheduleLow; 183212f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick bool RSchedLow = right->isScheduleLow; 183312f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick if (LSchedLow != RSchedLow) 183412f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick return LSchedLow < RSchedLow ? 1 : -1; 183512f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick return 0; 183612f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick} 183712f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick 1838117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman/// CalcNodeSethiUllmanNumber - Compute Sethi Ullman number. 1839117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman/// Smaller number is the higher priority. 1840c6be777208f4539af400ac694d9d1dc8b992bc80Evan Chengstatic unsigned 1841117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan GohmanCalcNodeSethiUllmanNumber(const SUnit *SU, std::vector<unsigned> &SUNumbers) { 1842c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng unsigned &SethiUllmanNumber = SUNumbers[SU->NodeNum]; 1843c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng if (SethiUllmanNumber != 0) 1844c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng return SethiUllmanNumber; 1845c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng 1846c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng unsigned Extra = 0; 1847c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 1848c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng I != E; ++I) { 184954e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isCtrl()) continue; // ignore chain preds 185054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *PredSU = I->getSUnit(); 1851117f3e9ee426ab7eb120b5ca1b65763baae2a824Dan Gohman unsigned PredSethiUllman = CalcNodeSethiUllmanNumber(PredSU, SUNumbers); 1852c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng if (PredSethiUllman > SethiUllmanNumber) { 1853c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng SethiUllmanNumber = PredSethiUllman; 1854c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng Extra = 0; 18558182347d70413174f2e80ea429801e887aee5cc3Evan Cheng } else if (PredSethiUllman == SethiUllmanNumber) 1856c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng ++Extra; 1857c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng } 1858c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng 1859c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng SethiUllmanNumber += Extra; 1860c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng 1861c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng if (SethiUllmanNumber == 0) 1862c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng SethiUllmanNumber = 1; 186338036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1864c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng return SethiUllmanNumber; 1865c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng} 1866c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng 1867f697c8a19adf962a933b055383952e72789a0e20Andrew Trick/// CalculateSethiUllmanNumbers - Calculate Sethi-Ullman numbers of all 1868f697c8a19adf962a933b055383952e72789a0e20Andrew Trick/// scheduling units. 1869f697c8a19adf962a933b055383952e72789a0e20Andrew Trickvoid RegReductionPQBase::CalculateSethiUllmanNumbers() { 1870f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SethiUllmanNumbers.assign(SUnits->size(), 0); 187138036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1872f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (unsigned i = 0, e = SUnits->size(); i != e; ++i) 1873f697c8a19adf962a933b055383952e72789a0e20Andrew Trick CalcNodeSethiUllmanNumber(&(*SUnits)[i], SethiUllmanNumbers); 1874f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 18752da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 1876f697c8a19adf962a933b055383952e72789a0e20Andrew Trickvoid RegReductionPQBase::addNode(const SUnit *SU) { 1877f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned SUSize = SethiUllmanNumbers.size(); 1878f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SUnits->size() > SUSize) 1879f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SethiUllmanNumbers.resize(SUSize*2, 0); 1880f697c8a19adf962a933b055383952e72789a0e20Andrew Trick CalcNodeSethiUllmanNumber(SU, SethiUllmanNumbers); 1881f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 1882a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 1883f697c8a19adf962a933b055383952e72789a0e20Andrew Trickvoid RegReductionPQBase::updateNode(const SUnit *SU) { 1884f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SethiUllmanNumbers[SU->NodeNum] = 0; 1885f697c8a19adf962a933b055383952e72789a0e20Andrew Trick CalcNodeSethiUllmanNumber(SU, SethiUllmanNumbers); 1886f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 1887a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng 18880bc308600b0069f07ba722b472c68588573ebd28Andrew Trick// Lower priority means schedule further down. For bottom-up scheduling, lower 18890bc308600b0069f07ba722b472c68588573ebd28Andrew Trick// priority SUs are scheduled before higher priority SUs. 1890f697c8a19adf962a933b055383952e72789a0e20Andrew Trickunsigned RegReductionPQBase::getNodePriority(const SUnit *SU) const { 1891f697c8a19adf962a933b055383952e72789a0e20Andrew Trick assert(SU->NodeNum < SethiUllmanNumbers.size()); 1892f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned Opc = SU->getNode() ? SU->getNode()->getOpcode() : 0; 1893f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (Opc == ISD::TokenFactor || Opc == ISD::CopyToReg) 1894f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // CopyToReg should be close to its uses to facilitate coalescing and 1895f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // avoid spilling. 1896f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return 0; 1897f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (Opc == TargetOpcode::EXTRACT_SUBREG || 1898f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Opc == TargetOpcode::SUBREG_TO_REG || 1899f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Opc == TargetOpcode::INSERT_SUBREG) 1900f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // EXTRACT_SUBREG, INSERT_SUBREG, and SUBREG_TO_REG nodes should be 1901f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // close to their uses to facilitate coalescing. 1902f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return 0; 1903f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SU->NumSuccs == 0 && SU->NumPreds != 0) 1904f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // If SU does not have a register use, i.e. it doesn't produce a value 1905f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // that would be consumed (e.g. store), then it terminates a chain of 1906f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // computation. Give it a large SethiUllman number so it will be 1907f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // scheduled right before its predecessors that it doesn't lengthen 1908f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // their live ranges. 1909f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return 0xffff; 1910f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SU->NumPreds == 0 && SU->NumSuccs != 0) 1911f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // If SU does not have a register def, schedule it close to its uses 1912f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // because it does not lengthen any live ranges. 1913f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return 0; 1914554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng#if 1 1915f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return SethiUllmanNumbers[SU->NodeNum]; 1916554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng#else 1917554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng unsigned Priority = SethiUllmanNumbers[SU->NodeNum]; 1918554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng if (SU->isCallOp) { 1919554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // FIXME: This assumes all of the defs are used as call operands. 1920554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng int NP = (int)Priority - SU->getNode()->getNumValues(); 1921554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng return (NP > 0) ? NP : 0; 1922554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng } 1923554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng return Priority; 1924554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng#endif 1925f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 1926e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1927f697c8a19adf962a933b055383952e72789a0e20Andrew Trick//===----------------------------------------------------------------------===// 1928f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Register Pressure Tracking 1929f697c8a19adf962a933b055383952e72789a0e20Andrew Trick//===----------------------------------------------------------------------===// 1930247fb4ebd363323b6642ce2bb08e53db705ca094Bill Wendling 1931f697c8a19adf962a933b055383952e72789a0e20Andrew Trickvoid RegReductionPQBase::dumpRegPressure() const { 1932b720be6a50f4e1b3280d2b029ee38dda14577525Manman Ren#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 1933f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (TargetRegisterInfo::regclass_iterator I = TRI->regclass_begin(), 1934f697c8a19adf962a933b055383952e72789a0e20Andrew Trick E = TRI->regclass_end(); I != E; ++I) { 1935f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const TargetRegisterClass *RC = *I; 1936f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned Id = RC->getID(); 1937f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned RP = RegPressure[Id]; 1938f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!RP) continue; 1939f697c8a19adf962a933b055383952e72789a0e20Andrew Trick DEBUG(dbgs() << RC->getName() << ": " << RP << " / " << RegLimit[Id] 1940f697c8a19adf962a933b055383952e72789a0e20Andrew Trick << '\n'); 1941f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 194277e300e8f0b8db8eec448cae9c87d7c5bfad9757Manman Ren#endif 1943f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 194415a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 1945f697c8a19adf962a933b055383952e72789a0e20Andrew Trickbool RegReductionPQBase::HighRegPressure(const SUnit *SU) const { 1946f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!TLI) 1947f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return false; 194838036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 1949f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (SUnit::const_pred_iterator I = SU->Preds.begin(),E = SU->Preds.end(); 1950f697c8a19adf962a933b055383952e72789a0e20Andrew Trick I != E; ++I) { 1951f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (I->isCtrl()) 1952f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 1953f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SUnit *PredSU = I->getSUnit(); 195492e946630d5f9bb092853b93501387dd216899b9Andrew Trick // NumRegDefsLeft is zero when enough uses of this node have been scheduled 195592e946630d5f9bb092853b93501387dd216899b9Andrew Trick // to cover the number of registers defined (they are all live). 195692e946630d5f9bb092853b93501387dd216899b9Andrew Trick if (PredSU->NumRegDefsLeft == 0) { 1957f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 1958e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 195992e946630d5f9bb092853b93501387dd216899b9Andrew Trick for (ScheduleDAGSDNodes::RegDefIter RegDefPos(PredSU, scheduleDAG); 196092e946630d5f9bb092853b93501387dd216899b9Andrew Trick RegDefPos.IsValid(); RegDefPos.Advance()) { 196177b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson unsigned RCId, Cost; 1962397fc4874efe9c17e737d4c5c50bd19dc3bf27f5Jakob Stoklund Olesen GetCostForDef(RegDefPos, TLI, TII, TRI, RCId, Cost, MF); 196377b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson 1964f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if ((RegPressure[RCId] + Cost) >= RegLimit[RCId]) 1965f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return true; 1966a6fb1b6743ee1411accf2d6e636f73f2ee0a7f5bEvan Cheng } 1967f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1968f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return false; 1969f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 1970e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 1971e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickbool RegReductionPQBase::MayReduceRegPressure(SUnit *SU) const { 1972f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const SDNode *N = SU->getNode(); 19734f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 1974f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!N->isMachineOpcode() || !SU->NumSuccs) 1975f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return false; 19764f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 1977f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs(); 1978f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (unsigned i = 0; i != NumDefs; ++i) { 1979860e7cdab9d5eceda5ac52ae0ddfb4bdab0067f2Patrik Hagglund MVT VT = N->getSimpleValueType(i); 1980f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!N->hasAnyUseOfValue(i)) 1981f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 1982f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 1983f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (RegPressure[RCId] >= RegLimit[RCId]) 1984f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return true; 1985f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 1986f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return false; 1987f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 19884f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 1989e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick// Compute the register pressure contribution by this instruction by count up 1990e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick// for uses that are not live and down for defs. Only count register classes 1991e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick// that are already under high pressure. As a side effect, compute the number of 1992e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick// uses of registers that are already live. 1993e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick// 1994e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick// FIXME: This encompasses the logic in HighRegPressure and MayReduceRegPressure 1995e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick// so could probably be factored. 1996e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trickint RegReductionPQBase::RegPressureDiff(SUnit *SU, unsigned &LiveUses) const { 1997e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick LiveUses = 0; 1998e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick int PDiff = 0; 1999e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick for (SUnit::const_pred_iterator I = SU->Preds.begin(),E = SU->Preds.end(); 2000e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick I != E; ++I) { 2001e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (I->isCtrl()) 2002e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick continue; 2003e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick SUnit *PredSU = I->getSUnit(); 2004e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick // NumRegDefsLeft is zero when enough uses of this node have been scheduled 2005e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick // to cover the number of registers defined (they are all live). 2006e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (PredSU->NumRegDefsLeft == 0) { 2007e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (PredSU->getNode()->isMachineOpcode()) 2008e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick ++LiveUses; 2009e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick continue; 2010e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick } 2011e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick for (ScheduleDAGSDNodes::RegDefIter RegDefPos(PredSU, scheduleDAG); 2012e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick RegDefPos.IsValid(); RegDefPos.Advance()) { 2013860e7cdab9d5eceda5ac52ae0ddfb4bdab0067f2Patrik Hagglund MVT VT = RegDefPos.GetValue(); 2014e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 2015e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (RegPressure[RCId] >= RegLimit[RCId]) 2016e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick ++PDiff; 2017e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick } 2018e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick } 2019e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick const SDNode *N = SU->getNode(); 2020e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 202129449448b0f0420dfcf52e278fc01adbf1690d70Eric Christopher if (!N || !N->isMachineOpcode() || !SU->NumSuccs) 2022e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick return PDiff; 2023e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 2024e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs(); 2025e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick for (unsigned i = 0; i != NumDefs; ++i) { 2026860e7cdab9d5eceda5ac52ae0ddfb4bdab0067f2Patrik Hagglund MVT VT = N->getSimpleValueType(i); 2027e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (!N->hasAnyUseOfValue(i)) 2028e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick continue; 2029e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 2030e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (RegPressure[RCId] >= RegLimit[RCId]) 2031e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick --PDiff; 2032e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick } 2033e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick return PDiff; 2034e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick} 2035e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 2036953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trickvoid RegReductionPQBase::scheduledNode(SUnit *SU) { 2037f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!TracksRegPressure) 2038f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return; 20394f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 204029449448b0f0420dfcf52e278fc01adbf1690d70Eric Christopher if (!SU->getNode()) 204129449448b0f0420dfcf52e278fc01adbf1690d70Eric Christopher return; 20420d93a110e31b384f59d91d6be27388d8ded5f03cAndrew Trick 2043f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 2044f697c8a19adf962a933b055383952e72789a0e20Andrew Trick I != E; ++I) { 2045f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (I->isCtrl()) 2046f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 2047f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SUnit *PredSU = I->getSUnit(); 204892e946630d5f9bb092853b93501387dd216899b9Andrew Trick // NumRegDefsLeft is zero when enough uses of this node have been scheduled 204992e946630d5f9bb092853b93501387dd216899b9Andrew Trick // to cover the number of registers defined (they are all live). 205092e946630d5f9bb092853b93501387dd216899b9Andrew Trick if (PredSU->NumRegDefsLeft == 0) { 2051f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 2052f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 205392e946630d5f9bb092853b93501387dd216899b9Andrew Trick // FIXME: The ScheduleDAG currently loses information about which of a 205492e946630d5f9bb092853b93501387dd216899b9Andrew Trick // node's values is consumed by each dependence. Consequently, if the node 205592e946630d5f9bb092853b93501387dd216899b9Andrew Trick // defines multiple register classes, we don't know which to pressurize 205692e946630d5f9bb092853b93501387dd216899b9Andrew Trick // here. Instead the following loop consumes the register defs in an 205792e946630d5f9bb092853b93501387dd216899b9Andrew Trick // arbitrary order. At least it handles the common case of clustered loads 205892e946630d5f9bb092853b93501387dd216899b9Andrew Trick // to the same class. For precise liveness, each SDep needs to indicate the 205992e946630d5f9bb092853b93501387dd216899b9Andrew Trick // result number. But that tightly couples the ScheduleDAG with the 206092e946630d5f9bb092853b93501387dd216899b9Andrew Trick // SelectionDAG making updates tricky. A simpler hack would be to attach a 206192e946630d5f9bb092853b93501387dd216899b9Andrew Trick // value type or register class to SDep. 206292e946630d5f9bb092853b93501387dd216899b9Andrew Trick // 206392e946630d5f9bb092853b93501387dd216899b9Andrew Trick // The most important aspect of register tracking is balancing the increase 206492e946630d5f9bb092853b93501387dd216899b9Andrew Trick // here with the reduction further below. Note that this SU may use multiple 206592e946630d5f9bb092853b93501387dd216899b9Andrew Trick // defs in PredSU. The can't be determined here, but we've already 206692e946630d5f9bb092853b93501387dd216899b9Andrew Trick // compensated by reducing NumRegDefsLeft in PredSU during 206792e946630d5f9bb092853b93501387dd216899b9Andrew Trick // ScheduleDAGSDNodes::AddSchedEdges. 206892e946630d5f9bb092853b93501387dd216899b9Andrew Trick --PredSU->NumRegDefsLeft; 206992e946630d5f9bb092853b93501387dd216899b9Andrew Trick unsigned SkipRegDefs = PredSU->NumRegDefsLeft; 207092e946630d5f9bb092853b93501387dd216899b9Andrew Trick for (ScheduleDAGSDNodes::RegDefIter RegDefPos(PredSU, scheduleDAG); 207192e946630d5f9bb092853b93501387dd216899b9Andrew Trick RegDefPos.IsValid(); RegDefPos.Advance(), --SkipRegDefs) { 207292e946630d5f9bb092853b93501387dd216899b9Andrew Trick if (SkipRegDefs) 2073f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 207477b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson 207577b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson unsigned RCId, Cost; 2076397fc4874efe9c17e737d4c5c50bd19dc3bf27f5Jakob Stoklund Olesen GetCostForDef(RegDefPos, TLI, TII, TRI, RCId, Cost, MF); 207777b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson RegPressure[RCId] += Cost; 207892e946630d5f9bb092853b93501387dd216899b9Andrew Trick break; 2079f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 2080f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 208189ba74d117577f5a1bd62ee209dc2ecb090172e3Evan Cheng 208292e946630d5f9bb092853b93501387dd216899b9Andrew Trick // We should have this assert, but there may be dead SDNodes that never 208392e946630d5f9bb092853b93501387dd216899b9Andrew Trick // materialize as SUnits, so they don't appear to generate liveness. 208492e946630d5f9bb092853b93501387dd216899b9Andrew Trick //assert(SU->NumRegDefsLeft == 0 && "not all regdefs have scheduled uses"); 208592e946630d5f9bb092853b93501387dd216899b9Andrew Trick int SkipRegDefs = (int)SU->NumRegDefsLeft; 208692e946630d5f9bb092853b93501387dd216899b9Andrew Trick for (ScheduleDAGSDNodes::RegDefIter RegDefPos(SU, scheduleDAG); 208792e946630d5f9bb092853b93501387dd216899b9Andrew Trick RegDefPos.IsValid(); RegDefPos.Advance(), --SkipRegDefs) { 208892e946630d5f9bb092853b93501387dd216899b9Andrew Trick if (SkipRegDefs > 0) 208992e946630d5f9bb092853b93501387dd216899b9Andrew Trick continue; 209077b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson unsigned RCId, Cost; 2091397fc4874efe9c17e737d4c5c50bd19dc3bf27f5Jakob Stoklund Olesen GetCostForDef(RegDefPos, TLI, TII, TRI, RCId, Cost, MF); 209277b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson if (RegPressure[RCId] < Cost) { 209392e946630d5f9bb092853b93501387dd216899b9Andrew Trick // Register pressure tracking is imprecise. This can happen. But we try 209492e946630d5f9bb092853b93501387dd216899b9Andrew Trick // hard not to let it happen because it likely results in poor scheduling. 209592e946630d5f9bb092853b93501387dd216899b9Andrew Trick DEBUG(dbgs() << " SU(" << SU->NodeNum << ") has too many regdefs\n"); 209692e946630d5f9bb092853b93501387dd216899b9Andrew Trick RegPressure[RCId] = 0; 209792e946630d5f9bb092853b93501387dd216899b9Andrew Trick } 209892e946630d5f9bb092853b93501387dd216899b9Andrew Trick else { 209977b4b13c2a525faf646a6784b24692cf0459b75eOwen Anderson RegPressure[RCId] -= Cost; 21004f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 2101f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 2102f697c8a19adf962a933b055383952e72789a0e20Andrew Trick dumpRegPressure(); 2103f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 21044f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 2105953be893e8cffa0ef9bf410036cd96aeb526e98aAndrew Trickvoid RegReductionPQBase::unscheduledNode(SUnit *SU) { 2106f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!TracksRegPressure) 2107f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return; 21084f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 2109f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const SDNode *N = SU->getNode(); 211029449448b0f0420dfcf52e278fc01adbf1690d70Eric Christopher if (!N) return; 21110d93a110e31b384f59d91d6be27388d8ded5f03cAndrew Trick 2112f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!N->isMachineOpcode()) { 2113f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (N->getOpcode() != ISD::CopyToReg) 2114f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return; 2115f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } else { 2116f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned Opc = N->getMachineOpcode(); 2117f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (Opc == TargetOpcode::EXTRACT_SUBREG || 2118f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Opc == TargetOpcode::INSERT_SUBREG || 2119f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Opc == TargetOpcode::SUBREG_TO_REG || 2120f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Opc == TargetOpcode::REG_SEQUENCE || 2121f697c8a19adf962a933b055383952e72789a0e20Andrew Trick Opc == TargetOpcode::IMPLICIT_DEF) 2122f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return; 2123f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 21244f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 2125f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 2126f697c8a19adf962a933b055383952e72789a0e20Andrew Trick I != E; ++I) { 2127f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (I->isCtrl()) 2128f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 2129f697c8a19adf962a933b055383952e72789a0e20Andrew Trick SUnit *PredSU = I->getSUnit(); 21300bc308600b0069f07ba722b472c68588573ebd28Andrew Trick // NumSuccsLeft counts all deps. Don't compare it with NumSuccs which only 21310bc308600b0069f07ba722b472c68588573ebd28Andrew Trick // counts data deps. 21320bc308600b0069f07ba722b472c68588573ebd28Andrew Trick if (PredSU->NumSuccsLeft != PredSU->Succs.size()) 2133f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 2134f697c8a19adf962a933b055383952e72789a0e20Andrew Trick const SDNode *PN = PredSU->getNode(); 2135f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!PN->isMachineOpcode()) { 2136f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (PN->getOpcode() == ISD::CopyFromReg) { 2137860e7cdab9d5eceda5ac52ae0ddfb4bdab0067f2Patrik Hagglund MVT VT = PN->getSimpleValueType(0); 2138f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 2139f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); 2140f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 2141f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 21424f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 2143f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned POpc = PN->getMachineOpcode(); 2144f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (POpc == TargetOpcode::IMPLICIT_DEF) 2145f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 21464ef4c171dba8e479f5f3fe7acb22e9fe97a4d6f8Andrew Trick if (POpc == TargetOpcode::EXTRACT_SUBREG || 21474ef4c171dba8e479f5f3fe7acb22e9fe97a4d6f8Andrew Trick POpc == TargetOpcode::INSERT_SUBREG || 21484ef4c171dba8e479f5f3fe7acb22e9fe97a4d6f8Andrew Trick POpc == TargetOpcode::SUBREG_TO_REG) { 2149860e7cdab9d5eceda5ac52ae0ddfb4bdab0067f2Patrik Hagglund MVT VT = PN->getSimpleValueType(0); 2150f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 2151f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); 2152f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 2153e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 2154f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned NumDefs = TII->get(PN->getMachineOpcode()).getNumDefs(); 2155f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (unsigned i = 0; i != NumDefs; ++i) { 2156860e7cdab9d5eceda5ac52ae0ddfb4bdab0067f2Patrik Hagglund MVT VT = PN->getSimpleValueType(i); 2157f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!PN->hasAnyUseOfValue(i)) 2158f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 2159f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 2160f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (RegPressure[RCId] < TLI->getRepRegClassCostFor(VT)) 2161f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // Register pressure tracking is imprecise. This can happen. 2162f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegPressure[RCId] = 0; 2163f697c8a19adf962a933b055383952e72789a0e20Andrew Trick else 2164f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegPressure[RCId] -= TLI->getRepRegClassCostFor(VT); 21654f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng } 2166f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 21674f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng 2168f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // Check for isMachineOpcode() as PrescheduleNodesWithMultipleUses() 2169f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // may transfer data dependencies to CopyToReg. 2170f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SU->NumSuccs && N->isMachineOpcode()) { 2171f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs(); 2172f697c8a19adf962a933b055383952e72789a0e20Andrew Trick for (unsigned i = NumDefs, e = N->getNumValues(); i != e; ++i) { 2173860e7cdab9d5eceda5ac52ae0ddfb4bdab0067f2Patrik Hagglund MVT VT = N->getSimpleValueType(i); 2174f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (VT == MVT::Glue || VT == MVT::Other) 2175f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 2176f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (!N->hasAnyUseOfValue(i)) 2177f697c8a19adf962a933b055383952e72789a0e20Andrew Trick continue; 2178f697c8a19adf962a933b055383952e72789a0e20Andrew Trick unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); 2179f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); 21802da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick } 2181f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 21822da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 2183f697c8a19adf962a933b055383952e72789a0e20Andrew Trick dumpRegPressure(); 2184e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 2185e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 2186f697c8a19adf962a933b055383952e72789a0e20Andrew Trick//===----------------------------------------------------------------------===// 2187f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Dynamic Node Priority for Register Pressure Reduction 2188f697c8a19adf962a933b055383952e72789a0e20Andrew Trick//===----------------------------------------------------------------------===// 2189f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2190c6deb3d44707de57e82e16642ab845bc8b9e9e01Evan Cheng/// closestSucc - Returns the scheduled cycle of the successor which is 2191b398fca15bcca895526699f20336fa6c6a624013Dan Gohman/// closest to the current cycle. 219261230d18d21a5dca1378e994f43934e4b314e595Evan Chengstatic unsigned closestSucc(const SUnit *SU) { 21933f23744df4809eba94284e601e81489212c974d4Dan Gohman unsigned MaxHeight = 0; 219461230d18d21a5dca1378e994f43934e4b314e595Evan Cheng for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 2195c6deb3d44707de57e82e16642ab845bc8b9e9e01Evan Cheng I != E; ++I) { 2196f0e366a929a1acb4bc14df5ef831cce74607a967Evan Cheng if (I->isCtrl()) continue; // ignore chain succs 21973f23744df4809eba94284e601e81489212c974d4Dan Gohman unsigned Height = I->getSUnit()->getHeight(); 2198c6deb3d44707de57e82e16642ab845bc8b9e9e01Evan Cheng // If there are bunch of CopyToRegs stacked up, they should be considered 2199c6deb3d44707de57e82e16642ab845bc8b9e9e01Evan Cheng // to be at the same position. 220054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->getSUnit()->getNode() && 220154e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman I->getSUnit()->getNode()->getOpcode() == ISD::CopyToReg) 22023f23744df4809eba94284e601e81489212c974d4Dan Gohman Height = closestSucc(I->getSUnit())+1; 22033f23744df4809eba94284e601e81489212c974d4Dan Gohman if (Height > MaxHeight) 22043f23744df4809eba94284e601e81489212c974d4Dan Gohman MaxHeight = Height; 2205c6deb3d44707de57e82e16642ab845bc8b9e9e01Evan Cheng } 22063f23744df4809eba94284e601e81489212c974d4Dan Gohman return MaxHeight; 220761230d18d21a5dca1378e994f43934e4b314e595Evan Cheng} 220861230d18d21a5dca1378e994f43934e4b314e595Evan Cheng 2209d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng/// calcMaxScratches - Returns an cost estimate of the worse case requirement 22108182347d70413174f2e80ea429801e887aee5cc3Evan Cheng/// for scratch registers, i.e. number of data dependencies. 2211d6c0758944b31bb5316b36cad37f4610a77f784dEvan Chengstatic unsigned calcMaxScratches(const SUnit *SU) { 2212d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng unsigned Scratches = 0; 2213d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 2214f2b14715d11e52adbb17a5860d1ce42f82f85a0cEvan Cheng I != E; ++I) { 221554e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isCtrl()) continue; // ignore chain preds 2216f2b14715d11e52adbb17a5860d1ce42f82f85a0cEvan Cheng Scratches++; 2217f2b14715d11e52adbb17a5860d1ce42f82f85a0cEvan Cheng } 2218d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng return Scratches; 2219d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng} 2220d6c0758944b31bb5316b36cad37f4610a77f784dEvan Cheng 222187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick/// hasOnlyLiveInOpers - Return true if SU has only value predecessors that are 222287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick/// CopyFromReg from a virtual register. 222387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trickstatic bool hasOnlyLiveInOpers(const SUnit *SU) { 222487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick bool RetVal = false; 222587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 222687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick I != E; ++I) { 222787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (I->isCtrl()) continue; 222887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick const SUnit *PredSU = I->getSUnit(); 222987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (PredSU->getNode() && 223087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick PredSU->getNode()->getOpcode() == ISD::CopyFromReg) { 223187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick unsigned Reg = 223287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick cast<RegisterSDNode>(PredSU->getNode()->getOperand(1))->getReg(); 223387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (TargetRegisterInfo::isVirtualRegister(Reg)) { 223487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick RetVal = true; 223587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick continue; 223687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick } 223787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick } 223887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick return false; 223987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick } 224087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick return RetVal; 224187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick} 224287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 224387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick/// hasOnlyLiveOutUses - Return true if SU has only value successors that are 2244089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng/// CopyToReg to a virtual register. This SU def is probably a liveout and 2245089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng/// it has no other use. It should be scheduled closer to the terminator. 2246089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Chengstatic bool hasOnlyLiveOutUses(const SUnit *SU) { 2247089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng bool RetVal = false; 2248089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); 2249089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng I != E; ++I) { 2250089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng if (I->isCtrl()) continue; 2251089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng const SUnit *SuccSU = I->getSUnit(); 2252089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng if (SuccSU->getNode() && SuccSU->getNode()->getOpcode() == ISD::CopyToReg) { 2253089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng unsigned Reg = 2254089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng cast<RegisterSDNode>(SuccSU->getNode()->getOperand(1))->getReg(); 2255089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng if (TargetRegisterInfo::isVirtualRegister(Reg)) { 2256089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng RetVal = true; 2257089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng continue; 2258089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng } 2259089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng } 2260089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng return false; 2261089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng } 2262089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng return RetVal; 2263089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng} 2264089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng 226587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// Set isVRegCycle for a node with only live in opers and live out uses. Also 226687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// set isVRegCycle for its CopyFromReg operands. 226787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// 226887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// This is only relevant for single-block loops, in which case the VRegCycle 226987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// node is likely an induction variable in which the operand and target virtual 227087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// registers should be coalesced (e.g. pre/post increment values). Setting the 227187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// isVRegCycle flag helps the scheduler prioritize other uses of the same 227287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// CopyFromReg so that this node becomes the virtual register "kill". This 227387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// avoids interference between the values live in and out of the block and 227487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// eliminates a copy inside the loop. 227587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trickstatic void initVRegCycle(SUnit *SU) { 227687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (DisableSchedVRegCycle) 227787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick return; 227887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 227987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (!hasOnlyLiveInOpers(SU) || !hasOnlyLiveOutUses(SU)) 228087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick return; 228187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 228287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick DEBUG(dbgs() << "VRegCycle: SU(" << SU->NodeNum << ")\n"); 228387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 228487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick SU->isVRegCycle = true; 228587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 228687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); 2287089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng I != E; ++I) { 228887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (I->isCtrl()) continue; 228987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick I->getSUnit()->isVRegCycle = true; 2290089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng } 229187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick} 229287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 229387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// After scheduling the definition of a VRegCycle, clear the isVRegCycle flag of 229487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// CopyFromReg operands. We should no longer penalize other uses of this VReg. 229587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trickstatic void resetVRegCycle(SUnit *SU) { 229687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (!SU->isVRegCycle) 229787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick return; 229887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 229987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick for (SUnit::const_pred_iterator I = SU->Preds.begin(),E = SU->Preds.end(); 2300089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng I != E; ++I) { 2301089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng if (I->isCtrl()) continue; // ignore chain preds 230287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick SUnit *PredSU = I->getSUnit(); 230387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (PredSU->isVRegCycle) { 230487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick assert(PredSU->getNode()->getOpcode() == ISD::CopyFromReg && 230587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick "VRegCycle def must be CopyFromReg"); 230687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick I->getSUnit()->isVRegCycle = 0; 230787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick } 2308089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng } 2309089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng} 2310089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng 231187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// Return true if this SUnit uses a CopyFromReg node marked as a VRegCycle. This 231287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick// means a node that defines the VRegCycle has not been scheduled yet. 231387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trickstatic bool hasVRegCycleUse(const SUnit *SU) { 231487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // If this SU also defines the VReg, don't hoist it as a "use". 231587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (SU->isVRegCycle) 231687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick return false; 231787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 231887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick for (SUnit::const_pred_iterator I = SU->Preds.begin(),E = SU->Preds.end(); 231987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick I != E; ++I) { 232054699765064842fd08d1466adc93453660bc2a85Andrew Trick if (I->isCtrl()) continue; // ignore chain preds 232187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (I->getSUnit()->isVRegCycle && 232287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick I->getSUnit()->getNode()->getOpcode() == ISD::CopyFromReg) { 232387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick DEBUG(dbgs() << " VReg cycle use: SU (" << SU->NodeNum << ")\n"); 232487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick return true; 232554699765064842fd08d1466adc93453660bc2a85Andrew Trick } 232654699765064842fd08d1466adc93453660bc2a85Andrew Trick } 232754699765064842fd08d1466adc93453660bc2a85Andrew Trick return false; 232854699765064842fd08d1466adc93453660bc2a85Andrew Trick} 232954699765064842fd08d1466adc93453660bc2a85Andrew Trick 2330f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Check for either a dependence (latency) or resource (hazard) stall. 2331f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// 2332f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Note: The ScheduleHazardRecognizer interface requires a non-const SU. 2333f697c8a19adf962a933b055383952e72789a0e20Andrew Trickstatic bool BUHasStall(SUnit *SU, int Height, RegReductionPQBase *SPQ) { 2334f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if ((int)SPQ->getCurCycle() < Height) return true; 2335f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SPQ->getHazardRec()->getHazardType(SU, 0) 2336f697c8a19adf962a933b055383952e72789a0e20Andrew Trick != ScheduleHazardRecognizer::NoHazard) 2337f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return true; 2338f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return false; 2339f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 2340f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2341f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Return -1 if left has higher priority, 1 if right has higher priority. 2342f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Return 0 if latency-based priority is equivalent. 2343f697c8a19adf962a933b055383952e72789a0e20Andrew Trickstatic int BUCompareLatency(SUnit *left, SUnit *right, bool checkPref, 2344f697c8a19adf962a933b055383952e72789a0e20Andrew Trick RegReductionPQBase *SPQ) { 234587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // Scheduling an instruction that uses a VReg whose postincrement has not yet 234687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // been scheduled will induce a copy. Model this as an extra cycle of latency. 234787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick int LPenalty = hasVRegCycleUse(left) ? 1 : 0; 234887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick int RPenalty = hasVRegCycleUse(right) ? 1 : 0; 234987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick int LHeight = (int)left->getHeight() + LPenalty; 235087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick int RHeight = (int)right->getHeight() + RPenalty; 2351f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2352692c1d85353249124caa1885cfeda513146c6d81Dan Gohman bool LStall = (!checkPref || left->SchedulingPref == Sched::ILP) && 2353f697c8a19adf962a933b055383952e72789a0e20Andrew Trick BUHasStall(left, LHeight, SPQ); 2354692c1d85353249124caa1885cfeda513146c6d81Dan Gohman bool RStall = (!checkPref || right->SchedulingPref == Sched::ILP) && 2355f697c8a19adf962a933b055383952e72789a0e20Andrew Trick BUHasStall(right, RHeight, SPQ); 2356f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2357f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // If scheduling one of the node will cause a pipeline stall, delay it. 2358f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // If scheduling either one of the node will cause a pipeline stall, sort 2359f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // them according to their height. 2360f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (LStall) { 2361e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (!RStall) 2362f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return 1; 2363e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (LHeight != RHeight) 2364f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return LHeight > RHeight ? 1 : -1; 2365e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky } else if (RStall) 2366f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return -1; 2367f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2368c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick // If either node is scheduling for latency, sort them by height/depth 2369f697c8a19adf962a933b055383952e72789a0e20Andrew Trick // and latency. 2370692c1d85353249124caa1885cfeda513146c6d81Dan Gohman if (!checkPref || (left->SchedulingPref == Sched::ILP || 2371692c1d85353249124caa1885cfeda513146c6d81Dan Gohman right->SchedulingPref == Sched::ILP)) { 23724eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick // If neither instruction stalls (!LStall && !RStall) and HazardRecognizer 23734eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick // is enabled, grouping instructions by cycle, then its height is already 23744eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick // covered so only its depth matters. We also reach this point if both stall 23754eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick // but have the same height. 23764eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick if (!SPQ->getHazardRec()->isEnabled()) { 2377e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (LHeight != RHeight) 2378c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick return LHeight > RHeight ? 1 : -1; 2379c8bfd1d78ff9a307d1d4cb57cce4549b538e60f4Andrew Trick } 23804eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick int LDepth = left->getDepth() - LPenalty; 23814eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick int RDepth = right->getDepth() - RPenalty; 23824eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick if (LDepth != RDepth) { 23834eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick DEBUG(dbgs() << " Comparing latency of SU (" << left->NodeNum 23844eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick << ") depth " << LDepth << " vs SU (" << right->NodeNum 23854eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick << ") depth " << RDepth << "\n"); 23864eb4e5eb224b3d737558bcda8a0a369cc9d800e6Andrew Trick return LDepth < RDepth ? 1 : -1; 2387f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 2388e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (left->Latency != right->Latency) 2389f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return left->Latency > right->Latency ? 1 : -1; 2390f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 2391f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return 0; 2392f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 2393f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2394f697c8a19adf962a933b055383952e72789a0e20Andrew Trickstatic bool BURRSort(SUnit *left, SUnit *right, RegReductionPQBase *SPQ) { 239512f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick // Schedule physical register definitions close to their use. This is 239612f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick // motivated by microarchitectures that can fuse cmp+jump macro-ops. But as 239712f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick // long as shortening physreg live ranges is generally good, we can defer 239812f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick // creating a subtarget hook. 239912f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick if (!DisableSchedPhysRegJoin) { 240012f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick bool LHasPhysReg = left->hasPhysRegDefs; 240112f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick bool RHasPhysReg = right->hasPhysRegDefs; 240212f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick if (LHasPhysReg != RHasPhysReg) { 240312f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick #ifndef NDEBUG 2404e32981048244ecfa67d0bdc211af1bac2020a555Craig Topper const char *const PhysRegMsg[] = {" has no physreg"," defines a physreg"}; 240512f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick #endif 240612f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick DEBUG(dbgs() << " SU (" << left->NodeNum << ") " 240712f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick << PhysRegMsg[LHasPhysReg] << " SU(" << right->NodeNum << ") " 240812f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick << PhysRegMsg[RHasPhysReg] << "\n"); 240912f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick return LHasPhysReg < RHasPhysReg; 241012f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick } 241112f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick } 241212f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick 2413b16d06f88a81a5163e0caffad4bb268a8e1d0204Evan Cheng // Prioritize by Sethi-Ulmann number and push CopyToReg nodes down. 2414c8edc64188399437f5476d7fa45f714a92f2cb93Evan Cheng unsigned LPriority = SPQ->getNodePriority(left); 2415c8edc64188399437f5476d7fa45f714a92f2cb93Evan Cheng unsigned RPriority = SPQ->getNodePriority(right); 2416554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng 2417554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // Be really careful about hoisting call operands above previous calls. 2418554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // Only allows it if it would reduce register pressure. 2419554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng if (left->isCall && right->isCallOp) { 2420554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng unsigned RNumVals = right->getNode()->getNumValues(); 2421554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng RPriority = (RPriority > RNumVals) ? (RPriority - RNumVals) : 0; 2422554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng } 2423554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng if (right->isCall && left->isCallOp) { 2424554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng unsigned LNumVals = left->getNode()->getNumValues(); 2425554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng LPriority = (LPriority > LNumVals) ? (LPriority - LNumVals) : 0; 2426554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng } 2427554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng 2428e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (LPriority != RPriority) 242984d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng return LPriority > RPriority; 24300bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick 2431554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // One or both of the nodes are calls and their sethi-ullman numbers are the 2432554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // same, then keep source order. 2433554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng if (left->isCall || right->isCall) { 2434554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng unsigned LOrder = SPQ->getNodeOrdering(left); 2435554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng unsigned ROrder = SPQ->getNodeOrdering(right); 2436554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng 2437554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // Prefer an ordering where the lower the non-zero order number, the higher 2438554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // the preference. 2439554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng if ((LOrder || ROrder) && LOrder != ROrder) 2440554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng return LOrder != 0 && (LOrder < ROrder || ROrder == 0); 2441554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng } 2442554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng 244384d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // Try schedule def + use closer when Sethi-Ullman numbers are the same. 244484d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // e.g. 244584d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t1 = op t2, c1 244684d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t3 = op t4, c2 244784d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // 244884d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // and the following instructions are both ready. 244984d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t2 = op c3 245084d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t4 = op c4 245184d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // 245284d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // Then schedule t2 = op first. 245384d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // i.e. 245484d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t4 = op c4 245584d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t2 = op c3 245684d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t1 = op t2, c1 245784d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // t3 = op t4, c2 245884d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // 245984d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng // This creates more short live intervals. 246084d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng unsigned LDist = closestSucc(left); 246184d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng unsigned RDist = closestSucc(right); 2462e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (LDist != RDist) 246384d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng return LDist < RDist; 246484d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng 24658182347d70413174f2e80ea429801e887aee5cc3Evan Cheng // How many registers becomes live when the node is scheduled. 246684d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng unsigned LScratch = calcMaxScratches(left); 246784d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng unsigned RScratch = calcMaxScratches(right); 2468e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (LScratch != RScratch) 246984d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng return LScratch > RScratch; 247084d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng 2471554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // Comparing latency against a call makes little sense unless the node 2472554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // is register pressure-neutral. 2473554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng if ((left->isCall && RPriority > 0) || (right->isCall && LPriority > 0)) 2474554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng return (left->NodeQueueId > right->NodeQueueId); 2475554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng 2476554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng // Do not compare latencies when one or both of the nodes are calls. 2477554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng if (!DisableSchedCycles && 2478554daa67bd1c4f01fb7a00f2f4255a52b81e9fa3Evan Cheng !(left->isCall || right->isCall)) { 2479f697c8a19adf962a933b055383952e72789a0e20Andrew Trick int result = BUCompareLatency(left, right, false /*checkPref*/, SPQ); 2480f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (result != 0) 2481f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return result > 0; 2482f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 2483f697c8a19adf962a933b055383952e72789a0e20Andrew Trick else { 2484e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (left->getHeight() != right->getHeight()) 2485f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return left->getHeight() > right->getHeight(); 248638036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 2487e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (left->getDepth() != right->getDepth()) 2488f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return left->getDepth() < right->getDepth(); 2489f697c8a19adf962a933b055383952e72789a0e20Andrew Trick } 249084d4a2b4ad0874a46642bb568b45720d55e46b64Evan Cheng 249138036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick assert(left->NodeQueueId && right->NodeQueueId && 2492c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng "NodeQueueId cannot be zero"); 2493c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng return (left->NodeQueueId > right->NodeQueueId); 2494c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng} 2495c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng 2496187361b056823df4ff292561fe47468dad956872Bill Wendling// Bottom up 2497f697c8a19adf962a933b055383952e72789a0e20Andrew Trickbool bu_ls_rr_sort::operator()(SUnit *left, SUnit *right) const { 249812f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick if (int res = checkSpecialNodes(left, right)) 249912f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick return res > 0; 250012f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick 2501187361b056823df4ff292561fe47468dad956872Bill Wendling return BURRSort(left, right, SPQ); 2502187361b056823df4ff292561fe47468dad956872Bill Wendling} 2503187361b056823df4ff292561fe47468dad956872Bill Wendling 2504187361b056823df4ff292561fe47468dad956872Bill Wendling// Source order, otherwise bottom up. 2505f697c8a19adf962a933b055383952e72789a0e20Andrew Trickbool src_ls_rr_sort::operator()(SUnit *left, SUnit *right) const { 250612f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick if (int res = checkSpecialNodes(left, right)) 250712f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick return res > 0; 250812f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick 2509187361b056823df4ff292561fe47468dad956872Bill Wendling unsigned LOrder = SPQ->getNodeOrdering(left); 2510187361b056823df4ff292561fe47468dad956872Bill Wendling unsigned ROrder = SPQ->getNodeOrdering(right); 2511187361b056823df4ff292561fe47468dad956872Bill Wendling 2512187361b056823df4ff292561fe47468dad956872Bill Wendling // Prefer an ordering where the lower the non-zero order number, the higher 2513187361b056823df4ff292561fe47468dad956872Bill Wendling // the preference. 2514187361b056823df4ff292561fe47468dad956872Bill Wendling if ((LOrder || ROrder) && LOrder != ROrder) 2515187361b056823df4ff292561fe47468dad956872Bill Wendling return LOrder != 0 && (LOrder < ROrder || ROrder == 0); 2516187361b056823df4ff292561fe47468dad956872Bill Wendling 2517187361b056823df4ff292561fe47468dad956872Bill Wendling return BURRSort(left, right, SPQ); 2518187361b056823df4ff292561fe47468dad956872Bill Wendling} 2519187361b056823df4ff292561fe47468dad956872Bill Wendling 2520f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// If the time between now and when the instruction will be ready can cover 2521f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// the spill code, then avoid adding it to the ready queue. This gives long 2522f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// stalls highest priority and allows hoisting across calls. It should also 2523f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// speed up processing the available queue. 2524f697c8a19adf962a933b055383952e72789a0e20Andrew Trickbool hybrid_ls_rr_sort::isReady(SUnit *SU, unsigned CurCycle) const { 2525f697c8a19adf962a933b055383952e72789a0e20Andrew Trick static const unsigned ReadyDelay = 3; 2526f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2527f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SPQ->MayReduceRegPressure(SU)) return true; 2528f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2529f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SU->getHeight() > (CurCycle + ReadyDelay)) return false; 2530f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2531f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SPQ->getHazardRec()->getHazardType(SU, -ReadyDelay) 2532f697c8a19adf962a933b055383952e72789a0e20Andrew Trick != ScheduleHazardRecognizer::NoHazard) 2533f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return false; 2534f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2535f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return true; 2536f697c8a19adf962a933b055383952e72789a0e20Andrew Trick} 2537f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2538f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Return true if right should be scheduled with higher priority than left. 2539f697c8a19adf962a933b055383952e72789a0e20Andrew Trickbool hybrid_ls_rr_sort::operator()(SUnit *left, SUnit *right) const { 254012f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick if (int res = checkSpecialNodes(left, right)) 254112f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick return res > 0; 254212f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick 25438239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng if (left->isCall || right->isCall) 25448239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng // No way to compute latency of calls. 25458239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng return BURRSort(left, right, SPQ); 25468239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng 2547e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng bool LHigh = SPQ->HighRegPressure(left); 2548e0e925efb31ca98c78e53bf52db8529388a0390aEvan Cheng bool RHigh = SPQ->HighRegPressure(right); 254970017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng // Avoid causing spills. If register pressure is high, schedule for 255070017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng // register pressure reduction. 25510bc308600b0069f07ba722b472c68588573ebd28Andrew Trick if (LHigh && !RHigh) { 25520bc308600b0069f07ba722b472c68588573ebd28Andrew Trick DEBUG(dbgs() << " pressure SU(" << left->NodeNum << ") > SU(" 25530bc308600b0069f07ba722b472c68588573ebd28Andrew Trick << right->NodeNum << ")\n"); 25544a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng return true; 25550bc308600b0069f07ba722b472c68588573ebd28Andrew Trick } 25560bc308600b0069f07ba722b472c68588573ebd28Andrew Trick else if (!LHigh && RHigh) { 25570bc308600b0069f07ba722b472c68588573ebd28Andrew Trick DEBUG(dbgs() << " pressure SU(" << right->NodeNum << ") > SU(" 25580bc308600b0069f07ba722b472c68588573ebd28Andrew Trick << left->NodeNum << ")\n"); 25594a863e2c75145432fd660ee65e61b578c5e90ac9Evan Cheng return false; 25600bc308600b0069f07ba722b472c68588573ebd28Andrew Trick } 256187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (!LHigh && !RHigh) { 256287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick int result = BUCompareLatency(left, right, true /*checkPref*/, SPQ); 256387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (result != 0) 256487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick return result > 0; 256554699765064842fd08d1466adc93453660bc2a85Andrew Trick } 256615a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng return BURRSort(left, right, SPQ); 256715a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng} 256815a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 25692da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick// Schedule as many instructions in each cycle as possible. So don't make an 25702da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick// instruction available unless it is ready in the current cycle. 25712da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickbool ilp_ls_rr_sort::isReady(SUnit *SU, unsigned CurCycle) const { 2572f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SU->getHeight() > CurCycle) return false; 2573f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2574f697c8a19adf962a933b055383952e72789a0e20Andrew Trick if (SPQ->getHazardRec()->getHazardType(SU, 0) 2575f697c8a19adf962a933b055383952e72789a0e20Andrew Trick != ScheduleHazardRecognizer::NoHazard) 2576f697c8a19adf962a933b055383952e72789a0e20Andrew Trick return false; 2577f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2578a0807f57caed954545661c23607c507f8d5c3d64Andrew Trick return true; 25792da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick} 25802da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 2581d7cdc3e1f9df50be73e239c130dc92239c0e5d32Benjamin Kramerstatic bool canEnableCoalescing(SUnit *SU) { 25820bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick unsigned Opc = SU->getNode() ? SU->getNode()->getOpcode() : 0; 25830bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (Opc == ISD::TokenFactor || Opc == ISD::CopyToReg) 25840bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick // CopyToReg should be close to its uses to facilitate coalescing and 25850bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick // avoid spilling. 25860bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick return true; 25870bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick 25880bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (Opc == TargetOpcode::EXTRACT_SUBREG || 25890bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick Opc == TargetOpcode::SUBREG_TO_REG || 25900bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick Opc == TargetOpcode::INSERT_SUBREG) 25910bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick // EXTRACT_SUBREG, INSERT_SUBREG, and SUBREG_TO_REG nodes should be 25920bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick // close to their uses to facilitate coalescing. 25930bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick return true; 25940bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick 25950bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (SU->NumPreds == 0 && SU->NumSuccs != 0) 25960bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick // If SU does not have a register def, schedule it close to its uses 25970bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick // because it does not lengthen any live ranges. 25980bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick return true; 25990bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick 26000bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick return false; 26010bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick} 26020bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick 2603ab2e3e2d7074207e2a4bb15e2913fa83795bb1caAndrew Trick// list-ilp is currently an experimental scheduler that allows various 2604ab2e3e2d7074207e2a4bb15e2913fa83795bb1caAndrew Trick// heuristics to be enabled prior to the normal register reduction logic. 2605f697c8a19adf962a933b055383952e72789a0e20Andrew Trickbool ilp_ls_rr_sort::operator()(SUnit *left, SUnit *right) const { 260612f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick if (int res = checkSpecialNodes(left, right)) 260712f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick return res > 0; 260812f0dc6bb556976f22d89ebcf42bce273c9e7d38Andrew Trick 26098239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng if (left->isCall || right->isCall) 26108239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng // No way to compute latency of calls. 26118239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng return BURRSort(left, right, SPQ); 26128239daf7c83a65a189c352cce3191cdc3bbfe151Evan Cheng 26130bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick unsigned LLiveUses = 0, RLiveUses = 0; 26140bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick int LPDiff = 0, RPDiff = 0; 26150bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (!DisableSchedRegPressure || !DisableSchedLiveUses) { 26160bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick LPDiff = SPQ->RegPressureDiff(left, LLiveUses); 26170bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick RPDiff = SPQ->RegPressureDiff(right, RLiveUses); 26180bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick } 2619e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (!DisableSchedRegPressure && LPDiff != RPDiff) { 26200bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick DEBUG(dbgs() << "RegPressureDiff SU(" << left->NodeNum << "): " << LPDiff 26210bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick << " != SU(" << right->NodeNum << "): " << RPDiff << "\n"); 2622e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick return LPDiff > RPDiff; 2623e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick } 2624e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 26250bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (!DisableSchedRegPressure && (LPDiff > 0 || RPDiff > 0)) { 2626d7cdc3e1f9df50be73e239c130dc92239c0e5d32Benjamin Kramer bool LReduce = canEnableCoalescing(left); 2627d7cdc3e1f9df50be73e239c130dc92239c0e5d32Benjamin Kramer bool RReduce = canEnableCoalescing(right); 26280bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (LReduce && !RReduce) return false; 26290bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (RReduce && !LReduce) return true; 26300bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick } 26310bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick 26320bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (!DisableSchedLiveUses && (LLiveUses != RLiveUses)) { 26330bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick DEBUG(dbgs() << "Live uses SU(" << left->NodeNum << "): " << LLiveUses 26340bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick << " != SU(" << right->NodeNum << "): " << RLiveUses << "\n"); 2635e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick return LLiveUses < RLiveUses; 2636e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick } 2637e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 26380bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick if (!DisableSchedStalls) { 26390bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick bool LStall = BUHasStall(left, left->getHeight(), SPQ); 26400bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick bool RStall = BUHasStall(right, right->getHeight(), SPQ); 2641e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (LStall != RStall) 26420bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick return left->getHeight() > right->getHeight(); 2643e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick } 2644e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 2645afc7d235e91a769f74d87bbe745558ed1b692ff7Andrew Trick if (!DisableSchedCriticalPath) { 2646afc7d235e91a769f74d87bbe745558ed1b692ff7Andrew Trick int spread = (int)left->getDepth() - (int)right->getDepth(); 2647afc7d235e91a769f74d87bbe745558ed1b692ff7Andrew Trick if (std::abs(spread) > MaxReorderWindow) { 26480bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick DEBUG(dbgs() << "Depth of SU(" << left->NodeNum << "): " 26490bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick << left->getDepth() << " != SU(" << right->NodeNum << "): " 26500bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick << right->getDepth() << "\n"); 2651afc7d235e91a769f74d87bbe745558ed1b692ff7Andrew Trick return left->getDepth() < right->getDepth(); 2652afc7d235e91a769f74d87bbe745558ed1b692ff7Andrew Trick } 2653e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick } 2654e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick 2655e0ef509aeb47b396cf1bdc170ca4f468f799719fAndrew Trick if (!DisableSchedHeight && left->getHeight() != right->getHeight()) { 26560bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick int spread = (int)left->getHeight() - (int)right->getHeight(); 2657e77ae2d692a25034da908151761c0f6b7e071779Nick Lewycky if (std::abs(spread) > MaxReorderWindow) 26580bf56c821c454a960afc34cb5746006bc1f9e1a1Andrew Trick return left->getHeight() > right->getHeight(); 265970017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng } 266070017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 266170017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng return BURRSort(left, right, SPQ); 266270017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng} 266370017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 266487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trickvoid RegReductionPQBase::initNodes(std::vector<SUnit> &sunits) { 266587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick SUnits = &sunits; 266687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // Add pseudo dependency edges for two-address nodes. 2667623a7e146bd86747dc46a6f8bb9993fc217d6b78Evan Cheng if (!Disable2AddrHack) 2668623a7e146bd86747dc46a6f8bb9993fc217d6b78Evan Cheng AddPseudoTwoAddrDeps(); 266987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // Reroute edges to nodes with multiple uses. 2670479389a4da53ce72226366cc6d1cad13da158909Evan Cheng if (!TracksRegPressure && !SrcOrder) 267187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick PrescheduleNodesWithMultipleUses(); 267287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // Calculate node priorities. 267387896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick CalculateSethiUllmanNumbers(); 267487896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 267587896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick // For single block loops, mark nodes that look like canonical IV increments. 267687896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick if (scheduleDAG->BB->isSuccessor(scheduleDAG->BB)) { 267787896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick for (unsigned i = 0, e = sunits.size(); i != e; ++i) { 267887896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick initVRegCycle(&sunits[i]); 267987896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick } 268087896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick } 268187896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick} 268287896d9368e08d93493427ce7bf8272d1e5cca35Andrew Trick 2683f697c8a19adf962a933b055383952e72789a0e20Andrew Trick//===----------------------------------------------------------------------===// 2684f697c8a19adf962a933b055383952e72789a0e20Andrew Trick// Preschedule for Register Pressure 2685f697c8a19adf962a933b055383952e72789a0e20Andrew Trick//===----------------------------------------------------------------------===// 2686f697c8a19adf962a933b055383952e72789a0e20Andrew Trick 2687f697c8a19adf962a933b055383952e72789a0e20Andrew Trickbool RegReductionPQBase::canClobber(const SUnit *SU, const SUnit *Op) { 268895f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng if (SU->isTwoAddress) { 2689550f5afb68ce8f034991863cac65bef22a6554daDan Gohman unsigned Opc = SU->getNode()->getMachineOpcode(); 2690e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng const MCInstrDesc &MCID = TII->get(Opc); 2691e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng unsigned NumRes = MCID.getNumDefs(); 2692e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng unsigned NumOps = MCID.getNumOperands() - NumRes; 269395f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng for (unsigned i = 0; i != NumOps; ++i) { 2694e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng if (MCID.getOperandConstraint(i+NumRes, MCOI::TIED_TO) != -1) { 2695550f5afb68ce8f034991863cac65bef22a6554daDan Gohman SDNode *DU = SU->getNode()->getOperand(i).getNode(); 269694d7a5f8156e62532870fbaf197377b34e52ff2aDan Gohman if (DU->getNodeId() != -1 && 269794d7a5f8156e62532870fbaf197377b34e52ff2aDan Gohman Op->OrigNode == &(*SUnits)[DU->getNodeId()]) 269895f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng return true; 269995f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 270095f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 2701e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng } 2702e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng return false; 2703e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 2704e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 2705340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick/// canClobberReachingPhysRegUse - True if SU would clobber one of it's 2706340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick/// successor's explicit physregs whose definition can reach DepSU. 2707340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick/// i.e. DepSU should not be scheduled above SU. 2708340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trickstatic bool canClobberReachingPhysRegUse(const SUnit *DepSU, const SUnit *SU, 2709340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick ScheduleDAGRRList *scheduleDAG, 2710340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick const TargetInstrInfo *TII, 2711340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick const TargetRegisterInfo *TRI) { 2712fac259814923d091942b230e7bd002a8d1130bc3Craig Topper const uint16_t *ImpDefs 2713340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick = TII->get(SU->getNode()->getMachineOpcode()).getImplicitDefs(); 271416a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen const uint32_t *RegMask = getNodeRegMask(SU->getNode()); 271516a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if(!ImpDefs && !RegMask) 2716340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick return false; 2717340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick 2718340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick for (SUnit::const_succ_iterator SI = SU->Succs.begin(), SE = SU->Succs.end(); 2719340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick SI != SE; ++SI) { 2720340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick SUnit *SuccSU = SI->getSUnit(); 2721340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick for (SUnit::const_pred_iterator PI = SuccSU->Preds.begin(), 2722340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick PE = SuccSU->Preds.end(); PI != PE; ++PI) { 2723340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick if (!PI->isAssignedRegDep()) 2724340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick continue; 2725340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick 272616a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (RegMask && MachineOperand::clobbersPhysReg(RegMask, PI->getReg()) && 272716a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen scheduleDAG->IsReachable(DepSU, PI->getSUnit())) 272816a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen return true; 272916a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen 273016a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (ImpDefs) 2731fac259814923d091942b230e7bd002a8d1130bc3Craig Topper for (const uint16_t *ImpDef = ImpDefs; *ImpDef; ++ImpDef) 273216a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen // Return true if SU clobbers this physical register use and the 273316a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen // definition of the register reaches from DepSU. IsReachable queries 273416a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen // a topological forward sort of the DAG (following the successors). 273516a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (TRI->regsOverlap(*ImpDef, PI->getReg()) && 273616a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen scheduleDAG->IsReachable(DepSU, PI->getSUnit())) 273716a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen return true; 2738340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick } 2739340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick } 2740340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick return false; 2741340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick} 2742340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick 2743180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng/// canClobberPhysRegDefs - True if SU would clobber one of SuccSU's 27442f1d3108e481758da66662f72673741da86312daDan Gohman/// physical register defs. 2745430b8a22e2717d3dfb6b4f096bc23c9538fd7959Dan Gohmanstatic bool canClobberPhysRegDefs(const SUnit *SuccSU, const SUnit *SU, 2746180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng const TargetInstrInfo *TII, 27476f0d024a534af18d9e60b3ea757376cd8a3a980eDan Gohman const TargetRegisterInfo *TRI) { 2748550f5afb68ce8f034991863cac65bef22a6554daDan Gohman SDNode *N = SuccSU->getNode(); 2749e8be6c63915e0389f1eef6b53c64300d13b2ce99Dan Gohman unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs(); 2750fac259814923d091942b230e7bd002a8d1130bc3Craig Topper const uint16_t *ImpDefs = TII->get(N->getMachineOpcode()).getImplicitDefs(); 27512f1d3108e481758da66662f72673741da86312daDan Gohman assert(ImpDefs && "Caller should check hasPhysRegDefs"); 2752a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman for (const SDNode *SUNode = SU->getNode(); SUNode; 275329d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner SUNode = SUNode->getGluedNode()) { 2754a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman if (!SUNode->isMachineOpcode()) 275559932584a812685b16ad6a53a023b2bb3fc21819Dan Gohman continue; 2756fac259814923d091942b230e7bd002a8d1130bc3Craig Topper const uint16_t *SUImpDefs = 2757a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman TII->get(SUNode->getMachineOpcode()).getImplicitDefs(); 275816a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen const uint32_t *SURegMask = getNodeRegMask(SUNode); 275916a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (!SUImpDefs && !SURegMask) 276016a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen continue; 2761a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman for (unsigned i = NumDefs, e = N->getNumValues(); i != e; ++i) { 2762e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT VT = N->getValueType(i); 2763f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner if (VT == MVT::Glue || VT == MVT::Other) 2764a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman continue; 2765a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman if (!N->hasAnyUseOfValue(i)) 2766a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman continue; 2767a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman unsigned Reg = ImpDefs[i - NumDefs]; 276816a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (SURegMask && MachineOperand::clobbersPhysReg(SURegMask, Reg)) 276916a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen return true; 277016a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen if (!SUImpDefs) 277116a7ff31655aa2de65d86472edcb71cf2df3ac3aJakob Stoklund Olesen continue; 2772a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman for (;*SUImpDefs; ++SUImpDefs) { 2773a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman unsigned SUReg = *SUImpDefs; 2774a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman if (TRI->regsOverlap(Reg, SUReg)) 2775a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman return true; 2776a5c8ae233ebaef270b8fa96565ca7bf4fdcb3e15Dan Gohman } 2777180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng } 2778180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng } 2779180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng return false; 2780180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng} 2781180c210a1d96a56ae0611d4f8de81e1ada5559ebEvan Cheng 2782002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// PrescheduleNodesWithMultipleUses - Nodes with multiple uses 2783002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// are not handled well by the general register pressure reduction 2784002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// heuristics. When presented with code like this: 2785002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2786002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// N 2787002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// / | 2788002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// / | 2789002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// U store 2790002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// | 2791002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// ... 2792002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2793002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// the heuristics tend to push the store up, but since the 2794002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// operand of the store has another use (U), this would increase 2795002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// the length of that other use (the U->N edge). 2796002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2797002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// This function transforms code like the above to route U's 2798002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// dependence through the store when possible, like this: 2799002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2800002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// N 2801002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// || 2802002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// || 2803002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// store 2804002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// | 2805002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// U 2806002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// | 2807002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// ... 2808002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2809002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// This results in the store being scheduled immediately 2810002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// after N, which shortens the U->N live range, reducing 2811002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// register pressure. 2812002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman/// 2813f697c8a19adf962a933b055383952e72789a0e20Andrew Trickvoid RegReductionPQBase::PrescheduleNodesWithMultipleUses() { 2814002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Visit all the nodes in topological order, working top-down. 2815002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman for (unsigned i = 0, e = SUnits->size(); i != e; ++i) { 2816002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman SUnit *SU = &(*SUnits)[i]; 2817002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // For now, only look at nodes with no data successors, such as stores. 2818002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // These are especially important, due to the heuristics in 2819002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // getNodePriority for nodes with no data successors. 2820002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SU->NumSuccs != 0) 2821002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2822002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // For now, only look at nodes with exactly one data predecessor. 2823002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SU->NumPreds != 1) 2824002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2825002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Avoid prescheduling copies to virtual registers, which don't behave 2826002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // like other nodes from the perspective of scheduling heuristics. 2827002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SDNode *N = SU->getNode()) 2828002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (N->getOpcode() == ISD::CopyToReg && 2829002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman TargetRegisterInfo::isVirtualRegister 2830002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman (cast<RegisterSDNode>(N->getOperand(1))->getReg())) 2831002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2832002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman 2833002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Locate the single data predecessor. 2834002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman SUnit *PredSU = 0; 2835002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman for (SUnit::const_pred_iterator II = SU->Preds.begin(), 2836002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman EE = SU->Preds.end(); II != EE; ++II) 2837002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (!II->isCtrl()) { 2838002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman PredSU = II->getSUnit(); 2839002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman break; 2840002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman } 2841002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman assert(PredSU); 2842002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman 2843002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Don't rewrite edges that carry physregs, because that requires additional 2844002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // support infrastructure. 2845002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (PredSU->hasPhysRegDefs) 2846002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2847002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Short-circuit the case where SU is PredSU's only data successor. 2848002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (PredSU->NumSuccs == 1) 2849002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2850002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Avoid prescheduling to copies from virtual registers, which don't behave 285192e946630d5f9bb092853b93501387dd216899b9Andrew Trick // like other nodes from the perspective of scheduling heuristics. 2852002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SDNode *N = SU->getNode()) 2853002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (N->getOpcode() == ISD::CopyFromReg && 2854002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman TargetRegisterInfo::isVirtualRegister 2855002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman (cast<RegisterSDNode>(N->getOperand(1))->getReg())) 2856002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman continue; 2857002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman 2858002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Perform checks on the successors of PredSU. 2859002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman for (SUnit::const_succ_iterator II = PredSU->Succs.begin(), 2860002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman EE = PredSU->Succs.end(); II != EE; ++II) { 2861002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman SUnit *PredSuccSU = II->getSUnit(); 2862002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (PredSuccSU == SU) continue; 2863002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // If PredSU has another successor with no data successors, for 2864002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // now don't attempt to choose either over the other. 2865002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (PredSuccSU->NumSuccs == 0) 2866002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman goto outer_loop_continue; 2867002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Don't break physical register dependencies. 2868002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SU->hasPhysRegClobbers && PredSuccSU->hasPhysRegDefs) 2869002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (canClobberPhysRegDefs(PredSuccSU, SU, TII, TRI)) 2870002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman goto outer_loop_continue; 2871002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Don't introduce graph cycles. 2872002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (scheduleDAG->IsReachable(SU, PredSuccSU)) 2873002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman goto outer_loop_continue; 2874002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman } 2875002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman 2876002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // Ok, the transformation is safe and the heuristics suggest it is 2877002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman // profitable. Update the graph. 287815a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng DEBUG(dbgs() << " Prescheduling SU #" << SU->NodeNum 287915a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng << " next to PredSU #" << PredSU->NodeNum 2880bbbfa99d3d18fe9f20265305e833666645ada528Chris Lattner << " to guide scheduling in the presence of multiple uses\n"); 2881002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman for (unsigned i = 0; i != PredSU->Succs.size(); ++i) { 2882002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman SDep Edge = PredSU->Succs[i]; 2883002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman assert(!Edge.isAssignedRegDep()); 2884002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman SUnit *SuccSU = Edge.getSUnit(); 2885002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman if (SuccSU != SU) { 2886002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman Edge.setSUnit(PredSU); 2887002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman scheduleDAG->RemovePred(SuccSU, Edge); 2888002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman scheduleDAG->AddPred(SU, Edge); 2889002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman Edge.setSUnit(SU); 2890002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman scheduleDAG->AddPred(SuccSU, Edge); 2891002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman --i; 2892002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman } 2893002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman } 2894002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman outer_loop_continue:; 2895002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman } 2896002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman} 2897002b44f1d7c0b301b78dec1f24d406d4b351fdaaDan Gohman 2898e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// AddPseudoTwoAddrDeps - If two nodes share an operand and one of them uses 2899e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// it as a def&use operand. Add a pseudo control edge from it to the other 2900e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng/// node (if it won't create a cycle) so the two-address one will be scheduled 290122a529990bb4bb86bdb2ae1cfce7340320a6ca7fEvan Cheng/// first (lower in the schedule). If both nodes are two-address, favor the 290222a529990bb4bb86bdb2ae1cfce7340320a6ca7fEvan Cheng/// one that has a CopyToReg use (more likely to be a loop induction update). 290322a529990bb4bb86bdb2ae1cfce7340320a6ca7fEvan Cheng/// If both are two-address, but one is commutable while the other is not 290422a529990bb4bb86bdb2ae1cfce7340320a6ca7fEvan Cheng/// commutable, favor the one that's not commutable. 2905ef0b3ca3a8935b5390633dc7bb4adcdb99e0c26aDuncan Sandsvoid RegReductionPQBase::AddPseudoTwoAddrDeps() { 290695f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng for (unsigned i = 0, e = SUnits->size(); i != e; ++i) { 2907430b8a22e2717d3dfb6b4f096bc23c9538fd7959Dan Gohman SUnit *SU = &(*SUnits)[i]; 290895f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng if (!SU->isTwoAddress) 290995f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng continue; 291095f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng 2911550f5afb68ce8f034991863cac65bef22a6554daDan Gohman SDNode *Node = SU->getNode(); 291229d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner if (!Node || !Node->isMachineOpcode() || SU->getNode()->getGluedNode()) 291395f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng continue; 291495f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng 2915089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng bool isLiveOut = hasOnlyLiveOutUses(SU); 2916e8be6c63915e0389f1eef6b53c64300d13b2ce99Dan Gohman unsigned Opc = Node->getMachineOpcode(); 2917e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng const MCInstrDesc &MCID = TII->get(Opc); 2918e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng unsigned NumRes = MCID.getNumDefs(); 2919e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng unsigned NumOps = MCID.getNumOperands() - NumRes; 292095f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng for (unsigned j = 0; j != NumOps; ++j) { 2921e837dead3c8dc3445ef6a0e2322179c57e264a13Evan Cheng if (MCID.getOperandConstraint(j+NumRes, MCOI::TIED_TO) == -1) 2922c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman continue; 2923c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman SDNode *DU = SU->getNode()->getOperand(j).getNode(); 2924c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if (DU->getNodeId() == -1) 2925c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman continue; 2926c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman const SUnit *DUSU = &(*SUnits)[DU->getNodeId()]; 2927c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if (!DUSU) continue; 2928c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman for (SUnit::const_succ_iterator I = DUSU->Succs.begin(), 2929c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman E = DUSU->Succs.end(); I != E; ++I) { 293054e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman if (I->isCtrl()) continue; 293154e4c36a7349e94a84773afb56eccd4ca65b49e9Dan Gohman SUnit *SuccSU = I->getSUnit(); 2932c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if (SuccSU == SU) 29337da8f399bf09e9a03fe8bdd8c8eef6e5a7d87327Evan Cheng continue; 2934c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman // Be conservative. Ignore if nodes aren't at roughly the same 2935c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman // depth and height. 29363f23744df4809eba94284e601e81489212c974d4Dan Gohman if (SuccSU->getHeight() < SU->getHeight() && 29373f23744df4809eba94284e601e81489212c974d4Dan Gohman (SU->getHeight() - SuccSU->getHeight()) > 1) 2938c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman continue; 29390e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman // Skip past COPY_TO_REGCLASS nodes, so that the pseudo edge 29400e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman // constrains whatever is using the copy, instead of the copy 29410e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman // itself. In the case that the copy is coalesced, this 29420e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman // preserves the intent of the pseudo two-address heurietics. 29430e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman while (SuccSU->Succs.size() == 1 && 29440e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman SuccSU->getNode()->isMachineOpcode() && 29450e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman SuccSU->getNode()->getMachineOpcode() == 2946518bb53485df640d7b7e3f6b0544099020c42aa7Chris Lattner TargetOpcode::COPY_TO_REGCLASS) 29470e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman SuccSU = SuccSU->Succs.front().getSUnit(); 29480e4042d30b2b47329166f5f784ef3c70f6b408ccDan Gohman // Don't constrain non-instruction nodes. 2949c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if (!SuccSU->getNode() || !SuccSU->getNode()->isMachineOpcode()) 2950c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman continue; 2951c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman // Don't constrain nodes with physical register defs if the 2952c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman // predecessor can clobber them. 29538f4aa333d02d0f48f90f4604d894a73ee53edcb5Dan Gohman if (SuccSU->hasPhysRegDefs && SU->hasPhysRegClobbers) { 2954c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman if (canClobberPhysRegDefs(SuccSU, SU, TII, TRI)) 295532dfbeada7292167bb488f36a71a5a6a519ddaffEvan Cheng continue; 2956c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman } 29578af808a3d64dfdcc5d126e6ad762f57b1483671cDan Gohman // Don't constrain EXTRACT_SUBREG, INSERT_SUBREG, and SUBREG_TO_REG; 29588af808a3d64dfdcc5d126e6ad762f57b1483671cDan Gohman // these may be coalesced away. We want them close to their uses. 2959c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman unsigned SuccOpc = SuccSU->getNode()->getMachineOpcode(); 2960518bb53485df640d7b7e3f6b0544099020c42aa7Chris Lattner if (SuccOpc == TargetOpcode::EXTRACT_SUBREG || 2961518bb53485df640d7b7e3f6b0544099020c42aa7Chris Lattner SuccOpc == TargetOpcode::INSERT_SUBREG || 2962518bb53485df640d7b7e3f6b0544099020c42aa7Chris Lattner SuccOpc == TargetOpcode::SUBREG_TO_REG) 2963c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman continue; 2964340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick if (!canClobberReachingPhysRegUse(SuccSU, SU, scheduleDAG, TII, TRI) && 2965340d78f4e7b95c235d0ecf315b755bebefd0a168Andrew Trick (!canClobber(SuccSU, DUSU) || 2966089751535d6e9adf65842e2ca5867bf9a70e1e95Evan Cheng (isLiveOut && !hasOnlyLiveOutUses(SuccSU)) || 2967c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman (!SU->isCommutable && SuccSU->isCommutable)) && 2968c2f9062ea4915ae034417eaeead3c5942921f24dDan Gohman !scheduleDAG->IsReachable(SuccSU, SU)) { 296915a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng DEBUG(dbgs() << " Adding a pseudo-two-addr edge from SU #" 2970bbbfa99d3d18fe9f20265305e833666645ada528Chris Lattner << SU->NodeNum << " to SU #" << SuccSU->NodeNum << "\n"); 2971a78d3228e8b2a14915ea9908dbaaf2c934803e11Andrew Trick scheduleDAG->AddPred(SU, SDep(SuccSU, SDep::Artificial)); 297295f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 297395f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 297495f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 297595f6edeff5ab6de9cf5589f662c8e7a6ba119c2cEvan Cheng } 2976e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 2977e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 2978e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 2979e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng// Public Constructor Functions 2980e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng//===----------------------------------------------------------------------===// 2981e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 298247ac0f0c7c39289f5970688154e385be22b7f293Dan Gohmanllvm::ScheduleDAGSDNodes * 29832da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickllvm::createBURRListDAGScheduler(SelectionDAGISel *IS, 29842da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CodeGenOpt::Level OptLevel) { 298579ce276083ced01256a0eb7d80731e4948ca6e87Dan Gohman const TargetMachine &TM = IS->TM; 298679ce276083ced01256a0eb7d80731e4948ca6e87Dan Gohman const TargetInstrInfo *TII = TM.getInstrInfo(); 298779ce276083ced01256a0eb7d80731e4948ca6e87Dan Gohman const TargetRegisterInfo *TRI = TM.getRegisterInfo(); 298838036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 29894f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng BURegReductionPriorityQueue *PQ = 2990479389a4da53ce72226366cc6d1cad13da158909Evan Cheng new BURegReductionPriorityQueue(*IS->MF, false, false, TII, TRI, 0); 29912da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, false, PQ, OptLevel); 2992c6be777208f4539af400ac694d9d1dc8b992bc80Evan Cheng PQ->setScheduleDAG(SD); 299338036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick return SD; 2994e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng} 2995e165a78551a91d8420cd8f074d97701e8788f8b5Evan Cheng 299647ac0f0c7c39289f5970688154e385be22b7f293Dan Gohmanllvm::ScheduleDAGSDNodes * 29972da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickllvm::createSourceListDAGScheduler(SelectionDAGISel *IS, 29982da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CodeGenOpt::Level OptLevel) { 2999187361b056823df4ff292561fe47468dad956872Bill Wendling const TargetMachine &TM = IS->TM; 3000187361b056823df4ff292561fe47468dad956872Bill Wendling const TargetInstrInfo *TII = TM.getInstrInfo(); 3001187361b056823df4ff292561fe47468dad956872Bill Wendling const TargetRegisterInfo *TRI = TM.getRegisterInfo(); 300238036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 30034f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng SrcRegReductionPriorityQueue *PQ = 3004479389a4da53ce72226366cc6d1cad13da158909Evan Cheng new SrcRegReductionPriorityQueue(*IS->MF, false, true, TII, TRI, 0); 30052da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, false, PQ, OptLevel); 300615a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng PQ->setScheduleDAG(SD); 300738036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick return SD; 300815a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng} 300915a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng 301015a16def6e70c8f7df1023da80ceb89887203b40Evan Chengllvm::ScheduleDAGSDNodes * 30112da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickllvm::createHybridListDAGScheduler(SelectionDAGISel *IS, 30122da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CodeGenOpt::Level OptLevel) { 301315a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng const TargetMachine &TM = IS->TM; 301415a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng const TargetInstrInfo *TII = TM.getInstrInfo(); 301515a16def6e70c8f7df1023da80ceb89887203b40Evan Cheng const TargetRegisterInfo *TRI = TM.getRegisterInfo(); 30164f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng const TargetLowering *TLI = &IS->getTargetLowering(); 301738036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 30184f6b4674be5473319ac5e70c76fd5cb964da2128Evan Cheng HybridBURRPriorityQueue *PQ = 3019479389a4da53ce72226366cc6d1cad13da158909Evan Cheng new HybridBURRPriorityQueue(*IS->MF, true, false, TII, TRI, TLI); 30202da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick 30212da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, PQ, OptLevel); 3022187361b056823df4ff292561fe47468dad956872Bill Wendling PQ->setScheduleDAG(SD); 302338036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick return SD; 3024187361b056823df4ff292561fe47468dad956872Bill Wendling} 302570017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng 302670017e44cdba1946cc478ce1856a3e855a767e28Evan Chengllvm::ScheduleDAGSDNodes * 30272da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trickllvm::createILPListDAGScheduler(SelectionDAGISel *IS, 30282da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick CodeGenOpt::Level OptLevel) { 302970017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng const TargetMachine &TM = IS->TM; 303070017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng const TargetInstrInfo *TII = TM.getInstrInfo(); 303170017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng const TargetRegisterInfo *TRI = TM.getRegisterInfo(); 303270017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng const TargetLowering *TLI = &IS->getTargetLowering(); 303338036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick 303470017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng ILPBURRPriorityQueue *PQ = 3035479389a4da53ce72226366cc6d1cad13da158909Evan Cheng new ILPBURRPriorityQueue(*IS->MF, true, false, TII, TRI, TLI); 30362da8bc8a5f7705ac131184cd247f48500da0d74eAndrew Trick ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, PQ, OptLevel); 303770017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng PQ->setScheduleDAG(SD); 303838036d8b36fa5d1b2f3530f47fd716e9da7bffabAndrew Trick return SD; 303970017e44cdba1946cc478ce1856a3e855a767e28Evan Cheng} 3040